deepline 0.1.182 → 0.1.183

package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts

@@ -0,0 +1,42 @@
+export const PLAY_RUNTIME_CONTRACT = 2;
+export const PLAY_RUNTIME_CONTRACT_MIN_SUPPORTED = 1;
+export const PLAY_RUNTIME_CONTRACT_HEADER = 'x-deepline-runtime-contract';
+
+export const RUNTIME_CONTRACT_CHANGELOG = [
+  {
+    contract: 2,
+    changed:
+      'Runtime receipt batch responses use positional nulls and v2 lease-aware receipt semantics.',
+    compatOwner: 'runtime',
+    compatRemoval: 'ledger cutover milestone M1',
+  },
+] as const;
+
+export type RuntimeContractResolution =
+  | { ok: true; contract: number }
+  | {
+      ok: false;
+      requestedContract: number;
+      currentContract: number;
+      minSupportedContract: number;
+    };
+
+export function resolvePlayRuntimeContract(
+  rawContract: string | null | undefined,
+): RuntimeContractResolution {
+  const trimmed = rawContract?.trim();
+  const requestedContract = trimmed ? Number(trimmed) : 1;
+  if (
+    !Number.isInteger(requestedContract) ||
+    requestedContract < PLAY_RUNTIME_CONTRACT_MIN_SUPPORTED ||
+    requestedContract > PLAY_RUNTIME_CONTRACT
+  ) {
+    return {
+      ok: false,
+      requestedContract,
+      currentContract: PLAY_RUNTIME_CONTRACT,
+      minSupportedContract: PLAY_RUNTIME_CONTRACT_MIN_SUPPORTED,
+    };
+  }
+  return { ok: true, contract: requestedContract };
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts

@@ -0,0 +1,183 @@
+export type RuntimeSheetRowStatus =
+  | 'pending'
+  | 'running'
+  | 'failed'
+  | 'enriched'
+  | 'stale';
+
+export type RuntimeSheetAttemptIdentity = {
+  attemptId: string | null;
+  attemptOwnerRunId: string | null;
+  attemptExpiresAt: string | null;
+  attemptSeq?: number | null;
+};
+
+export type RuntimeSheetRowAttemptInput = RuntimeSheetAttemptIdentity & {
+  status: RuntimeSheetRowStatus;
+  runId: string | null;
+};
+
+export type RuntimeSheetAttemptRelation =
+  | { kind: 'unowned' }
+  | { kind: 'same_attempt' }
+  | { kind: 'expired' }
+  | { kind: 'same_owner_newer_attempt' }
+  | { kind: 'same_owner_stale_attempt' }
+  | { kind: 'other_active_owner' }
+  | {
+      kind: 'malformed';
+      reason: 'invalid_row_expiry';
+    };
+
+export type RuntimeSheetPrepareDecision =
+  | { kind: 'claim_executable' }
+  | { kind: 'reuse_completed' }
+  | { kind: 'blocked_by_active_owner' };
+
+export type RuntimeSheetTerminalDecision =
+  | { kind: 'write_terminal' }
+  | { kind: 'reject_stale_attempt' };
+
+export type RuntimeSheetReleaseDecision =
+  | { kind: 'release' }
+  | { kind: 'skip' };
+
+function normalize(value: string | null | undefined): string | null {
+  const trimmed = value?.trim();
+  return trimmed ? trimmed : null;
+}
+
+function parseOptionalMs(value: string | null): number | null {
+  if (!value) return null;
+  const parsed = Date.parse(value);
+  return Number.isFinite(parsed) ? parsed : Number.NaN;
+}
+
+function normalizeAttemptSeq(value: number | null | undefined): number {
+  return Number.isFinite(value) ? Math.trunc(value as number) : 0;
+}
+
+export function classifyRuntimeSheetAttemptRelation(input: {
+  row: RuntimeSheetRowAttemptInput;
+  request: RuntimeSheetAttemptIdentity;
+  nowMs?: number;
+}): RuntimeSheetAttemptRelation {
+  const nowMs = input.nowMs ?? Date.now();
+  const rowAttemptId = normalize(input.row.attemptId);
+  const requestAttemptId = normalize(input.request.attemptId);
+  if (!rowAttemptId) return { kind: 'unowned' };
+  if (rowAttemptId === requestAttemptId) return { kind: 'same_attempt' };
+
+  const rowExpiresAt = normalize(input.row.attemptExpiresAt);
+  const rowExpiresAtMs = parseOptionalMs(rowExpiresAt);
+  if (rowExpiresAtMs === null) {
+    return { kind: 'malformed', reason: 'invalid_row_expiry' };
+  }
+  if (Number.isNaN(rowExpiresAtMs)) {
+    return { kind: 'malformed', reason: 'invalid_row_expiry' };
+  }
+  if (rowExpiresAtMs <= nowMs) return { kind: 'expired' };
+
+  const rowOwner =
+    normalize(input.row.attemptOwnerRunId) ?? normalize(input.row.runId);
+  const requestOwner = normalize(input.request.attemptOwnerRunId);
+  const rowAttemptSeq = normalizeAttemptSeq(input.row.attemptSeq);
+  const requestAttemptSeq = normalizeAttemptSeq(input.request.attemptSeq);
+  if (rowOwner && requestOwner && rowOwner === requestOwner) {
+    return requestAttemptSeq > rowAttemptSeq
+      ? { kind: 'same_owner_newer_attempt' }
+      : { kind: 'same_owner_stale_attempt' };
+  }
+
+  return { kind: 'other_active_owner' };
+}
+
+export function decideRuntimeSheetPrepare(input: {
+  row: RuntimeSheetRowAttemptInput;
+  request: RuntimeSheetAttemptIdentity;
+  missingRequiredOutput?: boolean;
+  nowMs?: number;
+}): RuntimeSheetPrepareDecision {
+  if (input.row.status === 'stale' || input.row.status === 'failed') {
+    return { kind: 'claim_executable' };
+  }
+  const relation = classifyRuntimeSheetAttemptRelation(input);
+  if (input.row.status === 'pending' || input.row.status === 'running') {
+    return relation.kind === 'other_active_owner' || relation.kind === 'malformed'
+      ? { kind: 'blocked_by_active_owner' }
+      : { kind: 'claim_executable' };
+  }
+  if (input.row.status === 'enriched') {
+    if (!input.missingRequiredOutput) return { kind: 'reuse_completed' };
+    return relation.kind === 'same_owner_stale_attempt' ||
+      relation.kind === 'malformed'
+      ? { kind: 'blocked_by_active_owner' }
+      : { kind: 'claim_executable' };
+  }
+  return assertNever(input.row.status);
+}
+
+export function decideRuntimeSheetTerminalWrite(input: {
+  row: RuntimeSheetRowAttemptInput;
+  request: RuntimeSheetAttemptIdentity;
+  nowMs?: number;
+}): RuntimeSheetTerminalDecision {
+  const relation = classifyRuntimeSheetAttemptRelation(input);
+  if (
+    relation.kind === 'unowned' ||
+    relation.kind === 'same_attempt' ||
+    relation.kind === 'expired' ||
+    relation.kind === 'same_owner_newer_attempt'
+  ) {
+    return { kind: 'write_terminal' };
+  }
+  if (input.row.status === 'enriched') {
+    const rowAttemptSeq = normalizeAttemptSeq(input.row.attemptSeq);
+    const requestAttemptSeq = normalizeAttemptSeq(input.request.attemptSeq);
+    if (requestAttemptSeq > rowAttemptSeq) {
+      return { kind: 'write_terminal' };
+    }
+  }
+  return { kind: 'reject_stale_attempt' };
+}
+
+/**
+ * Decide whether a controlled run-fatal teardown may release (clear the attempt
+ * lease on) a row. Only rows the releasing attempt currently owns are cleared:
+ * the row must carry an attempt lease whose owner run id and attempt generation
+ * (seq) match the releasing run. When the request carries a concrete attempt id
+ * the row's attempt id must match it too; when the request attempt id is null,
+ * every attempt id the run holds at that generation is releasable (the workers
+ * runner mints a distinct attempt id per map chunk under one owner + seq). Row
+ * status is irrelevant — the lease strands work receipts and blocks the write
+ * fence whether the row is pending, running, failed, or enriched — but an
+ * unowned row is never touched.
+ */
+export function decideRuntimeSheetRelease(input: {
+  row: RuntimeSheetRowAttemptInput;
+  request: RuntimeSheetAttemptIdentity;
+}): RuntimeSheetReleaseDecision {
+  const rowAttemptId = normalize(input.row.attemptId);
+  if (!rowAttemptId) return { kind: 'skip' };
+  const rowOwner =
+    normalize(input.row.attemptOwnerRunId) ?? normalize(input.row.runId);
+  const requestOwner = normalize(input.request.attemptOwnerRunId);
+  if (!rowOwner || !requestOwner || rowOwner !== requestOwner) {
+    return { kind: 'skip' };
+  }
+  if (
+    normalizeAttemptSeq(input.row.attemptSeq) !==
+    normalizeAttemptSeq(input.request.attemptSeq)
+  ) {
+    return { kind: 'skip' };
+  }
+  const requestAttemptId = normalize(input.request.attemptId);
+  if (requestAttemptId && requestAttemptId !== rowAttemptId) {
+    return { kind: 'skip' };
+  }
+  return { kind: 'release' };
+}
+
+function assertNever(value: never): never {
+  throw new Error(`Unhandled runtime sheet row status: ${String(value)}`);
+}

package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts

@@ -0,0 +1,88 @@
+import { resolveMapRowOutcomeKey } from './map-row-outcome';
+
+export interface RuntimeSheetBlockedRowDetail {
+  rowKey: string | null;
+  attemptId: string | null;
+  attemptOwnerRunId: string | null;
+  attemptExpiresAt: string | null;
+}
+
+function stringField(
+  row: Record<string, unknown>,
+  ...fieldNames: string[]
+): string | null {
+  for (const fieldName of fieldNames) {
+    const value = row[fieldName];
+    if (typeof value === 'string' && value.trim()) {
+      return value;
+    }
+  }
+  return null;
+}
+
+function runtimeSheetBlockedRowDetail(
+  row: Record<string, unknown>,
+): RuntimeSheetBlockedRowDetail {
+  return {
+    rowKey: resolveMapRowOutcomeKey(row),
+    attemptId: stringField(
+      row,
+      'attemptId',
+      '_attempt_id',
+      '__deeplineAttemptId',
+    ),
+    attemptOwnerRunId: stringField(
+      row,
+      'attemptOwnerRunId',
+      '_attempt_owner_run_id',
+      '__deeplineAttemptOwnerRunId',
+    ),
+    attemptExpiresAt: stringField(
+      row,
+      'attemptExpiresAt',
+      '_attempt_expires_at',
+      '__deeplineAttemptExpiresAt',
+    ),
+  };
+}
+
+export class RuntimeSheetRowsBlockedError extends Error {
+  readonly tableNamespace: string;
+  readonly rowKeys: string[];
+  readonly owningAttempts: string[];
+  readonly blockedRowDetails: RuntimeSheetBlockedRowDetail[];
+
+  constructor(input: {
+    tableNamespace: string;
+    blockedRows: Record<string, unknown>[];
+  }) {
+    const blockedRowDetails = input.blockedRows.map(
+      runtimeSheetBlockedRowDetail,
+    );
+    const rowKeys = blockedRowDetails
+      .map((detail) => detail.rowKey)
+      .filter((key): key is string => Boolean(key))
+      .slice(0, 20);
+    const owningAttempts = blockedRowDetails
+      .map((detail) => detail.attemptId)
+      .filter((attempt): attempt is string => Boolean(attempt));
+    const uniqueOwningAttempts = Array.from(new Set(owningAttempts)).slice(
+      0,
+      20,
+    );
+    super(
+      `ctx.dataset("${input.tableNamespace}") is blocked by ${input.blockedRows.length} active runtime sheet row lease(s) owned by another attempt. Retry after the owning attempt finishes or the row leases expire.` +
+        (rowKeys.length > 0
+          ? ` Blocked row keys: ${rowKeys.join(', ')}.`
+          : '') +
+        (uniqueOwningAttempts.length > 0
+          ? ` Owning attempts: ${uniqueOwningAttempts.join(', ')}.`
+          : ''),
+    );
+    this.name = 'RuntimeSheetRowsBlockedError';
+    this.tableNamespace = input.tableNamespace;
+    this.rowKeys = rowKeys;
+    this.owningAttempts = uniqueOwningAttempts;
+    this.blockedRowDetails = blockedRowDetails.slice(0, 20);
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -19,6 +19,7 @@ import type { ExecutionPlan } from './execution-plan';
 import type { PlayRuntimeManifestMap } from '../plays/compiler-manifest';
 import type { PreloadedRuntimeDbSession } from './db-session';
 import type { PlayRunnerRuntimeTiming } from './protocol';
+import type { RuntimeTestPolicyOverrides } from './test-runtime-seams';
 
 export const PLAY_SCHEDULER_BACKENDS = {
   temporal: 'temporal',
@@ -83,7 +84,9 @@ export type PlaySchedulerSubmitInput = {
   artifactHash: string;
   graphHash: string;
   input: Record<string, unknown>;
-  /** Run-level --force cache bypass for dataset rows and ctx.tools.execute receipts. */
+  /** Start a fresh run graph and recompute runtime-sheet rows. */
+  force?: boolean;
+  /** Explicit cache bypass for completed ctx.tools.execute receipts. */
   forceToolRefresh?: boolean | null;
   inputFile?: {
     name?: string;
@@ -149,6 +152,10 @@ export type PlaySchedulerSubmitInput = {
   runtimeDeployVersion?: string | null;
   /** Request-scoped Vercel Deployment Protection bypass for preview runtime callbacks. */
   vercelProtectionBypassToken?: string | null;
+  /** Request-scoped, dev-only runtime fault injection header for black-box durability tests. */
+  runtimeTestFaultHeader?: string | null;
+  /** Request-scoped, dev-only runtime policy overrides for black-box durability tests. */
+  testPolicyOverrides?: RuntimeTestPolicyOverrides | null;
   /** Millisecond epoch timestamp captured immediately before scheduler submit. */
   submittedAtMs?: number;
   /** Scheduler backend that owns orchestration for this run. */

package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts

@@ -0,0 +1,103 @@
+export function activeRuntimeSheetAttemptFenceSql(
+  tableAlias: string,
+  attemptIdExpression: string,
+  attemptOwnerRunIdExpression: string,
+  attemptExpiresAtExpression: string,
+  attemptSeqExpression?: string,
+): string {
+  const activeSameOwnerNewerAttempt =
+    attemptSeqExpression === undefined
+      ? `${tableAlias}._attempt_expires_at < ${attemptExpiresAtExpression}`
+      // Same owner + same sequence is a retry of the same logical run attempt.
+      // It may reclaim non-terminal rows; terminal rows are still guarded by
+      // newerTerminalRuntimeSheetRowSql before this predicate is considered.
+      : `COALESCE(${tableAlias}._attempt_seq, 0) <= ${attemptSeqExpression}::integer`;
+  return `(
+    /* owner: runtime; remove null-attempt compat after ledger cutover M1 */
+    ${tableAlias}._attempt_id IS NULL
+    OR ${tableAlias}._attempt_id = ${attemptIdExpression}
+    OR ${tableAlias}._attempt_expires_at <= now()
+    OR (
+      coalesce(${tableAlias}._attempt_owner_run_id, ${tableAlias}._run_id) = ${attemptOwnerRunIdExpression}
+      AND ${activeSameOwnerNewerAttempt}
+    )
+  )`;
+}
+
+/**
+ * Fence for a controlled attempt-lease RELEASE (run-fatal teardown). Matches
+ * only rows the releasing attempt currently owns: the row must carry an attempt
+ * lease whose owner run id and attempt generation match the releasing run. When
+ * `attemptIdExpression` is a non-null param it further pins the exact attempt id
+ * (single-attempt callers like the in-process app runner); when it is NULL the
+ * predicate releases every attempt id the run holds at that generation (the
+ * workers runner mints a distinct attempt id per map chunk under one owner run
+ * id + seq). It never matches a row owned by a different run, so a release can
+ * never weaken another live attempt's lease.
+ */
+export function releasableRuntimeSheetAttemptFenceSql(
+  tableAlias: string,
+  attemptIdExpression: string,
+  attemptOwnerRunIdExpression: string,
+  attemptSeqExpression: string,
+): string {
+  return `(
+    ${tableAlias}._attempt_id IS NOT NULL
+    AND coalesce(${tableAlias}._attempt_owner_run_id, ${tableAlias}._run_id) = ${attemptOwnerRunIdExpression}
+    AND COALESCE(${tableAlias}._attempt_seq, 0) = ${attemptSeqExpression}::integer
+    AND (${attemptIdExpression} IS NULL OR ${tableAlias}._attempt_id = ${attemptIdExpression})
+  )`;
+}
+
+export function sameOwnerTerminalAttemptEpochSql(
+  tableAlias: string,
+  attemptOwnerRunIdExpression: string,
+  attemptSeqExpression: string,
+): string {
+  return `(
+    ${tableAlias}._status = 'enriched'
+    AND coalesce(${tableAlias}._attempt_owner_run_id, ${tableAlias}._run_id) = ${attemptOwnerRunIdExpression}
+    AND COALESCE(${tableAlias}._attempt_seq, 0) = ${attemptSeqExpression}::integer
+  )`;
+}
+
+export function newerTerminalRuntimeSheetRowSql(
+  tableAlias: string,
+  attemptExpiresAtExpression: string,
+  attemptSeqExpression?: string,
+  attemptIdExpression?: string,
+  attemptOwnerRunIdExpression?: string,
+): string {
+  if (attemptSeqExpression !== undefined) {
+    const sameSeqDifferentAttemptSql = attemptOwnerRunIdExpression
+      ? `OR (
+          COALESCE(${tableAlias}._attempt_seq, 0) = ${attemptSeqExpression}::integer
+          AND ${tableAlias}._attempt_id IS NOT NULL
+          AND ${attemptIdExpression} IS NOT NULL
+          AND ${tableAlias}._attempt_id IS DISTINCT FROM ${attemptIdExpression}
+          AND (
+            coalesce(${tableAlias}._attempt_owner_run_id, ${tableAlias}._run_id) IS DISTINCT FROM ${attemptOwnerRunIdExpression}
+            OR ${tableAlias}._attempt_expires_at > ${attemptExpiresAtExpression}
+          )
+        )`
+      : `OR (
+          COALESCE(${tableAlias}._attempt_seq, 0) = ${attemptSeqExpression}::integer
+          AND ${tableAlias}._attempt_id IS NOT NULL
+          AND ${attemptIdExpression} IS NOT NULL
+          AND ${tableAlias}._attempt_id IS DISTINCT FROM ${attemptIdExpression}
+        )`;
+    return `(
+      ${tableAlias}._status = 'enriched'
+      AND (
+        COALESCE(${tableAlias}._attempt_seq, 0) > ${attemptSeqExpression}::integer
+        ${attemptIdExpression ? sameSeqDifferentAttemptSql : ''}
+      )
+    )`;
+  }
+  return `(
+    ${tableAlias}._status = 'enriched'
+    AND ${tableAlias}._attempt_expires_at IS NOT NULL
+    AND ${attemptExpiresAtExpression} IS NOT NULL
+    AND ${tableAlias}._attempt_expires_at > ${attemptExpiresAtExpression}
+  )`;
+}

package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts

@@ -17,6 +17,20 @@ export type PlayExecutionSuspension =
         eventKey: string;
         timeoutMs: number;
       }>;
+    }
+  | {
+      kind: 'child_play';
+      boundaryId: string;
+      childRunId: string;
+      childPlayName: string;
+    }
+  | {
+      kind: 'child_play_batch';
+      boundaries: Array<{
+        boundaryId: string;
+        childRunId: string;
+        childPlayName: string;
+      }>;
     };
 
 export class PlayExecutionSuspendedError extends Error {
@@ -28,7 +42,11 @@ export class PlayExecutionSuspendedError extends Error {
         ? `Play execution suspended for durable sleep (${suspension.delayMs}ms).`
         : suspension.kind === 'integration_event_batch'
           ? `Play execution suspended waiting for ${suspension.boundaries.length} integration events.`
-          : `Play execution suspended waiting for integration event ${JSON.stringify(suspension.eventKey)}.`,
+          : suspension.kind === 'child_play_batch'
+            ? `Play execution suspended waiting for ${suspension.boundaries.length} child plays.`
+            : suspension.kind === 'child_play'
+              ? `Play execution suspended waiting for child play ${JSON.stringify(suspension.childPlayName)}.`
+              : `Play execution suspended waiting for integration event ${JSON.stringify(suspension.eventKey)}.`,
     );
     this.name = 'PlayExecutionSuspendedError';
     this.suspension = suspension;