deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts

@@ -6,16 +6,17 @@ import {
   deleteModalSandboxById,
   readDetachedModalRuntimeCompletion,
 } from './backends/modal';
+import {
+  readPersistedRuntimeSandboxRef,
+  type RuntimeSandboxProviderId,
+  type RuntimeSandboxRef,
+} from '../runtime-sandbox-contract';
 import type { DetachedRunnerSuspension } from '../suspension';
 
-export type RuntimeSandboxProviderId = 'daytona' | 'modal';
-
-export type RuntimeSandboxRef = {
-  schemaVersion: 1;
-  provider: RuntimeSandboxProviderId;
-  resourceId: string;
-  routingDomain: string | null;
-};
+export type {
+  RuntimeSandboxProviderId,
+  RuntimeSandboxRef,
+} from '../runtime-sandbox-contract';
 
 export type RuntimeSandboxReconciliationAdapter = {
   provider: RuntimeSandboxProviderId;
@@ -138,41 +139,11 @@ export function runtimeSandboxRefFromSuspension(
   const storedRef = (suspension as { runtimeSandboxRef?: unknown })
     .runtimeSandboxRef;
   if (storedRef !== undefined) {
-    if (
-      !storedRef ||
-      typeof storedRef !== 'object' ||
-      Array.isArray(storedRef)
-    ) {
-      throw new Error('Persisted Runtime Sandbox Ref is not an object.');
-    }
-    const candidate = storedRef as Record<string, unknown>;
-    const resourceId =
-      typeof candidate.resourceId === 'string'
-        ? candidate.resourceId.trim()
-        : '';
-    const provider = candidate.provider;
-    const routingDomain = candidate.routingDomain;
-    if (
-      candidate.schemaVersion !== 1 ||
-      (provider !== 'daytona' && provider !== 'modal') ||
-      !resourceId ||
-      resourceId !== suspension.sandboxId.trim() ||
-      (routingDomain !== null && typeof routingDomain !== 'string') ||
-      (typeof routingDomain === 'string' && !routingDomain.trim()) ||
-      (suspension.sandboxProvider !== undefined &&
-        suspension.sandboxProvider !== provider)
-    ) {
-      throw new Error(
-        `Persisted Runtime Sandbox Ref is invalid or inconsistent for sandbox ${suspension.sandboxId}.`,
-      );
-    }
-    return {
-      schemaVersion: 1,
-      provider,
-      resourceId,
-      routingDomain:
-        typeof routingDomain === 'string' ? routingDomain.trim() : null,
-    };
+    return readPersistedRuntimeSandboxRef({
+      raw: storedRef,
+      expectedSandboxId: suspension.sandboxId,
+      expectedProvider: suspension.sandboxProvider,
+    });
   }
   if (
     suspension.sandboxProvider !== undefined &&

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -68,6 +68,7 @@ import {
   stringifyPostgresJson,
 } from './postgres-json';
 import { RECEIPT_STATUS_CODE, receiptStatusFromCode } from './receipt-status';
+import { WORK_RECEIPT_EXECUTION_LOCK_TABLE } from './work-receipt-storage-contract';
 import {
   workReceiptClaimableStatusCodes,
   workReceiptActiveOwnerPredicateSql,
@@ -298,7 +299,6 @@ const dbSessionCache = new Map<string, DbSessionCacheEntry>();
 const dbSessionInFlight = new Map<string, Promise<CreateDbSessionResponse>>();
 const postgresPools = new Map<string, CachedRuntimePostgresPool>();
 const runtimePostgresAdmissions = new Map<string, RuntimePostgresAdmission>();
-const runtimeWorkReceiptEnsureCache = new Map<string, Promise<void>>();
 const runtimeSheetEnsureCache = new Map<
   string,
   { expiresAt: number; promise: Promise<void> }
@@ -313,19 +313,6 @@ const APPEND_KEY_SUFFIX_LENGTH = 12;
 // 60-min alignment fixed. A shorter TTL only causes periodic re-mints (cheap;
 // the tenant runtime role is long-lived), not run failures.
 const RUNTIME_DB_SESSION_TTL_SECONDS = 10 * 60;
-const RUNTIME_WORK_RECEIPT_LEASE_COLUMNS = [
-  'lease_id',
-  'lease_owner_run_id',
-  'lease_owner_attempt',
-  'lease_expires_at',
-] as const;
-const RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN = 'failure_kind';
-const RUNTIME_WORK_RECEIPT_ERROR_PAYLOAD_COLUMN = 'error_payload';
-const RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS = [
-  ...RUNTIME_WORK_RECEIPT_LEASE_COLUMNS,
-  RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN,
-  RUNTIME_WORK_RECEIPT_ERROR_PAYLOAD_COLUMN,
-] as const;
 const RUNTIME_SHEET_ATTEMPT_COLUMNS = [
   '_attempt_id',
   '_attempt_owner_run_id',
@@ -2483,11 +2470,7 @@ function workReceiptTable(session: RuntimePostgresSession): string {
 }
 
 function receiptExecutionLockTable(session: RuntimePostgresSession): string {
-  return fqRuntimeTable(session, '_deepline_receipt_execution_locks');
-}
-
-function postgresUrlCacheKey(value: string): string {
-  return createHash('sha256').update(value).digest('hex').slice(0, 24);
+  return fqRuntimeTable(session, WORK_RECEIPT_EXECUTION_LOCK_TABLE);
 }
 
 function workReceiptKeyHex(key: string): string {
@@ -2548,134 +2531,6 @@ function mapRuntimeWorkReceiptRow(raw: Record<string, unknown>): WorkReceipt {
   };
 }
 
-function isMissingRuntimeWorkReceiptSelfHealColumnError(
-  error: unknown,
-): boolean {
-  if (!error || typeof error !== 'object') {
-    return false;
-  }
-  const code = 'code' in error ? String(error.code) : '';
-  const message = 'message' in error ? String(error.message) : '';
-  if (code !== '42703' && !/column .* does not exist/i.test(message)) {
-    return false;
-  }
-  return RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS.some((column) =>
-    message.includes(column),
-  );
-}
-
-function runtimeWorkReceiptEnsureCacheKey(
-  session: RuntimePostgresSession,
-): string {
-  return `${postgresUrlCacheKey(session.postgresUrl)}::${session.postgres.schema}::${session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE}`;
-}
-
-async function missingRuntimeWorkReceiptSelfHealColumns(
-  session: RuntimePostgresSession,
-  client: RuntimeQueryClient,
-): Promise<string[]> {
-  const result = await client.query(
-    `
-      SELECT column_name
-      FROM information_schema.columns
-      WHERE table_schema = $1
-        AND table_name = $2
-        AND column_name = ANY($3::text[])
-    `,
-    [
-      session.postgres.schema,
-      session.postgres.receiptTable ?? RUNTIME_WORK_RECEIPT_POSTGRES_TABLE,
-      [...RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS],
-    ],
-  );
-  const present = new Set(
-    result.rows.map((row) => String(row.column_name ?? '')),
-  );
-  return RUNTIME_WORK_RECEIPT_SELF_HEAL_COLUMNS.filter(
-    (column) => !present.has(column),
-  );
-}
-
-// COMPAT / SELF-HEAL — owner: runtime. Removal milestone: runtime-ledger
-// cutover. The customer-Neon
-// work-receipt table (lease columns + failure_kind) is the dying surface. The
-// F2 rolling migration (scripts/migrate-play-runtime-storage.ts) backfills these
-// columns; this lazy CREATE/ALTER self-heal is a compat-window safety net and is
-// deleted together with the Neon receipt path at M1.
-async function ensureRuntimeWorkReceiptTable(
-  session: RuntimePostgresSession,
-  client: RuntimeQueryClient,
-): Promise<void> {
-  const cacheKey = runtimeWorkReceiptEnsureCacheKey(session);
-  const cached = runtimeWorkReceiptEnsureCache.get(cacheKey);
-  if (cached) {
-    await cached;
-    return;
-  }
-  const promise = client
-    .query(
-      `
-    CREATE TABLE IF NOT EXISTS ${workReceiptTable(session)} (
-      k bytea PRIMARY KEY,
-      status smallint NOT NULL DEFAULT 0,
-      output jsonb,
-      error text,
-      error_payload jsonb,
-      failure_kind smallint NOT NULL DEFAULT 0,
-      run_id text,
-      lease_id text,
-      lease_owner_run_id text,
-      lease_owner_attempt integer,
-      lease_expires_at timestamptz,
-      updated_at timestamptz NOT NULL DEFAULT now()
-    )
-  `,
-    )
-    .then(async () => {
-      await client.query(`
-        CREATE TABLE IF NOT EXISTS ${receiptExecutionLockTable(session)} (
-          k bytea PRIMARY KEY,
-          owner_execution_id text NOT NULL,
-          expires_at timestamptz NOT NULL,
-          updated_at timestamptz NOT NULL DEFAULT now()
-        )
-      `);
-      const missingColumns = await missingRuntimeWorkReceiptSelfHealColumns(
-        session,
-        client,
-      );
-      if (missingColumns.length === 0) return;
-      await client.query(`
-        ALTER TABLE ${workReceiptTable(session)}
-          ${missingColumns
-            .map((column) => {
-              const type =
-                column === 'lease_expires_at'
-                  ? 'timestamptz'
-                  : column === 'lease_owner_attempt'
-                    ? 'integer'
-                    : column === RUNTIME_WORK_RECEIPT_FAILURE_KIND_COLUMN
-                      ? 'smallint NOT NULL DEFAULT 0'
-                      : column === RUNTIME_WORK_RECEIPT_ERROR_PAYLOAD_COLUMN
-                        ? 'jsonb'
-                        : 'text';
-              return `ADD COLUMN IF NOT EXISTS ${column} ${type}`;
-            })
-            .join(',\n          ')}
-      `);
-    })
-    .then(() => undefined);
-  runtimeWorkReceiptEnsureCache.set(cacheKey, promise);
-  try {
-    await promise;
-  } catch (error) {
-    if (runtimeWorkReceiptEnsureCache.get(cacheKey) === promise) {
-      runtimeWorkReceiptEnsureCache.delete(cacheKey);
-    }
-    throw error;
-  }
-}
-
 export async function acquireRuntimeReceiptExecutionLock(
   context: RuntimeApiContext,
   input: {
@@ -2752,13 +2607,10 @@ async function withRuntimeWorkReceiptClient<T>(
   session: RuntimePostgresSession,
   operation: (client: RuntimeQueryClient) => Promise<T>,
 ): Promise<T> {
-  // Receipt tables are part of customer Postgres bootstrap, so the hot path
-  // should not pay DDL on every fresh runtime isolate. Try the receipt query
-  // first, then self-heal once on a missing relation/column. Permission loss
-  // during an operational receipt query must fail loud: silently repairing it
-  // can let provider work continue after durable receipt persistence is gone.
-  // Only the schema-migration branch below may repair ownership before its
-  // retry. A genuinely missing schema or second failure also rethrows loudly.
+  // Receipt tables are customer-Postgres bootstrap state. The operation path
+  // never performs DDL or schema repair: a missing table/column is the real
+  // operation failure and must remain visible to the caller. We retry only
+  // transient connection/transport failures, never a semantic SQL error.
   const withClient = async <R>(
     run: (client: RuntimeQueryClient) => Promise<R>,
   ): Promise<R> =>
@@ -2768,8 +2620,6 @@ async function withRuntimeWorkReceiptClient<T>(
           cachePool: !context.disablePostgresPoolCache,
           signal: context.abortSignal,
         });
-  let selfHealAttempted = false;
-
   for (
     let attempt = 1;
     attempt <= RUNTIME_WORK_RECEIPT_QUERY_MAX_ATTEMPTS;
@@ -2778,34 +2628,6 @@ async function withRuntimeWorkReceiptClient<T>(
     try {
       return await withClient(operation);
     } catch (error) {
-      const missingStorage =
-        isMissingRelationError(error) ||
-        isMissingRuntimeWorkReceiptSelfHealColumnError(error);
-      if (missingStorage && !selfHealAttempted) {
-        selfHealAttempted = true;
-        runtimeWorkReceiptEnsureCache.delete(
-          runtimeWorkReceiptEnsureCacheKey(session),
-        );
-        try {
-          await withClient((client) =>
-            ensureRuntimeWorkReceiptTable(session, client),
-          );
-        } catch (ensureError) {
-          if (!isPostgresPermissionDeniedError(ensureError)) {
-            throw ensureError;
-          }
-          runtimeWorkReceiptEnsureCache.delete(
-            runtimeWorkReceiptEnsureCacheKey(session),
-          );
-          await repairRuntimeStorageGrants(context, {
-            playName: context.playName?.trim() || session.playName,
-          });
-          await withClient((client) =>
-            ensureRuntimeWorkReceiptTable(session, client),
-          );
-        }
-        continue;
-      }
       if (
         attempt >= RUNTIME_WORK_RECEIPT_QUERY_MAX_ATTEMPTS ||
         !isTransientRuntimePostgresOperationError(error)
@@ -3361,8 +3183,8 @@ async function writeRuntimeRows(
         const insertedRowsCte =
           input.mode === 'upsert'
             ? `inserted_rows AS (
-                INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${physicalInsertColumnsSql})
-                SELECT _key, 'pending', $4::text, _input_index${physicalInsertValuesSql}
+                INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _created_at${physicalInsertColumnsSql})
+                SELECT _key, 'pending', $4::text, _input_index, now()${physicalInsertValuesSql}
                 FROM input_rows
                 ON CONFLICT (_key) DO UPDATE SET
                   _status = 'pending',
@@ -3374,8 +3196,8 @@ async function writeRuntimeRows(
                 RETURNING _key
               ),`
             : `inserted_rows AS (
-                INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index${physicalInsertColumnsSql})
-                SELECT _key, 'pending', $4::text, _input_index${physicalInsertValuesSql}
+                INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _created_at${physicalInsertColumnsSql})
+                SELECT _key, 'pending', $4::text, _input_index, now()${physicalInsertValuesSql}
                 FROM input_rows
                 RETURNING _key
               ),`;
@@ -3660,8 +3482,8 @@ async function prepareRuntimeSheetDatasetRows(
           RETURNING target._key
         ),
         inserted_rows AS (
-          INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq, _write_version, _writer_run_id${input.physicalInsertColumnsSql})
-          SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer, $12::bigint, $4::text${input.physicalInsertValuesSql}
+          INSERT INTO ${sheetTable(session)} (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id, _attempt_expires_at, _attempt_seq, _write_version, _writer_run_id, _created_at${input.physicalInsertColumnsSql})
+          SELECT _key, 'pending', $4::text, _input_index, $7::text, $8::text, $9::timestamptz, $10::integer, $12::bigint, $4::text, now()${input.physicalInsertValuesSql}
           FROM input_rows
           ON CONFLICT (_key) DO UPDATE SET
             _status = 'pending',
@@ -3991,9 +3813,9 @@ async function prepareNetNewRuntimeSheetDatasetRows(
        inserted_rows AS (
          INSERT INTO ${sheetTable(session)}
          (_key, _status, _run_id, _input_index, _attempt_id, _attempt_owner_run_id,
-          _attempt_expires_at, _attempt_seq, _write_version, _writer_run_id${input.physicalInsertColumnsSql})
+          _attempt_expires_at, _attempt_seq, _write_version, _writer_run_id, _created_at${input.physicalInsertColumnsSql})
        SELECT _key, 'pending', $4::text, _input_index, $5::text, $6::text,
-              $7::timestamptz, $8::integer, $9::bigint, $4::text${input.physicalInsertValuesSql}
+              $7::timestamptz, $8::integer, $9::bigint, $4::text, now()${input.physicalInsertValuesSql}
        FROM input_rows
        ON CONFLICT (_key) DO NOTHING
        RETURNING _key
@@ -7047,11 +6869,6 @@ async function completeRuntimeMapRowChunks(
         OR coalesce(target._attempt_owner_run_id, target._run_id) IS DISTINCT FROM $10::text
       )
     )`;
-    const sameOwnerTerminalEpochSql = sameOwnerTerminalAttemptEpochSql(
-      'target',
-      '$10::text',
-      '$12',
-    );
     const { rows } = await client.query<{
       updated: number;
       matched_keys: string[];
@@ -7106,6 +6923,15 @@ async function completeRuntimeMapRowChunks(
               target._status <> 'enriched'
               OR (${targetChangedPatchedCellSql})
             )
+            -- First terminal wins: suppress conflict for same owner+epoch already-terminal row.
+            -- Durable replay (omitted_terminal_replay) from same attempt epoch is idempotent.
+            AND NOT (
+              $10::text IS NOT NULL
+              AND $12::integer IS NOT NULL
+              AND target._status = 'enriched'
+              AND coalesce(target._attempt_owner_run_id, target._run_id) = $10::text
+              AND COALESCE(target._attempt_seq, 0) = $12::integer
+            )
        ),
        applied_rows AS (
          UPDATE ${sheetTable(session)} AS target
@@ -7127,13 +6953,21 @@ async function completeRuntimeMapRowChunks(
 		           AND ($13::bigint IS NULL OR target._write_version = $13::bigint)
                    AND ($13::bigint IS NOT NULL OR target._status <> 'enriched')
 		           AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
-           AND NOT (${sameOwnerTerminalEpochSql})
            AND (
              ($9::text IS NULL AND target._run_id = $5::text)
              OR ($9::text IS NOT NULL AND ${writableNonTerminalAttemptSql})
              OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
 		             OR ($9::text IS NOT NULL AND ${writableEnrichedAttemptSql})
            )
+           -- First terminal wins: skip UPDATE for already-terminal same-owner+epoch row.
+           -- Even though we excluded from conflicts, we still no-op here for idempotency.
+           AND NOT (
+             $10::text IS NOT NULL
+             AND $12::integer IS NOT NULL
+             AND target._status = 'enriched'
+             AND coalesce(target._attempt_owner_run_id, target._run_id) = $10::text
+             AND COALESCE(target._attempt_seq, 0) = $12::integer
+           )
            AND (
              target._status <> 'enriched'
              OR target._run_id IS DISTINCT FROM $5::text
@@ -7339,11 +7173,6 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
       '$11::timestamptz',
       '$12',
     );
-    const sameOwnerTerminalEpochSql = sameOwnerTerminalAttemptEpochSql(
-      'target',
-      '$10::text',
-      '$12',
-    );
     const { rows } = await client.query<{
       _key?: string;
       fenced_keys?: string[];
@@ -7403,6 +7232,15 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
               target._status <> 'enriched'
               OR (${targetChangedPatchedCellSql})
             )
+            -- First terminal wins: suppress conflict for same owner+epoch already-terminal row.
+            -- Durable replay (omitted_terminal_replay) from same attempt epoch is idempotent.
+            AND NOT (
+              $10::text IS NOT NULL
+              AND $12::integer IS NOT NULL
+              AND target._status = 'enriched'
+              AND coalesce(target._attempt_owner_run_id, target._run_id) = $10::text
+              AND COALESCE(target._attempt_seq, 0) = $12::integer
+            )
        ),
        applied_rows AS (
          UPDATE ${sheetTable(session)} AS target
@@ -7425,7 +7263,6 @@ async function completeRuntimeMapRowChunksWithInputIndexRepair(
            AND ($13::bigint IS NULL OR target._write_version = $13::bigint)
            AND ($13::bigint IS NOT NULL OR target._status <> 'enriched')
 		           AND NOT (${newerTerminalRuntimeSheetRowSql('target', '$11::timestamptz', '$12::integer', '$9::text', '$10::text')})
-           AND NOT (${sameOwnerTerminalEpochSql})
            AND (
              ($9::text IS NULL AND target._run_id = $5::text)
              OR ($9::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$9::text', '$10::text', '$11::timestamptz', '$12::integer')})
@@ -7642,6 +7479,7 @@ async function insertMissingCompletedMapRowChunks(
            _attempt_owner_run_id,
            _attempt_expires_at,
            _attempt_seq,
+           _created_at,
            _cell_meta${physicalInsertColumnsSql}
          )
          SELECT _key,
@@ -7652,6 +7490,7 @@ async function insertMissingCompletedMapRowChunks(
                 $9::text,
                 $10::timestamptz,
                 $11::integer,
+                now(),
                 cell_meta_patch${physicalInsertValuesSql}
            FROM missing_rows
          ON CONFLICT (_key) DO NOTHING
@@ -7860,6 +7699,16 @@ async function failRuntimeMapRowChunks(
               OR target._error IS DISTINCT FROM updates.error
               OR (${targetChangedPatchedCellSql})
             )
+            -- Same-owner terminal replay is not a conflict. Normal replay
+            -- preserves the first value below; explicit force-failure may update it.
+            -- Different owners and attempt epochs still conflict.
+            AND NOT (
+              $12::text IS NOT NULL
+              AND $14::integer IS NOT NULL
+              AND target._status IN ('failed', 'enriched')
+              AND coalesce(target._attempt_owner_run_id, target._run_id) = $12::text
+              AND COALESCE(target._attempt_seq, 0) = $14::integer
+            )
        ),
        applied_rows AS (
          UPDATE ${sheetTable(session)} AS target
@@ -7886,13 +7735,15 @@ async function failRuntimeMapRowChunks(
              ($11::text IS NULL AND target._run_id = $6::text)
              OR ($11::text IS NOT NULL AND ${activeRuntimeSheetAttemptFenceSql('target', '$11::text', '$12::text', '$13::timestamptz', '$14::integer')})
 	           )
+           -- First terminal wins for failed rows too, BUT only when NOT force-failing.
+           -- Normal replay: no-op when already-failed OR already-enriched from same owner+epoch.
            AND NOT (
-             $16::bigint IS NOT NULL
-             AND target._write_version = $16::bigint
-             AND target._run_id = $6::text
-             AND target._status = 'failed'
-             AND target._error IS NOT DISTINCT FROM updates.error
-             AND NOT (${targetChangedPatchedCellSql})
+             $15::boolean = FALSE
+             AND $12::text IS NOT NULL
+             AND $14::integer IS NOT NULL
+             AND target._status IN ('failed', 'enriched')
+             AND coalesce(target._attempt_owner_run_id, target._run_id) = $12::text
+             AND COALESCE(target._attempt_seq, 0) = $14::integer
            )
          RETURNING target._key, prev._status AS prev_status
        ),
@@ -8004,9 +7855,12 @@ async function failRuntimeMapRowChunks(
         input.writeVersion,
       ],
     );
-    updated += Number(rows[0]?.updated ?? 0);
-    fencedKeys.push(...(rows[0]?.fenced_keys ?? []));
-    conflictKeys.push(...(rows[0]?.conflict_keys ?? []));
+    const appliedCount = Number(rows[0]?.updated ?? 0);
+    const chunkFencedKeys = rows[0]?.fenced_keys ?? [];
+    const chunkConflictKeys = rows[0]?.conflict_keys ?? [];
+    updated += appliedCount;
+    fencedKeys.push(...chunkFencedKeys);
+    conflictKeys.push(...chunkConflictKeys);
   }
   return { updated, fencedKeys, conflictKeys };
 }