deepline 0.3.174 → 0.3.176

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -25,6 +25,13 @@ import {
   type PhysicalSheetColumnProjection,
 } from '../play-data-plane/sheet-contract';
 import { RUNTIME_SHEET_CELL_META_COLUMN } from '../play-data-plane/runtime-sheet-storage-contract';
+import {
+  createRuntimeSheetOrm,
+  isRuntimeSheetOwnerMetadataConstraintError,
+  RUNTIME_SHEET_ATTEMPT_COLUMNS,
+  RUNTIME_SHEET_TIMESTAMP_COLUMNS,
+  RUNTIME_SHEET_VERSION_SEQUENCE,
+} from '../play-data-plane/runtime-sheet-orm';
 import type {
   CreateDbSessionResponse,
   DbLogicalTable,
@@ -316,13 +323,6 @@ const APPEND_KEY_SUFFIX_LENGTH = 12;
 // 60-min alignment fixed. A shorter TTL only causes periodic re-mints (cheap;
 // the tenant runtime role is long-lived), not run failures.
 const RUNTIME_DB_SESSION_TTL_SECONDS = 10 * 60;
-const RUNTIME_SHEET_ATTEMPT_COLUMNS = [
-  '_attempt_id',
-  '_attempt_owner_run_id',
-  '_attempt_seq',
-  '_attempt_expires_at',
-] as const;
-const RUNTIME_SHEET_TIMESTAMP_COLUMNS = ['_created_at', '_updated_at'] as const;
 const RUNTIME_SHEET_ENSURE_CACHE_TTL_MS = 10 * 60_000;
 const RUNTIME_SHEET_READY_AFTER_ENSURE_RETRY_DELAYS_MS = [
   100, 250, 500, 1_000, 2_000,
@@ -519,6 +519,7 @@ type RuntimeApiActionRequest =
       tableNamespace: string;
       sheetContract?: PlaySheetContract | null;
       userEmail?: string | null;
+      forceRepair?: boolean;
     }
   | {
       action: 'create_db_session';
@@ -1135,62 +1136,21 @@ async function isRuntimeSheetSchemaReady(
        FROM ${sheetTable(session)}
        LIMIT 0`,
     );
-    const attemptColumns = await client.query<Record<string, unknown>>(
-      `
-        SELECT count(*)::int AS present_count
-        FROM information_schema.columns
-        WHERE table_schema = $1
-          AND table_name = $2
-          AND column_name = ANY($3::text[])
-      `,
-      [
-        session.postgres.schema,
-        session.postgres.sheetTable,
-        [...RUNTIME_SHEET_ATTEMPT_COLUMNS],
-      ],
-    );
-    const timestampColumns = await client.query<Record<string, unknown>>(
-      `
-        SELECT count(*)::int AS ready_count
-        FROM information_schema.columns
-        WHERE table_schema = $1
-          AND table_name = $2
-          AND column_name = ANY($3::text[])
-          AND data_type = 'timestamp with time zone'
-          AND is_nullable = 'NO'
-          AND column_default IS NOT NULL
-      `,
-      [
-        session.postgres.schema,
-        session.postgres.sheetTable,
-        [...RUNTIME_SHEET_TIMESTAMP_COLUMNS],
-      ],
-    );
-    const cellMetaColumn = await client.query<Record<string, unknown>>(
-      `
-        SELECT count(*)::int AS ready_count
-        FROM information_schema.columns
-        WHERE table_schema = $1
-          AND table_name = $2
-          AND column_name = $3
-          AND data_type = $4
-          AND is_nullable = 'NO'
-          AND column_default IS NOT NULL
-      `,
-      [
-        session.postgres.schema,
-        session.postgres.sheetTable,
-        RUNTIME_SHEET_CELL_META_COLUMN.name,
-        RUNTIME_SHEET_CELL_META_COLUMN.dataType,
-      ],
-    );
-    return (
-      Number(attemptColumns.rows[0]?.present_count ?? 0) ===
-        RUNTIME_SHEET_ATTEMPT_COLUMNS.length &&
-      Number(timestampColumns.rows[0]?.ready_count ?? 0) ===
-        RUNTIME_SHEET_TIMESTAMP_COLUMNS.length &&
-      Number(cellMetaColumn.rows[0]?.ready_count ?? 0) === 1
-    );
+    const orm = createRuntimeSheetOrm({
+      schema: session.postgres.schema,
+      table: session.postgres.sheetTable,
+      query: async <Row = Record<string, unknown>>(
+        sql: string,
+        params?: readonly unknown[],
+      ): Promise<{ rows: Row[] }> => {
+        const result = await client.query<Row & Record<string, unknown>>(
+          sql,
+          params === undefined ? undefined : [...params],
+        );
+        return { rows: result.rows as Row[] };
+      },
+    });
+    return (await orm.readiness()).ready;
   };
   try {
     if (isRuntimeOneShotQueryFactoryRegistered()) {
@@ -1250,15 +1210,31 @@ async function readRuntimeSheetSchemaDiagnostic(
         `${quoteIdentifier(session.postgres.schema)}.${quoteIdentifier(session.postgres.sheetTable)}`,
       ],
     );
+    const orm = createRuntimeSheetOrm({
+      schema: session.postgres.schema,
+      table: session.postgres.sheetTable,
+      query: async <Row = Record<string, unknown>>(
+        sql: string,
+        params?: readonly unknown[],
+      ): Promise<{ rows: Row[] }> => {
+        const result = await client.query<Row & Record<string, unknown>>(
+          sql,
+          params === undefined ? undefined : [...params],
+        );
+        return { rows: result.rows as Row[] };
+      },
+    });
+    const ownerSchema = await orm.inspect();
+    const ownerReadiness = await orm.readiness(ownerSchema);
     const visible = new Set(visibleColumns.rows.map((row) => row.column_name));
     const timestampColumns = new Map(
-      visibleColumns.rows
-        .filter((row) =>
-          RUNTIME_SHEET_TIMESTAMP_COLUMNS.includes(
-            row.column_name as (typeof RUNTIME_SHEET_TIMESTAMP_COLUMNS)[number],
-          ),
-        )
-        .map((row) => [row.column_name, row]),
+      [...ownerSchema.columns.entries()].filter(([column]) =>
+        RUNTIME_SHEET_TIMESTAMP_COLUMNS.includes(column),
+      ),
+    );
+    const versionColumn = ownerSchema.columns.get('_version');
+    const cellMetaColumn = ownerSchema.columns.get(
+      RUNTIME_SHEET_CELL_META_COLUMN.name,
     );
     return {
       schema: session.postgres.schema,
@@ -1268,6 +1244,16 @@ async function readRuntimeSheetSchemaDiagnostic(
       canSelect: privileges.rows[0]?.can_select ?? null,
       canInsert: privileges.rows[0]?.can_insert ?? null,
       canUpdate: privileges.rows[0]?.can_update ?? null,
+      ownerSchemaReady: ownerReadiness.ready,
+      ownerSchemaMissingColumns: ownerReadiness.missingColumns,
+      ownerSchemaIncompatibleColumns: ownerReadiness.incompatibleColumns,
+      ownerSchemaMissingDefaults: ownerReadiness.missingDefaults,
+      versionSequenceExists: ownerSchema.sequenceExists,
+      versionColumnState: {
+        dataType: versionColumn?.data_type ?? null,
+        isNullable: versionColumn?.is_nullable ?? null,
+        hasDefault: versionColumn?.column_default != null,
+      },
       visibleColumnCount: visible.size,
       missingAttemptColumns: RUNTIME_SHEET_ATTEMPT_COLUMNS.filter(
         (column) => !visible.has(column),
@@ -1282,14 +1268,10 @@ async function readRuntimeSheetSchemaDiagnostic(
         };
       }),
       cellMetaColumnState: (() => {
-        const state = visibleColumns.rows.find(
-          (column) =>
-            column.column_name === RUNTIME_SHEET_CELL_META_COLUMN.name,
-        );
         return {
-          dataType: state?.data_type ?? null,
-          isNullable: state?.is_nullable ?? null,
-          hasDefault: state?.column_default != null,
+          dataType: cellMetaColumn?.data_type ?? null,
+          isNullable: cellMetaColumn?.is_nullable ?? null,
+          hasDefault: cellMetaColumn?.column_default != null,
         };
       })(),
     };
@@ -2483,6 +2465,7 @@ async function withRuntimeSheetProvisioningRetry<T>(
     playName: string;
     tableNamespace: string;
     sheetContract: PlaySheetContract;
+    session: RuntimePostgresSession;
     timings?: RuntimeSheetTiming[];
   },
   operation: () => Promise<T>,
@@ -2505,12 +2488,33 @@ async function withRuntimeSheetProvisioningRetry<T>(
     });
     if (isMissingRelationError(error)) {
       const ensureStartedAt = Date.now();
-      await ensureRuntimeSheet(context, input);
+      await ensureRuntimeSheet(context, {
+        playName: input.playName,
+        tableNamespace: input.tableNamespace,
+        sheetContract: input.sheetContract,
+      });
       input.timings?.push({
         phase: 'ensure_sheet_after_provisioning_error',
         ms: Date.now() - ensureStartedAt,
         retried: true,
       });
+    } else if (
+      isRuntimeSheetOwnerMetadataConstraintError(error, {
+        table: input.session.postgres.sheetTable,
+      })
+    ) {
+      const ensureStartedAt = Date.now();
+      await ensureRuntimeSheet(context, {
+        playName: input.playName,
+        tableNamespace: input.tableNamespace,
+        sheetContract: input.sheetContract,
+        forceRepair: true,
+      });
+      input.timings?.push({
+        phase: 'force_repair_sheet_after_metadata_constraint_error',
+        ms: Date.now() - ensureStartedAt,
+        retried: true,
+      });
     } else if (isPostgresPermissionDeniedError(error)) {
       const repairStartedAt = Date.now();
       await repairRuntimeStorageGrants(context, {
@@ -2551,7 +2555,7 @@ async function withRuntimeSheetQueryClient<T>(
   const totalStartedAt = Date.now();
   const result = await withRuntimeSheetProvisioningRetry(
     context,
-    input,
+    { ...input, session },
     async () => {
       if (
         !input.transactional &&
@@ -2865,12 +2869,10 @@ async function withRuntimeWorkReceiptClient<T>(
   throw new Error('Runtime work receipt query failed after retries.');
 }
 
-const PLAY_INTERNAL_SHEET_VERSION_SEQUENCE = '_deepline_sheet_version_seq';
-
 function nextRuntimeSheetVersionExpression(
   session: RuntimePostgresSession,
 ): string {
-  return `nextval(${quoteLiteral(`${session.postgres.schema}.${PLAY_INTERNAL_SHEET_VERSION_SEQUENCE}`)}::regclass)`;
+  return `nextval(${quoteLiteral(`${session.postgres.schema}.${RUNTIME_SHEET_VERSION_SEQUENCE}`)}::regclass)`;
 }
 
 function missingOutputCellSql(
@@ -3476,6 +3478,23 @@ async function writeRuntimeRows(
   });
 }
 
+async function writeRuntimeRowsWithProvisioningRetry(
+  context: RuntimeApiContext,
+  session: RuntimePostgresSession,
+  input: Parameters<typeof writeRuntimeRows>[1],
+): Promise<RowsWriteResponse> {
+  return await withRuntimeSheetProvisioningRetry(
+    context,
+    {
+      playName: session.playName,
+      tableNamespace: input.tableNamespace,
+      sheetContract: input.sheetContract,
+      session,
+    },
+    () => writeRuntimeRows(session, input),
+  );
+}
+
 export async function resolveRuntimeReferencedPlay(
   context: RuntimeApiContext,
   playRef: string,
@@ -3509,6 +3528,7 @@ export async function ensureRuntimeSheet(
     playName: string;
     tableNamespace: string;
     sheetContract: PlaySheetContract;
+    forceRepair?: boolean;
   },
 ): Promise<void> {
   const request = {
@@ -3516,6 +3536,7 @@ export async function ensureRuntimeSheet(
     ...input,
     runId: context.runId ?? null,
     userEmail: normalizeRuntimeUserEmail(context.userEmail),
+    ...(input.forceRepair === true ? { forceRepair: true } : {}),
   };
   try {
     await postRuntimeApi<{ ok: true }>(context, request);
@@ -9124,18 +9145,22 @@ export async function upsertRuntimeRows(
     },
   );
   return rowsWriteResponseSchema.parse(
-    await writeRuntimeRows(requireRuntimePostgresSession(session), {
-      tableNamespace: input.tableNamespace,
-      rows: input.rows,
-      runId: input.runId,
-      sheetContract: input.sheetContract,
-      idempotencyKey: createHash('sha1')
-        .update(
-          `${input.playName}:${input.tableNamespace}:${input.runId}:${JSON.stringify(input.rows)}`,
-        )
-        .digest('hex'),
-      mode: 'upsert',
-    }),
+    await writeRuntimeRowsWithProvisioningRetry(
+      context,
+      requireRuntimePostgresSession(session),
+      {
+        tableNamespace: input.tableNamespace,
+        rows: input.rows,
+        runId: input.runId,
+        sheetContract: input.sheetContract,
+        idempotencyKey: createHash('sha1')
+          .update(
+            `${input.playName}:${input.tableNamespace}:${input.runId}:${JSON.stringify(input.rows)}`,
+          )
+          .digest('hex'),
+        mode: 'upsert',
+      },
+    ),
   );
 }
 
@@ -9170,14 +9195,18 @@ export async function appendRuntimeRows(
     },
   );
   return rowsWriteResponseSchema.parse(
-    await writeRuntimeRows(requireRuntimePostgresSession(session), {
-      tableNamespace: input.tableNamespace,
-      rows: input.rows,
-      runId: input.runId,
-      sheetContract: input.sheetContract,
-      idempotencyKey: input.idempotencyKey,
-      mode: 'append',
-    }),
+    await writeRuntimeRowsWithProvisioningRetry(
+      context,
+      requireRuntimePostgresSession(session),
+      {
+        tableNamespace: input.tableNamespace,
+        rows: input.rows,
+        runId: input.runId,
+        sheetContract: input.sheetContract,
+        idempotencyKey: input.idempotencyKey,
+        mode: 'append',
+      },
+    ),
   );
 }
 
@@ -9212,14 +9241,18 @@ export async function replaceRuntimeRows(
     },
   );
   return rowsWriteResponseSchema.parse(
-    await writeRuntimeRows(requireRuntimePostgresSession(session), {
-      tableNamespace: input.tableNamespace,
-      rows: input.rows,
-      runId: input.runId,
-      sheetContract: input.sheetContract,
-      idempotencyKey: input.idempotencyKey,
-      mode: 'replace',
-    }),
+    await writeRuntimeRowsWithProvisioningRetry(
+      context,
+      requireRuntimePostgresSession(session),
+      {
+        tableNamespace: input.tableNamespace,
+        rows: input.rows,
+        runId: input.runId,
+        sheetContract: input.sheetContract,
+        idempotencyKey: input.idempotencyKey,
+        mode: 'replace',
+      },
+    ),
   );
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd.ts

@@ -14,33 +14,27 @@
  *             Persist the absurd task id
  *             on the run row so a later cancel can target it.
  *   observe/result → reuse the postgres LISTEN/NOTIFY run handle.
- *   cancel  → durable Absurd cancellation lane + bounded postgres terminal
- *             projection (with worker reconciliation when locks are busy).
+ *   cancel  → one bounded transaction cancels the Absurd task and terminalizes
+ *             the postgres scheduler run. Success means both states committed.
  *   signal  → durable signal row (authoritative payload) + absurd emitEvent
  *             (the wakeup only; payload is re-read from the signal row).
  */
 import { recordPerfTrace } from '../perf-trace';
 import { CTX_RUN_PLAY_INLINE_ONLY } from '../child-execution-strategy';
 import { PLAY_ADMISSION_MAX_ACTIVE_LIFECYCLES } from '../admission-transport-policy';
-import {
-  defaultRuntimeSandboxReconciliation,
-  runtimeSandboxDeleteSucceeded,
-  runtimeSandboxRefIdentityKey,
-  runtimeSandboxRefFromResource,
-  type RuntimeSandboxRef,
-} from '../runner-backends/runtime-sandbox-reconciliation';
 import {
   PLAY_SCHEDULER_BACKENDS,
-  RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE,
   type PlaySchedulerBackend,
   type PlaySchedulerRunHandle,
   type PlaySchedulerSubmitInput,
 } from '../scheduler-backend';
 import {
+  cancelPostgresSchedulerRun,
+  postgresSchedulerCancellationOutboxLease,
   postgresSchedulerRunHandle,
   readPostgresSchedulerLaunch,
   readPostgresSchedulerAbsurdTaskId,
-  readPostgresSchedulerRuntimeResourcesForRun,
+  readPostgresSchedulerResult,
   recordPostgresSchedulerSignal,
 } from './postgres';
 import {
@@ -57,7 +51,6 @@ import { withRuntimeSchedulerSubmitAdmission } from './submit-admission';
 import {
   ABSURD_PLAY_RUN_MAX_ATTEMPTS,
   ABSURD_PLAY_RUN_RETRY_STRATEGY,
-  ABSURD_DAYTONA_SANDBOX_CLEANUP_TASK,
   absurdIntegrationEventName,
   absurdQueueNameForRelease,
   absurdReleaseIdForLaunch,
@@ -67,7 +60,6 @@ import {
   normalizeAbsurdSchedulerOptions,
   resolveAbsurdPoolOptions,
   serverlessAdmissionSubmitMaxActive,
-  type AbsurdDaytonaSandboxCleanupTaskParams,
   type AbsurdSchedulerOptions,
 } from './absurd-shared';
 
@@ -162,168 +154,98 @@ function absurdRunHandle(input: {
   const runsQueue = absurdQueueNameForRelease(releaseId);
   const cancellationKey = `${poolOptions.url}::${effectiveOptions.schema ?? ''}::${input.runId}`;
 
-  const cancelAbsurdTask = async () => {
-    const taskId = await withRuntimeSchedulerClient(poolOptions, (client) =>
-      readPostgresSchedulerAbsurdTaskId(client, input.runId, effectiveOptions),
-    );
-    if (!taskId) return;
-    // Keep cancellation on the same bounded control-plane contract as the
-    // scheduler terminal write. This is a durable lane transition, not an
-    // unbounded best-effort query which can pile up behind a worker lock.
-    await withRuntimeSchedulerTransaction(
+  const cancelAbsurdRunSynchronously = async (reason?: string) =>
+    withRuntimeSchedulerTransaction(
       poolOptions,
       {
         lockTimeoutMs: RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
         statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
       },
       async (client) => {
-        await client.query(`SELECT absurd.cancel_task($1, $2::uuid)`, [
-          runsQueue,
-          taskId,
-        ]);
-        // Older v4 Absurd functions do not materialize a manual cancellation
-        // deadline or reason. Stamp both in the adapter so existing queues
-        // participate in the same durable reconciliation lane without a
-        // schema cutover. The explicit marker keeps a user cancel distinct
-        // from a task cancelled by its configured max_delay sweep.
-        await client.query(
-          `UPDATE absurd.${quotePostgresIdentifier(`t_${runsQueue}`)}
+        // Resolve the immutable release lane inside the same checkout and
+        // transaction that performs cancellation. The stop endpoint therefore
+        // has no preliminary lane lookup that can succeed or fail separately.
+        const launch = await readPostgresSchedulerLaunch(
+          client,
+          input.runId,
+          effectiveOptions,
+          { caller: 'play_run_control' },
+        );
+        if (!launch) {
+          throw new Error(
+            `Cannot cancel play run "${input.runId}": scheduler launch is missing.`,
+          );
+        }
+        const cancellationReleaseId = absurdReleaseIdForLaunch(launch);
+        const cancellationQueue = absurdQueueNameForRelease(
+          cancellationReleaseId,
+        );
+        const taskId = await readPostgresSchedulerAbsurdTaskId(
+          client,
+          input.runId,
+          effectiveOptions,
+        );
+        if (taskId) {
+          await client.query(`SELECT absurd.cancel_task($1, $2::uuid)`, [
+            cancellationQueue,
+            taskId,
+          ]);
+          // Older v4 Absurd functions do not materialize a manual cancellation
+          // deadline or reason. Stamp both in the adapter so existing queues
+          // participate in the same durable reconciliation lane without a
+          // schema cutover. The explicit marker keeps a user cancel distinct
+          // from a task cancelled by its configured max_delay sweep.
+          await client.query(
+            `UPDATE absurd.${quotePostgresIdentifier(`t_${cancellationQueue}`)}
               SET cancel_at = COALESCE(cancel_at, clock_timestamp()),
                   cancellation = COALESCE(cancellation, '{}'::jsonb)
                     || jsonb_build_object('manual', true)
             WHERE task_id = $1::uuid
               AND state = 'cancelled'`,
-          [taskId],
+            [taskId],
+          );
+        }
+        const cancelled = await cancelPostgresSchedulerRun(
+          client,
+          {
+            runId: input.runId,
+            reason,
+            outboxLease: postgresSchedulerCancellationOutboxLease(input.runId),
+          },
+          effectiveOptions,
         );
+        // The update returning a row is proof that this transaction wrote the
+        // terminal state and outbox event. Only the idempotent retry/race path
+        // needs one read to prove another terminal writer already won.
+        if (!cancelled) {
+          const terminal = await readPostgresSchedulerResult(
+            client,
+            input.runId,
+            effectiveOptions,
+          );
+          if (!terminal) {
+            throw new Error(
+              `Cancellation did not terminalize scheduler run "${input.runId}".`,
+            );
+          }
+        }
       },
       {
         runId: input.runId,
-        phase: 'cancel_engine_task',
+        phase: 'cancel_run_synchronously',
         priority: 'critical',
       },
-      { maxAttempts: 1 },
     );
-  };
-
-  const cancellationPendingError = (cause: unknown): Error => {
-    const error = new Error(
-      `Run cancellation was durably requested, but scheduler terminalization is pending for ${input.runId}.`,
-    );
-    Object.assign(error, {
-      code: RUNTIME_SCHEDULER_CANCELLATION_PENDING_CODE,
-      cause,
-    });
-    return error;
-  };
 
   const requestCancellation = async (reason?: string) => {
     await runCancellationSingleFlight(cancellationKey, async () => {
-      // First move the durable engine task into its cancellation lane. If the
-      // scheduler tables are briefly locked, the worker reconciler can finish
-      // the terminal write later without another user request piling up.
-      await cancelAbsurdTask();
-      try {
-        await delegate.cancel(reason);
-      } catch (error) {
-        throw cancellationPendingError(error);
-      }
-      try {
-        await cleanupPersistedRuntimeSandboxes();
-      } catch (error) {
-        // The scheduler terminal transaction already won. Its cleanup jobs are
-        // durable, so a best-effort eager cleanup failure must not turn a real
-        // cancellation into a false 502 or cause the caller to retry the
-        // terminal write.
-        console.warn('[absurd.runtime_sandbox_cleanup_deferred]', {
-          runId: input.runId,
-          error: error instanceof Error ? error.message : String(error),
-        });
-      }
+      // Engine cancellation and the scheduler terminal write commit together.
+      // A timeout rolls both back, so returning means the run is terminal and
+      // throwing means the caller can safely retry one unchanged operation.
+      await cancelAbsurdRunSynchronously(reason);
     });
   };
 
-  const cleanupPersistedRuntimeSandboxes = async () => {
-    const resources = await withRuntimeSchedulerClient(poolOptions, (client) =>
-      readPostgresSchedulerRuntimeResourcesForRun(
-        client,
-        { runId: input.runId },
-        effectiveOptions,
-      ),
-    );
-    const productionRefs = new Map<string, RuntimeSandboxRef>();
-    const workerOwnedRefs = new Map<string, RuntimeSandboxRef>();
-    for (const resource of resources) {
-      if (
-        !resource ||
-        typeof resource !== 'object' ||
-        Array.isArray(resource)
-      ) {
-        continue;
-      }
-      const record = resource as Record<string, unknown>;
-      const ref = runtimeSandboxRefFromResource(record);
-      if (!ref) continue;
-      const refKey = runtimeSandboxRefIdentityKey(ref);
-      // Modal credentials and all preview provider credentials are worker-only.
-      // This explicit task keeps deletion on the owning release lane. Legacy
-      // Daytona records predate runtimeEnvironment, so retain their marker.
-      const runtimeEnvironment =
-        record.runtimeEnvironment === 'preview' ||
-        record.daytonaEnvironment === 'preview'
-          ? 'preview'
-          : 'production';
-      if (runtimeEnvironment === 'preview' || ref.provider === 'modal') {
-        workerOwnedRefs.set(refKey, ref);
-        continue;
-      }
-      productionRefs.set(refKey, ref);
-    }
-    if (workerOwnedRefs.size > 0) {
-      const absurd = createAbsurdRuntimeClient(poolOptions, runsQueue);
-      await Promise.all(
-        [...workerOwnedRefs.values()].map((runtimeSandboxRef) => {
-          // sandboxId remains in the payload for workers deployed before the
-          // provider-scoped cleanup cutover. New workers use the versioned ref.
-          const params: AbsurdDaytonaSandboxCleanupTaskParams & {
-            runtimeSandboxRef: RuntimeSandboxRef;
-          } = {
-            runId: input.runId,
-            sandboxId: runtimeSandboxRef.resourceId,
-            runtimeSandboxRef,
-          };
-          return absurd.spawn(ABSURD_DAYTONA_SANDBOX_CLEANUP_TASK, params, {
-            queue: runsQueue,
-            idempotencyKey: `sandbox-cleanup:${input.runId}:${runtimeSandboxRefIdentityKey(runtimeSandboxRef)}`,
-            maxAttempts: ABSURD_PLAY_RUN_MAX_ATTEMPTS,
-            retryStrategy: ABSURD_PLAY_RUN_RETRY_STRATEGY,
-          });
-        }),
-      );
-    }
-    if (productionRefs.size === 0) return;
-    const cleanupResults = await Promise.allSettled(
-      [...productionRefs.values()].map(async (ref) => {
-        const outcome = await defaultRuntimeSandboxReconciliation.delete(ref);
-        if (!runtimeSandboxDeleteSucceeded(outcome)) {
-          throw new Error(
-            `${ref.provider} sandbox cleanup did not delete ${ref.resourceId}.`,
-          );
-        }
-      }),
-    );
-    for (const result of cleanupResults) {
-      if (result.status === 'rejected') {
-        console.warn('[absurd.runtime_sandbox_cleanup_failed]', {
-          runId: input.runId,
-          error:
-            result.reason instanceof Error
-              ? result.reason.message
-              : String(result.reason),
-        });
-      }
-    }
-  };
-
   return {
     ...delegate,
     runId: input.runId,