deepline 0.3.235 → 0.3.237

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -136,6 +136,8 @@ import {
   completePostgresSchedulerComputeBillingSessionBootstrap,
   deferPostgresSchedulerAttemptForCapacity,
   enqueuePostgresSchedulerSandboxCleanupForAttempt,
+  enqueuePostgresSchedulerSandboxCleanupForRun,
+  hasPostgresSchedulerRecoverableDetachedRunner,
   executePostgresSchedulerClaim,
   ensurePostgresSchedulerTerminalComputeBillingJob,
   failPostgresSchedulerAttempt,
@@ -221,6 +223,7 @@ import {
   type RuntimeSchedulerPoolOptions,
   withRuntimeSchedulerClient,
 } from './postgres-pool';
+import { runWithAbsurdSchedulerQuerySignal } from './absurd-pool';
 import {
   createPostgresSchedulerProgressReporter,
   type PostgresSchedulerProgressReporter,
@@ -550,7 +553,10 @@ export async function runWithAbsurdExecutionLeaseHeartbeat<T>(input: {
     lastHeartbeatAt = now;
     heartbeatState.inFlight = (async () => {
       try {
-        await input.ctx.heartbeat();
+        await runWithAbsurdSchedulerQuerySignal(
+          cancellationController.signal,
+          () => input.ctx.heartbeat(),
+        );
       } catch (error) {
         if (!isTransientHeartbeatTimeout(error) || stopped) throw error;
         console.warn('[absurd-runtime.worker] lease_heartbeat_timeout_retry', {
@@ -559,10 +565,22 @@ export async function runWithAbsurdExecutionLeaseHeartbeat<T>(input: {
         });
         await sleep(heartbeatTimeoutRetryDelayMs);
         if (stopped) return;
-        await input.ctx.heartbeat();
+        await runWithAbsurdSchedulerQuerySignal(
+          cancellationController.signal,
+          () => input.ctx.heartbeat(),
+        );
       }
     })()
       .catch((error) => {
+        // The handoff deadline owns this cancellation. A cancelled local SQL
+        // call does not prove that the engine cancelled the durable task.
+        if (
+          cancellationController.signal.aborted &&
+          error === cancellationController.signal.reason &&
+          error instanceof AbsurdExecutionOperationTimeoutError
+        ) {
+          return;
+        }
         const engineTerminal =
           error instanceof CancelledTask || error instanceof FailedTask;
         if (engineTerminal) {
@@ -595,9 +613,22 @@ export async function runWithAbsurdExecutionLeaseHeartbeat<T>(input: {
   timer.unref?.();
   let operationTimeout: ReturnType<typeof setTimeout> | null = null;
   try {
-    const operation = Promise.resolve().then(() =>
-      input.operation(cancellationController.signal),
-    );
+    // A completed backend still cannot park until its in-flight lease update
+    // settles. Keep that drain inside the same supervised handoff deadline.
+    const operation = (async () => {
+      try {
+        operationResult = await Promise.resolve().then(() =>
+          input.operation(cancellationController.signal),
+        );
+      } catch (error) {
+        operationError = error;
+      } finally {
+        stopped = true;
+        stopHeartbeatTimer();
+        await heartbeatState.inFlight;
+      }
+      return operationResult;
+    })();
     if (operationTimeoutMs !== null) {
       operationResult = await Promise.race([
         operation,
@@ -619,15 +650,11 @@ export async function runWithAbsurdExecutionLeaseHeartbeat<T>(input: {
       operationResult = await operation;
     }
   } catch (error) {
-    operationError = error;
+    operationError ??= error;
   } finally {
     if (operationTimeout) clearTimeout(operationTimeout);
     stopped = true;
     stopHeartbeatTimer();
-    const pendingHeartbeat = heartbeatState.inFlight;
-    if (pendingHeartbeat) {
-      await pendingHeartbeat.catch(() => undefined);
-    }
   }
   if (heartbeatFailure) {
     if (
@@ -735,7 +762,7 @@ export type AbsurdRuntimeWorkerDeps = {
   deleteRuntimeSandboxRef?: (
     ref: RuntimeSandboxRef,
   ) => Promise<RuntimeSandboxDeleteOutcome>;
-  deleteRuntimeSandboxWithOutcome: (
+  deleteRuntimeSandboxWithOutcome?: (
     sandboxId: string,
     expectedOrganizationId: string | null,
   ) => Promise<DaytonaSandboxDeleteOutcome>;
@@ -860,40 +887,48 @@ function buildDefaultDeps(absurd: Absurd): AbsurdRuntimeWorkerDeps {
  * of silently leaving the CLI stale.
  */
 async function projectLifecycleOutbox(input: {
+  ctx: TaskContext;
   deps: AbsurdRuntimeWorkerDeps;
   launch: PlaySchedulerSubmitInput;
   lifecycle: PostgresSchedulerLifecycleOutboxEvent;
   options?: Pick<PostgresSchedulerOptions, 'schema'>;
   transition: 'waiting' | 'resumed';
 }): Promise<void> {
-  try {
-    const result = await input.deps.projectOutbox(
-      input.launch,
-      input.lifecycle.runId,
-      input.options,
-      input.lifecycle.eventId,
-    );
-    if (result && !result.targetDelivered) {
-      console.error('[run-ledger-outbox.lifecycle_projection_pending]', {
-        runId: input.lifecycle.runId,
-        eventId: input.lifecycle.eventId,
-        transition: input.transition,
-        projected: result.projected,
-        failed: result.failed,
-        failedEventIds: result.failedEventIds,
-      });
-    }
-  } catch (error) {
-    // Scheduler truth and the outbox row were committed atomically before this
-    // handoff. Keep the durable continuation live so recovery can heal a
-    // transient app-runtime failure, but do not hide the visibility gap.
-    console.error('[run-ledger-outbox.lifecycle_projection_failed]', {
-      runId: input.lifecycle.runId,
-      eventId: input.lifecycle.eventId,
-      transition: input.transition,
-      error: error instanceof Error ? error.message : String(error),
-    });
-  }
+  await runWithAbsurdExecutionLeaseHeartbeat({
+    ctx: input.ctx,
+    runId: input.lifecycle.runId,
+    operation: async (signal) => {
+      try {
+        const result = await input.deps.projectOutbox(
+          input.launch,
+          input.lifecycle.runId,
+          input.options,
+          input.lifecycle.eventId,
+          signal,
+        );
+        if (result && !result.targetDelivered) {
+          console.error('[run-ledger-outbox.lifecycle_projection_pending]', {
+            runId: input.lifecycle.runId,
+            eventId: input.lifecycle.eventId,
+            transition: input.transition,
+            projected: result.projected,
+            failed: result.failed,
+            failedEventIds: result.failedEventIds,
+          });
+        }
+      } catch (error) {
+        // Scheduler truth and the outbox row were committed atomically before this
+        // handoff. Keep the durable continuation live so recovery can heal a
+        // transient app-runtime failure, but do not hide the visibility gap.
+        console.error('[run-ledger-outbox.lifecycle_projection_failed]', {
+          runId: input.lifecycle.runId,
+          eventId: input.lifecycle.eventId,
+          transition: input.transition,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    },
+  });
 }
 
 function nonJsonResponseTitle(
@@ -2397,23 +2432,26 @@ async function executeAbsurdAttemptAdapter(
   let executionLaunch = launch;
   try {
     const tokenStartedAt = performance.now();
-    const executorToken = await deps
-      .mintAttemptExecutorToken({
-        launch,
-        attempt: generation,
-        leaseToken,
-        schedulerSchema: options?.schema ?? null,
-      })
-      .finally(() => {
-        console.info(
-          '[absurd-runtime.worker.phase] ' +
-            JSON.stringify({
-              runId,
-              phase: 'executor_token_mint',
-              totalMs: performance.now() - tokenStartedAt,
-            }),
-        );
-      });
+    const executorToken = await runWithAbsurdExecutionLeaseHeartbeat({
+      ctx,
+      runId,
+      operation: () =>
+        deps.mintAttemptExecutorToken({
+          launch,
+          attempt: generation,
+          leaseToken,
+          schedulerSchema: options?.schema ?? null,
+        }),
+    }).finally(() => {
+      console.info(
+        '[absurd-runtime.worker.phase] ' +
+          JSON.stringify({
+            runId,
+            phase: 'executor_token_mint',
+            totalMs: performance.now() - tokenStartedAt,
+          }),
+      );
+    });
     executionLaunch = {
       ...launch,
       executorToken,
@@ -2434,12 +2472,56 @@ async function executeAbsurdAttemptAdapter(
       runtimeApi,
     });
   } catch (error) {
-    if (
-      error instanceof SuspendTask ||
-      error instanceof CancelledTask ||
-      error instanceof FailedTask ||
-      error instanceof DeferTask
-    ) {
+    if (error instanceof SuspendTask || error instanceof DeferTask) throw error;
+    let detached: Awaited<
+      ReturnType<typeof resumePostgresSchedulerDetachedRunnerAttempt>
+    >;
+    try {
+      detached = await deps.runWithClient((client) =>
+        resumePostgresSchedulerDetachedRunnerAttempt(
+          client,
+          { runId, attempt: generation, leaseToken },
+          options,
+        ),
+      );
+    } catch (recoveryError) {
+      console.error(
+        '[absurd-runtime.worker] detached_runner_recovery_unavailable',
+        JSON.stringify({
+          runId,
+          attempt: generation,
+          engineTaskId: ctx.taskID,
+          error: error instanceof Error ? error.message : String(error),
+          recoveryError:
+            recoveryError instanceof Error
+              ? recoveryError.message
+              : String(recoveryError),
+          cleanupAuthorized: false,
+        }),
+      );
+      throw new AggregateError(
+        [error, recoveryError],
+        'Scheduler could not prove detached runner cleanup authority.',
+      );
+    }
+    if (detached) {
+      console.error(
+        '[absurd-runtime.worker] detached_runner_handoff_recovered',
+        JSON.stringify({
+          runId,
+          attempt: generation,
+          engineAttempt: ctx.attempt,
+          engineTaskId: ctx.taskID,
+          error: error instanceof Error ? error.message : String(error),
+          cleanupAuthorized: false,
+        }),
+      );
+      throw new DeferTask(
+        `Detached runner ${runId}/${generation} retains execution after scheduler handoff failure.`,
+        1,
+      );
+    }
+    if (error instanceof CancelledTask || error instanceof FailedTask) {
       // Park/cancel signals belong to the engine; never treat them as leg
       // failures.
       throw error;
@@ -2761,11 +2843,6 @@ async function executeAssignedAbsurdAttempt(input: {
         `absurd runtime worker: suspended runner result for ${runId} carried no suspension.`,
       );
     }
-    await input.progressReporter.finalize(input.runnerResult, {
-      status: 'running',
-      replayAllTerminalLogs: input.fromDetachedRunner,
-      terminalLogProducerAttempt: input.terminalLogProducerAttempt,
-    });
     const suspendedCheckpoint = absurdCheckpointBase(
       input.runnerResult.checkpoint ?? checkpoint ?? null,
     );
@@ -2812,16 +2889,29 @@ async function executeAssignedAbsurdAttempt(input: {
       store: attemptStore,
       engine,
       lifecycle: {
-        waiting: async (lifecycle) =>
+        waiting: async (lifecycle) => {
+          await runWithAbsurdExecutionLeaseHeartbeat({
+            ctx,
+            runId,
+            operation: () =>
+              input.progressReporter.finalize(input.runnerResult, {
+                status: 'running',
+                replayAllTerminalLogs: input.fromDetachedRunner,
+                terminalLogProducerAttempt: input.terminalLogProducerAttempt,
+              }),
+          });
           await projectLifecycleOutbox({
+            ctx,
             deps,
             launch,
             lifecycle,
             options,
             transition: 'waiting',
-          }),
+          });
+        },
         resumed: async (lifecycle) =>
           await projectLifecycleOutbox({
+            ctx,
             deps,
             launch,
             lifecycle,
@@ -2916,6 +3006,7 @@ async function executeAssignedAbsurdAttempt(input: {
         return { ok: true, runId, schedulerStatus: 'idle' };
       }
       await projectLifecycleOutbox({
+        ctx,
         deps,
         launch,
         lifecycle: marked,
@@ -2962,6 +3053,7 @@ async function executeAssignedAbsurdAttempt(input: {
       return { ok: true, runId, schedulerStatus: 'idle' };
     }
     await projectLifecycleOutbox({
+      ctx,
       deps,
       launch,
       lifecycle: running,
@@ -3043,6 +3135,16 @@ async function executeAssignedAbsurdAttempt(input: {
     error: unknown,
     runnerConfig: PlayRunnerExecutionConfig,
   ): Promise<void> => {
+    if (
+      await deps.runWithClient((client) =>
+        hasPostgresSchedulerRecoverableDetachedRunner(
+          client,
+          { runId, attempt: generation, leaseToken },
+          options,
+        ),
+      )
+    )
+      return;
     const cleanupTerminalReason =
       error instanceof CancelledTask ? 'lease_expired' : 'runner_failed';
     await deps
@@ -3102,33 +3204,52 @@ async function executeAssignedAbsurdAttempt(input: {
         ),
       )
       .catch(() => attemptResources);
-    // Deleting without a durable obligation turns a transient provider failure
-    // into an untracked occupied sandbox. Preserve the recorded resource for
-    // terminal/retry recovery when the enqueue cannot commit; receipt and
-    // sheet release below remain safe and attempt-scoped.
-    const immediateSandboxGcAllowed =
-      !attemptResourceReadFailed &&
-      (attemptResources.length === 0 ||
-        (await persistAttemptSandboxCleanupBeforeBestEffortGc({
-          deps,
-          runId,
-          attempt: generation,
+    // All provider deletion goes through the durable cleanup lease, including
+    // retired generations. Never pass a read-only resource snapshot to raw GC.
+    if (!attemptResourceReadFailed) {
+      await deps.runWithClient((client) =>
+        enqueuePostgresSchedulerSandboxCleanupForRun(
+          client,
+          { runId },
           options,
-        })));
+        ),
+      );
+      const environments = new Set(
+        allResources.flatMap((resource) => {
+          if (!runtimeSandboxRefFromResource(resource)) return [];
+          const value = resource as {
+            runtimeEnvironment?: string;
+            daytonaEnvironment?: string;
+          };
+          return [
+            value.runtimeEnvironment === 'preview' ||
+            value.daytonaEnvironment === 'preview'
+              ? ('preview' as const)
+              : ('production' as const),
+          ];
+        }),
+      );
+      for (const environment of environments) {
+        await drainPostgresSchedulerSandboxCleanupOnce({
+          runWithClient: deps.runWithClient,
+          deleteRuntimeSandbox: deps.deleteRuntimeSandbox,
+          deleteRuntimeSandboxRef: deps.deleteRuntimeSandboxRef,
+          deleteRuntimeSandboxWithOutcome: deps.deleteRuntimeSandboxWithOutcome,
+          readSandboxCrashDiagnostic: deps.readSandboxCrashDiagnostic,
+          leaseOwner: `attempt-error:${runId}:${generation}`,
+          runId,
+          environment,
+          schedulerOptions: options,
+          timeoutMs: deps.sandboxGcTimeoutMs,
+        });
+      }
+    }
     const runtimeStateCleanup = releaseAbsurdAttemptRuntimeState({
       config: runnerConfig,
       runId,
       runAttempt: generation,
       resources: attemptResources,
-      sandboxRefs: runtimeSandboxResourceRefs(allResources),
       reason: cleanupTerminalReason,
-      deleteRuntimeSandbox: immediateSandboxGcAllowed
-        ? deps.deleteRuntimeSandbox
-        : undefined,
-      deleteRuntimeSandboxRef: immediateSandboxGcAllowed
-        ? deps.deleteRuntimeSandboxRef
-        : undefined,
-      sandboxGcTimeoutMs: deps.sandboxGcTimeoutMs,
     });
     if (ctx.maxAttempts !== null && ctx.attempt >= ctx.maxAttempts) {
       // A final engine attempt cannot retry. Its durable failed terminal must
@@ -3165,30 +3286,41 @@ async function executeAssignedAbsurdAttempt(input: {
       pendingDetachedRunner = null;
       let runnerResult: PlayRunnerResult;
       try {
-        runnerResult = await resolveDetachedRunnerOutcome({
+        runnerResult = await runWithAbsurdExecutionLeaseHeartbeat({
+          ctx,
           runId,
-          suspension: detached,
-          checkpoint: absurdCheckpointBase(checkpoint),
-          readRunnerTerminal,
-          recordRunnerTerminal,
-          readRunnerLiveness: () =>
-            deps.runWithClient((client) =>
-              readPostgresSchedulerRunnerLiveness(
-                client,
-                { runId, attempt: detached.runnerAttempt },
-                options,
-              ),
-            ),
-          inspectDetachedRunner: deps.inspectDetachedRunner,
-          deleteRuntimeSandbox: deps.deleteRuntimeSandbox,
-          deleteRuntimeSandboxRef: deps.deleteRuntimeSandboxRef,
-          sandboxGcTimeoutMs: deps.sandboxGcTimeoutMs,
+          operation: () =>
+            resolveDetachedRunnerOutcome({
+              runId,
+              suspension: detached,
+              checkpoint: absurdCheckpointBase(checkpoint),
+              readRunnerTerminal,
+              recordRunnerTerminal,
+              readRunnerLiveness: () =>
+                deps.runWithClient((client) =>
+                  readPostgresSchedulerRunnerLiveness(
+                    client,
+                    { runId, attempt: detached.runnerAttempt },
+                    options,
+                  ),
+                ),
+              inspectDetachedRunner: deps.inspectDetachedRunner,
+              deleteRuntimeSandbox: deps.deleteRuntimeSandbox,
+              deleteRuntimeSandboxRef: deps.deleteRuntimeSandboxRef,
+              sandboxGcTimeoutMs: deps.sandboxGcTimeoutMs,
+            }),
         });
       } catch (error) {
         // The terminal fast path skips normal runner setup. Build its cleanup
         // context only on failure, then preserve the same pre-retry resource
         // release and sandbox-GC fence as the full execution path.
-        const preparedRunnerConfig = await prepareRunnerConfig(claim);
+        const preparedRunnerConfig = await runWithAbsurdExecutionLeaseHeartbeat(
+          {
+            ctx,
+            runId,
+            operation: () => prepareRunnerConfig(claim),
+          },
+        );
         try {
           await cleanupAfterExecutionError(
             error,
@@ -3236,7 +3368,11 @@ async function executeAssignedAbsurdAttempt(input: {
     }
 
     const progressReporter = deps.createProgressReporter(launch, generation);
-    const preparedRunnerConfig = await prepareRunnerConfig(claim);
+    const preparedRunnerConfig = await runWithAbsurdExecutionLeaseHeartbeat({
+      ctx,
+      runId,
+      operation: () => prepareRunnerConfig(claim),
+    });
     const runnerConfig = preparedRunnerConfig.runnerConfig;
 
     let runnerResult: PlayRunnerResult;
@@ -5003,6 +5139,9 @@ export function createAbsurdRuntimeTasks(
   if (deps?.deleteRuntimeSandbox && !deps.deleteRuntimeSandboxRef) {
     resolved.deleteRuntimeSandboxRef = undefined;
   }
+  if (deps?.deleteRuntimeSandbox && !deps.deleteRuntimeSandboxWithOutcome) {
+    resolved.deleteRuntimeSandboxWithOutcome = undefined;
+  }
   absurd.registerTask({ name: ABSURD_PLAY_RUN_TASK, queue }, (params, ctx) =>
     executeAbsurdRuntimeLaunch(
       params as AbsurdPlayRunTaskParams,
@@ -5120,11 +5259,15 @@ async function readOpenAbsurdEngineRunId(input: {
  * can win that same terminalization race while a replacement is starting. In
  * that case Absurd rejects `complete_run` because the engine is already
  * terminal; that is an idempotent success, not a reason to flap the new worker.
+ * A receipt can also arrive before its wake is claimed. Retire that pending or
+ * sleeping task with the existing cancellation primitive after scheduler truth
+ * is durable, rather than failing recovery because complete_run needs a claim.
  */
 async function completeAbsurdEngineRunIfStillRunning(input: {
   client: PostgresSchedulerQueryClient;
   queue: string;
   absurdRunId: string;
+  absurdTaskId: string;
   result: unknown;
 }): Promise<boolean> {
   try {
@@ -5153,6 +5296,13 @@ async function completeAbsurdEngineRunIfStillRunning(input: {
     ) {
       return false;
     }
+    if (state === 'pending' || state === 'sleeping') {
+      return await cancelAbsurdEngineTaskAfterSchedulerTerminal({
+        client: input.client,
+        queue: input.queue,
+        absurdTaskId: input.absurdTaskId,
+      });
+    }
     throw error;
   }
 }
@@ -5206,9 +5356,9 @@ async function cancelAbsurdEngineTaskAfterSchedulerTerminal(input: {
  * its eventual lease reclaim would allocate a fresh scheduler generation and
  * re-run a play that has already completed. This recovery uses the original
  * attempt lease and the exact same terminal path as the normal wake leg, then
- * marks the engine run complete. It is deliberately limited to `completed`
- * results: failure and suspension retain the engine's normal retry/park
- * semantics once the scaler brings a recovery worker back for the lane.
+ * closes an engine run when it still exists. Completed and failed receipts
+ * remain authoritative even after the final engine claim expires. Suspended
+ * receipts retain the normal continuation handoff.
  */
 export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
   client: PostgresSchedulerQueryClient;
@@ -5228,6 +5378,7 @@ export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
     {
       limit: input.limit,
       absurdReleaseId: input.absurdReleaseId,
+      includeFailed: true,
     },
     options,
   );
@@ -5236,7 +5387,10 @@ export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
 
   for (const terminal of terminals) {
     const runnerResult = parseDetachedRunnerResult(terminal.result);
-    if (runnerResult?.status !== 'completed') {
+    if (
+      runnerResult?.status !== 'completed' &&
+      runnerResult?.status !== 'failed'
+    ) {
       skipped += 1;
       continue;
     }
@@ -5245,10 +5399,6 @@ export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
       queue: input.queue,
       schedulerRunId: terminal.runId,
     });
-    if (!absurdRun) {
-      skipped += 1;
-      continue;
-    }
     const launch = await readPostgresSchedulerLaunch(
       input.client,
       terminal.runId,
@@ -5267,6 +5417,16 @@ export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
       let schedulerStatus: 'completed' | 'failed';
       let terminalizedByAnotherWorker = false;
       try {
+        await markPostgresSchedulerAttemptRunning(
+          input.client,
+          {
+            runId: terminal.runId,
+            attempt: terminal.attempt,
+            leaseToken: terminal.leaseToken,
+            preserveHeartbeatAt: true,
+          },
+          options,
+        );
         const settled = await executePostgresSchedulerClaim(
           input.client,
           {
@@ -5296,22 +5456,26 @@ export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
           terminal.runId,
           options,
         );
-        if (observed?.status !== 'completed') throw error;
-        schedulerStatus = 'completed';
+        if (observed?.status !== 'completed' && observed?.status !== 'failed')
+          throw error;
+        schedulerStatus = observed.status;
         terminalizedByAnotherWorker = true;
       }
 
-      const engineCompleted = await completeAbsurdEngineRunIfStillRunning({
-        client: input.client,
-        queue: input.queue,
-        absurdRunId: absurdRun.runId,
-        result: {
-          recovered: terminalizedByAnotherWorker
-            ? 'completed_runner_terminal_race'
-            : 'completed_runner_terminal',
-          schedulerStatus,
-        },
-      });
+      const engineCompleted = absurdRun
+        ? await completeAbsurdEngineRunIfStillRunning({
+            client: input.client,
+            queue: input.queue,
+            absurdRunId: absurdRun.runId,
+            absurdTaskId: absurdRun.taskId,
+            result: {
+              recovered: terminalizedByAnotherWorker
+                ? 'completed_runner_terminal_race'
+                : 'completed_runner_terminal',
+              schedulerStatus,
+            },
+          })
+        : false;
       if (terminalizedByAnotherWorker) {
         if (engineCompleted) recovered += 1;
         else skipped += 1;
@@ -6333,6 +6497,7 @@ function unavailableSandboxCrashDiagnostic(
 async function claimPostgresSchedulerSandboxCleanupBatch(input: {
   runWithClient: AbsurdRuntimeWorkerDeps['runWithClient'];
   leaseOwner: string;
+  runId?: string;
   environment: 'preview' | 'production';
   schedulerOptions: Pick<PostgresSchedulerOptions, 'schema'>;
   limit: number;
@@ -6343,6 +6508,7 @@ async function claimPostgresSchedulerSandboxCleanupBatch(input: {
       client,
       {
         leaseOwner: input.leaseOwner,
+        runId: input.runId,
         environment: input.environment,
         limit: input.limit,
         providers: input.providers,
@@ -6630,7 +6796,8 @@ async function processPostgresSchedulerSandboxCleanupJob(input: {
       provider: job.provider,
       sandboxId: job.sandboxId,
       cleanupAttempt: job.attemptCount,
-      terminalStatus: job.runStatusAtClaim,
+      runStatusAtClaim: job.runStatusAtClaim,
+      cleanupAuthority: 'durable_cleanup_claim',
       providerOutcome: outcome.providerOutcome,
       completionOutcome: outcome.completion,
     });
@@ -6656,6 +6823,7 @@ export async function drainPostgresSchedulerSandboxCleanupOnce(input: {
   deleteRuntimeSandboxRef?: AbsurdRuntimeWorkerDeps['deleteRuntimeSandboxRef'];
   deleteRuntimeSandboxWithOutcome?: AbsurdRuntimeWorkerDeps['deleteRuntimeSandboxWithOutcome'];
   leaseOwner: string;
+  runId?: string;
   environment: 'preview' | 'production';
   schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
   limit?: number;
@@ -6677,6 +6845,7 @@ export async function drainPostgresSchedulerSandboxCleanupOnce(input: {
   const jobs = await claimPostgresSchedulerSandboxCleanupBatch({
     runWithClient: input.runWithClient,
     leaseOwner: input.leaseOwner,
+    runId: input.runId,
     environment: input.environment,
     schedulerOptions: options,
     limit,