deepline 0.3.461 → 0.3.463

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -53,6 +53,7 @@ import { RUNTIME_CLAIM_HEARTBEAT_DIVISOR } from '../runtime-constants';
 import { runtimeLimitExceededCause } from '../run-failure';
 import {
   playRunRecoveryLogDetail,
+  playRunRecoveryProviderText,
   recoveryForDaytonaSandboxAcquisitionUnavailable,
   recoveryForRuntimeInitializationTransportFailure,
   recoveryForRuntimeSchedulerAdmissionFailure,
@@ -66,10 +67,14 @@ import type {
   PlayRunnerExecutionConfig,
   PlayRunnerResult,
 } from '../protocol';
+import { resolveRunnerHeartbeatSilenceMs } from '../test-runtime-seams';
 import {
   RuntimeResourceFenceLostError,
   type PlayRunnerRuntimeResource,
 } from '../runner-backends/types';
+import {
+  daytonaSandboxMatchesOperationIdentity,
+} from '../runner-backends/backends/daytona-labels';
 import type { PlayBundleArtifact } from '../../plays/artifact-types';
 import {
   readRuntimeSheetRowOutcomeCounts,
@@ -161,8 +166,13 @@ import {
   markPostgresSchedulerAttemptWaiting,
   materializePostgresSchedulerFiles,
   listPostgresSchedulerCompletedRunnerTerminals,
+  listPostgresSchedulerDetachedRunnerLivenessWork,
   type PostgresSchedulerCompletedRunnerTerminal,
+  type PostgresSchedulerStaleDetachedRunner,
+  type PostgresSchedulerUnsettledRunnerTerminal,
+  type RunLedgerOutboxWithClient,
   listPostgresSchedulerAmbiguousModalCreateIntents,
+  recordPostgresSchedulerDetachedRunnerLivenessVerdict,
   recordPostgresSchedulerAttemptRuntimeResource,
   recordPostgresSchedulerAmbiguousModalCreateIntent,
   recordPostgresSchedulerLateManagedSandboxOperationForCleanup,
@@ -173,6 +183,7 @@ import {
   recordPostgresSchedulerRuntimeLifecycleEvent,
   recordPostgresSchedulerSandboxCrashDiagnostic,
   recordPostgresSchedulerSandboxCleanupEvent,
+  readPostgresSchedulerSandboxDeleteAcceptance,
   recordPostgresSchedulerRunnerTerminal,
   resumePostgresSchedulerDetachedRunnerAttempt,
   retryPostgresSchedulerComputeBillingJob,
@@ -204,8 +215,10 @@ import {
   type PostgresSchedulerOptions,
   type PostgresSchedulerQueryClient,
   type PostgresSchedulerSandboxCleanupJob,
+  type PostgresSchedulerSandboxDeleteAcceptance,
   type PostgresSchedulerSandboxCrashDiagnosticRequest,
   type PostgresSchedulerSignalRecord,
+  cancelPostgresSchedulerRun,
 } from './postgres';
 import {
   createPlayRunAttemptSuspensionJob,
@@ -224,15 +237,17 @@ import {
 } from './postgres-rate-state';
 import { verifyRuntimeReceiptGatewayHealth } from './runtime-receipt-gateway-health';
 import {
-  runtimeSandboxPlacementPoliciesForWorkerEnv,
+  runtimeSandboxPlacementPoliciesRunnableByWorker,
   runtimeSandboxProviderCapabilities,
 } from './absurd-worker-required-env';
 import {
   acquirePgNotifyHub,
   isConnectionCategoryError,
   isRuntimeSchedulerCapacityError,
+  isRuntimeSchedulerCircuitOpenError,
   isRuntimeSchedulerQueryCancellationError,
   isRuntimeSchedulerRetryableTransactionError,
+  isRuntimeSchedulerServerUnavailableError,
   parseRuntimeSchedulerCapacityError,
   RuntimeSchedulerCapacityError,
   runtimeSchedulerCapacityRetryAfterMs,
@@ -296,7 +311,7 @@ import {
   bindRuntimeSandboxCapacityLease,
   createRuntimeSandboxOperationReservation,
   readReservedDaytonaCancellationOperation,
-  recordRecoveredDaytonaCancellationSandbox,
+  recordDaytonaSandboxCleanupIntent,
   reserveRuntimeSandboxOperation,
   runtimeSandboxCapacityAdmissionMode,
   RuntimeSandboxCapacityLimitError,
@@ -828,6 +843,7 @@ export type AbsurdRuntimeWorkerDeps = {
   deleteRuntimeSandboxWithOutcome?: (
     sandboxId: string,
     expectedOrganizationId: string | null,
+    deleteAcceptance?: PostgresSchedulerSandboxDeleteAcceptance | null,
   ) => Promise<DaytonaSandboxDeleteOutcome>;
   /** Maximum time a best-effort sandbox deletion may occupy a worker slot. */
   sandboxGcTimeoutMs?: number;
@@ -859,6 +875,36 @@ function defaultProgressReporter(
   });
 }
 
+/** What projecting a committed terminal needs: the progress fold and the
+ * run-ledger outbox delivery a lane worker runs after its own terminal. */
+export type AbsurdRuntimeTerminalProjectionDeps = Pick<
+  AbsurdRuntimeWorkerDeps,
+  'runWithClient' | 'createProgressReporter' | 'projectOutbox'
+>;
+
+export function createAbsurdRuntimeTerminalProjectionDeps(input: {
+  runWithClient: AbsurdRuntimeWorkerDeps['runWithClient'];
+  outboxWithClient: RunLedgerOutboxWithClient;
+}): AbsurdRuntimeTerminalProjectionDeps {
+  return {
+    runWithClient: input.runWithClient,
+    createProgressReporter: defaultProgressReporter,
+    projectOutbox: async (launch, runId, options, targetEventId, signal) =>
+      await createRunLedgerProjectionJob({
+        outbox: createGatewayRunLedgerProjectionAdapter({
+          runId,
+          launch,
+          runtimeApi: {
+            ...absurdRuntimeApiFromLaunch(launch),
+            ...(signal ? { signal } : {}),
+          },
+          withClient: input.outboxWithClient,
+          schedulerOptions: options,
+        }),
+      }).deliver(targetEventId ? { targetEventId } : {}),
+  };
+}
+
 function buildDefaultDeps(absurd: Absurd): AbsurdRuntimeWorkerDeps {
   // Resolve the pool lazily: when a caller (tests) overrides `runWithClient`,
   // the runtime scheduler URL is never required.
@@ -868,29 +914,18 @@ function buildDefaultDeps(absurd: Absurd): AbsurdRuntimeWorkerDeps {
     process.env.DEEPLINE_PLAY_RUNNER_BACKEND || 'daytona',
   );
   return {
-    runWithClient: (fn, trace) =>
-      withRuntimeSchedulerClient(getPool(), fn, trace),
+    ...createAbsurdRuntimeTerminalProjectionDeps({
+      runWithClient: (fn, trace) =>
+        withRuntimeSchedulerClient(getPool(), fn, trace),
+      outboxWithClient: (operation, trace) =>
+        withRuntimeSchedulerClient(getPool(), operation, trace),
+    }),
     absurd,
     configuredRuntimeBackend,
     resolveBackend: (backendId, runtimeSandboxPlacementPolicyId) =>
       runtimeSandboxPlacementPolicyId
         ? resolveRuntimeSandboxPlacementBackend(runtimeSandboxPlacementPolicyId)
         : resolvePlayRunnerBackend(backendId),
-    createProgressReporter: defaultProgressReporter,
-    projectOutbox: async (launch, runId, options, targetEventId, signal) =>
-      await createRunLedgerProjectionJob({
-        outbox: createGatewayRunLedgerProjectionAdapter({
-          runId,
-          launch,
-          runtimeApi: {
-            ...absurdRuntimeApiFromLaunch(launch),
-            ...(signal ? { signal } : {}),
-          },
-          withClient: (operation, trace) =>
-            withRuntimeSchedulerClient(getPool(), operation, trace),
-          schedulerOptions: options,
-        }),
-      }).deliver(targetEventId ? { targetEventId } : {}),
     runtimeApiFromLaunch: absurdRuntimeApiFromLaunch,
     mintAttemptExecutorToken: requestAttemptExecutorToken,
     deleteRuntimeSandbox: (sandboxId) =>
@@ -900,10 +935,15 @@ function buildDefaultDeps(absurd: Absurd): AbsurdRuntimeWorkerDeps {
       }),
     deleteRuntimeSandboxRef: (ref) =>
       defaultRuntimeSandboxReconciliation.delete(ref),
-    deleteRuntimeSandboxWithOutcome: (sandboxId, expectedOrganizationId) =>
+    deleteRuntimeSandboxWithOutcome: (
+      sandboxId,
+      expectedOrganizationId,
+      deleteAcceptance,
+    ) =>
       deleteDaytonaSandboxByIdWithOutcome({
         sandboxId,
         expectedOrganizationId,
+        deleteAcceptance,
         timeoutSeconds: ABSURD_SANDBOX_GC_TIMEOUT_MS / 1_000,
       }),
     sandboxGcTimeoutMs: ABSURD_SANDBOX_GC_TIMEOUT_MS,
@@ -1511,6 +1551,7 @@ export async function parkAndResolveSuspension(input: {
    * durable park can wake at the runner's heartbeat deadline. */
   readRunnerLiveness?: (runnerAttempt: number) => Promise<{
     heartbeatAt: string | null;
+    heartbeatIdentity?: string | null;
     updatedAt: string | null;
     status: string;
   } | null>;
@@ -1542,15 +1583,11 @@ export async function parkAndResolveSuspension(input: {
     if (existing) {
       return absurdCheckpointBase(checkpoint);
     }
-    // Legacy parked rows predate heartbeatTimeoutMs. Never let malformed
-    // durable JSON turn the deadline into NaN, which Absurd treats as no timer.
-    const configuredHeartbeatTimeoutMs = Number(suspension.heartbeatTimeoutMs);
-    const heartbeatTimeoutMs =
-      Number.isFinite(configuredHeartbeatTimeoutMs) &&
-      configuredHeartbeatTimeoutMs >= 0
-        ? configuredHeartbeatTimeoutMs
-        : RUNTIME_CLAIM_LEASE_SECONDS * 1000;
+    // Never let malformed durable JSON turn the deadline into NaN, which Absurd
+    // treats as no timer.
+    const heartbeatTimeoutMs = detachedRunnerHeartbeatTimeoutMs(suspension);
     const ceilingDeadlineMs = suspension.startedAtMs + suspension.ceilingMs;
+    let silenceConfirmation: PlayCheckpoint['detachedRunnerSilenceConfirmation'];
     const readRunnerTerminal = input.readRunnerTerminal;
     const readRunnerLiveness = input.readRunnerLiveness;
     let rearms = 0;
@@ -1616,7 +1653,7 @@ export async function parkAndResolveSuspension(input: {
       let quietUntilMs = 0;
       const verdict = await confirmDetachedRunnerSilence({
         ctx,
-        stepPrefix: `${suspension.boundaryId}:liveness-confirm:${Number.isFinite(renewedAtMs) ? renewedAtMs : 'none'}`,
+        stepPrefix: `${suspension.boundaryId}:liveness-confirm-heartbeat:${renewed?.heartbeatIdentity ?? renewed?.heartbeatAt ?? 'none'}`,
         cadenceMs: Math.floor(
           heartbeatTimeoutMs / RUNTIME_CLAIM_HEARTBEAT_DIVISOR,
         ),
@@ -1628,10 +1665,10 @@ export async function parkAndResolveSuspension(input: {
           if (Date.now() < quietUntilMs) return false;
           if (await readRunnerTerminal(suspension.runnerAttempt)) return true;
           const current = await readRunnerLiveness(suspension.runnerAttempt);
-          const currentAtMs = current?.heartbeatAt
-            ? Date.parse(current.heartbeatAt)
-            : Number.NaN;
-          if (Number.isFinite(currentAtMs) && !(currentAtMs <= renewedAtMs)) {
+          if (
+            (current?.heartbeatIdentity ?? current?.heartbeatAt ?? null) !==
+            (renewed?.heartbeatIdentity ?? renewed?.heartbeatAt ?? null)
+          ) {
             return true;
           }
           quietUntilMs = Date.now() + LIVENESS_REPLAY_READ_REUSE_MS;
@@ -1639,11 +1676,24 @@ export async function parkAndResolveSuspension(input: {
         },
       });
       if (verdict === 'responded') continue;
+      if (verdict === 'gone') {
+        silenceConfirmation = {
+          boundaryId: suspension.boundaryId,
+          runnerAttempt: suspension.runnerAttempt,
+          heartbeatIdentity:
+            renewed?.heartbeatIdentity ?? renewed?.heartbeatAt ?? null,
+        };
+      }
       break;
     }
     // Truth is the persisted terminal (read by the caller's wake path); the
-    // absurd event carries no payload we trust.
-    return absurdCheckpointBase(checkpoint);
+    // absurd event carries no payload we trust. Discard any older proof when
+    // this wake did not complete a confirmation of the same heartbeat.
+    const resolved = absurdCheckpointBase(checkpoint);
+    delete resolved.detachedRunnerSilenceConfirmation;
+    if (silenceConfirmation)
+      resolved.detachedRunnerSilenceConfirmation = silenceConfirmation;
+    return resolved;
   }
 
   if (suspension.kind === 'sleep') {
@@ -1738,6 +1788,7 @@ export function createAbsurdEngineSuspensionAdapter(input: {
   readRunnerTerminal?: (runnerAttempt: number) => Promise<unknown | null>;
   readRunnerLiveness?: (runnerAttempt: number) => Promise<{
     heartbeatAt: string | null;
+    heartbeatIdentity?: string | null;
     updatedAt: string | null;
     status: string;
   } | null>;
@@ -2334,8 +2385,8 @@ type ObservedTerminalSource =
  * outbox controller owns retry after this attempt, so a worker lease never
  * determines whether a completed Run becomes durable or eventually visible.
  */
-function requestTerminalOutboxProjection(input: {
-  deps: AbsurdRuntimeWorkerDeps;
+type TerminalOutboxProjectionRequest = {
+  deps: Pick<AbsurdRuntimeWorkerDeps, 'projectOutbox'>;
   launch: PlaySchedulerSubmitInput;
   runId: string;
   options: Pick<PostgresSchedulerOptions, 'schema'>;
@@ -2343,7 +2394,19 @@ function requestTerminalOutboxProjection(input: {
   trigger?: string;
   /** When this worker began post-commit projection for the terminal. */
   projectionStartedAtMs?: number;
-}): void {
+  signal?: AbortSignal;
+};
+
+function requestTerminalOutboxProjection(
+  input: TerminalOutboxProjectionRequest,
+): void {
+  void dispatchTerminalOutboxProjection(input);
+}
+
+/** Never rejects: the terminal and its outbox fact are already committed. */
+function dispatchTerminalOutboxProjection(
+  input: TerminalOutboxProjectionRequest,
+): Promise<void> {
   const targetEventId = `run-terminal:${input.runId}`;
   const requestedAtMs = Date.now();
   // Terminal-visibility evidence: the request itself, and how long the
@@ -2357,8 +2420,14 @@ function requestTerminalOutboxProjection(input: {
         ? null
         : requestedAtMs - input.projectionStartedAtMs,
   });
-  void input.deps
-    .projectOutbox(input.launch, input.runId, input.options, targetEventId)
+  return input.deps
+    .projectOutbox(
+      input.launch,
+      input.runId,
+      input.options,
+      targetEventId,
+      input.signal,
+    )
     .then((projection) => {
       // Terminal-visibility latency evidence (ADR-0022 budget): when this
       // worker asked for delivery and how long the delivery took.
@@ -2635,21 +2704,21 @@ async function prepareAbsurdAttempt(input: {
       const launchBackend = normalizePlayRuntimeBackend(
         launch.runtimeBackend || configuredBackend,
       );
-      const configuredPlacementPolicies =
-        runtimeSandboxPlacementPoliciesForWorkerEnv(configuredBackend);
+      const runnablePlacementPolicies =
+        runtimeSandboxPlacementPoliciesRunnableByWorker(configuredBackend);
       const launchPlacementPolicy = resolveLaunchRuntimeSandboxPlacementPolicy({
         runtimeBackend: launchBackend,
         runtimeSandboxPlacementPolicyId: launch.runtimeSandboxPlacementPolicyId,
       });
       if (
-        launchBackend !== configuredBackend ||
-        (launchPlacementPolicy !== null &&
-          !configuredPlacementPolicies.some(
-            (policy) => policy.id === launchPlacementPolicy.id,
-          ))
+        launchPlacementPolicy === null
+          ? launchBackend !== configuredBackend
+          : !runnablePlacementPolicies.some(
+              (policy) => policy.id === launchPlacementPolicy.id,
+            )
       ) {
         throw new DeferTask(
-          `Runtime lane placement mismatch: worker=${configuredPlacementPolicies.map((policy) => policy.id).join(',') || configuredBackend}, launch=${launchPlacementPolicy?.id ?? launchBackend}. Reconcile the namespace before retrying.`,
+          `Runtime lane placement mismatch: worker=${runnablePlacementPolicies.map((policy) => policy.id).join(',') || configuredBackend}, launch=${launchPlacementPolicy?.id ?? launchBackend}. Reconcile the namespace before retrying.`,
           30,
         );
       }
@@ -2791,6 +2860,108 @@ async function prepareAbsurdAttempt(input: {
   );
 }
 
+function isRetryableAttemptPreparationError(error: unknown): boolean {
+  return (
+    isConnectionCategoryError(error) ||
+    isRuntimeSchedulerServerUnavailableError(error) ||
+    isRuntimeSchedulerCircuitOpenError(error) ||
+    isRuntimeSchedulerRetryableTransactionError(error) ||
+    isRuntimeSchedulerQueryCancellationError(error)
+  );
+}
+
+/**
+ * Preparation runs before an attempt exists, so nothing downstream can settle
+ * the run if it throws. A scheduler outage parks the task without spending an
+ * engine attempt: a terminal write needs that same database, so a deadline here
+ * could not settle anything sooner. Any other failure spends an attempt, and
+ * the final one settles the run rather than leaving it queued with no task.
+ */
+async function prepareAbsurdAttemptForTask(input: {
+  ctx: TaskContext;
+  deps: AbsurdRuntimeWorkerDeps;
+  options: Pick<PostgresSchedulerOptions, 'schema'>;
+  runId: string;
+}) {
+  try {
+    return await prepareAbsurdAttempt(input);
+  } catch (error) {
+    if (
+      error instanceof DeferTask ||
+      error instanceof SuspendTask ||
+      isRuntimeSchedulerCapacityError(error)
+    ) {
+      throw error;
+    }
+    if (isRetryableAttemptPreparationError(error)) {
+      const reason = error instanceof Error ? error.message : String(error);
+      console.warn('[absurd-runtime.worker] attempt_preparation_deferred', {
+        runId: input.runId,
+        reason,
+      });
+      throw new DeferTask(`Attempt preparation deferred: ${reason}`, 30);
+    }
+    await settleUnstartedRunOnFinalPreparationFailure({ ...input, error });
+    throw error;
+  }
+}
+
+async function settleUnstartedRunOnFinalPreparationFailure(input: {
+  ctx: TaskContext;
+  deps: AbsurdRuntimeWorkerDeps;
+  options: Pick<PostgresSchedulerOptions, 'schema'>;
+  runId: string;
+  error: unknown;
+}): Promise<void> {
+  const { ctx, deps, options, runId, error } = input;
+  if (ctx.maxAttempts === null || ctx.attempt < ctx.maxAttempts) return;
+  const reason = error instanceof Error ? error.message : String(error);
+  try {
+    // The fence skips a run that already has an attempt or changed state.
+    const cancelled = await deps.runWithClient((client) =>
+      cancelPostgresSchedulerRun(
+        client,
+        {
+          runId,
+          reason: `SCHEDULER_ATTEMPT_PREPARATION_FAILED: ${reason}`,
+          onlyIfStrandedPreExecution: {
+            releaseId: deps.currentReleaseId(),
+            absurdTaskId: ctx.taskID,
+            engineTaskTable: null,
+          },
+        },
+        options,
+      ),
+    );
+    console.error('[absurd-runtime.worker] attempt_preparation_exhausted', {
+      runId,
+      reason,
+      cancelled,
+    });
+  } catch (settleError) {
+    console.error(
+      '[absurd-runtime.worker] attempt_preparation_settlement_failed',
+      {
+        runId,
+        reason,
+        error:
+          settleError instanceof Error
+            ? settleError.message
+            : String(settleError),
+      },
+    );
+    // The original error would fail the task and strand the run, so keep this
+    // final attempt until the cancellation can commit.
+    if (isRuntimeSchedulerCapacityError(settleError)) throw settleError;
+    if (isRetryableAttemptPreparationError(settleError)) {
+      throw new DeferTask(
+        `Attempt preparation settlement deferred: ${reason}`,
+        30,
+      );
+    }
+  }
+}
+
 async function executeAbsurdAttemptAdapter(
   params: AbsurdPlayRunTaskParams,
   ctx: TaskContext,
@@ -2802,7 +2973,9 @@ async function executeAbsurdAttemptAdapter(
     schedulerOptions.schema ? { schema: schedulerOptions.schema } : {};
   const runId = params.runId;
 
-  let prepared = await prepareAbsurdAttempt({ deps, options, runId });
+  const prepareAttempt = () =>
+    prepareAbsurdAttemptForTask({ ctx, deps, options, runId });
+  let prepared = await prepareAttempt();
   while (prepared.kind === 'created_projection_pending') {
     const delaySeconds = prepared.delaySeconds;
     // The preparation checkout is already released. Absurd retains a winning
@@ -2819,11 +2992,19 @@ async function executeAbsurdAttemptAdapter(
     } catch (error) {
       if (!(error instanceof TimeoutError)) throw error;
     }
-    prepared = await prepareAbsurdAttempt({ deps, options, runId });
+    prepared = await prepareAttempt();
     if (signalled && prepared.kind === 'created_projection_pending') {
-      throw new Error(
+      const error = new Error(
         `Run-created wake has no durable projection acknowledgement for ${runId}.`,
       );
+      await settleUnstartedRunOnFinalPreparationFailure({
+        ctx,
+        deps,
+        options,
+        runId,
+        error,
+      });
+      throw error;
     }
   }
   if (prepared.kind === 'missing') {
@@ -3297,11 +3478,15 @@ async function executeAssignedAbsurdAttempt(input: {
         options,
       ),
     );
-  const recordRunnerTerminal = (runnerAttempt: number, result: unknown) =>
+  const recordRunnerTerminal = (
+    runnerAttempt: number,
+    result: unknown,
+    expectedHeartbeatIdentity?: string | null,
+  ) =>
     deps.runWithClient((client) =>
       recordPostgresSchedulerRunnerTerminal(
         client,
-        { runId, attempt: runnerAttempt, result },
+        { runId, attempt: runnerAttempt, result, expectedHeartbeatIdentity },
         options,
       ),
     );
@@ -3328,46 +3513,19 @@ async function executeAssignedAbsurdAttempt(input: {
       fromDetachedRunner: input.fromDetachedRunner,
       terminalLogProducerAttempt: input.terminalLogProducerAttempt,
     });
-    const canarySelection = runtimeSandboxPlacementPolicy
-      ? selectRuntimeSandboxCanary({
-          policy: runtimeSandboxPlacementPolicy,
-          runId,
-          production: isRuntimeSandboxCanaryProduction({
-            nodeEnv: process.env.NODE_ENV,
-            runtimeSchedulerSchema: options.schema,
-          }),
-        })
-      : null;
-    if (
-      input.fromDetachedRunner &&
-      input.detachedSuspension &&
-      canarySelection?.canary === 'modal' &&
-      runtimeSandboxRefFromSuspension(input.detachedSuspension).provider ===
-        'modal'
-    ) {
-      console.warn(
-        '[play-runner.sandbox_provider_canary]',
-        JSON.stringify({
-          runId,
-          workflowId: launch.runtimeAuthority?.workflowId ?? null,
-          runAttempt: generation,
-          policyId: runtimeSandboxPlacementPolicy?.id ?? null,
-          provider: 'modal',
-          bucket: canarySelection.bucket,
-          basisPoints: canarySelection.basisPoints,
-          runtimeEnvironment: 'production',
-          outcome: runnerResult.status,
-          phase: 'detached_terminal',
-          // A normal failed Play is customer/workflow truth, not evidence that
-          // Modal is unhealthy. Only a runner that disappeared after Modal
-          // accepted it is provider-infrastructure evidence at this boundary.
-          providerHealthFailure:
-            runnerResult.status === 'failed' &&
-            runnerResult.error.includes('RUNTIME_RUNNER_LOST:'),
-          errorName: null,
-          recoveryReason: null,
+    if (input.fromDetachedRunner && input.detachedSuspension) {
+      reportDetachedTerminalToModalCanary({
+        launch,
+        runId,
+        attempt: generation,
+        policy: runtimeSandboxPlacementPolicy,
+        production: isRuntimeSandboxCanaryProduction({
+          nodeEnv: process.env.NODE_ENV,
+          runtimeSchedulerSchema: options.schema,
         }),
-      );
+        suspension: input.detachedSuspension,
+        runnerResult,
+      });
     }
     const sandboxCrashDiagnostic =
       runnerResult.status === 'failed'
@@ -3813,6 +3971,9 @@ async function executeAssignedAbsurdAttempt(input: {
     // canonicalizes an omitted schema to the explicit production schema so
     // every runner-originated operation stays pinned to this physical lane.
     const absurdSchema = baseRunnerConfig.context.runtimeSchedulerSchema;
+    const heartbeatSilenceMs = resolveRunnerHeartbeatSilenceMs(
+      launch.runtimeTestFaultHeader,
+    );
     return {
       runnerConfig: {
         ...baseRunnerConfig,
@@ -3830,6 +3991,7 @@ async function executeAssignedAbsurdAttempt(input: {
                 runnerPushExecution: {
                   leaseSeconds: RUNTIME_CLAIM_LEASE_SECONDS,
                   runId,
+                  ...(heartbeatSilenceMs > 0 ? { heartbeatSilenceMs } : {}),
                 },
               }
             : {}),
@@ -4493,10 +4655,33 @@ async function executeAssignedAbsurdAttempt(input: {
                     providerOperationToken: reserved.providerOperationToken,
                     resumeExistingProviderOperation:
                       reserved.resumeExistingProviderOperation,
+                    resumePendingDelete: reserved.resumePendingDelete,
                     existingProviderOperationAgeMs:
                       reserved.existingProviderOperationAgeMs,
                   };
                 },
+                async recordDaytonaCleanupIntent(intent) {
+                  const recorded = await deps.runWithClient((client) =>
+                    recordDaytonaSandboxCleanupIntent(
+                      client,
+                      {
+                        operationKey: intent.leaseId,
+                        providerOperationToken:
+                          intent.providerOperationToken,
+                        orgId: intent.orgId,
+                        runId: intent.runId,
+                        sandboxId: intent.sandboxId,
+                        routingDomain: intent.routingDomain,
+                      },
+                      options,
+                    ),
+                  );
+                  if (!recorded) {
+                    throw new Error(
+                      'Daytona cleanup intent did not persist under its exact operation identity; cleanup and fallback suppressed.',
+                    );
+                  }
+                },
                 async releaseCapacity({ leaseId }) {
                   await deps.runWithClient((client) =>
                     abandonRuntimeSandboxOperationReservation(
@@ -4794,6 +4979,7 @@ async function executeAssignedAbsurdAttempt(input: {
         // Log-only. The persisted reason groups several provider conditions
         // into one decision; the detail names the one that was observed.
         const detail = playRunRecoveryLogDetail(recoveryError);
+        const providerText = playRunRecoveryProviderText(recoveryError);
         console.warn('[absurd-runtime.worker] pre_code_recovery_deferred', {
           runId,
           generation,
@@ -4803,6 +4989,7 @@ async function executeAssignedAbsurdAttempt(input: {
           deadlineAtMs: recovery.deadlineAtMs,
           delaySeconds: recovery.delaySeconds,
           ...(detail ? { detail } : {}),
+          ...(providerText ? { providerText } : {}),
         });
         throw new DeferTask(
           `Platform recovery deferred for ${recovery.dependency}: ${recovery.reason}`,
@@ -4878,48 +5065,22 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
   fromDetachedRunner: boolean;
   terminalLogProducerAttempt?: number;
 }): PlayRunAttemptWorkerTerminalProjectionPort {
-  /**
-   * Every terminal releases what the attempt claimed. Best-effort by design:
-   * terminal truth is already durable, and a failed release must never turn a
-   * committed terminal into a worker error.
-   */
-  const releaseAttemptReceipts = async (reason: string): Promise<void> => {
-    await input.deps
-      .runWithClient((client) =>
-        releasePostgresSchedulerReceiptsForAttempt(
-          client,
-          { runId: input.runId, runAttempt: input.generation },
-          { ...input.options, orgId: input.launch.orgId },
-        ),
-      )
-      .then((released) => {
-        if (released.released > 0) {
-          console.info('[absurd-runtime.worker] released terminal receipts', {
-            runId: input.runId,
-            generation: input.generation,
-            reason,
-            releasedReceipts: released.released,
-          });
-        }
-      })
-      .catch((error) => {
-        console.warn(
-          '[absurd-runtime.worker] terminal receipt release failed',
-          {
-            runId: input.runId,
-            generation: input.generation,
-            reason,
-            error: error instanceof Error ? error.message : String(error),
-          },
-        );
-      });
+  const attempt = {
+    runWithClient: input.deps.runWithClient,
+    launch: input.launch,
+    runId: input.runId,
+    generation: input.generation,
+    options: input.options,
   };
   return createPlayRunAttemptWorkerTerminalProjection({
     async observeFenceLost() {
       // A lost fence means this attempt is dead, which a cancelled run is the
       // loudest case of: its receipts must not stay leased to a generation
       // that will never complete them.
-      await releaseAttemptReceipts('fence_lost');
+      await releaseTerminalAttemptReceipts({
+        ...attempt,
+        reason: 'fence_lost',
+      });
       await settleObservedTerminal({
         deps: input.deps,
         launch: input.launch,
@@ -4941,55 +5102,7 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
       });
     },
     async releaseFailedAttempt() {
-      await releaseAttemptReceipts('failed_terminal');
-
-      // A normal runner-reported failure does not enter the thrown-error
-      // cleanup path above. Its terminal is already durable, so leaving these
-      // sheet leases until their TTL would make a corrected forced rerun wait
-      // behind a run that can no longer write. Keep this best-effort exactly
-      // like receipt release: terminal truth never depends on cleanup.
-      const resources = await input.deps
-        .runWithClient((client) =>
-          readPostgresSchedulerRuntimeResourcesForRun(
-            client,
-            { runId: input.runId, attempt: input.generation },
-            input.options,
-          ),
-        )
-        .catch((error) => {
-          console.warn(
-            '[absurd-runtime.worker] failed terminal sheet-release resource read failed',
-            {
-              runId: input.runId,
-              generation: input.generation,
-              error: error instanceof Error ? error.message : String(error),
-            },
-          );
-          return [];
-        });
-      if (resources.length === 0) return;
-      try {
-        await releaseRuntimeSheetAttemptResources({
-          config: {
-            context: buildPostgresSchedulerRunnerContext(
-              input.launch,
-              input.generation,
-            ),
-          },
-          runId: input.runId,
-          runAttempt: input.generation,
-          resources,
-        });
-      } catch (error) {
-        console.warn(
-          '[absurd-runtime.worker] failed terminal sheet-attempt release failed',
-          {
-            runId: input.runId,
-            generation: input.generation,
-            error: error instanceof Error ? error.message : String(error),
-          },
-        );
-      }
+      await releaseFailedTerminalAttempt(attempt);
     },
     queueProgressFinalization({ runnerResult }) {
       void input.reporter
@@ -5019,33 +5132,181 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
   });
 }
 
-function terminalEnvelopeFromRunnerResult(
-  result: PlayRunnerResult,
-): PlayRunAttemptTerminalEnvelope {
-  if (result.status === 'completed' || result.status === 'failed') {
-    return result;
-  }
-  throw new Error(
-    'Attempt terminal projection received a suspended runner result before suspension resolution.',
-  );
+type TerminalAttemptRelease = {
+  runWithClient: AbsurdRuntimeWorkerDeps['runWithClient'];
+  launch: PlaySchedulerSubmitInput;
+  runId: string;
+  generation: number;
+  options: Pick<PostgresSchedulerOptions, 'schema'>;
+};
+
+/**
+ * Every terminal releases what the attempt claimed. Best-effort by design:
+ * terminal truth is already durable, and a failed release must never turn a
+ * committed terminal into a worker error.
+ */
+async function releaseTerminalAttemptReceipts(
+  input: TerminalAttemptRelease & { reason: string },
+): Promise<void> {
+  await input
+    .runWithClient((client) =>
+      releasePostgresSchedulerReceiptsForAttempt(
+        client,
+        { runId: input.runId, runAttempt: input.generation },
+        { ...input.options, orgId: input.launch.orgId },
+      ),
+    )
+    .then((released) => {
+      if (released.released > 0) {
+        console.info('[absurd-runtime.worker] released terminal receipts', {
+          runId: input.runId,
+          generation: input.generation,
+          reason: input.reason,
+          releasedReceipts: released.released,
+        });
+      }
+    })
+    .catch((error) => {
+      console.warn('[absurd-runtime.worker] terminal receipt release failed', {
+        runId: input.runId,
+        generation: input.generation,
+        reason: input.reason,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    });
 }
 
-const DAYTONA_RUNNER_CRASH_WITHOUT_TERMINAL =
-  /Daytona play runner exited with code .*without pushing a terminal \(crash containment epilogue\)/i;
-const DAYTONA_RUNNER_LOST_WITHOUT_TERMINAL = /RUNTIME_RUNNER_LOST:/i;
+async function releaseFailedTerminalAttempt(
+  input: TerminalAttemptRelease,
+): Promise<void> {
+  await releaseTerminalAttemptReceipts({
+    ...input,
+    reason: 'failed_terminal',
+  });
 
-function sandboxCrashDiagnosticRequest(input: {
-  error: string;
-  suspension?: DetachedRunnerSuspension;
-}): PostgresSchedulerSandboxCrashDiagnosticRequest | undefined {
-  const suspension = input.suspension;
-  if (
-    !suspension ||
-    (!DAYTONA_RUNNER_CRASH_WITHOUT_TERMINAL.test(input.error) &&
-      !DAYTONA_RUNNER_LOST_WITHOUT_TERMINAL.test(input.error))
-  ) {
-    return undefined;
-  }
+  // A normal runner-reported failure does not enter the thrown-error cleanup
+  // path. Its terminal is already durable, so leaving these sheet leases until
+  // their TTL would make a corrected forced rerun wait behind a run that can no
+  // longer write. Keep this best-effort exactly like receipt release: terminal
+  // truth never depends on cleanup.
+  const resources = await input
+    .runWithClient((client) =>
+      readPostgresSchedulerRuntimeResourcesForRun(
+        client,
+        { runId: input.runId, attempt: input.generation },
+        input.options,
+      ),
+    )
+    .catch((error) => {
+      console.warn(
+        '[absurd-runtime.worker] failed terminal sheet-release resource read failed',
+        {
+          runId: input.runId,
+          generation: input.generation,
+          error: error instanceof Error ? error.message : String(error),
+        },
+      );
+      return [];
+    });
+  if (resources.length === 0) return;
+  try {
+    await releaseRuntimeSheetAttemptResources({
+      config: {
+        context: buildPostgresSchedulerRunnerContext(
+          input.launch,
+          input.generation,
+        ),
+      },
+      runId: input.runId,
+      runAttempt: input.generation,
+      resources,
+    });
+  } catch (error) {
+    console.warn(
+      '[absurd-runtime.worker] failed terminal sheet-attempt release failed',
+      {
+        runId: input.runId,
+        generation: input.generation,
+        error: error instanceof Error ? error.message : String(error),
+      },
+    );
+  }
+}
+
+function reportDetachedTerminalToModalCanary(input: {
+  launch: PlaySchedulerSubmitInput;
+  runId: string;
+  attempt: number;
+  policy: ReturnType<typeof resolveLaunchRuntimeSandboxPlacementPolicy>;
+  production: boolean;
+  suspension: DetachedRunnerSuspension;
+  runnerResult: PlayRunnerResult;
+}): void {
+  const selection = input.policy
+    ? selectRuntimeSandboxCanary({
+        policy: input.policy,
+        runId: input.runId,
+        production: input.production,
+      })
+    : null;
+  if (
+    selection?.canary !== 'modal' ||
+    runtimeSandboxRefFromSuspension(input.suspension).provider !== 'modal'
+  ) {
+    return;
+  }
+  console.warn(
+    '[play-runner.sandbox_provider_canary]',
+    JSON.stringify({
+      runId: input.runId,
+      workflowId: input.launch.runtimeAuthority?.workflowId ?? null,
+      runAttempt: input.attempt,
+      policyId: input.policy?.id ?? null,
+      provider: 'modal',
+      bucket: selection.bucket,
+      basisPoints: selection.basisPoints,
+      runtimeEnvironment: 'production',
+      outcome: input.runnerResult.status,
+      phase: 'detached_terminal',
+      // A normal failed Play is customer/workflow truth, not evidence that
+      // Modal is unhealthy. Only a runner that disappeared after Modal
+      // accepted it is provider-infrastructure evidence at this boundary.
+      providerHealthFailure:
+        input.runnerResult.status === 'failed' &&
+        input.runnerResult.error.includes('RUNTIME_RUNNER_LOST:'),
+      errorName: null,
+      recoveryReason: null,
+    }),
+  );
+}
+
+function terminalEnvelopeFromRunnerResult(
+  result: PlayRunnerResult,
+): PlayRunAttemptTerminalEnvelope {
+  if (result.status === 'completed' || result.status === 'failed') {
+    return result;
+  }
+  throw new Error(
+    'Attempt terminal projection received a suspended runner result before suspension resolution.',
+  );
+}
+
+const DAYTONA_RUNNER_CRASH_WITHOUT_TERMINAL =
+  /Daytona play runner exited with code .*without pushing a terminal \(crash containment epilogue\)/i;
+const DAYTONA_RUNNER_LOST_WITHOUT_TERMINAL = /RUNTIME_RUNNER_LOST:/i;
+
+function sandboxCrashDiagnosticRequest(input: {
+  error: string;
+  suspension?: DetachedRunnerSuspension;
+}): PostgresSchedulerSandboxCrashDiagnosticRequest | undefined {
+  const suspension = input.suspension;
+  if (
+    !suspension ||
+    (!DAYTONA_RUNNER_CRASH_WITHOUT_TERMINAL.test(input.error) &&
+      !DAYTONA_RUNNER_LOST_WITHOUT_TERMINAL.test(input.error))
+  ) {
+    return undefined;
+  }
   try {
     if (runtimeSandboxRefFromSuspension(suspension).provider !== 'daytona') {
       return undefined;
@@ -5199,9 +5460,11 @@ export async function resolveDetachedRunnerOutcome(input: {
   recordRunnerTerminal: (
     runnerAttempt: number,
     result: PlayRunnerResult,
+    expectedHeartbeatIdentity?: string | null,
   ) => Promise<boolean>;
   readRunnerLiveness: () => Promise<{
     heartbeatAt: string | null;
+    heartbeatIdentity?: string | null;
     updatedAt: string | null;
     status: string;
   } | null>;
@@ -5247,13 +5510,34 @@ export async function resolveDetachedRunnerOutcome(input: {
       },
     };
   };
+  const suspend = (): PlayRunnerResult => ({
+    status: 'suspended',
+    suspension,
+    logs: [],
+    stats: {},
+    steps: [],
+    checkpoint: input.checkpoint,
+    tableNamespace: null,
+  });
   const settleFirstWriteWinner = async (
     candidate: PlayRunnerResult,
+    expectedHeartbeatIdentity?: string | null,
   ): Promise<PlayRunnerResult> => {
     const attemptExists = await input.recordRunnerTerminal(
       suspension.runnerAttempt,
       candidate,
+      expectedHeartbeatIdentity,
     );
+    if (!attemptExists && expectedHeartbeatIdentity !== undefined) {
+      // A heartbeat can win while sandbox inspection or the terminal write is
+      // in flight. The SQL fence refuses only this synthetic outcome; preserve
+      // a concurrent pushed result and otherwise confirm the new heartbeat.
+      const winner = parseDetachedRunnerResult(
+        await input.readRunnerTerminal(suspension.runnerAttempt),
+      );
+      if (winner) return settle(winner);
+      if (await input.readRunnerLiveness()) return suspend();
+    }
     if (!attemptExists) {
       throw new Error(
         `Detached runner attempt ${suspension.runnerAttempt} for ${input.runId} disappeared before terminal arbitration.`,
@@ -5297,51 +5581,31 @@ export async function resolveDetachedRunnerOutcome(input: {
     ? Math.max(0, now - heartbeatAtMs)
     : null;
   const reachedCeiling = now >= suspension.startedAtMs + suspension.ceilingMs;
-  const configuredHeartbeatTimeoutMs = Number(suspension.heartbeatTimeoutMs);
-  const heartbeatTimeoutMs =
-    Number.isFinite(configuredHeartbeatTimeoutMs) &&
-    configuredHeartbeatTimeoutMs >= 0
-      ? configuredHeartbeatTimeoutMs
-      : RUNTIME_CLAIM_LEASE_SECONDS * 1000;
-  const reparkDetachedRunner = (): PlayRunnerResult => ({
-    status: 'suspended',
-    suspension,
-    logs: [],
-    stats: {},
-    steps: [],
-    checkpoint: input.checkpoint,
-    tableNamespace: null,
-  });
+  const heartbeatTimeoutMs = detachedRunnerHeartbeatTimeoutMs(suspension);
   if (!reachedCeiling && heartbeatAgeMs !== null) {
     if (heartbeatAgeMs < heartbeatTimeoutMs) {
       // We woke on an older deadline while the runner renewed liveness. Park
       // again from the new durable timestamp; do not touch Daytona.
-      return reparkDetachedRunner();
+      return suspend();
     }
   }
-  const reportLivenessExpired = () => {
-    // This event is consumed directly by the operational-health dashboard.
-    // Keep its second argument strict JSON so Fly's text log stream remains an
-    // executable producer contract, independent of Bun's object-inspect format.
-    console.error(
-      '[absurd-runtime.worker] detached_runner_liveness_expired',
-      JSON.stringify({
-        runId: input.runId,
-        runnerAttempt: suspension.runnerAttempt,
-        sandboxId: suspension.sandboxId,
-        sessionId: suspension.sessionId,
-        cmdId: suspension.cmdId,
-        startedAtMs: suspension.startedAtMs,
-        heartbeatTimeoutMs,
-        ceilingMs: suspension.ceilingMs,
-        observedAtMs: now,
-        reachedCeiling,
-        liveness,
-        heartbeatAgeMs,
-        pushedTerminalPresent: false,
-      }),
-    );
-  };
+  const confirmation = input.checkpoint.detachedRunnerSilenceConfirmation;
+  const heartbeatIdentity =
+    liveness?.heartbeatIdentity ?? liveness?.heartbeatAt ?? null;
+  const silenceConfirmed = Boolean(
+    confirmation &&
+    confirmation.boundaryId === suspension.boundaryId &&
+    confirmation.runnerAttempt === suspension.runnerAttempt &&
+    confirmation.heartbeatIdentity === heartbeatIdentity,
+  );
+  const reportLivenessExpired = () =>
+    reportDetachedRunnerLivenessExpired({
+      runId: input.runId,
+      suspension,
+      observedAtMs: now,
+      liveness,
+      heartbeatAgeMs,
+    });
   if (reachedCeiling) {
     reportLivenessExpired();
     return settleFirstWriteWinner({
@@ -5407,7 +5671,7 @@ export async function resolveDetachedRunnerOutcome(input: {
     }
   }
   // Inspection takes seconds. A terminal or heartbeat that landed meanwhile
-  // outranks everything read before it.
+  // outranks everything read before it, including a sub-millisecond renewal.
   const storedDuringInspection = parseDetachedRunnerResult(
     await input.readRunnerTerminal(suspension.runnerAttempt),
   );
@@ -5415,23 +5679,92 @@ export async function resolveDetachedRunnerOutcome(input: {
     return settle(storedDuringInspection);
   }
   const latestLiveness = await input.readRunnerLiveness();
-  const latestHeartbeatAtMs = latestLiveness?.heartbeatAt
-    ? Date.parse(latestLiveness.heartbeatAt)
-    : Number.NaN;
   if (
-    Number.isFinite(latestHeartbeatAtMs) &&
-    !(latestHeartbeatAtMs <= heartbeatAtMs)
+    (latestLiveness?.heartbeatIdentity ??
+      latestLiveness?.heartbeatAt ??
+      null) !== heartbeatIdentity
   ) {
-    return reparkDetachedRunner();
+    return suspend();
   }
+  if (!silenceConfirmed) return suspend();
   if (
     input.inspectDetachedRunner &&
     !provenGone &&
     !detachedRunnerInspectionProvesGone(inspection)
   ) {
-    return reparkDetachedRunner();
+    return suspend();
   }
   reportLivenessExpired();
+  return settleFirstWriteWinner(
+    detachedRunnerLostResult({
+      suspension,
+      observedAtMs: now,
+      livenessStatus: liveness?.status,
+      heartbeatAtMs,
+      heartbeatAgeMs,
+      inspectionAttempted: Boolean(input.inspectDetachedRunner),
+      inspection,
+      inspectionFailed,
+    }),
+    heartbeatIdentity,
+  );
+}
+
+/** Legacy parked rows predate heartbeatTimeoutMs, and the field is durable JSON. */
+function detachedRunnerHeartbeatTimeoutMs(
+  suspension: DetachedRunnerSuspension,
+): number {
+  const configured = Number(suspension.heartbeatTimeoutMs);
+  return Number.isFinite(configured) && configured >= 0
+    ? configured
+    : RUNTIME_CLAIM_LEASE_SECONDS * 1000;
+}
+
+function reportDetachedRunnerLivenessExpired(input: {
+  runId: string;
+  suspension: DetachedRunnerSuspension;
+  observedAtMs: number;
+  liveness: unknown;
+  heartbeatAgeMs: number | null;
+}): void {
+  const { suspension } = input;
+  // This event is consumed directly by the operational-health dashboard.
+  // Keep its second argument strict JSON so Fly's text log stream remains an
+  // executable producer contract, independent of Bun's object-inspect format.
+  console.error(
+    '[absurd-runtime.worker] detached_runner_liveness_expired',
+    JSON.stringify({
+      runId: input.runId,
+      runnerAttempt: suspension.runnerAttempt,
+      sandboxId: suspension.sandboxId,
+      sandboxProvider: suspension.sandboxProvider ?? 'daytona',
+      sessionId: suspension.sessionId,
+      cmdId: suspension.cmdId,
+      startedAtMs: suspension.startedAtMs,
+      heartbeatTimeoutMs: detachedRunnerHeartbeatTimeoutMs(suspension),
+      ceilingMs: suspension.ceilingMs,
+      observedAtMs: input.observedAtMs,
+      reachedCeiling:
+        input.observedAtMs >= suspension.startedAtMs + suspension.ceilingMs,
+      liveness: input.liveness,
+      heartbeatAgeMs: input.heartbeatAgeMs,
+      pushedTerminalPresent: false,
+    }),
+  );
+}
+
+/** The one failed terminal for a runner the provider proved gone. */
+function detachedRunnerLostResult(input: {
+  suspension: DetachedRunnerSuspension;
+  observedAtMs: number;
+  livenessStatus: unknown;
+  heartbeatAtMs: number;
+  heartbeatAgeMs: number | null;
+  inspectionAttempted: boolean;
+  inspection: DetachedRunnerInspection | null;
+  inspectionFailed: boolean;
+}): PlayRunnerResult {
+  const { suspension, inspection } = input;
   const safeDiagnosticLabel = (value: unknown): string =>
     typeof value === 'string' && /^[a-z0-9_]{1,80}$/i.test(value)
       ? value
@@ -5439,36 +5772,37 @@ export async function resolveDetachedRunnerOutcome(input: {
   const runnerLiveness = {
     event: 'detached_runner_liveness_expired',
     runnerAttempt: suspension.runnerAttempt,
-    livenessStatus: safeDiagnosticLabel(liveness?.status),
-    lastHeartbeatAt: Number.isFinite(heartbeatAtMs)
-      ? new Date(heartbeatAtMs).toISOString()
+    livenessStatus: safeDiagnosticLabel(input.livenessStatus),
+    lastHeartbeatAt: Number.isFinite(input.heartbeatAtMs)
+      ? new Date(input.heartbeatAtMs).toISOString()
       : null,
-    heartbeatAgeMs,
-    heartbeatTimeoutMs,
-    runElapsedMs: Math.max(0, now - suspension.startedAtMs),
+    heartbeatAgeMs: input.heartbeatAgeMs,
+    heartbeatTimeoutMs: detachedRunnerHeartbeatTimeoutMs(suspension),
+    runElapsedMs: Math.max(0, input.observedAtMs - suspension.startedAtMs),
     runtimeCeilingMs: suspension.ceilingMs,
-    reachedRuntimeCeiling: reachedCeiling,
+    reachedRuntimeCeiling:
+      input.observedAtMs >= suspension.startedAtMs + suspension.ceilingMs,
     terminalResultFound: false,
-    inspectionAttempted: Boolean(input.inspectDetachedRunner),
+    inspectionAttempted: input.inspectionAttempted,
     inspectionStage: inspection
       ? safeDiagnosticLabel(inspection.diagnosis?.stage)
-      : inspectionFailed
+      : input.inspectionFailed
         ? 'inspection_failed'
-        : input.inspectDetachedRunner
+        : input.inspectionAttempted
           ? 'inspection_empty'
           : 'not_configured',
     inspectionExitCode: inspection?.exitCode ?? null,
     inspectedResultStatus: inspection?.result?.status ?? null,
   };
-  return settleFirstWriteWinner({
+  return {
     status: 'failed',
-    error: `RUNTIME_RUNNER_LOST: Detached runner heartbeat expired after ${heartbeatAgeMs ?? 'unknown'}ms without a terminal result. Deepline did not automatically retry because external actions may already have completed.`,
+    error: `RUNTIME_RUNNER_LOST: Detached runner heartbeat expired after ${input.heartbeatAgeMs ?? 'unknown'}ms without a terminal result. Deepline did not automatically retry because external actions may already have completed.`,
     logs: [`[runtime.failure_diagnostic] ${JSON.stringify(runnerLiveness)}`],
     stats: { runnerLiveness },
     steps: [],
     checkpoint: null,
     tableNamespace: null,
-  });
+  };
 }
 
 type RuntimeCleanupApiContext = Parameters<
@@ -6283,6 +6617,71 @@ async function cancelAbsurdEngineTaskAfterSchedulerTerminal(input: {
   }
 }
 
+type SettledRunnerTerminal = {
+  terminal: PostgresSchedulerCompletedRunnerTerminal;
+  launch: PlaySchedulerSubmitInput;
+  runnerResult: PlayRunnerResult;
+  schedulerStatus: 'completed' | 'failed';
+  suspension: DetachedRunnerSuspension | null;
+  /** Another writer committed this terminal and owns what follows it. */
+  terminalizedByAnotherWorker: boolean;
+};
+
+/**
+ * What a lane worker does after its own terminal commits: fold the run's
+ * durable facts into progress, finalize it, and ask for the terminal outbox
+ * row. Never rejects: terminal truth is already durable, and the run-ledger
+ * projector delivers the outbox row without this.
+ */
+async function projectRecoveredRunnerTerminal(input: {
+  deps: AbsurdRuntimeTerminalProjectionDeps;
+  launch: PlaySchedulerSubmitInput;
+  runId: string;
+  attempt: number;
+  runnerResult: PlayRunnerResult;
+  options: Pick<PostgresSchedulerOptions, 'schema'>;
+  signal?: AbortSignal;
+}): Promise<void> {
+  const progress = (async () => {
+    const reporter = input.deps.createProgressReporter(
+      input.launch,
+      input.attempt,
+    );
+    await reconcileTerminalTruthIntoReporter({
+      deps: input.deps,
+      launch: input.launch,
+      runId: input.runId,
+      attempt: input.attempt,
+      options: input.options,
+      reporter,
+    });
+    await reporter.finalize(input.runnerResult, {
+      status: 'running',
+      skipTerminalLogReplay: true,
+      terminalLogProducerAttempt: input.attempt,
+    });
+  })().catch((error) => {
+    console.warn(
+      '[absurd-runtime.worker] recovered_terminal_projection_failed',
+      {
+        runId: input.runId,
+        attempt: input.attempt,
+        error: error instanceof Error ? error.message : String(error),
+      },
+    );
+  });
+  await Promise.all([
+    progress,
+    dispatchTerminalOutboxProjection({
+      deps: input.deps,
+      launch: input.launch,
+      runId: input.runId,
+      options: input.options,
+      ...(input.signal ? { signal: input.signal } : {}),
+    }),
+  ]);
+}
+
 /**
  * Close completed runner handoffs before this lane's claim loop starts.
  *
@@ -6299,9 +6698,11 @@ async function cancelAbsurdEngineTaskAfterSchedulerTerminal(input: {
 export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
   client: PostgresSchedulerQueryClient;
   /** A lane worker supplies post-terminal projection and sandbox cleanup. The
-   * fixed controller intentionally omits these: the terminal transaction's
-   * durable outbox is delivered by its singleton outbox controller, avoiding a
-   * worker → gateway retry chain during global recovery. */
+   * fixed controller's lane-wide recovery intentionally omits these: the
+   * terminal transaction's durable outbox is delivered by its singleton outbox
+   * controller, avoiding a worker → gateway retry chain during global
+   * recovery. The liveness sweep projects what it settles itself, inside its
+   * pass budget. */
   deps?: AbsurdRuntimeWorkerDeps;
   queue: string;
   schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
@@ -6309,6 +6710,8 @@ export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
   limit?: number;
   /** Preselected by the fixed controller's one-query cross-lane scan. */
   terminals?: readonly PostgresSchedulerCompletedRunnerTerminal[];
+  /** Told of each terminal now durable, whichever writer won it. */
+  onTerminalSettled?: (settled: SettledRunnerTerminal) => void;
 }): Promise<{ recovered: number; skipped: number }> {
   const options = input.schedulerOptions ?? {};
   const terminals =
@@ -6356,6 +6759,22 @@ export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
       // outbox semantics instead of inventing a recovery-only terminal write.
       let schedulerStatus: 'completed' | 'failed';
       let terminalizedByAnotherWorker = false;
+      const parsedSuspension = parseOverdueDetachedRunnerSuspension(
+        terminal.suspension,
+      );
+      // The run's suspension can name a newer attempt's sandbox. Only this
+      // terminal's own sandbox may supply its crash diagnostic or canary fact.
+      const suspension =
+        parsedSuspension?.runnerAttempt === terminal.attempt
+          ? parsedSuspension
+          : null;
+      const sandboxCrashDiagnostic =
+        runnerResult.status === 'failed' && suspension
+          ? sandboxCrashDiagnosticRequest({
+              error: runnerResult.error,
+              suspension,
+            })
+          : undefined;
       try {
         await markPostgresSchedulerAttemptRunning(
           input.client,
@@ -6379,6 +6798,7 @@ export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
           {
             ...options,
             terminalLogOutbox: 'ctx_log_tail',
+            ...(sandboxCrashDiagnostic ? { sandboxCrashDiagnostic } : {}),
           },
         );
         if (settled.status !== 'completed' && settled.status !== 'failed') {
@@ -6422,44 +6842,25 @@ export async function recoverAbsurdRuntimeCompletedRunnerTerminals(input: {
       } else {
         recovered += 1;
       }
+      input.onTerminalSettled?.({
+        terminal,
+        launch,
+        runnerResult,
+        schedulerStatus,
+        suspension,
+        terminalizedByAnotherWorker,
+      });
 
       // The worker which won the scheduler terminal can still disappear after
       // its atomic write but before best-effort projection. Cleanup is already
       // a durable obligation owned by the independent cleanup controller.
       if (input.deps) {
-        const workerDeps = input.deps;
-        void (async () => {
-          const reporter = workerDeps.createProgressReporter(
-            launch,
-            terminal.attempt,
-          );
-          await reconcileTerminalTruthIntoReporter({
-            deps: workerDeps,
-            launch,
-            runId: terminal.runId,
-            attempt: terminal.attempt,
-            options,
-            reporter,
-          });
-          await reporter.finalize(runnerResult, {
-            status: 'running',
-            skipTerminalLogReplay: true,
-            terminalLogProducerAttempt: terminal.attempt,
-          });
-        })().catch((error) => {
-          console.warn(
-            '[absurd-runtime.worker] recovered_terminal_projection_failed',
-            {
-              runId: terminal.runId,
-              attempt: terminal.attempt,
-              error: error instanceof Error ? error.message : String(error),
-            },
-          );
-        });
-        requestTerminalOutboxProjection({
+        void projectRecoveredRunnerTerminal({
           deps: input.deps,
           launch,
           runId: terminal.runId,
+          attempt: terminal.attempt,
+          runnerResult,
           options,
         });
       }
@@ -6735,6 +7136,661 @@ export async function recoverAbsurdRuntimeOverdueDetachedRunnerCeilingsAcrossAll
   return recoverAbsurdRuntimeOverdueDetachedRunnerCeilings(input);
 }
 
+/** Inside `ABSURD_OVERDUE_DETACHED_RUNNER_RECOVERY_INTERVAL_MS`. */
+export const ABSURD_DETACHED_RUNNER_LIVENESS_PASS_BUDGET_MS = 10_000;
+/** The fixed controller stops waiting for a pass here, even on a stuck store. */
+export const ABSURD_DETACHED_RUNNER_LIVENESS_HARD_STOP_MS = 12_000;
+/** Reads hold memory in a 256 MiB controller, so keep them few. Hung reads keep
+ * their slots across passes, which stops a stuck provider piling them up. */
+export const ABSURD_DETACHED_RUNNER_LIVENESS_READS_PER_PROVIDER = 3;
+/** Each provider's read has a shorter deadline of its own. This backstop gives
+ * back the slot of any read that outlives it instead of blinding the sweep
+ * until the controller restarts. */
+export const ABSURD_DETACHED_RUNNER_LIVENESS_READ_DEADLINE_MS = 60_000;
+/** Past this many unsettled reads a provider gets no more, which bounds what
+ * a provider that never answers can hold. */
+export const ABSURD_DETACHED_RUNNER_LIVENESS_MAX_ABANDONED_READS =
+  2 * ABSURD_DETACHED_RUNNER_LIVENESS_READS_PER_PROVIDER;
+export const ABSURD_DETACHED_RUNNER_LIVENESS_PAGE_LIMIT = 100;
+/** How soon an unproven sandbox may be read again. */
+export const ABSURD_DETACHED_RUNNER_LIVENESS_RECHECK_MS = 60_000;
+
+const ABSURD_DETACHED_RUNNER_LIVENESS_PROVIDERS = ['daytona', 'modal'] as const;
+/** Inspection stages that carry no answer from the provider about the sandbox. */
+const DETACHED_RUNNER_UNANSWERED_STAGES: ReadonlySet<string> = new Set([
+  'sandbox_lookup_failed',
+  'missing_routing_domain',
+  'wrong_routing_domain',
+]);
+type DetachedRunnerLivenessProvider =
+  (typeof ABSURD_DETACHED_RUNNER_LIVENESS_PROVIDERS)[number];
+
+/**
+ * Process-local on purpose. Losing it costs each candidate one extra read, and
+ * `nextCheckAtMs` is bounded by construction: a pass adds at most a page per
+ * provider, and every entry is pruned once its recheck time passes.
+ */
+export type AbsurdDetachedRunnerLivenessSweepState = {
+  generation: number;
+  nextCheckAtMs: Map<string, number>;
+  afterRunIds: Record<DetachedRunnerLivenessProvider, string | null>;
+  readsInFlight: Record<DetachedRunnerLivenessProvider, number>;
+  /** Reads past their deadline that have still not settled. */
+  abandonedReads: Record<DetachedRunnerLivenessProvider, number>;
+  /** A settlement runs, perhaps for a pass that has returned. No other starts
+   * until it ends, so a stalled store holds one settlement, not one per pass. */
+  settling: boolean;
+  /** Likewise for a settled terminal's progress and outbox projection. */
+  projecting: boolean;
+};
+
+export function createAbsurdDetachedRunnerLivenessSweepState(): AbsurdDetachedRunnerLivenessSweepState {
+  return {
+    generation: 0,
+    nextCheckAtMs: new Map(),
+    afterRunIds: { daytona: null, modal: null },
+    readsInFlight: { daytona: 0, modal: 0 },
+    abandonedReads: { daytona: 0, modal: 0 },
+    settling: false,
+    projecting: false,
+  };
+}
+
+export type AbsurdDetachedRunnerLivenessOutcome = {
+  stale: number;
+  notDue: number;
+  inspected: number;
+  /** Enforce mode: recorded terminals this pass tried to settle. */
+  recorded: number;
+  unproven: number;
+  proven: number;
+  /** Observe mode: proven gone, and nothing was written. */
+  wouldSettle: number;
+  /** A heartbeat, terminal or cancellation moved during the read and won. */
+  fenced: number;
+  settled: number;
+  settlementFailed: number;
+  invalidSuspensions: number;
+  /** No answer: the read failed, hung, or was not started. Never proof. */
+  unreadable: number;
+  /** Providers with due sandboxes and no answered read this pass, so their
+   * silence in `wouldSettle` says nothing about false positives. */
+  blindProviders: DetachedRunnerLivenessProvider[];
+  budgetExhausted: boolean;
+};
+
+async function runDetachedRunnerLivenessLane<T>(input: {
+  items: readonly T[];
+  concurrency: number;
+  active: () => boolean;
+  visit: (item: T) => Promise<void>;
+}): Promise<number> {
+  let visited = 0;
+  await Promise.all(
+    Array.from(
+      { length: Math.min(input.concurrency, input.items.length) },
+      async () => {
+        while (visited < input.items.length && input.active()) {
+          const item = input.items[visited]!;
+          visited += 1;
+          await input.visit(item);
+        }
+      },
+    ),
+  );
+  return visited;
+}
+
+/**
+ * One pass of the fixed controller's liveness sweep. Parked confirmation slices
+ * wait behind their own org's backlog in org-ranked claims, which delayed 298
+ * lost verdicts about 27 minutes on Oct 29, so this reads each stale sandbox
+ * outside those claims. An ended sandbox settles at once. A running or
+ * unreadable one is left alone, however long it stays silent.
+ *
+ * Discovery reads, then no scheduler client is held across a provider read, and
+ * the verdict is one conditional first write. That write is the same runner
+ * terminal the parked attempt would record, so a racing wake commits the same
+ * result, and the same settlement path leaves the same cleanup, billing and
+ * outbox facts on any lane. No customer code runs and nothing is replayed.
+ * A terminal this pass commits is then projected and released as its parked
+ * worker would have; one another writer committed is left to that writer.
+ *
+ * The pass waits on verdict writes, settlement and projection only until its
+ * budget. A settlement or projection that overruns keeps its slot, so no pass
+ * starts another until it ends; a verdict that commits late settles next pass.
+ *
+ * In `observe` mode the pass reads providers and reports what it would settle,
+ * and writes nothing.
+ */
+export async function recoverAbsurdRuntimeStaleDetachedRunners(input: {
+  withClient: <T>(
+    fn: (client: PostgresSchedulerQueryClient) => Promise<T>,
+  ) => Promise<T>;
+  state: AbsurdDetachedRunnerLivenessSweepState;
+  mode: 'observe' | 'enforce';
+  inspect?: (
+    suspension: DetachedRunnerSuspension,
+  ) => Promise<DetachedRunnerInspection | null>;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+  /** Whether a Modal canary run's terminal counts as production evidence. */
+  canaryProduction: boolean;
+  /** Projects each terminal this sweep settles, as its parked worker would
+   * have. Without it, only the run-ledger projector delivers the terminal. */
+  projection?: AbsurdRuntimeTerminalProjectionDeps;
+  pageLimit?: number;
+  concurrency?: number;
+  clock?: {
+    now: () => number;
+    budgetElapsed?: Promise<void>;
+    readDeadline?: () => Promise<void>;
+  };
+}): Promise<AbsurdDetachedRunnerLivenessOutcome> {
+  const { state } = input;
+  state.abandonedReads ??= { daytona: 0, modal: 0 };
+  state.settling ??= false;
+  state.projecting ??= false;
+  const readDeadline =
+    input.clock?.readDeadline ??
+    (() =>
+      new Promise<void>((resolve) => {
+        setTimeout(
+          resolve,
+          ABSURD_DETACHED_RUNNER_LIVENESS_READ_DEADLINE_MS,
+        ).unref?.();
+      }));
+  const answeredReads: Record<DetachedRunnerLivenessProvider, number> = {
+    daytona: 0,
+    modal: 0,
+  };
+  const options = input.schedulerOptions ?? {};
+  const now = input.clock?.now ?? Date.now;
+  const limit = input.pageLimit ?? ABSURD_DETACHED_RUNNER_LIVENESS_PAGE_LIMIT;
+  const concurrency =
+    input.concurrency ?? ABSURD_DETACHED_RUNNER_LIVENESS_READS_PER_PROVIDER;
+  // A provider-state read only: see `skipCapturedOutput`.
+  const inspect =
+    input.inspect ??
+    ((suspension: DetachedRunnerSuspension) =>
+      defaultRuntimeSandboxReconciliation.inspectDetachedRunner(
+        runtimeSandboxRefFromSuspension(suspension),
+        suspension,
+        { skipCapturedOutput: true },
+      ));
+  state.generation += 1;
+  const generation = state.generation;
+  const deadlineAtMs = now() + ABSURD_DETACHED_RUNNER_LIVENESS_PASS_BUDGET_MS;
+  const active = () => state.generation === generation && now() < deadlineAtMs;
+  for (const [key, dueAtMs] of state.nextCheckAtMs) {
+    if (dueAtMs <= now()) state.nextCheckAtMs.delete(key);
+  }
+  const outcome: AbsurdDetachedRunnerLivenessOutcome = {
+    stale: 0,
+    notDue: 0,
+    inspected: 0,
+    recorded: 0,
+    unproven: 0,
+    proven: 0,
+    wouldSettle: 0,
+    fenced: 0,
+    settled: 0,
+    settlementFailed: 0,
+    invalidSuspensions: 0,
+    unreadable: 0,
+    blindProviders: [],
+    budgetExhausted: false,
+  };
+  const work = await input.withClient((client) =>
+    listPostgresSchedulerDetachedRunnerLivenessWork(
+      client,
+      {
+        limit,
+        defaultHeartbeatTimeoutMs: RUNTIME_CLAIM_LEASE_SECONDS * 1000,
+        afterRunIds: state.afterRunIds,
+      },
+      options,
+    ),
+  );
+  outcome.stale = work.stale.length;
+
+  let budgetTimer: ReturnType<typeof setTimeout> | undefined;
+  const budgetElapsed =
+    input.clock?.budgetElapsed ??
+    new Promise<void>((resolve) => {
+      budgetTimer = setTimeout(
+        resolve,
+        ABSURD_DETACHED_RUNNER_LIVENESS_PASS_BUDGET_MS,
+      );
+    });
+
+  /** Never waits on a projection an earlier settlement left running. */
+  const projectTerminal = async (
+    settled: SettledRunnerTerminal,
+  ): Promise<void> => {
+    if (!input.projection) return;
+    const { terminal } = settled;
+    if (state.projecting) {
+      // The run-ledger projector still delivers this terminal.
+      console.warn(
+        '[absurd-runtime.worker] detached_runner_liveness_projection_skipped',
+        { runId: terminal.runId, attempt: terminal.attempt },
+      );
+      return;
+    }
+    state.projecting = true;
+    const deadline = new AbortController();
+    void budgetElapsed.then(() => deadline.abort());
+    await projectRecoveredRunnerTerminal({
+      deps: input.projection,
+      launch: settled.launch,
+      runId: terminal.runId,
+      attempt: terminal.attempt,
+      runnerResult: settled.runnerResult,
+      options,
+      signal: deadline.signal,
+    }).finally(() => {
+      state.projecting = false;
+    });
+  };
+
+  const releaseAfterTerminal = async (settled: SettledRunnerTerminal) => {
+    const { terminal, launch, runnerResult, suspension } = settled;
+    try {
+      if (suspension) {
+        reportDetachedTerminalToModalCanary({
+          launch,
+          runId: terminal.runId,
+          attempt: terminal.attempt,
+          policy: resolveLaunchRuntimeSandboxPlacementPolicy({
+            runtimeBackend: normalizePlayRuntimeBackend(
+              launch.runtimeBackend || 'daytona',
+            ),
+            runtimeSandboxPlacementPolicyId:
+              launch.runtimeSandboxPlacementPolicyId,
+          }),
+          production: input.canaryProduction,
+          suspension,
+          runnerResult,
+        });
+      }
+    } catch (error) {
+      console.warn(
+        '[absurd-runtime.worker] detached_runner_liveness_canary_failed',
+        {
+          runId: terminal.runId,
+          error: error instanceof Error ? error.message : String(error),
+        },
+      );
+    }
+    // A customer database or the app can stall here, so this waits only for
+    // the budget. Terminal truth is committed and an expired lease is the
+    // crash path.
+    await Promise.race([
+      Promise.all([
+        projectTerminal(settled),
+        settled.schedulerStatus === 'failed'
+          ? releaseFailedTerminalAttempt({
+              runWithClient: input.withClient,
+              launch,
+              runId: terminal.runId,
+              generation: terminal.attempt,
+              options,
+            })
+          : undefined,
+      ]),
+      budgetElapsed,
+    ]);
+  };
+
+  type Settlement =
+    | { kind: 'settled'; entries: SettledRunnerTerminal[] }
+    | { kind: 'deferred'; reason: 'pass_ended' | 'store_busy' };
+  // One at a time, across passes too: a settlement loads the launch, which
+  // can be large, and a store that stalls one must not gain one per pass.
+  let settlementTurn: Promise<unknown> = Promise.resolve();
+  const settleTerminal = async (
+    terminal: PostgresSchedulerUnsettledRunnerTerminal,
+  ): Promise<void> => {
+    const settle = settlementTurn.then(async (): Promise<Settlement> => {
+      // Queued behind a stalled one, it never starts after its pass's budget.
+      if (!active()) return { kind: 'deferred', reason: 'pass_ended' };
+      if (state.settling) return { kind: 'deferred', reason: 'store_busy' };
+      state.settling = true;
+      try {
+        const entries: SettledRunnerTerminal[] = [];
+        await input.withClient(async (client) => {
+          await recoverAbsurdRuntimeCompletedRunnerTerminals({
+            client,
+            // The run's immutable launch names its lane, so no lane list is
+            // needed and a long-retired lane settles like a live one.
+            queue: absurdQueueNameForRelease(
+              absurdReleaseIdForLaunch({ absurdReleaseId: terminal.releaseId }),
+            ),
+            schedulerOptions: options,
+            terminals: [
+              {
+                ...terminal,
+                result: await readPostgresSchedulerRunnerTerminal(
+                  client,
+                  { runId: terminal.runId, attempt: terminal.attempt },
+                  options,
+                ),
+              },
+            ],
+            onTerminalSettled: (entry) => entries.push(entry),
+          });
+        });
+        return { kind: 'settled', entries };
+      } finally {
+        state.settling = false;
+      }
+    });
+    settlementTurn = settle.catch(() => undefined);
+    const logFailure = (error: unknown, afterPass: boolean) =>
+      console.error(
+        '[absurd-runtime.worker] detached_runner_liveness_settlement_failed',
+        {
+          runId: terminal.runId,
+          attempt: terminal.attempt,
+          afterPass,
+          error: error instanceof Error ? error.message : String(error),
+        },
+      );
+    try {
+      const settlement = await Promise.race([
+        settle,
+        budgetElapsed.then(() => 'budget_spent' as const),
+      ]);
+      if (settlement === 'budget_spent') {
+        // It holds the store until it ends, and no settlement starts before.
+        // A terminal it commits still releases its attempt and folds its
+        // progress. The spent budget has already cancelled its outbox
+        // request, so the run-ledger projector delivers that row.
+        console.warn(
+          '[absurd-runtime.worker] detached_runner_liveness_settlement_overran',
+          { runId: terminal.runId, attempt: terminal.attempt },
+        );
+        void settle
+          .then(async (late) => {
+            if (late.kind !== 'settled') return;
+            for (const entry of late.entries) await releaseAfterTerminal(entry);
+          })
+          .catch((error: unknown) => logFailure(error, true));
+        return;
+      }
+      if (settlement.kind === 'deferred') {
+        // The recorded terminal stays for the next pass or the parked attempt.
+        if (settlement.reason === 'store_busy') {
+          console.warn(
+            '[absurd-runtime.worker] detached_runner_liveness_settlement_deferred',
+            { runId: terminal.runId, attempt: terminal.attempt },
+          );
+        }
+        return;
+      }
+      for (const entry of settlement.entries) {
+        // A terminal another writer committed counts as theirs, but its
+        // cleanup still runs here: the controller's lane recovery does none
+        // of it, and a worker can die between its commit and its cleanup.
+        // Each step is safe to repeat, as the lane recovery's projection is.
+        if (entry.terminalizedByAnotherWorker) outcome.fenced += 1;
+        else outcome.settled += 1;
+        await releaseAfterTerminal(entry);
+      }
+    } catch (error) {
+      // The terminal stays on the attempt, so the next pass retries it and the
+      // parked attempt can still commit the same verdict.
+      outcome.settlementFailed += 1;
+      logFailure(error, false);
+    }
+  };
+
+  const inspectStaleRunner = async (
+    candidate: PostgresSchedulerStaleDetachedRunner,
+  ): Promise<void> => {
+    const provider = candidate.provider;
+    const recheckLater = () =>
+      state.nextCheckAtMs.set(
+        `${candidate.runId}:${candidate.attempt}`,
+        now() + ABSURD_DETACHED_RUNNER_LIVENESS_RECHECK_MS,
+      );
+    const suspension = parseOverdueDetachedRunnerSuspension(
+      candidate.suspension,
+    );
+    if (!suspension) {
+      outcome.invalidSuspensions += 1;
+      recheckLater();
+      console.warn(
+        '[absurd-runtime.worker] detached_runner_liveness_invalid_suspension',
+        { runId: candidate.runId, attempt: candidate.attempt },
+      );
+      return;
+    }
+    if (
+      state.abandonedReads[provider] >=
+      ABSURD_DETACHED_RUNNER_LIVENESS_MAX_ABANDONED_READS
+    ) {
+      outcome.unreadable += 1;
+      return;
+    }
+    outcome.inspected += 1;
+    state.readsInFlight[provider] += 1;
+    let slotHeld = true;
+    let abandoned = false;
+    const releaseSlot = () => {
+      if (!slotHeld) return;
+      slotHeld = false;
+      state.readsInFlight[provider] -= 1;
+    };
+    const read = Promise.resolve()
+      .then(() => inspect(suspension))
+      .catch((error: unknown) => {
+        console.warn(
+          '[absurd-runtime.worker] detached_runner_liveness_evidence_failed',
+          JSON.stringify({
+            runId: candidate.runId,
+            sandboxId: suspension.sandboxId,
+            error: error instanceof Error ? error.message : String(error),
+          }),
+        );
+        return null;
+      })
+      .finally(() => {
+        if (abandoned) state.abandonedReads[provider] -= 1;
+        releaseSlot();
+      });
+    void readDeadline().then(() => {
+      if (!slotHeld) return;
+      abandoned = true;
+      state.abandonedReads[provider] += 1;
+      releaseSlot();
+      console.warn(
+        '[absurd-runtime.worker] detached_runner_liveness_read_abandoned',
+        JSON.stringify({
+          runId: candidate.runId,
+          sandboxId: suspension.sandboxId,
+          provider,
+          abandonedReads: state.abandonedReads[provider],
+        }),
+      );
+    });
+    const inspection = await Promise.race([
+      read,
+      budgetElapsed.then(() => null),
+    ]);
+    recheckLater();
+    if (
+      inspection &&
+      !DETACHED_RUNNER_UNANSWERED_STAGES.has(inspection.diagnosis?.stage ?? '')
+    ) {
+      answeredReads[provider] += 1;
+    } else {
+      outcome.unreadable += 1;
+    }
+    if (!inspection || !detachedRunnerInspectionProvesGone(inspection)) {
+      outcome.unproven += 1;
+      console.warn(
+        '[absurd-runtime.worker] detached_runner_liveness_unproven',
+        JSON.stringify({
+          runId: candidate.runId,
+          runnerAttempt: candidate.attempt,
+          sandboxId: suspension.sandboxId,
+          diagnosis: inspection?.diagnosis?.stage ?? 'unavailable',
+        }),
+      );
+      return;
+    }
+    outcome.proven += 1;
+    if (input.mode === 'observe') {
+      outcome.wouldSettle += 1;
+      console.info(
+        '[absurd-runtime.worker] detached_runner_liveness_would_settle',
+        JSON.stringify({
+          runId: candidate.runId,
+          runnerAttempt: candidate.attempt,
+          sandboxId: suspension.sandboxId,
+          diagnosis: inspection.diagnosis?.stage ?? 'unknown',
+        }),
+      );
+      return;
+    }
+    const observedAtMs = now();
+    const heartbeatAtMs = candidate.heartbeatAtMs ?? Number.NaN;
+    const heartbeatAgeMs = Number.isFinite(heartbeatAtMs)
+      ? Math.max(0, observedAtMs - heartbeatAtMs)
+      : null;
+    const verdict: PlayRunnerResult =
+      inspection.result && inspection.result.status !== 'suspended'
+        ? inspection.result
+        : detachedRunnerLostResult({
+            suspension,
+            observedAtMs,
+            livenessStatus: 'waiting',
+            heartbeatAtMs,
+            heartbeatAgeMs,
+            inspectionAttempted: true,
+            inspection,
+            inspectionFailed: false,
+          });
+    const logVerdictFailure = (error: unknown, afterPass: boolean) =>
+      console.error(
+        '[absurd-runtime.worker] detached_runner_liveness_verdict_failed',
+        {
+          runId: candidate.runId,
+          attempt: candidate.attempt,
+          afterPass,
+          error: error instanceof Error ? error.message : String(error),
+        },
+      );
+    const reportExpired = () => {
+      if (verdict.status !== 'failed' || verdict === inspection.result) return;
+      reportDetachedRunnerLivenessExpired({
+        runId: candidate.runId,
+        suspension,
+        observedAtMs,
+        liveness: { heartbeatAt: candidate.heartbeatAt, status: 'waiting' },
+        heartbeatAgeMs,
+      });
+    };
+    const write = input.withClient((client) =>
+      recordPostgresSchedulerDetachedRunnerLivenessVerdict(
+        client,
+        {
+          runId: candidate.runId,
+          attempt: candidate.attempt,
+          leaseToken: candidate.leaseToken,
+          observedHeartbeatAt: candidate.heartbeatAt,
+          // Fallback can leave `runtimeBackend` naming a provider that never
+          // ran this sandbox, so record the one that did.
+          result: {
+            ...verdict,
+            stats: { ...verdict.stats, sandboxProvider: provider },
+          },
+        },
+        options,
+      ),
+    );
+    try {
+      const recorded = await Promise.race([
+        write,
+        budgetElapsed.then(() => 'budget_spent' as const),
+      ]);
+      if (recorded === 'budget_spent') {
+        // If it commits, the next pass settles it like any recorded terminal.
+        // The scheduler pool's acquire and query timeouts end it otherwise.
+        console.warn(
+          '[absurd-runtime.worker] detached_runner_liveness_verdict_overran',
+          { runId: candidate.runId, attempt: candidate.attempt },
+        );
+        void write
+          .then((late) => {
+            if (late) reportExpired();
+          })
+          .catch((error: unknown) => logVerdictFailure(error, true));
+        return;
+      }
+      if (!recorded) {
+        outcome.fenced += 1;
+        return;
+      }
+    } catch (error) {
+      outcome.settlementFailed += 1;
+      logVerdictFailure(error, false);
+      return;
+    }
+    reportExpired();
+    await settleTerminal(candidate);
+  };
+
+  try {
+    await Promise.all([
+      input.mode === 'enforce'
+        ? runDetachedRunnerLivenessLane({
+            items: work.terminals,
+            concurrency: 1,
+            active,
+            visit: settleTerminal,
+          }).then((visited) => {
+            outcome.recorded = visited;
+          })
+        : null,
+      ...ABSURD_DETACHED_RUNNER_LIVENESS_PROVIDERS.map(async (provider) => {
+        const page = work.stale.filter(
+          (candidate) => candidate.provider === provider,
+        );
+        const due = page.filter(
+          (candidate) =>
+            !state.nextCheckAtMs.has(`${candidate.runId}:${candidate.attempt}`),
+        );
+        outcome.notDue += page.length - due.length;
+        const visited = await runDetachedRunnerLivenessLane({
+          items: due,
+          concurrency: Math.max(0, concurrency - state.readsInFlight[provider]),
+          active,
+          visit: inspectStaleRunner,
+        });
+        if (due.length > 0 && answeredReads[provider] === 0) {
+          outcome.blindProviders.push(provider);
+        }
+        // Walk the set rather than rereading its head, so a stale set larger
+        // than a page still reaches every sandbox. A short page wraps.
+        if (visited < due.length) {
+          if (visited > 0) {
+            state.afterRunIds[provider] = due[visited - 1]!.runId;
+          }
+        } else {
+          state.afterRunIds[provider] =
+            page.length < limit ? null : (page.at(-1)?.runId ?? null);
+        }
+      }),
+    ]);
+  } finally {
+    if (budgetTimer) clearTimeout(budgetTimer);
+  }
+  outcome.budgetExhausted = now() >= deadlineAtMs;
+  return outcome;
+}
+
 /**
  * Non-terminal task states on a lane queue's `absurd.t_<queue>` table. A queue
  * with zero of these has no runnable/parked work left — the lane is drainable.
@@ -7006,12 +8062,16 @@ export async function runAbsurdLaneAdoptionSupervisor(input: {
   startAdoptionWorker: (queue: string) => Promise<AbsurdAdoptedLaneWorker>;
   intervalMs?: number;
   maxQueues?: number;
-  sleep?: (ms: number) => Promise<void>;
+  sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+  signal?: AbortSignal;
   shouldStop?: () => boolean;
   onAttached?: (queues: readonly string[]) => void;
   onError?: (error: unknown) => void;
 }): Promise<void> {
-  const wait = input.sleep ?? ((ms: number) => sleep(ms));
+  const wait =
+    input.sleep ??
+    ((ms: number, signal?: AbortSignal) => sleep(ms, undefined, { signal }));
+  const stopped = () => input.signal?.aborted || input.shouldStop?.();
   const intervalMs =
     input.intervalMs ?? ABSURD_LANE_ADOPTION_DISCOVERY_INTERVAL_MS;
   const maxQueues = input.maxQueues ?? ABSURD_LANE_ADOPTION_MAX_QUEUES;
@@ -7045,9 +8105,10 @@ export async function runAbsurdLaneAdoptionSupervisor(input: {
   };
   try {
     for (;;) {
-      if (input.shouldStop?.()) return;
+      if (stopped()) return;
       try {
         const adoptableReleaseIds = await input.discoverAdoptableReleaseIds();
+        if (stopped()) return;
         const allQueues = absurdAdoptionQueuesForReleases({
           workerReleaseId: input.workerReleaseId,
           adoptableReleaseIds,
@@ -7076,10 +8137,12 @@ export async function runAbsurdLaneAdoptionSupervisor(input: {
           console.info('[absurd-runtime.worker] adoption_detached', { queue });
         }
         for (const queue of wanted) {
+          if (stopped()) return;
           if (attached.has(queue) || draining.has(queue)) continue;
           if (attached.size + draining.size >= maxQueues) break;
           const worker = await input.startAdoptionWorker(queue);
           attached.set(queue, worker);
+          if (stopped()) return;
           console.info('[absurd-runtime.worker] adoption_attached', {
             queue,
             workerReleaseId: input.workerReleaseId,
@@ -7105,8 +8168,18 @@ export async function runAbsurdLaneAdoptionSupervisor(input: {
           error: error instanceof Error ? error.message : String(error),
         });
       }
-      if (input.shouldStop?.()) return;
-      await wait(intervalMs);
+      if (stopped()) return;
+      try {
+        await wait(intervalMs, input.signal);
+      } catch (error) {
+        if (
+          input.signal?.aborted &&
+          error instanceof Error &&
+          error.name === 'AbortError'
+        )
+          return;
+        throw error;
+      }
     }
   } finally {
     await closeAll();
@@ -7680,6 +8753,9 @@ async function processPostgresSchedulerSandboxCleanupJob(input: {
     cleanupAttempt: job.attemptCount,
     failureCount: job.failureCount,
   } as const;
+  const deleteProof: {
+    acceptance: PostgresSchedulerSandboxDeleteAcceptance | null;
+  } = { acceptance: null };
   const cleanup = createSandboxCleanupJob({
     store: createPostgresSandboxCleanupStore({
       runWithClient: input.runWithClient,
@@ -7692,7 +8768,27 @@ async function processPostgresSchedulerSandboxCleanupJob(input: {
         timeoutMs,
       }): Promise<SandboxCleanupDeleteOutcome> => {
         try {
-          await recordCleanupEvent('provider_delete_started', { timeoutMs });
+          // Only a job claimed before can have an earlier accepted delete. A
+          // failed read retries, so a 404 is never judged without this proof.
+          deleteProof.acceptance =
+            resource.provider === 'daytona' && resource.cleanupAttempt > 1
+              ? await input.runWithClient((client) =>
+                  readPostgresSchedulerSandboxDeleteAcceptance(
+                    client,
+                    {
+                      provider: resource.provider,
+                      sandboxId: resource.sandboxId,
+                      routingDomain: resource.routingDomain,
+                      runId: resource.runId,
+                      attempt: resource.attempt,
+                    },
+                    input.schedulerOptions,
+                  ),
+                )
+              : null;
+          if (!deleteProof.acceptance) {
+            await recordCleanupEvent('provider_delete_started', { timeoutMs });
+          }
           const providerOutcome = await deleteRuntimeSandboxBounded({
             deleteRuntimeSandbox: async (sandboxId) => {
               if (resource.provider === 'modal') {
@@ -7728,6 +8824,7 @@ async function processPostgresSchedulerSandboxCleanupJob(input: {
                 return await input.deleteRuntimeSandboxWithOutcome(
                   sandboxId,
                   resource.routingDomain,
+                  deleteProof.acceptance,
                 );
               }
               return (await input.deleteRuntimeSandbox(sandboxId))
@@ -7754,6 +8851,24 @@ async function processPostgresSchedulerSandboxCleanupJob(input: {
               routingDomain: providerOutcome.organizationId,
             };
           }
+          if (providerOutcome.kind === 'delete_pending') {
+            // The first acceptance is durable proof for the read-back, and the
+            // moment billing keeps as this sandbox's end.
+            if (providerOutcome.deleteAccepted && !deleteProof.acceptance) {
+              await recordCleanupEvent('provider_delete_accepted', {
+                routingDomain: providerOutcome.organizationId,
+              });
+            } else {
+              await recordCleanupEvent('provider_delete_pending', {
+                code: providerOutcome.code,
+              });
+            }
+            return {
+              kind: 'pending',
+              code: providerOutcome.code,
+              detail: `${providerOutcome.code}: ${providerOutcome.detail}`,
+            };
+          }
           await recordCleanupEvent('provider_delete_failed', {
             code:
               'code' in providerOutcome
@@ -7815,6 +8930,15 @@ async function processPostgresSchedulerSandboxCleanupJob(input: {
       cleanupAuthority: 'durable_cleanup_claim',
       providerOutcome: outcome.providerOutcome,
       completionOutcome: outcome.completion,
+      // An upper bound: the 404 was seen on a paced read, not as it happened.
+      ...(deleteProof.acceptance
+        ? {
+            disappearedAfterMs: Math.max(
+              0,
+              Date.now() - deleteProof.acceptance.acceptedAtMs,
+            ),
+          }
+        : {}),
     });
     return { deleted: 1, retried: 0 };
   }
@@ -8000,7 +9124,7 @@ export async function reconcilePostgresSchedulerAmbiguousModalCreatesOnce(input:
   return { observed: intents.length, resourcesRecorded, resolvedAbsent };
 }
 
-async function reconcileReservedDaytonaCancellationOnce(input: {
+export async function reconcileReservedDaytonaCancellationOnce(input: {
   runWithClient: AbsurdRuntimeWorkerDeps['runWithClient'];
   environment: 'preview' | 'production';
   schedulerOptions: Pick<PostgresSchedulerOptions, 'schema'>;
@@ -8037,7 +9161,11 @@ async function reconcileReservedDaytonaCancellationOnce(input: {
     await input.runWithClient((client) =>
       abandonRuntimeSandboxOperationReservation(
         client,
-        { operationKey: operation.operationKey },
+        {
+          operationKey: operation.operationKey,
+          // Execution recovery ending is not proof a sent create cannot land.
+          onlyWhileRunActive: true,
+        },
         input.schedulerOptions,
       ),
     );
@@ -8045,27 +9173,75 @@ async function reconcileReservedDaytonaCancellationOnce(input: {
   } finally {
     control.dispose();
   }
-  if (sandbox.labels?.operationKey !== operation.providerOperationToken) {
+  await persistReservedDaytonaCancellationCleanupIntent({
+    operation,
+    sandbox,
+    configuredOrganizationId: process.env.DAYTONA_ORGANIZATION_ID?.trim(),
+    recordCleanupIntent: (identity) =>
+      input.runWithClient((client) =>
+        recordDaytonaSandboxCleanupIntent(
+          client,
+          identity,
+          input.schedulerOptions,
+        ),
+      ),
+  });
+}
+
+export async function persistReservedDaytonaCancellationCleanupIntent(input: {
+  operation: {
+    operationKey: string;
+    providerOperationToken: string;
+    orgId: string;
+    runId: string;
+  };
+  sandbox: {
+    id: string;
+    name?: string | null;
+    organizationId?: string | null;
+    labels?: Record<string, string | undefined> | null;
+  };
+  configuredOrganizationId?: string | null;
+  recordCleanupIntent: (identity: {
+    operationKey: string;
+    providerOperationToken: string;
+    orgId: string;
+    runId: string;
+    sandboxId: string;
+    routingDomain: string;
+  }) => Promise<boolean>;
+}): Promise<void> {
+  const sandboxName = `dl-${input.operation.providerOperationToken}`;
+  if (
+    !daytonaSandboxMatchesOperationIdentity({
+      sandbox: input.sandbox,
+      sandboxName,
+      providerOperationToken: input.operation.providerOperationToken,
+      orgId: input.operation.orgId,
+      runId: input.operation.runId,
+      configuredOrganizationId: input.configuredOrganizationId,
+    })
+  ) {
     throw new Error(
-      `Daytona cancellation lookup ${sandboxName} lacks its exact operation label.`,
+      `Daytona cancellation lookup ${sandboxName} has a mismatched sandbox identity or organization routing; cleanup suppressed.`,
     );
   }
-  if (!sandbox.organizationId) {
+  const routingDomain = input.sandbox.organizationId?.trim();
+  if (!routingDomain) {
     throw new Error(
-      `Daytona cancellation lookup ${sandboxName} lacks its routing organization.`,
+      `Daytona cancellation lookup ${sandboxName} lacks its routing organization; cleanup suppressed.`,
+    );
+  }
+  const recorded = await input.recordCleanupIntent({
+    ...input.operation,
+    sandboxId: input.sandbox.id,
+    routingDomain,
+  });
+  if (!recorded) {
+    throw new Error(
+      `Daytona cancellation cleanup intent for ${sandboxName} was not durably recorded.`,
     );
   }
-  await input.runWithClient((client) =>
-    recordRecoveredDaytonaCancellationSandbox(
-      client,
-      {
-        operationKey: operation.operationKey,
-        sandboxId: sandbox.id,
-        routingDomain: sandbox.organizationId,
-      },
-      input.schedulerOptions,
-    ),
-  );
 }
 
 export async function runPostgresSchedulerSandboxCleanupController(input: {
@@ -8773,7 +9949,7 @@ export async function startRuntimeWorkerHost(
   // claiming and still hold parked durable waits, so a retired lane does not
   // have to stay alive until its last sleeping run wakes. A draining lane
   // attaches nothing — it is on its way out itself.
-  let adoptionStopped = false;
+  const adoptionStop = new AbortController();
   let adoptionSupervisor: Promise<void> | null = null;
   if (!drainExit) {
     const releaseEnvironment =
@@ -8782,7 +9958,7 @@ export async function startRuntimeWorkerHost(
         : ('production' as const);
     adoptionSupervisor = runAbsurdLaneAdoptionSupervisor({
       workerReleaseId: releaseId,
-      shouldStop: () => adoptionStopped,
+      signal: adoptionStop.signal,
       discoverAdoptableReleaseIds: async () => {
         const lanes = await withRuntimeSchedulerClient(poolOptions, (client) =>
           listPostgresSchedulerAdoptableReleaseLanes(
@@ -8903,7 +10079,7 @@ export async function startRuntimeWorkerHost(
       drainStopped = true;
       billingStopped = true;
       cleanupStopped = true;
-      adoptionStopped = true;
+      adoptionStop.abort();
       overdueCeilingStopped = true;
     },
     controllers: [