deepline 0.3.257 → 0.3.259

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -134,7 +134,10 @@ import {
   cleanupPostgresSchedulerMaterializedFiles,
   completePostgresSchedulerComputeBillingJob,
   completePostgresSchedulerComputeBillingSessionBootstrap,
+  claimPostgresSchedulerWaitingDetachedAttemptForTerminal,
   deferPostgresSchedulerAttemptForCapacity,
+  deferPostgresSchedulerAttemptForExecutorTokenOutage,
+  detachedRunnerCeilingDeadlineMsFromSuspension,
   enqueuePostgresSchedulerSandboxCleanupForAttempt,
   enqueuePostgresSchedulerSandboxCleanupForRun,
   hasPostgresSchedulerRecoverableDetachedRunner,
@@ -225,9 +228,11 @@ import {
 } from './postgres-pool';
 import { runWithAbsurdSchedulerQuerySignal } from './absurd-pool';
 import {
+  buildFailedTerminalSettledProgressEvents,
   createPostgresSchedulerProgressReporter,
   type PostgresSchedulerProgressReporter,
 } from './postgres-progress';
+import type { PlayRunLedgerEvent } from '../run-ledger';
 import {
   ABSURD_PLAY_RUN_MAX_ATTEMPTS,
   ABSURD_PLAY_RUN_RETRY_STRATEGY,
@@ -310,6 +315,43 @@ class AttemptExecutorTokenCapacityError extends RuntimeSchedulerCapacityError {
   }
 }
 
+/**
+ * The executor-token route was unreachable or answered with a transient
+ * front-door/server failure (transport error, timeout, 5xx, 408/429, or a
+ * non-JSON 2xx gateway page). The mint precedes any Play code, so the handler
+ * defers the durable task — bounded by wall-clock — instead of spending an
+ * engine attempt. Deterministic 4xx rejections (auth, lease fence) stay
+ * ordinary failures.
+ */
+export class AttemptExecutorTokenUnavailableError extends Error {
+  constructor(
+    message: string,
+    readonly status: number | null,
+  ) {
+    super(message);
+    this.name = 'AttemptExecutorTokenUnavailableError';
+  }
+}
+
+/** Wall-clock budget for one executor-token request (headers and body). */
+const ATTEMPT_EXECUTOR_TOKEN_MINT_TIMEOUT_MS = 30_000;
+/**
+ * Longest consecutive executor-token outage a fresh attempt may defer
+ * through. A detached-runner wake is bounded by its own persisted hard
+ * ceiling instead. Past either bound the run is terminalized loudly.
+ */
+export const ABSURD_EXECUTOR_TOKEN_OUTAGE_MAX_MS = 10 * 60_000;
+const ABSURD_EXECUTOR_TOKEN_OUTAGE_DEFER_SECONDS = 5;
+
+function isTransientExecutorTokenStatus(
+  status: number,
+  jsonBody: boolean,
+): boolean {
+  if (status >= 500 || status === 408 || status === 429) return true;
+  // A 2xx that is not JSON is a front-door interstitial, not the route.
+  return status >= 200 && status < 300 && !jsonBody;
+}
+
 // No `installPgRuntimePoolDriver()` here: the runtime Postgres driver belongs to
 // the process that owns this module's boot — `apps/absurd-runtime-worker` and
 // `apps/absurd-release-scaler` install it at their entrypoints. `apps/deepline-api`
@@ -976,26 +1018,48 @@ export async function requestAttemptExecutorToken(input: {
     attempt: input.attempt,
     traceId,
   });
-  const response = await fetch(url, {
-    method: 'POST',
-    headers: {
-      'content-type': 'application/json',
-      'x-deepline-internal-token': internalToken,
-      'x-deepline-runtime-trace-id': traceId,
-      ...vercelProtectionHeaders,
-    },
-    body: JSON.stringify({
+  const requestSignal = AbortSignal.timeout(
+    ATTEMPT_EXECUTOR_TOKEN_MINT_TIMEOUT_MS,
+  );
+  let response: Response;
+  try {
+    response = await fetch(url, {
+      signal: requestSignal,
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        'x-deepline-internal-token': internalToken,
+        'x-deepline-runtime-trace-id': traceId,
+        ...vercelProtectionHeaders,
+      },
+      body: JSON.stringify({
+        runId: input.launch.runId,
+        attempt: input.attempt,
+        leaseToken: input.leaseToken,
+        schedulerSchema: input.schedulerSchema ?? null,
+        // Expand-before-enable rollout fence. An old worker never sends this,
+        // so a newly deployed app cannot activate direct terminal reduction
+        // before the owning Fly lane has the cleanup controller.
+        terminalReducerContract: DIRECT_COMPLETED_TERMINAL_CONTRACT,
+      }),
+      redirect: 'manual',
+    });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    console.warn('[runtime-attempt-token.request_failed]', {
       runId: input.launch.runId,
       attempt: input.attempt,
-      leaseToken: input.leaseToken,
-      schedulerSchema: input.schedulerSchema ?? null,
-      // Expand-before-enable rollout fence. An old worker never sends this,
-      // so a newly deployed app cannot activate direct terminal reduction
-      // before the owning Fly lane has the cleanup controller.
-      terminalReducerContract: DIRECT_COMPLETED_TERMINAL_CONTRACT,
-    }),
-    redirect: 'manual',
-  });
+      traceId,
+      durationMs: Math.round(performance.now() - startedAt),
+      status: null,
+      code: 'TRANSPORT',
+      error: message,
+    });
+    throw new AttemptExecutorTokenUnavailableError(
+      `Attempt executor token mint failed [TRANSPORT] at ${new URL(url).host}: ${message}`,
+      null,
+    );
+  }
   const responseForFingerprint = response.clone();
   const body = (await response.json().catch(() => null)) as {
     executorToken?: unknown;
@@ -1038,9 +1102,11 @@ export async function requestAttemptExecutorToken(input: {
         response.status,
       );
     }
-    throw new Error(
-      `Attempt executor token mint failed [${code}] (${response.status}) at ${new URL(url).host}: ${typeof body?.error === 'string' ? body.error : 'invalid response'}`,
-    );
+    const failure = `Attempt executor token mint failed [${code}] (${response.status}) at ${new URL(url).host}: ${typeof body?.error === 'string' ? body.error : 'invalid response'}`;
+    if (isTransientExecutorTokenStatus(response.status, body !== null)) {
+      throw new AttemptExecutorTokenUnavailableError(failure, response.status);
+    }
+    throw new Error(failure);
   }
   console.info('[runtime-attempt-token.request_succeeded]', {
     runId: input.launch.runId,
@@ -1857,6 +1923,24 @@ export async function reconcileTerminalTruthIntoReporter(input: {
     });
   }
 
+  const outcomes = await readTerminalSheetRowOutcomes(input);
+  if (outcomes.length > 0) {
+    input.reporter.applyTerminalSheetOutcomes(outcomes);
+  }
+}
+
+/**
+ * Each runtime sheet's settled row split for this run — the data plane's
+ * account of the run's rows (ADR 0016 rule 3). Best-effort per table: a table
+ * whose sheet never materialized has no counts and is skipped loudly.
+ */
+async function readTerminalSheetRowOutcomes(input: {
+  launch: PlaySchedulerSubmitInput;
+  runId: string;
+  attempt: number;
+}): Promise<
+  Array<{ tableNamespace: string; completed: number; failed: number; total: number }>
+> {
   const sheetNamespaces = [
     ...new Set(
       (input.launch.preloadedDbSessions ?? [])
@@ -1866,7 +1950,7 @@ export async function reconcileTerminalTruthIntoReporter(input: {
     ),
   ];
   if (sheetNamespaces.length === 0) {
-    return;
+    return [];
   }
   const runtimeContext = {
     baseUrl: input.launch.baseUrl,
@@ -1907,8 +1991,35 @@ export async function reconcileTerminalTruthIntoReporter(input: {
       });
     }
   }
-  if (outcomes.length > 0) {
-    input.reporter.applyTerminalSheetOutcomes(outcomes);
+  return outcomes;
+}
+
+/**
+ * A failed runner result's settled map counts, read from the runtime sheets
+ * BEFORE the terminal write so they ride the `run.failed` outbox row ahead of
+ * the terminal fact (`buildFailedTerminalSettledProgressEvents`). Never
+ * blocks the terminal: a read that fails leaves the ledger at its last live
+ * counts, as before.
+ */
+async function collectFailedTerminalSettledProgressEvents(input: {
+  launch: PlaySchedulerSubmitInput;
+  runId: string;
+  attempt: number;
+}): Promise<PlayRunLedgerEvent[]> {
+  try {
+    const outcomes = await readTerminalSheetRowOutcomes(input);
+    return buildFailedTerminalSettledProgressEvents({
+      launch: input.launch,
+      outcomes,
+      occurredAt: Date.now(),
+    });
+  } catch (error) {
+    console.warn('[absurd-runtime.worker] failed_terminal_settled_counts_failed', {
+      runId: input.runId,
+      attempt: input.attempt,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return [];
   }
 }
 type ObservedTerminalSource =
@@ -1931,11 +2042,48 @@ function requestTerminalOutboxProjection(input: {
   launch: PlaySchedulerSubmitInput;
   runId: string;
   options: Pick<PostgresSchedulerOptions, 'schema'>;
+  /** Where the request came from, for terminal-visibility attribution. */
+  trigger?: string;
+  /** When this worker began post-commit projection for the terminal. */
+  projectionStartedAtMs?: number;
 }): void {
   const targetEventId = `run-terminal:${input.runId}`;
+  const requestedAtMs = Date.now();
+  // Terminal-visibility evidence: the request itself, and how long the
+  // post-commit work before it took, so a slow first claim is attributable.
+  console.info('[run-ledger-outbox.immediate_dispatch_requested]', {
+    runId: input.runId,
+    trigger: input.trigger ?? 'unspecified',
+    requestedAtMs,
+    sinceProjectionStartMs:
+      input.projectionStartedAtMs === undefined
+        ? null
+        : requestedAtMs - input.projectionStartedAtMs,
+  });
   void input.deps
     .projectOutbox(input.launch, input.runId, input.options, targetEventId)
     .then((projection) => {
+      // Terminal-visibility latency evidence (ADR-0022 budget): when this
+      // worker asked for delivery and how long the delivery took.
+      console.info('[run-ledger-outbox.immediate_dispatch_settled]', {
+        runId: input.runId,
+        requestedAtMs,
+        durationMs: Date.now() - requestedAtMs,
+        targetDelivered: projection?.targetDelivered ?? null,
+        attempted: projection?.attempted ?? null,
+        projected: projection?.projected ?? null,
+      });
+      if (projection && projection.attempted === 0 && !projection.targetDelivered) {
+        // Nothing was claimable for this run: another dispatcher holds its
+        // outbox lease or a predecessor is not yet due. Delivery now depends
+        // on that holder or the projector's next wave.
+        console.info('[run-ledger-outbox.immediate_dispatch_empty_claim]', {
+          runId: input.runId,
+          targetEventId,
+          requestedAtMs,
+          durationMs: Date.now() - requestedAtMs,
+        });
+      }
       // An empty drain means another concurrent projector already claimed and
       // delivered the terminal fact. It is not pending work and warning here
       // turns the normal exactly-once race into misleading operator noise.
@@ -2081,6 +2229,12 @@ async function settleObservedTerminal(input: {
       )
     )?.status;
   if (status !== 'completed' && status !== 'failed' && status !== 'cancelled') {
+    console.info('[run-ledger-outbox.immediate_dispatch_skipped]', {
+      runId: input.runId,
+      source: input.source,
+      reason: 'run_not_terminal',
+      status: status ?? null,
+    });
     return;
   }
   const backfilledBilling = await input.deps.runWithClient((client) =>
@@ -2100,7 +2254,7 @@ async function settleObservedTerminal(input: {
       },
     );
   }
-  requestTerminalOutboxProjection(input);
+  requestTerminalOutboxProjection({ ...input, trigger: input.source });
 }
 
 /**
@@ -2516,16 +2670,66 @@ async function executeAbsurdAttemptAdapter(
           cleanupAuthorized: false,
         }),
       );
-      throw new DeferTask(
-        `Detached runner ${runId}/${generation} retains execution after scheduler handoff failure.`,
-        1,
-      );
+      // A failed handoff never destroys the live runner (no cleanup is
+      // authorized here) and does not terminalize a Run whose runner may still
+      // finish (#6781). The wait stays bounded by wall-clock time: an
+      // executor-token outage defers only until the detached runner's hard
+      // ceiling; past it the Run is terminalized exactly once under its fence,
+      // and runner cleanup follows only from that recorded terminal outcome.
+      if (!(error instanceof AttemptExecutorTokenUnavailableError)) {
+        throw new DeferTask(
+          `Detached runner ${runId}/${generation} retains execution after scheduler handoff failure.`,
+          1,
+        );
+      }
+      // Throws DeferTask while inside the wall-clock bound.
+      await deferAbsurdAttemptForExecutorTokenOutage({
+        deps,
+        options,
+        runId,
+        generation,
+        leaseToken,
+        detachedRecovery: true,
+        detachedSuspension: detached.suspension,
+        error,
+      });
+      await salvageAbsurdFinalAttemptTerminal({
+        ctx,
+        deps,
+        options,
+        runId,
+        launch: executionLaunch,
+        generation,
+        leaseToken,
+        error,
+        detachedRecovery: true,
+        force: true,
+      });
+      throw error;
     }
     if (error instanceof CancelledTask || error instanceof FailedTask) {
       // Park/cancel signals belong to the engine; never treat them as leg
       // failures.
       throw error;
     }
+    let forceTerminalSalvage = false;
+    if (error instanceof AttemptExecutorTokenUnavailableError) {
+      // Throws DeferTask while inside the wall-clock bound; returns only when
+      // the bound is exhausted and this attempt must write the terminal.
+      await deferAbsurdAttemptForExecutorTokenOutage({
+        deps,
+        options,
+        runId,
+        generation,
+        leaseToken,
+        detachedRecovery: isDetachedRecovery,
+        detachedSuspension: isDetachedRecovery
+          ? (assigned.suspension ?? null)
+          : null,
+        error,
+      });
+      forceTerminalSalvage = true;
+    }
     // Token minting and other pre-run setup can fail after a sandbox has
     // already been recorded. There is no runner config available on this
     // outer path, so persist the controller-owned cleanup obligation here.
@@ -2577,6 +2781,8 @@ async function executeAbsurdAttemptAdapter(
       generation,
       leaseToken,
       error,
+      detachedRecovery: isDetachedRecovery,
+      force: forceTerminalSalvage,
     });
     throw error;
   } finally {
@@ -2781,6 +2987,16 @@ async function executeAssignedAbsurdAttempt(input: {
       });
     }
 
+    // A failed run's settled counts must reach the ledger BEFORE its terminal
+    // makes the steps immutable, so they ride the terminal outbox row.
+    const failedTerminalProgressEvents =
+      runnerResult.status === 'failed'
+        ? await collectFailedTerminalSettledProgressEvents({
+            launch,
+            runId,
+            attempt: generation,
+          })
+        : [];
     const result = await deps.runWithClient((client) =>
       executePostgresSchedulerClaim(
         client,
@@ -2790,6 +3006,9 @@ async function executeAssignedAbsurdAttempt(input: {
           ...options,
           terminalLogOutbox: input.fromDetachedRunner ? 'ctx_log_tail' : 'omit',
           sandboxCrashDiagnostic,
+          ...(failedTerminalProgressEvents.length > 0
+            ? { failedTerminalProgressEvents }
+            : {}),
         },
       ),
     );
@@ -2802,6 +3021,13 @@ async function executeAssignedAbsurdAttempt(input: {
         `absurd runtime worker: unexpected non-terminal claim status "${result.status}" for ${runId}.`,
       );
     }
+    console.info('[absurd-runtime.worker] terminal_committed', {
+      runId,
+      generation,
+      status: result.status,
+      fromDetachedRunner: input.fromDetachedRunner,
+      committedAtMs: Date.now(),
+    });
 
     await terminalProjection.projectCommitted({
       runnerResult: terminalEnvelopeFromRunnerResult(runnerResult),
@@ -4149,6 +4375,7 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
         );
       });
   };
+  let projectionStartedAtMs: number | undefined;
   return createPlayRunAttemptWorkerTerminalProjection({
     async observeFenceLost() {
       // A lost fence means this attempt is dead, which a cancelled run is the
@@ -4164,6 +4391,7 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
       });
     },
     async reconcileTerminalTruth() {
+      projectionStartedAtMs ??= Date.now();
       // ADR 0016 rule 3: detached Daytona runs may provide no live events, so
       // fold stored docflow/runtime-sheet facts into progress after truth wins.
       await reconcileTerminalTruthIntoReporter({
@@ -4247,6 +4475,10 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
         launch: input.launch,
         runId: input.runId,
         options: input.options,
+        trigger: 'committed_terminal',
+        ...(projectionStartedAtMs === undefined
+          ? {}
+          : { projectionStartedAtMs }),
       });
     },
   });
@@ -4960,6 +5192,111 @@ async function releaseAbsurdAttemptRuntimeState(input: {
   }
 }
 
+/**
+ * An executor-token outage happens before any Play code runs under this
+ * attempt, so it must not spend an engine attempt: a few minutes of app
+ * restart would otherwise exhaust max_attempts. Defer the durable task while
+ * inside a wall-clock bound, then return so the caller terminalizes loudly.
+ *
+ * - Fresh attempt: retire it and requeue the run (like a capacity deferral);
+ *   the bound is the start of the unbroken outage recorded on attempt rows.
+ * - Detached-runner wake: the attempt stays `waiting` under its original
+ *   fence, because a live runner may still push its terminal. Nothing is
+ *   written; the bound is the runner's persisted hard ceiling.
+ */
+async function deferAbsurdAttemptForExecutorTokenOutage(input: {
+  deps: AbsurdRuntimeWorkerDeps;
+  options: Pick<PostgresSchedulerOptions, 'schema'>;
+  runId: string;
+  generation: number;
+  leaseToken: string;
+  detachedRecovery: boolean;
+  detachedSuspension: unknown;
+  error: AttemptExecutorTokenUnavailableError;
+}): Promise<void> {
+  const nowMs = Date.now();
+  if (input.detachedRecovery) {
+    const deadlineMs = detachedRunnerCeilingDeadlineMsFromSuspension(
+      input.detachedSuspension,
+    );
+    if (deadlineMs !== null && nowMs < deadlineMs) {
+      const delaySeconds = Math.max(
+        1,
+        Math.min(
+          ABSURD_EXECUTOR_TOKEN_OUTAGE_DEFER_SECONDS,
+          Math.ceil((deadlineMs - nowMs) / 1_000),
+        ),
+      );
+      console.warn('[absurd-runtime.worker] executor_token_outage_deferred', {
+        runId: input.runId,
+        generation: input.generation,
+        detachedRecovery: true,
+        status: input.error.status,
+        deadlineMs,
+        delaySeconds,
+      });
+      throw new DeferTask(input.error.message, delaySeconds);
+    }
+    console.error(
+      '[absurd-runtime.worker] executor_token_outage_bound_exhausted',
+      {
+        runId: input.runId,
+        generation: input.generation,
+        detachedRecovery: true,
+        status: input.error.status,
+        deadlineMs,
+      },
+    );
+    return;
+  }
+  // The retired attempt may already own a recorded sandbox; persist its
+  // cleanup obligation before the attempt leaves the running fence.
+  await persistAttemptSandboxCleanupBeforeBestEffortGc({
+    deps: input.deps,
+    runId: input.runId,
+    attempt: input.generation,
+    options: input.options,
+  });
+  const outcome = await input.deps.runWithClient((client) =>
+    deferPostgresSchedulerAttemptForExecutorTokenOutage(
+      client,
+      {
+        runId: input.runId,
+        attempt: input.generation,
+        leaseToken: input.leaseToken,
+        reason: input.error.message,
+        maxOutageMs: ABSURD_EXECUTOR_TOKEN_OUTAGE_MAX_MS,
+      },
+      input.options,
+    ),
+  );
+  if (outcome.deferred) {
+    console.warn('[absurd-runtime.worker] executor_token_outage_deferred', {
+      runId: input.runId,
+      generation: input.generation,
+      detachedRecovery: false,
+      status: input.error.status,
+      outageStartedAtMs: outcome.outageStartedAtMs,
+      delaySeconds: ABSURD_EXECUTOR_TOKEN_OUTAGE_DEFER_SECONDS,
+    });
+    throw new DeferTask(
+      input.error.message,
+      ABSURD_EXECUTOR_TOKEN_OUTAGE_DEFER_SECONDS,
+    );
+  }
+  console.error(
+    '[absurd-runtime.worker] executor_token_outage_bound_exhausted',
+    {
+      runId: input.runId,
+      generation: input.generation,
+      detachedRecovery: false,
+      status: input.error.status,
+      outageStartedAtMs: outcome.outageStartedAtMs,
+      maxOutageMs: ABSURD_EXECUTOR_TOKEN_OUTAGE_MAX_MS,
+    },
+  );
+}
+
 /**
  * Final-attempt terminal salvage. When the LAST engine attempt of a play-run
  * task dies with an unexpected error (Daytona/app-runtime/DB throw — NOT a park
@@ -4984,6 +5321,12 @@ async function salvageAbsurdFinalAttemptTerminal(input: {
   generation: number;
   leaseToken: string;
   error: unknown;
+  /** The attempt was claimed as a parked detached-runner wake and may still
+   * be `waiting` under its original fence. */
+  detachedRecovery?: boolean;
+  /** A wall-clock bound, not the engine attempt count, requires the terminal
+   * now (executor-token outage past its bound). */
+  force?: boolean;
 }): Promise<void> {
   const attempt = input.ctx.attempt;
   const maxAttempts = input.ctx.maxAttempts;
@@ -5001,12 +5344,15 @@ async function salvageAbsurdFinalAttemptTerminal(input: {
       { cause: input.error },
     );
   }
-  if (attempt < maxAttempts) return;
+  if (!input.force && attempt < maxAttempts) return;
   const message =
     input.error instanceof Error ? input.error.message : String(input.error);
   const failedResult: PlayRunnerResult = {
     status: 'failed',
-    error: `absurd play-run task failed on its final engine attempt (${attempt}/${maxAttempts}): ${message}`,
+    error:
+      input.force && attempt < maxAttempts
+        ? `absurd play-run task exhausted its executor-token outage bound on engine attempt ${attempt}/${maxAttempts}: ${message}`
+        : `absurd play-run task failed on its final engine attempt (${attempt}/${maxAttempts}): ${message}`,
     logs: [],
     stats: {},
     steps: [],
@@ -5028,14 +5374,33 @@ async function salvageAbsurdFinalAttemptTerminal(input: {
       salvageAttempt += 1
     ) {
       try {
-        result = await input.deps.runWithClient((client) =>
-          executePostgresSchedulerClaim(
+        result = await input.deps.runWithClient(async (client) => {
+          // A detached-runner wake keeps its attempt `waiting` until setup
+          // succeeds. Take that same fence to `running` (the overdue-ceiling
+          // claim's transition) so the failure terminal can be written. When
+          // the attempt is no longer waiting — it already resumed, or a newer
+          // fence or pushed terminal owns it — the ordinary fenced path below
+          // decides.
+          if (input.detachedRecovery) {
+            await claimPostgresSchedulerWaitingDetachedAttemptForTerminal(
+              client,
+              {
+                runId: input.runId,
+                attempt: input.generation,
+                leaseToken: input.leaseToken,
+              },
+              input.options,
+            );
+          }
+          return await executePostgresSchedulerClaim(
             client,
             claim,
             async () => failedResult,
-            input.options,
-          ),
-        );
+            input.detachedRecovery
+              ? { ...input.options, preserveRunnerTerminal: true }
+              : input.options,
+          );
+        });
         break;
       } catch (salvageError) {
         try {
@@ -5720,6 +6085,57 @@ export async function recoverAbsurdRuntimeOverdueDetachedRunnerCeilings(input: {
   return { recovered, skipped };
 }
 
+export const ABSURD_OVERDUE_DETACHED_RUNNER_RECOVERY_INTERVAL_MS = 15_000;
+
+/**
+ * In-process hard-ceiling recovery loop for a topology with no release
+ * scaler (local dev). Production keeps this obligation on the fixed scaler.
+ * A pass that settled anything runs again at once: each claimed run leaves the
+ * `waiting` candidate set in its own statement, so the backlog drains by
+ * construction; an empty pass sleeps one interval.
+ */
+export async function runAbsurdOverdueDetachedRunnerCeilingController(input: {
+  runOnce: () => Promise<{ recovered: number; skipped: number }>;
+  shouldStop: () => boolean;
+  intervalMs?: number;
+  sleep?: (ms: number) => Promise<void>;
+}): Promise<void> {
+  const intervalMs = Math.max(
+    1,
+    Math.floor(
+      input.intervalMs ?? ABSURD_OVERDUE_DETACHED_RUNNER_RECOVERY_INTERVAL_MS,
+    ),
+  );
+  const wait = input.sleep ?? ((ms: number) => sleep(ms));
+  const reportFailure = createRateLimitedRuntimeLogger({
+    log: (...args) => console.error(...args),
+  });
+  while (!input.shouldStop()) {
+    let settled = 0;
+    try {
+      const outcome = await input.runOnce();
+      settled = outcome.recovered + outcome.skipped;
+      if (settled > 0) {
+        console.info(
+          '[absurd-runtime.worker] overdue_detached_runner_recovery',
+          outcome,
+        );
+      }
+    } catch (error) {
+      reportFailure(
+        '[absurd-runtime.worker] overdue_detached_runner_recovery_failed',
+        { error: error instanceof Error ? error.message : String(error) },
+      );
+    }
+    if (settled > 0) continue;
+    // Sleep in short slices so host shutdown is not held for a full interval.
+    const deadline = Date.now() + intervalMs;
+    while (!input.shouldStop() && Date.now() < deadline) {
+      await wait(Math.min(500, Math.max(1, deadline - Date.now())));
+    }
+  }
+}
+
 /**
  * Fixed-controller sweep across every persisted release, including lanes no
  * longer present in the live release registry. The run's immutable launch
@@ -7427,6 +7843,12 @@ export type RuntimeWorkerHostConfig = {
   /** Invoked once the lane has drained (drain mode only). The entrypoint wires
    * this to a clean `process.exit(0)`. */
   onDrained?: () => void;
+  /**
+   * Run the detached-runner hard-ceiling recovery in this process. Only for a
+   * topology without the fixed release scaler (the local dev supervisor);
+   * production leaves it to the scaler. Default off.
+   */
+  recoverOverdueDetachedRunnerCeilings?: boolean;
 };
 
 export type RuntimeWorkerHost = {
@@ -7675,6 +8097,19 @@ export async function startRuntimeWorkerHost(
     readSandboxCrashDiagnostic: workerDeps.readSandboxCrashDiagnostic,
     shouldStop: () => cleanupStopped,
   });
+  let overdueCeilingStopped = false;
+  const overdueCeilingController = options.recoverOverdueDetachedRunnerCeilings
+    ? runAbsurdOverdueDetachedRunnerCeilingController({
+        runOnce: () =>
+          withRuntimeSchedulerClient(poolOptions, (client) =>
+            recoverAbsurdRuntimeOverdueDetachedRunnerCeilingsAcrossAllLanes({
+              client,
+              schedulerOptions,
+            }),
+          ),
+        shouldStop: () => overdueCeilingStopped,
+      })
+    : null;
   // Drain self-exit (old-lane retirement). Off unless the deploy system opts in.
   let drainStopped = false;
   let drainWatch: Promise<'drained' | 'stopped'> | null = null;
@@ -7828,6 +8263,8 @@ export async function startRuntimeWorkerHost(
     releaseId,
     drainExit,
     laneAdoption: !drainExit,
+    overdueDetachedRunnerRecovery:
+      options.recoverOverdueDetachedRunnerCeilings === true,
     workerName: workerBase,
   });
 
@@ -7841,8 +8278,10 @@ export async function startRuntimeWorkerHost(
       billingStopped = true;
       cleanupStopped = true;
       adoptionStopped = true;
+      overdueCeilingStopped = true;
     },
     controllers: [
+      overdueCeilingController,
       drainWatch,
       adoptionSupervisor,
       billingController,