deepline 0.3.128 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -40,7 +40,6 @@ import {
   fallbackRuntimeTrafficPolicyForUnavailableControlPlane,
   getRuntimeTrafficPolicyViaAppRuntime,
   isLegacyRuntimeTrafficPolicyControlPlaneResponse,
-  recordFirstClaimLatencyViaAppRuntime,
   upsertComputeBillingSessionViaAppRuntime,
   type WorkerRuntimeApiContext,
 } from '../app-runtime-api';
@@ -49,6 +48,7 @@ import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
 import type { RuntimeTrafficPolicy } from '../runtime-traffic-policy';
 import { runtimeLimitExceededCause } from '../run-failure';
 import {
+  recoveryForDaytonaSandboxAcquisitionUnavailable,
   recoveryForRuntimeInitializationTransportFailure,
   recoveryForRuntimeTrafficPolicy,
 } from '../play-run-recovery-policy';
@@ -60,7 +60,10 @@ import type {
   PlayRunnerExecutionConfig,
   PlayRunnerResult,
 } from '../protocol';
-import { RuntimeResourceFenceLostError } from '../runner-backends/types';
+import {
+  RuntimeResourceFenceLostError,
+  type PlayRunnerRuntimeResource,
+} from '../runner-backends/types';
 import type { PlayBundleArtifact } from '../../plays/artifact-types';
 import {
   readRuntimeSheetRowOutcomeCounts,
@@ -73,6 +76,7 @@ import { installPgRuntimePoolDriver } from '../runtime-pg-driver-pg';
 import {
   isManagedSandboxRuntimeBackend,
   normalizePlayRuntimeBackend,
+  PLAY_RUNTIME_BACKENDS,
   type PlayRuntimeBackendId,
 } from '../backend';
 import {
@@ -84,7 +88,6 @@ import { retryableRuntimeInitializationFailureReason } from '../runner-backends/
 import {
   isRuntimeSandboxCanaryProduction,
   resolveLaunchRuntimeSandboxPlacementPolicy,
-  runtimeSandboxPlacementPoliciesForBackend,
   selectRuntimeSandboxCanary,
 } from '../runtime-sandbox-placement-policy';
 import { shouldUseRuntimeSchedulerDirectNotify } from '../runtime-scheduler-topology';
@@ -149,6 +152,7 @@ import {
   recordPostgresSchedulerLateManagedSandboxOperationForCleanup,
   recordPostgresSchedulerLateRuntimeResourceForCleanup,
   recordPostgresSchedulerManagedSandboxOperationLive,
+  readPostgresSchedulerManagedSandboxOperationAlreadyLive,
   recordPostgresSchedulerReconciledModalResourcesForCleanup,
   recordPostgresSchedulerRuntimeLifecycleEvent,
   recordPostgresSchedulerSandboxCrashDiagnostic,
@@ -194,9 +198,15 @@ import {
   type PlayRunAttemptWorkerTerminalProjectionPort,
 } from '../play-runs/attempts';
 import { transitionPreCodePlayRunRecovery } from './play-run-recovery';
-import { acquirePostgresRateState } from './postgres-rate-state';
+import {
+  acquirePostgresRateState,
+  penalizePostgresRateState,
+} from './postgres-rate-state';
 import { verifyRuntimeReceiptGatewayHealth } from './runtime-receipt-gateway-health';
-import { runtimeSandboxProviderCapabilities } from './absurd-worker-required-env';
+import {
+  runtimeSandboxPlacementPoliciesForWorkerEnv,
+  runtimeSandboxProviderCapabilities,
+} from './absurd-worker-required-env';
 import {
   acquirePgNotifyHub,
   isRuntimeSchedulerCapacityError,
@@ -261,6 +271,7 @@ import {
   runtimeSandboxCapacityAdmissionMode,
   RuntimeSandboxCapacityLimitError,
 } from './runtime-sandbox-capacity';
+import { runtimeSandboxProviderForResourceKind } from './runtime-sandbox-operation';
 import {
   createPostgresSandboxCleanupStore,
   createSandboxCleanupJob,
@@ -718,6 +729,16 @@ export type AbsurdRuntimeWorkerDeps = {
   readDetachedRunnerRuntimeCompletion?: (
     suspension: DetachedRunnerSuspension,
   ) => Promise<number | null>;
+  /**
+   * A bounded provider inspection of the captured terminal written by a
+   * detached runner that lost its gateway. This never executes customer code:
+   * it can only recover the result the sandbox already emitted before exit.
+   */
+  inspectDetachedRunner?: (suspension: DetachedRunnerSuspension) => Promise<{
+    exitCode: number | null;
+    result: PlayRunnerResult | null;
+    diagnosis: { stage: string; detail: string | null };
+  } | null>;
   readSandboxCrashDiagnostic?: (
     request: PostgresSchedulerSandboxCrashDiagnosticRequest,
     routingDomain: string | null,
@@ -789,6 +810,11 @@ function buildDefaultDeps(absurd: Absurd): AbsurdRuntimeWorkerDeps {
             suspension.runtimeCompletedPath,
           )
         : Promise.resolve(null),
+    inspectDetachedRunner: (suspension) =>
+      defaultRuntimeSandboxReconciliation.inspectDetachedRunner(
+        runtimeSandboxRefFromSuspension(suspension),
+        suspension,
+      ),
     readSandboxCrashDiagnostic: (request, routingDomain) =>
       readDetachedDaytonaCrashDiagnostic({
         sandboxId: request.sandboxId,
@@ -1783,6 +1809,28 @@ function sandboxCapacityExceededCause(): string {
   return `${SANDBOX_CAPACITY_EXCEEDED_CODE}: This organization's execution concurrency limit is currently full.`;
 }
 
+/**
+ * The existing capacity lease and operation tables own Daytona and Modal only.
+ * Keep this narrowing at their adapter seam: the shared Sandbox Runs contract
+ * remains provider-neutral, while a future provider must add its durable
+ * capacity/cleanup owner before it can receive one of these leases.
+ */
+type ManagedSandboxCapacityResource = PlayRunnerRuntimeResource & {
+  kind: 'daytona_sandbox' | 'modal_sandbox';
+  sandboxCapacityLeaseId: string;
+};
+
+function isManagedSandboxCapacityResource(
+  resource: PlayRunnerRuntimeResource,
+): resource is ManagedSandboxCapacityResource {
+  return (
+    (resource.kind === 'daytona_sandbox' ||
+      resource.kind === 'modal_sandbox') &&
+    typeof resource.sandboxCapacityLeaseId === 'string' &&
+    resource.sandboxCapacityLeaseId.trim().length > 0
+  );
+}
+
 async function terminalizeSandboxCapacityRejection(input: {
   deps: AbsurdRuntimeWorkerDeps;
   options: Pick<PostgresSchedulerOptions, 'schema'>;
@@ -1969,7 +2017,7 @@ async function prepareAbsurdAttempt(input: {
         launch.runtimeBackend || configuredBackend,
       );
       const configuredPlacementPolicies =
-        runtimeSandboxPlacementPoliciesForBackend(configuredBackend);
+        runtimeSandboxPlacementPoliciesForWorkerEnv(configuredBackend);
       const launchPlacementPolicy = resolveLaunchRuntimeSandboxPlacementPolicy({
         runtimeBackend: launchBackend,
         runtimeSandboxPlacementPolicyId: launch.runtimeSandboxPlacementPolicyId,
@@ -2210,27 +2258,13 @@ async function executeAbsurdAttemptAdapter(
     });
   }
 
-  if (firstClaimLatencyMs !== undefined) {
-    void Promise.resolve()
-      .then(() =>
-        recordFirstClaimLatencyViaAppRuntime(
-          deps.runtimeApiFromLaunch(launch),
-          {
-            runId,
-            firstClaimLatencyMs,
-          },
-        ),
-      )
-      .catch((error) => {
-        console.warn(
-          '[absurd-runtime.worker.first-claim-latency-write-failed]',
-          {
-            runId,
-            error: error instanceof Error ? error.message : String(error),
-          },
-        );
-      });
-  }
+  // The Postgres `run.started` outbox projection already carries this
+  // scheduler-owned measurement into the initial Convex admission. Do not
+  // issue a second fire-and-forget write against the hot `playRuns` document:
+  // it races the worker's first ledger append and turns an observability
+  // metric into an OCC retry on the runtime critical path. Keep the public
+  // app-runtime action registered for mixed-version workers, but let the
+  // durable start projection be the single writer here.
 
   let executionLaunch = launch;
   try {
@@ -2991,6 +3025,7 @@ async function executeAssignedAbsurdAttempt(input: {
                 options,
               ),
             ),
+          inspectDetachedRunner: deps.inspectDetachedRunner,
           deleteRuntimeSandbox: deps.deleteRuntimeSandbox,
           deleteRuntimeSandboxRef: deps.deleteRuntimeSandboxRef,
           sandboxGcTimeoutMs: deps.sandboxGcTimeoutMs,
@@ -3054,6 +3089,10 @@ async function executeAssignedAbsurdAttempt(input: {
     let terminalResultFromDetachedRunner = false;
     let terminalLogProducerAttempt: number | undefined;
     let terminalDetachedSuspension: DetachedRunnerSuspension | undefined;
+    let daytonaStartRateConfig: {
+      ratePerMinute: number;
+      maximumRatePerMinute: number;
+    } | null = null;
     try {
       if (pendingDetachedRunner) {
         // Push execution (B2-final): this leg woke a `detached_runner` park.
@@ -3078,6 +3117,7 @@ async function executeAssignedAbsurdAttempt(input: {
                 options,
               ),
             ),
+          inspectDetachedRunner: deps.inspectDetachedRunner,
           readDetachedRunnerRuntimeCompletion:
             deps.readDetachedRunnerRuntimeCompletion,
           deleteRuntimeSandbox: deps.deleteRuntimeSandbox,
@@ -3193,8 +3233,29 @@ async function executeAssignedAbsurdAttempt(input: {
                 });
                 trafficPolicy = fallback;
               }
+              const defaultDaytonaStartRatePerMinute =
+                launch.runtimeBackend === PLAY_RUNTIME_BACKENDS.daytona
+                  ? RUNTIME_CAPACITY_POLICY.sandboxAcquisition
+                      .daytonaInitialStartsPerMinute
+                  : null;
+              const effectiveDaytonaStartRatePerMinute =
+                trafficPolicy.sandboxStartsPerMinute ??
+                defaultDaytonaStartRatePerMinute;
+              daytonaStartRateConfig =
+                effectiveDaytonaStartRatePerMinute === null
+                  ? null
+                  : {
+                      ratePerMinute: effectiveDaytonaStartRatePerMinute,
+                      maximumRatePerMinute:
+                        trafficPolicy.sandboxStartsPerMinute === null
+                          ? RUNTIME_CAPACITY_POLICY.sandboxAcquisition
+                              .daytonaMaximumStartsPerMinute
+                          : effectiveDaytonaStartRatePerMinute,
+                    };
               await enforceRuntimeTrafficPolicyBeforeSandboxStart({
                 policy: trafficPolicy,
+                defaultSandboxStartsPerMinute:
+                  defaultDaytonaStartRatePerMinute,
                 nowMs: Date.now(),
                 acquireStartPermit: async (sandboxStartsPerMinute) =>
                   await deps.runWithClient((client) =>
@@ -3210,8 +3271,16 @@ async function executeAssignedAbsurdAttempt(input: {
                           // start-rate ceiling rather than a minute-sized burst.
                           windowMs: Math.ceil(60_000 / sandboxStartsPerMinute),
                           maxConcurrency: null,
+                          adaptiveMaxRps:
+                            (daytonaStartRateConfig?.maximumRatePerMinute ??
+                              sandboxStartsPerMinute) /
+                            60,
                         },
                       ],
+                      // The permit itself is the successful admission
+                      // observation that lets the fleet-wide AIMD controller
+                      // ramp. Provider pressure is fed back in the catch path.
+                      observedSuccesses: 1,
                     }),
                   ),
               });
@@ -3263,6 +3332,11 @@ async function executeAssignedAbsurdAttempt(input: {
                   return { kind: 'allowed' };
                 },
                 async reserveCapacity({ provider, providerAttempt }) {
+                  if (provider === 'agentcore_runtime') {
+                    throw new Error(
+                      'AgentCore cannot use the Daytona/Modal capacity ledger before its provider-specific durable capacity owner is installed.',
+                    );
+                  }
                   const mode = runtimeSandboxCapacityAdmissionMode();
                   const reserved = await deps.runWithClient((client) =>
                     reserveRuntimeSandboxOperation(
@@ -3345,22 +3419,38 @@ async function executeAssignedAbsurdAttempt(input: {
                       resource,
                     );
                   const recorded = await deps.runWithClient(async (client) => {
-                    if (
-                      runnerResource.sandboxCapacityLeaseId &&
-                      (runnerResource.kind === 'modal_sandbox' ||
-                        runnerResource.kind === 'daytona_sandbox')
-                    ) {
-                      return await recordPostgresSchedulerManagedSandboxOperationLive(
+                    if (isManagedSandboxCapacityResource(runnerResource)) {
+                      const promoted =
+                        await recordPostgresSchedulerManagedSandboxOperationLive(
+                          client,
+                          {
+                            runId,
+                            attempt: generation,
+                            leaseToken,
+                            resource: runnerResource,
+                          },
+                          options,
+                        );
+                      if (promoted) return true;
+                      // The promotion is a one-shot reserved->live compare-and-set,
+                      // so it also rejects this attempt re-reporting a sandbox it
+                      // already promoted -- which a provider retry inside one
+                      // attempt does whenever it retires an earlier sandbox. Prove
+                      // from durable state which case this is instead of assuming
+                      // either: only an intact fence over an already-live operation
+                      // for this exact sandbox is idempotent success. Everything
+                      // else stays a loud fence loss.
+                      return await readPostgresSchedulerManagedSandboxOperationAlreadyLive(
                         client,
                         {
                           runId,
                           attempt: generation,
                           leaseToken,
-                          resource: {
-                            ...runnerResource,
-                            sandboxCapacityLeaseId:
-                              runnerResource.sandboxCapacityLeaseId,
-                          },
+                          operationKey: runnerResource.sandboxCapacityLeaseId,
+                          provider: runtimeSandboxProviderForResourceKind(
+                            runnerResource.kind,
+                          ),
+                          sandboxId: runnerResource.sandboxId,
                         },
                         options,
                       );
@@ -3384,22 +3474,8 @@ async function executeAssignedAbsurdAttempt(input: {
                         ) || runtimeSandboxCapacityAdmissionMode() === 'observe'
                       );
                     }
-                    return await bindRuntimeSandboxCapacityLease(
-                      client,
-                      {
-                        leaseId: runnerResource.sandboxCapacityLeaseId,
-                        provider:
-                          runnerResource.kind === 'modal_sandbox'
-                            ? 'modal'
-                            : 'daytona',
-                        sandboxId: runnerResource.sandboxId,
-                        routingDomain:
-                          runnerResource.kind === 'modal_sandbox'
-                            ? runnerResource.modalAppId?.trim() || null
-                            : runnerResource.daytonaOrganizationId?.trim() ||
-                              null,
-                      },
-                      options,
+                    throw new Error(
+                      `Runtime resource ${runnerResource.kind} carried a capacity lease without a matching managed provider owner.`,
                     );
                   });
                   return recorded
@@ -3417,18 +3493,14 @@ async function executeAssignedAbsurdAttempt(input: {
                     );
                   }
                   const recorded = await deps.runWithClient(async (client) => {
-                    if (runnerResource.sandboxCapacityLeaseId) {
+                    if (isManagedSandboxCapacityResource(runnerResource)) {
                       return await recordPostgresSchedulerLateManagedSandboxOperationForCleanup(
                         client,
                         {
                           runId,
                           attempt: generation,
                           leaseToken,
-                          resource: {
-                            ...runnerResource,
-                            sandboxCapacityLeaseId:
-                              runnerResource.sandboxCapacityLeaseId,
-                          },
+                          resource: runnerResource,
                         },
                         options,
                       );
@@ -3446,19 +3518,8 @@ async function executeAssignedAbsurdAttempt(input: {
                       );
                     if (!resourceRecorded) return false;
                     if (!runnerResource.sandboxCapacityLeaseId) return true;
-                    return await bindRuntimeSandboxCapacityLease(
-                      client,
-                      {
-                        leaseId: runnerResource.sandboxCapacityLeaseId,
-                        provider: 'modal',
-                        sandboxId: runnerResource.sandboxId,
-                        routingDomain:
-                          runnerResource.kind === 'modal_sandbox'
-                            ? runnerResource.modalAppId?.trim() || null
-                            : runnerResource.daytonaOrganizationId?.trim() ||
-                              null,
-                      },
-                      options,
+                    throw new Error(
+                      'Late Modal resource carried a capacity lease without a matching managed provider owner.',
                     );
                   });
                   return recorded
@@ -3569,9 +3630,35 @@ async function executeAssignedAbsurdAttempt(input: {
       }
     } catch (error) {
       await cleanupAfterExecutionError(error, runnerConfig);
+      const recoveryError =
+        recoveryForDaytonaSandboxAcquisitionUnavailable(error) ?? error;
+      if (
+        recoveryError !== error &&
+        daytonaStartRateConfig !== null
+      ) {
+        await deps.runWithClient((client) =>
+          penalizePostgresRateState(client, {
+            bucketId: 'runtime:traffic:sandbox-start',
+            schedulerSchema: options.schema ?? null,
+            cooldownMs: 15_000,
+            rules: [
+              {
+                ruleId: 'runtime:traffic:sandbox-start',
+                requestsPerWindow: 1,
+                windowMs: Math.ceil(
+                  60_000 / daytonaStartRateConfig!.ratePerMinute,
+                ),
+                adaptiveMaxRps:
+                  daytonaStartRateConfig!.maximumRatePerMinute / 60,
+                maxConcurrency: null,
+              },
+            ],
+          }),
+        );
+      }
       const recovery = await deps.runWithClient((client) =>
         transitionPreCodePlayRunRecovery({
-          error,
+          error: recoveryError,
           runId,
           attempt: generation,
           leaseToken,
@@ -3664,8 +3751,48 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
   fromDetachedRunner: boolean;
   terminalLogProducerAttempt?: number;
 }): PlayRunAttemptWorkerTerminalProjectionPort {
+  /**
+   * Every terminal releases what the attempt claimed. Best-effort by design:
+   * terminal truth is already durable, and a failed release must never turn a
+   * committed terminal into a worker error.
+   */
+  const releaseAttemptReceipts = async (reason: string): Promise<void> => {
+    await input.deps
+      .runWithClient((client) =>
+        releasePostgresSchedulerReceiptsForAttempt(
+          client,
+          { runId: input.runId, runAttempt: input.generation },
+          { ...input.options, orgId: input.launch.orgId },
+        ),
+      )
+      .then((released) => {
+        if (released.released > 0) {
+          console.info('[absurd-runtime.worker] released terminal receipts', {
+            runId: input.runId,
+            generation: input.generation,
+            reason,
+            releasedReceipts: released.released,
+          });
+        }
+      })
+      .catch((error) => {
+        console.warn(
+          '[absurd-runtime.worker] terminal receipt release failed',
+          {
+            runId: input.runId,
+            generation: input.generation,
+            reason,
+            error: error instanceof Error ? error.message : String(error),
+          },
+        );
+      });
+  };
   return createPlayRunAttemptWorkerTerminalProjection({
     async observeFenceLost() {
+      // A lost fence means this attempt is dead, which a cancelled run is the
+      // loudest case of: its receipts must not stay leased to a generation
+      // that will never complete them.
+      await releaseAttemptReceipts('fence_lost');
       await settleObservedTerminal({
         deps: input.deps,
         launch: input.launch,
@@ -3687,36 +3814,7 @@ function createAbsurdWorkerTerminalProjectionAdapter(input: {
       });
     },
     async releaseFailedAttempt() {
-      await input.deps
-        .runWithClient((client) =>
-          releasePostgresSchedulerReceiptsForAttempt(
-            client,
-            { runId: input.runId, runAttempt: input.generation },
-            { ...input.options, orgId: input.launch.orgId },
-          ),
-        )
-        .then((released) => {
-          if (released.released > 0) {
-            console.info(
-              '[absurd-runtime.worker] released failed terminal receipts',
-              {
-                runId: input.runId,
-                generation: input.generation,
-                releasedReceipts: released.released,
-              },
-            );
-          }
-        })
-        .catch((error) => {
-          console.warn(
-            '[absurd-runtime.worker] failed terminal receipt release failed',
-            {
-              runId: input.runId,
-              generation: input.generation,
-              error: error instanceof Error ? error.message : String(error),
-            },
-          );
-        });
+      await releaseAttemptReceipts('failed_terminal');
 
       // A normal runner-reported failure does not enter the thrown-error
       // cleanup path above. Its terminal is already durable, so leaving these
@@ -3870,12 +3968,13 @@ function throwRetryableRuntimeInitializationFailure(
 
 /**
  * Consume one global start permit before a first sandbox create. The limiter's
- * durable bucket is intentionally advisory only when no short-lived incident
- * policy exists; an active policy turns a long wait into the same finite
+ * durable bucket is used for the compiled Daytona AIMD start ceiling and for
+ * any short-lived operator policy. A long wait becomes the same finite
  * recovery ledger used for platform outages.
  */
 export async function enforceRuntimeTrafficPolicyBeforeSandboxStart(input: {
   policy: RuntimeTrafficPolicy;
+  defaultSandboxStartsPerMinute?: number | null;
   nowMs: number;
   acquireStartPermit: (sandboxStartsPerMinute: number) => Promise<{
     granted: number;
@@ -3888,20 +3987,41 @@ export async function enforceRuntimeTrafficPolicyBeforeSandboxStart(input: {
   if (input.policy.admissionMode === 'hold_new_sandbox_starts') {
     throw recoveryForRuntimeTrafficPolicy({ mode: 'hold' });
   }
-  const sandboxStartsPerMinute = input.policy.sandboxStartsPerMinute;
+  const sandboxStartsPerMinute =
+    input.policy.sandboxStartsPerMinute ??
+    input.defaultSandboxStartsPerMinute ??
+    null;
   if (sandboxStartsPerMinute === null) return;
-  const permit = await input.acquireStartPermit(sandboxStartsPerMinute);
-  const scheduledAtMs = permit.scheduledAtMs?.[0] ?? input.nowMs;
-  const schedulerNowMs = permit.serverNowMs ?? input.nowMs;
-  const delayMs = Math.max(0, scheduledAtMs - schedulerNowMs);
-  // A sub-second schedule retains smooth pacing without spending a worker
-  // attempt. Longer waits become a durable, bounded defer, allowing the
-  // scheduler to serve unrelated traffic during a deliberate shed.
-  if (permit.granted < 1 || delayMs > 1_000) {
-    throw recoveryForRuntimeTrafficPolicy({ mode: 'throttle' });
-  }
-  if (delayMs > 0) {
-    await (input.sleep ?? sleep)(delayMs);
+  let locallyWaitedMs = 0;
+  for (;;) {
+    const permit = await input.acquireStartPermit(sandboxStartsPerMinute);
+    const scheduledAtMs = permit.scheduledAtMs?.[0] ?? input.nowMs;
+    const schedulerNowMs = permit.serverNowMs ?? input.nowMs;
+    const delayMs =
+      permit.granted > 0
+        ? Math.max(0, scheduledAtMs - schedulerNowMs)
+        : Math.max(
+            0,
+            permit.waitMs,
+            scheduledAtMs > input.nowMs
+              ? scheduledAtMs - schedulerNowMs
+              : 0,
+          );
+    // A sub-second schedule retains smooth pacing without spending a worker
+    // attempt. Longer waits become a durable, bounded defer, allowing the
+    // scheduler to serve unrelated traffic during a deliberate shed.
+    if (delayMs > 1_000 || locallyWaitedMs + delayMs > 1_000) {
+      throw recoveryForRuntimeTrafficPolicy({ mode: 'throttle' });
+    }
+    if (permit.granted > 0) {
+      if (delayMs > 0) {
+        await (input.sleep ?? sleep)(delayMs);
+      }
+      return;
+    }
+    const waitMs = Math.max(1, delayMs);
+    await (input.sleep ?? sleep)(waitMs);
+    locallyWaitedMs += waitMs;
   }
 }
 
@@ -3946,6 +4066,7 @@ async function resolveDetachedRunnerOutcome(input: {
     updatedAt: string | null;
     status: string;
   } | null>;
+  inspectDetachedRunner?: AbsurdRuntimeWorkerDeps['inspectDetachedRunner'];
   readDetachedRunnerRuntimeCompletion?: AbsurdRuntimeWorkerDeps['readDetachedRunnerRuntimeCompletion'];
   deleteRuntimeSandbox: AbsurdRuntimeWorkerDeps['deleteRuntimeSandbox'];
   deleteRuntimeSandboxRef?: AbsurdRuntimeWorkerDeps['deleteRuntimeSandboxRef'];
@@ -4114,6 +4235,54 @@ async function resolveDetachedRunnerOutcome(input: {
       tableNamespace: null,
     });
   }
+  // The runner writes a structured terminal to its sandbox output before its
+  // bounded terminal push. A receipt-gateway outage can exhaust that push and
+  // expire liveness even though the sandbox cleanly stopped itself. Recover
+  // only that already-written result before synthesizing RUNNER_LOST: this
+  // does not execute customer code, and an unavailable inspection preserves
+  // the existing conservative failure below.
+  if (input.inspectDetachedRunner) {
+    try {
+      const inspection = await input.inspectDetachedRunner(suspension);
+      if (inspection?.result && inspection.result.status !== 'suspended') {
+        console.info(
+          '[absurd-runtime.worker] detached_runner_liveness_salvaged',
+          JSON.stringify({
+            runId: input.runId,
+            runnerAttempt: suspension.runnerAttempt,
+            sandboxId: suspension.sandboxId,
+            exitCode: inspection.exitCode,
+            diagnosis: inspection.diagnosis.stage,
+            resultStatus: inspection.result.status,
+          }),
+        );
+        return settleFirstWriteWinner(inspection.result);
+      }
+      if (inspection) {
+        console.warn(
+          '[absurd-runtime.worker] detached_runner_liveness_inspection_empty',
+          JSON.stringify({
+            runId: input.runId,
+            runnerAttempt: suspension.runnerAttempt,
+            sandboxId: suspension.sandboxId,
+            exitCode: inspection.exitCode,
+            diagnosis: inspection.diagnosis.stage,
+            resultStatus: inspection.result?.status ?? null,
+          }),
+        );
+      }
+    } catch (error) {
+      console.warn(
+        '[absurd-runtime.worker] detached_runner_liveness_inspection_failed',
+        JSON.stringify({
+          runId: input.runId,
+          runnerAttempt: suspension.runnerAttempt,
+          sandboxId: suspension.sandboxId,
+          error: error instanceof Error ? error.message : String(error),
+        }),
+      );
+    }
+  }
   return settleFirstWriteWinner({
     status: 'failed',
     error: `RUNTIME_RUNNER_LOST: Detached runner heartbeat expired after ${heartbeatAgeMs ?? 'unknown'}ms without a terminal result. Sandbox ${suspension.sandboxId}. The run was stopped and was not retried because provider side effects may already exist; completed row state remains persisted.`,

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts

@@ -30,7 +30,7 @@ import { assertKnownResolvedBoundary } from '../checkpoint-boundaries';
 import type { Absurd, Log, RetryStrategy } from '@deepline/absurd/sdk';
 import { isIsolatedRuntimeSchedulerSchema } from '../runtime-scheduler-topology';
 import {
-  ensureAbsurdSchema,
+  assertAbsurdSchemaReady,
   failPostgresSchedulerScheduledChildWaits,
   type PostgresSchedulerWorkerOnceResult,
 } from './postgres';
@@ -717,10 +717,12 @@ export function createAbsurdRuntimeClient(
 const absurdReadyPromises = new Map<string, Promise<void>>();
 
 /**
- * Ensure, once per process (keyed by pool url + schema): the vendored absurd
- * schema and the shared queue with its
- * policies. Idempotent and safe under concurrent submits — the memoized promise
- * is shared; a failure evicts the cache so the next caller retries loudly.
+ * Verify, once per process (keyed by pool url + schema), the deployment-owned
+ * Absurd schema contract and provision the shared queue with its policies.
+ * Schema DDL belongs exclusively to the migration gate: a worker or Vercel
+ * cold start may read this contract, but may never take the global bootstrap
+ * lock. The memoized promise is shared; a failure evicts the cache so the next
+ * caller retries loudly.
  */
 export function ensureAbsurdRuntimeReady(
   poolOptions: RuntimeSchedulerPoolOptions,
@@ -744,7 +746,7 @@ export function ensureAbsurdRuntimeReady(
   if (existing) return existing;
   const promise = (async () => {
     await withRuntimeSchedulerClient(poolOptions, (client) =>
-      ensureAbsurdSchema(client),
+      assertAbsurdSchemaReady(client),
     );
     // Hard-cutover cleanup: an old worker may have persisted a run waiting
     // on a scheduled ctx.runPlay child. This runtime intentionally has no such