deepline 0.3.128 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts

@@ -41,6 +41,8 @@ import {
   runtimeLimitExceededFailure,
 } from '../run-failure';
 import { createPlayContext } from '../context';
+import type { RuntimeApiResponseMeta } from '../app-runtime-api';
+import { onMemoryPressure } from '../memory-pressure-signal';
 import {
   shouldEmitProgressSidecarEvent,
   shouldEmitStdoutEvent,
@@ -58,6 +60,7 @@ import {
   createRunnerProgressPusher,
   type RunnerProgressPusher,
 } from './runtime-progress-push';
+import { collectStartupEgressProbeLines } from './egress-probe';
 import {
   pushRunnerTerminal,
   runnerTerminalResultForTransport,
@@ -92,6 +95,12 @@ import {
 } from '../dataset-lifecycle-log';
 import {
   assertCustomerPlayReturnWithinLimit,
+  RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES,
+  RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_ITEMS,
+  RUNTIME_RECEIPT_COMPLETION_MAX_IN_FLIGHT_BATCHES,
+  RUNTIME_RECEIPT_CONTROL_BUFFER_MAX_BYTES,
+  RUNTIME_RECEIPT_CONTROL_BUFFER_MAX_ITEMS,
+  RUNTIME_RECEIPT_CONTROL_MAX_IN_FLIGHT_BATCHES,
   RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES,
 } from '../output-size-limits';
 import {
@@ -100,7 +109,10 @@ import {
   readRuntimeSheetDatasetRowKeys,
   readRuntimeSheetDatasetRows,
   releaseRuntimeWorkReceipts,
+  RuntimeSheetAttemptSupersededError,
   startRuntimeSheetDataset,
+  supersededRuntimeMapRows,
+  type RuntimeMapRowsWriteResult,
 } from '../runtime-api';
 import { installPgRuntimePoolDriver } from '../runtime-pg-driver-pg';
 import { createRuntimeToolAuthScopeDigestResolver } from '../auth-scope-resolver';
@@ -141,6 +153,32 @@ function loadSafeOutboundFetch(): Promise<SafeOutboundFetchModule> {
   return safeOutboundFetchModule;
 }
 
+type ReceiptGatewayResponseObserverTarget = Pick<
+  ReturnType<typeof createPlayContext>,
+  'observeRuntimeGatewayResponse'
+>;
+
+/**
+ * The receipt adapter is created before the Play Context because the Context
+ * needs its durable handlers. Keep the response observer late-bound so those
+ * completion requests still contribute their gateway/transport split to the
+ * Context's terminal runtime summary.
+ */
+export function createReceiptGatewayResponseObserver(): {
+  attach: (target: ReceiptGatewayResponseObserverTarget) => void;
+  onResponseMeta: (meta: RuntimeApiResponseMeta) => void;
+} {
+  let target: ReceiptGatewayResponseObserverTarget | null = null;
+  return {
+    attach: (nextTarget) => {
+      target = nextTarget;
+    },
+    onResponseMeta: (meta) => {
+      target?.observeRuntimeGatewayResponse(meta);
+    },
+  };
+}
+
 async function safeNodeOutboundFetch(
   input: string | URL,
   init?: RequestInit,
@@ -250,6 +288,36 @@ const RUNTIME_RECEIPT_GATEWAY_REQUEST_TIMEOUT_MS =
   RUNTIME_CAPACITY_POLICY.receiptGateway.requestTimeoutMs;
 const RETRY_SAFE_EVENT_WAIT_ARM_TOOLS = new Set(['test_wait_for_event']);
 
+/**
+ * The Context already bounds receipt completion to sixteen flights. The
+ * adapter is the same receipt writer at the gateway boundary, so leaving its
+ * default single flight serializes those independent batches again after they
+ * have left the Context. Keep the two writers on the same bounded lanes and
+ * make the adapter's retained command envelope explicit.
+ */
+export function playRunnerReceiptStoreAdapterOptions() {
+  return {
+    maxBatchSize: RUNTIME_RECEIPT_GATEWAY_BATCH_SIZE,
+    maxFlushMs: RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_WAIT_MS,
+    maxInFlightBatches:
+      RUNTIME_RECEIPT_COMPLETION_MAX_IN_FLIGHT_BATCHES +
+      RUNTIME_RECEIPT_CONTROL_MAX_IN_FLIGHT_BATCHES,
+    maxBufferedItems:
+      RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_ITEMS +
+      RUNTIME_RECEIPT_CONTROL_BUFFER_MAX_ITEMS,
+    maxBufferedBytes:
+      RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES +
+      RUNTIME_RECEIPT_CONTROL_BUFFER_MAX_BYTES,
+    maxCompletionInFlightBatches:
+      RUNTIME_RECEIPT_COMPLETION_MAX_IN_FLIGHT_BATCHES,
+    maxControlInFlightBatches: RUNTIME_RECEIPT_CONTROL_MAX_IN_FLIGHT_BATCHES,
+    maxCompletionBufferedItems: RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_ITEMS,
+    maxCompletionBufferedBytes: RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES,
+    maxControlBufferedItems: RUNTIME_RECEIPT_CONTROL_BUFFER_MAX_ITEMS,
+    maxControlBufferedBytes: RUNTIME_RECEIPT_CONTROL_BUFFER_MAX_BYTES,
+  };
+}
+
 // The one receipt backend: one Play Durability Store Adapter bound to the
 // co-located receipt gateway. There is no host-injected or app-runtime path.
 type RuntimeStepReceiptHandlers = RuntimeReceiptStoreHandlers;
@@ -296,8 +364,9 @@ export function assertRuntimeSheetAttemptPersistedTerminalRows(input: {
   fencedKeys?: string[];
 }): void {
   // Kept as a temporary source-compatible seam for callers compiled against
-  // the lease model. A partial update now means a newer scheduled write won;
-  // it is an observable stale drop, not a run-fatal ownership error.
+  // the lease model. A partial update is classified from the terminal-write
+  // result (`assertRuntimeSheetTerminalWriteNotSuperseded`), never from this
+  // count.
   void input;
 }
 
@@ -320,6 +389,99 @@ export function deterministicRuntimeSheetAttemptId(input: {
   return `sheet-attempt:${digest}`;
 }
 
+/** The sheet's admission version for one table; a value, not a wire marker. */
+type SheetWriteVersionValue = number;
+
+export type RunnerRuntimeSheetAttempt = {
+  attemptId: string;
+  attemptOwnerRunId: string;
+  attemptExpiresAt: string;
+  attemptSeq: number;
+  writeVersion?: SheetWriteVersionValue;
+};
+
+/**
+ * The sheet attempt record to keep after a map start admitted one source page
+ * of a table. A runtime-sheet-backed map admits its source one page at a time
+ * while earlier pages' rows are still executing, and every terminal write is
+ * fenced by the write version the runner sends. If each page minted its own
+ * version, every completion flushed after the next page's admission would
+ * carry a version the earlier rows were never admitted under and be rejected
+ * as stale (the 2026-09-18 20,000-row incident: 4,350 rows executed, 45
+ * persisted). So the first admission's write version is requested again for
+ * every later page (see `runtimeSheetStartWriteVersion`) and must come back
+ * unchanged; anything else means the sheet no longer belongs to this runner.
+ */
+export function runtimeSheetAttemptForMapStart(input: {
+  tableNamespace: string;
+  runId: string;
+  existing: RunnerRuntimeSheetAttempt | undefined;
+  mapStartResult: {
+    attemptId: string;
+    attemptOwnerRunId: string;
+    attemptExpiresAt: string;
+    attemptSeq?: number | null;
+    writeVersion?: SheetWriteVersionValue | null;
+  };
+  runnerRunAttempt: number;
+}): RunnerRuntimeSheetAttempt {
+  const requested = input.existing?.writeVersion;
+  const returned = input.mapStartResult.writeVersion ?? undefined;
+  if (requested != null && returned !== requested) {
+    throw new Error(
+      `Runtime sheet start for table "${input.tableNamespace}" (run ${input.runId}) ` +
+        `was asked to keep write version ${requested} for a later source page but ` +
+        `returned ${returned ?? 'none'}; the runner cannot persist rows under two ` +
+        `write versions, so this map cannot continue.`,
+    );
+  }
+  return {
+    attemptId: input.mapStartResult.attemptId,
+    attemptOwnerRunId: input.mapStartResult.attemptOwnerRunId,
+    attemptExpiresAt: input.mapStartResult.attemptExpiresAt,
+    attemptSeq: input.mapStartResult.attemptSeq ?? input.runnerRunAttempt,
+    ...(returned != null ? { writeVersion: returned } : {}),
+  };
+}
+
+/** The write version a later source page of the same table must be admitted under. */
+export function runtimeSheetStartWriteVersion(
+  existing: RunnerRuntimeSheetAttempt | undefined,
+): { writeVersion: SheetWriteVersionValue } | Record<string, never> {
+  return existing?.writeVersion != null
+    ? { writeVersion: existing.writeVersion }
+    : {};
+}
+
+/**
+ * Terminal writes rejected as stale for any reason other than a verified
+ * same-run settlement or an unversioned already-terminal row mean the sheet
+ * rows belong to a different write version than this attempt's: a newer
+ * admission (rerun, resumed attempt, concurrent run) owns them. Throw so the
+ * map fails and the persistence latch trips, instead of logging a perf line
+ * and spending provider calls on rows that can never be persisted.
+ */
+export function assertRuntimeSheetTerminalWriteNotSuperseded(input: {
+  tableNamespace: string;
+  runId: string;
+  writeVersion: SheetWriteVersionValue | null;
+  submittedRows: number;
+  result: Pick<RuntimeMapRowsWriteResult, 'staleKeys' | 'staleRows'>;
+  recoveredStaleKeys: Iterable<string>;
+}): void {
+  const supersededRows = supersededRuntimeMapRows(input.result, {
+    exceptKeys: input.recoveredStaleKeys,
+  });
+  if (supersededRows.length === 0) return;
+  throw new RuntimeSheetAttemptSupersededError({
+    tableNamespace: input.tableNamespace,
+    runId: input.runId,
+    writeVersionValue: input.writeVersion,
+    submittedRows: input.submittedRows,
+    supersededRows,
+  });
+}
+
 async function vercelBypassHeaders(
   config: PlayRunnerExecutionConfig,
 ): Promise<Record<string, string>> {
@@ -882,6 +1044,13 @@ async function createStreamingCsvDataset(input: {
 export interface ExecutePlayRunnerOptions {
   emitEvent?: (event: PlayRunnerEvent) => void;
   signal?: AbortSignal;
+  /** Stops future row admission while already admitted rows can persist. */
+  rowAdmissionSignal?: AbortSignal;
+  /** Runner-local terminal state made available to the deadline drain. */
+  onRuntimeReady?: (runtime: {
+    log: (line: string) => void;
+    timeoutResult: (runtimeLimitSeconds: number) => PlayRunnerResult;
+  }) => void;
 }
 
 function emitRunnerEvent(
@@ -1646,6 +1815,18 @@ async function releaseRuntimeLeasesOnAppRunnerSettlement(input: {
   }
 }
 
+function terminalLogsWithPhaseSummary(ctx: {
+  getLogs(): string[];
+  getRuntimeResourceSnapshot(): unknown;
+  getLogDiagnostics(): unknown;
+}): string[] {
+  return [
+    ...ctx.getLogs(),
+    `[runtime.logs] ${JSON.stringify(ctx.getLogDiagnostics())}`,
+    `[runtime.phase_summary] ${JSON.stringify(ctx.getRuntimeResourceSnapshot())}`,
+  ];
+}
+
 async function run(
   config: PlayRunnerExecutionConfig,
   options?: ExecutePlayRunnerOptions,
@@ -1690,6 +1871,7 @@ async function run(
     vercelProtectionBypassToken: config.context.vercelProtectionBypassToken,
     runtimeTestFaultHeader: config.context.runtimeTestFaultHeader ?? null,
   };
+  const receiptGatewayResponseObserver = createReceiptGatewayResponseObserver();
   const receiptGatewayBaseUrl = config.context.receiptGatewayBaseUrl?.trim();
   // Receipt state has exactly two backends: handlers provided by the host
   // process (inline executor) or the co-located receipt gateway named by the
@@ -1701,6 +1883,7 @@ async function run(
         baseUrl: receiptGatewayBaseUrl,
         boundary: 'receipt_gateway' as const,
         requestTimeoutMs: RUNTIME_RECEIPT_GATEWAY_REQUEST_TIMEOUT_MS,
+        onResponseMeta: receiptGatewayResponseObserver.onResponseMeta,
       }
     : null;
   const canUseRuntimeStepReceipts = playRunnerRuntimeReceiptsEnabled(
@@ -1718,8 +1901,7 @@ async function run(
             config.context.runId ??
             config.context.workflowId ??
             'anonymous-run',
-          maxBatchSize: RUNTIME_RECEIPT_GATEWAY_BATCH_SIZE,
-          maxFlushMs: RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_WAIT_MS,
+          ...playRunnerReceiptStoreAdapterOptions(),
           onRetryTelemetry: (line) => emitSystemLog(line, options),
         })
       : null;
@@ -1836,6 +2018,7 @@ async function run(
   });
   const ctx = createPlayContext({
     artifactRuntimeContract,
+    rowAdmissionSignal: options?.rowAdmissionSignal,
     executorToken: config.context.executorToken,
     baseUrl: config.context.baseUrl,
     // Runtime control stays on baseUrl (the receipt gateway); provider calls
@@ -2164,6 +2347,15 @@ async function run(
           `Missing sheet contract for play "${playName}" and table namespace "${tableNamespace}".`,
         );
       }
+      const attemptKey = runtimeSheetAttemptKey({
+        playName,
+        runId,
+        tableNamespace,
+      });
+      // A later source page of a table this runner already admitted keeps the
+      // first page's write version: rows of earlier pages are still executing
+      // and their terminal writes are fenced by it.
+      const existingAttempt = runtimeSheetAttempts.get(attemptKey);
       const mapStartResult = await retryRuntimeSheetPersistence({
         description: `runtime map start table=${tableNamespace}`,
         log: (line) => emitWorkerLog(line, options),
@@ -2205,6 +2397,7 @@ async function run(
               }),
               attemptOwnerRunId: runId,
               attemptSeq: runnerRunAttempt,
+              ...runtimeSheetStartWriteVersion(existingAttempt),
               ...(playRunnerContextRuntimeSheetForce(config.context)
                 ? { force: true }
                 : {}),
@@ -2274,14 +2467,20 @@ async function run(
         mapStartResult.attemptExpiresAt
       ) {
         runtimeSheetAttempts.set(
-          runtimeSheetAttemptKey({ playName, runId, tableNamespace }),
-          {
-            attemptId: mapStartResult.attemptId,
-            attemptOwnerRunId: mapStartResult.attemptOwnerRunId,
-            attemptExpiresAt: mapStartResult.attemptExpiresAt,
-            attemptSeq: mapStartResult.attemptSeq ?? runnerRunAttempt,
-            writeVersion: mapStartResult.writeVersion,
-          },
+          attemptKey,
+          runtimeSheetAttemptForMapStart({
+            tableNamespace,
+            runId,
+            existing: existingAttempt,
+            mapStartResult: {
+              attemptId: mapStartResult.attemptId,
+              attemptOwnerRunId: mapStartResult.attemptOwnerRunId,
+              attemptExpiresAt: mapStartResult.attemptExpiresAt,
+              attemptSeq: mapStartResult.attemptSeq,
+              writeVersion: mapStartResult.writeVersion,
+            },
+            runnerRunAttempt,
+          }),
         );
       }
       emitWorkerLog(
@@ -2487,6 +2686,22 @@ async function run(
           `[info] dropped ${staleKeys.size} stale runtime sheet write(s) for table=${input.tableNamespace}; ${recoveredStaleKeys.length} same-run terminal write(s) were accepted idempotently`,
           options,
         );
+        try {
+          assertRuntimeSheetTerminalWriteNotSuperseded({
+            tableNamespace: input.tableNamespace,
+            runId,
+            writeVersion: attempt?.writeVersion ?? null,
+            submittedRows: input.rows.length,
+            result,
+            recoveredStaleKeys,
+          });
+        } catch (error) {
+          emitWorkerLog(
+            `[error] ${error instanceof Error ? error.message : String(error)}`,
+            options,
+          );
+          throw error;
+        }
       }
       const failedRows = input.rows.filter(
         (row) =>
@@ -2511,7 +2726,9 @@ async function run(
         failedRows: 0,
       };
       // Only exact-version accepted rows belong to this run's lifecycle
-      // settlement. Stale rows belong to the newer writer and stay excluded.
+      // settlement. The stale keys that reach this point were all recovered as
+      // same-run terminal settlements, counted when they first committed, so
+      // they stay excluded here.
       settlement.persistedRows += committedTerminalRows;
       settlement.succeededRows += succeededRows;
       settlement.failedRows += failedRows;
@@ -2527,6 +2744,7 @@ async function run(
       };
     },
   });
+  receiptGatewayResponseObserver.attach(ctx);
 
   const csvSourceCache = new Map<string, ReplayableCsvSource>();
   const csvDatasetCache = new Map<
@@ -2666,6 +2884,53 @@ async function run(
     `[perf] context setup elapsed_ms=${Date.now() - contextSetupStartedAt}`,
     options,
   );
+  for (const line of await collectStartupEgressProbeLines({
+    // A local/direct executor can use baseUrl without a receipt gateway.
+    // Probe only the explicit data-plane hop.
+    receiptGatewayBaseUrl: config.context.receiptGatewayBaseUrl ?? null,
+    executionRelayBaseUrl: config.context.executionGatewayBaseUrl ?? null,
+    executorToken: config.context.executorToken ?? null,
+  })) {
+    ctx.logRunnerDiagnostic(line);
+  }
+  const logRecovery = () => {
+    const runId = config.context.runId ?? config.context.workflowId;
+    for (const saved of [...runtimeMapDatasetSettlements.values()].slice(
+      0,
+      16,
+    )) {
+      ctx.logRunnerDiagnostic(
+        formatDatasetAvailabilityLog({
+          runId: saved.runId,
+          dataset: saved,
+          persistedRows: saved.persistedRows,
+        }),
+      );
+    }
+    if (runId) {
+      ctx.logRunnerDiagnostic(
+        `Run stopped before returning. Inspect saved datasets: deepline runs get ${runId} --json. Export saved data before rerunning unfinished work.`,
+      );
+    }
+  };
+  options?.onRuntimeReady?.({
+    log: (line) => ctx.logRunnerDiagnostic(line),
+    timeoutResult: (runtimeLimitSeconds) => {
+      logRecovery();
+      return {
+        status: 'failed',
+        error: runtimeLimitExceededCause(runtimeLimitSeconds),
+        errors: [runtimeLimitExceededFailure(runtimeLimitSeconds)],
+        logs: terminalLogsWithPhaseSummary(ctx),
+        stats: ctx.getStats(),
+        steps: ctx.getSteps(),
+        tableNamespace: null,
+        totalRows,
+        inserted,
+        skipped,
+      };
+    },
+  });
 
   const playInput: PlayRunInputPayload =
     runtimeInputAlias && !Array.isArray(runtimeInput)
@@ -2683,6 +2948,7 @@ async function run(
       runtimeInput: playInput,
       options,
     });
+    ctx.assertRowAdmissionOpen();
     // A durable ctx boundary deliberately unwinds execution. Customer code can
     // catch that error, so the context keeps a sticky handoff that remains
     // authoritative until this invocation returns it to the scheduler.
@@ -2850,7 +3116,7 @@ async function run(
       reason: 'completed',
     });
 
-    const logs = ctx.getLogs();
+    const logs = terminalLogsWithPhaseSummary(ctx);
     const runtimeDiagnostics = Array.from(
       new Map(
         logs
@@ -2888,9 +3154,13 @@ async function run(
     // sticky handoff is still authoritative until this runner returns it to
     // the scheduler. Resolve that precedence before classifying the terminal
     // result so a replacement error cannot turn a suspended run into failed.
-    let error: unknown = caughtError;
+    let error: unknown = options?.rowAdmissionSignal?.aborted
+      ? options.rowAdmissionSignal.reason
+      : caughtError;
     try {
-      ctx.assertNoPendingDurableSuspension();
+      if (!options?.rowAdmissionSignal?.aborted) {
+        ctx.assertNoPendingDurableSuspension();
+      }
     } catch (suspension) {
       error = suspension;
     }
@@ -2921,7 +3191,7 @@ async function run(
       return {
         status: 'suspended',
         suspension: error.suspension,
-        logs: ctx.getLogs(),
+        logs: terminalLogsWithPhaseSummary(ctx),
         stats: ctx.getStats(),
         steps: ctx.getSteps(),
         checkpoint: ctx.getCheckpoint(),
@@ -2949,11 +3219,12 @@ async function run(
 
     ensureSourceMapSupport();
     const failure = normalizePlayRunFailure(error);
+    logRecovery();
     return {
       status: 'failed',
       error: formatRunnerFailure(error),
       errors: [failure],
-      logs: ctx.getLogs(),
+      logs: terminalLogsWithPhaseSummary(ctx),
       stats: ctx.getStats(),
       steps: ctx.getSteps(),
       checkpoint: ctx.getCheckpoint(),
@@ -3220,6 +3491,10 @@ async function main() {
   // is reachable instead of relying only on the detached ceiling salvage.
   runnerTerminalPushContext = terminalPushContextFromConfig(parsedConfig);
   const executionController = new AbortController();
+  const rowAdmissionController = new AbortController();
+  let runtimeState:
+    | Parameters<NonNullable<ExecutePlayRunnerOptions['onRuntimeReady']>>[0]
+    | undefined;
   const heartbeat = startRunnerPushHeartbeatFromConfig(parsedConfig, () => {
     executionController.abort(
       new Error('Play runner stopped by the runtime heartbeat channel.'),
@@ -3238,14 +3513,31 @@ async function main() {
     runnerProgressPusher = startRunnerProgressPusherFromConfig(config);
     const runtimeDeadline = startConfiguredPlayRunnerRuntimeDeadline({
       config,
+      onDrain: (runtimeLimitSeconds) => {
+        runtimeState?.log(
+          `[runtime.log_transport] ${JSON.stringify(runnerProgressPusher?.snapshot() ?? null)}`,
+        );
+        runtimeState?.log(
+          `Runtime limit approaching (${runtimeLimitSeconds}s configured); stopping new rows and saving started work for up to 4 seconds.`,
+        );
+        rowAdmissionController.abort(
+          new Error(runtimeLimitExceededCause(runtimeLimitSeconds)),
+        );
+      },
       onDeadline: async (runtimeLimitSeconds) => {
+        runtimeState?.log(
+          `[runtime.log_transport] ${JSON.stringify(runnerProgressPusher?.snapshot() ?? null)}`,
+        );
         executionController.abort(
           new Error(
             `Play runner reached its ${runtimeLimitSeconds}-second runtime limit.`,
           ),
         );
         heartbeat?.stop();
-        await terminateRunnerForRuntimeDeadline(runtimeLimitSeconds);
+        await terminateRunnerForRuntimeDeadline(
+          runtimeLimitSeconds,
+          runtimeState?.timeoutResult(runtimeLimitSeconds),
+        );
       },
       onError: (error) => {
         console.error('[play-runner.runtime-deadline] terminal_failed', {
@@ -3263,6 +3555,10 @@ async function main() {
     try {
       result = await executePlayRunner(config, {
         signal: executionController.signal,
+        rowAdmissionSignal: rowAdmissionController.signal,
+        onRuntimeReady: (runtime) => {
+          runtimeState = runtime;
+        },
       });
     } catch (error) {
       executionFailed = true;
@@ -3586,7 +3882,7 @@ function terminateRunnerForHeartbeatStop(
       ? stop.terminal === 'cancelled'
         ? 'Play run was cancelled; sandbox runner self-terminated on the receipt gateway heartbeat cancellation channel.'
         : 'Play run already failed in the scheduler; sandbox runner self-terminated on the receipt gateway heartbeat cancellation channel.'
-      : `Sandbox runner could not reach the receipt gateway for ${stop.reason === 'unreachable' ? stop.sinceMs : 0}ms (past the lease window); self-terminating — the parked attempt's ceiling timeout salvages this result from the captured output. ${formatRunnerGatewayTransportSummary(stop.reason === 'unreachable' ? stop.gatewayTransport : undefined)}`;
+      : `RUNTIME_CONTROL_PLANE_UNAVAILABLE: Sandbox runner could not reach the receipt gateway for ${stop.reason === 'unreachable' ? stop.sinceMs : 0}ms (past the lease window); self-terminating — the parked attempt's ceiling timeout salvages this result from the captured output. ${formatRunnerGatewayTransportSummary(stop.reason === 'unreachable' ? stop.gatewayTransport : undefined)}`;
   const failedResult = runnerTerminalResultForTransport({
     status: 'failed',
     error,
@@ -3606,17 +3902,20 @@ function terminateRunnerForHeartbeatStop(
 
 async function terminateRunnerForRuntimeDeadline(
   runtimeLimitSeconds: number,
+  snapshot?: PlayRunnerResult,
 ): Promise<never> {
   const error = runtimeLimitExceededCause(runtimeLimitSeconds);
-  const failedResult = runnerTerminalResultForTransport({
-    status: 'failed',
-    error,
-    errors: [runtimeLimitExceededFailure(runtimeLimitSeconds, error)],
-    logs: [],
-    stats: {},
-    steps: [],
-    tableNamespace: null,
-  });
+  const failedResult = runnerTerminalResultForTransport(
+    snapshot ?? {
+      status: 'failed',
+      error,
+      errors: [runtimeLimitExceededFailure(runtimeLimitSeconds, error)],
+      logs: [],
+      stats: {},
+      steps: [],
+      tableNamespace: null,
+    },
+  );
   // Stop customer code immediately. The structured stdout event is captured by
   // the Daytona shell wrapper, whose crash epilogue owns the bounded gateway
   // push during the outer grace period. Awaiting network/progress work here
@@ -3630,6 +3929,17 @@ async function terminateRunnerForRuntimeDeadline(
   );
 }
 
+// The external watchdog samples the sandbox cgroup and raises SIGUSR2 at its
+// pressure threshold. Register at module load: Node would otherwise terminate
+// before map admission can react and release retained result bodies.
+onMemoryPressure((sample) => {
+  emitWorkerLog(
+    `[memory] pressure signal received usage=${sample.usageBytes ?? 'unknown'} limit=${sample.limitBytes ?? 'unknown'}`,
+  );
+  // The shared sandbox argv enables explicit collection for this recovery path.
+  (globalThis as { gc?: () => void }).gc?.();
+});
+
 export function runPlayRunnerMain(): void {
   main().catch(async (error) => {
     ensureSourceMapSupport();

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts

@@ -7,6 +7,9 @@ import {
 export interface PlayRunnerRuntimeDeadlineOptions {
   limitMs: number;
   onDeadline: () => void | Promise<void>;
+  /** Stop new work while leaving a bounded window for durable writes. */
+  onDrain?: () => void;
+  drainLeadMs?: number;
   onError?: (error: unknown) => void;
   setTimeoutImpl?: (callback: () => void, delayMs: number) => unknown;
   clearTimeoutImpl?: (handle: unknown) => void;
@@ -22,6 +25,7 @@ export interface PlayRunnerRuntimeDeadlineHandle {
 export function startConfiguredPlayRunnerRuntimeDeadline(input: {
   config: Pick<PlayRunnerExecutionConfig, 'context'>;
   onDeadline: (runtimeLimitSeconds: number) => void | Promise<void>;
+  onDrain?: (runtimeLimitSeconds: number) => void;
   onError?: (error: unknown) => void;
   setTimeoutImpl?: (callback: () => void, delayMs: number) => unknown;
   clearTimeoutImpl?: (handle: unknown) => void;
@@ -33,7 +37,13 @@ export function startConfiguredPlayRunnerRuntimeDeadline(input: {
   );
   const runtimeLimitSeconds = runtimeLimits.timeoutSeconds;
   return startPlayRunnerRuntimeDeadline({
-    limitMs: runtimeLimitSeconds * 1_000,
+    // Reserve one second for captured terminal stdout before the independent
+    // watchdog's hard deadline. Legacy callers keep their exact deadline.
+    limitMs: runtimeLimitSeconds * 1_000 - (input.onDrain ? 1_000 : 0),
+    onDrain: input.onDrain
+      ? () => input.onDrain!(runtimeLimitSeconds)
+      : undefined,
+    drainLeadMs: 4_000,
     onDeadline: () => input.onDeadline(runtimeLimitSeconds),
     onError: input.onError,
     setTimeoutImpl: input.setTimeoutImpl,
@@ -62,6 +72,22 @@ export function startPlayRunnerRuntimeDeadline(
   const deadlineSettled = new Promise<void>((resolve) => {
     settleDeadline = resolve;
   });
+  const drainHandle = options.onDrain
+    ? setTimeoutImpl(
+        () => {
+          if (fired || completionClaimed) return;
+          try {
+            options.onDrain?.();
+          } catch (error) {
+            options.onError?.(error);
+          }
+        },
+        Math.max(
+          1,
+          limitMs - Math.min(options.drainLeadMs ?? 4_000, limitMs / 2),
+        ),
+      )
+    : null;
   const handle = setTimeoutImpl(() => {
     if (fired || completionClaimed) return;
     fired = true;
@@ -77,6 +103,7 @@ export function startPlayRunnerRuntimeDeadline(
       if (fired || completionClaimed) return false;
       completionClaimed = true;
       clearTimeoutImpl(handle);
+      if (drainHandle !== null) clearTimeoutImpl(drainHandle);
       return true;
     },
     deadlineSettled,