deepline 0.3.128 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts

@@ -1,10 +1,132 @@
+import { STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS } from '../../../plays/sandbox-runtime-limits';
+import { RUNTIME_SANDBOX_MEMORY_ABORT_CODE } from '../../run-failure';
+import {
+  RUNTIME_MEMORY_LEDGER_PATH_ENV,
+  RUNTIME_MEMORY_PRESSURE_PATH_ENV,
+} from '../../memory-pressure-signal';
+
 /** Exit code reserved for the external runner watchdog's runtime deadline. */
 export const DAYTONA_RUNTIME_WATCHDOG_EXIT_CODE = 124;
 
+/** The cgroup memory envelope one standard sandbox is given, in MiB. */
+export const STANDARD_PLAY_SANDBOX_MEMORY_MIB =
+  STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS.memoryGiB * 1024;
+
+/**
+ * Share of the sandbox cgroup that V8 may use for its old generation.
+ *
+ * This used to be 0.75, which was not an envelope at all: 768 MiB of old space
+ * plus the default ~192 MiB young generation let V8 alone reach 960 MiB inside
+ * a 1024 MiB cgroup before a single native buffer, code page, or in-flight
+ * transport body was counted. The cgroup kills on RSS, and RSS runs 1.3-1.5x
+ * the V8 live bytes, so V8 never had to collect before the kernel killed the
+ * process. Run 34064408068 died exactly that way at residentPeak ~1 GiB.
+ *
+ * At 0.5 the V8 heap ceiling sits at 512 MiB of the 1024 MiB cgroup, so V8
+ * collects hard while there is still cgroup headroom for everything V8 does
+ * not own.
+ */
+export const STANDARD_PLAY_RUNNER_OLD_SPACE_SHARE = 0.5;
+
+/**
+ * `--max-old-space-size` for the customer runner. This is the single V8
+ * envelope for both the Daytona and local-process substrates; map admission
+ * samples this actual V8 ceiling.
+ */
+export const STANDARD_PLAY_RUNNER_MAX_OLD_SPACE_SIZE_MIB = Math.floor(
+  STANDARD_PLAY_SANDBOX_MEMORY_MIB * STANDARD_PLAY_RUNNER_OLD_SPACE_SHARE,
+);
+
+/**
+ * `--max-semi-space-size` for the customer runner. V8 reserves three
+ * semi-spaces, so 16 MiB adds ~48 MiB to `heap_size_limit` where the 64-bit
+ * default (64 MiB) adds ~192 MiB. Capping the young generation is what makes
+ * the flag list an envelope rather than an old-space number with an
+ * unaccounted nursery bolted on top.
+ */
+export const STANDARD_PLAY_RUNNER_MAX_SEMI_SPACE_SIZE_MIB = 16;
+
+/**
+ * Approximately what `v8.getHeapStatistics().heap_size_limit` reports under
+ * the flags below: 512 + 3 x 16 = 560 MiB of the 1024 MiB sandbox.
+ */
+export const STANDARD_PLAY_RUNNER_HEAP_SIZE_LIMIT_MIB =
+  STANDARD_PLAY_RUNNER_MAX_OLD_SPACE_SIZE_MIB +
+  3 * STANDARD_PLAY_RUNNER_MAX_SEMI_SPACE_SIZE_MIB;
+
+/**
+ * The one Node exec argv every customer-runner spawn site uses. `--expose-gc`
+ * is not a debug flag here: the runner calls `globalThis.gc()` when this
+ * watchdog signals memory pressure, which is the only way a hold can return
+ * bytes to the cgroup before the abort threshold.
+ */
+export const PLAY_RUNNER_NODE_EXEC_ARGV: readonly string[] = [
+  `--max-old-space-size=${STANDARD_PLAY_RUNNER_MAX_OLD_SPACE_SIZE_MIB}`,
+  `--max-semi-space-size=${STANDARD_PLAY_RUNNER_MAX_SEMI_SPACE_SIZE_MIB}`,
+  '--expose-gc',
+  // The sandbox reports the host's 48-64 cores while its cgroup quota is one
+  // core, so V8 would spawn parallel and concurrent GC helpers that only
+  // contend with the main thread for that core (51-run Daytona CPU
+  // experiment, 2026-09-07: gzip on the threadpool accounted for every GC
+  // event and every event-loop stall). One core gets one GC thread.
+  '--single-threaded-gc',
+];
+
 /**
- * Build the tiny parent process that owns the hard runtime deadline outside
- * the customer-code Node event loop. A synchronous infinite loop can block the
- * runner's own timer, but it cannot block this separate process.
+ * libuv threadpool size for the customer runner. The default is 4 and it is
+ * sized for a machine, not a one-core cgroup: zlib and fs work on four threads
+ * plus the main thread thrash on one core. Two keeps gzip and file I/O off the
+ * main thread without oversubscribing it.
+ */
+export const PLAY_RUNNER_UV_THREADPOOL_SIZE = 2;
+
+/** How often the watchdog parent samples the cgroup's memory usage. */
+export const RUNNER_MEMORY_WATCH_INTERVAL_MS = 250;
+
+/**
+ * Fraction of the cgroup limit at which the watchdog signals the runner
+ * (SIGUSR2) to pause new admission, release caches, and collect.
+ */
+export const RUNNER_MEMORY_PRESSURE_RATIO = 0.8;
+
+/**
+ * Fraction of the cgroup limit at which the watchdog terminates the runner
+ * itself. A controlled abort carrying the runtime's byte ledger is a failure
+ * the run can report; the kernel's OOM kill is not.
+ */
+export const RUNNER_MEMORY_ABORT_RATIO = 0.92;
+
+/** Minimum spacing between pressure signals while usage stays above the ratio. */
+export const RUNNER_MEMORY_PRESSURE_SIGNAL_INTERVAL_MS = 2_000;
+
+/** Grace between SIGTERM and SIGKILL when the watchdog stops the runner. */
+export const RUNNER_TERMINATION_KILL_GRACE_MS = 1_000;
+
+/** Default cgroup mount point. Overridable by argv so tests can inject a root. */
+export const RUNNER_MEMORY_CGROUP_ROOT = '/sys/fs/cgroup';
+
+/**
+ * cgroup v1 reports "no limit" as a near-2^63 sentinel. Anything at or above
+ * this is not a real sandbox envelope, so the watch treats it as absent.
+ */
+export const RUNNER_MEMORY_CGROUP_UNLIMITED_BYTES = 2 ** 50;
+
+/** `reason` written into the termination diagnostic on a memory abort. */
+export const RUNTIME_SANDBOX_MEMORY_ABORT_REASON =
+  RUNTIME_SANDBOX_MEMORY_ABORT_CODE;
+
+/**
+ * Build the tiny parent process that owns the hard runtime deadline and the
+ * sandbox memory envelope outside the customer-code Node event loop. A
+ * synchronous infinite loop can block the runner's own timer, but it cannot
+ * block this separate process; nor can a runner whose heap is thrashing run
+ * its own memory watch reliably.
+ *
+ * argv (after `node -e <source>`):
+ *   1 runnerPath, 2 configPath, 3 runtimeStartedPath, 4 runtimeCompletedPath,
+ *   5 limitMs, 6 startupLimitMs, 7 terminalGraceMs, 8 runtimeLimitMarkerPath,
+ *   9 terminationDiagnosticPath, 10 memoryLedgerPath, 11 memoryPressurePath,
+ *   12 cgroupRoot.
  */
 export function buildDaytonaRuntimeWatchdogSource(): string {
   return `
@@ -19,18 +141,30 @@ const startupLimitMs = Math.max(1, Number.parseInt(process.argv[6] || '', 10) ||
 const terminalGraceMs = Math.max(1, Number.parseInt(process.argv[7] || '', 10) || 1);
 const runtimeLimitMarkerPath = process.argv[8] || '';
 const terminationDiagnosticPath = process.argv[9] || '';
+const memoryLedgerPath = process.argv[10] || '';
+const memoryPressurePath = process.argv[11] || '';
+const cgroupRoot = process.argv[12] || ${JSON.stringify(RUNNER_MEMORY_CGROUP_ROOT)};
 const runtimeLimitSeconds = Math.max(1, Math.ceil(limitMs / 1_000));
 const child = spawn(
   process.execPath,
-  ['--max-old-space-size=512', runnerPath, configPath],
+  [${PLAY_RUNNER_NODE_EXEC_ARGV.map((flag) => JSON.stringify(flag)).join(', ')}, runnerPath, configPath],
   {
-  env: process.env,
+  env: {
+    ...process.env,
+    ...(memoryLedgerPath
+      ? { ${JSON.stringify(RUNTIME_MEMORY_LEDGER_PATH_ENV)}: memoryLedgerPath }
+      : {}),
+    ...(memoryPressurePath
+      ? { ${JSON.stringify(RUNTIME_MEMORY_PRESSURE_PATH_ENV)}: memoryPressurePath }
+      : {}),
+  },
   stdio: 'inherit',
   },
 );
 let timedOut = false;
 let startupTimedOut = false;
 let terminalFlushTimedOut = false;
+let memoryAborted = false;
 let settled = false;
 let forceKillTimer = null;
 let runtimeTimer = null;
@@ -56,7 +190,7 @@ function writeTerminationDiagnostic(reason, details = {}) {
 function stopChild(reason, details) {
   writeTerminationDiagnostic(reason, details);
   child.kill('SIGTERM');
-  forceKillTimer = setTimeout(() => child.kill('SIGKILL'), 1_000);
+  forceKillTimer = setTimeout(() => child.kill('SIGKILL'), ${RUNNER_TERMINATION_KILL_GRACE_MS});
 }
 
 function markRuntimeLimit() {
@@ -66,6 +200,144 @@ function markRuntimeLimit() {
   } catch {}
 }
 
+// --- sandbox memory envelope ------------------------------------------------
+// Reads the cgroup the sandbox actually lives in, from OUTSIDE the customer
+// event loop. cgroup v2 first, then v1; where neither exists (macOS
+// local-process, for instance) the watch reports itself off once and stays off.
+let memorySource = null;
+let memoryWatchDisabledReported = false;
+let lastMemoryPressureSignalAt = 0;
+
+function readMemoryNumberFile(path) {
+  try {
+    const raw = fs.readFileSync(path, 'utf8').trim();
+    if (!raw) return null;
+    if (raw === 'max') return Number.POSITIVE_INFINITY;
+    const value = Number.parseInt(raw, 10);
+    return Number.isFinite(value) ? value : null;
+  } catch {}
+  return null;
+}
+
+function memorySourceCandidates() {
+  if (memorySource) return [memorySource];
+  return [
+    {
+      kind: 'cgroup_v2',
+      usagePath: cgroupRoot + '/memory.current',
+      limitPath: cgroupRoot + '/memory.max',
+    },
+    {
+      kind: 'cgroup_v1',
+      usagePath: cgroupRoot + '/memory.usage_in_bytes',
+      limitPath: cgroupRoot + '/memory.limit_in_bytes',
+    },
+    {
+      kind: 'cgroup_v1_nested',
+      usagePath: cgroupRoot + '/memory/memory.usage_in_bytes',
+      limitPath: cgroupRoot + '/memory/memory.limit_in_bytes',
+    },
+  ];
+}
+
+function readCgroupMemory() {
+  const candidates = memorySourceCandidates();
+  for (let index = 0; index < candidates.length; index += 1) {
+    const candidate = candidates[index];
+    const usageBytes = readMemoryNumberFile(candidate.usagePath);
+    const limitBytes = readMemoryNumberFile(candidate.limitPath);
+    if (usageBytes === null || limitBytes === null) continue;
+    if (!(limitBytes > 0)) continue;
+    if (limitBytes >= ${RUNNER_MEMORY_CGROUP_UNLIMITED_BYTES}) continue;
+    memorySource = candidate;
+    return {
+      source: candidate.kind,
+      usageBytes: usageBytes,
+      limitBytes: limitBytes,
+      ratio: usageBytes / limitBytes,
+    };
+  }
+  return null;
+}
+
+function reportMemoryWatchDisabled() {
+  if (memoryWatchDisabledReported) return;
+  memoryWatchDisabledReported = true;
+  // stdout, not stderr: an absent cgroup is an expected substrate fact on a
+  // host without memory accounting, and stderr is reserved for terminations.
+  process.stdout.write(
+    '[memory] sandbox memory watch disabled: no cgroup memory accounting under ' +
+      cgroupRoot +
+      '\\n',
+  );
+}
+
+function readMemoryLedger() {
+  if (!memoryLedgerPath) return null;
+  try {
+    const raw = fs.readFileSync(memoryLedgerPath, 'utf8');
+    if (!raw.trim()) return null;
+    return JSON.parse(raw);
+  } catch {}
+  return null;
+}
+
+function publishMemoryPressure(sample) {
+  if (!memoryPressurePath) return;
+  try {
+    fs.writeFileSync(
+      memoryPressurePath,
+      JSON.stringify({
+        schemaVersion: 1,
+        source: sample.source,
+        usageBytes: sample.usageBytes,
+        limitBytes: sample.limitBytes,
+        ratio: sample.ratio,
+        at: Date.now(),
+      }),
+    );
+  } catch {}
+}
+
+const memoryWatch = setInterval(() => {
+  if (settled || memoryAborted) return;
+  const sample = readCgroupMemory();
+  if (!sample) {
+    reportMemoryWatchDisabled();
+    return;
+  }
+  if (sample.ratio >= ${RUNNER_MEMORY_ABORT_RATIO}) {
+    memoryAborted = true;
+    clearInterval(memoryWatch);
+    process.stderr.write(
+      '${RUNTIME_SANDBOX_MEMORY_ABORT_REASON}: sandbox memory usage ' +
+        sample.usageBytes +
+        ' of ' +
+        sample.limitBytes +
+        ' bytes reached the abort threshold; the runner was stopped before the kernel could kill it.\\n',
+    );
+    stopChild('${RUNTIME_SANDBOX_MEMORY_ABORT_REASON}', {
+      memorySource: sample.source,
+      memoryUsageBytes: sample.usageBytes,
+      memoryLimitBytes: sample.limitBytes,
+      memoryUsageRatio: Math.round(sample.ratio * 10_000) / 10_000,
+      memoryLedger: readMemoryLedger(),
+    });
+    return;
+  }
+  if (sample.ratio >= ${RUNNER_MEMORY_PRESSURE_RATIO}) {
+    const now = Date.now();
+    if (now - lastMemoryPressureSignalAt < ${RUNNER_MEMORY_PRESSURE_SIGNAL_INTERVAL_MS}) {
+      return;
+    }
+    lastMemoryPressureSignalAt = now;
+    publishMemoryPressure(sample);
+    try {
+      child.kill('SIGUSR2');
+    } catch {}
+  }
+}, ${RUNNER_MEMORY_WATCH_INTERVAL_MS});
+
 function armRuntimeDeadline() {
   if (settled || runtimeTimer) return;
   clearTimeout(startupTimer);
@@ -124,6 +396,7 @@ function finish(code) {
   if (settled) return;
   settled = true;
   clearInterval(runtimeBoundaryPoll);
+  clearInterval(memoryWatch);
   clearTimeout(startupTimer);
   if (runtimeTimer) clearTimeout(runtimeTimer);
   if (terminalGraceTimer) clearTimeout(terminalGraceTimer);
@@ -134,6 +407,8 @@ function finish(code) {
       ? 1
     : terminalFlushTimedOut
       ? 1
+    : memoryAborted
+      ? 1
     : typeof code === 'number'
       ? code
       : 1;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -511,6 +511,8 @@ export async function inspectDetachedDaytonaRunner(input: {
   cmdId: string;
   outputPath: string;
   exitCodePath: string;
+  /** Refuse to read a sandbox outside the launch-pinned Daytona organization. */
+  expectedOrganizationId?: string | null;
 }): Promise<{
   exitCode: number | null;
   result: PlayRunnerResult | null;
@@ -521,7 +523,8 @@ export async function inspectDetachedDaytonaRunner(input: {
       | 'result_missing'
       | 'command_inspection_failed'
       | 'sandbox_missing'
-      | 'sandbox_lookup_failed';
+      | 'sandbox_lookup_failed'
+      | 'wrong_routing_domain';
     detail: string | null;
   };
 }> {
@@ -529,6 +532,21 @@ export async function inspectDetachedDaytonaRunner(input: {
     const { clientOptions } = loadDaytonaRequiredConfig();
     const daytona = daytonaSdkClientFactory.createFull(clientOptions);
     const sandbox = (await daytona.get(input.sandboxId)) as DaytonaSandbox;
+    const expectedOrganizationId = input.expectedOrganizationId?.trim() || null;
+    const observedOrganizationId = sandbox.organizationId?.trim() || null;
+    if (
+      expectedOrganizationId &&
+      observedOrganizationId !== expectedOrganizationId
+    ) {
+      return {
+        exitCode: null,
+        result: null,
+        diagnosis: {
+          stage: 'wrong_routing_domain',
+          detail: 'Sandbox belongs to a different Daytona organization.',
+        },
+      };
+    }
     let exitCode: number | null = null;
     let commandFound = false;
     let commandInspectionError: string | null = null;
@@ -911,6 +929,7 @@ export function retryableRuntimeInitializationFailureReason(
   | 'runtime_receipt_gateway_transport'
   | 'daytona_public_status'
   | 'daytona_runner_startup_transport'
+  | 'daytona_infrastructure'
   | null {
   if (result.status !== 'failed') return null;
   if (RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN.test(result.error)) {
@@ -941,6 +960,13 @@ export function retryableRuntimeInitializationFailureReason(
   ) {
     return 'runtime_api_transport';
   }
+  if (
+    rowsProcessed === 0 &&
+    result.steps.length === 0 &&
+    DAYTONA_INFRASTRUCTURE_RETRY_PATTERN.test(result.error)
+  ) {
+    return 'daytona_infrastructure';
+  }
   return null;
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts

@@ -441,6 +441,22 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
                 workspaceRoot,
                 'runtime-limit.marker',
               );
+              const terminationDiagnosticPath = join(
+                workspaceRoot,
+                'runtime-termination.json',
+              );
+              // The local substrate shares the sandbox memory envelope: same
+              // watchdog, same flags, same signal contract. On a host without
+              // cgroup memory accounting the watch reports itself off once and
+              // the runtime deadline behaviour is unchanged.
+              const memoryLedgerPath = join(
+                workspaceRoot,
+                'memory-ledger.json',
+              );
+              const memoryPressurePath = join(
+                workspaceRoot,
+                'memory-pressure.json',
+              );
               const result = await runCommand(
                 'node',
                 [
@@ -454,6 +470,9 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
                   String(PLAY_RUNNER_STARTUP_GRACE_SECONDS * 1_000),
                   String(PLAY_RUNNER_TERMINAL_GRACE_SECONDS * 1_000),
                   runtimeLimitMarkerPath,
+                  terminationDiagnosticPath,
+                  memoryLedgerPath,
+                  memoryPressurePath,
                 ],
                 {
                   cwd: workspaceRoot,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts

@@ -6,6 +6,7 @@ import {
   SandboxRunnerPreCodeDeferralError,
 } from '../types';
 import { buildPlayRunnerBundle } from '../bundle';
+import { findPlayRunnerResult, parsePlayRunnerEvents } from '../runner-events';
 import type {
   PlayRunnerExecutionConfig,
   PlayRunnerResult,
@@ -39,6 +40,7 @@ const MODAL_SANDBOX_CREATE_TIMEOUT_MS =
   RUNTIME_RELIABILITY_POLICY.sandbox.modalSandboxCreateTimeoutMs;
 const MODAL_PAYLOAD_UPLOAD_TIMEOUT_MS = 90_000;
 const MODAL_TERMINATION_TIMEOUT_MS = 10_000;
+const MODAL_DETACHED_INSPECTION_TIMEOUT_MS = 10_000;
 const MODAL_CAPACITY_RELEASE_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
 const MODAL_LATE_CREATE_PERSIST_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
 type ModalSandbox = Awaited<ReturnType<ModalClient['sandboxes']['create']>>;
@@ -820,6 +822,112 @@ export async function findModalSandboxIdsByTags(input: {
   return [...new Set(sandboxIds)];
 }
 
+/**
+ * Best-effort terminal recovery for an already-detached Modal runner. This
+ * reads only the payload wrapper's captured files; it never executes Play
+ * code or launches a replacement sandbox.
+ */
+export async function inspectDetachedModalRunner(input: {
+  sandboxId: string;
+  outputPath: string;
+  exitCodePath: string;
+  expectedAppId?: string | null;
+}): Promise<{
+  exitCode: number | null;
+  result: PlayRunnerResult | null;
+  diagnosis: { stage: string; detail: string | null };
+}> {
+  const expectedAppId = input.expectedAppId?.trim() || null;
+  if (!expectedAppId) {
+    return {
+      exitCode: null,
+      result: null,
+      diagnosis: {
+        stage: 'missing_routing_domain',
+        detail: 'Modal sandbox inspection requires the launch-pinned app.',
+      },
+    };
+  }
+
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      (async () => {
+        const { client } = await loadModalClientConfig();
+        for await (const sandbox of client.sandboxes.list({
+          appId: expectedAppId,
+        })) {
+          if (sandbox.sandboxId !== input.sandboxId) continue;
+          const [output, exitCode] = await Promise.allSettled([
+            sandbox.filesystem.readText(input.outputPath),
+            sandbox.filesystem.readText(input.exitCodePath),
+          ]);
+          if (output.status !== 'fulfilled') {
+            return {
+              exitCode: null,
+              result: null,
+              diagnosis: {
+                stage: 'result_missing',
+                detail: 'Captured runner output is unavailable.',
+              },
+            };
+          }
+          const parsedExitCode =
+            exitCode.status === 'fulfilled' &&
+            /^-?\d+$/u.test(exitCode.value.trim())
+              ? Number(exitCode.value.trim())
+              : null;
+          const result =
+            findPlayRunnerResult(parsePlayRunnerEvents(output.value)) ?? null;
+          return {
+            exitCode:
+              parsedExitCode !== null && Number.isSafeInteger(parsedExitCode)
+                ? parsedExitCode
+                : null,
+            result,
+            diagnosis: result
+              ? { stage: 'salvaged', detail: null }
+              : { stage: 'result_missing', detail: null },
+          };
+        }
+        return {
+          exitCode: null,
+          result: null,
+          diagnosis: {
+            stage: 'sandbox_missing',
+            detail: 'Sandbox was not found in the launch-pinned Modal app.',
+          },
+        };
+      })(),
+      new Promise<never>((_resolve, reject) => {
+        timer = setTimeout(() => {
+          reject(
+            new Error(
+              `Modal detached-runner inspection exceeded ${MODAL_DETACHED_INSPECTION_TIMEOUT_MS}ms.`,
+            ),
+          );
+        }, MODAL_DETACHED_INSPECTION_TIMEOUT_MS);
+      }),
+    ]);
+  } catch (error) {
+    console.warn('[play-runner.modal.detached_inspect_failed]', {
+      sandboxId: input.sandboxId,
+      expectedAppId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return {
+      exitCode: null,
+      result: null,
+      diagnosis: {
+        stage: 'sandbox_lookup_failed',
+        detail: 'Modal sandbox inspection could not complete.',
+      },
+    };
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+
 export async function readDetachedModalRuntimeCompletion(input: {
   sandboxId: string;
   runtimeCompletedPath: string;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts

@@ -1,9 +1,11 @@
 import {
   deleteDaytonaSandboxByIdWithOutcome,
+  inspectDetachedDaytonaRunner,
   readDetachedDaytonaRuntimeCompletion,
 } from './backends/daytona';
 import {
   deleteModalSandboxById,
+  inspectDetachedModalRunner,
   readDetachedModalRuntimeCompletion,
 } from './backends/modal';
 import {
@@ -12,6 +14,7 @@ import {
   type RuntimeSandboxRef,
 } from '../runtime-sandbox-contract';
 import type { DetachedRunnerSuspension } from '../suspension';
+import type { PlayRunnerResult } from '../protocol';
 
 export type {
   RuntimeSandboxProviderId,
@@ -21,6 +24,15 @@ export type {
 export type RuntimeSandboxReconciliationAdapter = {
   provider: RuntimeSandboxProviderId;
   delete(ref: RuntimeSandboxRef): Promise<RuntimeSandboxDeleteOutcome>;
+  /**
+   * Read an already-captured detached-runner terminal without executing Play
+   * code. The scheduler owns whether this is safe to commit; adapters only
+   * implement their provider's file read.
+   */
+  inspectDetachedRunner(
+    ref: RuntimeSandboxRef,
+    suspension: DetachedRunnerSuspension,
+  ): Promise<RuntimeSandboxDetachedRunnerInspection | null>;
   readWorkCompletedAt(
     ref: RuntimeSandboxRef,
     path: string,
@@ -29,12 +41,22 @@ export type RuntimeSandboxReconciliationAdapter = {
 
 export type RuntimeSandboxReconciliation = {
   delete(ref: RuntimeSandboxRef): Promise<RuntimeSandboxDeleteOutcome>;
+  inspectDetachedRunner(
+    ref: RuntimeSandboxRef,
+    suspension: DetachedRunnerSuspension,
+  ): Promise<RuntimeSandboxDetachedRunnerInspection | null>;
   readWorkCompletedAt(
     ref: RuntimeSandboxRef,
     path: string,
   ): Promise<number | null>;
 };
 
+export type RuntimeSandboxDetachedRunnerInspection = {
+  exitCode: number | null;
+  result: PlayRunnerResult | null;
+  diagnosis: { stage: string; detail: string | null };
+};
+
 export type RuntimeSandboxDeleteOutcome =
   | {
       kind: 'deleted' | 'already_absent';
@@ -94,6 +116,8 @@ export function createRuntimeSandboxReconciliation(input: {
   };
   return {
     delete: (ref) => resolve(ref).delete(ref),
+    inspectDetachedRunner: (ref, suspension) =>
+      resolve(ref).inspectDetachedRunner(ref, suspension),
     readWorkCompletedAt: (ref, path) =>
       resolve(ref).readWorkCompletedAt(ref, path),
   };
@@ -181,6 +205,15 @@ export const defaultRuntimeSandboxReconciliation =
             routingDomain: outcome.organizationId,
           };
         },
+        inspectDetachedRunner: (ref, suspension) =>
+          inspectDetachedDaytonaRunner({
+            sandboxId: ref.resourceId,
+            sessionId: suspension.sessionId,
+            cmdId: suspension.cmdId,
+            outputPath: suspension.outputPath,
+            exitCodePath: suspension.exitCodePath,
+            expectedOrganizationId: ref.routingDomain,
+          }),
         readWorkCompletedAt: (ref, path) =>
           readDetachedDaytonaRuntimeCompletion({
             sandboxId: ref.resourceId,
@@ -200,6 +233,13 @@ export const defaultRuntimeSandboxReconciliation =
             routingDomain: outcome.appId,
           };
         },
+        inspectDetachedRunner: (ref, suspension) =>
+          inspectDetachedModalRunner({
+            sandboxId: ref.resourceId,
+            outputPath: suspension.outputPath,
+            exitCodePath: suspension.exitCodePath,
+            expectedAppId: ref.routingDomain,
+          }),
         readWorkCompletedAt: (ref, path) =>
           readDetachedModalRuntimeCompletion({
             sandboxId: ref.resourceId,