deepline 0.1.252 → 0.1.254

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -236,10 +236,26 @@ export const RUNNER_TERMINAL_CHECKPOINT_DISPOSITION = {
 export type RunnerTerminalCheckpointDisposition =
   (typeof RUNNER_TERMINAL_CHECKPOINT_DISPOSITION)[keyof typeof RUNNER_TERMINAL_CHECKPOINT_DISPOSITION];
 
+export type PlayRunOutputWarning = {
+  code: 'RUN_OUTPUT_TRUNCATED';
+  reason: 'array_preview_limit' | 'byte_limit';
+  message: string;
+  originalBytes?: number;
+  originalBytesAtLeast?: number;
+  retainedBytes?: number;
+  limitBytes?: number;
+  limitItems?: number;
+  originalItems?: number;
+  retainedItems?: number;
+  retryable: false;
+  recoverableFrom: 'runtime_sheet' | 'none';
+};
+
 export type PlayRunnerResult =
   | {
       status: 'completed';
       output: unknown;
+      outputWarnings?: PlayRunOutputWarning[];
       outputRowCount?: number;
       logs: string[];
       stats: Record<string, unknown>;

package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts

@@ -58,6 +58,10 @@ export interface RuntimeResourceGovernor {
     estimatedCost?: number | null;
     signal?: AbortSignal;
   }): Promise<RuntimeResourceLease>;
+  acquireProviderPermit(input: {
+    toolId: string;
+    signal?: AbortSignal;
+  }): Promise<RuntimeResourceLease>;
   acquireSheetFlush(input?: {
     estimatedBytes?: number | null;
     rowCount?: number | null;
@@ -290,6 +294,13 @@ export function createRuntimeResourceGovernor(input: {
       return lease;
     },
 
+    acquireProviderPermit(providerInput) {
+      return input.executionGovernor.acquireProviderPermit(
+        providerInput.toolId,
+        { signal: providerInput.signal },
+      );
+    },
+
     async acquireSheetFlush(sheetInput) {
       const startedAt = Date.now();
       const lease = await sheetFlushAdmission.acquire(sheetInput?.signal);

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -6,7 +6,11 @@ const CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE =
   /The requested endpoint could not be found, or you don't have access to it\. Please check the provided ID and try again\./i;
 const VERCEL_RUNTIME_API_DEPLOYMENT_MISSING_RE =
   /runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i;
+const RUNTIME_LIMIT_EXCEEDED_RE = /\bRUNTIME_LIMIT_EXCEEDED\b/i;
+const RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_RE =
+  /\bRUNTIME_SANDBOX_INSPECTION_UNAVAILABLE\b/i;
 const RUNTIME_SANDBOX_LOST_RE = /\bRUNTIME_SANDBOX_LOST\b/i;
+const OUTPUT_TOO_LARGE_RE = /\b(?:OUTPUT_TOO_LARGE|OutputTooLarge)\b/;
 
 export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
   'Run interrupted by a platform deploy. Deepline retries this automatically when possible; if this error is still visible, re-run the same command.';
@@ -19,6 +23,10 @@ export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
   'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.';
 export const RUNTIME_SANDBOX_LOST_MESSAGE =
   'The execution sandbox became unreachable before returning a terminal result. Re-run the same command; if this keeps happening, contact Deepline support with the run ID.';
+export const RUNTIME_LIMIT_EXCEEDED_MESSAGE =
+  'The play reached its 30 minute runtime limit and was stopped. Completed row state was preserved; run a smaller batch or continue from the persisted rows.';
+export const RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_MESSAGE =
+  'The play stopped after Deepline could not verify the execution sandbox state. Completed row state was preserved. Re-run from the persisted rows; if this keeps happening, contact Deepline support with the run ID.';
 
 export const WORKSPACE_STORAGE_NOT_READY_CODE = 'WORKSPACE_STORAGE_NOT_READY';
 
@@ -164,7 +172,10 @@ function boundedFailureText(value: string): string {
 function boundedFailureStack(error: Error): string | undefined {
   if (typeof error.stack !== 'string' || !error.stack.trim()) return undefined;
   return boundedFailureText(
-    error.stack.split('\n').slice(0, PUBLIC_FAILURE_STACK_LINE_LIMIT).join('\n'),
+    error.stack
+      .split('\n')
+      .slice(0, PUBLIC_FAILURE_STACK_LINE_LIMIT)
+      .join('\n'),
   );
 }
 
@@ -205,6 +216,38 @@ function toErrorText(error: unknown): string {
 export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
   const rawCause = toErrorText(error);
   const cause = boundedFailureText(rawCause);
+  if (
+    (error &&
+      typeof error === 'object' &&
+      (error as { code?: unknown }).code === 'OUTPUT_TOO_LARGE') ||
+    OUTPUT_TOO_LARGE_RE.test(rawCause)
+  ) {
+    return {
+      code: 'OUTPUT_TOO_LARGE',
+      phase: 'runtime',
+      message: cause,
+      retryable: false,
+      cause,
+    };
+  }
+  if (RUNTIME_LIMIT_EXCEEDED_RE.test(rawCause)) {
+    return {
+      code: 'RUNTIME_LIMIT_EXCEEDED',
+      phase: 'runtime',
+      message: RUNTIME_LIMIT_EXCEEDED_MESSAGE,
+      retryable: false,
+      cause,
+    };
+  }
+  if (RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_RE.test(rawCause)) {
+    return {
+      code: 'RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE',
+      phase: 'infrastructure',
+      message: RUNTIME_SANDBOX_INSPECTION_UNAVAILABLE_MESSAGE,
+      retryable: false,
+      cause,
+    };
+  }
   if (RUNTIME_SANDBOX_LOST_RE.test(rawCause)) {
     const stack = error instanceof Error ? boundedFailureStack(error) : null;
     const causes = error instanceof Error ? failureCauseTexts(error) : [];

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -4,13 +4,16 @@ import type {
   PlayRunnerResult,
 } from '@shared_libs/play-runtime/protocol';
 import { isIsolatedRuntimeSchedulerSchema } from '@shared_libs/play-runtime/runtime-scheduler-topology';
+import { PLAY_RUNNER_TIMEOUT_SECONDS } from '@shared_libs/play-runtime/runtime-constants';
 
 const DAYTONA_CREATE_TIMEOUT_SECONDS = 10;
 const DAYTONA_CREATE_RETRY_DELAYS_MS = [0, 500, 1_500] as const;
-// A detached runner is bounded by both the play runtime ceiling and Daytona's
-// own inactivity stop. No background activity refresh keeps a wedged sandbox
-// alive past those limits.
-const DAYTONA_AUTO_STOP_INTERVAL_MINUTES = 30;
+// Explicit runner deadline + scheduler GC own the normal lifecycle. Daytona's
+// inactivity stop is a wider crash backstop measured from sandbox creation, so
+// setup time cannot consume the terminal-flush grace.
+const DAYTONA_AUTO_STOP_INTERVAL_MINUTES = Math.ceil(
+  PLAY_RUNNER_TIMEOUT_SECONDS / 60,
+);
 const DAYTONA_AUTO_ARCHIVE_INTERVAL_MINUTES = 15;
 const DAYTONA_SANDBOX_LABEL_SOURCE = 'deepline-play-runner';
 // The sandbox must be created at its final resource boundary. Do not add a

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -9,12 +9,22 @@ import {
   PLAY_RUNTIME_CONTRACT,
   PLAY_RUNTIME_CONTRACT_HEADER,
 } from '@shared_libs/play-runtime/runtime-contract';
+import {
+  PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
+  PLAY_RUNNER_STARTUP_GRACE_SECONDS,
+  STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
+} from '@shared_libs/play-runtime/runtime-constants';
+import {
+  RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES,
+  RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES,
+} from '@shared_libs/play-runtime/output-size-limits';
 import { compactPlayArtifactForRuntimeTransport } from '@shared_libs/plays/artifact-transport';
 import type {
   DaytonaExecutionContext,
   DaytonaSandbox,
   DaytonaStageEmitter,
 } from './daytona-lifecycle';
+import { buildDaytonaRuntimeWatchdogSource } from './daytona-runtime-watchdog';
 
 type DaytonaRemoteRuntimeContext = PlayRunnerExecutionConfig['context'];
 
@@ -28,6 +38,20 @@ type DaytonaPayloadEnvelope = {
   crashPusherCode: string;
 };
 
+/**
+ * The crash pusher must be able to recover one complete terminal event after
+ * a runner exits before its primary gateway push. A stdout `result` event is
+ * smaller than the accepted runner-terminal request envelope (the request also
+ * carries action + runId), so this covers every terminal that the gateway can
+ * have accepted, including a resume-required suspended checkpoint. Leave room
+ * for the runner's primary-push retry diagnostics written after that event.
+ */
+export const DAYTONA_CRASH_PUSHER_POST_EVENT_DIAGNOSTIC_HEADROOM_BYTES =
+  RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES;
+export const DAYTONA_CRASH_PUSHER_OUTPUT_TAIL_MAX_BYTES =
+  RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES +
+  DAYTONA_CRASH_PUSHER_POST_EVENT_DIAGNOSTIC_HEADROOM_BYTES;
+
 export type StagedDaytonaPayload = {
   workDir: string;
   command: string;
@@ -99,8 +123,32 @@ const fs = require('node:fs');
 const configPath = process.argv[2];
 const exitCodeRaw = Number.parseInt(process.argv[3] ?? '', 10);
 const outputPath = process.argv[4];
+const runtimeLimitMarkerPath = process.argv[5] || '';
 const exitCode = Number.isFinite(exitCodeRaw) ? exitCodeRaw : null;
-const MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
+const MAX_OUTPUT_BYTES = ${DAYTONA_CRASH_PUSHER_OUTPUT_TAIL_MAX_BYTES};
+const MAX_POST_TERMINAL_DIAGNOSTIC_BYTES = ${RUNNER_POST_TERMINAL_DIAGNOSTIC_MAX_BYTES};
+let postTerminalDiagnosticBytesRemaining = MAX_POST_TERMINAL_DIAGNOSTIC_BYTES;
+
+function truncateUtf8(value, maxBytes) {
+  if (Buffer.byteLength(value, 'utf8') <= maxBytes) return value;
+  const encoded = Buffer.from(value, 'utf8');
+  for (let end = Math.min(encoded.length, maxBytes); end > 0; end -= 1) {
+    const candidate = encoded.subarray(0, end).toString('utf8');
+    if (Buffer.byteLength(candidate, 'utf8') <= maxBytes) return candidate;
+  }
+  return '';
+}
+
+function writeDiagnostic(line) {
+  if (postTerminalDiagnosticBytesRemaining <= 1) return;
+  const bounded = truncateUtf8(
+    String(line),
+    postTerminalDiagnosticBytesRemaining - 1,
+  );
+  if (!bounded) return;
+  console.log(bounded);
+  postTerminalDiagnosticBytesRemaining -= Buffer.byteLength(bounded, 'utf8') + 1;
+}
 
 function readResultFromOutput() {
   try {
@@ -170,17 +218,34 @@ async function main() {
   const gateway = String(context.receiptGatewayBaseUrl || '').replace(/\\/$/, '');
   const token = context.executorToken;
   if (!push || !push.runId || !gateway || !token) {
-    console.log('[crash-terminal] no push config; skipping');
+    writeDiagnostic('[crash-terminal] no push config; skipping');
     return;
   }
   const parsedResult = readResultFromOutput();
-  const result = terminalResultForTransport(parsedResult || {
-    status: 'failed',
-    error:
+  // Exit codes are authored-program output too: a play may intentionally call
+  // process.exit(124). Only the separate parent watchdog writes this marker.
+  const runtimeLimitExceeded = Boolean(
+    runtimeLimitMarkerPath && fs.existsSync(runtimeLimitMarkerPath),
+  );
+  const synthesizedError = runtimeLimitExceeded
+    ? 'RUNTIME_LIMIT_EXCEEDED: The play reached its 30 minute runtime limit and was stopped.'
+    :
       'Daytona play runner exited with code ' +
       (exitCode === null ? 'unknown' : exitCode) +
       (exitCode === 137 ? ' (sandbox OOM or forced SIGKILL)' : '') +
-      ' without pushing a terminal (crash containment epilogue).',
+      ' without pushing a terminal (crash containment epilogue).';
+  const result = terminalResultForTransport(parsedResult || {
+    status: 'failed',
+    error: synthesizedError,
+    errors: runtimeLimitExceeded
+      ? [{
+          code: 'RUNTIME_LIMIT_EXCEEDED',
+          phase: 'runtime',
+          message: 'The play reached its 30 minute runtime limit and was stopped. Completed row state was preserved; run a smaller batch or continue from the persisted rows.',
+          retryable: false,
+          cause: synthesizedError,
+        }]
+      : undefined,
     logs: [],
     stats: {},
     steps: [],
@@ -209,24 +274,24 @@ async function main() {
       });
       clearTimeout(timer);
       if (response.ok) {
-        console.log('[crash-terminal] pushed status=' + result.status);
+        writeDiagnostic('[crash-terminal] pushed status=' + result.status);
         return;
       }
-      console.log('[crash-terminal] rejected http=' + response.status);
+      writeDiagnostic('[crash-terminal] rejected http=' + response.status);
       // Deterministic scope/fence rejections cannot succeed on retry.
       if (response.status >= 400 && response.status < 500) return;
     } catch (error) {
-      console.log(
+      writeDiagnostic(
         '[crash-terminal] transport ' + (error && error.message ? error.message : String(error)),
       );
     }
     await new Promise((resolveDelay) => setTimeout(resolveDelay, attempt * 1000));
   }
-  console.log('[crash-terminal] exhausted retries');
+  writeDiagnostic('[crash-terminal] exhausted retries');
 }
 
 main().catch((error) => {
-  console.log('[crash-terminal] fatal ' + (error && error.message ? error.message : String(error)));
+  writeDiagnostic('[crash-terminal] fatal ' + (error && error.message ? error.message : String(error)));
 });
 `;
 }
@@ -389,14 +454,31 @@ export async function stageDaytonaRunnerPayload(input: {
   });
 
   const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
+  const crashPusherLogPath = `${workDir}/deepline-play-crash-terminal-${randomUUID()}.log`;
   const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
   const readyPath = `${workDir}/deepline-play-ready-${randomUUID()}.json`;
   const startAckPath = `${workDir}/deepline-play-start-ack-${randomUUID()}.json`;
+  const runtimeStartedPath = `${workDir}/deepline-play-runtime-started-${randomUUID()}.json`;
+  const runtimeCompletedPath = `${workDir}/deepline-play-runtime-completed-${randomUUID()}.json`;
+  const runtimeLimitMarkerPath = `${workDir}/deepline-play-runtime-limit-${randomUUID()}.txt`;
   const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
   const runnerTraceEnv =
     process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
       ? 'DEEPLINE_RUNTIME_RECEIPT_TRACE=1 '
       : '';
+  const watchedRunnerCommand = [
+    'node',
+    '-e',
+    shellQuote(buildDaytonaRuntimeWatchdogSource()),
+    shellQuote(runnerPath),
+    shellQuote(configPath),
+    shellQuote(runtimeStartedPath),
+    shellQuote(runtimeCompletedPath),
+    String(STANDARD_PLAY_RUNTIME_LIMIT_SECONDS * 1_000),
+    String(PLAY_RUNNER_STARTUP_GRACE_SECONDS * 1_000),
+    String(PLAY_RUNNER_TERMINAL_GRACE_SECONDS * 1_000),
+    shellQuote(runtimeLimitMarkerPath),
+  ].join(' ');
   const runnerCommand = `${nodeMaterializePayloadCommand({
     envelopePath,
     runnerPath,
@@ -404,13 +486,16 @@ export async function stageDaytonaRunnerPayload(input: {
     artifactCodePath,
     artifactSourceMapPath,
     crashPusherPath,
-  })} && DEEPLINE_PLAY_RUNNER_READY_PATH=${shellQuote(readyPath)} DEEPLINE_PLAY_RUNNER_START_ACK_PATH=${shellQuote(startAckPath)} DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
+  })} && DEEPLINE_PLAY_RUNNER_READY_PATH=${shellQuote(readyPath)} DEEPLINE_PLAY_RUNNER_START_ACK_PATH=${shellQuote(startAckPath)} DEEPLINE_PLAY_RUNNER_RUNTIME_STARTED_PATH=${shellQuote(runtimeStartedPath)} DEEPLINE_PLAY_RUNNER_RUNTIME_COMPLETED_PATH=${shellQuote(runtimeCompletedPath)} DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}${watchedRunnerCommand}`;
   // Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
   // pushes the parsed (or synthesized) terminal to the gateway so the parked
   // worker wakes within seconds of ANY runner death — process.exit abuse, OOM
   // SIGKILL, or a crash before the in-process push. Idempotent against a
   // successful in-runner push (terminals are first-write-wins).
-  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(readyPath)} ${shellQuote(startAckPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
+  // Its own retry diagnostics are kept in a separate log: appending them to
+  // `outputPath` after a near-gateway-sized result could hide that result from
+  // a later bounded-tail recovery read.
+  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(crashPusherLogPath)} ${shellQuote(exitCodePath)} ${shellQuote(readyPath)} ${shellQuote(startAckPath)} ${shellQuote(runtimeStartedPath)} ${shellQuote(runtimeCompletedPath)} ${shellQuote(runtimeLimitMarkerPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} ${shellQuote(runtimeLimitMarkerPath)} > ${shellQuote(crashPusherLogPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
 
   return {
     workDir: input.workDir,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts

@@ -0,0 +1,117 @@
+/** Exit code reserved for the external runner watchdog's runtime deadline. */
+export const DAYTONA_RUNTIME_WATCHDOG_EXIT_CODE = 124;
+
+/**
+ * Build the tiny parent process that owns the hard runtime deadline outside
+ * the customer-code Node event loop. A synchronous infinite loop can block the
+ * runner's own timer, but it cannot block this separate process.
+ */
+export function buildDaytonaRuntimeWatchdogSource(): string {
+  return `
+const fs = require('node:fs');
+const { spawn } = require('node:child_process');
+const runnerPath = process.argv[1];
+const configPath = process.argv[2];
+const runtimeStartedPath = process.argv[3];
+const runtimeCompletedPath = process.argv[4];
+const limitMs = Math.max(1, Number.parseInt(process.argv[5] || '', 10) || 1);
+const startupLimitMs = Math.max(1, Number.parseInt(process.argv[6] || '', 10) || 1);
+const terminalGraceMs = Math.max(1, Number.parseInt(process.argv[7] || '', 10) || 1);
+const runtimeLimitMarkerPath = process.argv[8] || '';
+const child = spawn(process.execPath, [runnerPath, configPath], {
+  env: process.env,
+  stdio: 'inherit',
+});
+let timedOut = false;
+let startupTimedOut = false;
+let terminalFlushTimedOut = false;
+let settled = false;
+let forceKillTimer = null;
+let runtimeTimer = null;
+let terminalGraceTimer = null;
+
+function stopChild() {
+  child.kill('SIGTERM');
+  forceKillTimer = setTimeout(() => child.kill('SIGKILL'), 1_000);
+}
+
+function markRuntimeLimit() {
+  if (!runtimeLimitMarkerPath) return;
+  try {
+    fs.writeFileSync(runtimeLimitMarkerPath, 'runtime_limit');
+  } catch {}
+}
+
+function armRuntimeDeadline() {
+  if (settled || runtimeTimer) return;
+  clearTimeout(startupTimer);
+  runtimeTimer = setTimeout(() => {
+    // Close the polling race at the exact deadline: customer code may have
+    // written the completion fence after the last 25ms poll.
+    if (fs.existsSync(runtimeCompletedPath)) {
+      armTerminalGrace();
+      return;
+    }
+    timedOut = true;
+    markRuntimeLimit();
+    process.stderr.write('RUNTIME_LIMIT_EXCEEDED: external runtime watchdog stopped the play.\\n');
+    stopChild();
+  }, limitMs);
+}
+
+function armTerminalGrace() {
+  if (settled || terminalGraceTimer) return;
+  clearInterval(runtimeBoundaryPoll);
+  clearTimeout(startupTimer);
+  if (runtimeTimer) {
+    clearTimeout(runtimeTimer);
+    runtimeTimer = null;
+  }
+  terminalGraceTimer = setTimeout(() => {
+    terminalFlushTimedOut = true;
+    process.stderr.write('RUNTIME_TERMINAL_FLUSH_LIMIT_EXCEEDED: terminal settlement exceeded its grace period.\\n');
+    stopChild();
+  }, terminalGraceMs);
+}
+
+const runtimeBoundaryPoll = setInterval(() => {
+  if (fs.existsSync(runtimeStartedPath)) armRuntimeDeadline();
+  if (fs.existsSync(runtimeCompletedPath)) armTerminalGrace();
+}, 25);
+const startupTimer = setTimeout(() => {
+  // The startup marker can land between the last poll and this callback.
+  if (fs.existsSync(runtimeStartedPath)) {
+    armRuntimeDeadline();
+    return;
+  }
+  startupTimedOut = true;
+  clearInterval(runtimeBoundaryPoll);
+  process.stderr.write('RUNTIME_STARTUP_LIMIT_EXCEEDED: runner never reached the customer-code boundary.\\n');
+  stopChild();
+}, startupLimitMs);
+if (fs.existsSync(runtimeStartedPath)) armRuntimeDeadline();
+if (fs.existsSync(runtimeCompletedPath)) armTerminalGrace();
+
+function finish(code) {
+  if (settled) return;
+  settled = true;
+  clearInterval(runtimeBoundaryPoll);
+  clearTimeout(startupTimer);
+  if (runtimeTimer) clearTimeout(runtimeTimer);
+  if (terminalGraceTimer) clearTimeout(terminalGraceTimer);
+  if (forceKillTimer) clearTimeout(forceKillTimer);
+  process.exitCode = timedOut
+    ? ${DAYTONA_RUNTIME_WATCHDOG_EXIT_CODE}
+    : startupTimedOut
+      ? 1
+    : terminalFlushTimedOut
+      ? 1
+    : typeof code === 'number'
+      ? code
+      : 1;
+}
+
+child.once('error', () => finish(1));
+child.once('exit', (code) => finish(code));
+`;
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -14,6 +14,7 @@ import type {
   PlayRunnerResult,
 } from '@shared_libs/play-runtime/protocol';
 import {
+  PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
   STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
   STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
 } from '@shared_libs/play-runtime/runtime-constants';
@@ -39,6 +40,8 @@ import {
 } from './daytona-session-execution';
 
 const DAYTONA_EXECUTE_TIMEOUT_SECONDS = STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
+const DAYTONA_DETACHED_CEILING_SECONDS =
+  STANDARD_PLAY_RUNTIME_LIMIT_SECONDS + PLAY_RUNNER_TERMINAL_GRACE_SECONDS;
 const STANDARD_WORKFLOW_RUNTIME_LIMIT_ERROR = `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`;
 const DAYTONA_COMMAND_RECOVERY_TIMEOUT_MS = 60_000;
 const DAYTONA_COMMAND_RECOVERY_POLL_MS = 2_000;
@@ -209,8 +212,10 @@ export async function inspectDetachedDaytonaRunner(input: {
   diagnosis: {
     stage:
       | 'salvaged'
+      | 'command_running'
       | 'result_missing'
       | 'command_inspection_failed'
+      | 'sandbox_missing'
       | 'sandbox_lookup_failed';
     detail: string | null;
   };
@@ -220,12 +225,14 @@ export async function inspectDetachedDaytonaRunner(input: {
     const daytona = daytonaSdkClientFactory.createFull(clientOptions);
     const sandbox = (await daytona.get(input.sandboxId)) as DaytonaSandbox;
     let exitCode: number | null = null;
+    let commandFound = false;
     let commandInspectionError: string | null = null;
     try {
       const command = await sandbox.process.getSessionCommand(
         input.sessionId,
         input.cmdId,
       );
+      commandFound = Boolean(command);
       exitCode =
         typeof command?.exitCode === 'number' ? command.exitCode : null;
     } catch (error) {
@@ -252,7 +259,9 @@ export async function inspectDetachedDaytonaRunner(input: {
               stage: 'command_inspection_failed',
               detail: commandInspectionError,
             }
-          : { stage: 'result_missing', detail: null },
+          : commandFound && exitCode === null
+            ? { stage: 'command_running', detail: null }
+            : { stage: 'result_missing', detail: null },
       };
     }
     const result =
@@ -275,7 +284,13 @@ export async function inspectDetachedDaytonaRunner(input: {
     return {
       exitCode: null,
       result: null,
-      diagnosis: { stage: 'sandbox_lookup_failed', detail: failure.detail },
+      diagnosis: {
+        stage:
+          failure.httpStatus === 404
+            ? 'sandbox_missing'
+            : 'sandbox_lookup_failed',
+        detail: failure.detail,
+      },
     };
   }
 }
@@ -874,6 +889,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             cmdId: start.cmdId,
             runnerAttempt,
             ceilingSeconds: DAYTONA_EXECUTE_TIMEOUT_SECONDS,
+            terminalGraceSeconds: PLAY_RUNNER_TERMINAL_GRACE_SECONDS,
             outputPath: stagedPayload.outputPath,
             exitCodePath: stagedPayload.exitCodePath,
             elapsedMs: Date.now() - startedAt,
@@ -890,7 +906,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               outputPath: stagedPayload.outputPath,
               exitCodePath: stagedPayload.exitCodePath,
               startedAtMs: Date.now(),
-              ceilingMs: DAYTONA_EXECUTE_TIMEOUT_SECONDS * 1_000,
+              ceilingMs: DAYTONA_DETACHED_CEILING_SECONDS * 1_000,
             },
             logs: [],
             stats: {},

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts

@@ -27,6 +27,7 @@ export type RuntimeResourceTerminalReason =
   | 'runner_failed'
   | 'sandbox_missing'
   | 'sandbox_oom'
+  | 'runtime_limit'
   | 'cancelled'
   | 'lease_expired';
 

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts

@@ -1,5 +1,9 @@
 export const PLAY_RUNTIME_API_COMPAT_PATH = '/api/v2/plays/internal/runtime';
 export const PLAY_RUNTIME_API_CURRENT_PATH = '/api/v2/internal/play-runtime';
+export const PLAY_RUNTIME_OPERATION_ID_HEADER =
+  'x-deepline-runtime-operation-id';
+export const PLAY_RUNTIME_OPERATION_ATTEMPT_HEADER =
+  'x-deepline-runtime-operation-attempt';
 export const PLAY_RUNTIME_ATTEMPT_EXECUTOR_TOKEN_PATH =
   '/api/v2/internal/play-runtime/executor-token';
 export const PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH =

package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts

@@ -2,6 +2,19 @@
 export const STANDARD_PLAY_RUNTIME_LIMIT_SECONDS = 30 * 60;
 export const STANDARD_PLAY_RUNTIME_LIMIT_LABEL = '30 minutes';
 
+/**
+ * Extra time after the user-code deadline for the sandbox runner to push its
+ * terminal, flush captured stdout, and let the crash epilogue finish.
+ */
+export const PLAY_RUNNER_TERMINAL_GRACE_SECONDS = 90;
+
+/**
+ * Hard ceiling for Daytona runner startup before the customer runtime clock is
+ * armed. This covers the first durable heartbeat and the worker's reverse
+ * startup acknowledgement without charging either interval to user code.
+ */
+export const PLAY_RUNNER_STARTUP_GRACE_SECONDS = 4 * 60;
+
 /**
  * Runner timeout includes setup, cleanup, and billing headroom after the
  * user-code runtime cap.

package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts

@@ -1,6 +1,28 @@
 export const PLAY_RUNTIME_CONTRACT = 3;
 export const PLAY_RUNTIME_CONTRACT_MIN_SUPPORTED = 1;
 export const PLAY_RUNTIME_CONTRACT_HEADER = 'x-deepline-runtime-contract';
+/**
+ * Opaque, per-request correlation key generated inside the sandbox runner.
+ * It contains no customer data and lets gateway logs distinguish a request that
+ * never reached Fly from one that was accepted but failed downstream.
+ */
+export const PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER =
+  'x-deepline-runtime-transport-attempt';
+
+const RUNTIME_TRANSPORT_ATTEMPT_ID =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+/**
+ * Validate the opaque runner-generated correlation value before it appears in
+ * any gateway log or response header. It is intentionally not a credential.
+ */
+export function resolvePlayRuntimeTransportAttemptId(
+  raw: string | string[] | null | undefined,
+): string | null {
+  const value = Array.isArray(raw) ? raw[0] : raw;
+  const trimmed = value?.trim();
+  return trimmed && RUNTIME_TRANSPORT_ATTEMPT_ID.test(trimmed) ? trimmed : null;
+}
 
 export const RUNTIME_CONTRACT_CHANGELOG = [
   {