deepline 0.1.211 → 0.1.213

package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts

@@ -30,7 +30,10 @@ export type RuntimeResourceSnapshot = {
   };
   sheetFlush: {
     acquired: number;
+    active: number;
+    waiting: number;
     bytes: number;
+    admissionWaitMsEwma: number | null;
     latencyMsEwma: number | null;
   };
   providers: Record<
@@ -90,8 +93,74 @@ function createEwma(alpha = 0.2): Ewma {
   };
 }
 
-function noOpLease(): RuntimeResourceLease {
-  return { release() {} };
+// Sheet persistence has its own local admission lane. Receipt lifecycle calls
+// are admitted by the receipt sink/gateway and must not queue behind a large
+// Runtime Sheet flush in the runner. Two flushes keep the database busy while
+// bounding resident payloads and gateway/pool pressure (ADR 0012).
+const MAX_CONCURRENT_SHEET_FLUSHES = 2;
+
+type SheetFlushWaiter = {
+  resolve: (lease: RuntimeResourceLease) => void;
+  reject: (error: unknown) => void;
+  signal?: AbortSignal;
+  onAbort?: () => void;
+};
+
+class SheetFlushAdmission {
+  #active = 0;
+  #waiters: SheetFlushWaiter[] = [];
+
+  get active(): number {
+    return this.#active;
+  }
+
+  get waiting(): number {
+    return this.#waiters.length;
+  }
+
+  acquire(signal?: AbortSignal): Promise<RuntimeResourceLease> {
+    if (signal?.aborted) {
+      return Promise.reject(signal.reason ?? new Error('Sheet flush aborted'));
+    }
+    if (this.#active < MAX_CONCURRENT_SHEET_FLUSHES) {
+      this.#active += 1;
+      return Promise.resolve(this.#lease());
+    }
+    return new Promise((resolve, reject) => {
+      const waiter: SheetFlushWaiter = {
+        resolve,
+        reject,
+        ...(signal ? { signal } : {}),
+      };
+      if (signal) {
+        waiter.onAbort = () => {
+          const index = this.#waiters.indexOf(waiter);
+          if (index >= 0) this.#waiters.splice(index, 1);
+          reject(signal.reason ?? new Error('Sheet flush aborted'));
+        };
+        signal.addEventListener('abort', waiter.onAbort, { once: true });
+      }
+      this.#waiters.push(waiter);
+    });
+  }
+
+  #lease(): RuntimeResourceLease {
+    let released = false;
+    return {
+      release: () => {
+        if (released) return;
+        released = true;
+        while (this.#waiters.length > 0) {
+          const waiter = this.#waiters.shift()!;
+          waiter.signal?.removeEventListener('abort', waiter.onAbort!);
+          if (waiter.signal?.aborted) continue;
+          waiter.resolve(this.#lease());
+          return;
+        }
+        this.#active = Math.max(0, this.#active - 1);
+      },
+    };
+  }
 }
 
 function elapsedSince(startedAt: number): number {
@@ -109,6 +178,8 @@ export function createRuntimeResourceGovernor(input: {
   const rowAdmissionWait = createEwma();
   const toolAdmissionWait = createEwma();
   const sheetFlushLatency = createEwma();
+  const sheetFlushAdmissionWait = createEwma();
+  const sheetFlushAdmission = new SheetFlushAdmission();
   const providers = new Map<
     string,
     {
@@ -219,11 +290,14 @@ export function createRuntimeResourceGovernor(input: {
     },
 
     async acquireSheetFlush(sheetInput) {
+      const startedAt = Date.now();
+      const lease = await sheetFlushAdmission.acquire(sheetInput?.signal);
       sheetFlushCounters.acquired += 1;
+      sheetFlushAdmissionWait.observe(elapsedSince(startedAt));
       observe({
         sheetFlushBytes: sheetInput?.estimatedBytes ?? null,
       });
-      return noOpLease();
+      return lease;
     },
 
     suggestedToolParallelism(toolId, fallback) {
@@ -258,7 +332,10 @@ export function createRuntimeResourceGovernor(input: {
         },
         sheetFlush: {
           acquired: sheetFlushCounters.acquired,
+          active: sheetFlushAdmission.active,
+          waiting: sheetFlushAdmission.waiting,
           bytes: sheetFlushCounters.bytes,
+          admissionWaitMsEwma: sheetFlushAdmissionWait.value,
           latencyMsEwma: sheetFlushLatency.value,
         },
         providers: Object.fromEntries(

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -7,6 +7,7 @@ import {
   nextPlayRunLedgerTerminalSource,
   normalizePlayRunLedgerTerminalSource,
 } from './run-terminal-source';
+import { MAX_LEDGER_LOG_LINES_PER_EVENT } from './ledger-safe-payload';
 
 export type PlayRunLedgerStatus =
   | 'queued'
@@ -84,6 +85,18 @@ export type PlayRunLedgerStepSnapshot = {
   progress?: PlayRunLedgerStepProgress | null;
 };
 
+export type PlayRunLedgerDatasetSnapshot = {
+  datasetId: string;
+  path: string;
+  tableNamespace: string;
+  phase: 'registered' | 'available' | 'failed';
+  persistedRows: number;
+  succeededRows: number;
+  failedRows: number;
+  complete: boolean;
+  updatedAt: number;
+};
+
 export type PlayRunLedgerSnapshot = {
   runId: string;
   playName?: string | null;
@@ -96,6 +109,8 @@ export type PlayRunLedgerSnapshot = {
   durationMs?: number | null;
   orderedStepIds: string[];
   stepsById: Record<string, PlayRunLedgerStepSnapshot>;
+  orderedDatasetIds: string[];
+  datasetsById: Record<string, PlayRunLedgerDatasetSnapshot>;
   /**
    * Bounded tail of the run's log lines (last {@link LOG_TAIL_LIMIT}).
    * Full retention lives in the Run Log Stream (Convex `playRunLogChunks`);
@@ -217,10 +232,34 @@ export type PlayRunLedgerEvent =
        * repeated identical lines while absorbing redundant re-sends.
        */
       channelOffset?: number;
+      /** Scheduler attempt that owns this positional log channel. */
+      producerAttempt?: number;
+      /**
+       * Terminal transport replay normally matches ctx.log lines after
+       * removing timestamps. Detached-attempt recovery uses exact matching so
+       * the same message from a later attempt remains a distinct occurrence.
+       */
+      transportDedupe?: 'semantic' | 'exact';
       /** Set by ingestion when this append crossed the retention cap. */
       logsTruncated?: boolean;
     })
   | (PlayRunLedgerBaseEvent & {
+      type: 'dataset.lifecycle';
+      datasetId: string;
+      path: string;
+      tableNamespace: string;
+      phase: 'registered' | 'available' | 'failed';
+      persistedRows?: number;
+      succeededRows?: number;
+      failedRows?: number;
+      complete?: boolean;
+    })
+  | (PlayRunLedgerBaseEvent & {
+      /**
+       * @deprecated Mixed-version deploy seam only. Owner: Play Runtime.
+       * Remove after 2026-07-19, once pre-dataset-lifecycle workers have
+       * exceeded the maximum run duration plus the seven-day drain window.
+       */
       type: 'sheet.summary.updated';
       tableNamespace: string;
       deltaCursor?: number;
@@ -241,6 +280,7 @@ export type PlayRunLedgerStatusPatch = {
    * (see `slicePositionalLogLines`), forwarded onto the `log.appended` event.
    */
   liveLogsChannelOffset?: number | null;
+  liveLogsProducerAttempt?: number | null;
   liveNodeProgress?: unknown;
   result?: unknown;
 };
@@ -401,6 +441,8 @@ export function createEmptyPlayRunLedgerSnapshot(input: {
         : null,
     orderedStepIds: [],
     stepsById: {},
+    orderedDatasetIds: [],
+    datasetsById: {},
     logTail: [],
     totalLogCount: 0,
     logsTruncated: false,
@@ -447,6 +489,26 @@ export function normalizePlayRunLedgerSnapshot(
       progress: rawProgress ? normalizeStepProgress(rawProgress) : null,
     };
   }
+  const rawDatasets = isRecord(value.datasetsById) ? value.datasetsById : {};
+  const datasetsById: Record<string, PlayRunLedgerDatasetSnapshot> = {};
+  for (const [datasetId, rawDataset] of Object.entries(rawDatasets)) {
+    if (!datasetId.trim() || !isRecord(rawDataset)) continue;
+    const phase =
+      rawDataset.phase === 'available' || rawDataset.phase === 'failed'
+        ? rawDataset.phase
+        : 'registered';
+    datasetsById[datasetId] = {
+      datasetId,
+      path: optionalString(rawDataset.path) ?? `datasets.${datasetId}`,
+      tableNamespace: optionalString(rawDataset.tableNamespace) ?? datasetId,
+      phase,
+      persistedRows: Math.max(0, finiteNumber(rawDataset.persistedRows) ?? 0),
+      succeededRows: Math.max(0, finiteNumber(rawDataset.succeededRows) ?? 0),
+      failedRows: Math.max(0, finiteNumber(rawDataset.failedRows) ?? 0),
+      complete: rawDataset.complete === true,
+      updatedAt: finiteNumber(rawDataset.updatedAt) ?? 0,
+    };
+  }
   const createdAt = finiteNumber(value.createdAt) ?? fallback.createdAt ?? null;
   const startedAt = finiteNumber(value.startedAt) ?? fallback.startedAt ?? null;
   const finishedAt =
@@ -484,6 +546,14 @@ export function normalizePlayRunLedgerSnapshot(
         : finiteNumber(value.durationMs),
     orderedStepIds: orderedStepIds.filter((stepId) => stepsById[stepId]),
     stepsById,
+    orderedDatasetIds: (Array.isArray(value.orderedDatasetIds)
+      ? value.orderedDatasetIds
+      : Object.keys(datasetsById)
+    ).filter(
+      (datasetId): datasetId is string =>
+        typeof datasetId === 'string' && Boolean(datasetsById[datasetId]),
+    ),
+    datasetsById,
     logTail: logTail.slice(-LOG_TAIL_LIMIT),
     // Snapshots persisted before totalLogCount existed only know the retained
     // tail, so the best lower bound for the cumulative count is the tail size.
@@ -1269,6 +1339,34 @@ export function reducePlayRunLedgerEvent(
             : null,
       });
     }
+    case 'dataset.lifecycle': {
+      const current = base.datasetsById[event.datasetId];
+      const next: PlayRunLedgerDatasetSnapshot = {
+        datasetId: event.datasetId,
+        path: event.path,
+        tableNamespace: event.tableNamespace,
+        phase: event.phase,
+        persistedRows: Math.max(
+          current?.persistedRows ?? 0,
+          event.persistedRows ?? 0,
+        ),
+        succeededRows: Math.max(
+          current?.succeededRows ?? 0,
+          event.succeededRows ?? 0,
+        ),
+        failedRows: Math.max(current?.failedRows ?? 0, event.failedRows ?? 0),
+        complete: current?.complete === true || event.complete === true,
+        updatedAt: occurredAt,
+      };
+      return withTiming({
+        ...base,
+        orderedDatasetIds: current
+          ? base.orderedDatasetIds
+          : [...base.orderedDatasetIds, event.datasetId],
+        datasetsById: { ...base.datasetsById, [event.datasetId]: next },
+        resultTableNamespace: base.resultTableNamespace ?? event.tableNamespace,
+      });
+    }
     case 'sheet.summary.updated':
       return withTiming({
         ...base,
@@ -1376,6 +1474,8 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
   source?: PlayRunLedgerEventSource;
   occurredAt?: number;
   channelOffset?: number | null;
+  producerAttempt?: number | null;
+  transportDedupe?: 'semantic' | 'exact';
   includeLogAppend?: boolean;
 }): PlayRunLedgerEvent[] {
   const source = input.source ?? 'worker';
@@ -1392,16 +1492,31 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
   if (newLines.length === 0 || input.includeLogAppend === false) {
     return [];
   }
-  return [
-    {
+  const events: PlayRunLedgerEvent[] = [];
+  for (
+    let index = 0;
+    index < newLines.length;
+    index += MAX_LEDGER_LOG_LINES_PER_EVENT
+  ) {
+    events.push({
       type: 'log.appended',
       runId: input.runId,
       source,
       occurredAt,
-      lines: newLines,
-      ...(positional ? { channelOffset: input.channelOffset! } : {}),
-    },
-  ];
+      lines: newLines.slice(index, index + MAX_LEDGER_LOG_LINES_PER_EVENT),
+      ...(positional ? { channelOffset: input.channelOffset! + index } : {}),
+      ...(positional &&
+      typeof input.producerAttempt === 'number' &&
+      Number.isFinite(input.producerAttempt) &&
+      input.producerAttempt >= 0
+        ? { producerAttempt: Math.trunc(input.producerAttempt) }
+        : {}),
+      ...(input.transportDedupe
+        ? { transportDedupe: input.transportDedupe }
+        : {}),
+    });
+  }
+  return events;
 }
 
 /**
@@ -1419,6 +1534,8 @@ export function buildTerminalLogReplayEvents(input: {
   source?: PlayRunLedgerEventSource;
   occurredAt?: number;
   liveLogTotalCount?: number;
+  dedupeMode?: 'semantic' | 'exact';
+  producerAttempt?: number;
 }): PlayRunLedgerEvent[] {
   const channelOffset = terminalLogReplayChannelOffset({
     liveLogTotalCount: input.liveLogTotalCount,
@@ -1430,6 +1547,10 @@ export function buildTerminalLogReplayEvents(input: {
     source: 'coordinator',
     occurredAt: input.occurredAt,
     channelOffset,
+    producerAttempt: input.producerAttempt,
+    transportDedupe:
+      input.dedupeMode ??
+      (typeof input.producerAttempt === 'number' ? 'exact' : undefined),
   });
 }
 
@@ -1466,6 +1587,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
         source,
         occurredAt: checkpointAt,
         channelOffset: patch.liveLogsChannelOffset ?? null,
+        producerAttempt: patch.liveLogsProducerAttempt ?? null,
       })
     : [];
   const earliestStartedAt =

package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts

@@ -4,6 +4,7 @@ import {
   normalizePlayRunLedgerSnapshot,
   summarizeRunRowOutcomes,
   type PlayRunLedgerSnapshot,
+  type PlayRunLedgerDatasetSnapshot,
   type PlayRunLedgerStepSnapshot,
   type PlayRunRowOutcomeSummary,
 } from './run-ledger';
@@ -76,6 +77,7 @@ export type PlayRunLiveSnapshot = {
   logsTruncated?: boolean;
   activeArtifactTableNamespace: string | null;
   resultTableNamespace: string | null;
+  datasets?: PlayRunLedgerDatasetSnapshot[];
   nodeStates: PlayRunStreamNodeState[];
   activeNodeId: string | null;
   /**
@@ -212,6 +214,11 @@ function buildSnapshotFromLedger(
     ...(snapshot.logsTruncated ? { logsTruncated: true } : {}),
     activeArtifactTableNamespace: snapshot.activeArtifactTableNamespace ?? null,
     resultTableNamespace: snapshot.resultTableNamespace ?? null,
+    datasets: snapshot.orderedDatasetIds
+      .map((datasetId) => snapshot.datasetsById[datasetId])
+      .filter((dataset): dataset is PlayRunLedgerDatasetSnapshot =>
+        Boolean(dataset),
+      ),
     nodeStates,
     activeNodeId: snapshot.activeStepId ?? null,
     ...(rowOutcomes ? { rowOutcomes } : {}),

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts

@@ -1,5 +1,99 @@
 import type { DaytonaSandbox } from './daytona-lifecycle';
 
+export type DetachedDaytonaStartupDiagnostic = {
+  commandFound: boolean;
+  exitCode: number | null;
+  sessionFound: boolean;
+  sessionCommandCount: number | null;
+  exitCodeFile: number | null;
+  processSummary: {
+    total: number;
+    node: number;
+    states: Record<string, number>;
+  } | null;
+  errors: string[];
+};
+
+function summarizeProcesses(value: unknown): {
+  total: number;
+  node: number;
+  states: Record<string, number>;
+} | null {
+  if (typeof value !== 'string' || !value.trim()) return null;
+  const rows = value
+    .split('\n')
+    .map((line) => line.trim().split(/\s+/, 2))
+    .filter(([state, command]) => Boolean(state && command));
+  const states: Record<string, number> = {};
+  let node = 0;
+  for (const [state, command] of rows) {
+    const normalizedState = state![0]?.toUpperCase() || '?';
+    states[normalizedState] = (states[normalizedState] ?? 0) + 1;
+    if (/^node(?:-|$)/i.test(command!)) node += 1;
+  }
+  return { total: rows.length, node, states };
+}
+
+/**
+ * Inspect an accepted detached command without exposing its command line or
+ * environment. Daytona's command model has no queued/running state, so the
+ * combination of session membership, a numeric exit marker, and a
+ * aggregate process counts are the narrowest useful startup diagnostic.
+ * Process names are reduced to a Node count before logging; runner logs,
+ * output, and arbitrary process names are deliberately excluded because no
+ * heuristic redactor can guarantee removal of arbitrary customer secrets.
+ */
+export async function inspectDetachedDaytonaStartup(input: {
+  sandbox: DaytonaSandbox;
+  sessionId: string;
+  cmdId: string;
+  exitCodePath: string;
+}): Promise<DetachedDaytonaStartupDiagnostic> {
+  const errors: string[] = [];
+  const diagnostic: DetachedDaytonaStartupDiagnostic = {
+    commandFound: false,
+    exitCode: null,
+    sessionFound: false,
+    sessionCommandCount: null,
+    exitCodeFile: null,
+    processSummary: null,
+    errors,
+  };
+
+  await Promise.all([
+    input.sandbox.process
+      .getSessionCommand(input.sessionId, input.cmdId)
+      .then((command) => {
+        diagnostic.commandFound = true;
+        diagnostic.exitCode =
+          typeof command.exitCode === 'number' ? command.exitCode : null;
+      })
+      .catch(() => errors.push('get_session_command_failed')),
+    input.sandbox.process
+      .getSession(input.sessionId)
+      .then((session) => {
+        diagnostic.sessionFound = true;
+        diagnostic.sessionCommandCount = session.commands.length;
+      })
+      .catch(() => errors.push('get_session_failed')),
+    input.sandbox.fs
+      .downloadFile(input.exitCodePath, 5)
+      .then((file) => {
+        const value = Number.parseInt(file.toString('utf-8').trim(), 10);
+        diagnostic.exitCodeFile = Number.isFinite(value) ? value : null;
+      })
+      .catch(() => errors.push('exit_code_file_unavailable')),
+    input.sandbox.process
+      .executeCommand('ps -eo stat=,comm=', undefined, {}, 5)
+      .then((result) => {
+        diagnostic.processSummary = summarizeProcesses(result.result);
+      })
+      .catch(() => errors.push('process_snapshot_failed')),
+  ]);
+
+  return diagnostic;
+}
+
 /**
  * Detached Daytona command start (push-execution B2-final).
  *

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -32,7 +32,10 @@ import {
   validateDaytonaExecutionContext,
 } from './daytona-lifecycle';
 import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
-import { startDetachedDaytonaCommand } from './daytona-session-execution';
+import {
+  inspectDetachedDaytonaStartup,
+  startDetachedDaytonaCommand,
+} from './daytona-session-execution';
 
 const DAYTONA_EXECUTE_TIMEOUT_SECONDS = STANDARD_PLAY_RUNTIME_LIMIT_SECONDS;
 const STANDARD_WORKFLOW_RUNTIME_LIMIT_ERROR = `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`;
@@ -42,6 +45,8 @@ const DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS = 2;
 const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
 const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
 const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
+const DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS = 15_000;
+const DAYTONA_STARTUP_TRACE_ENV = 'DEEPLINE_DAYTONA_STARTUP_TRACE';
 
 const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
   /\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
@@ -122,19 +127,34 @@ export async function inspectDetachedDaytonaRunner(input: {
   cmdId: string;
   outputPath: string;
   exitCodePath: string;
-}): Promise<{ exitCode: number | null; result: PlayRunnerResult | null }> {
+}): Promise<{
+  exitCode: number | null;
+  result: PlayRunnerResult | null;
+  diagnosis: {
+    stage:
+      | 'salvaged'
+      | 'result_missing'
+      | 'command_inspection_failed'
+      | 'sandbox_lookup_failed';
+    detail: string | null;
+  };
+}> {
   try {
     const { clientOptions } = loadDaytonaRequiredConfig();
     const daytona = daytonaSdkClientFactory.createFull(clientOptions);
     const sandbox = (await daytona.get(input.sandboxId)) as DaytonaSandbox;
     let exitCode: number | null = null;
+    let commandInspectionError: string | null = null;
     try {
       const command = await sandbox.process.getSessionCommand(
         input.sessionId,
         input.cmdId,
       );
-      exitCode = typeof command?.exitCode === 'number' ? command.exitCode : null;
+      exitCode =
+        typeof command?.exitCode === 'number' ? command.exitCode : null;
     } catch (error) {
+      commandInspectionError =
+        error instanceof Error ? error.message : String(error);
       console.warn('[play-runner.daytona.detached_inspect_command_failed]', {
         sandboxId: input.sandboxId,
         cmdId: input.cmdId,
@@ -148,7 +168,16 @@ export async function inspectDetachedDaytonaRunner(input: {
       timeoutMs: 10_000,
     });
     if (!recovered.execution) {
-      return { exitCode, result: null };
+      return {
+        exitCode,
+        result: null,
+        diagnosis: commandInspectionError
+          ? {
+              stage: 'command_inspection_failed',
+              detail: commandInspectionError,
+            }
+          : { stage: 'result_missing', detail: null },
+      };
     }
     const result =
       findPlayRunnerResult(parsePlayRunnerEvents(recovered.execution.result)) ??
@@ -156,14 +185,22 @@ export async function inspectDetachedDaytonaRunner(input: {
     return {
       exitCode: exitCode ?? recovered.execution.exitCode,
       result,
+      diagnosis: result
+        ? { stage: 'salvaged', detail: null }
+        : { stage: 'result_missing', detail: null },
     };
   } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
     console.warn('[play-runner.daytona.detached_inspect_failed]', {
       sandboxId: input.sandboxId,
       cmdId: input.cmdId,
-      error: error instanceof Error ? error.message : String(error),
+      error: detail,
     });
-    return { exitCode: null, result: null };
+    return {
+      exitCode: null,
+      result: null,
+      diagnosis: { stage: 'sandbox_lookup_failed', detail },
+    };
   }
 }
 
@@ -717,8 +754,56 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             cmdId: start.cmdId,
             runnerAttempt,
             ceilingSeconds: DAYTONA_EXECUTE_TIMEOUT_SECONDS,
+            outputPath: stagedPayload.outputPath,
+            exitCodePath: stagedPayload.exitCodePath,
             elapsedMs: Date.now() - startedAt,
           });
+          // Daytona acknowledges an async command with only a cmdId. It does
+          // not expose a queued/running state, and the backend parks before the
+          // runner contacts the runtime sheet. During startup investigations,
+          // retain a bounded snapshot while the sandbox is still alive; the
+          // normal cancellation path otherwise deletes the only evidence.
+          // This is opt-in because it adds Daytona control-plane calls to every
+          // run. The diagnostic excludes command lines and environment values.
+          if (process.env[DAYTONA_STARTUP_TRACE_ENV] === '1') {
+            const diagnosticTimer = setTimeout(() => {
+              void inspectDetachedDaytonaStartup({
+                sandbox,
+                sessionId: start.sessionId,
+                cmdId: start.cmdId,
+                exitCodePath: stagedPayload.exitCodePath,
+              }).then(
+                (diagnostic) =>
+                  emitDaytonaStage(
+                    callbacks,
+                    config.context,
+                    'execute:startup_diagnostic',
+                    {
+                      sandboxId: sandbox.id,
+                      sessionId: start.sessionId,
+                      cmdId: start.cmdId,
+                      diagnostic,
+                      elapsedMs: Date.now() - startedAt,
+                    },
+                  ),
+                (error: unknown) =>
+                  emitDaytonaStage(
+                    callbacks,
+                    config.context,
+                    'execute:startup_diagnostic_failed',
+                    {
+                      sandboxId: sandbox.id,
+                      sessionId: start.sessionId,
+                      cmdId: start.cmdId,
+                      errorType:
+                        error instanceof Error ? error.name : 'unknown',
+                      elapsedMs: Date.now() - startedAt,
+                    },
+                  ),
+              );
+            }, DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS);
+            diagnosticTimer.unref?.();
+          }
           return {
             status: 'suspended',
             suspension: {

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts

@@ -9,16 +9,25 @@ import type {
   PlayRunnerResult,
 } from '@shared_libs/play-runtime/protocol';
 
-export type PlayRunnerRuntimeResource =
-  | {
-      kind: 'daytona_sandbox';
-      sandboxId: string;
-      billingStartedAt: number;
-      billingEndedAt?: number | null;
-      cpu?: number | null;
-      memoryGiB?: number | null;
-      diskGiB?: number | null;
-    };
+export type PlayRunnerRuntimeResource = {
+  kind: 'daytona_sandbox';
+  sandboxId: string;
+  billingStartedAt: number;
+  billingEndedAt?: number | null;
+  terminalReason?: RuntimeResourceTerminalReason | null;
+  terminalAt?: number | null;
+  cpu?: number | null;
+  memoryGiB?: number | null;
+  diskGiB?: number | null;
+};
+
+export type RuntimeResourceTerminalReason =
+  | 'completed'
+  | 'runner_failed'
+  | 'sandbox_missing'
+  | 'sandbox_oom'
+  | 'cancelled'
+  | 'lease_expired';
 
 export class RuntimeResourceFenceLostError extends Error {
   constructor(message: string) {