deepline 0.1.213 → 0.1.215

package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts

@@ -38,7 +38,7 @@ import {
   PLAY_RUNTIME_CONTRACT,
   PLAY_RUNTIME_CONTRACT_HEADER,
 } from '@shared_libs/play-runtime/runtime-contract';
-import { PLAY_RUNTIME_API_COMPAT_PATH } from '@shared_libs/play-runtime/runtime-api-paths';
+import { PLAY_RUNTIME_API_CURRENT_PATH } from '@shared_libs/play-runtime/runtime-api-paths';
 import { PLAY_RUNTIME_TEST_FAULT_HEADER } from '@shared_libs/play-runtime/test-runtime-seams';
 import { vercelProtectionBypassHeaders } from '@shared_libs/play-runtime/vercel-protection';
 
@@ -569,6 +569,19 @@ function summarizeAppRuntimeErrorBody(body: string): string {
   return trimmed.length > 500 ? `${trimmed.slice(0, 500)}...` : trimmed;
 }
 
+function appRuntimeErrorCode(response: Response, body: string): string | null {
+  const header = response.headers.get('x-deepline-error-code')?.trim();
+  if (header) return header;
+  try {
+    const parsed = JSON.parse(body) as { code?: unknown };
+    return typeof parsed?.code === 'string' && parsed.code.trim()
+      ? parsed.code.trim()
+      : null;
+  } catch {
+    return null;
+  }
+}
+
 export function isTransientAppRuntimeFailureBody(body: string): boolean {
   return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated|RUNTIME_SCHEDULER_SATURATED|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open|OptimisticConcurrencyControlFailure|changed while this mutation was being run|Documents read from or written to the .* table changed while this mutation/i.test(
     body,
@@ -775,7 +788,7 @@ export class AppRuntimeApiTransportError extends Error {
 
 function resolveAppRuntimeApiUrl(context: WorkerRuntimeApiContext): string {
   const baseUrl = context.baseUrl.trim().replace(/\/$/, '');
-  return `${baseUrl}${PLAY_RUNTIME_API_COMPAT_PATH}`;
+  return `${baseUrl}${PLAY_RUNTIME_API_CURRENT_PATH}`;
 }
 
 async function postAppRuntimeApi<TResponse>(
@@ -892,8 +905,13 @@ async function postAppRuntimeApi<TResponse>(
       );
       continue;
     }
+    const code = appRuntimeErrorCode(response, responseText);
+    const requestId = response.headers.get('x-deepline-request-id')?.trim();
     throw new Error(
-      `App runtime API ${body.action} failed with status ${response.status}: ${summarizeAppRuntimeErrorBody(responseText)}`,
+      `App runtime API ${body.action} failed with status ${response.status}` +
+        `${code ? ` code=${code}` : ''}` +
+        `${requestId ? ` request_id=${requestId}` : ''}: ` +
+        summarizeAppRuntimeErrorBody(responseText),
     );
   }
 

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -48,8 +48,8 @@ import {
   SYNTHETIC_RUN_HEADER,
 } from './coordinator-headers';
 import {
-  PLAY_RUNTIME_API_COMPAT_PATH,
-  PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
+  PLAY_RUNTIME_API_CURRENT_PATH,
+  PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_CURRENT_PATH,
 } from './runtime-api-paths';
 import {
   createRootRunExecutionScope,
@@ -284,6 +284,11 @@ const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
 const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
 const FETCH_TRANSPORT_MAX_ATTEMPTS = 3;
 const FETCH_TRANSPORT_RETRY_DELAY_MS = 100;
+// cloudflared returns this branded HTML when its connection to the local app
+// resets. It is a transport failure before the app can reach a provider, not a
+// provider 502; require both the branded page and our development tunnel host.
+const DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN =
+  /\bdeeplinedeveloper\.com\s*\|\s*502\s*:\s*bad gateway\b/i;
 const CHILD_PLAY_LAUNCH_WRITE_CONFLICT_RETRY_DELAYS_MS = [
   50, 100, 200, 400, 800,
 ] as const;
@@ -518,6 +523,28 @@ export function resolveToolRuntimeTimeoutMs(
     : undefined;
 }
 
+function isDeeplineDeveloperTunnelOrigin502(input: {
+  url: string;
+  status: number;
+  bodyText: string;
+}): boolean {
+  if (
+    input.status !== 502 ||
+    !DEEPLINE_DEVELOPER_TUNNEL_ORIGIN_502_PATTERN.test(input.bodyText)
+  ) {
+    return false;
+  }
+  try {
+    const hostname = new URL(input.url).hostname.toLowerCase();
+    return (
+      hostname === 'deeplinedeveloper.com' ||
+      hostname.endsWith('.deeplinedeveloper.com')
+    );
+  } catch {
+    return false;
+  }
+}
+
 function loadSafeFetch(): Promise<SafeFetchModule> {
   safeFetchModule ??= import('@shared_libs/security/safe-fetch');
   return safeFetchModule;
@@ -1675,7 +1702,7 @@ export class PlayContextImpl {
     | null
   > {
     const response = await fetch(
-      `${this.runtimeApiBaseUrl()}${PLAY_RUNTIME_API_COMPAT_PATH}`,
+      `${this.runtimeApiBaseUrl()}${PLAY_RUNTIME_API_CURRENT_PATH}`,
       {
         method: 'POST',
         headers: await this.runtimeApiHeaders(),
@@ -1790,7 +1817,7 @@ export class PlayContextImpl {
       this.#options.runId
     ) {
       const response = await fetch(
-        `${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`,
+        `${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_CURRENT_PATH}`,
         {
           method: 'POST',
           headers: {
@@ -6680,7 +6707,7 @@ export class PlayContextImpl {
           : inlineChildGovernor!.snapshot();
         const childPlaySlot = launchThroughScheduler
           ? await this.governor.acquireChildSubmitSlot()
-          : null;
+          : await this.governor.acquireInlineChildSlot();
         const rowStore = rowContext.getStore();
         const producer = {
           kind: 'play' as const,
@@ -6937,7 +6964,7 @@ export class PlayContextImpl {
       return this.#options.executorToken;
     }
     const response = await fetch(
-      `${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH}`,
+      `${this.#options.baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_CURRENT_PATH}`,
       {
         method: 'POST',
         headers: {
@@ -8529,6 +8556,32 @@ export class PlayContextImpl {
           const retrySafeTransientHttp =
             retryPolicy?.retrySafeTransientHttp === true;
           let transportAttempt = 0;
+          const retryToolTransportFailure = async (
+            message: string,
+          ): Promise<void> => {
+            transportAttempt += 1;
+            if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
+              await this.governor.chargeBudget('retry');
+              const retryAfterMs =
+                TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
+              span.setAttribute(
+                'plays.transport_retry_attempt',
+                transportAttempt,
+              );
+              this.log(
+                `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
+              );
+              await options?.parkProviderCall?.();
+              await this.sleepWithCheckpointHeartbeat(retryAfterMs);
+              return;
+            }
+            throw new ToolHttpError(
+              `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
+              null,
+              0,
+              'repairable',
+            );
+          };
 
           while (true) {
             let response: Response;
@@ -8696,32 +8749,12 @@ export class PlayContextImpl {
                       `Tool ${toolId} runtime API call timed out after ${timeoutMs}ms.`,
                     )
                 : error;
-              transportAttempt += 1;
               const message =
                 transportError instanceof Error
                   ? transportError.message
                   : String(transportError);
-              if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
-                await this.governor.chargeBudget('retry');
-                const retryAfterMs =
-                  TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
-                span.setAttribute(
-                  'plays.transport_retry_attempt',
-                  transportAttempt,
-                );
-                this.log(
-                  `Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
-                );
-                await options?.parkProviderCall?.();
-                await this.sleepWithCheckpointHeartbeat(retryAfterMs);
-                continue;
-              }
-              throw new ToolHttpError(
-                `Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
-                null,
-                0,
-                'repairable',
-              );
+              await retryToolTransportFailure(message);
+              continue;
             } finally {
               if (timeoutHandle) {
                 clearTimeout(timeoutHandle);
@@ -8732,6 +8765,18 @@ export class PlayContextImpl {
 
             if (!response.ok) {
               const text = await response.text();
+              if (
+                isDeeplineDeveloperTunnelOrigin502({
+                  url,
+                  status: response.status,
+                  bodyText: text,
+                })
+              ) {
+                await retryToolTransportFailure(
+                  'the Deepline development tunnel returned a branded 502 before reaching the app origin',
+                );
+                continue;
+              }
               const authScopeChanged = parseToolExecuteAuthScopeChangedError({
                 status: response.status,
                 bodyText: text,

package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts

@@ -429,7 +429,11 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     receipt: RuntimeStepReceipt,
     source: DurableReceiptRecoverySource = 'cache',
   ): Promise<T> => {
-    input.log(`ctx.${input.operation}(${input.id}): reused completed work`);
+    input.log(
+      source === 'owner'
+        ? `ctx.${input.operation}(${input.id}): executed and converged through immutable publication`
+        : `ctx.${input.operation}(${input.id}): reused completed work`,
+    );
     if (receipt.output === undefined) {
       return receipt.output as T;
     }
@@ -489,8 +493,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
       const afterLock = await input.store.get(input.receiptKey);
       if (
         input.force !== true &&
-        (afterLock?.status === 'completed' ||
-          afterLock?.status === 'skipped')
+        (afterLock?.status === 'completed' || afterLock?.status === 'skipped')
       ) {
         await input.store.releaseExecutionLock({
           receiptKey: input.receiptKey,
@@ -566,6 +569,10 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
         completed?.status === 'completed' ||
         completed?.status === 'skipped'
       ) {
+        // Completion is a compact acknowledgement. This execution already
+        // owns the just-produced value; only a later receipt read needs the
+        // persisted payload for replay.
+        if (completed.output === undefined) return result;
         return await recoverCompletedReceipt(completed, 'owner');
       }
       const winner = await input.store.get(input.receiptKey);
@@ -828,6 +835,10 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
     (completed.status === 'completed' || completed.status === 'skipped')
   ) {
     logPhase('total', lifecycleStartedAt);
+    // Completion endpoints acknowledge ownership and status only. The caller
+    // already owns the live value; replay reads are the only path that needs
+    // the receipt payload returned over the transport.
+    if (completed.output === undefined) return result;
     return await recoverCompletedReceipt(completed, 'owner');
   }
   if (input.store.canPersistCompletion) {

package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts

@@ -106,6 +106,13 @@ export interface PlayExecutionGovernor {
   acquireRowSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
   /** Block until a child-play submit slot is free. Released after submit, not terminal. */
   acquireChildSubmitSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
+  /**
+   * Block until an in-process child-composition slot is free. Unlike a
+   * scheduled child's submit slot, this lease is held through child terminal
+   * execution so a large row fan-out cannot materialize thousands of child
+   * contexts and receipt closures at once.
+   */
+  acquireInlineChildSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
   /**
    * Block until a global tool-concurrency slot AND the per-(org,provider) pacer
    * permit are free, then charge the tool-call budget and return a lease. Order:
@@ -397,6 +404,7 @@ export function createPlayExecutionGovernor(
 
     acquireRowSlot: (opts) => rowSlots.acquire(opts?.signal),
     acquireChildSubmitSlot: (opts) => childPlaySlots.acquire(opts?.signal),
+    acquireInlineChildSlot: (opts) => childPlaySlots.acquire(opts?.signal),
 
     async acquireToolSlot(toolId, opts) {
       // 1. global tool-concurrency slot.
@@ -612,6 +620,17 @@ function createInlineChildGovernor(
     policy: root.policy,
     acquireRowSlot: (opts) => root.acquireRowSlot(opts),
     acquireChildSubmitSlot: (opts) => root.acquireChildSubmitSlot(opts),
+    async acquireInlineChildSlot(opts) {
+      if (opts?.signal?.aborted) {
+        throw opts.signal.reason instanceof Error
+          ? opts.signal.reason
+          : new Error('Slot acquire aborted.');
+      }
+      // This governor already runs inside a root inline-child lease. Descendant
+      // composition shares that admitted subtree slot; reacquiring the root
+      // semaphore here can deadlock when every admitted parent awaits a child.
+      return { release: () => {} };
+    },
     acquireToolSlot: (toolId, opts) => root.acquireToolSlot(toolId, opts),
     suggestedParallelism: (toolId, fallback) =>
       root.suggestedParallelism(toolId, fallback),

package/dist/bundling-sources/shared_libs/play-runtime/profiles.ts

@@ -6,6 +6,7 @@ import {
   PLAY_RUNTIME_PROVIDERS,
   PLAY_RUNTIME_PROVIDER_IDS,
   defaultPlayRuntimeProvider,
+  resolveEnabledPlayRuntimeProvider,
   resolvePlayRuntimeProvider,
   type PlayRuntimeProviderId,
 } from './providers';
@@ -50,3 +51,24 @@ export function resolveExecutionProfile(
     throw error;
   }
 }
+
+export function resolveEnabledExecutionProfile(
+  override?: string | null,
+): PlayExecutionProfile {
+  try {
+    return resolveEnabledPlayRuntimeProvider(override);
+  } catch (error) {
+    if (
+      override?.trim() &&
+      error instanceof Error &&
+      /Unsupported play runtime provider/.test(error.message)
+    ) {
+      throw new Error(
+        `Unknown execution profile "${override.trim()}". Expected one of: ${Object.keys(
+          PLAY_EXECUTION_PROFILES,
+        ).join(', ')}.`,
+      );
+    }
+    throw error;
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/providers.ts

@@ -27,6 +27,22 @@ export type PlayRuntimeProviderDescriptor = {
   label: string;
 };
 
+export const PLAY_RUNTIME_PROVIDER_DISABLED_CODE =
+  'PLAY_RUNTIME_PROVIDER_DISABLED' as const;
+
+export class PlayRuntimeProviderDisabledError extends Error {
+  readonly code = PLAY_RUNTIME_PROVIDER_DISABLED_CODE;
+  readonly profile: PlayRuntimeProviderId;
+
+  constructor(profile: PlayRuntimeProviderId) {
+    super(
+      `Play runtime profile "${profile}" is disabled. Deepline runs now execute on the "absurd" profile.`,
+    );
+    this.name = 'PlayRuntimeProviderDisabledError';
+    this.profile = profile;
+  }
+}
+
 export const PLAY_RUNTIME_PROVIDERS: Record<
   PlayRuntimeProviderId,
   PlayRuntimeProviderDescriptor
@@ -54,7 +70,15 @@ export const PLAY_RUNTIME_PROVIDERS: Record<
 };
 
 export function defaultPlayRuntimeProvider(): PlayRuntimeProviderDescriptor {
-  return PLAY_RUNTIME_PROVIDERS.workers_edge;
+  return PLAY_RUNTIME_PROVIDERS.absurd;
+}
+
+export function assertPlayRuntimeProviderEnabled(
+  provider: PlayRuntimeProviderDescriptor,
+): void {
+  if (provider.id === PLAY_RUNTIME_PROVIDER_IDS.workersEdge) {
+    throw new PlayRuntimeProviderDisabledError(provider.id);
+  }
 }
 
 export function resolvePlayRuntimeProvider(
@@ -73,3 +97,11 @@ export function resolvePlayRuntimeProvider(
   }
   return defaultPlayRuntimeProvider();
 }
+
+export function resolveEnabledPlayRuntimeProvider(
+  override?: string | null,
+): PlayRuntimeProviderDescriptor {
+  const provider = resolvePlayRuntimeProvider(override);
+  assertPlayRuntimeProviderEnabled(provider);
+  return provider;
+}

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -2,6 +2,8 @@ const CLOUDFLARE_DURABLE_OBJECT_RESET_RE =
   /Durable Object.*(?:code (?:was|has been) updated|storage caused object)/;
 const CLOUDFLARE_WORKER_SUBREQUEST_LIMIT_RE =
   /Too many subrequests by single Worker invocation/i;
+const CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE =
+  /The requested endpoint could not be found, or you don't have access to it\. Please check the provided ID and try again\./i;
 const VERCEL_RUNTIME_API_DEPLOYMENT_MISSING_RE =
   /runtime API 404:[\s\S]*(?:DeploymentNotFound|DEPLOYMENT_NOT_FOUND|requested deployment .*exist|deployment could not be found on Vercel)/i;
 
@@ -9,6 +11,8 @@ export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
   'Run interrupted by a platform deploy. Deepline retries this automatically when possible; if this error is still visible, re-run the same command.';
 export const PLATFORM_WORKER_SUBREQUEST_INTERRUPTED_MESSAGE =
   'Worker subrequest limit hit. Deepline retries automatically; if still visible, re-run.';
+export const PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE =
+  'Cloudflare workflow control was temporarily unavailable. Deepline retries automatically; if still visible, re-run.';
 
 export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
   'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.';
@@ -171,6 +175,15 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
       cause,
     };
   }
+  if (CLOUDFLARE_WORKFLOW_ENDPOINT_UNAVAILABLE_RE.test(cause)) {
+    return {
+      code: 'PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED',
+      phase: 'runtime',
+      message: PLATFORM_WORKFLOW_ENDPOINT_INTERRUPTED_MESSAGE,
+      retryable: true,
+      cause,
+    };
+  }
   const playDepthBudgetMatch = cause.match(
     /Play execution playDepth budget exceeded \((\d+)\/(\d+)\)\./,
   );

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -27,6 +27,7 @@ type DaytonaPayloadEnvelope = {
 export type StagedDaytonaPayload = {
   workDir: string;
   command: string;
+  readyPath: string;
   outputPath: string;
   exitCodePath: string;
   progressEventPath: string;
@@ -343,6 +344,7 @@ export async function stageDaytonaRunnerPayload(input: {
 
   const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
   const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
+  const readyPath = `${workDir}/deepline-play-ready-${randomUUID()}.json`;
   const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
   const runnerTraceEnv =
     process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
@@ -354,17 +356,18 @@ export async function stageDaytonaRunnerPayload(input: {
     configPath,
     artifactCodePath,
     crashPusherPath,
-  })} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
+  })} && DEEPLINE_PLAY_RUNNER_READY_PATH=${shellQuote(readyPath)} DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
   // Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
   // pushes the parsed (or synthesized) terminal to the gateway so the parked
   // worker wakes within seconds of ANY runner death — process.exit abuse, OOM
   // SIGKILL, or a crash before the in-process push. Idempotent against a
   // successful in-runner push (terminals are first-write-wins).
-  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
+  const command = `rm -f ${shellQuote(outputPath)} ${shellQuote(exitCodePath)} ${shellQuote(readyPath)} ${shellQuote(progressEventPath)} ${shellQuote(`${progressEventPath}.*`)}; ( ${runnerCommand} ) > ${shellQuote(outputPath)} 2>&1; code=$?; printf '%s' "$code" > ${shellQuote(exitCodePath)}; node ${shellQuote(crashPusherPath)} ${shellQuote(configPath)} "$code" ${shellQuote(outputPath)} >> ${shellQuote(outputPath)} 2>&1 || true; printf 'deepline runner output captured: %s\\n' ${shellQuote(outputPath)}; exit "$code"`;
 
   return {
     workDir: input.workDir,
     command,
+    readyPath,
     outputPath,
     exitCodePath,
     progressEventPath,

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts

@@ -112,6 +112,16 @@ export type DetachedDaytonaCommandStart = {
   cmdId: string;
 };
 
+const DAYTONA_RUNNER_READY_TIMEOUT_MS = 3_000;
+const DAYTONA_RUNNER_READY_POLL_MS = 250;
+
+export class DaytonaRunnerInitializationError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'DaytonaRunnerInitializationError';
+  }
+}
+
 /**
  * Start `command` detached in a fresh session. Returns the session + command id
  * immediately (no await on completion).
@@ -137,3 +147,59 @@ export async function startDetachedDaytonaCommand(input: {
   }
   return { sessionId: input.sessionId, cmdId };
 }
+
+/**
+ * A detached command id only proves that Daytona accepted the request. Before
+ * the worker parks, require a marker written by the initialized runner process.
+ * This closes the unobservable handoff where a sandbox or command disappears
+ * after executeSessionCommand but before the first gateway heartbeat.
+ */
+export async function confirmDetachedDaytonaRunnerReady(input: {
+  sandbox: DaytonaSandbox;
+  sessionId: string;
+  cmdId: string;
+  readyPath: string;
+  exitCodePath: string;
+  timeoutMs?: number;
+  pollMs?: number;
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+}): Promise<void> {
+  const now = input.now ?? Date.now;
+  const sleep =
+    input.sleep ??
+    ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  const timeoutMs = Math.max(
+    1,
+    input.timeoutMs ?? DAYTONA_RUNNER_READY_TIMEOUT_MS,
+  );
+  const pollMs = Math.max(1, input.pollMs ?? DAYTONA_RUNNER_READY_POLL_MS);
+  const deadline = now() + timeoutMs;
+  let markerDiagnosis = 'ready_marker_unavailable';
+
+  while (now() < deadline) {
+    try {
+      const marker = await input.sandbox.fs.downloadFile(input.readyPath, 2);
+      const parsed = JSON.parse(marker.toString('utf-8')) as {
+        ready?: unknown;
+      };
+      if (parsed.ready === true) return;
+      markerDiagnosis = 'ready_marker_invalid';
+    } catch {
+      markerDiagnosis = 'ready_marker_unavailable';
+    }
+    await sleep(Math.min(pollMs, Math.max(1, deadline - now())));
+  }
+
+  const diagnostic = await inspectDetachedDaytonaStartup({
+    sandbox: input.sandbox,
+    sessionId: input.sessionId,
+    cmdId: input.cmdId,
+    exitCodePath: input.exitCodePath,
+  });
+  const oomIndicated =
+    diagnostic.exitCode === 137 || diagnostic.exitCodeFile === 137;
+  throw new DaytonaRunnerInitializationError(
+    `RUNTIME_SANDBOX_START_FAILED: Daytona accepted detached command ${input.cmdId} in sandbox ${input.sandbox.id}, but the play runner did not initialize within ${timeoutMs}ms. Marker diagnosis: ${markerDiagnosis}. Daytona diagnosis: ${JSON.stringify(diagnostic)}. OOM is ${oomIndicated ? 'indicated by exit code 137' : 'not confirmed by the available Daytona evidence'}. The play was stopped and was not retried.`,
+  );
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -33,7 +33,8 @@ import {
 } from './daytona-lifecycle';
 import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
 import {
-  inspectDetachedDaytonaStartup,
+  confirmDetachedDaytonaRunnerReady,
+  DaytonaRunnerInitializationError,
   startDetachedDaytonaCommand,
 } from './daytona-session-execution';
 
@@ -45,8 +46,6 @@ const DAYTONA_RUNTIME_INITIALIZATION_MAX_ATTEMPTS = 2;
 const DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS = 2;
 const DAYTONA_UPLOAD_MAX_ATTEMPTS = 2;
 const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
-const DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS = 15_000;
-const DAYTONA_STARTUP_TRACE_ENV = 'DEEPLINE_DAYTONA_STARTUP_TRACE';
 
 const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
   /\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
@@ -397,6 +396,7 @@ export function retryableRuntimeInitializationFailureReason(
 }
 
 function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
+  if (error instanceof DaytonaRunnerInitializationError) return false;
   const message = error instanceof Error ? error.message : String(error);
   if (!message || /cancelled|aborted/i.test(message)) {
     return false;
@@ -715,12 +715,11 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
           });
           // Push execution (B2-final, park-and-wake): start the runner command
           // DETACHED via a Daytona session and return a `detached_runner`
-          // suspension immediately — the worker parks on it (releasing its
-          // claim, freeing the slot) and wakes on the runner's pushed terminal
-          // or the ceiling timeout. Nothing here awaits completion: no held
-          // socket, no poll loop, no per-second sandbox file reads. The sandbox
-          // MUST outlive this return (the runner is executing inside it); the
-          // wake leg owns finalize + sandbox GC.
+          // suspension after the runner writes its one-time ready marker. The
+          // marker is the handoff proof: a Daytona cmdId alone does not prove
+          // the sandbox process initialized. After that bounded startup check,
+          // the worker parks (releasing its claim and slot) and wakes only on
+          // the runner's pushed terminal or the ceiling timeout.
           const sessionId = `deepline-play-${randomUUID()}`;
           let start: Awaited<ReturnType<typeof startDetachedDaytonaCommand>>;
           try {
@@ -744,6 +743,16 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             // classification below decide fresh-sandbox retry vs loud failure.
             throw error;
           }
+          await Promise.race([
+            confirmDetachedDaytonaRunnerReady({
+              sandbox,
+              sessionId: start.sessionId,
+              cmdId: start.cmdId,
+              readyPath: stagedPayload.readyPath,
+              exitCodePath: stagedPayload.exitCodePath,
+            }),
+            cancellationPromise,
+          ]);
           const runnerAttempt = Math.max(
             0,
             Math.floor(config.context.runAttempt ?? 0),
@@ -758,52 +767,6 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             exitCodePath: stagedPayload.exitCodePath,
             elapsedMs: Date.now() - startedAt,
           });
-          // Daytona acknowledges an async command with only a cmdId. It does
-          // not expose a queued/running state, and the backend parks before the
-          // runner contacts the runtime sheet. During startup investigations,
-          // retain a bounded snapshot while the sandbox is still alive; the
-          // normal cancellation path otherwise deletes the only evidence.
-          // This is opt-in because it adds Daytona control-plane calls to every
-          // run. The diagnostic excludes command lines and environment values.
-          if (process.env[DAYTONA_STARTUP_TRACE_ENV] === '1') {
-            const diagnosticTimer = setTimeout(() => {
-              void inspectDetachedDaytonaStartup({
-                sandbox,
-                sessionId: start.sessionId,
-                cmdId: start.cmdId,
-                exitCodePath: stagedPayload.exitCodePath,
-              }).then(
-                (diagnostic) =>
-                  emitDaytonaStage(
-                    callbacks,
-                    config.context,
-                    'execute:startup_diagnostic',
-                    {
-                      sandboxId: sandbox.id,
-                      sessionId: start.sessionId,
-                      cmdId: start.cmdId,
-                      diagnostic,
-                      elapsedMs: Date.now() - startedAt,
-                    },
-                  ),
-                (error: unknown) =>
-                  emitDaytonaStage(
-                    callbacks,
-                    config.context,
-                    'execute:startup_diagnostic_failed',
-                    {
-                      sandboxId: sandbox.id,
-                      sessionId: start.sessionId,
-                      cmdId: start.cmdId,
-                      errorType:
-                        error instanceof Error ? error.name : 'unknown',
-                      elapsedMs: Date.now() - startedAt,
-                    },
-                  ),
-              );
-            }, DAYTONA_STARTUP_DIAGNOSTIC_DELAY_MS);
-            diagnosticTimer.unref?.();
-          }
           return {
             status: 'suspended',
             suspension: {