deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -92,6 +92,20 @@ export class DaytonaSandboxAcquisitionUnavailableError extends Error {
   }
 }
 
+/**
+ * A named Daytona create is still changing state after the client deadline.
+ * The scheduler must defer and resume this exact identity; falling back or
+ * creating another sandbox could execute the Play twice.
+ */
+export class DaytonaSandboxTransitionPendingError extends Error {
+  constructor(readonly sandboxName: string) {
+    super(
+      `Daytona sandbox ${sandboxName} is still transitioning after create timed out.`,
+    );
+    this.name = 'DaytonaSandboxTransitionPendingError';
+  }
+}
+
 export function resolveDaytonaSandboxAcquisitionUnavailableReason(
   errors: readonly string[],
 ): DaytonaSandboxAcquisitionUnavailableReason | null {
@@ -326,7 +340,8 @@ async function createOneShotDaytonaSandbox(input: {
 type TimedOutDaytonaSandboxReconciliation =
   | { kind: 'started'; sandbox: DaytonaSandbox }
   | { kind: 'deleted' }
-  | { kind: 'not_found' };
+  | { kind: 'not_found' }
+  | { kind: 'transition_pending' };
 
 async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
   daytona: DaytonaClient;
@@ -376,23 +391,28 @@ async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
     return { kind: 'deleted' };
   }
   if (lastDeleteConflict) {
-    throw new Error(
-      `Daytona timed-out sandbox ${input.sandboxName} remained in a state transition across the bounded cleanup window. Modal fallback suppressed.`,
-      { cause: lastDeleteConflict },
-    );
+    return { kind: 'transition_pending' };
   }
   return { kind: 'not_found' };
 }
 
+function daytonaSandboxName(providerOperationToken?: string | null): string {
+  return providerOperationToken
+    ? `dl-${providerOperationToken}`
+    : `dl-${crypto.randomUUID()}`;
+}
+
 async function createRetriedOneShotDaytonaSandbox(input: {
   daytona: DaytonaClient;
   orgId: string;
   context: DaytonaExecutionContext;
   emitStage: DaytonaStageEmitter;
   observeCreateCall?: DaytonaCreateCallObserver;
-  reserveSandboxCapacity?: (
-    providerAttempt: number,
-  ) => Promise<{ leaseId: string; providerOperationToken: string | null }>;
+  reserveSandboxCapacity?: (providerAttempt: number) => Promise<{
+    leaseId: string;
+    providerOperationToken: string | null;
+    resumeExistingProviderOperation?: boolean;
+  }>;
   releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
   nextProviderAttempt: () => number;
   startedAt: number;
@@ -400,7 +420,6 @@ async function createRetriedOneShotDaytonaSandbox(input: {
   const errors: string[] = [];
   for (const [index, delayMs] of DAYTONA_CREATE_RETRY_DELAYS_MS.entries()) {
     const attempt = input.nextProviderAttempt();
-    const sandboxName = `dl-${crypto.randomUUID()}`;
     if (delayMs > 0) {
       input.emitStage('create:retry', {
         attempt: index,
@@ -417,6 +436,40 @@ async function createRetriedOneShotDaytonaSandbox(input: {
       providerAttempt: attempt,
     });
     const reservation = await input.reserveSandboxCapacity?.(attempt);
+    const sandboxName = daytonaSandboxName(reservation?.providerOperationToken);
+    if (reservation?.resumeExistingProviderOperation) {
+      if (!input.daytona.get) {
+        throw new DaytonaSandboxTransitionPendingError(sandboxName);
+      }
+      try {
+        const sandbox = await input.daytona.get(sandboxName);
+        if (sandbox.state !== 'started') {
+          throw new DaytonaSandboxTransitionPendingError(sandboxName);
+        }
+        const acquiredAt = Date.now();
+        input.emitStage('create:resumed', {
+          attempt,
+          elapsedMs: acquiredAt - input.startedAt,
+          sandboxId: sandbox.id,
+        });
+        return {
+          sandbox,
+          attempt,
+          attemptElapsedMs: acquiredAt - attemptStartedAt,
+          sandboxCapacityLeaseId: reservation.leaseId,
+        };
+      } catch (error) {
+        if (error instanceof DaytonaSandboxTransitionPendingError) throw error;
+        if (!isDaytonaSandboxNotFound(error)) {
+          throw new DaytonaSandboxTransitionPendingError(sandboxName);
+        }
+        await input.releaseSandboxCapacity?.(reservation.leaseId);
+        throw new DaytonaSandboxAcquisitionUnavailableError(
+          'daytona_sandbox_start_timeout',
+          `Timed-out Daytona sandbox ${sandboxName} was confirmed absent before resume.`,
+        );
+      }
+    }
     let sandbox: DaytonaSandbox;
     try {
       sandbox = await createOneShotDaytonaSandbox({
@@ -496,6 +549,15 @@ async function createRetriedOneShotDaytonaSandbox(input: {
           };
         }
         await recordCreateFailure();
+        if (reconciliation.kind === 'transition_pending') {
+          if (reservation?.providerOperationToken) {
+            throw new DaytonaSandboxTransitionPendingError(sandboxName);
+          }
+          throw new Error(
+            `Daytona timed-out sandbox ${sandboxName} remained in a state transition across the bounded cleanup window. Modal fallback suppressed.`,
+            { cause: error },
+          );
+        }
         if (reconciliation.kind === 'deleted') {
           // The provider-confirmed reconciliation means this reservation can
           // no longer represent a live sandbox. Release before returning the
@@ -693,6 +755,7 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
   reserveSandboxCapacity?: (providerAttempt: number) => Promise<{
     leaseId: string;
     providerOperationToken: string | null;
+    resumeExistingProviderOperation?: boolean;
   }>;
   releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
   startAdmission?: DaytonaSandboxStartAdmission;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -26,6 +26,11 @@ import {
   RUNTIME_SANDBOX_KILLED_MESSAGE,
   RUNTIME_SANDBOX_OOM_MESSAGE,
 } from '../../run-failure';
+import {
+  buildDaytonaPayloadMaterializerSource,
+  type DaytonaPayloadEnvelope,
+} from '../../daytona-payload-contract';
+export { buildDaytonaPayloadMaterializerSource } from '../../daytona-payload-contract';
 import { compactPlayArtifactForRuntimeTransport } from '../../../plays/artifact-transport';
 import type {
   DaytonaExecutionContext,
@@ -36,16 +41,6 @@ import { buildDaytonaRuntimeWatchdogSource } from './daytona-runtime-watchdog';
 
 type DaytonaRemoteRuntimeContext = PlayRunnerExecutionConfig['context'];
 
-type DaytonaPayloadEnvelope = {
-  schemaVersion: 1;
-  runnerCode: string;
-  artifactBundledCode: string;
-  artifactSourceMap: string;
-  config: PlayRunnerExecutionConfig;
-  /** Crash-containment epilogue script (see buildDaytonaCrashTerminalPusherSource). */
-  crashPusherCode: string;
-};
-
 /**
  * The crash pusher must be able to recover one complete terminal event after
  * a runner exits before its primary gateway push. A stdout `result` event is
@@ -116,8 +111,7 @@ function nodeMaterializePayloadCommand(input: {
   artifactSourceMapPath: string;
   crashPusherPath: string;
 }): string {
-  const script =
-    "const fs=require('node:fs');const zlib=require('node:zlib');const p=JSON.parse(zlib.gunzipSync(fs.readFileSync(process.argv[1])));if(p.schemaVersion!==1)throw new Error('Unsupported Daytona payload envelope');fs.writeFileSync(process.argv[2],p.runnerCode);fs.writeFileSync(process.argv[3],JSON.stringify(p.config));fs.writeFileSync(process.argv[4],p.artifactBundledCode);fs.writeFileSync(process.argv[5],p.artifactSourceMap);fs.writeFileSync(process.argv[6],p.crashPusherCode);";
+  const script = buildDaytonaPayloadMaterializerSource();
   return [
     'node',
     '-e',

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -28,6 +28,7 @@ import {
 import {
   DAYTONA_CANCELLED_ERROR,
   DaytonaSandboxAcquisitionUnavailableError,
+  DaytonaSandboxTransitionPendingError,
   createDaytonaSandboxCleanupManager,
   createOneShotDaytonaSandboxLifecycle,
   type AcquiredDaytonaSandbox,
@@ -37,6 +38,7 @@ import {
   type OneShotDaytonaSandboxLifecycle,
   validateDaytonaExecutionContext,
 } from './daytona-lifecycle';
+import { recoveryForDaytonaSandboxTransitionPending } from '../../play-run-recovery-policy';
 import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
 import {
   DaytonaRunnerInitializationError,
@@ -69,6 +71,12 @@ const RUNTIME_API_TRANSPORT_RETRY_PATTERN =
 // with user code that has already had side effects.
 const DAYTONA_PUBLIC_STATUS_TRANSIENT_RETRY_PATTERN =
   /\bDaytonaValidationError:\s*bad request:\s*failed to get sandbox public status:\s*an unexpected error occurred\.?\b/i;
+// A detached runner sends its first authenticated heartbeat before it can run
+// customer code.  Persisted terminal results retain the bounded startup
+// diagnostic, so the worker can distinguish that proven pre-code transport
+// failure from a generic readiness timeout on a later wake.
+const DAYTONA_RUNNER_STARTUP_TRANSPORT_RETRY_PATTERN =
+  /\bDaytonaRunnerInitializationError:\s*RUNTIME_SANDBOX_START_FAILED:.*\bScheduler diagnosis:\s*runner_heartbeat_not_observed\..*"schedulerReadinessObserved":false.*"commandFound":true.*"exitCode":null.*"sessionFound":true.*"exitCodeFile":null.*"runnerStartup":\{"phase":"heartbeat_transport","outcome":"network_error","errorCode":"(?:ECONNRESET|ECONNREFUSED|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT)"/i;
 const DAYTONA_INFRASTRUCTURE_RETRY_PATTERN =
   /\b(?:Request failed with status code (?:408|429|500|502|503|504)|ECONNRESET|ECONNREFUSED|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|fetch failed|socket hang up|Daytona sandbox create failed across|Daytona sandbox create did not complete|Daytona readiness baseline)\b/i;
 
@@ -902,6 +910,7 @@ export function retryableRuntimeInitializationFailureReason(
   | 'runtime_api_transport'
   | 'runtime_receipt_gateway_transport'
   | 'daytona_public_status'
+  | 'daytona_runner_startup_transport'
   | null {
   if (result.status !== 'failed') return null;
   if (RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN.test(result.error)) {
@@ -911,6 +920,13 @@ export function retryableRuntimeInitializationFailureReason(
     return 'runtime_receipt_gateway_transport';
   }
   const rowsProcessed = Number(result.stats?.rowsProcessed ?? 0);
+  if (
+    rowsProcessed === 0 &&
+    result.steps.length === 0 &&
+    DAYTONA_RUNNER_STARTUP_TRANSPORT_RETRY_PATTERN.test(result.error)
+  ) {
+    return 'daytona_runner_startup_transport';
+  }
   if (
     rowsProcessed === 0 &&
     result.steps.length === 0 &&
@@ -1470,6 +1486,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
       if (error instanceof SandboxRunnerPreCodeDeferralError) {
         throw error;
       }
+      if (error instanceof DaytonaSandboxTransitionPendingError) {
+        throw recoveryForDaytonaSandboxTransitionPending(error);
+      }
       if (error instanceof DaytonaSandboxAcquisitionUnavailableError) {
         throw error;
       }

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/bundle.ts

@@ -206,7 +206,6 @@ export async function buildPlayRunnerBundle(): Promise<string> {
         resolve(process.cwd(), 'packages', 'play-runtime'),
         resolve(process.cwd(), 'packages', 'plays'),
         resolve(process.cwd(), 'packages', 'security'),
-        resolve(process.cwd(), 'shared_libs'),
       ];
       const sourceHash = await hashRunnerSources(sourceRoots);
       setSpanAttributes(span, {

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts

@@ -6,17 +6,17 @@ import {
   deleteModalSandboxById,
   readDetachedModalRuntimeCompletion,
 } from './backends/modal';
+import {
+  readPersistedRuntimeSandboxRef,
+  type RuntimeSandboxProviderId,
+  type RuntimeSandboxRef,
+} from '../runtime-sandbox-contract';
 import type { DetachedRunnerSuspension } from '../suspension';
 
-export type RuntimeSandboxProviderId = 'daytona' | 'modal';
-
-export type RuntimeSandboxRef = {
-  /** Frozen at 1: persisted in suspension_json, so readers must keep taking 1. */
-  schemaVersion: 1;
-  provider: RuntimeSandboxProviderId;
-  resourceId: string;
-  routingDomain: string | null;
-};
+export type {
+  RuntimeSandboxProviderId,
+  RuntimeSandboxRef,
+} from '../runtime-sandbox-contract';
 
 export type RuntimeSandboxReconciliationAdapter = {
   provider: RuntimeSandboxProviderId;
@@ -139,41 +139,11 @@ export function runtimeSandboxRefFromSuspension(
   const storedRef = (suspension as { runtimeSandboxRef?: unknown })
     .runtimeSandboxRef;
   if (storedRef !== undefined) {
-    if (
-      !storedRef ||
-      typeof storedRef !== 'object' ||
-      Array.isArray(storedRef)
-    ) {
-      throw new Error('Persisted Runtime Sandbox Ref is not an object.');
-    }
-    const candidate = storedRef as Record<string, unknown>;
-    const resourceId =
-      typeof candidate.resourceId === 'string'
-        ? candidate.resourceId.trim()
-        : '';
-    const provider = candidate.provider;
-    const routingDomain = candidate.routingDomain;
-    if (
-      candidate.schemaVersion !== 1 ||
-      (provider !== 'daytona' && provider !== 'modal') ||
-      !resourceId ||
-      resourceId !== suspension.sandboxId.trim() ||
-      (routingDomain !== null && typeof routingDomain !== 'string') ||
-      (typeof routingDomain === 'string' && !routingDomain.trim()) ||
-      (suspension.sandboxProvider !== undefined &&
-        suspension.sandboxProvider !== provider)
-    ) {
-      throw new Error(
-        `Persisted Runtime Sandbox Ref is invalid or inconsistent for sandbox ${suspension.sandboxId}.`,
-      );
-    }
-    return {
-      schemaVersion: 1,
-      provider,
-      resourceId,
-      routingDomain:
-        typeof routingDomain === 'string' ? routingDomain.trim() : null,
-    };
+    return readPersistedRuntimeSandboxRef({
+      raw: storedRef,
+      expectedSandboxId: suspension.sandboxId,
+      expectedProvider: suspension.sandboxProvider,
+    });
   }
   if (
     suspension.sandboxProvider !== undefined &&