deepline 0.3.247 → 0.3.249

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -1,4 +1,5 @@
-import { type Daytona } from '@daytonaio/sdk';
+import type { CreateSandboxFromSnapshotParams, Sandbox } from '@daytonaio/sdk';
+import { DaytonaLaunchOutcomeUnknownError } from './daytona-launch-transport';
 import type {
   PlayRunnerExecutionConfig,
   PlayRunnerResult,
@@ -40,8 +41,18 @@ const DAYTONA_NETWORK_ALLOW_LIST_ENV = 'DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST';
 
 export const DAYTONA_CANCELLED_ERROR = 'Daytona play runner cancelled';
 
-export type DaytonaClient = Pick<Daytona, 'create'> &
-  Partial<Pick<Daytona, 'get'>>;
+export type DaytonaClient = {
+  create(
+    params: CreateSandboxFromSnapshotParams,
+    options: { timeout: number },
+  ): Promise<Sandbox>;
+  get?(name: string): Promise<Sandbox>;
+  delete(sandbox: Pick<Sandbox, 'id'>, timeoutSeconds?: number): Promise<void>;
+  scope(
+    sandbox: Sandbox,
+    timeoutMs: number,
+  ): { sandbox: Sandbox; dispose(): void };
+};
 export type DaytonaSandbox = Awaited<ReturnType<DaytonaClient['create']>>;
 export type DaytonaExecutionContext = PlayRunnerExecutionConfig['context'];
 export type DaytonaStageEmitter = (
@@ -55,6 +66,7 @@ type DaytonaCreateCallObserver = (
 
 export type AcquiredDaytonaSandbox = {
   sandbox: DaytonaSandbox;
+  deleteSandbox: () => Promise<void>;
   daytonaOrganizationId: string;
   billingStartedAt: number;
   billingEndedAt?: number;
@@ -125,6 +137,25 @@ export function resolveDaytonaSandboxAcquisitionUnavailableReason(
   return null;
 }
 
+function waitForAcquisitionRetry(
+  ms: number,
+  signal?: AbortSignal,
+): Promise<void> {
+  signal?.throwIfAborted();
+  return new Promise((resolve, reject) => {
+    const abort = () => {
+      clearTimeout(timer);
+      reject(signal?.reason);
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener('abort', abort);
+      resolve();
+    }, ms);
+    signal?.addEventListener('abort', abort, { once: true });
+    if (signal?.aborted) abort();
+  });
+}
+
 function isDaytonaSandboxStartTimeout(error: string): boolean {
   return /Failed to create and start sandbox within \d+ seconds\. Operation timed out\./.test(
     error,
@@ -161,11 +192,12 @@ function daytonaCreateErrorClass(
 async function rejectAcquiredSandbox(
   sandbox: DaytonaSandbox,
   reason: string,
-  reservation?: { leaseId: string },
-  releaseSandboxCapacity?: (leaseId: string) => Promise<void>,
+  reservation: { leaseId: string } | undefined,
+  releaseSandboxCapacity: ((leaseId: string) => Promise<void>) | undefined,
+  deleteSandbox: () => Promise<void>,
 ): Promise<never> {
   try {
-    await sandbox.delete(30);
+    await deleteSandbox();
   } catch (error) {
     const cleanupError = error instanceof Error ? error.message : String(error);
     throw new Error(
@@ -346,20 +378,22 @@ type TimedOutDaytonaSandboxReconciliation =
 async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
   daytona: DaytonaClient;
   sandboxName: string;
+  cancellationSignal?: AbortSignal;
 }): Promise<TimedOutDaytonaSandboxReconciliation> {
   if (!input.daytona.get) return { kind: 'not_found' };
   let lastDeleteConflict: unknown = null;
   for (const delayMs of DAYTONA_TIMED_OUT_CREATE_RECONCILE_DELAYS_MS) {
+    input.cancellationSignal?.throwIfAborted();
     if (delayMs > 0) {
-      await new Promise((resolve) => setTimeout(resolve, delayMs));
+      await waitForAcquisitionRetry(delayMs, input.cancellationSignal);
     }
     let sandbox: DaytonaSandbox;
     try {
       sandbox = await input.daytona.get(input.sandboxName);
     } catch (error) {
-      // A confirmed absence cannot become a cleanup-addressable resource on a
-      // later read. Stop immediately; only uncertain lookup failures merit
-      // the bounded retry window below.
+      // A current absence is not proof that the sent create cannot commit.
+      // Return it to the durable operation owner without another create or
+      // releasing capacity; later recovery re-reads the same identity.
       if (isDaytonaSandboxNotFound(error)) return { kind: 'not_found' };
       continue;
     }
@@ -372,7 +406,7 @@ async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
       return { kind: 'started', sandbox };
     }
     try {
-      await sandbox.delete(30);
+      await input.daytona.delete(sandbox, 30);
     } catch (error) {
       // Daytona can make the named sandbox lookup-addressable before its
       // timed-out create transition has settled. That exact conflict is not a
@@ -416,9 +450,11 @@ async function createRetriedOneShotDaytonaSandbox(input: {
   releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
   nextProviderAttempt: () => number;
   startedAt: number;
+  cancellationSignal?: AbortSignal;
 }): Promise<DaytonaCreateResult> {
   const errors: string[] = [];
   for (const [index, delayMs] of DAYTONA_CREATE_RETRY_DELAYS_MS.entries()) {
+    input.cancellationSignal?.throwIfAborted();
     const attempt = input.nextProviderAttempt();
     if (delayMs > 0) {
       input.emitStage('create:retry', {
@@ -427,7 +463,7 @@ async function createRetriedOneShotDaytonaSandbox(input: {
         reason: 'bounded_backoff',
         delayMs,
       });
-      await new Promise((resolve) => setTimeout(resolve, delayMs));
+      await waitForAcquisitionRetry(delayMs, input.cancellationSignal);
     }
     const attemptStartedAt = Date.now();
     await input.observeCreateCall?.({
@@ -463,11 +499,9 @@ async function createRetriedOneShotDaytonaSandbox(input: {
         if (!isDaytonaSandboxNotFound(error)) {
           throw new DaytonaSandboxTransitionPendingError(sandboxName);
         }
-        await input.releaseSandboxCapacity?.(reservation.leaseId);
-        throw new DaytonaSandboxAcquisitionUnavailableError(
-          'daytona_sandbox_start_timeout',
-          `Timed-out Daytona sandbox ${sandboxName} was confirmed absent before resume.`,
-        );
+        // A lookup 404 does not prove that an earlier sent create cannot still
+        // commit. Keep the durable identity reserved; never hedge its outcome.
+        throw new DaytonaSandboxTransitionPendingError(sandboxName);
       }
     }
     let sandbox: DaytonaSandbox;
@@ -507,12 +541,16 @@ async function createRetriedOneShotDaytonaSandbox(input: {
       });
       const immediateFallbackReason =
         resolveDaytonaSandboxAcquisitionUnavailableReason([message]);
-      if (isDaytonaSandboxStartTimeout(message)) {
+      if (
+        error instanceof DaytonaLaunchOutcomeUnknownError ||
+        isDaytonaSandboxStartTimeout(message)
+      ) {
         let reconciliation: TimedOutDaytonaSandboxReconciliation;
         try {
           reconciliation = await reconcileAndDeleteTimedOutDaytonaSandbox({
             daytona: input.daytona,
             sandboxName,
+            cancellationSignal: input.cancellationSignal,
           });
         } catch (reconciliationError) {
           await recordCreateFailure();
@@ -571,6 +609,9 @@ async function createRetriedOneShotDaytonaSandbox(input: {
             `${message} Timed-out Daytona sandbox was reconciled and deleted before fallback.`,
           );
         }
+        if (reservation?.providerOperationToken) {
+          throw new DaytonaSandboxTransitionPendingError(sandboxName);
+        }
         throw new Error(
           `${message} Daytona did not return a cleanup-addressable sandbox identity; Modal fallback suppressed. The named ephemeral sandbox retains Daytona's auto-delete-on-stop backstop.`,
           { cause: error },
@@ -645,6 +686,7 @@ async function acquireOneShotDaytonaSandbox(input: {
   nextProviderAttempt: () => number;
   startedAt: number;
   startAdmission?: DaytonaSandboxStartAdmission;
+  cancellationSignal?: AbortSignal;
 }): Promise<AcquiredDaytonaSandbox> {
   const limits = validatePlaySandboxRuntimeLimits(
     input.context.sandboxRuntimeLimits ?? {
@@ -653,8 +695,9 @@ async function acquireOneShotDaytonaSandbox(input: {
   );
   input.emitStage('create:start');
   const result = input.startAdmission
-    ? await input.startAdmission.run(() =>
-        createRetriedOneShotDaytonaSandbox(input),
+    ? await input.startAdmission.run(
+        () => createRetriedOneShotDaytonaSandbox(input),
+        input.cancellationSignal,
       )
     : await createRetriedOneShotDaytonaSandbox(input);
   // The runner's egress probe measures the sandbox-to-estate round trip; the
@@ -682,6 +725,7 @@ async function acquireOneShotDaytonaSandbox(input: {
         ? { leaseId: result.sandboxCapacityLeaseId }
         : undefined,
       input.releaseSandboxCapacity,
+      () => input.daytona.delete(result.sandbox, 30),
     );
   }
   const configuredOrganizationId =
@@ -699,6 +743,7 @@ async function acquireOneShotDaytonaSandbox(input: {
         ? { leaseId: result.sandboxCapacityLeaseId }
         : undefined,
       input.releaseSandboxCapacity,
+      () => input.daytona.delete(result.sandbox, 30),
     );
   }
   let lookupOrganizationId: string | null = null;
@@ -725,6 +770,7 @@ async function acquireOneShotDaytonaSandbox(input: {
         ? { leaseId: result.sandboxCapacityLeaseId }
         : undefined,
       input.releaseSandboxCapacity,
+      () => input.daytona.delete(result.sandbox, 30),
     );
   }
   const billingStartedAt = Date.now();
@@ -745,6 +791,7 @@ async function acquireOneShotDaytonaSandbox(input: {
   });
   return {
     sandbox,
+    deleteSandbox: () => input.daytona.delete(sandbox, 30),
     daytonaOrganizationId,
     billingStartedAt,
     ...(result.sandboxCapacityLeaseId
@@ -765,6 +812,7 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
   }>;
   releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
   startAdmission?: DaytonaSandboxStartAdmission;
+  cancellationSignal?: AbortSignal;
   startedAt?: number;
 }): OneShotDaytonaSandboxLifecycle {
   const orgId = validateDaytonaExecutionContext(input.context);
@@ -783,6 +831,7 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
       reserveSandboxCapacity: input.reserveSandboxCapacity,
       releaseSandboxCapacity: input.releaseSandboxCapacity,
       startAdmission: input.startAdmission,
+      cancellationSignal: input.cancellationSignal,
       nextProviderAttempt: () => {
         providerAttempt += 1;
         return providerAttempt;
@@ -812,7 +861,7 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
       disposed = true;
       try {
         const acquired = await latestAcquiredSandboxPromise;
-        await acquired.sandbox.delete(30);
+        await acquired.deleteSandbox();
         // `dispose` has provider-confirmed that the sandbox is gone. This is
         // normally the narrow cancellation window before the scheduler has
         // durably recorded the resource, so no cleanup job will release an
@@ -846,7 +895,7 @@ export function createDaytonaSandboxCleanupManager(): {
   activate(acquired: AcquiredDaytonaSandbox): void;
   currentSandbox(): DaytonaSandbox | null;
   cleanupActiveSandboxForCancellation(): boolean;
-  stashActiveSandboxForRetry(): void;
+  stashActiveSandboxForRetry(): Promise<void>;
   withCleanup(
     result: PlayRunnerResult,
     input: { cancellationCleanupStarted: boolean },
@@ -855,6 +904,11 @@ export function createDaytonaSandboxCleanupManager(): {
   let sandbox: DaytonaSandbox | null = null;
   let sandboxBillingStartedAt: number | null = null;
   const deletedSandboxIds = new Set<string>();
+  let deleteActive: (() => Promise<void>) | null = null;
+  let retirement: Promise<void> | null = null;
+  // Cancellation, terminal cleanup, and retry retirement share the same
+  // provider operation. A rejected/ambiguous delete still blocks replacement.
+  const retireActive = () => (retirement ??= deleteActive!());
 
   const deleteSandboxOnce = (input: {
     reason: string;
@@ -866,7 +920,7 @@ export function createDaytonaSandboxCleanupManager(): {
     }
     deletedSandboxIds.add(target.id);
     const billingStartedAt = sandboxBillingStartedAt;
-    void target.delete(30).then(
+    void retireActive().then(
       () => {
         console.info('[play-runner.daytona.cleanup_done]', {
           reason: input.reason,
@@ -889,6 +943,8 @@ export function createDaytonaSandboxCleanupManager(): {
   return {
     activate(acquired) {
       sandbox = acquired.sandbox;
+      deleteActive = acquired.deleteSandbox;
+      retirement = null;
       sandboxBillingStartedAt = acquired.billingStartedAt;
     },
     currentSandbox() {
@@ -900,14 +956,15 @@ export function createDaytonaSandboxCleanupManager(): {
         sandboxToDelete: sandbox,
       });
     },
-    stashActiveSandboxForRetry() {
-      // The retired sandbox is dead — a fresh one replaces it for the retry.
-      // Delete NOW instead of deferring to a consumer that never existed.
-      deleteSandboxOnce({
-        reason: 'retired_for_retry',
-        sandboxToDelete: sandbox,
-      });
+    async stashActiveSandboxForRetry() {
+      if (!sandbox || !deleteActive) return;
+      // A response lost after command acceptance can leave a runner alive.
+      // No replacement may start until provider-confirmed retirement.
+      const target = sandbox;
+      await retireActive();
+      deletedSandboxIds.add(target.id);
       sandbox = null;
+      deleteActive = null;
       sandboxBillingStartedAt = null;
     },
     withCleanup(result, input) {

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts

@@ -358,6 +358,7 @@ export function prepareDetachedDaytonaRunner(input: {
   sessionId: string;
   readiness: DetachedRunnerReadinessPort;
   cancellation?: Promise<never>;
+  runControl<T>(operation: (sandbox: DaytonaSandbox) => Promise<T>): Promise<T>;
   /** Invoked immediately after Daytona accepts the detached command. */
   onCommandAccepted?: (
     state: Extract<DetachedDaytonaRunnerState, { phase: 'command_accepted' }>,
@@ -368,8 +369,8 @@ export function prepareDetachedDaytonaRunner(input: {
   ) => Promise<void> | void;
 }): DetachedDaytonaRunnerSupervisor {
   const cancellation = input.cancellation ?? new Promise<never>(() => {});
-  const sessionCreation = input.sandbox.process
-    .createSession(input.sessionId)
+  const sessionCreation = input
+    .runControl((sandbox) => sandbox.process.createSession(input.sessionId))
     .then(
       () => ({ ok: true as const }),
       (error: unknown) => ({ ok: false as const, error }),
@@ -399,10 +400,12 @@ export function prepareDetachedDaytonaRunner(input: {
       const baseline = await Promise.race([readinessBaseline, cancellation]);
       if (baseline.ok === false) throw baseline.error;
       const response = await Promise.race([
-        input.sandbox.process.executeSessionCommand(input.sessionId, {
-          command,
-          runAsync: true,
-        }),
+        input.runControl((sandbox) =>
+          sandbox.process.executeSessionCommand(input.sessionId, {
+            command,
+            runAsync: true,
+          }),
+        ),
         cancellation,
       ]);
       const cmdId = response?.cmdId;
@@ -427,6 +430,7 @@ export function prepareDetachedDaytonaRunner(input: {
           startupDiagnosticPath,
           readiness: input.readiness,
           baselineHeartbeatAt: state.baselineHeartbeatAt,
+          runControl: input.runControl,
         }),
         cancellation,
       ]);
@@ -452,6 +456,9 @@ export async function confirmDetachedDaytonaRunnerReady(input: {
   startupDiagnosticPath?: string;
   readiness: DetachedRunnerReadinessPort;
   baselineHeartbeatAt: string | null;
+  runControl?: <T>(
+    operation: (sandbox: DaytonaSandbox) => Promise<T>,
+  ) => Promise<T>;
   timeoutMs?: number;
   pollMs?: number;
   now?: () => number;
@@ -488,13 +495,17 @@ export async function confirmDetachedDaytonaRunnerReady(input: {
     retryDelayMs = Math.min(1_000, retryDelayMs * 2);
   }
 
-  const diagnostic = await inspectDetachedDaytonaStartup({
-    sandbox: input.sandbox,
-    sessionId: input.sessionId,
-    cmdId: input.cmdId,
-    exitCodePath: input.exitCodePath,
-    startupDiagnosticPath: input.startupDiagnosticPath,
-  });
+  const inspect = (sandbox: DaytonaSandbox) =>
+    inspectDetachedDaytonaStartup({
+      sandbox,
+      sessionId: input.sessionId,
+      cmdId: input.cmdId,
+      exitCodePath: input.exitCodePath,
+      startupDiagnosticPath: input.startupDiagnosticPath,
+    });
+  const diagnostic = await (input.runControl
+    ? input.runControl(inspect)
+    : inspect(input.sandbox));
   diagnostic.schedulerReadinessObserved = false;
   diagnostic.schedulerReadinessResponses = schedulerReadinessResponses;
   const oomIndicated =

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-start-admission.ts

@@ -1,7 +1,7 @@
 import { RUNTIME_CAPACITY_POLICY } from '../../runtime-capacity-policy';
 
 export type DaytonaSandboxStartAdmission = {
-  run<T>(operation: () => Promise<T>): Promise<T>;
+  run<T>(operation: () => Promise<T>, signal?: AbortSignal): Promise<T>;
 };
 
 /**
@@ -13,9 +13,8 @@ export type DaytonaSandboxStartAdmission = {
  * is still transitioning, recreating the unsafe overlap this gate prevents.
  */
 export function createDaytonaSandboxStartAdmission(
-  concurrency =
-    RUNTIME_CAPACITY_POLICY.sandboxAcquisition
-      .daytonaConcurrentStartsPerProcess,
+  concurrency = RUNTIME_CAPACITY_POLICY.sandboxAcquisition
+    .daytonaConcurrentStartsPerProcess,
 ): DaytonaSandboxStartAdmission {
   if (!Number.isInteger(concurrency) || concurrency <= 0) {
     throw new Error(
@@ -43,7 +42,11 @@ export function createDaytonaSandboxStartAdmission(
   };
 
   return {
-    async run<T>(operation: () => Promise<T>): Promise<T> {
+    async run<T>(
+      operation: () => Promise<T>,
+      signal?: AbortSignal,
+    ): Promise<T> {
+      signal?.throwIfAborted();
       // Preserve eager prepare semantics when capacity is free: the provider
       // create begins in this turn, just as it did before admission existed.
       // Only contended callers cross an asynchronous queue boundary.
@@ -53,8 +56,22 @@ export function createDaytonaSandboxStartAdmission(
               active += 1;
               return createRelease();
             })()
-          : await new Promise<() => void>((resolve) => waiters.push(resolve));
+          : await new Promise<() => void>((resolve, reject) => {
+              const accept = (release: () => void) => {
+                signal?.removeEventListener('abort', abort);
+                resolve(release);
+              };
+              const abort = () => {
+                const index = waiters.indexOf(accept);
+                if (index >= 0) waiters.splice(index, 1);
+                reject(signal?.reason);
+              };
+              waiters.push(accept);
+              signal?.addEventListener('abort', abort, { once: true });
+              if (signal?.aborted) abort();
+            });
       try {
+        signal?.throwIfAborted();
         return await operation();
       } finally {
         release();