deepline 0.3.453 → 0.3.454

package/dist/bundling-sources/sdk/src/release.ts

@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.453',
+  version: '0.3.454',
   updateSummary: {
     introducedIn: '0.3.90',
     summary:

package/dist/bundling-sources/sdk/src/types.ts

@@ -882,6 +882,8 @@ export interface PlayRunPackage {
     startedAt?: number | null;
     finishedAt?: number | null;
     durationMs?: number | null;
+    /** The provider that ran the sandbox, known once the run has ended. */
+    sandboxProvider?: 'daytona' | 'modal';
     outcome?: PlayRunOutcome;
     recovery?: {
       mode: 'replayed' | 'forced' | 'recovered' | 'joined';

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -309,6 +309,9 @@ export interface RunnerPushExecutionConfig {
   leaseSeconds: number;
   /** The run the runner heartbeats. Must match the executor token's run scope. */
   runId: string;
+  /** The provider that acquired this sandbox, which a Daytona-selected launch
+   * can differ from after fallback. Absent from older workers. */
+  sandboxProvider?: 'daytona' | 'modal';
 }
 
 export interface PlayRunnerExecutionConfig {

package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts

@@ -4076,6 +4076,12 @@ async function main() {
     (
       result as PlayRunnerResult & { workCompletedAtMs?: number }
     ).workCompletedAtMs = workCompletedAtMs;
+    if (runnerTerminalPushContext?.sandboxProvider) {
+      result.stats = {
+        ...result.stats,
+        sandboxProvider: runnerTerminalPushContext.sandboxProvider,
+      };
+    }
     const terminalResult = runnerTerminalPushContext
       ? runnerTerminalResultForTransport(result)
       : null;
@@ -4260,6 +4266,7 @@ let runnerTerminalPushContext: {
   gatewayBaseUrl: string;
   executorToken: string;
   runId: string;
+  sandboxProvider?: 'daytona' | 'modal';
 } | null = null;
 
 function terminalPushContextFromConfig(
@@ -4271,7 +4278,12 @@ function terminalPushContextFromConfig(
   if (!push || !gatewayBaseUrl || !executorToken) {
     return null;
   }
-  return { gatewayBaseUrl, executorToken, runId: push.runId };
+  return {
+    gatewayBaseUrl,
+    executorToken,
+    runId: push.runId,
+    ...(push.sandboxProvider ? { sandboxProvider: push.sandboxProvider } : {}),
+  };
 }
 
 async function pushRunnerTerminalIfConfigured(

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts

@@ -16,7 +16,9 @@ import {
 } from '@daytonaio/sdk';
 import axios, { type InternalAxiosRequestConfig } from 'axios';
 import { Readable } from 'node:stream';
+import { setTimeout as delay } from 'node:timers/promises';
 import type { DaytonaClientOptions } from '../../daytona-runtime-config';
+import type { DaytonaDeleteRetryAuthorization } from './daytona-lifecycle';
 import {
   DAYTONA_CONTROL_TIMEOUT_MS,
   DaytonaControlTimeoutError,
@@ -83,6 +85,10 @@ const DAYTONA_SDK_ERROR_NAMES: ReadonlySet<string> = new Set([
 ]);
 const DAYTONA_PROVIDER_ERROR_CODE_PATTERN = /^[A-Za-z][A-Za-z0-9_.-]{1,63}$/;
 const DAYTONA_PROVIDER_DETAIL_PATTERN = / \[provider [^\]]*\]/g;
+// A pass costs two provider calls and one fenced scheduler write, so a sandbox
+// stuck in transition must not repeat it every 100 ms for the whole phase.
+const DAYTONA_RETIREMENT_BACKOFF_INITIAL_MS = 100;
+const DAYTONA_RETIREMENT_BACKOFF_MAX_MS = 5_000;
 
 /** A provider state from Daytona's published enum, or `unrecognized`. */
 export function daytonaSandboxStateForLog(state: unknown): string | null {
@@ -599,10 +605,15 @@ export function createDaytonaLaunchClient(
         phase.dispose();
       }
     },
-    async delete(sandbox: Pick<Sandbox, 'id'>, timeoutSeconds = 30) {
+    async delete(
+      sandbox: Pick<Sandbox, 'id'>,
+      timeoutSeconds = 30,
+      retry?: DaytonaDeleteRetryAuthorization,
+    ) {
       // Cleanup must survive execution cancellation and must not trigger the
       // SDK's unawaited refreshDataSafe after a confirmed delete response.
       const phase = deadline(timeoutSeconds * 1_000);
+      let absenceOnlyConflict: DaytonaConflictError | null = null;
       try {
         const api = new SandboxApi(
           new Configuration({
@@ -610,10 +621,78 @@ export function createDaytonaLaunchClient(
             baseOptions: { ...configuration.baseOptions, signal: phase.signal },
           }),
         );
-        await api.deleteSandbox(sandbox.id, undefined, {
-          timeout: timeoutSeconds * 1_000,
-        });
+        let backoffMs = DAYTONA_RETIREMENT_BACKOFF_INITIAL_MS;
+        for (;;) {
+          phase.signal.throwIfAborted();
+          if (!absenceOnlyConflict) {
+            try {
+              await api.deleteSandbox(sandbox.id, undefined, {
+                timeout: timeoutSeconds * 1_000,
+              });
+              return;
+            } catch (caught) {
+              const error = providerError(caught);
+              // A prior accepted delete can already have removed this exact ID.
+              if (retry && error instanceof DaytonaNotFoundError) return;
+              if (
+                !retry ||
+                !(error instanceof DaytonaConflictError) ||
+                error.statusCode !== 409
+              )
+                throw error;
+              if (
+                error.message.trim() ===
+                'Sandbox was modified by another operation'
+              ) {
+                // This conflict does not authorize mutation replay. A concurrent
+                // cleanup is deleting this exact ID, so only prove its absence.
+                absenceOnlyConflict = error;
+              } else if (
+                !/^Sandbox state change in progress[.!]?$/i.test(
+                  error.message.trim(),
+                )
+              ) {
+                throw error;
+              }
+            }
+          }
+          // Give the in-progress transition (including durable cleanup) its
+          // backoff before observing absence, so a closed resource need not
+          // be re-reported just to learn that deletion succeeded.
+          await delay(backoffMs, undefined, { signal: phase.signal });
+          backoffMs = Math.min(backoffMs * 2, DAYTONA_RETIREMENT_BACKOFF_MAX_MS);
+          // Reconcile only the acquired ID. Never resolve a name, start it, or
+          // infer absence from a failed/ambiguous provider read.
+          try {
+            await api.getSandbox(sandbox.id, undefined, undefined, {
+              signal: phase.signal,
+            });
+          } catch (caught) {
+            const error = providerError(caught);
+            if (error instanceof DaytonaNotFoundError) return;
+            throw error;
+          }
+          if (absenceOnlyConflict) continue;
+          // Authorization and every provider call share this ONE phase budget.
+          // A delayed ACK after expiry must never authorize another DELETE.
+          const signal = phase.signal;
+          signal.throwIfAborted();
+          await new Promise<void>((resolve, reject) => {
+            const abort = () => reject(signal.reason);
+            signal.addEventListener('abort', abort, { once: true });
+            Promise.resolve()
+              // Only managed callers get here: the DELETE branch rethrows
+              // every failure when retry is absent.
+              .then(retry!.authorizeRetry)
+              .then(resolve, reject)
+              .finally(() => signal.removeEventListener('abort', abort));
+          });
+          phase.signal.throwIfAborted();
+        }
       } catch (error) {
+        // Absence never arrived, so the provider's conflict is the real cause.
+        if (absenceOnlyConflict && phase.signal.aborted)
+          throw absenceOnlyConflict;
         throw providerError(error);
       } finally {
         phase.dispose();

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -59,13 +59,21 @@ const DAYTONA_NETWORK_ALLOW_LIST_ENV = 'DEEPLINE_DAYTONA_NETWORK_ALLOW_LIST';
 
 export const DAYTONA_CANCELLED_ERROR = 'Daytona play runner cancelled';
 
+export type DaytonaDeleteRetryAuthorization = {
+  authorizeRetry: () => Promise<void>;
+};
+
 export type DaytonaClient = {
   create(
     params: CreateSandboxFromSnapshotParams,
     options: { timeout: number },
   ): Promise<Sandbox>;
   get?(name: string): Promise<Sandbox>;
-  delete(sandbox: Pick<Sandbox, 'id'>, timeoutSeconds?: number): Promise<void>;
+  delete(
+    sandbox: Pick<Sandbox, 'id'>,
+    timeoutSeconds?: number,
+    retry?: DaytonaDeleteRetryAuthorization,
+  ): Promise<void>;
   scope(
     sandbox: Sandbox,
     timeoutMs: number,
@@ -84,7 +92,7 @@ type DaytonaCreateCallObserver = (
 
 export type AcquiredDaytonaSandbox = {
   sandbox: DaytonaSandbox;
-  deleteSandbox: () => Promise<void>;
+  deleteSandbox: (retry?: DaytonaDeleteRetryAuthorization) => Promise<void>;
   daytonaOrganizationId: string;
   billingStartedAt: number;
   billingEndedAt?: number;
@@ -1117,7 +1125,7 @@ async function acquireOneShotDaytonaSandbox(input: {
   });
   return {
     sandbox,
-    deleteSandbox: () => input.daytona.delete(sandbox, 30),
+    deleteSandbox: (retry) => input.daytona.delete(sandbox, 30, retry),
     daytonaOrganizationId,
     billingStartedAt,
     ...(result.sandboxCapacityLeaseId

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts

@@ -560,6 +560,22 @@ async function timedRunnerPayloadUpload(input: {
   });
 }
 
+/** Records the acquired provider in the config the sandbox reads. */
+export function withRunnerSandboxProvider(
+  config: PlayRunnerExecutionConfig,
+  sandboxProvider: 'daytona' | 'modal',
+): PlayRunnerExecutionConfig {
+  const push = config.context.runnerPushExecution;
+  if (!push) return config;
+  return {
+    ...config,
+    context: {
+      ...config.context,
+      runnerPushExecution: { ...push, sandboxProvider },
+    },
+  };
+}
+
 export async function stageRunnerPayload(input: {
   sandbox: RemoteRunnerPayloadSandbox;
   bundlePromise: Promise<string>;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts

@@ -189,6 +189,8 @@ let forceKillTimer = null;
 let runtimeTimer = null;
 let terminalGraceTimer = null;
 let heartbeatTimer = null;
+let heartbeatRetryTimer = null;
+let heartbeatCadenceMs = 0;
 let heartbeatController = null;
 let heartbeatInFlight = false;
 let heartbeatStopped = false;
@@ -236,6 +238,10 @@ function stopRunnerHeartbeat() {
     clearInterval(heartbeatTimer);
     heartbeatTimer = null;
   }
+  if (heartbeatRetryTimer) {
+    clearTimeout(heartbeatRetryTimer);
+    heartbeatRetryTimer = null;
+  }
   if (heartbeatController) {
     heartbeatController.abort();
     heartbeatController = null;
@@ -266,6 +272,7 @@ function startRunnerHeartbeatSupervisor() {
       JSON.stringify({ runId: config.runId, cadenceMs, handoffHeartbeatAgeMs: Math.max(0, Date.now() - heartbeatLastSuccessAtMs) }) +
       '\\n',
   );
+  heartbeatCadenceMs = cadenceMs;
   void runnerHeartbeatTick();
   heartbeatTimer = setInterval(runnerHeartbeatTick, cadenceMs);
   heartbeatTimer.unref?.();
@@ -341,11 +348,24 @@ async function runnerHeartbeatTick() {
         '[play-runner.heartbeat] watchdog_owner_rejected ' + JSON.stringify(diagnostic) + '\\n',
       );
       stopChild('runner_heartbeat_rejected', diagnostic);
+      return;
     }
     // Otherwise keep renewing. An unreachable or failing control plane is not
     // evidence that this run is over: the scheduler owns the liveness verdict
     // and confirms it only after it can accept heartbeats again. The runtime
     // limit, the terminal channel above, and sandbox cleanup bound this process.
+    // The lease is only a few cadences long, so one shed renewal must not wait
+    // a whole cadence. Only a 429 retries early: it says nothing happened and
+    // to try again, whereas retrying a failing store would add to its load.
+    // Jitter keeps a fleet shed together from retrying together, and the
+    // in-flight guard stops a retry overlapping a tick.
+    if (errorCode === 'http_429' && !heartbeatRetryTimer) {
+      heartbeatRetryTimer = setTimeout(() => {
+        heartbeatRetryTimer = null;
+        void runnerHeartbeatTick();
+      }, (heartbeatCadenceMs / 4) * (1 + Math.random() / 2));
+      heartbeatRetryTimer.unref?.();
+    }
   } finally {
     clearTimeout(timeout);
     if (heartbeatController === controller) heartbeatController = null;

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -58,7 +58,10 @@ import {
   isPlayRunRecoveryError,
   recoveryForDaytonaSandboxTransitionPending,
 } from '../../play-run-recovery-policy';
-import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
+import {
+  stageDaytonaRunnerPayload,
+  withRunnerSandboxProvider,
+} from './daytona-payload-transport';
 import {
   DaytonaRunnerInitializationError,
   prepareDetachedDaytonaRunner,
@@ -819,6 +822,20 @@ export async function readDetachedDaytonaCrashDiagnostic(input: {
  * `{ exitCode: null, result: null }` and the attempt fails through the normal
  * engine retry/fencing path.
  */
+/**
+ * Daytona states in which no process in the sandbox can still be running.
+ * `error` is left out: nothing here shows it means the sandbox stopped rather
+ * than that Daytona lost track of it, and a live run must not fail on that.
+ */
+const DAYTONA_SANDBOX_STOPPED_STATES = new Set([
+  'stopped',
+  'stopping',
+  'archived',
+  'archiving',
+  'destroyed',
+  'destroying',
+]);
+
 export async function inspectDetachedDaytonaRunner(input: {
   sandboxId: string;
   sessionId: string;
@@ -836,6 +853,7 @@ export async function inspectDetachedDaytonaRunner(input: {
       | 'command_running'
       | 'result_missing'
       | 'command_inspection_failed'
+      | 'sandbox_stopped'
       | 'sandbox_missing'
       | 'sandbox_lookup_failed'
       | 'wrong_routing_domain';
@@ -882,6 +900,19 @@ export async function inspectDetachedDaytonaRunner(input: {
         error: error instanceof Error ? error.message : String(error),
       });
     }
+    const stopped = DAYTONA_SANDBOX_STOPPED_STATES.has(
+      String(sandbox.state ?? '').toLowerCase(),
+    );
+    if (!stopped && commandFound && exitCode === null) {
+      // A running command has no exit-code file to recover. Polling for one
+      // would hold a worker slot for the whole recovery budget, on every
+      // re-inspection of a run that is only quiet.
+      return {
+        exitCode,
+        result: null,
+        diagnosis: { stage: 'command_running', detail: null },
+      };
+    }
     // The full captured stdout file can scale with customer output. Metadata
     // time must not consume its existing independent file-recovery budget.
     const recoveryFiles = control.legacyOutputRecovery(sandbox);
@@ -896,13 +927,13 @@ export async function inspectDetachedDaytonaRunner(input: {
       return {
         exitCode,
         result: null,
-        diagnosis: commandInspectionError
-          ? {
-              stage: 'command_inspection_failed',
-              detail: commandInspectionError,
-            }
-          : commandFound && exitCode === null
-            ? { stage: 'command_running', detail: null }
+        diagnosis: stopped
+          ? { stage: 'sandbox_stopped', detail: String(sandbox.state) }
+          : commandInspectionError
+            ? {
+                stage: 'command_inspection_failed',
+                detail: commandInspectionError,
+              }
             : { stage: 'result_missing', detail: null },
       };
     }
@@ -1491,10 +1522,14 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
     const reportedSandboxIds = new Set<string>();
     const reportedSandboxEndTimes = new Map<string, number | null>();
     const runtimeResourceReportErrors = new Set<unknown>();
-    const reportRuntimeResource = async (acquired: AcquiredDaytonaSandbox) => {
+    const reportRuntimeResource = async (
+      acquired: AcquiredDaytonaSandbox,
+      revalidate = false,
+    ) => {
       const sandbox = acquired.sandbox;
       const billingEndedAt = acquired.billingEndedAt ?? null;
       if (
+        !revalidate &&
         reportedSandboxIds.has(sandbox.id) &&
         reportedSandboxEndTimes.get(sandbox.id) === billingEndedAt
       ) {
@@ -1598,7 +1633,21 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             cancellationPromise,
           ]);
           activeAcquiredResource = acquiredSandbox;
-          sandboxCleanup.activate(acquiredSandbox);
+          sandboxCleanup.activate({
+            ...acquiredSandbox,
+            deleteSandbox: () =>
+              acquiredSandbox.deleteSandbox(
+                callbacks?.onRuntimeResourceAcquired &&
+                  acquiredSandbox.sandboxCapacityLeaseId
+                  ? {
+                      // Bypass the local report cache: the scheduler must prove this
+                      // exact attempt still owns deletion before each mutating retry.
+                      authorizeRetry: () =>
+                        reportRuntimeResource(acquiredSandbox, true),
+                    }
+                  : undefined,
+              ),
+          });
           runtimeTiming.daytonaCreateMs =
             acquiredSandbox.billingStartedAt - startedAt;
           const sandbox = acquiredSandbox.sandbox;
@@ -1730,7 +1779,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               stageDaytonaRunnerPayload({
                 sandbox: uploadScope.sandbox,
                 bundlePromise,
-                config,
+                config: withRunnerSandboxProvider(config, 'daytona'),
                 workDir,
                 startedAt,
                 emitStage: (stage, extra) =>
@@ -1890,8 +1939,24 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
             if (sandboxForAttempt) {
               activeAcquiredResource!.billingEndedAt = Date.now();
               await reportRetiringRuntimeResource(activeAcquiredResource!);
-              await sandboxCleanup.stashActiveSandboxForRetry();
+              try {
+                await sandboxCleanup.stashActiveSandboxForRetry();
+              } catch (retirementError) {
+                if (runtimeResourceReportErrors.has(retirementError))
+                  throw retirementError;
+                // Preserve both causes without feeding a retryable startup
+                // cause back into replacement admission. The cleanup manager
+                // retains the active resource and its single retirement promise.
+                const failure = new Error(
+                  `Daytona sandbox retirement unconfirmed: ${formatDaytonaError(retirementError)}. Original startup failure: ${formatDaytonaError(error)}`,
+                  { cause: error },
+                );
+                failure.name = 'DaytonaSandboxRetirementUnconfirmedError';
+                Object.assign(failure, { retirementError });
+                throw failure;
+              }
             }
+            throwIfCancellationRequested();
             acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
             continue;
           }

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts

@@ -24,7 +24,10 @@ import {
 } from '../../modal-runtime-config';
 import type { PlaySandboxRuntimeLimits } from '../../../plays/sandbox-runtime-limits';
 import { validateDaytonaExecutionContext } from './daytona-lifecycle';
-import { stageRunnerPayload } from './daytona-payload-transport';
+import {
+  stageRunnerPayload,
+  withRunnerSandboxProvider,
+} from './daytona-payload-transport';
 import { captureDetachedDaytonaRunnerReadinessBaseline } from './daytona-session-execution';
 import {
   isPlayRunRecoveryError,
@@ -594,7 +597,7 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
                 }),
             },
             bundlePromise: buildPlayRunnerBundle(),
-            config,
+            config: withRunnerSandboxProvider(config, 'modal'),
             workDir: modalConfig.workdir,
             startedAt,
             emitStage: (stage, extra) =>