deepline 0.3.461 → 0.3.463

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -22,9 +22,27 @@ import {
   STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS,
   validatePlaySandboxRuntimeLimits,
 } from '../../../plays/sandbox-runtime-limits';
-import type { PlayRunnerRuntimeLifecycleEvent } from '../types';
-import { DAYTONA_PLAY_RUNNER_LABEL_SOURCE } from './daytona-labels';
+import type {
+  PlayRunnerDaytonaCleanupIntent,
+  PlayRunnerRuntimeLifecycleEvent,
+} from '../types';
+import {
+  daytonaSandboxMatchesOperationIdentity,
+  DAYTONA_PLAY_RUNNER_LABEL_SOURCE,
+} from './daytona-labels';
+import { logSandboxCreateOutcome } from '../sandbox-create-outcome';
 import type { DaytonaSandboxStartAdmission } from './daytona-start-admission';
+import { DaytonaControlTimeoutError } from './daytona-control-transport';
+
+function isDaytonaControlTimeout(error: unknown): boolean {
+  return (
+    error instanceof DaytonaControlTimeoutError ||
+    (error instanceof Error &&
+      error.name === 'DaytonaControlTimeoutError' &&
+      'code' in error &&
+      error.code === 'ETIMEDOUT')
+  );
+}
 
 // Match Daytona's supported default create window. Ten seconds is shorter
 // than normal provider startup under burst load: the SDK abandons an otherwise
@@ -69,6 +87,7 @@ export type DaytonaClient = {
     options: { timeout: number },
   ): Promise<Sandbox>;
   get?(name: string): Promise<Sandbox>;
+  placementForLog?(sandbox: Sandbox): Record<string, unknown>;
   delete(
     sandbox: Pick<Sandbox, 'id'>,
     timeoutSeconds?: number,
@@ -88,7 +107,7 @@ export type DaytonaStageEmitter = (
 
 type DaytonaCreateCallObserver = (
   event: PlayRunnerRuntimeLifecycleEvent,
-) => Promise<void>;
+) => Promise<void> | undefined;
 
 export type AcquiredDaytonaSandbox = {
   sandbox: DaytonaSandbox;
@@ -178,6 +197,7 @@ export type DaytonaSandboxTransitionPendingLogDetail =
     lookups: number;
     lookupErrors: number;
     deleteConflicts: number;
+    deleteTimeouts: number;
     elapsedMs: number;
     /** Known only in the process that sent the create. */
     createResponseReceived: boolean | null;
@@ -240,12 +260,16 @@ function daytonaTransitionPendingReasonDetail(
   phase: DaytonaSandboxTransitionPendingLogDetail['phase'],
   observation: DaytonaNamedSandboxObservation,
   deleteConflicts: number,
+  deleteTimeouts: number,
 ): string {
   if (observation.lookup !== 'found') {
     return `${phase}:lookup_${observation.lookup}`;
   }
-  // Reconcile reaches a pending decision with a visible sandbox only after
-  // every delete of it reported an in-progress state change.
+  // A visible sandbox may remain in its original create transition, or its
+  // deletion may have conflicted or timed out.
+  if (deleteTimeouts > 0) {
+    return `${phase}:delete_timeout:${observation.sandboxState}`;
+  }
   return deleteConflicts > 0
     ? `${phase}:delete_conflict:${observation.sandboxState}`
     : `${phase}:sandbox_state:${observation.sandboxState}`;
@@ -262,8 +286,14 @@ function daytonaTransitionPendingLogDetail(input: {
   launchOutcome: DaytonaLaunchOutcomeDiagnostic | null;
 }): DaytonaSandboxTransitionPendingLogDetail | null {
   try {
-    const { last, lookups, lookupErrors, deleteConflicts, elapsedMs } =
-      input.observed;
+    const {
+      last,
+      lookups,
+      lookupErrors,
+      deleteConflicts,
+      deleteTimeouts,
+      elapsedMs,
+    } = input.observed;
     return {
       ...last,
       phase: input.phase,
@@ -271,11 +301,13 @@ function daytonaTransitionPendingLogDetail(input: {
         input.phase,
         last,
         deleteConflicts,
+        deleteTimeouts,
       ),
       providerAttempt: input.providerAttempt,
       lookups,
       lookupErrors,
       deleteConflicts,
+      deleteTimeouts,
       elapsedMs,
       createResponseReceived:
         input.launchOutcome?.createResponseReceived ?? null,
@@ -340,6 +372,44 @@ function isDaytonaSandboxNotFound(error: unknown): boolean {
   return /\b(?:sandbox )?not found\b|\b404\b/i.test(message);
 }
 
+/** How long an in-process delete waits for Daytona to stop returning it. */
+export const DAYTONA_DELETE_CONFIRMATION_TIMEOUT_MS = 30_000;
+const DAYTONA_DELETE_CONFIRMATION_BACKOFF_INITIAL_MS = 100;
+const DAYTONA_DELETE_CONFIRMATION_BACKOFF_MAX_MS = 5_000;
+
+/**
+ * An accepted DELETE only starts Daytona's work, and the sandbox can keep
+ * running after it. Read the exact id until Daytona answers 404. A sandbox
+ * still returned at the deadline, or a read that fails some other way, proves
+ * nothing, so the caller keeps the sandbox's capacity and identity.
+ */
+async function confirmDaytonaSandboxDeleted(input: {
+  daytona: DaytonaClient;
+  sandboxId: string;
+  cancellationSignal?: AbortSignal;
+}): Promise<boolean> {
+  if (!input.daytona.get) return false;
+  const deadlineAt = Date.now() + DAYTONA_DELETE_CONFIRMATION_TIMEOUT_MS;
+  let backoffMs = DAYTONA_DELETE_CONFIRMATION_BACKOFF_INITIAL_MS;
+  for (;;) {
+    try {
+      await input.daytona.get(input.sandboxId);
+    } catch (error) {
+      return isDaytonaSandboxNotFound(error);
+    }
+    const remainingMs = deadlineAt - Date.now();
+    if (remainingMs <= 0) return false;
+    await waitForAcquisitionRetry(
+      Math.min(backoffMs, remainingMs),
+      input.cancellationSignal,
+    );
+    backoffMs = Math.min(
+      backoffMs * 2,
+      DAYTONA_DELETE_CONFIRMATION_BACKOFF_MAX_MS,
+    );
+  }
+}
+
 type DaytonaCreateResult = {
   sandbox: DaytonaSandbox;
   attempt: number;
@@ -363,6 +433,7 @@ async function rejectAcquiredSandbox(
   reservation: { leaseId: string } | undefined,
   releaseSandboxCapacity: ((leaseId: string) => Promise<void>) | undefined,
   deleteSandbox: () => Promise<void>,
+  confirmDeleted: () => Promise<boolean>,
 ): Promise<never> {
   try {
     await deleteSandbox();
@@ -376,6 +447,14 @@ async function rejectAcquiredSandbox(
     );
   }
   if (reservation) {
+    if (!(await confirmDeleted())) {
+      // Releasing now would count a sandbox Daytona still returns as gone.
+      // Nothing re-reads it once this run fails, so the reservation ends at
+      // its crash-backstop TTL, as it does when the delete itself fails above.
+      throw new Error(
+        `${reason} Daytona accepted the defensive deletion of sandbox ${sandbox.id} but still returns it, so its capacity stays held.`,
+      );
+    }
     await releaseSandboxCapacity?.(reservation.leaseId);
   }
   throw new Error(reason);
@@ -545,6 +624,7 @@ type TimedOutDaytonaSandboxReconcileObserved = {
   lookups: number;
   lookupErrors: number;
   deleteConflicts: number;
+  deleteTimeouts: number;
   elapsedMs: number;
 };
 
@@ -561,25 +641,32 @@ async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
   cancellationSignal?: AbortSignal;
   emitStage?: DaytonaStageEmitter;
   operationIdentity?: {
+    leaseId: string;
     providerOperationToken: string;
     orgId: string;
     runId?: string;
   };
+  recordCleanupIntent?: (
+    intent: PlayRunnerDaytonaCleanupIntent,
+  ) => Promise<void>;
 }): Promise<TimedOutDaytonaSandboxReconciliation> {
   const reconcileStartedAt = Date.now();
   let last = NO_DAYTONA_NAMED_SANDBOX_LOOKUP;
   let lookups = 0;
   let lookupErrors = 0;
   let deleteConflicts = 0;
+  let deleteTimeouts = 0;
   const observed = (): TimedOutDaytonaSandboxReconcileObserved => ({
     last,
     lookups,
     lookupErrors,
     deleteConflicts,
+    deleteTimeouts,
     elapsedMs: Date.now() - reconcileStartedAt,
   });
   if (!input.daytona.get) return { kind: 'not_found', observed: observed() };
   let lastDeleteConflict: unknown = null;
+  let sawDeleteTimeout = false;
   for (const delayMs of DAYTONA_TIMED_OUT_CREATE_RECONCILE_DELAYS_MS) {
     input.cancellationSignal?.throwIfAborted();
     if (delayMs > 0) {
@@ -613,36 +700,56 @@ async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
     input.emitStage?.('create:reconcile_state', {
       sandboxId: sandbox.id,
       state: sandbox.state ?? null,
+      ...input.daytona.placementForLog?.(sandbox),
     });
-    // `create` waits only a bounded interval, but the named sandbox may finish
-    // the same create transition immediately afterwards. It is already the
-    // durable identity associated with this provider operation, so reusing it
-    // is safer and faster than attempting a conflicting delete or creating a
-    // second sandbox.
-    if (sandbox.state === 'started') {
-      return { kind: 'started', sandbox, observed: observed() };
-    }
     last = observeDaytonaNamedSandbox(sandbox);
     if (input.operationIdentity) {
       const expected = input.operationIdentity;
       const configuredOrganizationId =
         process.env.DAYTONA_ORGANIZATION_ID?.trim();
       if (
-        sandbox.name !== input.sandboxName ||
-        sandbox.labels?.source !== DAYTONA_PLAY_RUNNER_LABEL_SOURCE ||
-        sandbox.labels?.operationKey !== expected.providerOperationToken ||
-        sandbox.labels?.orgId !== normalizeLabelValue(expected.orgId) ||
-        (normalizeLabelValue(expected.runId) &&
-          sandbox.labels?.runId !== normalizeLabelValue(expected.runId)) ||
-        !sandbox.organizationId?.trim() ||
-        (configuredOrganizationId &&
-          sandbox.organizationId !== configuredOrganizationId)
+        !daytonaSandboxMatchesOperationIdentity({
+          sandbox,
+          sandboxName: input.sandboxName,
+          providerOperationToken: expected.providerOperationToken,
+          orgId: expected.orgId,
+          runId: expected.runId,
+          configuredOrganizationId,
+        })
       ) {
         throw new Error(
           'Daytona resumed sandbox identity or organization routing does not match its reserved operation. Cleanup and fallback suppressed.',
         );
       }
     }
+    // Keep an in-progress create under its original reservation. A late start
+    // belongs to the same operation, unless a prior DELETE timed out and made
+    // that sandbox's future state ambiguous.
+    if (sandbox.state === 'creating' && !sawDeleteTimeout) continue;
+    if (sandbox.state === 'started' && !sawDeleteTimeout) {
+      return { kind: 'started', sandbox, observed: observed() };
+    }
+    // Persist the exact identity before handing stable or ambiguously-deleted
+    // resources to the cleanup worker. Never turn a still-creating sandbox
+    // into a delete intent: its create may complete and be safely resumed.
+    if (input.operationIdentity && input.recordCleanupIntent) {
+      if (!input.operationIdentity.runId || !sandbox.organizationId?.trim()) {
+        throw new Error(
+          'Daytona timed-out sandbox lacks the run or organization identity required for durable cleanup.',
+        );
+      }
+      await input.recordCleanupIntent({
+        leaseId: input.operationIdentity.leaseId,
+        providerOperationToken: input.operationIdentity.providerOperationToken,
+        orgId: input.operationIdentity.orgId,
+        runId: input.operationIdentity.runId,
+        sandboxId: sandbox.id,
+        routingDomain: sandbox.organizationId,
+      });
+      return { kind: 'transition_pending', observed: observed() };
+    }
+    // After a DELETE timed out, never run the sandbox until this same named
+    // identity is confirmed deleted or remains pending cleanup.
     try {
       await input.daytona.delete(sandbox, 30);
     } catch (error) {
@@ -656,14 +763,30 @@ async function reconcileAndDeleteTimedOutDaytonaSandbox(input: {
         deleteConflicts += 1;
         continue;
       }
+      if (isDaytonaControlTimeout(error)) {
+        sawDeleteTimeout = true;
+        deleteTimeouts += 1;
+        continue;
+      }
       throw new Error(
         `Daytona timed-out sandbox ${sandbox.id} was reconciled by name but could not be deleted. Modal fallback suppressed.`,
         { cause: error },
       );
     }
-    return { kind: 'deleted', observed: observed() };
+    if (
+      await confirmDaytonaSandboxDeleted({
+        daytona: input.daytona,
+        sandboxId: sandbox.id,
+        cancellationSignal: input.cancellationSignal,
+      })
+    ) {
+      return { kind: 'deleted', observed: observed() };
+    }
+    // Accepted is not gone. Keep this identity and its capacity: the
+    // scheduler resumes the same name and re-reads it.
+    return { kind: 'transition_pending', observed: observed() };
   }
-  if (lastDeleteConflict) {
+  if (lastDeleteConflict || sawDeleteTimeout || last.lookup === 'found') {
     return { kind: 'transition_pending', observed: observed() };
   }
   return { kind: 'not_found', observed: observed() };
@@ -685,9 +808,13 @@ async function createRetriedOneShotDaytonaSandbox(input: {
     leaseId: string;
     providerOperationToken: string | null;
     resumeExistingProviderOperation?: boolean;
+    resumePendingDelete?: boolean;
     existingProviderOperationAgeMs?: number;
   }>;
   releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
+  recordCleanupIntent?: (
+    intent: PlayRunnerDaytonaCleanupIntent,
+  ) => Promise<void>;
   nextProviderAttempt: () => number;
   startedAt: number;
   cancellationSignal?: AbortSignal;
@@ -716,6 +843,20 @@ async function createRetriedOneShotDaytonaSandbox(input: {
     const reservation = await input.reserveSandboxCapacity?.(attempt);
     const sandboxName = daytonaSandboxName(reservation?.providerOperationToken);
     let recreatedAfter: TimedOutDaytonaSandboxReconcileObserved | null = null;
+    if (reservation?.resumePendingDelete) {
+      input.emitStage('create:resume_delete_pending', {
+        attempt,
+        sandboxName,
+      });
+      throw new DaytonaSandboxTransitionPendingError(sandboxName);
+    }
+    if (
+      reservation &&
+      !reservation.providerOperationToken &&
+      input.recordCleanupIntent
+    ) {
+      throw new DaytonaSandboxTransitionPendingError(sandboxName);
+    }
     if (reservation?.resumeExistingProviderOperation) {
       if (!reservation.providerOperationToken) {
         throw new DaytonaSandboxTransitionPendingError(sandboxName);
@@ -728,10 +869,12 @@ async function createRetriedOneShotDaytonaSandbox(input: {
         cancellationSignal: input.cancellationSignal,
         emitStage: input.emitStage,
         operationIdentity: {
+          leaseId: reservation.leaseId,
           providerOperationToken: reservation.providerOperationToken,
           orgId: input.orgId,
           runId: input.context.runId ?? undefined,
         },
+        recordCleanupIntent: input.recordCleanupIntent,
       });
       if (reconciliation.kind === 'started') {
         const sandbox = reconciliation.sandbox;
@@ -798,13 +941,25 @@ async function createRetriedOneShotDaytonaSandbox(input: {
       // can be ambiguous, so retain its lease until the crash TTL rather than
       // admitting another sandbox against an unknown provider outcome.
       const message = error instanceof Error ? error.message : String(error);
-      const recordCreateFailure = () =>
-        input.observeCreateCall?.({
+      const recordCreateFailure = () => {
+        const errorClass = daytonaCreateErrorClass(error);
+        logSandboxCreateOutcome({
+          provider: 'daytona',
+          outcome:
+            error instanceof DaytonaLaunchOutcomeUnknownError ||
+            errorClass === 'timeout'
+              ? 'unknown'
+              : 'failed',
+          runId: input.context.runId,
+          reason: errorClass,
+        });
+        return input.observeCreateCall?.({
           type: 'daytona_create_call_failed',
           occurredAtMs: Date.now(),
           providerAttempt: attempt,
-          errorClass: daytonaCreateErrorClass(error),
+          errorClass,
         });
+      };
       errors.push(message);
       runErrors.push(
         isDaytonaProviderError(error)
@@ -848,6 +1003,19 @@ async function createRetriedOneShotDaytonaSandbox(input: {
             sandboxName,
             cancellationSignal: input.cancellationSignal,
             emitStage: input.emitStage,
+            ...(input.recordCleanupIntent &&
+            reservation?.providerOperationToken &&
+            reservation.leaseId
+              ? {
+                  operationIdentity: {
+                    leaseId: reservation.leaseId,
+                    providerOperationToken: reservation.providerOperationToken,
+                    orgId: input.orgId,
+                    runId: input.context.runId ?? undefined,
+                  },
+                  recordCleanupIntent: input.recordCleanupIntent,
+                }
+              : {}),
           });
         } catch (reconciliationError) {
           await recordCreateFailure();
@@ -1002,10 +1170,17 @@ async function acquireOneShotDaytonaSandbox(input: {
   context: DaytonaExecutionContext;
   emitStage: DaytonaStageEmitter;
   observeCreateCall?: DaytonaCreateCallObserver;
-  reserveSandboxCapacity?: (
-    providerAttempt: number,
-  ) => Promise<{ leaseId: string; providerOperationToken: string | null }>;
+  reserveSandboxCapacity?: (providerAttempt: number) => Promise<{
+    leaseId: string;
+    providerOperationToken: string | null;
+    resumeExistingProviderOperation?: boolean;
+    resumePendingDelete?: boolean;
+    existingProviderOperationAgeMs?: number;
+  }>;
   releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
+  recordCleanupIntent?: (
+    intent: PlayRunnerDaytonaCleanupIntent,
+  ) => Promise<void>;
   nextProviderAttempt: () => number;
   startedAt: number;
   startAdmission?: DaytonaSandboxStartAdmission;
@@ -1045,6 +1220,12 @@ async function acquireOneShotDaytonaSandbox(input: {
     diskGiB: result.sandbox.disk,
     gpu: result.sandbox.gpu ?? 0,
   };
+  const confirmDeleted = () =>
+    confirmDaytonaSandboxDeleted({
+      daytona: input.daytona,
+      sandboxId: result.sandbox.id,
+      cancellationSignal: input.cancellationSignal,
+    });
   if (
     granted.cpu !== limits.cpu ||
     granted.memoryGiB !== limits.memoryGiB ||
@@ -1059,6 +1240,7 @@ async function acquireOneShotDaytonaSandbox(input: {
         : undefined,
       input.releaseSandboxCapacity,
       () => input.daytona.delete(result.sandbox, 30),
+      confirmDeleted,
     );
   }
   const configuredOrganizationId =
@@ -1077,6 +1259,7 @@ async function acquireOneShotDaytonaSandbox(input: {
         : undefined,
       input.releaseSandboxCapacity,
       () => input.daytona.delete(result.sandbox, 30),
+      confirmDeleted,
     );
   }
   let lookupOrganizationId: string | null = null;
@@ -1104,6 +1287,7 @@ async function acquireOneShotDaytonaSandbox(input: {
         : undefined,
       input.releaseSandboxCapacity,
       () => input.daytona.delete(result.sandbox, 30),
+      confirmDeleted,
     );
   }
   const billingStartedAt = Date.now();
@@ -1143,9 +1327,13 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
     leaseId: string;
     providerOperationToken: string | null;
     resumeExistingProviderOperation?: boolean;
+    resumePendingDelete?: boolean;
     existingProviderOperationAgeMs?: number;
   }>;
   releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
+  recordCleanupIntent?: (
+    intent: PlayRunnerDaytonaCleanupIntent,
+  ) => Promise<void>;
   startAdmission?: DaytonaSandboxStartAdmission;
   cancellationSignal?: AbortSignal;
   startedAt?: number;
@@ -1165,6 +1353,7 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
       observeCreateCall: input.observeCreateCall,
       reserveSandboxCapacity: input.reserveSandboxCapacity,
       releaseSandboxCapacity: input.releaseSandboxCapacity,
+      recordCleanupIntent: input.recordCleanupIntent,
       startAdmission: input.startAdmission,
       cancellationSignal: input.cancellationSignal,
       nextProviderAttempt: () => {
@@ -1197,11 +1386,23 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
       try {
         const acquired = await latestAcquiredSandboxPromise;
         await acquired.deleteSandbox();
-        // `dispose` has provider-confirmed that the sandbox is gone. This is
-        // normally the narrow cancellation window before the scheduler has
-        // durably recorded the resource, so no cleanup job will release an
-        // unbound lease on our behalf.
+        // This is normally the narrow cancellation window before the scheduler
+        // has durably recorded the resource, so no cleanup job will release an
+        // unbound lease on our behalf. Release it only once Daytona stops
+        // returning the sandbox; otherwise the reservation keeps its name for
+        // the scheduler to resume and re-read.
         if (acquired.sandboxCapacityLeaseId) {
+          if (
+            !(await confirmDaytonaSandboxDeleted({
+              daytona: input.daytona,
+              sandboxId: acquired.sandbox.id,
+            }))
+          ) {
+            console.warn('[play-runner.daytona.dispose_delete_unconfirmed]', {
+              sandboxId: acquired.sandbox.id,
+            });
+            return;
+          }
           await input.releaseSandboxCapacity?.(acquired.sandboxCapacityLeaseId);
         }
       } catch (error) {