deepline 0.3.460 → 0.3.462

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -5,6 +5,8 @@ import {
   daytonaErrorStatusForLog,
   daytonaProviderDetailForRunError,
   daytonaProviderMessageForRunError,
+  daytonaSandboxPlacementForLog,
+  daytonaSandboxStateForLog,
   isDaytonaProviderError,
   redactDaytonaErrorDetail,
   withoutDaytonaProviderDetail,
@@ -28,6 +30,7 @@ import {
 } from '../types';
 import { buildPlayRunnerBundle } from '../bundle';
 import { findPlayRunnerResult, parsePlayRunnerEvents } from '../runner-events';
+import { logSandboxCreateOutcome } from '../sandbox-create-outcome';
 import type {
   PlayRunnerExecutionConfig,
   PlayRunnerRuntimeTiming,
@@ -57,6 +60,7 @@ import {
 import {
   isPlayRunRecoveryError,
   recoveryForDaytonaSandboxTransitionPending,
+  recoveryForRunnerStartupNotLive,
 } from '../../play-run-recovery-policy';
 import {
   stageDaytonaRunnerPayload,
@@ -844,6 +848,8 @@ export async function inspectDetachedDaytonaRunner(input: {
   exitCodePath: string;
   /** Refuse to read a sandbox outside the launch-pinned Daytona organization. */
   expectedOrganizationId?: string | null;
+  /** Answer from the sandbox's state alone. See the reconciliation seam. */
+  skipCapturedOutput?: boolean;
 }): Promise<{
   exitCode: number | null;
   result: PlayRunnerResult | null;
@@ -853,6 +859,7 @@ export async function inspectDetachedDaytonaRunner(input: {
       | 'command_running'
       | 'result_missing'
       | 'command_inspection_failed'
+      | 'sandbox_not_ended'
       | 'sandbox_stopped'
       | 'sandbox_missing'
       | 'sandbox_lookup_failed'
@@ -880,6 +887,20 @@ export async function inspectDetachedDaytonaRunner(input: {
         },
       };
     }
+    if (input.skipCapturedOutput) {
+      return {
+        exitCode: null,
+        result: null,
+        diagnosis: {
+          stage: DAYTONA_SANDBOX_STOPPED_STATES.has(
+            String(sandbox.state ?? '').toLowerCase(),
+          )
+            ? 'sandbox_stopped'
+            : 'sandbox_not_ended',
+          detail: daytonaSandboxStateForLog(sandbox.state),
+        },
+      };
+    }
     let exitCode: number | null = null;
     let commandFound = false;
     let commandInspectionError: string | null = null;
@@ -900,6 +921,18 @@ export async function inspectDetachedDaytonaRunner(input: {
         error: error instanceof Error ? error.message : String(error),
       });
     }
+    // The control API can answer while the sandbox itself does not. This is
+    // Daytona's account of a silent sandbox, and it is lost with the sandbox.
+    console.warn(
+      '[play-runner.daytona.detached_provider_snapshot]',
+      JSON.stringify({
+        sandboxId: input.sandboxId,
+        cmdId: input.cmdId,
+        toolboxResponded: commandInspectionError === null,
+        state: daytonaSandboxStateForLog(sandbox.state),
+        ...daytonaSandboxPlacementForLog(control.providerData(sandbox)),
+      }),
+    );
     const stopped = DAYTONA_SANDBOX_STOPPED_STATES.has(
       String(sandbox.state ?? '').toLowerCase(),
     );
@@ -954,13 +987,24 @@ export async function inspectDetachedDaytonaRunner(input: {
       cmdId: input.cmdId,
       failure,
     });
+    // A key for another organization gets not-found for every live sandbox, so
+    // a 404 proves loss only under the launch's own organization, as in delete.
+    const expectedOrganizationId = input.expectedOrganizationId?.trim() || null;
+    const configuredOrganizationId =
+      process.env.DAYTONA_ORGANIZATION_ID?.trim() || null;
+    const notFoundInLaunchOrganization =
+      failure.httpStatus === 404 &&
+      expectedOrganizationId !== null &&
+      (configuredOrganizationId === null ||
+        configuredOrganizationId === expectedOrganizationId);
     return {
       exitCode: null,
       result: null,
       diagnosis: {
-        stage:
-          failure.httpStatus === 404
-            ? 'sandbox_missing'
+        stage: notFoundInLaunchOrganization
+          ? 'sandbox_missing'
+          : failure.httpStatus === 404 && expectedOrganizationId !== null
+            ? 'wrong_routing_domain'
             : 'sandbox_lookup_failed',
         detail: failure.detail,
       },
@@ -1026,6 +1070,14 @@ export type DaytonaSandboxDeleteOutcome =
       kind: 'deleted' | 'already_absent';
       organizationId: string | null;
     }
+  | {
+      kind: 'delete_pending';
+      organizationId: string | null;
+      code: string;
+      detail: string;
+      /** A delete has been accepted, so only the 404 is outstanding. */
+      deleteAccepted: boolean;
+    }
   | {
       kind: 'timed_out' | 'rate_limited' | 'failed';
       organizationId: string | null;
@@ -1033,11 +1085,23 @@ export type DaytonaSandboxDeleteOutcome =
       detail: string;
     };
 
+/** Daytona has already accepted a delete, so another adds nothing. */
+const DAYTONA_SANDBOX_DELETING_STATES = new Set(['destroying', 'destroyed']);
+
+/**
+ * An accepted DELETE only starts the provider's work: on Oct 30 sandboxes
+ * recorded deleted stayed in Daytona's inventory for minutes, and one process
+ * kept posting 8 minutes later. Only a 404 for this exact sandbox settles it.
+ * That 404 reads Daytona's database rather than the host, so it proves the
+ * record is gone, which is the agreed bar.
+ */
 export async function deleteDaytonaSandboxByIdWithOutcome(input: {
   sandboxId: string;
   timeoutSeconds?: number;
   expectedOrganizationId?: string | null;
   allowUnscopedAlreadyAbsent?: boolean;
+  /** Durable proof that Daytona accepted this delete on an earlier pass. */
+  deleteAcceptance?: { organizationId: string | null } | null;
 }): Promise<DaytonaSandboxDeleteOutcome> {
   const sandboxId = input.sandboxId?.trim();
   const expectedOrganizationId = input.expectedOrganizationId?.trim() || null;
@@ -1050,6 +1114,7 @@ export async function deleteDaytonaSandboxByIdWithOutcome(input: {
     };
   }
   let control: ReturnType<typeof createDaytonaControlClient> | undefined;
+  let observedOrganizationId: string | null = null;
   try {
     const { clientOptions } = loadDaytonaRequiredConfig();
     control = daytonaSdkClientFactory.createControl(
@@ -1057,7 +1122,7 @@ export async function deleteDaytonaSandboxByIdWithOutcome(input: {
       (input.timeoutSeconds ?? 30) * 1_000,
     );
     const sandbox = await control.get(sandboxId);
-    const observedOrganizationId = sandbox.organizationId?.trim() || null;
+    observedOrganizationId = sandbox.organizationId?.trim() || null;
     if (
       expectedOrganizationId &&
       observedOrganizationId !== expectedOrganizationId
@@ -1069,10 +1134,19 @@ export async function deleteDaytonaSandboxByIdWithOutcome(input: {
         detail: 'Sandbox belongs to a different Daytona organization.',
       };
     }
-    await control.delete(sandbox);
+    if (
+      !DAYTONA_SANDBOX_DELETING_STATES.has(
+        String(sandbox.state ?? '').toLowerCase(),
+      )
+    ) {
+      await control.delete(sandbox);
+    }
     return {
-      kind: 'deleted',
+      kind: 'delete_pending',
       organizationId: observedOrganizationId,
+      code: 'daytona_delete_unconfirmed',
+      detail: `Daytona still returns sandbox ${sandboxId}; only a 404 confirms deletion.`,
+      deleteAccepted: true,
     };
   } catch (error) {
     const failure = describeDaytonaLookupFailure(error);
@@ -1087,6 +1161,27 @@ export async function deleteDaytonaSandboxByIdWithOutcome(input: {
     if (failure.httpStatus === 404) {
       const configuredOrganizationId =
         process.env.DAYTONA_ORGANIZATION_ID?.trim() || null;
+      if (input.deleteAcceptance) {
+        // The read-back confirming an accepted delete: a provider confirmation,
+        // so it may establish a legacy job's domain, which plain absence cannot.
+        const acceptedOrganizationId =
+          input.deleteAcceptance.organizationId?.trim() ||
+          expectedOrganizationId;
+        if (
+          configuredOrganizationId &&
+          acceptedOrganizationId &&
+          configuredOrganizationId !== acceptedOrganizationId
+        ) {
+          return {
+            kind: 'failed',
+            organizationId: configuredOrganizationId,
+            code: 'wrong_routing_domain',
+            detail:
+              'Daytona returned not-found from outside the organization that accepted the delete.',
+          };
+        }
+        return { kind: 'deleted', organizationId: acceptedOrganizationId };
+      }
       if (
         !input.allowUnscopedAlreadyAbsent &&
         (!expectedOrganizationId ||
@@ -1112,6 +1207,23 @@ export async function deleteDaytonaSandboxByIdWithOutcome(input: {
           configuredOrganizationId ?? expectedOrganizationId ?? null,
       };
     }
+    if (failure.httpStatus === 409) {
+      // A state change is in progress. Daytona took no delete, and nothing is
+      // wrong, so this waits for its next read without reporting a failure.
+      console.info(
+        '[play-runner.daytona.reclaim_sandbox_state_change_pending]',
+        {
+          sandboxId,
+        },
+      );
+      return {
+        kind: 'delete_pending',
+        organizationId: observedOrganizationId ?? expectedOrganizationId,
+        code: 'daytona_state_change_in_progress',
+        detail: failure.detail,
+        deleteAccepted: false,
+      };
+    }
     console.warn('[play-runner.daytona.reclaim_sandbox_delete_failed]', {
       sandboxId,
       failure,
@@ -1147,7 +1259,11 @@ export async function deleteDaytonaSandboxById(input: {
     ...input,
     allowUnscopedAlreadyAbsent: true,
   });
-  return outcome.kind === 'deleted' || outcome.kind === 'already_absent';
+  return (
+    outcome.kind === 'deleted' ||
+    outcome.kind === 'already_absent' ||
+    (outcome.kind === 'delete_pending' && outcome.deleteAccepted)
+  );
 }
 
 export function formatDaytonaError(error: unknown): string {
@@ -1360,6 +1476,96 @@ function isRetryableDaytonaInfrastructureFailure(error: unknown): boolean {
   return DAYTONA_INFRASTRUCTURE_RETRY_PATTERN.test(message);
 }
 
+// A new sandbox can outrun these: the sandbox is gone or busy, the request
+// timed out, or Daytona is limiting. Other 4xx answers, a refused credential
+// among them, would fail the same way on every sandbox and stay loud.
+const DAYTONA_REPLACEABLE_START_STATUSES: ReadonlySet<number> = new Set([
+  404, 408, 409, 429,
+]);
+// Codes that mean the request got no answer.
+const DAYTONA_UNANSWERED_REQUEST_CODES: ReadonlySet<string> = new Set([
+  'ECONNABORTED',
+  'ECONNREFUSED',
+  'ECONNRESET',
+  'EAI_AGAIN',
+  'EPIPE',
+  'ERR_NETWORK',
+  'ETIMEDOUT',
+]);
+
+/**
+ * Daytona failed a step of starting the runner: the payload upload, the
+ * session, the command, or a read of a runner that never reported live. No
+ * runner has been released yet, so a new sandbox cannot repeat customer work.
+ * Only Daytona's own failures count: anything else here is our bundle or
+ * config, which fails the same way on every sandbox. A stalled upload stays
+ * loud too, because its input may be too large for any sandbox.
+ */
+function isFailedDaytonaRunnerStart(error: unknown): boolean {
+  if (error instanceof DaytonaControlTimeoutError) return true;
+  if (!isDaytonaProviderError(error)) return false;
+  const status = daytonaErrorStatusForLog(error);
+  if (status !== null) {
+    return status >= 500 || DAYTONA_REPLACEABLE_START_STATUSES.has(status);
+  }
+  // A missing status is not proof of a missing answer: the SDK raises
+  // status-less errors for local failures too, such as a module it cannot
+  // load. The 56 October uploads to bad hosts ended in a bare `DaytonaError`
+  // with no status, no code and no message.
+  return (
+    error.name === 'DaytonaConnectionError' ||
+    error.name === 'DaytonaTimeoutError' ||
+    DAYTONA_UNANSWERED_REQUEST_CODES.has(daytonaErrorCodeForLog(error) ?? '') ||
+    error.message === '' ||
+    DAYTONA_INFRASTRUCTURE_RETRY_PATTERN.test(error.message)
+  );
+}
+
+/** Which failure ended a start, for the stage log: the run error has less. */
+function startFailureForLog(error: unknown) {
+  const diagnostic =
+    error instanceof DaytonaRunnerInitializationError
+      ? error.startupDiagnostic
+      : null;
+  const startup = diagnostic?.runnerStartup;
+  return {
+    errorName: error instanceof Error ? error.name : null,
+    providerStatus: daytonaErrorStatusForLog(error),
+    providerCode: daytonaErrorCodeForLog(error),
+    ...(diagnostic
+      ? {
+          // Retain the already-sanitized evidence before sandbox retirement.
+          // Omit inspection messages, process names and the growing state map.
+          startupDiagnostic: {
+            schedulerReadinessObserved: diagnostic.schedulerReadinessObserved,
+            schedulerReadinessResponses: diagnostic.schedulerReadinessResponses,
+            commandFound: diagnostic.commandFound,
+            exitCode: diagnostic.exitCode,
+            sessionFound: diagnostic.sessionFound,
+            sessionCommandCount: diagnostic.sessionCommandCount,
+            exitCodeFile: diagnostic.exitCodeFile,
+            processSummary: diagnostic.processSummary
+              ? {
+                  total: diagnostic.processSummary.total,
+                  node: diagnostic.processSummary.node,
+                }
+              : null,
+            runnerStartup: startup
+              ? {
+                  phase: startup.phase,
+                  outcome: startup.outcome,
+                  status: startup.status,
+                  errorCode: startup.errorCode,
+                  transportAttemptId: startup.transportAttemptId,
+                }
+              : null,
+            inspectionErrorCount: diagnostic.errors.length,
+          },
+        }
+      : {}),
+  };
+}
+
 function isRetryableUnstartedDaytonaRunner(
   error: unknown,
 ): error is DaytonaRunnerInitializationError {
@@ -1396,12 +1602,22 @@ function prepareDaytonaExecution(
     context: input.context,
     emitStage: (stage, extra) =>
       emitDaytonaStage(callbacks, input.context, stage, extra),
-    observeCreateCall: callbacks?.onRuntimeLifecycleEvent,
+    observeCreateCall: (event) => {
+      if (event.type === 'daytona_create_call_succeeded') {
+        logSandboxCreateOutcome({
+          provider: 'daytona',
+          outcome: 'succeeded',
+          runId: input.context.runId,
+        });
+      }
+      return callbacks?.onRuntimeLifecycleEvent?.(event);
+    },
     reserveSandboxCapacity: callbacks?.reserveSandboxCapacity
       ? (providerAttempt) =>
           callbacks.reserveSandboxCapacity!('daytona', providerAttempt)
       : undefined,
     releaseSandboxCapacity: callbacks?.releaseSandboxCapacity,
+    recordCleanupIntent: callbacks?.recordDaytonaCleanupIntent,
     startAdmission: daytonaSandboxStartAdmission,
     cancellationSignal: callbacks?.cancellationSignal,
   });
@@ -1626,6 +1842,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
       ) {
         let sandboxForAttempt: DaytonaSandbox | null = null;
         let resourceAccounting: Promise<void> | null = null;
+        // How far this sandbox's start got. The retry decision below reads it.
+        let commandSent = false;
+        let parkStarted = false;
         const attemptStartedAt = Date.now();
         try {
           const acquiredSandbox = await Promise.race([
@@ -1705,6 +1924,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               }
             },
             cancellation: cancellationPromise,
+            onCommandSending: () => {
+              commandSent = true;
+            },
             onCommandAccepted: async () => {
               commandAcceptedAtMs = Date.now();
               emitDaytonaStage(callbacks, config.context, 'execute:accepted', {
@@ -1716,6 +1938,7 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
               });
             },
             onReady: async (ready) => {
+              parkStarted = true;
               const heartbeatReadyAtMs = Date.now();
               emitDaytonaStage(
                 callbacks,
@@ -1922,44 +2145,74 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
           }
           const retryReason = isRetryableUnstartedDaytonaRunner(error)
             ? 'runner_startup_not_live'
-            : isRetryableDaytonaInfrastructureFailure(error)
-              ? 'daytona_infrastructure'
-              : null;
+            : sandboxForAttempt &&
+                !parkStarted &&
+                isFailedDaytonaRunnerStart(error)
+              ? 'runner_start_failed'
+              : isRetryableDaytonaInfrastructureFailure(error)
+                ? 'daytona_infrastructure'
+                : null;
+          const retireSandboxForAttempt = async () => {
+            if (!sandboxForAttempt) return;
+            activeAcquiredResource!.billingEndedAt = Date.now();
+            await reportRetiringRuntimeResource(activeAcquiredResource!);
+            try {
+              await sandboxCleanup.stashActiveSandboxForRetry();
+            } catch (retirementError) {
+              if (runtimeResourceReportErrors.has(retirementError))
+                throw retirementError;
+              // Preserve both causes without feeding a retryable startup
+              // cause back into replacement admission. The cleanup manager
+              // retains the active resource and its single retirement promise.
+              const failure = new Error(
+                `Daytona sandbox retirement unconfirmed: ${formatDaytonaError(retirementError)}. Original startup failure: ${formatDaytonaError(error)}`,
+                { cause: error },
+              );
+              failure.name = 'DaytonaSandboxRetirementUnconfirmedError';
+              Object.assign(failure, { retirementError });
+              throw failure;
+            }
+          };
+          // A sandbox that never got a command has no runner, so this attempt
+          // can replace it. Once a command went out, a runner may live until
+          // Daytona finishes a delete it has only accepted, and a replacement
+          // here would pass the same attempt fence and could be released with
+          // it. That sandbox moves to a new attempt, whose fence rejects it.
           if (
             executionAttempt < DAYTONA_INFRASTRUCTURE_MAX_ATTEMPTS &&
-            retryReason
+            retryReason &&
+            !commandSent
           ) {
             emitDaytonaStage(callbacks, config.context, 'execute:retry', {
               sandboxId: sandboxForAttempt?.id ?? null,
               attempt: executionAttempt + 1,
               reason: retryReason,
               error: error instanceof Error ? error.message : String(error),
+              ...startFailureForLog(error),
               elapsedMs: Date.now() - startedAt,
             });
-            if (sandboxForAttempt) {
-              activeAcquiredResource!.billingEndedAt = Date.now();
-              await reportRetiringRuntimeResource(activeAcquiredResource!);
-              try {
-                await sandboxCleanup.stashActiveSandboxForRetry();
-              } catch (retirementError) {
-                if (runtimeResourceReportErrors.has(retirementError))
-                  throw retirementError;
-                // Preserve both causes without feeding a retryable startup
-                // cause back into replacement admission. The cleanup manager
-                // retains the active resource and its single retirement promise.
-                const failure = new Error(
-                  `Daytona sandbox retirement unconfirmed: ${formatDaytonaError(retirementError)}. Original startup failure: ${formatDaytonaError(error)}`,
-                  { cause: error },
-                );
-                failure.name = 'DaytonaSandboxRetirementUnconfirmedError';
-                Object.assign(failure, { retirementError });
-                throw failure;
-              }
-            }
+            await retireSandboxForAttempt();
             throwIfCancellationRequested();
             acquiredSandboxPromise = sandboxLifecycle.createFreshSandbox();
             continue;
           }
+          if (
+            retryReason === 'runner_startup_not_live' ||
+            retryReason === 'runner_start_failed'
+          ) {
+            // No runner was released. Retire this sandbox first: the new
+            // attempt starts clean.
+            emitDaytonaStage(callbacks, config.context, 'execute:deferred', {
+              sandboxId: sandboxForAttempt?.id ?? null,
+              attempt: executionAttempt,
+              reason: retryReason,
+              ...startFailureForLog(error),
+              elapsedMs: Date.now() - startedAt,
+            });
+            await retireSandboxForAttempt();
+            throwIfCancellationRequested();
+            throw recoveryForRunnerStartupNotLive(error);
+          }
           throw error;
         }
       }