deepline 0.3.460 → 0.3.462

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts

@@ -7,6 +7,7 @@ import {
 } from '../types';
 import { buildPlayRunnerBundle } from '../bundle';
 import { findPlayRunnerResult, parsePlayRunnerEvents } from '../runner-events';
+import { logSandboxCreateOutcome } from '../sandbox-create-outcome';
 import type {
   PlayRunnerExecutionConfig,
   PlayRunnerResult,
@@ -21,6 +22,7 @@ import {
   MODAL_SANDBOX_MEMORY_MIB,
   loadModalClientConfig,
   loadModalRequiredConfig,
+  withModalCallSignal,
 } from '../../modal-runtime-config';
 import type { PlaySandboxRuntimeLimits } from '../../../plays/sandbox-runtime-limits';
 import { validateDaytonaExecutionContext } from './daytona-lifecycle';
@@ -33,6 +35,7 @@ import {
   isPlayRunRecoveryError,
   recoveryForOpenModalCircuit,
   recoveryForModalSandboxCreateFailure,
+  recoveryForRunnerStartupNotLive,
 } from '../../play-run-recovery-policy';
 import { shouldInjectModalSandboxCreateResourceExhausted } from '../../test-runtime-seams';
 import { RUNTIME_RELIABILITY_POLICY } from '../../runtime-reliability-policy';
@@ -44,8 +47,19 @@ const MODAL_SANDBOX_CREATE_TIMEOUT_MS =
 const MODAL_PAYLOAD_UPLOAD_TIMEOUT_MS = 90_000;
 const MODAL_TERMINATION_TIMEOUT_MS = 10_000;
 const MODAL_DETACHED_INSPECTION_TIMEOUT_MS = 10_000;
+/** Completion-marker reads still out, after their deadline or not. */
+export const MODAL_RUNTIME_COMPLETION_READS_MAX = 8;
+let modalRuntimeCompletionReadsOutstanding = 0;
+/**
+ * A state-only read serves the fixed controller's liveness sweep, so it ends
+ * well inside that sweep's 10-second pass. The deadline cancels the call, so
+ * a provider that never answers leaves no abandoned read holding a slot.
+ */
+export const MODAL_SANDBOX_STATE_READ_TIMEOUT_MS = 5_000;
 const MODAL_CAPACITY_RELEASE_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
 const MODAL_LATE_CREATE_PERSIST_RETRY_DELAYS_MS = [0, 100, 500, 2_000] as const;
+// The SDK does not export gRPC status codes.
+const MODAL_GRPC_STATUS_NOT_FOUND = 5;
 type ModalSandbox = Awaited<ReturnType<ModalClient['sandboxes']['create']>>;
 
 class ModalPayloadUploadTimeoutError extends Error {
@@ -57,6 +71,18 @@ class ModalPayloadUploadTimeoutError extends Error {
   }
 }
 
+/**
+ * A file did not reach the sandbox because Modal reports the sandbox
+ * unavailable, which another sandbox can outrun. Staging sends no command, so
+ * nothing could run. By name, as `isModalSandboxNotFoundError` reads the SDK's
+ * errors. Other write errors stay loud: the SDK folds a refused credential into
+ * the same `SandboxFilesystemError` as a provider fault, and a stalled upload
+ * may be input too large for any sandbox.
+ */
+function isUndeliveredModalRunnerFailure(error: unknown): boolean {
+  return error instanceof Error && error.name === 'NotFoundError';
+}
+
 async function writeModalFileWithDeadline(input: {
   sandbox: ModalSandbox;
   content: Buffer;
@@ -205,13 +231,15 @@ function emitModalStage(
   );
 }
 
-async function confirmDetachedModalRunnerReady(input: {
+export async function confirmDetachedModalRunnerReady(input: {
   readiness: NonNullable<PlayRunnerCallbacks['readDetachedRunnerReadiness']>;
   baselineHeartbeatAt: string | null;
   cancellation?: Promise<never>;
+  timeoutMs?: number;
 }): Promise<void> {
   const cancellation = input.cancellation ?? new Promise<never>(() => {});
-  const deadline = Date.now() + MODAL_RUNNER_READY_TIMEOUT_MS;
+  const timeoutMs = input.timeoutMs ?? MODAL_RUNNER_READY_TIMEOUT_MS;
+  const deadline = Date.now() + timeoutMs;
   let delayMs = 50;
   while (Date.now() < deadline) {
     const response = await Promise.race([
@@ -225,8 +253,10 @@ async function confirmDetachedModalRunnerReady(input: {
     ]);
     delayMs = Math.min(1_000, delayMs * 2);
   }
-  throw new Error(
-    `RUNTIME_SANDBOX_START_FAILED: Modal accepted the runner command, but the scheduler did not observe play-runner liveness within ${MODAL_RUNNER_READY_TIMEOUT_MS}ms. Customer play execution was not parked.`,
+  throw recoveryForRunnerStartupNotLive(
+    new Error(
+      `RUNTIME_SANDBOX_START_FAILED: Modal accepted the runner command, but the scheduler did not observe play-runner liveness within ${timeoutMs}ms. Customer play execution was not parked.`,
+    ),
   );
 }
 
@@ -355,6 +385,7 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
         attempt: String(config.context.runAttempt ?? 0),
       };
       let sandbox: ModalSandbox;
+      let createSent = false;
       try {
         // This is deliberately adjacent to the real provider boundary. The
         // reservation proves the scheduler observes the same no-resource
@@ -394,6 +425,7 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
             ? { sandboxCapacityLeaseId: capacityReservation.leaseId }
             : {}),
         });
+        createSent = true;
         sandbox = await awaitModalSandboxCreate({
           create: modalConfig.client.sandboxes.create(app, image, {
             cpu: MODAL_SANDBOX_CPU_CORES,
@@ -409,6 +441,12 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
               : {}),
           }),
           onLateSuccess: async (lateSandbox) => {
+            logSandboxCreateOutcome({
+              provider: 'modal',
+              outcome: 'succeeded',
+              runId: config.context.runId,
+              reason: 'late',
+            });
             // Do not directly delete a late sandbox. First store it as a
             // durable cleanup obligation, then let the normal cleanup worker
             // perform provider-confirmed deletion, capacity release, and
@@ -467,7 +505,25 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
             await callbacks?.onAmbiguousModalCreateResolved?.();
           },
         });
+        logSandboxCreateOutcome({
+          provider: 'modal',
+          outcome: 'succeeded',
+          runId: config.context.runId,
+        });
       } catch (error) {
+        if (createSent) {
+          logSandboxCreateOutcome({
+            provider: 'modal',
+            outcome:
+              error instanceof ModalSandboxCreateOutcomeUnknownError
+                ? 'unknown'
+                : 'failed',
+            runId: config.context.runId,
+            reason:
+              recoveryForModalSandboxCreateFailure(error)?.decision.reason ??
+              null,
+          });
+        }
         if (
           capacityReservation &&
           !(error instanceof ModalSandboxCreateOutcomeUnknownError)
@@ -604,7 +660,17 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
               emitModalStage(config.context, stage, extra),
           }),
           cancellationPromise,
-        ]);
+        ]).catch((error: unknown) => {
+          // The runner never reached this sandbox. `finally` terminates it,
+          // and the defer starts a new attempt on a new sandbox.
+          if (
+            !callbacks?.cancellationSignal?.aborted &&
+            isUndeliveredModalRunnerFailure(error)
+          ) {
+            throw recoveryForRunnerStartupNotLive(error);
+          }
+          throw error;
+        });
         runtimeTiming.modalUploadMs = Date.now() - uploadStartedAt;
         const baselineHeartbeatAt = await Promise.race([
           captureDetachedDaytonaRunnerReadinessBaseline({
@@ -628,10 +694,10 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
           provider: 'modal',
           sandboxId: sandbox.sandboxId,
         });
-        // A readiness observation after the command is accepted is ambiguous:
-        // the detached runner may already have received its durable park and
-        // started customer code while this worker cannot read the scheduler.
-        // Never translate that ambiguity into a fresh sandbox retry.
+        // The runner holds customer code until this worker parks the attempt,
+        // and that park comes after this check. A runner that never reports
+        // live therefore never released anything, and the scheduler's defer
+        // starts a new attempt whose fence rejects it.
         await confirmDetachedModalRunnerReady({
           readiness: readRunnerReadiness,
           baselineHeartbeatAt,
@@ -746,6 +812,46 @@ export const modalPlayRunnerBackend: PlayRunnerBackend = {
   },
 };
 
+/**
+ * `poll()` and `terminate()` pass Modal's gRPC failure through unchanged, so
+ * Modal not knowing an exact sandbox id arrives as a gRPC NOT_FOUND. The SDK's
+ * `NotFoundError` is what its other lookups raise for the same fact.
+ *
+ * Use this only on those calls. `sandbox.filesystem` also raises
+ * `NotFoundError` for transient failures such as UNAVAILABLE, so a failed file
+ * read proves nothing about the sandbox.
+ */
+function isModalSandboxNotFoundError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false;
+  if (error.name === 'NotFoundError') return true;
+  return (
+    error.name === 'ClientError' &&
+    (error as { code?: unknown }).code === MODAL_GRPC_STATUS_NOT_FOUND
+  );
+}
+
+/**
+ * What Modal says about one sandbox. A failed read throws: it is not a state,
+ * and a stale runner must stay unproven rather than be called lost.
+ */
+async function readModalSandboxState(
+  sandbox: ModalSandbox,
+): Promise<
+  | { kind: 'running' }
+  | { kind: 'stopped'; returnCode: number }
+  | { kind: 'missing' }
+> {
+  try {
+    const returnCode = await sandbox.poll();
+    return returnCode === null
+      ? { kind: 'running' }
+      : { kind: 'stopped', returnCode };
+  } catch (error) {
+    if (isModalSandboxNotFoundError(error)) return { kind: 'missing' };
+    throw error;
+  }
+}
+
 export async function deleteModalSandboxById(input: {
   sandboxId: string;
   expectedAppId?: string | null;
@@ -768,17 +874,19 @@ export async function deleteModalSandboxById(input: {
   }
   try {
     const { client } = await loadModalClientConfig();
-    for await (const sandbox of client.sandboxes.list({
-      appId: expectedAppId,
-    })) {
-      if (sandbox.sandboxId !== input.sandboxId) continue;
-      await sandbox.terminate({ wait: true });
-      return { kind: 'deleted', appId: expectedAppId };
-    }
-    // Cleanup means ensure absent. An exact app-scoped inventory proving the
-    // id is absent satisfies the obligation without an unsafe cross-app get.
-    return { kind: 'already_absent', appId: expectedAppId };
+    // Never walk the app here: each walk spends Modal's LIST limit, and on
+    // Oct 29 that throttled deletes while finished sandboxes kept billing.
+    // `fromId` makes no call. Modal has no public read of a sandbox's app, so
+    // the exact id persisted at create is the authority and `terminate` is
+    // the first thing to learn whether it still exists.
+    const sandbox = await client.sandboxes.fromId(input.sandboxId);
+    await sandbox.terminate({ wait: true });
+    return { kind: 'deleted', appId: expectedAppId };
   } catch (error) {
+    // Cleanup means ensure absent, and this answer is about this exact id.
+    if (isModalSandboxNotFoundError(error)) {
+      return { kind: 'already_absent', appId: expectedAppId };
+    }
     console.warn('[play-runner.modal.reclaim_sandbox_delete_failed]', {
       sandboxId: input.sandboxId,
       expectedAppId,
@@ -797,6 +905,8 @@ export async function deleteModalSandboxById(input: {
  * Read the provider inventory using the exact app + tag selector persisted
  * before a Modal create call.  This is deliberately a tiny provider adapter:
  * the scheduler owns claim/fencing, resource recording, cleanup, and policy.
+ * A list is required here: a create with an unknown outcome has no sandbox id
+ * to read, and these tags name one attempt, so this does not walk the app.
  */
 export async function findModalSandboxIdsByTags(input: {
   modalAppId: string;
@@ -827,14 +937,16 @@ export async function findModalSandboxIdsByTags(input: {
 
 /**
  * Best-effort terminal recovery for an already-detached Modal runner. This
- * reads only the payload wrapper's captured files; it never executes Play
- * code or launches a replacement sandbox.
+ * reads the sandbox's state and the payload wrapper's captured files; it never
+ * executes Play code or launches a replacement sandbox.
  */
 export async function inspectDetachedModalRunner(input: {
   sandboxId: string;
   outputPath: string;
   exitCodePath: string;
   expectedAppId?: string | null;
+  /** Answer from the sandbox's state alone. See the reconciliation seam. */
+  skipCapturedOutput?: boolean;
 }): Promise<{
   exitCode: number | null;
   result: PlayRunnerResult | null;
@@ -852,15 +964,59 @@ export async function inspectDetachedModalRunner(input: {
     };
   }
 
-  let timer: ReturnType<typeof setTimeout> | undefined;
+  const timeoutMs = input.skipCapturedOutput
+    ? MODAL_SANDBOX_STATE_READ_TIMEOUT_MS
+    : MODAL_DETACHED_INSPECTION_TIMEOUT_MS;
+  // The deadline cancels Modal's calls as well as the wait, so a read that
+  // times out is over and a stuck provider cannot pile reads up behind it.
+  const deadline = new AbortController();
+  const timer = setTimeout(() => {
+    deadline.abort(
+      new Error(
+        `Modal detached-runner inspection exceeded ${timeoutMs}ms.`,
+      ),
+    );
+  }, timeoutMs);
   try {
     return await Promise.race([
-      (async () => {
+      withModalCallSignal(deadline.signal, async () => {
         const { client } = await loadModalClientConfig();
-        for await (const sandbox of client.sandboxes.list({
-          appId: expectedAppId,
-        })) {
-          if (sandbox.sandboxId !== input.sandboxId) continue;
+        const sandbox = await client.sandboxes.fromId(input.sandboxId);
+        try {
+          // Proof of loss is a read of this one sandbox. The app list holds
+          // only running sandboxes and is rate limited, so absence from it
+          // proves nothing.
+          const state = await readModalSandboxState(sandbox);
+          if (state.kind === 'missing') {
+            return {
+              exitCode: null,
+              result: null,
+              diagnosis: {
+                stage: 'sandbox_missing',
+                detail: 'Modal has no sandbox with this id.',
+              },
+            };
+          }
+          if (state.kind === 'stopped') {
+            // Files cannot be read from a stopped sandbox. The return code is
+            // the sandbox's own, not the runner's, and 137 covers both a spend
+            // limit kill and a terminate, so it stays out of `exitCode`.
+            return {
+              exitCode: null,
+              result: null,
+              diagnosis: {
+                stage: 'sandbox_stopped',
+                detail: `Modal sandbox ended with return code ${state.returnCode}.`,
+              },
+            };
+          }
+          if (input.skipCapturedOutput) {
+            return {
+              exitCode: null,
+              result: null,
+              diagnosis: { stage: 'sandbox_not_ended', detail: null },
+            };
+          }
           const [output, exitCode] = await Promise.allSettled([
             sandbox.filesystem.readText(input.outputPath),
             sandbox.filesystem.readText(input.exitCodePath),
@@ -892,24 +1048,18 @@ export async function inspectDetachedModalRunner(input: {
               ? { stage: 'salvaged', detail: null }
               : { stage: 'result_missing', detail: null },
           };
+        } finally {
+          sandbox.detach();
         }
-        return {
-          exitCode: null,
-          result: null,
-          diagnosis: {
-            stage: 'sandbox_missing',
-            detail: 'Sandbox was not found in the launch-pinned Modal app.',
-          },
-        };
-      })(),
+      }),
+      // An auth token refresh is shared and keeps running, so do not wait on
+      // whatever the abort could not cancel.
       new Promise<never>((_resolve, reject) => {
-        timer = setTimeout(() => {
-          reject(
-            new Error(
-              `Modal detached-runner inspection exceeded ${MODAL_DETACHED_INSPECTION_TIMEOUT_MS}ms.`,
-            ),
-          );
-        }, MODAL_DETACHED_INSPECTION_TIMEOUT_MS);
+        deadline.signal.addEventListener(
+          'abort',
+          () => reject(deadline.signal.reason),
+          { once: true },
+        );
       }),
     ]);
   } catch (error) {
@@ -927,7 +1077,7 @@ export async function inspectDetachedModalRunner(input: {
       },
     };
   } finally {
-    if (timer) clearTimeout(timer);
+    clearTimeout(timer);
   }
 }
 
@@ -938,20 +1088,63 @@ export async function readDetachedModalRuntimeCompletion(input: {
 }): Promise<number | null> {
   const expectedAppId = input.expectedAppId?.trim() || null;
   if (!expectedAppId) return null;
+  // A read past its deadline keeps running, so a Modal that never answers
+  // would otherwise gather one per settlement. The marker is only telemetry.
+  if (
+    modalRuntimeCompletionReadsOutstanding >= MODAL_RUNTIME_COMPLETION_READS_MAX
+  ) {
+    console.warn('[play-runner.modal.runtime_completion_marker_unavailable]', {
+      sandboxId: input.sandboxId,
+      expectedAppId,
+      error: `${modalRuntimeCompletionReadsOutstanding} earlier reads have not answered.`,
+    });
+    return null;
+  }
+  modalRuntimeCompletionReadsOutstanding += 1;
+  const read = readModalRuntimeCompletionMarker(input, expectedAppId).finally(
+    () => {
+      modalRuntimeCompletionReadsOutstanding -= 1;
+    },
+  );
+  // Terminal settlement awaits this read. Modal's calls have no RPC deadline
+  // and retry throttling without limit, so settle without the marker rather
+  // than leave a finished run `running`. The read cannot be cancelled.
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  return await Promise.race([
+    read,
+    new Promise<null>((resolve) => {
+      timer = setTimeout(() => {
+        console.warn(
+          '[play-runner.modal.runtime_completion_marker_unavailable]',
+          {
+            sandboxId: input.sandboxId,
+            expectedAppId,
+            error: `No answer within ${MODAL_DETACHED_INSPECTION_TIMEOUT_MS}ms.`,
+          },
+        );
+        resolve(null);
+      }, MODAL_DETACHED_INSPECTION_TIMEOUT_MS);
+    }),
+  ]).finally(() => clearTimeout(timer));
+}
+
+async function readModalRuntimeCompletionMarker(
+  input: { sandboxId: string; runtimeCompletedPath: string },
+  expectedAppId: string,
+): Promise<number | null> {
+  let sandbox: ModalSandbox | undefined;
   try {
     const { client } = await loadModalClientConfig();
-    for await (const sandbox of client.sandboxes.list({
-      appId: expectedAppId,
-    })) {
-      if (sandbox.sandboxId !== input.sandboxId) continue;
-      const marker = JSON.parse(
-        await sandbox.filesystem.readText(input.runtimeCompletedPath),
-      ) as { at?: unknown };
-      return typeof marker.at === 'number' && Number.isFinite(marker.at)
-        ? marker.at
-        : null;
-    }
-    return null;
+    sandbox = await client.sandboxes.fromId(input.sandboxId);
+    // The marker went with a stopped sandbox. Check first: a file read of a
+    // sandbox with no task yet can wait minutes inside the SDK.
+    if ((await readModalSandboxState(sandbox)).kind !== 'running') return null;
+    const marker = JSON.parse(
+      await sandbox.filesystem.readText(input.runtimeCompletedPath),
+    ) as { at?: unknown };
+    return typeof marker.at === 'number' && Number.isFinite(marker.at)
+      ? marker.at
+      : null;
   } catch (error) {
     console.warn('[play-runner.modal.runtime_completion_marker_unavailable]', {
       sandboxId: input.sandboxId,
@@ -959,5 +1152,7 @@ export async function readDetachedModalRuntimeCompletion(input: {
       error: error instanceof Error ? error.message : String(error),
     });
     return null;
+  } finally {
+    sandbox?.detach();
   }
 }

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts

@@ -32,6 +32,7 @@ export type RuntimeSandboxReconciliationAdapter = {
   inspectDetachedRunner(
     ref: RuntimeSandboxRef,
     suspension: DetachedRunnerSuspension,
+    options?: RuntimeSandboxDetachedInspectionOptions,
   ): Promise<RuntimeSandboxDetachedRunnerInspection | null>;
   readWorkCompletedAt(
     ref: RuntimeSandboxRef,
@@ -44,6 +45,7 @@ export type RuntimeSandboxReconciliation = {
   inspectDetachedRunner(
     ref: RuntimeSandboxRef,
     suspension: DetachedRunnerSuspension,
+    options?: RuntimeSandboxDetachedInspectionOptions,
   ): Promise<RuntimeSandboxDetachedRunnerInspection | null>;
   readWorkCompletedAt(
     ref: RuntimeSandboxRef,
@@ -57,11 +59,28 @@ export type RuntimeSandboxDetachedRunnerInspection = {
   diagnosis: { stage: string; detail: string | null };
 };
 
+export type RuntimeSandboxDetachedInspectionOptions = {
+  /**
+   * Prove only whether the sandbox itself has ended. A runner's captured output
+   * grows with customer logs, so a small fixed controller must never load it,
+   * and an ended sandbox has no files left to read. A sandbox that has not
+   * ended stays unproven here, and its parked attempt can still salvage it.
+   */
+  skipCapturedOutput?: boolean;
+};
+
 export type RuntimeSandboxDeleteOutcome =
   | {
       kind: 'deleted' | 'already_absent';
       routingDomain: string | null;
     }
+  | {
+      kind: 'delete_pending';
+      routingDomain: string | null;
+      code: string;
+      detail: string;
+      deleteAccepted: boolean;
+    }
   | {
       kind: 'timed_out' | 'rate_limited' | 'failed';
       routingDomain: string | null;
@@ -116,8 +135,8 @@ export function createRuntimeSandboxReconciliation(input: {
   };
   return {
     delete: (ref) => resolve(ref).delete(ref),
-    inspectDetachedRunner: (ref, suspension) =>
-      resolve(ref).inspectDetachedRunner(ref, suspension),
+    inspectDetachedRunner: (ref, suspension, ...options) =>
+      resolve(ref).inspectDetachedRunner(ref, suspension, ...options),
     readWorkCompletedAt: (ref, path) =>
       resolve(ref).readWorkCompletedAt(ref, path),
   };
@@ -205,7 +224,7 @@ export const defaultRuntimeSandboxReconciliation =
             routingDomain: outcome.organizationId,
           };
         },
-        inspectDetachedRunner: (ref, suspension) =>
+        inspectDetachedRunner: (ref, suspension, options) =>
           inspectDetachedDaytonaRunner({
             sandboxId: ref.resourceId,
             sessionId: suspension.sessionId,
@@ -213,6 +232,9 @@ export const defaultRuntimeSandboxReconciliation =
             outputPath: suspension.outputPath,
             exitCodePath: suspension.exitCodePath,
             expectedOrganizationId: ref.routingDomain,
+            ...(options?.skipCapturedOutput
+              ? { skipCapturedOutput: true }
+              : {}),
           }),
         readWorkCompletedAt: (ref, path) =>
           readDetachedDaytonaRuntimeCompletion({
@@ -233,12 +255,15 @@ export const defaultRuntimeSandboxReconciliation =
             routingDomain: outcome.appId,
           };
         },
-        inspectDetachedRunner: (ref, suspension) =>
+        inspectDetachedRunner: (ref, suspension, options) =>
           inspectDetachedModalRunner({
             sandboxId: ref.resourceId,
             outputPath: suspension.outputPath,
             exitCodePath: suspension.exitCodePath,
             expectedAppId: ref.routingDomain,
+            ...(options?.skipCapturedOutput
+              ? { skipCapturedOutput: true }
+              : {}),
           }),
         readWorkCompletedAt: (ref, path) =>
           readDetachedModalRuntimeCompletion({

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/sandbox-create-outcome.ts

@@ -0,0 +1,27 @@
+export type SandboxCreateOutcome = 'succeeded' | 'failed' | 'unknown';
+
+/**
+ * One line for each create request that reached a provider, read by the
+ * provider outage alert. Log it only where a request was really sent: a run held
+ * behind the open Modal circuit makes no request, and account reads prove
+ * nothing, since Modal reported no limit on Oct 31 while refusing a real create.
+ * `unknown` means the request outlived its client deadline and may still land.
+ */
+export function logSandboxCreateOutcome(input: {
+  provider: 'daytona' | 'modal';
+  outcome: SandboxCreateOutcome;
+  runId: string | null | undefined;
+  reason?: string | null;
+}): void {
+  console.info('[runtime.sandbox.create_outcome]', {
+    event: 'runtime.sandbox.create_outcome',
+    provider: input.provider,
+    outcome: input.outcome,
+    runId: input.runId ?? null,
+    reason: input.reason ?? null,
+    runtimeEnvironment:
+      process.env.DEEPLINE_RUNTIME_ENVIRONMENT === 'preview'
+        ? 'preview'
+        : 'production',
+  });
+}

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts

@@ -46,6 +46,17 @@ export type PlayRunnerRuntimeResource = {
   sandboxCapacityLeaseId?: string;
 };
 
+/** Exact durable owner identity required before reconciling an ambiguous
+ * Daytona create by deleting the sandbox it produced. */
+export type PlayRunnerDaytonaCleanupIntent = {
+  leaseId: string;
+  providerOperationToken: string;
+  orgId: string;
+  runId: string;
+  sandboxId: string;
+  routingDomain: string;
+};
+
 /**
  * Durable identity for a Modal create that can outlive the caller's timeout.
  * These fields are only provider routing and correlation data; payloads,
@@ -183,9 +194,19 @@ export interface PlayRunnerCallbacks {
   reserveSandboxCapacity?: (
     provider: 'daytona' | 'modal' | 'agentcore_runtime',
     providerAttempt?: number,
-  ) => Promise<{ leaseId: string; providerOperationToken: string | null }>;
+  ) => Promise<{
+    leaseId: string;
+    providerOperationToken: string | null;
+    resumeExistingProviderOperation?: boolean;
+    resumePendingDelete?: boolean;
+    existingProviderOperationAgeMs?: number;
+  }>;
   /** Release an unbound reservation when provider create never produced a resource. */
   releaseSandboxCapacity?: (leaseId: string) => Promise<void>;
+  /** Persist a cleanup obligation before removing an ambiguously-created sandbox. */
+  recordDaytonaCleanupIntent?: (
+    intent: PlayRunnerDaytonaCleanupIntent,
+  ) => Promise<void>;
   /**
    * Scheduler-owned dependency circuit admission. A false result means no
    * provider call is safe to make yet; the backend must return/throw a typed