deepline 0.1.208 → 0.1.210

package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts

@@ -56,7 +56,25 @@ export interface PacingPermit {
   release(): void;
 }
 
+/**
+ * Discriminates the backend so the Governor can adapt its policy to it. Only
+ * `app_runtime_postgres` (the Absurd/Node substrate's DB-backed pacer) is
+ * db-authoritative: it runs the whole token-bucket + AIMD in Postgres, so the
+ * Governor's in-memory adaptive admission must become a passthrough and defer
+ * pacing to the row. The in-memory and coordinator backends leave `kind`
+ * undefined and keep the Governor's in-memory adaptive admission (byte-identical
+ * to today).
+ */
+export type RateStateBackendKind = 'app_runtime_postgres';
+
 export interface RateStateBackend {
+  /**
+   * When present, tells the Governor this backend owns pacing authoritatively.
+   * Absent on the in-memory and coordinator backends — those keep the Governor's
+   * in-memory adaptive admission.
+   */
+  readonly kind?: RateStateBackendKind;
+
   /**
    * Block until one outbound call is permitted for `bucketId` under all `rules`
    * (request windows always; per-rule `maxConcurrency` only on the in-memory

package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts

@@ -34,6 +34,17 @@ export type PlayRunnerRateStateAcquireResult = {
   waitMs: number;
   /** One independently-expiring concurrency reservation per granted permit. */
   leaseIds?: string[];
+  /**
+   * Absolute epoch-ms until which the pacer's own cooldown holds (internally
+   * capped, e.g. 45min probe cadence). Advisory: exposes the current backoff to
+   * observers; the client already sleeps on `waitMs`.
+   */
+  coolUntilMs?: number;
+  /**
+   * The most recent server Retry-After stored verbatim (uncapped) as an absolute
+   * epoch-ms deadline. Purely observational — never gates a grant on its own.
+   */
+  claimedRetryAtMs?: number;
 };
 
 export type PlayRunnerRateStateReleaseInput = {
@@ -126,9 +137,29 @@ export interface PlayRunnerContextConfig {
    * path or on a pre-release parent.
    */
   absurdReleaseId?: string | null;
+  /**
+   * Push-execution wiring for the Daytona sandbox runner. When present the
+   * runner keeps the Absurd run claim alive itself (heartbeating the receipt
+   * gateway on a `leaseSeconds / 3` cadence) instead of the worker babysitting a
+   * held `executeCommand` await. Absent for the worker-babysat local-process
+   * backend and every non-Daytona runtime, which keep their existing behavior.
+   */
+  runnerPushExecution?: RunnerPushExecutionConfig | null;
   governance?: GovernanceSnapshot | null;
 }
 
+/**
+ * Runner-side push-execution parameters. The receipt gateway resolves the run's
+ * queue server-side, so the runner only needs the lease window (to derive the
+ * heartbeat cadence) and the run identity it heartbeats.
+ */
+export interface RunnerPushExecutionConfig {
+  /** Absurd claim lease window in seconds; heartbeat cadence is this / 3. */
+  leaseSeconds: number;
+  /** The run the runner heartbeats. Must match the executor token's run scope. */
+  runId: string;
+}
+
 export interface PlayRunnerExecutionConfig {
   artifact: PlayBundleArtifact;
   artifactTransport?: {
@@ -172,15 +203,6 @@ export type PlayRunnerEvent =
       result: PlayRunnerResult;
     };
 
-export type PlayRunnerDeferredRuntimeTask = {
-  kind: 'daytona_sandbox_cleanup';
-  sandboxId: string;
-  billingStartedAt: number;
-  cpu: number;
-  memoryGiB: number;
-  diskGiB: number;
-};
-
 export type PlayRunnerRuntimeTiming = {
   backend: 'daytona';
   daytonaCreateMs?: number;
@@ -202,7 +224,6 @@ export type PlayRunnerResult =
       inserted?: number;
       skipped?: number;
       runtimeTiming?: PlayRunnerRuntimeTiming;
-      deferredRuntimeTasks?: PlayRunnerDeferredRuntimeTask[];
     }
   | {
       status: 'suspended';
@@ -216,7 +237,6 @@ export type PlayRunnerResult =
       inserted?: number;
       skipped?: number;
       runtimeTiming?: PlayRunnerRuntimeTiming;
-      deferredRuntimeTasks?: PlayRunnerDeferredRuntimeTask[];
     }
   | {
       status: 'failed';
@@ -230,5 +250,4 @@ export type PlayRunnerResult =
       inserted?: number;
       skipped?: number;
       runtimeTiming?: PlayRunnerRuntimeTiming;
-      deferredRuntimeTasks?: PlayRunnerDeferredRuntimeTask[];
     };

package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts

@@ -44,6 +44,89 @@ export class WorkspaceStorageNotReadyError extends Error {
   }
 }
 
+export const PROVIDER_EXHAUSTED_CODE = 'PROVIDER_EXHAUSTED';
+
+/**
+ * Format an absolute epoch-ms retry deadline as a human UTC string for the
+ * user-facing PROVIDER_EXHAUSTED message. Contains no provider internals.
+ */
+export function formatProviderRetryAtUtc(retryAtMs: number): string {
+  const millis = Number(retryAtMs);
+  if (!Number.isFinite(millis)) {
+    return 'an unknown time';
+  }
+  return new Date(millis).toUTCString();
+}
+
+/**
+ * Build the user-facing PROVIDER_EXHAUSTED message. Names only the provider and
+ * the retry time — never Deepline provider spend, SQL, or tenant internals — so
+ * it is safe to surface verbatim past redaction.
+ */
+export function providerExhaustedMessage(input: {
+  provider: string;
+  retryAtMs: number;
+}): string {
+  return (
+    `${input.provider} is exhausted and asked us to retry after ${formatProviderRetryAtUtc(input.retryAtMs)}. ` +
+    'This call was skipped with no spend. ' +
+    'For higher provider throughput guarantees, talk to us about an enterprise plan.'
+  );
+}
+
+/**
+ * Thrown by the DB-authoritative pacer (`app_runtime_postgres` rate-state
+ * backend) when a provider's hold exceeds the tolerable wait
+ * (PROVIDER_EXHAUSTED_MAX_WAIT_MS). The call is skipped BEFORE any tool
+ * execution request, so nothing is dispatched and nothing is billed. Row-failure
+ * isolation still applies: this is a row/step outcome, not a run-fatal error.
+ */
+export class ProviderExhaustedError extends Error {
+  readonly code = PROVIDER_EXHAUSTED_CODE;
+  readonly provider: string;
+  /** ISO 8601 retry deadline (from the server Retry-After / cooldown). */
+  readonly retryAt: string;
+  constructor(input: {
+    provider: string;
+    retryAtMs: number;
+    cause?: unknown;
+  }) {
+    // Prefix the code token so `String(error)` and any persisted error string
+    // carry it across the serialization boundary back to the CLI, exactly like
+    // WorkspaceStorageNotReadyError.
+    super(
+      `${PROVIDER_EXHAUSTED_CODE}: ${providerExhaustedMessage({
+        provider: input.provider,
+        retryAtMs: input.retryAtMs,
+      })}`,
+    );
+    this.name = 'ProviderExhaustedError';
+    this.provider = input.provider;
+    this.retryAt = Number.isFinite(Number(input.retryAtMs))
+      ? new Date(Number(input.retryAtMs)).toISOString()
+      : new Date(0).toISOString();
+    if (input.cause !== undefined) {
+      (this as { cause?: unknown }).cause = input.cause;
+    }
+  }
+}
+
+/**
+ * Derive the human provider name for a PROVIDER_EXHAUSTED error from a pacing
+ * bucketId. Bucket ids are `${orgId}:${provider}`; the provider itself is
+ * `tool:${toolId}` for undeclared tools (see `defaultPacingForTool`). Strip the
+ * org prefix, then unwrap a `tool:` prefix down to the toolId so the message
+ * names something the user recognizes.
+ */
+export function providerNameFromBucketId(bucketId: string): string {
+  const separator = bucketId.indexOf(':');
+  const provider =
+    separator >= 0 ? bucketId.slice(separator + 1) : bucketId;
+  return provider.startsWith('tool:')
+    ? provider.slice('tool:'.length)
+    : provider;
+}
+
 export type PlayRunFailureDetails = {
   code: string;
   phase: string;
@@ -100,6 +183,26 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
       cause,
     };
   }
+  if (
+    error instanceof ProviderExhaustedError ||
+    /\bPROVIDER_EXHAUSTED\b/.test(cause)
+  ) {
+    // The human message (provider + retry time) already lives in `cause` after
+    // the code token prefix; surface it verbatim minus the token so the run
+    // terminal reads cleanly. Contains no provider spend or tenant internals by
+    // construction (see providerExhaustedMessage).
+    const stripped = cause.replace(
+      new RegExp(`^${PROVIDER_EXHAUSTED_CODE}:\\s*`),
+      '',
+    );
+    return {
+      code: PROVIDER_EXHAUSTED_CODE,
+      phase: 'runtime',
+      message: stripped,
+      retryable: false,
+      cause,
+    };
+  }
   if (
     error instanceof WorkspaceStorageNotReadyError ||
     /\bWORKSPACE_STORAGE_NOT_READY\b/.test(cause)

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -316,6 +316,58 @@ export function isTerminalPlayRunLedgerStatus(
   return TERMINAL_STATUSES.has(status);
 }
 
+/**
+ * Aggregate, row-level outcome of a run derived purely from the settled step
+ * progress. This is an ADDITIVE truthfulness signal, NOT a status: a run can be
+ * `status: 'completed'` (all steps ran to the end) while `failedRows > 0`
+ * because row-failure isolation persists failed rows for retry instead of
+ * failing the whole run. Readers use `hasRowFailures` to render a truthful
+ * "completed with failures" surface without inventing a new public run status.
+ *
+ * `completedRows` here is the reconciled successes-only count summed from the
+ * producer's final step progress (`completed`), which is the durable/persisted
+ * success count — NOT a live in-flight estimate. It is what should be compared
+ * against persisted dataset rows; the two diverging was the 21-vs-15 symptom of
+ * children being lost before their rows settled.
+ */
+export type PlayRunRowOutcomeSummary = {
+  completedRows: number;
+  failedRows: number;
+  totalRows: number;
+  hasRowFailures: boolean;
+};
+
+export function summarizeRunRowOutcomes(
+  snapshot: Pick<PlayRunLedgerSnapshot, 'stepsById'>,
+): PlayRunRowOutcomeSummary {
+  let completedRows = 0;
+  let failedRows = 0;
+  let totalRows = 0;
+  for (const step of Object.values(snapshot.stepsById)) {
+    const progress = step.progress;
+    if (!progress) continue;
+    // `completed` is successes-only; `failed` is settled-but-failed rows. Sum
+    // both across every map/step so a multi-map play reports the whole run's
+    // row outcome, not just the last step.
+    completedRows += Math.max(0, finiteNumber(progress.completed) ?? 0);
+    failedRows += Math.max(0, finiteNumber(progress.failed) ?? 0);
+    // Prefer the settled denominator; fall back to completed+failed when a
+    // producer never stamped `total` (per the terminal invariant those agree).
+    const stepTotal = finiteNumber(progress.total);
+    totalRows +=
+      stepTotal !== null && stepTotal >= 0
+        ? stepTotal
+        : Math.max(0, finiteNumber(progress.completed) ?? 0) +
+          Math.max(0, finiteNumber(progress.failed) ?? 0);
+  }
+  return {
+    completedRows,
+    failedRows,
+    totalRows,
+    hasRowFailures: failedRows > 0,
+  };
+}
+
 export function createEmptyPlayRunLedgerSnapshot(input: {
   runId: string;
   playName?: string | null;
@@ -529,6 +581,32 @@ function mergeMonotonicStepProgress(input: {
   };
 }
 
+/**
+ * Zero the in-flight row accounting on a settled step. `activeRows` (calls in
+ * flight) and `waitingRows` (rows queued behind rate/concurrency limits) are
+ * live-only counters: once a step is terminal nothing is in flight or waiting,
+ * so leaving the last observed values renders a contradiction like
+ * "status: completed · 1 in flight · waiting 29". This preserves the settled
+ * counts (completed/failed/total/completedRows/startedRows) and only clears the
+ * two live accumulators. Idempotent: dropping already-absent fields is a no-op.
+ */
+function clearInFlightProgressOnTerminal(
+  progress: PlayRunLedgerStepProgress | null | undefined,
+): PlayRunLedgerStepProgress | null | undefined {
+  if (!progress) return progress;
+  if (progress.activeRows === undefined && progress.waitingRows === undefined) {
+    return progress;
+  }
+  const cleared = { ...progress };
+  delete cleared.activeRows;
+  delete cleared.waitingRows;
+  return cleared;
+}
+
+function isTerminalStepStatus(status: PlayRunLedgerStepStatus): boolean {
+  return status === 'completed' || status === 'failed' || status === 'skipped';
+}
+
 function normalizeStepStatus(value: unknown): PlayRunLedgerStepStatus | null {
   const normalized = String(value ?? '')
     .trim()
@@ -838,11 +916,11 @@ function settleRunningStepsOnTerminal(
           completedAt: step.completedAt ?? occurredAt,
           updatedAt: occurredAt,
           progress: step.progress
-            ? {
+            ? clearInFlightProgressOnTerminal({
                 ...step.progress,
                 completedAt: step.progress.completedAt ?? occurredAt,
                 updatedAt: occurredAt,
-              }
+              })
             : step.progress,
         },
       ];
@@ -1043,19 +1121,30 @@ export function reducePlayRunLedgerEvent(
     case 'step.progress': {
       const current = base.stepsById[event.stepId];
       const runTerminal = isTerminalPlayRunLedgerStatus(base.status);
-      const progress = {
+      const mergedProgress = {
         ...mergeMonotonicStepProgress({
           current: current?.progress,
           next: event.progress,
         }),
         updatedAt: event.progress.updatedAt ?? occurredAt,
       };
+      // An explicit `completedAt` on THIS event is the producer's authoritative
+      // completion signal and always settles the step. Absent that, the
+      // `completed >= total` heuristic is a stale-estimate trap while rows are
+      // still in flight (total can shrink below the in-flight count under child
+      // fan-out), so it must NOT infer completion while active rows remain.
+      const eventSignalsCompletion =
+        typeof event.progress.completedAt === 'number';
+      const hasActiveRows =
+        typeof mergedProgress.activeRows === 'number' &&
+        mergedProgress.activeRows > 0;
       const inferredStatus =
-        typeof progress.completedAt === 'number' ||
-        (typeof progress.total === 'number' &&
-          typeof progress.completed === 'number' &&
-          progress.total > 0 &&
-          progress.completed >= progress.total)
+        eventSignalsCompletion ||
+        (!hasActiveRows &&
+          typeof mergedProgress.total === 'number' &&
+          typeof mergedProgress.completed === 'number' &&
+          mergedProgress.total > 0 &&
+          mergedProgress.completed >= mergedProgress.total)
           ? 'completed'
           : 'running';
       const status =
@@ -1085,6 +1174,11 @@ export function reducePlayRunLedgerEvent(
         (status === 'completed' || status === 'failed'
           ? (completedAt ?? event.progress.updatedAt ?? occurredAt)
           : null);
+      // Once the step is settled, drop the live in-flight/waiting accumulators
+      // so a terminal node never reports active or waiting rows.
+      const progress = isTerminalStepStatus(status)
+        ? clearInFlightProgressOnTerminal(mergedProgress)
+        : mergedProgress;
       const nextStep: PlayRunLedgerStepSnapshot = {
         ...(current ?? { stepId: event.stepId, status }),
         stepId: event.stepId,
@@ -1092,7 +1186,7 @@ export function reducePlayRunLedgerEvent(
         kind: event.kind ?? current?.kind,
         status,
         artifactTableNamespace:
-          progress.artifactTableNamespace ??
+          progress?.artifactTableNamespace ??
           current?.artifactTableNamespace ??
           null,
         startedAt,
@@ -1155,7 +1249,8 @@ export function reducePlayRunLedgerEvent(
             : null),
         completedAt,
         updatedAt: occurredAt,
-        progress: current?.progress ?? null,
+        // Terminal step: clear live in-flight/waiting accounting.
+        progress: clearInFlightProgressOnTerminal(current?.progress) ?? null,
       };
       const runningStepIds = base.orderedStepIds.filter((stepId) => {
         if (stepId === event.stepId) return false;

package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts

@@ -2,8 +2,10 @@ import { resolveTimingWindow } from './live-events';
 import {
   LOG_TAIL_LIMIT,
   normalizePlayRunLedgerSnapshot,
+  summarizeRunRowOutcomes,
   type PlayRunLedgerSnapshot,
   type PlayRunLedgerStepSnapshot,
+  type PlayRunRowOutcomeSummary,
 } from './run-ledger';
 import {
   isActivePlayRunLifecycleStatus,
@@ -76,6 +78,14 @@ export type PlayRunLiveSnapshot = {
   resultTableNamespace: string | null;
   nodeStates: PlayRunStreamNodeState[];
   activeNodeId: string | null;
+  /**
+   * Additive, terminal-only aggregate of row outcomes across all steps. Present
+   * once the run is terminal so readers can render a truthful "completed with
+   * failures" surface (row-failure isolation persists failed rows for retry, so
+   * a run legitimately reports `status: 'completed'` with `rowOutcomes.failedRows
+   * > 0`). Absent on non-terminal snapshots and on runs with no row/map steps.
+   */
+  rowOutcomes?: PlayRunRowOutcomeSummary;
 };
 
 export function normalizePlayRunLiveStatus(value: unknown): PlayRunLiveStatus {
@@ -179,9 +189,18 @@ function buildSnapshotFromLedger(
       completedAt: step.completedAt ?? null,
       updatedAt: step.updatedAt ?? null,
     }));
+  const liveStatus = normalizePlayRunLiveStatus(snapshot.status);
+  // Only surface the row-outcome aggregate once the run is terminal and it
+  // actually observed row/map steps — mid-flight `failed` counts churn, and a
+  // no-map play has no meaningful row outcome to report.
+  const rowOutcomes =
+    isTerminalPlayRunLiveStatus(liveStatus) &&
+    Object.keys(snapshot.stepsById).length > 0
+      ? summarizeRunRowOutcomes(snapshot)
+      : null;
   return {
     runId: snapshot.runId,
-    status: normalizePlayRunLiveStatus(snapshot.status),
+    status: liveStatus,
     createdAt: snapshot.createdAt ?? null,
     startedAt: snapshot.startedAt ?? null,
     finishedAt: snapshot.finishedAt ?? null,
@@ -195,6 +214,7 @@ function buildSnapshotFromLedger(
     resultTableNamespace: snapshot.resultTableNamespace ?? null,
     nodeStates,
     activeNodeId: snapshot.activeStepId ?? null,
+    ...(rowOutcomes ? { rowOutcomes } : {}),
   };
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts

@@ -1,10 +1,8 @@
 import type { Daytona } from '@daytonaio/sdk';
 import type {
-  PlayRunnerDeferredRuntimeTask,
   PlayRunnerExecutionConfig,
   PlayRunnerResult,
 } from '@shared_libs/play-runtime/protocol';
-import { shouldCleanupSandboxInBackground } from '@shared_libs/play-runtime/daytona-runtime-config';
 import { isIsolatedRuntimeSchedulerSchema } from '@shared_libs/play-runtime/runtime-scheduler-topology';
 
 const DAYTONA_CREATE_TIMEOUT_SECONDS = 10;
@@ -337,19 +335,19 @@ export function createOneShotDaytonaSandboxLifecycle(input: {
   };
 }
 
-function attachDeferredDaytonaCleanup(
-  result: PlayRunnerResult,
-  task: PlayRunnerDeferredRuntimeTask | null,
-): PlayRunnerResult {
-  if (!task) {
-    return result;
-  }
-  return {
-    ...result,
-    deferredRuntimeTasks: [...(result.deferredRuntimeTasks ?? []), task],
-  };
-}
-
+/**
+ * Deterministic sandbox delete on ALL backend exits (FIX 1). The old shape
+ * attached `deferredRuntimeTasks` cleanup records that NOTHING consumed, gated
+ * eager deletion behind a `DAYTONA_BACKGROUND_CLEANUP` env that was unset
+ * everywhere — every run leaked paid idle compute until Daytona's autoStop
+ * fired. Now every exit path deletes directly (fire-and-forget with structured
+ * logs; a failed delete never throws into the result path). autoStop remains
+ * only the backstop for a crashed worker process.
+ *
+ * Push execution note: a `detached_runner` suspended return deliberately
+ * BYPASSES `withCleanup` — the runner is still executing in the sandbox; the
+ * worker's wake leg owns that delete.
+ */
 export function createDaytonaSandboxCleanupManager(): {
   activate(acquired: AcquiredDaytonaSandbox): void;
   currentSandbox(): DaytonaSandbox | null;
@@ -362,49 +360,31 @@ export function createDaytonaSandboxCleanupManager(): {
 } {
   let sandbox: DaytonaSandbox | null = null;
   let sandboxBillingStartedAt: number | null = null;
-  const deferredCleanupTasksFromPreviousAttempts: PlayRunnerDeferredRuntimeTask[] =
-    [];
-  const backgroundCleanupSandboxIds = new Set<string>();
-
-  const deferredCleanupTask = (): PlayRunnerDeferredRuntimeTask | null => {
-    if (!sandbox || sandboxBillingStartedAt === null) {
-      return null;
-    }
-    return {
-      kind: 'daytona_sandbox_cleanup',
-      sandboxId: sandbox.id,
-      billingStartedAt: sandboxBillingStartedAt,
-      cpu: typeof sandbox.cpu === 'number' ? sandbox.cpu : 1,
-      memoryGiB: typeof sandbox.memory === 'number' ? sandbox.memory : 1,
-      diskGiB: typeof sandbox.disk === 'number' ? sandbox.disk : 0,
-    };
-  };
+  const deletedSandboxIds = new Set<string>();
 
   const deleteSandboxOnce = (input: {
     reason: string;
-    task: PlayRunnerDeferredRuntimeTask | null;
     sandboxToDelete: DaytonaSandbox | null;
   }) => {
-    if (
-      !input.task ||
-      !input.sandboxToDelete ||
-      backgroundCleanupSandboxIds.has(input.task.sandboxId)
-    ) {
+    const target = input.sandboxToDelete;
+    if (!target || deletedSandboxIds.has(target.id)) {
       return false;
     }
-    backgroundCleanupSandboxIds.add(input.task.sandboxId);
-    void input.sandboxToDelete.delete(30).then(
+    deletedSandboxIds.add(target.id);
+    const billingStartedAt = sandboxBillingStartedAt;
+    void target.delete(30).then(
       () => {
         console.info('[play-runner.daytona.cleanup_done]', {
           reason: input.reason,
-          sandboxId: input.task!.sandboxId,
-          elapsedMs: Date.now() - input.task!.billingStartedAt,
+          sandboxId: target.id,
+          elapsedMs:
+            billingStartedAt === null ? null : Date.now() - billingStartedAt,
         });
       },
       (error: unknown) => {
         console.warn('[play-runner.daytona.cleanup_failed]', {
           reason: input.reason,
-          sandboxId: input.task!.sandboxId,
+          sandboxId: target.id,
           error: error instanceof Error ? error.message : String(error),
         });
       },
@@ -412,25 +392,6 @@ export function createDaytonaSandboxCleanupManager(): {
     return true;
   };
 
-  const scheduleBackgroundCleanup = (
-    task: PlayRunnerDeferredRuntimeTask | null,
-    sandboxToDelete: DaytonaSandbox | null,
-  ) => {
-    if (
-      !shouldCleanupSandboxInBackground() ||
-      !task ||
-      !sandboxToDelete ||
-      backgroundCleanupSandboxIds.has(task.sandboxId)
-    ) {
-      return false;
-    }
-    return deleteSandboxOnce({
-      reason: 'background',
-      task,
-      sandboxToDelete,
-    });
-  };
-
   return {
     activate(acquired) {
       sandbox = acquired.sandbox;
@@ -442,39 +403,24 @@ export function createDaytonaSandboxCleanupManager(): {
     cleanupActiveSandboxForCancellation() {
       return deleteSandboxOnce({
         reason: 'cancelled',
-        task: deferredCleanupTask(),
         sandboxToDelete: sandbox,
       });
     },
     stashActiveSandboxForRetry() {
-      const task = deferredCleanupTask();
-      const sandboxToDelete = sandbox;
-      if (!scheduleBackgroundCleanup(task, sandboxToDelete) && task) {
-        deferredCleanupTasksFromPreviousAttempts.push(task);
-      }
+      // The retired sandbox is dead — a fresh one replaces it for the retry.
+      // Delete NOW instead of deferring to a consumer that never existed.
+      deleteSandboxOnce({
+        reason: 'retired_for_retry',
+        sandboxToDelete: sandbox,
+      });
       sandbox = null;
       sandboxBillingStartedAt = null;
     },
     withCleanup(result, input) {
-      const resultWithPreviousCleanup =
-        deferredCleanupTasksFromPreviousAttempts.length === 0
-          ? result
-          : {
-              ...result,
-              deferredRuntimeTasks: [
-                ...deferredCleanupTasksFromPreviousAttempts,
-                ...(result.deferredRuntimeTasks ?? []),
-              ],
-            };
-      const task = deferredCleanupTask();
-      const sandboxToDelete = sandbox;
-      if (input.cancellationCleanupStarted) {
-        return resultWithPreviousCleanup;
-      }
-      if (scheduleBackgroundCleanup(task, sandboxToDelete)) {
-        return resultWithPreviousCleanup;
+      if (!input.cancellationCleanupStarted) {
+        deleteSandboxOnce({ reason: 'terminal', sandboxToDelete: sandbox });
       }
-      return attachDeferredDaytonaCleanup(resultWithPreviousCleanup, task);
+      return result;
     },
   };
 }