@opengeni/worker-bundle 1.0.0-canary.1 → 1.0.0

package/src/sandbox-resume.ts

@@ -20,21 +20,27 @@
 //
 // Liveness between turns is the lease refcount; there is no keepalive loop.
 
+import type { EventBus } from "@opengeni/events";
+import { publishDurableSessionEvents } from "./session-event-fanout";
 import {
   effectiveSandboxLifecycle,
   sandboxArchiveCaptureTimeoutMs,
   sandboxLifecycleTransitionWaitMs,
+  sandboxWarmRateMicrosPerSecond,
   type Settings,
 } from "@opengeni/config";
 import { randomUUID } from "node:crypto";
+import { retainsHolderForWarmCapture } from "./warm-capture-holder";
 import {
   acquireLease,
   adoptLegacyModalCheckpointArtifact,
   beginSandboxRematerialization,
+  sessionHoldsFreshWorkspaceRecovery,
   claimWorkspaceArchiveCapture,
   commitWarmingToWarm,
   failSandboxRematerialization,
   failWarmingToCold,
+  beginModalProviderCreate,
   getSandboxSessionEnvelope,
   heartbeatLeaseHolderStatus,
   markSandboxRestoreVerifying,
@@ -79,6 +85,7 @@ import {
   withoutSandboxProviderIdentity,
   type EstablishedSandboxSession,
   type RuntimeMetricsHooks,
+  type SandboxReadinessReplacementOutcome,
   type WorkspaceArchiveDescriptor,
 } from "@opengeni/runtime";
 import {
@@ -86,7 +93,9 @@ import {
   WorkspaceArchiveStorageError,
   type ObjectStorage,
 } from "@opengeni/storage";
-import type { Observability } from "@opengeni/observability";
+import { sandboxLeaseTelemetryKey, type Observability } from "@opengeni/observability";
+import { safeSnapshotError } from "./sandbox-snapshot-diagnostics";
+export { safeSnapshotError } from "./sandbox-snapshot-diagnostics";
 import { parseWorkspaceArchiveObjectRef } from "@opengeni/contracts";
 import {
   persistWorkspaceArchiveCandidate,
@@ -125,9 +134,7 @@ export type SandboxResumeServices = {
   db: Database;
   settings: Settings;
   objectStorage?: ObjectStorage | null;
-  /** Exact settings before a verified rig provider image overlaid the logical
-   * pack/deployment image. Fresh-create NotFound fallback must preserve an
-   * ID-only logical base instead of selecting the provider default. */
+
   logicalFallbackSettings?: Settings;
   sandboxMetrics?: RuntimeMetricsHooks;
   /** Structured logger for lifecycle decisions (holder release reasons). Falls
@@ -148,6 +155,26 @@ export type SandboxResumeServices = {
   /** Test seam for the bounded command-readiness proof performed before an
    * attached provider box is handed to the agent. */
   verifyAttachedSandboxReadiness?: (established: EstablishedSandboxSession) => Promise<void>;
+  /** Test seam for the bounded command-readiness proof the elected spawner runs
+   * on its freshly created box before publishing the lease warm. Production
+   * uses {@link waitForSandboxExecReadiness} with the shared 60 s budget. */
+  verifySpawnedSandboxReadiness?: (
+    established: EstablishedSandboxSession,
+    identity: { sandboxGroupId: string },
+  ) => Promise<void>;
+  /** Test seam for the jittered pause before the single fresh-box readiness
+   * replacement. Production uses {@link freshSandboxReadinessReplacementDelayMs}.
+   * It may be async so a test can observe the rolled-back lease in between. */
+  freshSandboxReadinessReplacementDelayMs?: () => number | Promise<number>;
+  /**
+   * The turn attempt's fresh-box readiness replacement budget. The lazy
+   * provisioner may call resumeBoxForTurn again after a typed lease
+   * supersession, so the turn creates the budget once and shares it with every
+   * call. Absent, the call gets its own single replacement.
+   */
+  freshSandboxReadinessReplacementBudget?: FreshSandboxReadinessReplacementBudget;
+  /** Live fanout for background commands a provider loss settled. */
+  bus?: EventBus | null;
   /** Called only by the observer that wins the exact warm->cold loss CAS. */
   onSandboxLost?: (input: {
     sandboxGroupId: string;
@@ -187,6 +214,9 @@ export type ResumeBoxIds = {
    * never conflicts).
    */
   image?: string;
+  /** Deployment/workspace image pins apply only to new creates; an existing
+   * group keeps its image. Omission preserves explicit image matching (B3). */
+  imagePolicy?: "require_match" | "new_creates_only";
   /**
    * RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
    * acquireLease, which stamps it on the cold-create and conflicts on a live box
@@ -316,6 +346,80 @@ export class SandboxSiblingWarmingTimeoutError extends SandboxWarmingTimeoutErro
   }
 }
 
+/**
+ * Out-of-band proof attached to one exact {@link SandboxExecReadinessTimeoutError}
+ * thrown by the elected spawner: the box that missed its readiness budget was
+ * freshly created by this caller (not an attached, resumed, or provider-continuity
+ * box), it was never published warm, the provider terminated it, and the holder
+ * was released. Only such an error may be replaced by re-entering admission.
+ */
+const disposedUnreadyFreshSandboxErrors = new WeakSet<object>();
+
+/** True only for a spawner readiness timeout whose unready fresh box is proven gone. */
+export function isReplaceableFreshSandboxReadinessTimeout(error: unknown): boolean {
+  return (
+    error instanceof SandboxExecReadinessTimeoutError &&
+    disposedUnreadyFreshSandboxErrors.has(error)
+  );
+}
+
+/**
+ * A burst of simultaneous turn starts can leave many freshly created provider
+ * boxes command-unready at once. The single replacement re-enters admission
+ * after a jittered pause so the replacement creates do not re-synchronize into
+ * the same burst. Bounded to [2 s, 10 s).
+ */
+export const FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS = 2_000;
+export const FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS = 8_000;
+
+export function freshSandboxReadinessReplacementDelayMs(
+  random: () => number = Math.random,
+): number {
+  const sample = random();
+  const unit = Number.isFinite(sample) ? Math.min(Math.max(sample, 0), 1 - Number.EPSILON) : 0;
+  return (
+    FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS +
+    Math.floor(unit * FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS)
+  );
+}
+
+/** Replacements allowed per turn attempt, across every resumeBoxForTurn call. */
+export const FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT = 1;
+
+export type FreshSandboxReadinessReplacementBudget = { remaining: number };
+
+export function createFreshSandboxReadinessReplacementBudget(): FreshSandboxReadinessReplacementBudget {
+  return { remaining: FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT };
+}
+
+function recordFreshSandboxReadinessReplacement(
+  metrics: RuntimeMetricsHooks | undefined,
+  backend: string,
+  outcome: SandboxReadinessReplacementOutcome,
+): void {
+  try {
+    metrics?.onSandboxReadinessReplacement?.({ backend, outcome });
+  } catch {
+    // Metrics emission must never affect sandbox recovery or error propagation.
+  }
+}
+
+async function sleepUnlessCancelled(ms: number, signal: AbortSignal | undefined): Promise<boolean> {
+  if (signal?.aborted) return false;
+  if (ms <= 0) return true;
+  return await new Promise<boolean>((resolve) => {
+    const onAbort = () => {
+      clearTimeout(timer);
+      resolve(false);
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve(true);
+    }, ms);
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
 /** The exact attached caller that won warm->cold after proving the provider
  * instance gone. It is a lease supersession (the same logical turn recovers),
  * plus the lost id needed for one durable observability event. */
@@ -348,9 +452,10 @@ export async function waitForSandboxExecReadiness(
   established: EstablishedSandboxSession,
   timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
   identity: { sandboxGroupId?: string | null } = {},
+  signal?: AbortSignal,
 ): Promise<void> {
   try {
-    await verifySandboxExecReadiness(established, timeoutMs);
+    await verifySandboxExecReadiness(established, timeoutMs, signal);
   } catch (error) {
     if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
       throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
@@ -369,54 +474,6 @@ class SnapshotTimeoutError extends Error {
   }
 }
 
-export function safeSnapshotError(error: unknown): {
-  errorClass: "SnapshotOperationError";
-  errorCode: "snapshot_operation_failed";
-  status?: number;
-  origin: "sandbox-resume";
-  causeName?: string;
-  integrityCode?: string;
-} {
-  const fields: {
-    errorClass: "SnapshotOperationError";
-    errorCode: "snapshot_operation_failed";
-    status?: number;
-    origin: "sandbox-resume";
-    causeName?: string;
-    integrityCode?: string;
-  } = {
-    errorClass: "SnapshotOperationError",
-    errorCode: "snapshot_operation_failed",
-    origin: "sandbox-resume",
-  };
-  try {
-    if (error && typeof error === "object") {
-      const candidate = error as {
-        name?: unknown;
-        code?: unknown;
-        status?: unknown;
-        statusCode?: unknown;
-      };
-      if (
-        typeof candidate.name === "string" &&
-        /^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)
-      ) {
-        fields.causeName = candidate.name;
-      }
-      if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
-        fields.integrityCode = candidate.code;
-      }
-      const rawStatus = candidate.status ?? candidate.statusCode;
-      const status = Number(rawStatus);
-      if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
-    }
-  } catch {
-    // Public diagnostics are best-effort and must never replace the exact
-    // internal snapshot failure.
-  }
-  return fields;
-}
-
 export async function waitForWarmSnapshot(
   snapshot: Promise<unknown>,
   timeoutMs: number,
@@ -582,8 +639,9 @@ async function materializeSpawnEnvelopeArchive(
   }
   if (!objectStorage) {
     throw new WorkspaceArchiveIntegrityError(
-      "archive_base64_invalid",
+      "archive_storage_unavailable",
       "workspace archive object storage is not configured",
+      { retryable: true },
     );
   }
   const descriptor = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
@@ -787,6 +845,19 @@ async function persistWarmWorkspaceSnapshot(
   registerOwnedContinuation: (continuation: Promise<void>) => void,
 ): Promise<boolean> {
   const { db, settings } = services;
+  const warnSnapshotFailure = (message: string, error: unknown): void => {
+    const fields = {
+      ...safeSnapshotError(error),
+      sandboxLeaseKey: sandboxLeaseTelemetryKey(ids.workspaceId, ids.sandboxGroupId),
+      leaseEpoch,
+    };
+    try {
+      if (services.observability) services.observability.warn(message, fields);
+      else console.warn(message, fields);
+    } catch {
+      // Diagnostics cannot change provider settlement or capture-gate cleanup.
+    }
+  };
   const intervalMs = settings.sandboxSnapshotIntervalMs;
   if (intervalMs <= 0 && !force) {
     return false;
@@ -896,6 +967,9 @@ async function persistWarmWorkspaceSnapshot(
         : {}),
     });
     if (claimed.status !== "claimed") {
+      // A scheduled interval not being due is normal, including after a failed
+      // attempt. Do not turn every ten-second heartbeat into an error log.
+      if (claimed.status === "throttled") return false;
       console.error("mid-session workspace snapshot skipped (capture claim)", {
         sandboxGroupId: ids.sandboxGroupId,
         status: claimed.status,
@@ -951,6 +1025,8 @@ async function persistWarmWorkspaceSnapshot(
     // turn signal resolves first. Its finally block is the only normal release
     // of the exact admission gate; a late callback cannot release a successor.
     const captureAndPublish = (async (): Promise<boolean> => {
+      const captureStarted = performance.now();
+      let captureOutcome: "completed" | "failed" = "failed";
       let archive: VerifiedWorkspaceArchivePayload | undefined;
       let candidate: { id: string } | null = null;
       let publicationAttempted = false;
@@ -1001,6 +1077,7 @@ async function persistWarmWorkspaceSnapshot(
         if (!wrote && candidate) {
           await abandonCandidate(candidate.id, "snapshot_publication_fenced");
         }
+        if (wrote) captureOutcome = "completed";
         return wrote;
       } catch (error) {
         if (candidate && !publicationAttempted)
@@ -1018,20 +1095,24 @@ async function persistWarmWorkspaceSnapshot(
           expectedEpoch: leaseEpoch,
           expectedInstanceId: instanceId,
         }).catch((error) => {
-          console.error(
-            "mid-session workspace capture gate release failed",
-            safeSnapshotError(error),
-          );
+          captureOutcome = "failed";
+          warnSnapshotFailure("mid-session workspace capture gate release failed", error);
         });
+        try {
+          services.sandboxMetrics?.onWorkspaceCapture?.({
+            backend: lease.backend,
+            outcome: captureOutcome,
+            durationSeconds: Math.max(0, performance.now() - captureStarted) / 1_000,
+          });
+        } catch {
+          // A completed capture and released gate cannot depend on telemetry.
+        }
       }
     })();
     const settled = captureAndPublish.then(
       (persisted) => ({ kind: "settled" as const, persisted }),
       (error) => {
-        console.error(
-          "mid-session workspace snapshot failed (turn unaffected)",
-          safeSnapshotError(error),
-        );
+        warnSnapshotFailure("mid-session workspace snapshot failed (turn unaffected)", error);
         return { kind: "settled" as const, persisted: false };
       },
     );
@@ -1063,12 +1144,10 @@ async function persistWarmWorkspaceSnapshot(
     // exact claim cleanup, so it cannot leak an unhandled promise or artifact.
     return outcome.kind === "settled" ? outcome.persisted : false;
   } catch (error) {
-    // Protection, not a dependency: a failed snapshot must never fail (or slow
-    // down retrying) the turn. The next heartbeat/turn-end tick retries.
-    console.error(
-      "mid-session workspace snapshot failed (turn unaffected)",
-      safeSnapshotError(error),
-    );
+    // A failed snapshot does not fail the turn. Physical captures can still
+    // fence commands until their exact gate settles; the caller timeout is not
+    // permission to release that gate. The next eligible tick can retry.
+    warnSnapshotFailure("mid-session workspace snapshot failed (turn unaffected)", error);
     return false;
   }
 }
@@ -1082,12 +1161,103 @@ async function persistWarmWorkspaceSnapshot(
  * holderId is the globally unique durable turn-attempt id. It must not be a
  * Temporal activity id, because activity ids are only workflow-local and
  * collide when sibling sessions share one sandbox group.
+ *
+ * A freshly created box that misses its bounded command-readiness budget is
+ * replaced at most once per turn attempt (the budget is shared by every call
+ * the turn's provisioner makes). The first establish has already terminated that exact
+ * unpublished box, rolled its exact warming epoch back to cold (advancing the
+ * epoch), and released its holder, so the replacement is an ordinary new
+ * admission: it re-runs the cold->warming CAS, persists its own provider
+ * instance before readiness, and may instead attach to a sibling that won the
+ * CAS first. Nothing model- or tool-visible ran on the discarded box, so no
+ * side effect is replayed. A second readiness timeout fails the turn exactly
+ * as before; attached/resumed boxes and unconfirmed terminations never
+ * replace.
  */
 export async function resumeBoxForTurn(
   services: SandboxResumeServices,
   ids: ResumeBoxIds,
   kind: "turn",
   holderId: TurnSandboxLeaseHolderId,
+): Promise<ResumedTurnSandbox> {
+  try {
+    return await resumeBoxForTurnOnce(services, ids, kind, holderId);
+  } catch (error) {
+    if (!isReplaceableFreshSandboxReadinessTimeout(error)) throw error;
+    const readiness = error as SandboxExecReadinessTimeoutError;
+    const warn = (message: string, fields: Parameters<Observability["warn"]>[1]) => {
+      if (services.observability) services.observability.warn(message, fields);
+      else console.warn(message, fields);
+    };
+    const identity = {
+      workspaceId: ids.workspaceId,
+      sessionId: ids.sessionId,
+      sandboxGroupId: ids.sandboxGroupId,
+      backend: readiness.backend,
+      instanceId: readiness.instanceId,
+      readinessTimeoutMs: readiness.timeoutMs,
+    };
+    const budget =
+      services.freshSandboxReadinessReplacementBudget ??
+      createFreshSandboxReadinessReplacementBudget();
+    if (budget.remaining <= 0) {
+      recordFreshSandboxReadinessReplacement(
+        services.sandboxMetrics,
+        readiness.backend,
+        "budget_spent",
+      );
+      warn(
+        "sandbox command-readiness timed out on a fresh box; this turn attempt already used its replacement",
+        identity,
+      );
+      throw error;
+    }
+    budget.remaining -= 1;
+    const delayMs = await (
+      services.freshSandboxReadinessReplacementDelayMs ?? freshSandboxReadinessReplacementDelayMs
+    )();
+    warn("sandbox command-readiness timed out on a fresh box; replacing it once after jitter", {
+      ...identity,
+      replacementDelayMs: delayMs,
+    });
+    if (!(await sleepUnlessCancelled(delayMs, services.cancellationSignal))) {
+      // Cancellation owns the turn boundary; surface the original typed
+      // readiness failure rather than starting a replacement.
+      recordFreshSandboxReadinessReplacement(
+        services.sandboxMetrics,
+        readiness.backend,
+        "cancelled",
+      );
+      throw error;
+    }
+    try {
+      const replaced = await resumeBoxForTurnOnce(services, ids, kind, holderId);
+      recordFreshSandboxReadinessReplacement(
+        services.sandboxMetrics,
+        readiness.backend,
+        "replaced",
+      );
+      return replaced;
+    } catch (replacementError) {
+      recordFreshSandboxReadinessReplacement(
+        services.sandboxMetrics,
+        readiness.backend,
+        services.cancellationSignal?.aborted
+          ? "cancelled"
+          : replacementError instanceof SandboxExecReadinessTimeoutError
+            ? "failed_again"
+            : "replacement_failed",
+      );
+      throw replacementError;
+    }
+  }
+}
+
+async function resumeBoxForTurnOnce(
+  services: SandboxResumeServices,
+  ids: ResumeBoxIds,
+  kind: "turn",
+  holderId: TurnSandboxLeaseHolderId,
 ): Promise<ResumedTurnSandbox> {
   const { db, settings } = services;
   const os = ids.os ?? "linux";
@@ -1179,13 +1349,18 @@ export async function resumeBoxForTurn(
     holderId,
     subjectId: ids.sessionId,
     backend: ids.backend,
+    warmBilling: {
+      mode: settings.sandboxWarmBillingMode,
+      rateMicrosPerSecond: sandboxWarmRateMicrosPerSecond(settings, ids.backend),
+    },
     os,
     // IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
-    // conflicts on a live box already running a different image. A
+    // conflicts on a live box already running a different required image. A
     // SandboxImageConflictError propagates while another holder is active; a
     // solo change requests a capture-and-drain rotation and this attempt retries
     // after the cold successor can safely stamp the new image.
     ...(ids.image ? { image: ids.image } : {}),
+    ...(ids.imagePolicy ? { imagePolicy: ids.imagePolicy } : {}),
     // RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
     // + conflicts on a live box under a different rig. A SandboxRigConflictError
     // propagates while another holder is active; a solo change uses the same
@@ -1276,7 +1451,16 @@ export async function resumeBoxForTurn(
   // provider operation may still be mutating /workspace. Both loops therefore
   // converge on "holder alive -> never release"; finalization performs the
   // proof-bearing release.
-  const releaseDeadHolder = (reason: string): void => {
+  const releaseDeadHolder = async (reason: string): Promise<void> => {
+    // Logical closure deliberately fences heartbeat authority. It is not proof
+    // that the already-admitted capture has physically settled: the finalizer
+    // owns that wait and the original deadline bounds dead-worker recovery.
+    // Match the reaper's retention rule without touching holder/lease TTLs.
+    if (reason === "holder_gone" && kind === "turn" && holderLeaseHeartbeat) {
+      const expectedEpoch = holderLeaseHeartbeat.expectedEpoch;
+      const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
+      if (retainsHolderForWarmCapture(lease, expectedEpoch)) return;
+    }
     const message = "sandbox resume holder released: holder no longer live";
     const fields = {
       workspaceId: ids.workspaceId,
@@ -1300,14 +1484,15 @@ export async function resumeBoxForTurn(
         holderId,
         expectedEpoch: heartbeat.expectedEpoch,
         leaseTtlMs: heartbeat.leaseTtlMs,
+        billingMode: services.settings.sandboxWarmBillingMode,
       })
-        .then((status) => {
+        .then(async (status) => {
           if (heartbeat !== holderLeaseHeartbeat) return;
           if (!status.holderAlive) {
             // Reaped, released, or its exact attempt is no longer the active
             // writer. Stop this otherwise-unbounded provider operation and
             // idempotently drop any remaining holder state.
-            releaseDeadHolder(status.fence);
+            await releaseDeadHolder(status.fence);
             return;
           }
           if (status.fence === "epoch") {
@@ -1316,7 +1501,7 @@ export async function resumeBoxForTurn(
             // instance replaced, re-established by a later turn). The holder
             // row is stale authority for an instance that no longer exists;
             // drop it promptly instead of pinning a successor's lease.
-            releaseDeadHolder(status.fence);
+            await releaseDeadHolder(status.fence);
             return;
           }
           // The box is still serving this live holder: keep the provider TTL
@@ -1338,13 +1523,13 @@ export async function resumeBoxForTurn(
       kind,
       holderId,
     })
-      .then((touched) => {
+      .then(async (touched) => {
         if (heartbeat !== holderLeaseHeartbeat) return;
         // A canonical turn holder is rejected once its exact attempt is no
         // longer the active writer (touch is attempt-fenced only; it never
         // consults lease authority, so false here always means holder death).
         if (!touched) {
-          releaseDeadHolder("holder_gone");
+          await releaseDeadHolder("holder_gone");
           return;
         }
         void maybeRenewProviderExpiration().catch(() => undefined);
@@ -1391,6 +1576,8 @@ export async function resumeBoxForTurn(
   if (acquired.role === "spawner") {
     const expectedEpoch = acquired.lease.leaseEpoch;
     let createdEstablished: EstablishedSandboxSession | null = null;
+    let providerCreateOperationId: string | undefined;
+    let providerCreateBindingKey: string | undefined;
     let rematerialization: {
       id: string;
       selectedRevision: string;
@@ -1402,29 +1589,53 @@ export async function resumeBoxForTurn(
       > | null;
     } | null = null;
     try {
-      const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
+      // An audited decision to continue a definitively lost workspace on a new
+      // EMPTY box: hydrate nothing (not even a per-session legacy archive) and
+      // resume no prior provider identity. Every group member was already told
+      // the workspace is empty; the DB accepts only this exact publication.
+      const freshWorkspaceRecoveryId = acquired.lease.freshWorkspaceRecoveryId ?? null;
+      const envelope = freshWorkspaceRecoveryId
+        ? null
+        : await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
       // The lease is authoritative. A legacy per-session fallback archive may
       // only be used after beginSandboxRematerialization imports its archive
       // fields under the warming-row lock and records one selected revision.
       // Select the fallback only when the lease has no archive truth at all. A
       // lease-carried unverified/invalid archive must fail closed rather than
       // silently substituting another revision.
+      // Backstop: a session already told its workspace is empty never gets a
+      // legacy per-session archive back, even if the lease marker was lost.
+      const legacyFallbackRefused =
+        !freshWorkspaceRecoveryId &&
+        acquired.lease.recovery.archive.status === "none" &&
+        workspaceArchiveFieldsFromEnvelope(envelope) !== null &&
+        (await sessionHoldsFreshWorkspaceRecovery(db, ids.workspaceId, ids.sessionId));
       const fallbackArchiveEnvelope =
+        !freshWorkspaceRecoveryId &&
+        !legacyFallbackRefused &&
         acquired.lease.recovery.archive.status === "none" &&
         workspaceArchiveFieldsFromEnvelope(envelope) !== null
           ? withoutSandboxProviderIdentity(envelope)
           : null;
-      let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
+      let spawnEnvelope = freshWorkspaceRecoveryId
+        ? null
+        : legacyFallbackRefused
+          ? (acquired.lease.resumeState ?? null)
+          : (fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope);
       const archiveSource =
         acquired.lease.recovery.archive.status === "none"
           ? fallbackArchiveEnvelope
           : acquired.lease.resumeState;
       const continuityRecovery = acquired.lease.recovery.continuity;
+      // A fresh-workspace spawn is deliberately archive-free; commitWarmingToWarm
+      // verifies the exact decision before publication.
       if (
-        (acquired.lease.recovery.archive.status === "available" &&
-          acquired.lease.archiveComplete) ||
-        (acquired.lease.recovery.archive.status === "none" &&
-          workspaceArchiveFieldsFromEnvelope(archiveSource) !== null)
+        !freshWorkspaceRecoveryId &&
+        ((acquired.lease.recovery.archive.status === "available" &&
+          (acquired.lease.archiveComplete ||
+            acquired.lease.historicalRecoveryAuthorized === true)) ||
+          (acquired.lease.recovery.archive.status === "none" &&
+            workspaceArchiveFieldsFromEnvelope(archiveSource) !== null))
       ) {
         const rematerializationId = crypto.randomUUID();
         const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
@@ -1467,7 +1678,11 @@ export async function resumeBoxForTurn(
           legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
           legacyProviderBinding: null,
         };
-      } else if (acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
+      } else if (
+        !freshWorkspaceRecoveryId &&
+        acquired.lease.recovery.archive.status !== "none" &&
+        !continuityRecovery
+      ) {
         throw new SandboxLeaseRecoveryBlockedError(
           ids.sandboxGroupId,
           expectedEpoch,
@@ -1526,6 +1741,34 @@ export async function resumeBoxForTurn(
         ...(services.logicalFallbackSettings
           ? { logicalFallbackSettings: services.logicalFallbackSettings }
           : {}),
+        onBeforeSandboxCreate: async (createSettings, intent, providerContext) => {
+          if (ids.backend !== "modal") return;
+          if (!intent || !providerContext)
+            throw new Error("Modal create requires the provider dispatch boundary");
+          throwIfReleasedOrCancelled();
+          const binding = await resolveModalCheckpointProviderBindingForSession(
+            createSettings,
+            providerContext,
+          );
+          const operationId = intent.operationId;
+          await beginModalProviderCreate(db, {
+            accountId: ids.accountId,
+            workspaceId: ids.workspaceId,
+            sandboxGroupId: ids.sandboxGroupId,
+            expectedEpoch,
+            operationId,
+            providerBindingKey: binding.key,
+            rematerializationId: rematerialization?.id ?? null,
+            selectedRevision: rematerialization?.selectedRevision ?? null,
+            imageId: intent.imageId,
+            imageRef: createSettings.modalImageRef ?? null,
+            appId: intent.appId,
+            providerName: intent.name,
+            requestSha256: intent.requestSha256,
+          });
+          providerCreateOperationId = operationId;
+          providerCreateBindingKey = binding.key;
+        },
         onSandboxCreated: async (created) => {
           createdEstablished = created;
           providerRenewalTarget = {
@@ -1533,7 +1776,16 @@ export async function resumeBoxForTurn(
             instanceId: created.instanceId,
           };
           providerRenewedAtMs = Date.now();
-          throwIfReleasedOrCancelled();
+          if (
+            providerCreateBindingKey &&
+            !(await modalSessionMatchesCheckpointProviderBinding(
+              settings,
+              created.session,
+              providerCreateBindingKey,
+            ))
+          ) {
+            throw new Error("Modal creation receipt crossed the fenced provider namespace");
+          }
           if (
             rematerialization &&
             (rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)
@@ -1584,6 +1836,7 @@ export async function resumeBoxForTurn(
             sandboxGroupId: ids.sandboxGroupId,
             expectedEpoch,
             rematerializationId: rematerialization?.id ?? null,
+            ...(providerCreateOperationId ? { providerCreateOperationId } : {}),
             ...(created.providerContinuity
               ? { continuityRecovery: created.providerContinuity }
               : {}),
@@ -1666,11 +1919,22 @@ export async function resumeBoxForTurn(
       // A sandbox handle is not sufficient evidence that an asynchronous
       // provider's command router is live. Do not publish a warm lease until
       // one bounded no-op exec works.
-      // On timeout the catch below terminates the box and rolls warming -> cold,
-      // so the next turn cold-creates instead of hanging forever on first use.
-      await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
-        sandboxGroupId: ids.sandboxGroupId,
-      });
+      // On timeout the catch below terminates the box and rolls warming -> cold;
+      // resumeBoxForTurn may then replace a proven-terminated fresh box once.
+      if (services.verifySpawnedSandboxReadiness) {
+        await services.verifySpawnedSandboxReadiness(established, {
+          sandboxGroupId: ids.sandboxGroupId,
+        });
+      } else {
+        await waitForSandboxExecReadiness(
+          established,
+          MODAL_EXEC_READINESS_TIMEOUT_MS,
+          {
+            sandboxGroupId: ids.sandboxGroupId,
+          },
+          cancellationSignal,
+        );
+      }
       await maybeRenewProviderExpiration(true);
       throwIfReleasedOrCancelled();
       // Fold the LIVE box into a re-resumable envelope and persist it as the
@@ -1716,16 +1980,18 @@ export async function resumeBoxForTurn(
         dataPlaneUrl: null,
         resumeBackendId: established.backendId,
         resumeState: resumeEnvelope,
-        ...(established.providerContinuity
-          ? { continuityRecovery: established.providerContinuity }
-          : rematerialization
-            ? {
-                rematerialization: {
-                  id: rematerialization.id,
-                  verifiedRevision: rematerialization.selectedRevision,
-                },
-              }
-            : {}),
+        ...(freshWorkspaceRecoveryId
+          ? { freshWorkspace: { operationId: freshWorkspaceRecoveryId } }
+          : established.providerContinuity
+            ? { continuityRecovery: established.providerContinuity }
+            : rematerialization
+              ? {
+                  rematerialization: {
+                    id: rematerialization.id,
+                    verifiedRevision: rematerialization.selectedRevision,
+                  },
+                }
+              : {}),
         leaseTtlMs,
       });
       if (!committed.committed || !committed.lease) {
@@ -1806,6 +2072,16 @@ export async function resumeBoxForTurn(
       }
       await release();
       recordSandboxWarmingTimeout(services.sandboxMetrics, error);
+      if (
+        terminated &&
+        createdEstablished &&
+        error instanceof SandboxExecReadinessTimeoutError &&
+        error.instanceId === createdEstablished.instanceId &&
+        (createdEstablished.origin === "created" || createdEstablished.origin === "restored") &&
+        !createdEstablished.providerContinuity
+      ) {
+        disposedUnreadyFreshSandboxErrors.add(error);
+      }
       throw sandboxProvisionStageError(rematerialization ? "archive_recovery" : "create", error);
     }
   }
@@ -1855,9 +2131,14 @@ export async function resumeBoxForTurn(
       if (services.verifyAttachedSandboxReadiness) {
         await services.verifyAttachedSandboxReadiness(established);
       } else {
-        await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
-          sandboxGroupId: ids.sandboxGroupId,
-        });
+        await waitForSandboxExecReadiness(
+          established,
+          MODAL_EXEC_READINESS_TIMEOUT_MS,
+          {
+            sandboxGroupId: ids.sandboxGroupId,
+          },
+          cancellationSignal,
+        );
       }
       providerRenewalTarget = {
         backend: ids.backend,
@@ -1875,8 +2156,14 @@ export async function resumeBoxForTurn(
         sandboxGroupId: ids.sandboxGroupId,
         expectedEpoch: leaseEpoch,
         expectedInstanceId: live.instanceId,
+        expectedBackend: ids.backend,
       });
       if (marked.status === "marked") {
+        await publishDurableSessionEvents(
+          services.bus,
+          ids.workspaceId,
+          marked.backgroundCommandEvents,
+        );
         await services.onSandboxLost?.({
           sandboxGroupId: ids.sandboxGroupId,
           instanceId: live.instanceId,