@opengeni/runtime 0.6.0 → 0.7.0

package/src/sandbox/index.ts

@@ -20,14 +20,18 @@
 
 import type { Settings } from "@opengeni/config";
 import { collectSandboxEnvironment, parseExposedPorts } from "@opengeni/config";
-import { DESKTOP_STREAM_PORT, TERMINAL_STREAM_PORT, type SandboxBackend } from "@opengeni/contracts";
+import {
+  DESKTOP_STREAM_PORT,
+  TERMINAL_STREAM_PORT,
+  type SandboxBackend,
+} from "@opengeni/contracts";
 import type {
   SandboxClient,
   SandboxSessionLike,
   SandboxSessionState,
 } from "@openai/agents/sandbox";
 import { PROVIDER_REGISTRY } from "./providers";
-import { SandboxConfigError } from "./errors";
+import { SandboxConfigError, SandboxResumeStateUnavailableError } from "./errors";
 import { isSelfhostedProviderNotFoundError } from "./selfhosted/session";
 import type { RuntimeMetricsHooks } from "../metrics";
 
@@ -47,7 +51,11 @@ export {
   assertDescriptorRegistryInvariants,
   type CapabilityDescriptor,
 } from "./capabilities";
-export { SandboxConfigError, SandboxProviderUnavailableError } from "./errors";
+export {
+  SandboxConfigError,
+  SandboxProviderUnavailableError,
+  SandboxResumeStateUnavailableError,
+} from "./errors";
 export {
   PROVIDER_REGISTRY,
   assertProviderRegistryInvariants,
@@ -130,11 +138,12 @@ export {
 } from "./stream-port";
 
 // P4.3 recording loop — plain functions over a live session handle (no agent
-// loop); finalize reads bytes + PUTs to storage in the holding process (F10).
+// loop); finalize uploads box -> object storage without buffering in the worker.
 export {
   startRecording,
   stopRecording,
-  readRecordingBytes,
+  inspectRecordingArtifact,
+  uploadRecordingArtifact,
   deleteRecordingArtifacts,
   recordingStorageKey,
   contentTypeForCodec,
@@ -145,7 +154,7 @@ export {
   type RecordingContentType,
   type StartRecordingInput,
   type RecordingProcess,
-  type FinalizeRecordingResult,
+  type RecordingArtifactMetadata,
 } from "./recording";
 
 // P4.4 Channel-A structured services — the provider-agnostic SandboxChannelAService
@@ -166,11 +175,14 @@ export {
   parsePorcelainV2,
   parseNumstatZ,
   parseUnifiedPatch,
+  REPOSITORY_DISCOVERY_LIMIT,
   type ChannelASession,
   type ChannelAExecArgs,
   type ChannelAExecResult,
   type ChannelAEmitter,
   type SandboxChannelAServiceOptions,
+  type RepositoryDiscoveryDegradedReason,
+  type RepositoryDiscoveryResult,
   type NumstatEntry,
 } from "./channel-a";
 
@@ -190,9 +202,43 @@ export {
   timeoutControlResponse,
   offlineAgentError,
   timeoutAgentError,
+  drainingExhaustedError,
+  payloadTooLargeMessage,
+  drainingMessage,
+  execDeadlineHint,
+  NEVER_SENT_DETAIL_KEY,
   type NatsRequestConnection,
   type SelfhostedUnavailableReason,
 } from "./selfhosted/control-rpc";
+// The transport-agnostic per-op observation seam (out-of-band telemetry — metrics
+// + machine.* events; the op-engine's op-stream client emits through this too).
+export type { SelfhostedOpObserver, SelfhostedOpObservation } from "./selfhosted/op-observer";
+// The four-field in-band fault renderer (the failure-visibility doctrine).
+export {
+  selfhostedFaultClass,
+  SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
+  renderSelfhostedFault,
+  FAULT_FIELD_WHAT_HAPPENED,
+  FAULT_FIELD_WHICH_LAYER,
+  FAULT_FIELD_WHAT_PRESERVED,
+  FAULT_FIELD_WHAT_TO_TRY,
+} from "./selfhosted/fault-rendering";
+// The selfhosted control-op retry policy (the pure decision + the injected clock).
+export {
+  decideSelfhostedRetry,
+  selfhostedRetryBackoffMs,
+  defaultSelfhostedRetryClock,
+  SELFHOSTED_IDEMPOTENT_READONLY_OPS,
+  SELFHOSTED_DRAINING_MAX_RETRIES,
+  SELFHOSTED_EXEC_DRAINING_MAX_RETRIES,
+  SELFHOSTED_TIMEOUT_MAX_RETRIES,
+  SELFHOSTED_NEVER_SENT_MAX_RETRIES,
+  SELFHOSTED_RETRY_BACKOFF_BASE_MS,
+  SELFHOSTED_RETRY_BACKOFF_FACTOR,
+  SELFHOSTED_RETRY_BACKOFF_CAP_MS,
+  type SelfhostedRetryClock,
+  type SelfhostedRetryDecision,
+} from "./selfhosted/retry-policy";
 export {
   SelfhostedSession,
   SelfhostedSandboxClient,
@@ -210,7 +256,32 @@ export {
   type SelfhostedApplyDiff,
   type SelfhostedEditor,
   type SelfhostedImageOutput,
+  type SelfhostedOpStreamDeps,
 } from "./selfhosted/session";
+// The op-stream exec transport (op-stream protocol v1.1 — streaming exec to a
+// Connected Machine runner). The worker injects `NatsOpStreamTransport` (over
+// the same bus connection as the control rpc) plus an `OpStreamJournal`
+// adapted onto Temporal; sessions without the injection keep the legacy exec.
+export {
+  NatsOpStreamTransport,
+  OpStreamUnavailableError,
+  opAckSubject,
+  opFrameSubject,
+  type NatsOpStreamConnection,
+  type OpStreamSubscription,
+  type OpStreamTransport,
+} from "./selfhosted/op-transport";
+export {
+  OP_STREAM_ACK_INTERVAL_MS,
+  OP_STREAM_DEFAULT_WINDOW_BYTES,
+  OP_STREAM_RECONNECT_HOLD_MS,
+  OP_STREAM_SILENCE_TIMEOUT_MS,
+  type OpStreamExecOutcome,
+  type OpStreamJournal,
+} from "./selfhosted/op-stream";
+// The durable op-id correlation seam (B1): the runtime barrel BINDS it around
+// the SDK shell tool; the sandbox leaf READS it when minting op ids.
+export { nextDurableOpId, runWithToolCallCorrelation, sanitizeOpIdToken } from "./op-correlation";
 export {
   negotiateSelfhostedCapabilities,
   selfhostedLiveness,
@@ -219,7 +290,11 @@ export {
   type SelfhostedLivenessState,
   type SelfhostedEnrollment,
 } from "./selfhosted/capabilities";
-export { MockAgentResponder, type MockAgentResponderOptions, type MockExecHandler } from "./selfhosted/testing";
+export {
+  MockAgentResponder,
+  type MockAgentResponderOptions,
+  type MockExecHandler,
+} from "./selfhosted/testing";
 
 // The hot-swap routing proxy (M7): ONE stable session-shaped object the SDK binds
 // to, which re-reads the per-session active pointer per op and dispatches to the
@@ -237,8 +312,11 @@ export {
 export {
   makeActiveBackendResolver,
   ActiveBackendUnresolvableError,
+  swapTargetEstablishability,
   type ActiveBackendResolverDeps,
   type RoutableSandbox,
+  type BackendUnresolvableCode,
+  type SwapTargetEstablishability,
 } from "./routing/backend-resolver";
 
 /**
@@ -252,8 +330,15 @@ export {
  * ports on demand (modal/runloop/e2b pre-declare; blaxel resolves on demand).
  * Existing modal/docker/local construction is behavior-preserved.
  */
-export function createSandboxClient(settings: Settings, environment = collectSandboxEnvironment(settings)): unknown {
-  return createSandboxClientForBackend(settings.sandboxBackend as SandboxBackend, settings, environment);
+export function createSandboxClient(
+  settings: Settings,
+  environment = collectSandboxEnvironment(settings),
+): unknown {
+  return createSandboxClientForBackend(
+    settings.sandboxBackend as SandboxBackend,
+    settings,
+    environment,
+  );
 }
 
 /**
@@ -287,10 +372,10 @@ export function createSandboxClientForBackend(
   // on-demand → must pre-declare). blaxel is on-demand → skipped here.
   const desktop = registration.descriptor.capabilities.DesktopStream;
   if (
-    desktop.available
-    && settings.sandboxDesktopEnabled
-    && !registration.descriptor.portExposure.supportsOnDemandPorts
-    && !exposedPorts.includes(DESKTOP_STREAM_PORT)
+    desktop.available &&
+    settings.sandboxDesktopEnabled &&
+    !registration.descriptor.portExposure.supportsOnDemandPorts &&
+    !exposedPorts.includes(DESKTOP_STREAM_PORT)
   ) {
     exposedPorts.push(DESKTOP_STREAM_PORT);
   }
@@ -299,10 +384,10 @@ export function createSandboxClientForBackend(
   // at construction for resolveExposedPort(7681) to succeed later on a fresh box.
   // Same condition as the 6080 merge (a desktop-capable image bakes ttyd too).
   if (
-    desktop.available
-    && settings.sandboxDesktopEnabled
-    && !registration.descriptor.portExposure.supportsOnDemandPorts
-    && !exposedPorts.includes(TERMINAL_STREAM_PORT)
+    desktop.available &&
+    settings.sandboxDesktopEnabled &&
+    !registration.descriptor.portExposure.supportsOnDemandPorts &&
+    !exposedPorts.includes(TERMINAL_STREAM_PORT)
   ) {
     exposedPorts.push(TERMINAL_STREAM_PORT);
   }
@@ -328,14 +413,48 @@ function withDockerNetwork(client: SandboxClient, network: string | undefined):
   };
   return {
     backendId: client.backendId,
-    ...(client.supportsDefaultOptions !== undefined ? { supportsDefaultOptions: client.supportsDefaultOptions } : {}),
-    ...(client.create ? { create: async (...args: any[]) => await wrapSession(await (client.create as any)(...args)) } : {}),
-    ...(client.resume ? { resume: async (state: SandboxSessionState) => await wrapSession(await client.resume!(state)) } : {}),
-    ...(client.delete ? { delete: async (state: SandboxSessionState) => await client.delete!(state) } : {}),
-    ...(client.serializeSessionState ? { serializeSessionState: async (state: SandboxSessionState, options) => await client.serializeSessionState!(state, options) } : {}),
-    ...(client.canPersistOwnedSessionState ? { canPersistOwnedSessionState: async (state: SandboxSessionState) => await client.canPersistOwnedSessionState!(state) } : {}),
-    ...(client.canReusePreservedOwnedSession ? { canReusePreservedOwnedSession: async (state: SandboxSessionState) => await client.canReusePreservedOwnedSession!(state) } : {}),
-    ...(client.deserializeSessionState ? { deserializeSessionState: async (state: Record<string, unknown>) => await client.deserializeSessionState!(state) } : {}),
+    ...(client.supportsDefaultOptions !== undefined
+      ? { supportsDefaultOptions: client.supportsDefaultOptions }
+      : {}),
+    ...(client.create
+      ? {
+          create: async (...args: any[]) =>
+            await wrapSession(await (client.create as any)(...args)),
+        }
+      : {}),
+    ...(client.resume
+      ? {
+          resume: async (state: SandboxSessionState) =>
+            await wrapSession(await client.resume!(state)),
+        }
+      : {}),
+    ...(client.delete
+      ? { delete: async (state: SandboxSessionState) => await client.delete!(state) }
+      : {}),
+    ...(client.serializeSessionState
+      ? {
+          serializeSessionState: async (state: SandboxSessionState, options) =>
+            await client.serializeSessionState!(state, options),
+        }
+      : {}),
+    ...(client.canPersistOwnedSessionState
+      ? {
+          canPersistOwnedSessionState: async (state: SandboxSessionState) =>
+            await client.canPersistOwnedSessionState!(state),
+        }
+      : {}),
+    ...(client.canReusePreservedOwnedSession
+      ? {
+          canReusePreservedOwnedSession: async (state: SandboxSessionState) =>
+            await client.canReusePreservedOwnedSession!(state),
+        }
+      : {}),
+    ...(client.deserializeSessionState
+      ? {
+          deserializeSessionState: async (state: Record<string, unknown>) =>
+            await client.deserializeSessionState!(state),
+        }
+      : {}),
   };
 }
 
@@ -351,7 +470,9 @@ async function connectDockerNetwork(network: string, containerId: string): Promi
   if (stderr.includes("already exists")) {
     return;
   }
-  throw new Error(`Failed to connect Docker sandbox container to network ${network}: ${stderr.trim()}`);
+  throw new Error(
+    `Failed to connect Docker sandbox container to network ${network}: ${stderr.trim()}`,
+  );
 }
 
 /**
@@ -364,14 +485,15 @@ export function sandboxStateEntryFromRunState(state: unknown): Record<string, un
   if (!sandboxState) {
     return null;
   }
-  const entry = sandboxState.sessionsByAgent?.[sandboxState.currentAgentKey]
-    ?? (sandboxState.currentAgentKey && sandboxState.sessionState
+  const entry =
+    sandboxState.sessionsByAgent?.[sandboxState.currentAgentKey] ??
+    (sandboxState.currentAgentKey && sandboxState.sessionState
       ? {
-        backendId: sandboxState.backendId,
-        currentAgentKey: sandboxState.currentAgentKey,
-        currentAgentName: sandboxState.currentAgentName,
-        sessionState: sandboxState.sessionState,
-      }
+          backendId: sandboxState.backendId,
+          currentAgentKey: sandboxState.currentAgentKey,
+          currentAgentName: sandboxState.currentAgentName,
+          sessionState: sandboxState.sessionState,
+        }
       : null);
   if (!entry || !entry.sessionState) {
     return null;
@@ -384,7 +506,10 @@ export function sandboxStateEntryFromRunState(state: unknown): Record<string, un
  * sandbox session state from a stored entry (as produced by
  * sandboxStateEntryFromRunState) instead of from a RunState blob.
  */
-export async function restoredSandboxSessionStateFromEntry(entry: Record<string, unknown>, client: unknown): Promise<SandboxSessionState | undefined> {
+export async function restoredSandboxSessionStateFromEntry(
+  entry: Record<string, unknown>,
+  client: unknown,
+): Promise<SandboxSessionState | undefined> {
   if (!client || !entry || typeof entry !== "object" || !("sessionState" in entry)) {
     return undefined;
   }
@@ -409,7 +534,9 @@ export async function restoredSandboxSessionStateFromEntry(entry: Record<string,
  * and the SDK's deserializeSessionState must NOT receive it (it is an opaque
  * runtime-level field, not provider state).
  */
-export function readWorkspaceArchiveFromEnvelopeSessionState(sessionState: unknown): Uint8Array | undefined {
+export function readWorkspaceArchiveFromEnvelopeSessionState(
+  sessionState: unknown,
+): Uint8Array | undefined {
   if (!sessionState || typeof sessionState !== "object") {
     return undefined;
   }
@@ -424,6 +551,24 @@ export function readWorkspaceArchiveFromEnvelopeSessionState(sessionState: unkno
   }
 }
 
+function readWorkspaceArchivePairFromEnvelopeSessionState(sessionState: unknown): {
+  current?: Uint8Array;
+  previous?: Uint8Array;
+} {
+  if (!sessionState || typeof sessionState !== "object") {
+    return {};
+  }
+  const state = sessionState as { workspaceArchivePrev?: unknown };
+  const current = readWorkspaceArchiveFromEnvelopeSessionState(sessionState);
+  const previous = readWorkspaceArchiveFromEnvelopeSessionState({
+    workspaceArchive: state.workspaceArchivePrev,
+  });
+  return {
+    ...(current ? { current } : {}),
+    ...(previous ? { previous } : {}),
+  };
+}
+
 // The native snapshot-ref prefixes the @openai/agents-extensions modal client
 // encodes (snapshots.mjs `NATIVE_SNAPSHOT_PREFIXES`). The ref is
 // `<PREFIX>\n{"snapshot_id":"...",...}`. We re-implement the decode here because
@@ -473,7 +618,10 @@ export function decodeModalSnapshotId(archive: Uint8Array): string | undefined {
  * A tar archive (no snapshot id) is a no-op. Returns the deleted snapshot id (or
  * undefined when nothing was deleted) for observability.
  */
-export async function deletePriorPersistedSnapshot(session: unknown, priorArchiveBase64: string | null | undefined): Promise<string | undefined> {
+export async function deletePriorPersistedSnapshot(
+  session: unknown,
+  priorArchiveBase64: string | null | undefined,
+): Promise<string | undefined> {
   if (!priorArchiveBase64) {
     return undefined;
   }
@@ -487,7 +635,8 @@ export async function deletePriorPersistedSnapshot(session: unknown, priorArchiv
   if (!snapshotId) {
     return undefined;
   }
-  const modal = (session as { modal?: { images?: { delete?: (id: string) => Promise<unknown> } } }).modal;
+  const modal = (session as { modal?: { images?: { delete?: (id: string) => Promise<unknown> } } })
+    .modal;
   const del = modal?.images?.delete;
   if (typeof del !== "function") {
     return undefined;
@@ -500,12 +649,17 @@ export async function deletePriorPersistedSnapshot(session: unknown, priorArchiv
   }
 }
 
-export async function deserializeSandboxSessionStateEnvelope(client: SandboxClient, envelope: unknown): Promise<SandboxSessionState | undefined> {
+export async function deserializeSandboxSessionStateEnvelope(
+  client: SandboxClient,
+  envelope: unknown,
+): Promise<SandboxSessionState | undefined> {
   if (!envelope || typeof envelope !== "object") {
     return undefined;
   }
   if (!client.deserializeSessionState) {
-    throw new Error("Sandbox client must implement deserializeSessionState() to resume RunState sandbox state");
+    throw new Error(
+      "Sandbox client must implement deserializeSessionState() to resume RunState sandbox state",
+    );
   }
   const state = envelope as {
     providerState?: Record<string, unknown>;
@@ -520,8 +674,12 @@ export async function deserializeSandboxSessionStateEnvelope(client: SandboxClie
     ...(state.providerState ?? {}),
     manifest: state.manifest,
     ...(state.snapshot !== undefined ? { snapshot: state.snapshot } : {}),
-    ...(state.snapshotFingerprint !== undefined ? { snapshotFingerprint: state.snapshotFingerprint } : {}),
-    ...(state.snapshotFingerprintVersion !== undefined ? { snapshotFingerprintVersion: state.snapshotFingerprintVersion } : {}),
+    ...(state.snapshotFingerprint !== undefined
+      ? { snapshotFingerprint: state.snapshotFingerprint }
+      : {}),
+    ...(state.snapshotFingerprintVersion !== undefined
+      ? { snapshotFingerprintVersion: state.snapshotFingerprintVersion }
+      : {}),
     workspaceReady: state.workspaceReady,
     ...(state.exposedPorts ? { exposedPorts: structuredClone(state.exposedPorts) } : {}),
   });
@@ -531,14 +689,18 @@ export async function deserializeSandboxSessionStateEnvelope(client: SandboxClie
 // The ONE resume / recovery primitive (P1.2).
 //
 // establishSandboxSessionFromEnvelope is the single re-establish-from-envelope
-// path the stateless model leans on: a turn (or any API-direct op) resolves the
-// group lease, hands us the recovery envelope, and gets back a LIVE non-owned
-// session. On a warm box this is a no-lock warm reattach by id (Modal fromId,
-// e2b reconnect — R4-safe, a stray second handle never spawns a second box).
-// When the provider reports the box genuinely gone (NotFound) we cold-restore
-// from the snapshot via create(). NEVER create() on any OTHER resume error
-// (only on NotFound) — a resume-conflict means the box is alive and the caller
-// must back off, not spawn a rival.
+// path the stateless model leans on. Creation authority is explicit:
+//
+//   - `create-or-restore` is reserved for the caller that won the durable
+//     cold->warming lease transition. It may create a cold box, or replace a
+//     resumable box that the provider proves is gone.
+//   - `resume-only` is for attached turns and API-direct operations. It may
+//     resume the leased box by id, but a provider NotFound propagates to the
+//     caller so the lease can be atomically marked cold. It NEVER creates.
+//
+// This ownership boundary is the double-spawn guard. A provider NotFound alone
+// is not creation authority: many attached callers can observe the same dead id
+// concurrently, while only one caller may own cold->warming.
 // ============================================================================
 
 /** A live, externally-owned sandbox session re-established from the group lease
@@ -551,6 +713,16 @@ export type EstablishedSandboxSession = {
   sessionState: unknown;
   instanceId: string;
   backendId: string;
+  /** How this establish reached a live box: warm reattach by id ("resumed"),
+   *  fresh create ("created"), or fresh create + archive hydration
+   *  ("restored"). Callers use it to emit the durable sandbox.box.* lifecycle
+   *  events — box transitions were previously unrecorded, which made both
+   *  2026-07-06 incidents near-unattributable. Optional so external/legacy
+   *  constructors of this shape stay valid. */
+  origin?: "resumed" | "created" | "restored";
+  /** Set when a create-authorized reattach found the envelope's box GONE
+   *  (provider NotFound) and fell through to cold-restore. */
+  lostInstanceId?: string;
 };
 
 export type SandboxCreatedCallback = (established: EstablishedSandboxSession) => Promise<void>;
@@ -590,14 +762,22 @@ export function isProviderSandboxNotFoundError(backendId: string, error: unknown
   if (!error) {
     return false;
   }
-  const status = (error as { status?: unknown; statusCode?: unknown }).status
-    ?? (error as { statusCode?: unknown }).statusCode;
+  const status =
+    (error as { status?: unknown; statusCode?: unknown }).status ??
+    (error as { statusCode?: unknown }).statusCode;
   if (status === 404) {
     return true;
   }
-  const name = typeof (error as { name?: unknown }).name === "string" ? (error as { name: string }).name : "";
-  const code = typeof (error as { code?: unknown }).code === "string" ? (error as { code: string }).code : "";
-  const message = error instanceof Error ? error.message : typeof error === "string" ? error : String((error as { message?: unknown })?.message ?? "");
+  const name =
+    typeof (error as { name?: unknown }).name === "string" ? (error as { name: string }).name : "";
+  const code =
+    typeof (error as { code?: unknown }).code === "string" ? (error as { code: string }).code : "";
+  const message =
+    error instanceof Error
+      ? error.message
+      : typeof error === "string"
+        ? error
+        : String((error as { message?: unknown })?.message ?? "");
   const haystack = `${name} ${code} ${message}`.toLowerCase();
   // Provider-agnostic "gone" markers (Modal: "sandbox … not found" / terminated;
   // e2b/daytona/runloop: "not found" / "no longer running" / "terminated" /
@@ -619,7 +799,11 @@ export function isProviderSandboxNotFoundError(backendId: string, error: unknown
   ];
   // A "running"/"already exists" resume-conflict is explicitly NOT NotFound — the
   // box is alive; recreating would double-spawn.
-  if (haystack.includes("already running") || haystack.includes("still running") || haystack.includes("already exists")) {
+  if (
+    haystack.includes("already running") ||
+    haystack.includes("still running") ||
+    haystack.includes("already exists")
+  ) {
     return false;
   }
   return goneMarkers.some((marker) => haystack.includes(marker));
@@ -627,53 +811,82 @@ export function isProviderSandboxNotFoundError(backendId: string, error: unknown
 
 function readInstanceId(session: unknown): string {
   const state = (session as { state?: Record<string, unknown> }).state ?? {};
-  const candidate = state.sandboxId ?? state.instanceId ?? state.id ?? state.hostId ?? state.containerId;
+  const candidate =
+    state.sandboxId ??
+    state.instanceId ??
+    state.id ??
+    state.hostId ??
+    state.containerId ??
+    state.workspaceRootPath;
   return typeof candidate === "string" && candidate.length > 0 ? candidate : "";
 }
 
-async function terminateCreatedSandbox(client: ResumeCapableClient, session: unknown, sessionState: unknown): Promise<void> {
+async function terminateCreatedSandbox(
+  client: ResumeCapableClient,
+  session: unknown,
+  sessionState: unknown,
+): Promise<void> {
   const clientWithDelete = client as { delete?: (state: unknown) => Promise<unknown> };
   if (typeof clientWithDelete.delete === "function" && sessionState !== undefined) {
-    try { await clientWithDelete.delete(sessionState); } catch { /* best-effort */ }
+    try {
+      await clientWithDelete.delete(sessionState);
+    } catch {
+      /* best-effort */
+    }
     return;
   }
-  const sess = session as { close?: () => Promise<unknown>; terminate?: () => Promise<unknown>; kill?: () => Promise<unknown> };
-  try { await (sess.terminate ?? sess.kill ?? sess.close)?.(); } catch { /* best-effort */ }
+  const sess = session as {
+    close?: () => Promise<unknown>;
+    terminate?: () => Promise<unknown>;
+    kill?: () => Promise<unknown>;
+  };
+  try {
+    await (sess.terminate ?? sess.kill ?? sess.close)?.();
+  } catch {
+    /* best-effort */
+  }
 }
 
 /**
- * Resume the one box by id from its recovery envelope, or cold-restore from the
- * snapshot when the provider reports it gone. The envelope is the lease's
- * box-identity descriptor (the same per-turn `_sandbox` envelope upserted by the
- * turn activity). A null envelope means a cold session that was never warmed →
- * create() directly.
+ * Establish from one recovery envelope under an explicit creation policy. The
+ * envelope is the lease's box-identity descriptor (the same per-turn `_sandbox`
+ * envelope upserted by the turn activity).
  *
  *  - `opts.backendOverride ?? envelope.backendId ?? settings.sandboxBackend`
  *    selects the backend; the client is built for THAT backend (resume-by-id is
  *    fenced to the original provider).
  *  - warm reattach: deserialize the envelope sessionState → client.resume(state)
- *    (no lock; R4-safe). On a provider NotFound, cold-restore via create().
- *  - cold restore / cold session: client.create() — the ONLY create() site.
+ *    (no lock; R4-safe). `resume-only` propagates provider NotFound.
+ *  - `create-or-restore`: the elected owner may replace a missing warm instance,
+ *    or create directly from a cold/null envelope.
  */
 export async function establishSandboxSessionFromEnvelope(
   settings: Settings,
   envelope: Record<string, unknown> | null,
   opts: {
     sessionId: string;
+    recovery: "create-or-restore" | "resume-only";
     backendOverride?: SandboxBackend;
     environment?: Record<string, string>;
     onSandboxCreated?: SandboxCreatedCallback;
     metrics?: RuntimeMetricsHooks;
   },
 ): Promise<EstablishedSandboxSession> {
-  const envelopeBackend = typeof envelope?.backendId === "string" ? (envelope.backendId as SandboxBackend) : undefined;
-  const backend = (opts.backendOverride ?? envelopeBackend ?? (settings.sandboxBackend as SandboxBackend));
+  const envelopeBackend =
+    typeof envelope?.backendId === "string" ? (envelope.backendId as SandboxBackend) : undefined;
+  const backend =
+    opts.backendOverride ?? envelopeBackend ?? (settings.sandboxBackend as SandboxBackend);
   const environment = opts.environment ?? collectSandboxEnvironment(settings);
-  const client = createSandboxClientForBackend(backend, settings, environment) as ResumeCapableClient | undefined;
+  const client = createSandboxClientForBackend(backend, settings, environment) as
+    | ResumeCapableClient
+    | undefined;
   if (!client) {
-    throw new SandboxConfigError(backend, `Cannot establish a sandbox session for backend "${backend}" (no client; sandboxBackend=none?)`);
+    throw new SandboxConfigError(
+      backend,
+      `Cannot establish a sandbox session for backend "${backend}" (no client; sandboxBackend=none?)`,
+    );
   }
-  if (!client.create) {
+  if (opts.recovery === "create-or-restore" && !client.create) {
     throw new SandboxConfigError(backend, `Sandbox backend "${backend}" does not support create()`);
   }
 
@@ -693,7 +906,10 @@ export async function establishSandboxSessionFromEnvelope(
   // The serialized provider state the box was last persisted as. The envelope
   // shape is the per-turn `_sandbox` entry; its `sessionState` is the provider
   // payload deserializeSandboxSessionStateEnvelope re-hydrates.
-  const envelopeSessionState = envelope && typeof envelope === "object" ? (envelope as { sessionState?: unknown }).sessionState : undefined;
+  const envelopeSessionState =
+    envelope && typeof envelope === "object"
+      ? (envelope as { sessionState?: unknown }).sessionState
+      : undefined;
 
   // The persisted /workspace snapshot the reaper folded onto the lease envelope
   // (sandbox-file-persistence). Present on a re-warm whose box was drain-persisted:
@@ -701,7 +917,8 @@ export async function establishSandboxSessionFromEnvelope(
   //   - COLD lease re-warm (confirmDrainCold preserved a MINIMAL archive-only
   //     envelope `{ sessionState: { workspaceArchive } }` — NO sandboxId, so the
   //     warm-reattach branch must NOT try resume()-by-id; it cold-creates+hydrates).
-  const workspaceArchive = readWorkspaceArchiveFromEnvelopeSessionState(envelopeSessionState);
+  const workspaceArchives = readWorkspaceArchivePairFromEnvelopeSessionState(envelopeSessionState);
+  const workspaceArchive = workspaceArchives.current;
 
   // create() a FRESH box, THEN replay the persisted /workspace snapshot via
   // session.hydrateWorkspace(archive) when one rode the envelope. hydrateWorkspace
@@ -709,7 +926,10 @@ export async function establishSandboxSessionFromEnvelope(
   // image (restoreSnapshotFilesystem); no archive -> a clean empty box. This is the
   // SOLE archive-replay seam, shared by the NotFound warm-reattach path AND the
   // cold-restore branch (b) below.
-  const coldRestore = async (resumeFallbackState?: unknown): Promise<EstablishedSandboxSession> => {
+  const coldRestore = async (
+    resumeFallbackState?: unknown,
+    skipWorkspaceHydrate = false,
+  ): Promise<EstablishedSandboxSession> => {
     const createStarted = Date.now();
     let restored: Awaited<ReturnType<NonNullable<typeof client.create>>>;
     try {
@@ -735,25 +955,53 @@ export async function establishSandboxSessionFromEnvelope(
         throw createCallbackError;
       }
     }
-    if (workspaceArchive) {
-      const hydrate = (restored as { hydrateWorkspace?: (data: Uint8Array) => Promise<void> }).hydrateWorkspace;
+    // Whether an archive was ACTUALLY applied to /workspace — drives `origin`
+    // (and the worker's sandbox.box.created `hydrated` field). A clean-box
+    // fallback, a backend without hydrateWorkspace, or an all-archives-failed
+    // path leaves this false so we never report `hydrated: "archive"` for an
+    // empty workspace.
+    let hydrationApplied = false;
+    if (!skipWorkspaceHydrate && (workspaceArchive || workspaceArchives.previous)) {
+      const hydrate = (restored as { hydrateWorkspace?: (data: Uint8Array) => Promise<void> })
+        .hydrateWorkspace;
       if (typeof hydrate === "function") {
-        try {
-          // hydrateWorkspace may internally REPLACE the underlying box
-          // (restoreSnapshotFilesystem creates a replacement sandbox and terminates
-          // the placeholder), so the instanceId must be re-read AFTER.
-          await hydrate.call(restored, workspaceArchive);
-        } catch (hydrateError) {
-          // sandbox-file-persistence: if hydrateWorkspace throws (snapshot GC'd,
-          // provider timeout, corrupt archive), the placeholder box created above is
-          // live but unhydrated — it would leak up to the full idle/hard lifetime
-          // (3600s) if we just re-throw. Best-effort delete/terminate it BEFORE
-          // re-throwing so no box leaks. The original error semantics are preserved
-          // (the re-throw propagates to the caller). This mirrors the reaper's
-          // discipline: NEVER leave an orphaned box running.
+        let hydrated = false;
+        for (const candidate of [
+          { archive: workspaceArchive, label: "current" },
+          { archive: workspaceArchives.previous, label: "previous" },
+        ]) {
+          if (!candidate.archive) {
+            continue;
+          }
+          try {
+            // hydrateWorkspace may internally REPLACE the underlying box
+            // (restoreSnapshotFilesystem creates a replacement sandbox and terminates
+            // the placeholder), so the instanceId must be re-read AFTER.
+            await hydrate.call(restored, candidate.archive);
+            console.info(
+              `[sandbox] cold-restore hydrated workspace from ${candidate.label} archive`,
+            );
+            hydrated = true;
+            break;
+          } catch (hydrateError) {
+            console.warn(
+              `[sandbox] cold-restore failed to hydrate workspace from ${candidate.label} archive${
+                candidate.label === "current" && workspaceArchives.previous
+                  ? "; trying previous archive"
+                  : ""
+              }`,
+              hydrateError,
+            );
+          }
+        }
+        if (!hydrated) {
+          // If both retained archives are unusable, drop the placeholder and
+          // fall through to the same clean-box behavior as an envelope with no
+          // archive. Restore failure is fail-open; it must not strand a turn.
           await terminateCreatedSandbox(client, restored, restoredState);
-          throw hydrateError;
+          return await coldRestore(resumeFallbackState, true);
         }
+        hydrationApplied = true;
         const hydratedState = (restored as { state?: unknown }).state;
         const hydratedInstanceId = readInstanceId(restored);
         if (hydratedInstanceId && hydratedInstanceId !== established.instanceId) {
@@ -782,6 +1030,7 @@ export async function establishSandboxSessionFromEnvelope(
       sessionState: restoredState ?? resumeFallbackState,
       instanceId: readInstanceId(restored),
       backendId: client.backendId,
+      origin: hydrationApplied ? ("restored" as const) : ("created" as const),
     };
   };
 
@@ -792,44 +1041,78 @@ export async function establishSandboxSessionFromEnvelope(
   // NotFound, so it would propagate instead of cold-restoring. Gate the resume
   // branch on a present sandbox identity so an archive-only envelope falls straight
   // through to the cold-restore+hydrate path (b).
-  const envelopeProviderState = envelopeSessionState && typeof envelopeSessionState === "object"
-    ? (envelopeSessionState as { providerState?: Record<string, unknown> }).providerState
-    : undefined;
+  const envelopeProviderState =
+    envelopeSessionState && typeof envelopeSessionState === "object"
+      ? (envelopeSessionState as { providerState?: Record<string, unknown> }).providerState
+      : undefined;
   const hasResumableInstance = Boolean(
-    envelopeProviderState
-    && typeof envelopeProviderState === "object"
-    && (envelopeProviderState.sandboxId
-      || envelopeProviderState.instanceId
-      || envelopeProviderState.id
-      || envelopeProviderState.containerId),
+    envelopeProviderState &&
+    typeof envelopeProviderState === "object" &&
+    (envelopeProviderState.sandboxId ||
+      envelopeProviderState.instanceId ||
+      envelopeProviderState.id ||
+      envelopeProviderState.containerId ||
+      envelopeProviderState.workspaceRootPath),
   );
 
   // (a) WARM REATTACH BY ID — only when the envelope carries a resumable box id.
-  if (hasResumableInstance && envelopeSessionState && client.resume && client.deserializeSessionState) {
+  if (
+    hasResumableInstance &&
+    envelopeSessionState &&
+    client.resume &&
+    client.deserializeSessionState
+  ) {
     let resumedState: unknown;
     try {
-      resumedState = await deserializeSandboxSessionStateEnvelope(client as unknown as SandboxClient, envelopeSessionState);
+      resumedState = await deserializeSandboxSessionStateEnvelope(
+        client as unknown as SandboxClient,
+        envelopeSessionState,
+      );
     } catch (error) {
-      throw new SandboxConfigError(backend, `Failed to deserialize sandbox resume envelope for backend "${backend}": ${error instanceof Error ? error.message : String(error)}`);
+      throw new SandboxConfigError(
+        backend,
+        `Failed to deserialize sandbox resume envelope for backend "${backend}": ${error instanceof Error ? error.message : String(error)}`,
+      );
     }
     if (resumedState !== undefined) {
       try {
         const session = await client.resume(resumedState);
-        return { client, session, sessionState: resumedState, instanceId: readInstanceId(session), backendId: client.backendId };
+        return {
+          client,
+          session,
+          sessionState: resumedState,
+          instanceId: readInstanceId(session),
+          backendId: client.backendId,
+          origin: "resumed",
+        };
       } catch (error) {
-        // ONLY a provider NotFound (box gone) licenses a cold-restore. Anything
-        // else (transient/auth/network/resume-conflict) propagates: the caller
-        // backs off and re-fences — NEVER spawns a rival box.
-        if (!isProviderSandboxNotFoundError(client.backendId, error)) {
+        // Attached callers never own replacement. Propagate the provider
+        // NotFound so their lease-aware caller can atomically mark this exact
+        // warm epoch/instance cold and re-enter normal admission. Only the
+        // cold->warming winner may replace the missing box.
+        if (
+          !isProviderSandboxNotFoundError(client.backendId, error) ||
+          opts.recovery === "resume-only"
+        ) {
           throw error;
         }
         // COLD-RESTORE: the box is genuinely gone. Modal does NOT restore via
         // create({ snapshot }) — passing `snapshot` to ModalSandboxClient.create()
         // THROWS (assertCoreSnapshotUnsupported). Modal's real persistence is an
-        // OPAQUE ARCHIVE captured by session.persistWorkspace() at reaper-drain
-        // time and folded onto the lease envelope (sandbox-file-persistence). The
-        // shared coldRestore() seam creates a fresh box and replays that archive.
-        return await coldRestore(resumedState);
+        // OPAQUE ARCHIVE captured by session.persistWorkspace() at drain/turn-
+        // snapshot time and folded onto the lease envelope
+        // (sandbox-file-persistence). The shared coldRestore() seam creates a
+        // fresh box and replays that archive. `lostInstanceId` carries the gone
+        // box's id so the caller can emit the durable sandbox.box.lost event.
+        const lostInstanceId = [
+          envelopeProviderState?.sandboxId,
+          envelopeProviderState?.instanceId,
+          envelopeProviderState?.id,
+          envelopeProviderState?.containerId,
+          envelopeProviderState?.workspaceRootPath,
+        ].find((value): value is string => typeof value === "string" && value.length > 0);
+        const restoredSession = await coldRestore(resumedState);
+        return lostInstanceId ? { ...restoredSession, lostInstanceId } : restoredSession;
       }
     }
   }
@@ -839,6 +1122,9 @@ export async function establishSandboxSessionFromEnvelope(
   // envelope confirmDrainCold preserves across draining->cold), replay it so
   // /workspace survives the box churn (sandbox-file-persistence). No archive -> a
   // clean empty box (a never-warmed session).
+  if (opts.recovery === "resume-only") {
+    throw new SandboxResumeStateUnavailableError(backend);
+  }
   return await coldRestore();
 }