@agent-compose/sdk 0.5.6 → 0.5.8

package/src/sandbox.ts

@@ -8,10 +8,14 @@
 import { promises as fs } from "node:fs";
 import { dirname } from "node:path";
 import { spawn } from "node:child_process";
-import { Sandbox } from "e2b";
+import { Sandbox, SandboxNotFoundError, RateLimitError } from "e2b";
 import { Sandbox as Desktop } from "@e2b/desktop";
 import pRetry from "p-retry";
+import type { FailedAttemptError } from "p-retry";
+import { SandboxUnavailableError } from "./sandbox-errors.js";
 import type { SandboxProvider, DesktopSandboxProvider, SandboxCommandResult } from "./types/sandbox.js";
+import type { ConnectorRequestRules } from "./types/workflow-metadata.js";
+import type { NetworkPolicy as VercelNetworkPolicy, NetworkPolicyRule as VercelNetworkPolicyRule } from "@vercel/sandbox";
 
 export type { SandboxProvider, DesktopSandboxProvider, SandboxCommandRunOptions, SandboxCommandResult } from "./types/sandbox.js";
 
@@ -34,10 +38,13 @@ export const AGENT_COMPOSE_TAG = process.env.AGENT_COMPOSE_TAG ??
 const VERCEL_VM_LIFETIME_WINDOW_MS = 6 * 60 * 60 * 1000;
 
 /**
- * Vercel-compatible network policy for outbound HTTPS requests.
- * When a sandbox makes a request matching a domain in `allow`, the Vercel
- * firewall injects the specified headers before forwarding — credentials never
- * exist inside the VM. E2B ignores this field (future self-hosted mapping TBD).
+ * Network policy for outbound HTTPS requests — ONE shape for every provider;
+ * only the enforcement point differs. When a sandbox makes a request matching
+ * a domain in `allow`, the egress layer injects the specified headers before
+ * forwarding — credentials never exist inside the VM. Vercel's firewall
+ * enforces this natively (`requestRules` translate to native `match` rules);
+ * E2B enforces it via the embedded iron-proxy, which pulls the identical
+ * resolved policy from the server.
  */
 export interface SandboxNetworkHeaderTransform {
   headers?: Record<string, string>;
@@ -45,6 +52,10 @@ export interface SandboxNetworkHeaderTransform {
 
 export interface SandboxNetworkAllowRule {
   transform?: SandboxNetworkHeaderTransform[];
+  /** Tier-2 request gate (method/path) — the transform (and, on iron-proxy,
+   *  the request itself) only applies when the request matches. Present on
+   *  connector-auth rules; see `ConnectorRequestRules`. */
+  requestRules?: ConnectorRequestRules;
 }
 
 export interface SandboxNetworkSubnetPolicy {
@@ -60,20 +71,59 @@ export type SandboxNetworkPolicy =
       subnets?: SandboxNetworkSubnetPolicy;
     };
 
+/** Sandbox machine size. A coarse small/medium/large knob that maps to
+ *  provider machine specs at create time. Vercel honours it natively via
+ *  `resources.vcpus` (2048 MB RAM per vCPU). E2B sizing is baked into the
+ *  template, so E2B ignores this field. Default: "small". */
+/** Sandbox hardware SKU. Named for the actual machine spec (vCPU + RAM) rather
+ *  than abstract t-shirt sizes. Memory is always 2048 MB per vCPU:
+ *    2vcpu-4gb   =  2 vCPU /  4 GiB  (Vercel's own default machine)
+ *    4vcpu-8gb   =  4 vCPU /  8 GiB
+ *    8vcpu-16gb  =  8 vCPU / 16 GiB  (per-sandbox ceiling on STANDARD accounts —
+ *                                     probed live: 16 & 32 vCPU 400 on dev)
+ *    32vcpu-64gb = 32 vCPU / 64 GiB  (ENTERPRISE ONLY — standard accounts reject >8 vCPU) */
+export type SandboxSize = "2vcpu-4gb" | "4vcpu-8gb" | "8vcpu-16gb" | "32vcpu-64gb";
+
+/** SKU → Vercel vCPU count (RAM follows at 2048 MB/vCPU). */
+export const SANDBOX_VCPUS: Record<SandboxSize, number> = {
+  "2vcpu-4gb":    2,
+  "4vcpu-8gb":    4,
+  "8vcpu-16gb":   8,
+  "32vcpu-64gb": 32,
+};
+
+/** SDK fallback size when neither the caller nor the deployment specifies one.
+ *  Deliberately conservative — the OPERATIONAL default is chosen per-environment
+ *  by the server via the `SANDBOX_DEFAULT_SIZE` env var (prod = Enterprise →
+ *  `32vcpu-64gb`; dev = `8vcpu-16gb`, since the dev account caps at 8 vCPU).
+ *  TODO(sandbox-size): the prod default is temporarily `32vcpu-64gb` for a
+ *  memory-hungry workload — lower it back when no longer needed. */
+export const DEFAULT_SANDBOX_SIZE: SandboxSize = "2vcpu-4gb";
+
 export interface SandboxCreateOpts {
   envs:           Record<string, string>;
   metadata:       Record<string, string>;
   timeoutMs:      number;
-  /** Provider-specific template/snapshot identifier. E2B: template ID; Vercel: snapshot ID. */
+  /** Provider-specific template/snapshot identifier. E2B: template id or
+   *  snapshot id (omit → E2B's default base). Vercel: snapshot id (omit → node24). */
   template?:      string;
-  /** Outbound request policy. Vercel only — E2B silently ignores. */
+  /** Machine size. Vercel maps it to `resources.vcpus`; E2B ignores it
+   *  (size is template-defined). Omit → `DEFAULT_SANDBOX_SIZE`. */
+  size?:          SandboxSize;
+  /** Outbound request policy with header transforms — ONE shape for every
+   *  provider; only the enforcement point differs. Vercel's firewall
+   *  enforces + injects natively from this value. E2B enforces it via an
+   *  EMBEDDED iron-proxy inside the VM (loopback DNS + iptables, started
+   *  at boot), which fetches the identical resolved policy from the
+   *  server's per-run egress-policy endpoint — so this field is not passed
+   *  to E2B's API. See server/src/sandbox/iron-proxy.ts. */
   networkPolicy?: SandboxNetworkPolicy;
 }
 
 /**
  * What the provider reports as "currently alive" — the single input to orphan
  * reconciliation. `metadata` is best-effort: E2B populates it from sandbox
- * labels, Vercel leaves it empty (the API has no metadata field). Callers
+ * labels, Vercel from sandbox tags (max 5, stamped at create). Callers
  * that need runId correlation cross-reference `sandboxId` against their own
  * state (server: `workflow_runs.sandbox_id` + `run_agent_sandboxes.provider_sandbox_id`).
  */
@@ -92,18 +142,17 @@ interface SandboxProviderDef {
    * Returns the number of sandboxes currently running against this provider.
    * Used for quota observability — account-wide, not per-instance.
    *
-   * E2B scopes by our AGENT_COMPOSE_TAG metadata so each machine only reports
-   * its own sandboxes; DD aggregates across instances with `sum by {provider}`.
-   * Vercel has no user metadata support, so it counts every running sandbox in
-   * the configured project (assumed dedicated to agent-compose).
+   * Both providers scope by our AGENT_COMPOSE_TAG (E2B metadata labels,
+   * Vercel tags) so each fleet only reports its own sandboxes; DD aggregates
+   * across instances with `sum by {provider}`.
    */
   getActiveCount?: (env: Record<string, string>) => Promise<number>;
   /**
    * Provider's view of what's currently alive for our account/fleet.
    * The reconciliation SoT — a sandbox missing from this list IS dead,
    * regardless of what our DB says. Implemented by listing the provider's
-   * running sandboxes; E2B filters by metadata tag, Vercel lists the whole
-   * project (it has no metadata search). 200-row cap applies to Vercel.
+   * running sandboxes; E2B filters by metadata tag, Vercel by the `executor`
+   * sandbox tag.
    */
   listOwned?: (env: Record<string, string>) => Promise<OwnedSandbox[]>;
   /**
@@ -147,6 +196,80 @@ export function makeSandboxProvider(sb: Sandbox | Desktop): SandboxProvider {
   };
 }
 
+/** E2B caps a sandbox's lifetime per plan (Hobby 1h, Pro 24h) and 400s when the
+ *  create timeout exceeds it — and our `AC_SANDBOX_DEADLINE` (4.5h) tops Hobby.
+ *  Clamp the E2B create timeout to this max (raise on Pro via `E2B_MAX_SANDBOX_MS`).
+ *  Read at call time so the workflow bundle stays env-free. */
+function e2bMaxSandboxMs(): number {
+  return Number(process.env.E2B_MAX_SANDBOX_MS) || 60 * 60 * 1000;
+}
+
+/** Is this a FLEETING sandbox-provider race worth retrying — the sandbox being
+ *  paused / reclaimed / stopped under load, an upstream 5xx/429, or a dropped
+ *  connection — vs a PERMANENT failure that must fail fast?
+ *
+ *  E2B raises TYPED errors, so prefer `instanceof` (upgrade-stable — survives any
+ *  message rewording): a reconnect/snapshot that hits a sandbox mid-pause throws
+ *  `SandboxNotFoundError`, rate limits throw `RateLimitError`. (Permanent
+ *  template/snapshot-not-found only occurs on CREATE, which is no longer retried.)
+ *  The message regex is the FALLBACK for the untyped Vercel transport + raw socket
+ *  errors — and deliberately omits bare "timeout"/"network": E2B's permanent
+ *  plan-cap 400 ("timeout … exceeds the maximum sandbox lifetime") contains
+ *  "timeout" and must fail fast, not retry 4×. */
+function isTransientSandboxError(error: unknown): boolean {
+  if (error instanceof SandboxNotFoundError || error instanceof RateLimitError) return true;
+  const message = error instanceof Error ? error.message : String(error ?? "");
+  // The 5xx match is anchored to a status-code context ("status 503",
+  // "502 Bad Gateway", …) — a bare \b5\d\d\b would classify ANY standalone
+  // 500-599 number in a message (durations, row counts) as retryable.
+  return /paus|stopp|resum|transition|status(?:\s+code)?[:\s]+5\d\d\b|\b5\d\d\s+(?:internal|bad gateway|service|gateway)|\b429\b|ECONNRESET|ECONNREFUSED|ETIMEDOUT|socket hang up|fetch failed/i
+    .test(message);
+}
+
+/** The ONE place the sandbox retry policy lives. Provisioning, reconnecting, and
+ *  snapshotting all race with the sandbox being paused/reclaimed; this runs the call
+ *  through p-retry's generic backoff, retrying ONLY the transient race (and failing
+ *  fast otherwise). reconnectSandbox and provider.snapshot() all go through it, so
+ *  no provider re-implements the pattern or can forget it. `shouldRetry` is
+ *  overridable for call sites where the default classification is wrong (see
+ *  reconnectSandbox's SandboxNotFoundError handling). */
+function withSandboxRetry<T>(
+  fn: () => Promise<T>,
+  shouldRetry: (error: FailedAttemptError) => boolean = isTransientSandboxError,
+): Promise<T> {
+  return pRetry(fn, { retries: 4, minTimeout: 400, factor: 2, shouldRetry });
+}
+
+/** Add snapshot-retry to a freshly-obtained provider. create/reconnect are retried at
+ *  the registry call (`createSandbox`/`reconnectSandbox`); snapshot is a method, so it
+ *  gets the same `withSandboxRetry` here. `commands.run` is deliberately NOT wrapped —
+ *  retrying user-code execution could double-apply side effects, so step execution
+ *  relies on the engine's pre-step sandbox recovery instead. No-op without a snapshot. */
+function withSnapshotRetry(p: SandboxProvider): SandboxProvider {
+  if (!p.snapshot) return p;
+  const snapshot = p.snapshot.bind(p);
+  return { ...p, snapshot: () => withSandboxRetry(snapshot) };
+}
+
+/** E2B base provider + live-filesystem snapshot. E2B's `createSnapshot()` captures
+ *  the running sandbox as a persistent snapshot whose id is usable as a
+ *  `Sandbox.create()` source and outlives the origin sandbox — so E2B reaches
+ *  snapshot / `bootFrom` parity with Vercel, and snapshot-backed `ctx.pause` works
+ *  on E2B. (Desktop intentionally omits this — it is registry-only, not selectable.) */
+function makeE2bSandboxProvider(sb: Sandbox): SandboxProvider {
+  return {
+    ...makeSandboxProvider(sb),
+    async snapshot() {
+      // Raw capture — the transient pause/reclaim race is retried centrally:
+      // createSandbox/reconnectSandbox wrap every provider's snapshot() in
+      // withSandboxRetry (via withSnapshotRetry). create itself is deliberately
+      // NOT retried — see createSandbox.
+      const { snapshotId } = await sb.createSnapshot();
+      return { snapshotId };
+    },
+  };
+}
+
 export function makeDesktopSandboxProvider(sb: Desktop): DesktopSandboxProvider {
   return {
     ...makeSandboxProvider(sb),
@@ -209,6 +332,88 @@ export async function parseSseExecStream(
 
 // ── Vercel helpers ────────────────────────────────────────────────────────────
 
+/** Matches a path containing a dot-dot segment — literal (`/../`) or
+ *  percent-encoded (`%2e%2e`, `%2f` boundaries). `DOT_SEGMENT_PATH_RE2` is
+ *  the RE2 form handed to Vercel's matcher: wrapped in `.*` so it behaves
+ *  identically under search and full-match semantics (RE2 has no lookaround,
+ *  so the guard is a positive match on the traversal pattern, used as a
+ *  credential-stripping first rule). `DOT_SEGMENT_RE` is the local JS twin
+ *  used to validate DECLARED prefixes. */
+export const DOT_SEGMENT_PATH_RE2 = ".*(?:^|/|%2[fF])(?:\\.|%2[eE]){2}(?:/|%2[fF]|$).*";
+const DOT_SEGMENT_RE = /(?:^|\/|%2f)(?:\.|%2e){2}(?:\/|%2f|$)/i;
+
+/** A Tier-2 path gate decides where a brokered credential may ride, so a
+ *  malformed declared prefix is a config bug that must fail closed, not a
+ *  string we forward verbatim to a third-party matcher. */
+function assertSafePathPrefix(domain: string, prefix: string): void {
+  if (!prefix.startsWith("/") || DOT_SEGMENT_RE.test(prefix)) {
+    throw new Error(
+      `Invalid Tier-2 pathPrefix ${JSON.stringify(prefix)} for "${domain}": prefixes must start with "/" and contain no dot-segments`,
+    );
+  }
+}
+
+/**
+ * Translate our policy shape into Vercel's native `NetworkPolicy`.
+ *
+ * The shapes are identical except for Tier-2: our `requestRules`
+ * (`{ methods, pathPrefixes }`) become Vercel `match` rules
+ * (`{ method, path: { startsWith } }`) — one rule per path prefix, since a
+ * Vercel matcher carries a single path.
+ *
+ * Dot-segment defense: prefix matching is raw — `/repos/acme/../../user`
+ * starts with `/repos/acme/` but the origin normalizes it to `/user`, riding
+ * the credential outside its declared scope. We don't get to assume the
+ * enforcer RFC-3986-normalizes before matching, so every domain with a path
+ * gate gets a transform-free FIRST rule matching dot-segment paths
+ * (first-match-wins ⇒ such requests go out credential-free), and declared
+ * prefixes themselves are validated (must start with "/", no dot-segments).
+ *
+ * Accepted divergence from iron-proxy (E2B): off-gate requests to an allowed
+ * domain pass WITHOUT the transform (Vercel terminates TLS only for
+ * transform-bearing domains and applies first-match-wins) — iron-proxy
+ * REFUSES them instead. Reachability differs; the property that matters is
+ * preserved on both substrates: the brokered credential rides only declared
+ * method/path combinations. Pinned by vercel-network-policy.test.ts.
+ */
+export function toVercelNetworkPolicy(policy: SandboxNetworkPolicy): VercelNetworkPolicy {
+  if (typeof policy === "string" || !policy.allow || Array.isArray(policy.allow)) {
+    return policy as VercelNetworkPolicy;
+  }
+  const allow: Record<string, VercelNetworkPolicyRule[]> = {};
+  for (const [domain, rules] of Object.entries(policy.allow)) {
+    const translated = rules.flatMap((rule): VercelNetworkPolicyRule[] => {
+      const { requestRules, ...rest } = rule;
+      const methods = requestRules?.methods ?? [];
+      const prefixes = requestRules?.pathPrefixes ?? [];
+      if (methods.length === 0 && prefixes.length === 0) return [rest];
+      const methodMatch = methods.length > 0 ? { method: methods } : {};
+      if (prefixes.length === 0) return [{ ...rest, match: methodMatch }];
+      for (const prefix of prefixes) assertSafePathPrefix(domain, prefix);
+      return prefixes.map((prefix) => ({ ...rest, match: { ...methodMatch, path: { startsWith: prefix } } }));
+    });
+    const hasPathGate = rules.some((rule) => (rule.requestRules?.pathPrefixes?.length ?? 0) > 0);
+    allow[domain] = hasPathGate
+      ? [{ match: { path: { regex: DOT_SEGMENT_PATH_RE2 } } }, ...translated]
+      : translated;
+  }
+  return { ...policy, allow };
+}
+
+/** Vercel caps sandbox tags at 5. Build the tag set with the fleet `executor`
+ *  tag always present and always winning over caller metadata — `listOwned` /
+ *  orphan reconciliation scope on it, so a clobbered or dropped executor tag
+ *  makes the sandbox invisible to cleanup. When metadata overflows the cap,
+ *  keep executor + the lexicographically-first 4 metadata keys, so what gets
+ *  dropped is deterministic rather than dependent on object insertion order. */
+export const VERCEL_MAX_TAGS = 5;
+export function buildVercelTags(metadata: Record<string, string>): Record<string, string> {
+  const tags: Record<string, string> = { ...metadata, executor: AGENT_COMPOSE_TAG };
+  const extras = Object.keys(tags).filter((k) => k !== "executor").sort();
+  for (const key of extras.slice(VERCEL_MAX_TAGS - 1)) delete tags[key];
+  return tags;
+}
+
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function makeVercelSandboxProvider(sb: any, globalEnvs?: Record<string, string>): SandboxProvider {
   // Vercel's Sandbox.create({ env }) does NOT flow to runCommand subprocesses — they start fresh shells.
@@ -217,8 +422,21 @@ function makeVercelSandboxProvider(sb: any, globalEnvs?: Record<string, string>)
   const mergeEnvs = (cmdEnvs?: Record<string, string>) =>
     globalEnvs ? { ...globalEnvs, ...cmdEnvs } : cmdEnvs;
 
+  // Wrap a sandbox-side failure as `SandboxUnavailableError`. `retryable` is
+  // decided purely by *where* the error was caught, not by inspecting its
+  // shape: true before the runner launched any user code (file write / command
+  // launch / reconnect — safe to re-provision and retry), false once the
+  // runner streamed (user code may have run). The original message is
+  // preserved for server-side diagnostics.
+  const asUnavailable = (err: unknown, retryable: boolean): never => {
+    throw new SandboxUnavailableError(err instanceof Error ? err.message : String(err), { retryable, sandboxId: sb.name });
+  };
+
   return {
-    sandboxId: sb.sandboxId,
+    // @vercel/sandbox 2.x is name-keyed: `name` is the stable identifier
+    // (`Sandbox.get({ name })`); the v1 `sandboxId` getter is gone. Our
+    // provider-facing field keeps its name — it's "the provider-native id".
+    sandboxId: sb.name,
     commands: {
       async run(cmd, opts) {
         const signal = opts?.timeoutMs ? AbortSignal.timeout(opts.timeoutMs) : undefined;
@@ -226,34 +444,75 @@ function makeVercelSandboxProvider(sb: any, globalEnvs?: Record<string, string>)
         let stderr = "";
         // `sudo: true` is Vercel's native root flag — applied to the `sh`
         // invocation so the whole shell (and its children) runs as root.
+        // A launch failure means the command never started — nothing
+        // user-side ran, so any error here is safe to re-provision + retry.
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const handle: any = await sb.runCommand({ cmd: "sh", args: ["-c", cmd], cwd: opts?.cwd, env: mergeEnvs(opts?.envs), detached: true, signal, ...(opts?.sudo ? { sudo: true } : {}) });
-        // Reconnect to the already-running command on transient stream failures (e.g. BrotliDecompressionError).
-        // `h.logs()` replays from the start on reconnect, so reset accumulators per
-        // attempt to avoid double-counting. The streaming callbacks may still fire
-        // for duplicate chunks during retries — an acceptable tradeoff for resilience.
-        await pRetry(async (attempt) => {
-          // eslint-disable-next-line @typescript-eslint/no-explicit-any
-          const h: any = attempt === 1 ? handle : await sb.getCommand(handle.cmdId);
-          if (attempt > 1) { stdout = ""; stderr = ""; }
-          for await (const log of h.logs()) {
-            if (log.stream === "stdout") { stdout += log.data; opts?.onStdout?.(log.data); }
-            else { stderr += log.data; opts?.onStderr?.(log.data); }
-          }
-        }, { retries: 3, minTimeout: 1_000, factor: 2 });
-        const finished = await handle.wait();
-        return { exitCode: finished.exitCode, stdout, stderr };
+        const handle: any = await sb.runCommand({ cmd: "sh", args: ["-c", cmd], cwd: opts?.cwd, env: mergeEnvs(opts?.envs), detached: true, signal, ...(opts?.sudo ? { sudo: true } : {}) })
+          .catch((err: unknown) => asUnavailable(err, true));
+        // Stream + wait. Reconnect to the already-running command on transient
+        // stream failures (e.g. BrotliDecompressionError). `h.logs()` replays
+        // from the start on reconnect, so reset accumulators per attempt to
+        // avoid double-counting.
+        const collect = async () => {
+          await pRetry(async (attempt) => {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const h: any = attempt === 1 ? handle : await sb.getCommand(handle.cmdId);
+            if (attempt > 1) { stdout = ""; stderr = ""; }
+            for await (const log of h.logs()) {
+              if (log.stream === "stdout") { stdout += log.data; opts?.onStdout?.(log.data); }
+              else { stderr += log.data; opts?.onStderr?.(log.data); }
+            }
+          }, { retries: 3, minTimeout: 1_000, factor: 2 });
+          const finished = await handle.wait();
+          return { exitCode: finished.exitCode, stdout, stderr };
+        };
+        // `timeoutMs` MUST be authoritative. The AbortSignal alone is not: a
+        // command that RUNS but emits nothing (e.g. a wedged `archil checkout`
+        // on a blocked data plane) leaves `logs()`/`wait()` pending and the
+        // abort never interrupts the await — the call hangs for the activity's
+        // whole multi-hour ceiling. Race a hard client-side deadline so a hung
+        // command fails fast and the caller's degrade/retry policy takes over.
+        let timer: ReturnType<typeof setTimeout> | undefined;
+        const deadline = opts?.timeoutMs
+          ? new Promise<never>((_, reject) => {
+              timer = setTimeout(
+                () => reject(new Error(`command timed out after ${opts.timeoutMs}ms: ${cmd.slice(0, 200)}`)),
+                opts.timeoutMs);
+            })
+          : undefined;
+        try {
+          return await (deadline ? Promise.race([collect(), deadline]) : collect());
+        } catch (err) {
+          // A timeout (hard deadline or AbortSignal) is a COMMAND overrunning
+          // its budget, not the sandbox dying — the VM is alive. Surface it as
+          // a plain error so the caller's own retry/deadline policy decides
+          // (classifying it as terminal sandbox-unavailable failed whole runs
+          // over a single slow mount probe, observed live).
+          if (err instanceof Error && err.message.startsWith("command timed out after")) throw err;
+          if (signal?.aborted) throw new Error(`command timed out after ${opts?.timeoutMs}ms: ${cmd.slice(0, 200)}`);
+          return asUnavailable(err, false);
+        } finally {
+          if (timer) clearTimeout(timer);
+        }
       },
     },
     files: {
       async write(path: string, content: string) {
-        await sb.writeFiles([{ path, content }]);
+        // File writes happen before the runner launches (step input / pause
+        // resolution files), so a failure here means nothing ran — retryable.
+        await sb.writeFiles([{ path, content }]).catch((err: unknown) => asUnavailable(err, true));
       },
     },
     // Propagate errors — `killAllRunSandboxes` relies on kill failures being
     // observable so it can leave `sandbox_id` set for `findOrphanedSandboxes`
     // to retry on next boot. Swallowing here makes the orphan retry loop blind.
-    async kill() { await sb.stop(); },
+    //
+    // kill must DESTROY: in @vercel/sandbox 2.x stop() only halts the VM — the
+    // name-keyed sandbox record persists (reserving the name; commands against
+    // a stale handle can transparently resume a stopped VM). delete() is the
+    // destroying call ("after deletion the instance becomes inert"), so stop
+    // then delete.
+    async kill() { await sb.stop(); await sb.delete(); },
     // Vercel native snapshot — used by sandbox-environments to capture the
     // configured VM after the customer's `setup()` completes. `expiration: 0`
     // is Vercel's "never expires" value; env snapshots are long-lived by
@@ -267,6 +526,15 @@ function makeVercelSandboxProvider(sb: any, globalEnvs?: Record<string, string>)
         ...(sizeBytes !== undefined ? { sizeBytes } : {}),
       };
     },
+    // Push a freshly-resolved egress policy onto the live sandbox —
+    // @vercel/sandbox 2.x `update({ networkPolicy })`. Lets the server
+    // re-resolve the run policy (re-minting connector access tokens)
+    // before each step instead of living with the policy baked at create.
+    // E2B leaves this undefined: its enforcement (iron-proxy) lives inside
+    // the VM and is configured at boot.
+    async updateNetworkPolicy(policy) {
+      await sb.update({ networkPolicy: toVercelNetworkPolicy(policy) });
+    },
   };
 }
 
@@ -329,10 +597,33 @@ const SANDBOX_PROVIDERS: Record<string, SandboxProviderDef> = {
       const { Sandbox: VercelSandbox } = await import("@vercel/sandbox");
       const creds = { token: env.VERCEL_ACCESS_TOKEN, teamId: env.VERCEL_TEAM_ID, projectId: env.VERCEL_PROJECT_ID };
       // `template` is a snapshot id (runtime is baked in); absent → fresh node24.
-      const np = opts.networkPolicy;
-      const sb = await VercelSandbox.create(opts.template
-        ? { source: { type: "snapshot" as const, snapshotId: opts.template }, timeout: opts.timeoutMs, env: opts.envs, ...(np ? { networkPolicy: np } : {}), ...creds }
-        : { runtime: "node24" as const, timeout: opts.timeoutMs, env: opts.envs, ...(np ? { networkPolicy: np } : {}), ...creds },
+      const np = opts.networkPolicy ? { networkPolicy: toVercelNetworkPolicy(opts.networkPolicy) } : {};
+      // Caller metadata rides as Vercel tags — `listOwned` filters on the
+      // `executor` key, mirroring E2B's metadata-tag scoping. buildVercelTags
+      // guarantees executor wins over metadata and survives the 5-tag cap.
+      const tags = buildVercelTags(opts.metadata);
+      // No explicit template/bootFrom → VERCEL_DEFAULT_SNAPSHOT if set (the
+      // platform agent-env base: claude, archil, rtk, bun, agentc CLI, the
+      // SDK in /workspace/node_modules, /ac:* skills, AGENTS.md — built by
+      // .agentc/environments/agent-env.ts), else raw node24. The E2B
+      // analogue is E2B_DEFAULT_TEMPLATE below. Because `saveLatest`/
+      // `reuse` captures snapshot the whole filesystem, everything a run
+      // bakes on top of this base chains from it — the base tools are in
+      // every derived snapshot for free.
+      const tmpl = opts.template ?? process.env.VERCEL_DEFAULT_SNAPSHOT;
+      // Machine size → vCPUs (RAM auto-follows at 2048 MB/vCPU). Always sent
+      // explicitly so the spec is deterministic and self-documenting rather
+      // than riding Vercel's implicit default; "small" maps to that default
+      // anyway, so existing runs are unchanged.
+      const resources = { vcpus: SANDBOX_VCPUS[opts.size ?? DEFAULT_SANDBOX_SIZE] };
+      // `persistent: false` — @vercel/sandbox 2.x creates persistent-by-default
+      // sandboxes: stop() auto-snapshots (and keeps billing storage), commands
+      // transparently resume a stopped VM, and kill no longer destroys. Our
+      // sandboxes are single-run and lifecycle-managed by the engine (explicit
+      // snapshot() / kill()), so opt out in BOTH branches.
+      const sb = await VercelSandbox.create(tmpl
+        ? { source: { type: "snapshot" as const, snapshotId: tmpl }, timeout: opts.timeoutMs, env: opts.envs, tags, persistent: false, resources, ...np, ...creds }
+        : { runtime: "node24" as const, timeout: opts.timeoutMs, env: opts.envs, tags, persistent: false, resources, ...np, ...creds },
       );
       // Pass envs as globalEnvs so they're injected into every runCommand subprocess.
       // (Vercel's Sandbox.create env parameter does not flow to runCommand subprocesses.)
@@ -345,33 +636,58 @@ const SANDBOX_PROVIDERS: Record<string, SandboxProviderDef> = {
         teamId:    process.env.VERCEL_TEAM_ID ?? "",
         projectId: process.env.VERCEL_PROJECT_ID ?? "",
       };
-      return makeVercelSandboxProvider(await VercelSandbox.get({ sandboxId, ...creds }));
+      // Reconnecting happens before any user code runs, so any failure here
+      // (sandbox already stopping, API blip) is a retryable sandbox-unavailable
+      // — the workflow re-provisions a fresh one rather than failing the run.
+      // 2.x is name-keyed: the id we persisted IS the sandbox name.
+      const sb = await VercelSandbox.get({ name: sandboxId, ...creds }).catch((err: unknown) => {
+        throw new SandboxUnavailableError(err instanceof Error ? err.message : String(err), { retryable: true, sandboxId });
+      });
+      return makeVercelSandboxProvider(sb);
     },
     getActiveCount: async (env) => (await SANDBOX_PROVIDERS.vercel.listOwned!(env)).length,
     listOwned: async (env) => {
       const { Sandbox: VercelSandbox } = await import("@vercel/sandbox");
       const creds = { token: env.VERCEL_ACCESS_TOKEN, teamId: env.VERCEL_TEAM_ID, projectId: env.VERCEL_PROJECT_ID };
-      // Bounds the scan to `VERCEL_VM_LIFETIME_WINDOW_MS` — Pro/Enterprise caps
-      // VM lifetime at 5h, +1h slack for clock skew and `extendTimeout()`.
-      // Without this, projects with thousands of historical terminal sandboxes
-      // page forever (10k+ in v0.6.28). Truncating silently would hide orphans,
-      // so we throw at the 50-page cap instead. Each page is wrapped in pRetry
-      // so a transient 5xx/429/network blip doesn't cost a reconcile tick.
+      // Scoped to our fleet by the `executor` tag (stamped at create) and
+      // bounded to `VERCEL_VM_LIFETIME_WINDOW_MS` — Pro/Enterprise caps VM
+      // lifetime at 5h, +1h slack for clock skew and `extendTimeout()`. The
+      // sort is pinned to createdAt-desc explicitly — the early-stop below is
+      // only correct under that order, and the server's default sort is not a
+      // contract we can rely on — so we stop at the first item older than the
+      // window instead of paging through thousands of historical terminal
+      // sandboxes (10k+ in v0.6.28). Truncating silently would hide orphans,
+      // so we throw at the page cap instead — 200 pages × the API's 50-item
+      // limit preserves the previous 10k scan bound (v1 paged 200 × 50). Each
+      // page is wrapped in pRetry so a transient 5xx/429/network blip doesn't
+      // cost a reconcile tick.
+      //
+      // 2.x cutover caveat: sandboxes created by the pre-2.x code carry no
+      // tags, so for up to VERCEL_VM_LIFETIME_WINDOW_MS after a deploy they
+      // are invisible here (and their persisted v1 sandbox ids don't resolve
+      // via `Sandbox.get({ name })`). Self-limiting — the provider's 5h
+      // lifetime cap expires them — but drain in-flight Vercel runs before
+      // deploying if losing their sandbox state matters.
       const since = Date.now() - VERCEL_VM_LIFETIME_WINDOW_MS;
       const out: OwnedSandbox[] = [];
-      let until: number | undefined;
-      for (let page = 0; page < 50; page++) {
-        const { json } = await pRetry(
-          () => VercelSandbox.list({ ...creds, limit: 200, since, ...(until !== undefined ? { until } : {}) }),
+      let cursor: string | undefined;
+      for (let page = 0; page < 200; page++) {
+        const result = await pRetry(
+          // limit is capped at 50 by the v2 list API — 51+ answers 400
+          // (probed live 2026-06-10; not documented in the SDK types).
+          () => VercelSandbox.list({ ...creds, limit: 50, sortBy: "createdAt", sortOrder: "desc", tags: { executor: AGENT_COMPOSE_TAG }, ...(cursor !== undefined ? { cursor } : {}) }),
           { retries: 3, minTimeout: 500, factor: 2 },
         );
-        for (const sb of json.sandboxes) if (sb.status === "running") {
-          out.push({ sandboxId: sb.id, createdAt: new Date(sb.createdAt), metadata: {} });
+        for (const sb of result.sandboxes) {
+          if (sb.createdAt < since) return out;
+          if (sb.status === "running") {
+            out.push({ sandboxId: sb.name, createdAt: new Date(sb.createdAt), metadata: sb.tags ?? {} });
+          }
         }
-        if (json.pagination.next === null) return out;
-        until = json.pagination.next;
+        if (result.pagination.next === null) return out;
+        cursor = result.pagination.next;
       }
-      throw new Error("Vercel listSandboxes exceeded 50 pages within the lifetime window — refuse to silently truncate");
+      throw new Error("Vercel listSandboxes exceeded 200 pages (10k sandboxes) within the lifetime window — refuse to silently truncate");
     },
     deleteSnapshot: async (snapshotId, env) => {
       const { Snapshot } = await import("@vercel/sandbox");
@@ -382,11 +698,45 @@ const SANDBOX_PROVIDERS: Record<string, SandboxProviderDef> = {
   },
   "e2b": {
     requiredEnv: { E2B_API_KEY: "E2B API key — e2b.dev/dashboard" },
-    create: async ({ template, ...sandboxOpts }) => {
-      if (!template) throw new Error("E2B provider requires an explicit `template` (Dockerfile-based — no default base image)");
-      return makeSandboxProvider(await Sandbox.create(template, sandboxOpts));
+    // `size` dropped here: E2B has no create-time resource knob (specs are
+    // baked into the template/snapshot), so sizing on E2B = a pre-sized
+    // template, not this field. Honoured only on Vercel.
+    create: async ({ template, timeoutMs, networkPolicy: _np, size: _size, ...rest }) => {
+      // `template` is an E2B template id, a snapshot id (a valid create source
+      // that persists beyond its origin sandbox — bootFrom parity), or absent →
+      // E2B's default base (Debian + node + npm/python/git), mirroring Vercel's
+      // node24 default. With no explicit template/bootFrom, fall back to
+      // E2B_DEFAULT_TEMPLATE if set (a prebuilt base — e.g. one with the claude CLI
+      // + chromium baked in and more RAM than the stock 482MB base), the
+      // per-deployment analogue of Vercel's node24; else E2B's stock base.
+      // Self-provisioning runtimes (claude/codex/amp via bootFrom:"reuse") install
+      // their CLI on the base and cache it in the captured snapshot, so a
+      // template-less first run boots, installs, snapshots. Clamp the lifetime to
+      // E2B's per-plan cap (see e2bMaxSandboxMs) so a 4.5h AC_SANDBOX_DEADLINE
+      // doesn't 400 on smaller plans.
+      //
+      // `networkPolicy` is deliberately NOT passed to E2B's API. E2B has no
+      // header-injecting firewall; enforcement happens INSIDE the VM via
+      // the embedded iron-proxy started at sandbox boot, which pulls the
+      // identical resolved policy from the server. One policy shape, two
+      // enforcement points. See server/src/sandbox/iron-proxy.ts.
+      const maxMs = e2bMaxSandboxMs();
+      if (timeoutMs > maxMs) {
+        // The clamp turns E2B's explicit create-time 400 into a silent mid-run
+        // sandbox death at the cap — say so up front instead of letting a run
+        // budgeted past the cap discover it as a terminal infra failure.
+        console.warn(
+          `[sandbox] e2b create timeout clamped: requested ${timeoutMs}ms exceeds the plan cap ${maxMs}ms — ` +
+          `the sandbox dies at the cap, not the requested deadline. Raise E2B_MAX_SANDBOX_MS on plans that allow more.`,
+        );
+      }
+      const sandboxOpts = { ...rest, timeoutMs: Math.min(timeoutMs, maxMs) };
+      const tmpl = template ?? process.env.E2B_DEFAULT_TEMPLATE;
+      return makeE2bSandboxProvider(
+        await (tmpl ? Sandbox.create(tmpl, sandboxOpts) : Sandbox.create(sandboxOpts)),
+      );
     },
-    reconnect: async (sandboxId) => makeSandboxProvider(
+    reconnect: async (sandboxId) => makeE2bSandboxProvider(
       await Sandbox.connect(sandboxId, { apiKey: process.env.E2B_API_KEY ?? "", timeoutMs: 60 * 60 * 1000 }),
     ),
     killAll: async () => {
@@ -413,12 +763,25 @@ const SANDBOX_PROVIDERS: Record<string, SandboxProviderDef> = {
       }
       return out;
     },
+    deleteSnapshot: async (snapshotId, env) => {
+      // E2B snapshots are team-scoped; delete by id. Best-effort like Vercel's.
+      await Sandbox.deleteSnapshot(snapshotId, { apiKey: env.E2B_API_KEY });
+    },
   },
   "e2b-desktop": {
     requiredEnv: { E2B_API_KEY: "E2B API key — e2b.dev/dashboard" },
-    create: async ({ template, ...sandboxOpts }) => {
+    // Mirror "e2b": `networkPolicy` is deliberately NOT passed to the SDK —
+    // its transforms carry live connector tokens, and a credential-bearing
+    // object must never cross into a third-party library's option bag.
+    // Lifetime is clamped to E2B's per-plan cap the same way.
+    // `size` dropped here: E2B has no create-time resource knob (specs are
+    // baked into the template/snapshot), so sizing on E2B = a pre-sized
+    // template, not this field. Honoured only on Vercel.
+    create: async ({ template, timeoutMs, networkPolicy: _np, size: _size, ...rest }) => {
       if (!template) throw new Error("E2B Desktop provider requires an explicit `template` (Dockerfile-based — no default base image)");
-      return makeDesktopSandboxProvider(await Desktop.create(template, sandboxOpts));
+      return makeDesktopSandboxProvider(
+        await Desktop.create(template, { ...rest, timeoutMs: Math.min(timeoutMs, e2bMaxSandboxMs()) }),
+      );
     },
   },
 };
@@ -431,14 +794,28 @@ export async function createSandbox(provider: SandboxProviderName, opts: Sandbox
     .filter(([key]) => !process.env[key])
     .map(([key, desc]) => `  ${key}  — ${desc}`);
   if (missing.length > 0) throw new Error(`Sandbox provider "${provider}" requires env vars:\n${missing.join("\n")}`);
-  return def.create(opts, Object.fromEntries(Object.keys(def.requiredEnv).map(k => [k, process.env[k]!])));
+  const env = Object.fromEntries(Object.keys(def.requiredEnv).map((k) => [k, process.env[k]!]));
+  // Provision once — do NOT wrap create in withSandboxRetry: a create whose RESPONSE
+  // is lost (client-side timeout after the provider already provisioned) would, on
+  // retry, mint a SECOND billed sandbox and orphan the first. Provision failures are
+  // re-attempted at the workflow layer. The result is still wrapped so snapshot()
+  // retries the transient pause/reclaim race (which is where the race actually lives).
+  return withSnapshotRetry(await def.create(opts, env));
 }
 
 /** Reconnect to an existing sandbox by provider + provider-native sandbox ID. */
 export async function reconnectSandbox(provider: SandboxProviderName, sandboxId: string): Promise<SandboxProvider> {
   const def = SANDBOX_PROVIDERS[provider];
   if (!def?.reconnect) throw new Error(`Provider "${provider}" does not support reconnect`);
-  return def.reconnect(sandboxId);
+  // Retry the transient reconnect/resume race, then add snapshot-retry to the
+  // result. On RECONNECT, SandboxNotFoundError usually means the sandbox is
+  // genuinely gone (killed / lifetime-expired) — permanent — but it can also
+  // be the transient mid-pause race, so it gets exactly ONE retry instead of
+  // burning the full 4-attempt backoff before surfacing.
+  return withSnapshotRetry(await withSandboxRetry(
+    () => def.reconnect!(sandboxId),
+    (error) => error instanceof SandboxNotFoundError ? error.attemptNumber <= 1 : isTransientSandboxError(error),
+  ));
 }
 
 /** Delete a snapshot by id on the named provider. No live sandbox needed. */