@coreplane/switchboard 1.227.0 → 1.229.0

package/dist/assets/src/core/runUsage.ts

@@ -0,0 +1,199 @@
+import type { RunEvent } from "./runEvents.js";
+
+// What a run cost in tokens, and who it belongs to — the data behind "cost by
+// user" on the costs page (docs/reference/specs/costs.md). Every provider call a
+// run makes is one `model.turn` span with the provider's own token counts as
+// attrs (metered by the model proxy or the runner; docs/reference/specs/tracing.md), so a
+// run's usage is the sum of those spans, per model. It is computed ONCE, at
+// finish, from the events still in memory (`assembleRunRecord`), and rides the
+// record — a record's events may be cut to fit the byte budget, so an aggregate
+// taken then is more faithful than one re-read later. A record written before
+// the field existed has none; the store fills it in from the run's stored
+// events on demand (the lazy backfill), and reports how many still wait.
+
+export interface ModelUsage {
+  turns: number;
+  inputTokens: number;
+  outputTokens: number;
+  cacheReadTokens: number;
+  cacheWriteTokens: number;
+}
+
+export interface RunUsage {
+  /** Every `model.turn` span, whatever its model. */
+  turns: number;
+  /** Per `<provider>/<model>` as the span named it; `unknown` for a turn without a model attr. */
+  byModel: Record<string, ModelUsage>;
+}
+
+export const UNKNOWN_MODEL = "unknown";
+
+const MODEL_TURN = "model.turn";
+
+const num = (v: unknown): number => (typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : 0);
+
+export const emptyUsage = (): RunUsage => ({ turns: 0, byModel: {} });
+
+/** The run's usage from its events: one `model.turn` span end per provider call. */
+export function usageOfEvents(events: readonly RunEvent[]): RunUsage {
+  const usage = emptyUsage();
+  for (const e of events) {
+    if (e.type !== "span_end" || e.name !== MODEL_TURN) continue;
+    const attrs = (e.attrs ?? {}) as Record<string, unknown>;
+    const model = typeof attrs.model === "string" && attrs.model ? attrs.model : UNKNOWN_MODEL;
+    const m = usage.byModel[model] ?? {
+      turns: 0,
+      inputTokens: 0,
+      outputTokens: 0,
+      cacheReadTokens: 0,
+      cacheWriteTokens: 0,
+    };
+    m.turns += 1;
+    m.inputTokens += num(attrs.inputTokens);
+    m.outputTokens += num(attrs.outputTokens);
+    m.cacheReadTokens += num(attrs.cacheReadTokens);
+    m.cacheWriteTokens += num(attrs.cacheWriteTokens);
+    usage.byModel[model] = m;
+    usage.turns += 1;
+  }
+  return usage;
+}
+
+export function addUsage(a: RunUsage, b: RunUsage): RunUsage {
+  const out: RunUsage = { turns: a.turns + b.turns, byModel: {} };
+  for (const src of [a.byModel, b.byModel]) {
+    for (const [model, m] of Object.entries(src)) {
+      const acc = out.byModel[model] ?? {
+        turns: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        cacheReadTokens: 0,
+        cacheWriteTokens: 0,
+      };
+      acc.turns += m.turns;
+      acc.inputTokens += m.inputTokens;
+      acc.outputTokens += m.outputTokens;
+      acc.cacheReadTokens += m.cacheReadTokens;
+      acc.cacheWriteTokens += m.cacheWriteTokens;
+      out.byModel[model] = acc;
+    }
+  }
+  return out;
+}
+
+const isModelUsage = (v: unknown): v is ModelUsage =>
+  typeof v === "object" &&
+  v !== null &&
+  (["turns", "inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTokens"] as const).every(
+    (k) => typeof (v as Record<string, unknown>)[k] === "number",
+  );
+
+export function isRunUsage(v: unknown): v is RunUsage {
+  if (typeof v !== "object" || v === null) return false;
+  const u = v as Record<string, unknown>;
+  if (typeof u.turns !== "number") return false;
+  if (typeof u.byModel !== "object" || u.byModel === null || Array.isArray(u.byModel)) return false;
+  return Object.values(u.byModel).every(isModelUsage);
+}
+
+// ---- the aggregate: who spent what, per UTC day ---------------------------------------
+
+/** One finished run as the aggregate sees it — the record's identity fields and its usage. */
+export interface UsageRun {
+  id: string;
+  userId: string;
+  userName?: string;
+  /** A child run is billed to whoever started its parent (run-history item 46). */
+  parentRunId?: string;
+  startedAt: number;
+  finishedAt: number;
+  /** Absent on a record written before usage existed and not yet backfilled. */
+  usage?: RunUsage;
+}
+
+export interface UserDayUsage {
+  userId: string;
+  userName?: string;
+  /** The UTC day the run finished, `YYYY-MM-DD`. */
+  day: string;
+  runs: number;
+  /** Summed wall-clock of the runs (finish − start), for allocating shared cloud spend. */
+  wallMs: number;
+  usage: RunUsage;
+}
+
+export interface RunUsageQuery {
+  /** Runs that finished at or after this epoch ms … */
+  sinceMs: number;
+  /** … and before this one. */
+  untilMs: number;
+}
+
+export interface RunUsageReport {
+  rows: UserDayUsage[];
+  /** Runs in range whose usage is not known yet (written before the field; backfill outstanding). */
+  pending: number;
+  /** The oldest finish the store still holds, so a page can bound its range to the data. */
+  earliestFinishedAt?: number;
+  retentionDays: number;
+}
+
+export const dayOf = (epochMs: number): string => new Date(epochMs).toISOString().slice(0, 10);
+
+/** Who a run is billed to: its parent's requester when it is a child and the
+ *  parent is known (in the batch, or through `lookupParent`), else its own. */
+export function billedTo(
+  run: UsageRun,
+  batch: ReadonlyMap<string, UsageRun>,
+  lookupParent: (id: string) => Pick<UsageRun, "userId" | "userName"> | undefined,
+): Pick<UsageRun, "userId" | "userName"> {
+  if (!run.parentRunId) return { userId: run.userId, ...(run.userName ? { userName: run.userName } : {}) };
+  const parent = batch.get(run.parentRunId) ?? lookupParent(run.parentRunId);
+  if (!parent) return { userId: run.userId, ...(run.userName ? { userName: run.userName } : {}) };
+  return { userId: parent.userId, ...(parent.userName ? { userName: parent.userName } : {}) };
+}
+
+/** Pure: the runs summed per (billed user, UTC day of finish). A run without
+ *  usage counts as pending and contributes its run and wall-clock only. Rows
+ *  come out oldest day first, then by user id. */
+export function aggregateUsageByUser(
+  runs: readonly UsageRun[],
+  lookupParent: (id: string) => Pick<UsageRun, "userId" | "userName"> | undefined = () => undefined,
+): { rows: UserDayUsage[]; pending: number } {
+  const batch = new Map(runs.map((r) => [r.id, r]));
+  const rows = new Map<string, UserDayUsage>();
+  let pending = 0;
+  for (const run of runs) {
+    const who = billedTo(run, batch, lookupParent);
+    const day = dayOf(run.finishedAt);
+    const key = `${day} ${who.userId}`;
+    const row = rows.get(key) ?? { userId: who.userId, day, runs: 0, wallMs: 0, usage: emptyUsage() };
+    if (who.userName && !row.userName) row.userName = who.userName;
+    row.runs += 1;
+    row.wallMs += Math.max(0, run.finishedAt - run.startedAt);
+    if (run.usage) row.usage = addUsage(row.usage, run.usage);
+    else pending += 1;
+    rows.set(key, row);
+  }
+  return {
+    rows: [...rows.values()].sort((a, b) => (a.day < b.day ? -1 : a.day > b.day ? 1 : a.userId < b.userId ? -1 : 1)),
+    pending,
+  };
+}
+
+export function isRunUsageReport(v: unknown): v is RunUsageReport {
+  if (typeof v !== "object" || v === null) return false;
+  const r = v as Record<string, unknown>;
+  if (!Array.isArray(r.rows) || typeof r.pending !== "number" || typeof r.retentionDays !== "number") return false;
+  if (r.earliestFinishedAt !== undefined && typeof r.earliestFinishedAt !== "number") return false;
+  return r.rows.every(
+    (row) =>
+      typeof row === "object" &&
+      row !== null &&
+      typeof (row as UserDayUsage).userId === "string" &&
+      typeof (row as UserDayUsage).day === "string" &&
+      typeof (row as UserDayUsage).runs === "number" &&
+      typeof (row as UserDayUsage).wallMs === "number" &&
+      isRunUsage((row as UserDayUsage).usage),
+  );
+}

package/dist/assets/src/core/ship/coordinator.ts

@@ -424,7 +424,9 @@ export interface UnitPipelineInput {
   /** Each child preset's own wall-clock budget (its `maxMinutes`), the number a
    *  round's budget is clipped from — supplied by the bot, which holds the registry. */
   childMinutes: Readonly<Record<ChildPreset, number>>;
-  /** Who merges: the runner (a plan branch, under its grant) or a person (any other branch). */
+  /** Who merges: the instance's `merge` field as the plan route answers it —
+   *  `runner` (a seeded plan, under its grant) or `person` (a task, or a
+   *  record without the field). */
   merge: "runner" | "person";
   /** Resume at review: an open pull request of ship's own the requester named. */
   resume?: { pr: number; headSha?: string; url?: string };
@@ -1102,7 +1104,7 @@ export function renderUnitReport(s: UnitPipelineState): string {
         `✅ Merge-ready after ${rounds}: ${e.pr.url}`,
         verdictLine,
         declinedLine,
-        "Remaining gate: a person's merge — the runner merges only a plan branch's pull request, and ship never approves.",
+        "Remaining gate: a person's merge — the runner merges only when the instance's `merge` field says runner, and ship never approves.",
       ].join("\n");
     case "merge_refused":
       return join([

package/dist/assets/src/execution/residentDepCache.ts

@@ -213,6 +213,10 @@ export interface DepCacheScriptParse {
   /** Raw `find` output for the hardlinked node_modules (the exact
    *  `mutableCacheFindArgv` shape), for `mutableCachePaths`. */
   mutableListing: string[];
+  /** The swap script's `skipped=` lines: tool-managed paths (relative to
+   *  node_modules) it left in place because the source has no counterpart —
+   *  tree-private entries, not shared inodes (see `mutableCacheSwapScript`). */
+  skipped: string[];
   failedStep: string | null;
 }
 
@@ -275,6 +279,7 @@ export function depCacheScript(
 export function parseDepCacheScriptOutput(stdout: string): DepCacheScriptParse {
   let mech: DepCacheMaterialization | "none" = "none";
   const mutableListing: string[] = [];
+  const skipped: string[] = [];
   let failedStep: string | null = null;
   for (const raw of stdout.split("\n")) {
     const line = raw.trim();
@@ -285,19 +290,34 @@ export function parseDepCacheScriptOutput(stdout: string): DepCacheScriptParse {
       }
     } else if ((m = /^mutable=(.+)$/.exec(line))) {
       mutableListing.push(m[1]);
+    } else if ((m = /^skipped=(.+)$/.exec(line))) {
+      skipped.push(m[1]);
     } else if ((m = /^err=(.+)$/.exec(line))) {
       failedStep ??= m[1];
     }
   }
-  return { mech, mutableListing, failedStep };
+  return { mech, mutableListing, skipped, failedStep };
 }
 
 /** The per-path swaps for a hardlinked node_modules' tool-managed entries
  *  (`mutableCachePaths` output), all in one fork: `rm -rf` the shared
- *  subtree, `cp -R` the warm checkout's matching subpath (fresh inodes),
- *  `chown -Rh` to the thread user (-h: a postinstall-planted symlink is
+ *  subtree, `cp -R` the source's matching subpath (fresh inodes), `chmod -R
+ *  u+w`, `chown -Rh` to the thread user (-h: a postinstall-planted symlink is
  *  re-owned as a LINK, never followed to an out-of-tree target). Same steps,
- *  same order, same flags as the old per-spawn loop. */
+ *  same order, same flags as the old per-spawn loop.
+ *
+ *  Each swap is gated on the counterpart existing in the source. The paths
+ *  come from a `find` over the TREE, and the tree's listing can name an
+ *  entry the source cannot stat — a top-level dot entry the store entry
+ *  lacks (one repo's tree listed `node_modules/.eports.d.ts`). Such an entry
+ *  is not a shared inode to swap: whatever is at that path is already
+ *  tree-private. Deleting it is a regression, and failing on it took the
+ *  whole refresh down — the `rm` had run, the `cp` died on the missing
+ *  source, and the resident degraded on every cycle after. It is left in
+ *  place and named on a `skipped=<path relative to node_modules>` line so
+ *  the step's output says so (`DepCacheScriptParse.skipped`). `-L` beside
+ *  `-e`: `-e` follows symlinks, and a dangling link in the source is still
+ *  an entry `cp -R` copies as a link. */
 export function mutableCacheSwapScript(
   srcRoot: string,
   dstRoot: string,
@@ -309,13 +329,19 @@ export function mutableCacheSwapScript(
   const lines: string[] = [];
   for (const p of paths) {
     const rel = p.slice(root.length);
-    lines.push(`rm -rf ${shellQuote(p)} || { echo err=deps-mutable-rm; exit 1; }`);
-    lines.push(`cp -R ${shellQuote(`${srcRoot}${rel}`)} ${shellQuote(p)} || { echo err=deps-mutable-copy; exit 1; }`);
+    const src = shellQuote(`${srcRoot}${rel}`);
+    const dst = shellQuote(p);
+    lines.push(`if [ -e ${src} ] || [ -L ${src} ]; then`);
+    lines.push(`  rm -rf ${dst} || { echo err=deps-mutable-rm; exit 1; }`);
+    lines.push(`  cp -R ${src} ${dst} || { echo err=deps-mutable-copy; exit 1; }`);
     // cp copies mode bits: a store entry's files are owner-read-only (item 59,
     // hardened so no consumer can write through the shared inodes), and a
     // cache the tree's own tools must rewrite in place has to be writable.
-    lines.push(`chmod -R u+w ${shellQuote(p)} || { echo err=deps-mutable-chmod; exit 1; }`);
-    lines.push(`chown -Rh ${owner} ${shellQuote(p)} || { echo err=deps-mutable-chown; exit 1; }`);
+    lines.push(`  chmod -R u+w ${dst} || { echo err=deps-mutable-chmod; exit 1; }`);
+    lines.push(`  chown -Rh ${owner} ${dst} || { echo err=deps-mutable-chown; exit 1; }`);
+    lines.push(`else`);
+    lines.push(`  echo ${shellQuote(`skipped=${rel.replace(/^\/+/, "")}`)}`);
+    lines.push(`fi`);
   }
   return lines.join("\n");
 }

package/dist/assets/src/execution/residentRebind.ts

@@ -65,6 +65,66 @@ export function parseRefByDefault(value: unknown): ParsedRefByDefault {
   return { error: "refByDefault must be a boolean when present" };
 }
 
+/** A branch a run pushed and the pull request it heads — what the run's
+ *  release hands the resident (`/detach` body `pushed`), read off the run's
+ *  own `pr_opened` events. */
+export interface PushedBranch {
+  ref: string;
+  pr: number;
+}
+
+/** The same fact as the binding remembers it, with when it was told. */
+export interface OwnBranch extends PushedBranch {
+  at: string;
+}
+
+/** The most branches one release may hand over: a run pushes a handful at most. */
+export const PUSHED_MAX = 20;
+/** The most branches a binding remembers: the newest are kept. */
+export const OWN_BRANCHES_MAX = 50;
+
+export type ParsedPushed = { pushed: PushedBranch[] } | { error: string };
+
+const PUSHED_ERROR = "pushed must be a list of {ref: <branch>, pr: <positive integer>} when present";
+
+/** `/detach` body field `pushed`: absent → nothing (the body every bot always
+ *  sent); a list of up to `PUSHED_MAX` well-formed `{ref, pr}` → itself;
+ *  anything else → a 400-shaped error. Each ref's PATTERN is the Worker's to
+ *  check (`parseRef`), like every ref. */
+export function parsePushed(value: unknown): ParsedPushed {
+  if (value === undefined) return { pushed: [] };
+  if (!Array.isArray(value) || value.length > PUSHED_MAX) return { error: PUSHED_ERROR };
+  const pushed: PushedBranch[] = [];
+  for (const entry of value) {
+    if (typeof entry !== "object" || entry === null) return { error: PUSHED_ERROR };
+    const { ref, pr } = entry as { ref?: unknown; pr?: unknown };
+    if (typeof ref !== "string" || ref.length === 0) return { error: PUSHED_ERROR };
+    if (typeof pr !== "number" || !Number.isSafeInteger(pr) || pr <= 0) return { error: PUSHED_ERROR };
+    pushed.push({ ref, pr });
+  }
+  return { pushed };
+}
+
+/** The binding's memory after a release: every branch handed over is
+ *  remembered once — a branch pushed again moves to the end with its current
+ *  pull request and time — and only the newest `OWN_BRANCHES_MAX` are kept.
+ *  Stored BEFORE any eviction decision, so the fact survives the tree. */
+export function rememberOwnBranches(
+  existing: readonly OwnBranch[] | undefined,
+  pushed: readonly PushedBranch[],
+  at: string,
+): OwnBranch[] {
+  const refs = new Set(pushed.map((p) => p.ref));
+  const kept = (existing ?? []).filter((b) => !refs.has(b.ref));
+  const added = pushed.map((p) => ({ ref: p.ref, pr: p.pr, at }));
+  return [...kept, ...added].slice(-OWN_BRANCHES_MAX);
+}
+
+/** Whether the thread's own runs pushed `ref`, as the binding remembers it. */
+export function isOwnBranch(binding: { ownBranches?: readonly OwnBranch[] }, ref: string): boolean {
+  return (binding.ownBranches ?? []).some((b) => b.ref === ref);
+}
+
 /** How a binding's ref was chosen: the repo default for want of a named
  *  branch, or a branch someone named. */
 export type BoundBy = "default" | "name";
@@ -109,6 +169,8 @@ export interface RebindableBinding {
   evicted?: boolean;
   boundBy?: BoundBy;
   rebound?: Rebound;
+  /** The branches the thread's own runs pushed, handed over at each release. */
+  ownBranches?: OwnBranch[];
 }
 
 export type RebindPlan =
@@ -117,8 +179,14 @@ export type RebindPlan =
   | { kind: "none" }
   /** The binding alone rules it out; nothing on disk is consulted. */
   | { kind: "refuse"; refused: RebindRefused }
-  /** The binding allows it; the tree decides (`rebindVerdict`). */
-  | { kind: "measure"; from: string; to: string; pr: number };
+  /** The binding allows it and has a live tree; the tree decides
+   *  (`rebindVerdict`). `own`: the thread's own runs pushed the branch, so a
+   *  tree that turns out missing may still be recreated at it. */
+  | { kind: "measure"; from: string; to: string; pr: number; own: boolean }
+  /** The binding allows it, its tree was evicted, and the thread's own runs
+   *  pushed the branch: the binding moves and the attach recreates the tree
+   *  at the branch — once the mirror is known to hold it. */
+  | { kind: "recreate"; from: string; to: string; pr: number };
 
 export function rebindRefused(plan: { to: string; pr: number }, reason: RebindRefusal, why: string): RebindRefused {
   return { to: plan.to, pr: plan.pr, reason, why };
@@ -157,17 +225,19 @@ export function rebindPlan(input: {
       ),
     };
   }
+  const own = isOwnBranch(binding, plan.to);
   if (binding.evicted || !binding.user) {
+    if (own) return { kind: "recreate", from: binding.ref, ...plan };
     return {
       kind: "refuse",
       refused: rebindRefused(
         plan,
         "branch-absent",
-        "the thread's worktree was evicted; the branch cannot be verified there",
+        `the thread's worktree was evicted and none of its runs pushed ${JSON.stringify(plan.to)}; only a branch this thread's own run pushed moves it`,
       ),
     };
   }
-  return { kind: "measure", from: binding.ref, ...plan };
+  return { kind: "measure", from: binding.ref, ...plan, own };
 }
 
 /** What the attach measured about the thread's tree, as the thread user. Each
@@ -185,17 +255,24 @@ export interface RebindTreeFacts {
   dirty?: boolean;
 }
 
-export type RebindVerdict = { kind: "rebind" } | { kind: "refuse"; refused: RebindRefused };
+export type RebindVerdict =
+  /** Check the branch out in the existing tree. */
+  | { kind: "rebind" }
+  /** The tree is gone (a slept container) and the thread's own runs pushed the
+   *  branch: move the binding and let the attach recreate the tree at it. */
+  | { kind: "recreate" }
+  | { kind: "refuse"; refused: RebindRefused };
 
 /** The tree's verdict on a measured plan. */
-export function rebindVerdict(plan: { to: string; pr: number }, tree: RebindTreeFacts): RebindVerdict {
+export function rebindVerdict(plan: { to: string; pr: number; own?: boolean }, tree: RebindTreeFacts): RebindVerdict {
   if (!tree.exists) {
+    if (plan.own === true) return { kind: "recreate" };
     return {
       kind: "refuse",
       refused: rebindRefused(
         plan,
         "branch-absent",
-        "the thread's worktree is missing; the branch cannot be verified there",
+        `the thread's worktree is missing and none of its runs pushed ${JSON.stringify(plan.to)}; the branch cannot be verified there`,
       ),
     };
   }

package/dist/assets/src/execution/sandboxErrors.ts

@@ -90,16 +90,6 @@ export function isFleetBusyError(err: unknown): boolean {
   );
 }
 
-/** The 0.12.x Durable Object's answer while its container is still booting:
- *  no session exists yet and nothing ran, so the SAME request can be re-sent
- *  after a short pause — the one retry the Worker still does itself. */
-export const CONTAINER_STARTING_PATTERN = /^Container is starting\. Please retry in a moment\.?$/i;
-
-export function isContainerStarting(err: unknown): boolean {
-  const { message } = thrownShape(err);
-  return !!message && CONTAINER_STARTING_PATTERN.test(message.trim());
-}
-
 /** The Worker's answer on /read and /write (sent as HTTP 503): the named
  *  reason plus an `error` that keeps the SDK's own message as the cause, so
  *  the logs and the model can still see what the platform actually said. */
@@ -160,35 +150,21 @@ export function thrownText(shape: ThrownShape): string {
 
 // ---- A runtime that did not answer (docs/reference/specs/execution.md item 9) ----
 //
-// The SDK proxies every command to the container's port through the
-// `@cloudflare/containers` base class. When nothing answers there — the SDK's
-// server exited (an uncaught exception ends it), or is being started again by
-// the image's PID 1 — that class answers HTTP 500 with a plain-text body, not
-// the JSON the SDK client expects, and the client renders it as the bare
-// `HTTP error! status: 500`: no container, no cause, nothing a card can act
-// on. The Worker reads the body first and names the condition instead.
+// Every command and file operation reaches the container through the SDK's
+// control connection to the container's port. When nothing answers there —
+// the SDK's server exited, or the image's PID 1 is starting it again — the
+// SDK's connect aborts after 30 s with a bare `The operation was aborted`:
+// no container, no cause, nothing a card can act on. The Durable Object
+// names the condition instead, with the facts a reader needs.
 
 /** The named reason the Worker answers with, like `fleet-busy`: a machine
- *  token the executor and the harness match on, never the SDK's text. */
+ *  token the executor matches on, never the SDK's text. */
 export const RUNTIME_UNREACHABLE_REASON = "runtime-unreachable" as const;
 
 /** The name the Worker's typed error carries across the Durable Object RPC
  *  boundary (which keeps `name`/`message` and drops the prototype). */
 export const RUNTIME_UNREACHABLE_ERROR_NAME = "SandboxRuntimeUnreachableError";
 
-/** The base class's two plain-text 500 bodies for a failed proxy to the
- *  container's port: the fetch itself failed (nothing listening — the server
- *  exited, or has not come back yet), and a connection lost mid-request. A
- *  JSON 500 from the server itself is neither: that server was alive. */
-const PROXY_FAILURE_BODIES: readonly RegExp[] = [
-  /^Error proxying request to container\b/,
-  /^Container suddenly disconnected\b/,
-];
-
-export function isRuntimeProxyFailure(status: number, body: string): boolean {
-  return status === 500 && PROXY_FAILURE_BODIES.some((re) => re.test(body.trim()));
-}
-
 export interface RuntimeUnreachableFacts {
   /** The Durable Object's id — the same word the `containers` log dataset carries as the container id. */
   containerId: string;
@@ -196,7 +172,7 @@ export interface RuntimeUnreachableFacts {
   running: boolean | undefined;
   /** The Worker's `@cloudflare/sandbox` pin, which is also the image tag. */
   sdkVersion: string;
-  /** The base class's body, verbatim: what the platform said. */
+  /** The SDK's own text, verbatim: what it saw. */
   cause: string;
 }
 
@@ -209,13 +185,13 @@ export function runtimeUnreachableMessage(f: RuntimeUnreachableFacts): string {
   return (
     `${RUNTIME_UNREACHABLE_REASON}: the sandbox container's runtime did not answer ` +
     `(container ${f.containerId}, sandbox SDK ${f.sdkVersion}; the platform reports the container ${platform}) — ` +
-    "its server exited and the image's PID 1 starts it again within seconds; this command may not have run and " +
-    `/workspace is intact: wait a moment, check, then retry (${f.cause.trim() || "no detail from the platform"})`
+    "nothing ran; if the container still runs, /workspace is intact and its runtime is being started again: " +
+    `wait a moment, then retry (${f.cause.trim() || "no detail from the SDK"})`
   );
 }
 
-/** The Worker's typed error for a failed proxy: thrown from `containerFetch`
- *  inside the Durable Object, caught by the Worker's routes on the other side
+/** The Worker's typed error for a silent control port: built inside the
+ *  Durable Object with its facts, read by the fetch handler on the other side
  *  of the RPC boundary, so it is matched by name and by its message token,
  *  never by `instanceof`. */
 export class SandboxRuntimeUnreachableError extends Error {
@@ -245,8 +221,8 @@ export function runtimeUnreachableAnswer(message: string): {
 
 /** The `/exec` answer, in-body under the streamed HTTP 200 like every other exec
  *  failure: the dual `error` + exit-127/stderr shape (item 3) plus the reason.
- *  Never re-sent by anyone: a command in flight when the server died may have
- *  run; the text tells the model to check before it retries. */
+ *  Never re-sent by anyone: the executor re-sends nothing in-body, and the
+ *  text tells the model to wait, then retry. */
 export function runtimeUnreachableExecAnswer(message: string): {
   error: string;
   reason: typeof RUNTIME_UNREACHABLE_REASON;

package/dist/assets/src/execution/sandboxLifecycle.ts

@@ -0,0 +1,78 @@
+// The per-thread sandbox's lifecycle facts the Worker and the bot share
+// (docs/reference/specs/execution.md items 1, 2 and 9): how long an idle
+// container stays warm, and how a container replaced or restarted under a
+// command is recognized and named. Deliberately free of node: imports so
+// wrangler can bundle it into the sandbox Worker.
+
+/** How long an IDLE container stays warm before the Durable Object stops it,
+ *  in the Container class's own `<n>[smh]` grammar. Idle means idle: the SDK
+ *  renews the activity timeout every second while a command's stream is open
+ *  (its control connection's busy poll), so a running command never counts
+ *  toward it. 5 minutes frees a finished thread's slot (`max_instances`)
+ *  sooner than the SDK default (10 min), while a follow-up inside 5 minutes
+ *  still lands on the same warm workspace; a later one re-clones, which is the
+ *  documented per-thread degradation (item 1). */
+export const SANDBOX_SLEEP_AFTER = "5m";
+
+/** The texts the SDK produces when the container is torn down under a
+ *  command, across generations. 0.3.x: the exec handler's generic wrapper (its
+ *  real cause, "Session terminated", sat in a field the client discarded), the
+ *  cause itself, and the stale-session answer the same attempt got once the
+ *  sessions were cleared. 0.12.x: the typed `SessionTerminatedError` text
+ *  (`Session '<id>' shell exited (exit code: <n>)`) and the
+ *  `OperationInterruptedError` text for a container that stopped under a
+ *  pending call, and the disconnect text for a sandbox `destroy()`ed under a
+ *  pending call. Anything else — a transport error, a file-op failure — is
+ *  never recycle-shaped, whenever it arrives. */
+const RECYCLE_SHAPED: readonly RegExp[] = [
+  /^Command execution failed$/,
+  /^Session terminated$/i,
+  /^Session '[^']*' not found$/i,
+  /^Session '[^']*' shell exited \(exit code: -?\d+\)$/i,
+  /^The sandbox container stopped while the operation was pending\.?$/i,
+  /^The sandbox was destroyed while the operation was pending\.?$/i,
+];
+
+/** The typed errors that MEAN the container's runtime went away under the
+ *  call, across the 0.12 and 0.13 lines. Matched by name, not `instanceof`:
+ *  the fetch handler sees them after the Durable Object RPC boundary, which
+ *  keeps `name`/`message` and drops the prototype (inside the Durable Object,
+ *  `instanceof` holds and decides first). */
+export const RECYCLE_ERROR_NAMES: readonly string[] = [
+  "SessionTerminatedError",
+  "OperationInterruptedError",
+  "StaleProcessHandleError",
+  "RuntimeIdentityInactiveError",
+];
+
+/** Type first, text second: a typed recycle error, or a recycle-shaped text. */
+export function isRecycleError(err: { name?: string; message?: string }): boolean {
+  if (err.name && RECYCLE_ERROR_NAMES.includes(err.name)) return true;
+  return !!err.message && RECYCLE_SHAPED.some((re) => re.test(err.message!.trim()));
+}
+
+/** Grace inside which a recycle-shaped TEXT is taken at face value: a session
+ *  that fails to start does so in seconds, not minutes. A typed recycle error
+ *  needs no grace — the SDK is stating the runtime went away. */
+const RECYCLE_SUSPECT_AFTER_MS = 60_000;
+
+/** The message `/exec` puts in-body when a command's failure looks like the
+ *  container's runtime went away under it: a typed recycle error (`certain`),
+ *  or a recycle-shaped text that arrived more than a minute into THIS attempt
+ *  (a startup failure shows in seconds). Any other text, however late, is
+ *  returned unchanged — timing alone never rewords an unrelated error. The
+ *  exit code stays 127: it IS an infra failure and the command did not finish,
+ *  so a faked exit 124 would tell the model to shorten a command that was
+ *  never the problem. What the workspace holds afterwards depends on what
+ *  went away: a replaced container has an empty `/workspace` (re-clone); a
+ *  runtime the image's PID 1 started again (item 21) left it intact. The text
+ *  says to look before assuming either. */
+export function recycledMidCommandMessage(elapsedMs: number, msg: string, certain = false): string {
+  const shaped = RECYCLE_SHAPED.some((re) => re.test(msg.trim()));
+  if (!certain && (!shaped || elapsedMs <= RECYCLE_SUSPECT_AFTER_MS)) return msg;
+  const secs = Math.round(elapsedMs / 1_000);
+  return (
+    `sandbox recycled mid-command after ${secs}s — the container's runtime was replaced or restarted under the command, which did not finish; ` +
+    `check /workspace before continuing: it is empty if the container was replaced (re-clone), intact if only its runtime restarted (${msg})`
+  );
+}