@coreplane/switchboard 1.242.0 → 1.244.0

package/dist/assets/deploy/cloudflare-resident/worker.ts

@@ -69,6 +69,7 @@
 //      (docs/decisions/0009-residents-second-credential-domain.md).
 import {
   isDurableObjectCodeUpdateReset,
+  isPlatformTransientError,
   OperationInterruptedError,
   ProcessWaitTimeoutError,
   RPCTransportError,
@@ -179,6 +180,7 @@ import {
   isRuntimeUnreachableSignal,
   killStaleBuildProcessesCommand,
   planRefresh,
+  RUNTIME_MOVED_WORDING,
   RUNTIME_REPLACEMENT_WORDING,
   RUNTIME_UNREACHABLE_DOWN_AT,
   runtimeUnreachableReason,
@@ -195,6 +197,7 @@ import {
   type RefreshPlan,
   type RestoreSample,
   type RefreshFailure,
+  fetchFailureIsMirrors,
 } from "../../src/execution/residentRefresh.js";
 import { autoRebuildDecision, isAutoRebuildEligible } from "../../src/execution/residentAutoRebuild.js";
 import {
@@ -682,18 +685,60 @@ export function validateEnvNames(vars: Record<string, string>): void {
  *  decides (idempotent read/write retry; exec is handed to the model). */
 class RuntimeReplacedError extends Error {
   constructor(
-    readonly phase: "spawn" | "collect",
+    /** Where the replacement was met: the command's spawn, its collect, or
+     *  the Worker's call into the Durable Object itself (`threadRejectionErr`),
+     *  before the method could answer. */
+    readonly phase: "spawn" | "collect" | "call",
     readonly cause: unknown,
+    /** Whether the replacement was KNOWN where the failure was classified
+     *  (`replacementKnown`): the SDK vouched the runtime moved, or a restore is
+     *  under way for this resident. One decision, made once at the exec choke
+     *  point, and it gates the word on `/exec` alone (`replacedExecAnswer`) —
+     *  the incarnation swap is unconditional there. Unknown, the failure only
+     *  says the container is down or the transport was lost — a merely asleep
+     *  or starting container, or a network blip, say the same — so the answer
+     *  is the SDK's words and the harness's one more command decides. At the
+     *  Worker's call nothing vouched, so it is `false` there. */
+    readonly known: boolean,
   ) {
     super(
       `runtime-replaced: the resident runtime was replaced (a deploy) while this command was ${
-        phase === "spawn" ? "starting" : "running"
+        phase === "spawn" ? "starting" : phase === "collect" ? "running" : "pending at the Worker"
       }; its output is lost (${errMsg(cause)})`,
     );
     this.name = "RuntimeReplacedError";
   }
 }
 
+/** The resident's own Durable Object was reset while a command was in flight —
+ *  a `wrangler deploy` of the Worker code (not the container image) supersedes
+ *  this DO's isolate, so the SDK's control session to the container is lost
+ *  mid-command (`isDurableObjectCodeUpdateReset`). This is NOT a runtime
+ *  replacement: the container and every process the run holds in it are exactly
+ *  as they were — only the DO that was driving them reset. The command's
+ *  outcome is unknown (the reset may have raced its start or its finish), so
+ *  the resident never re-issues it here; the thread routes answer the NAMED
+ *  `control-reset` error and the client re-sends an idempotent op or resolves a
+ *  write by pi's echo (docs/reference/specs/harness-pi.md item 16). Distinct
+ *  from `RuntimeReplacedError` precisely so the harness never mistakes a DO
+ *  reset over a live pi for a replaced container and orphans that pi. */
+class ControlResetError extends Error {
+  constructor(
+    /** Where the reset was met: the command's spawn, its collect, or the
+     *  Worker's call into the Durable Object itself (`threadRejectionErr`),
+     *  before the method could answer. */
+    readonly phase: "spawn" | "collect" | "call",
+    readonly cause: unknown,
+  ) {
+    super(
+      `control-reset: the resident's Durable Object was reset (a deploy) while this command was ${
+        phase === "spawn" ? "starting" : phase === "collect" ? "running" : "pending at the Worker"
+      }; the container and its processes are as they were; the command's outcome is unknown (${errMsg(cause)})`,
+    );
+    this.name = "ControlResetError";
+  }
+}
+
 /** Every wording the pinned SDK (@cloudflare/sandbox@0.13.0-next.751.1) uses
  *  when the runtime incarnation changed under a call, for the message-based
  *  fallback below. Two of these come from classes the SDK does NOT export
@@ -752,27 +797,194 @@ const RPC_TRANSPORT_LOSS_KINDS = new Set(["peer_closed", "connection_failed", "u
 /** Does this SDK error mean the runtime incarnation changed under us? Typed
  *  checks first (`StaleProcessHandleError`, `RuntimeIdentityInactiveError`, an
  *  `OperationInterruptedError` with one of `RUNTIME_REPLACED_REASONS`, an
- *  `RPCTransportError` with one of `RPC_TRANSPORT_LOSS_KINDS`, the platform's
- *  superseded-isolate reset), then the SDK's message wording — on the error AND
- *  its cause chain — as a belt-and-braces fallback. Anything else (a real spawn
- *  failure, a timeout, a wire-format error) is NOT a runtime replacement. */
+ *  `RPCTransportError` with one of `RPC_TRANSPORT_LOSS_KINDS`), then the SDK's
+ *  message wording — on the error AND its cause chain — as a belt-and-braces
+ *  fallback. Anything else (a real spawn failure, a timeout, a wire-format
+ *  error) is NOT a runtime replacement.
+ *
+ *  Deliberately IMMUNE to the platform's superseded-isolate reset (a DO
+ *  code-update reset): that is this Durable Object resetting over a container it
+ *  did not touch, the container unchanged, so it is never a runtime replacement
+ *  and never swaps the incarnation on the `/exec` path. `run()` answers it as a
+ *  `control-reset` (`isControlReset`), and the restore path folds it in
+ *  explicitly at its own site (`isRuntimeReplacement(err) || isControlReset(err)`)
+ *  — the ONE place a DO reset shares the replacement disposition, since a reset
+ *  under a live restore leaves nothing to land and re-restores onto the container
+ *  that comes back. Keeping it out of this predicate makes that use the only one. */
 function isRuntimeReplacement(err: unknown): boolean {
   if (err instanceof StaleProcessHandleError) return true;
   if (err instanceof RuntimeIdentityInactiveError) return true;
   if (err instanceof OperationInterruptedError && RUNTIME_REPLACED_REASONS.has(err.reason)) return true;
   if (err instanceof RPCTransportError && RPC_TRANSPORT_LOSS_KINDS.has(err.kind)) return true;
-  if (isDurableObjectCodeUpdateReset(err)) return true;
   for (const link of selfAndCauses(err)) if (RUNTIME_REPLACEMENT_WORDING.test(errMsg(link))) return true;
   return false;
 }
 
+/** Whether the SDK's failure vouches that the runtime under the command MOVED
+ *  — a new incarnation serves the thread, so the container the command was
+ *  aimed at is gone whatever else the resident knows: the typed classes for
+ *  a stale process handle and an inactive runtime identity, an interruption
+ *  whose reason is a replaced runtime, and the SDK's wordings for the same
+ *  (`RUNTIME_MOVED_WORDING`, on the error and its cause chain). NOT the words
+ *  for a container that is merely down (`STOPPED_CONTAINER_WORDING`: "The
+ *  container is not running", "Process supervisor is closed") and NOT a
+ *  transport lost under the call (`RPCTransportError`): an asleep or starting
+ *  container and a network blip answer those too, so on `/exec` they are the
+ *  word only when the resident itself knows the container it held is gone
+ *  (`replacedExecAnswer`). NOT the platform's superseded-isolate reset either:
+ *  that is this Durable Object resetting over a container it did not touch —
+ *  `run()` intercepts it first, in both phases, as a `control-reset`
+ *  (`isControlReset`), so a DO-reset cause never reaches this predicate, and a
+ *  vouch for it here would say the opposite of that word. The restore path
+ *  keeps the union by naming both explicitly at its site
+ *  (`isRuntimeReplacement(err) || isControlReset(err)`), so a step interrupted
+ *  either way is retried onto the container that comes back — while this
+ *  predicate and `isRuntimeReplacement` stay immune to the reset. */
+function sdkVouchesRuntimeMoved(err: unknown): boolean {
+  if (err instanceof StaleProcessHandleError) return true;
+  if (err instanceof RuntimeIdentityInactiveError) return true;
+  if (err instanceof OperationInterruptedError && RUNTIME_REPLACED_REASONS.has(err.reason)) return true;
+  for (const link of selfAndCauses(err)) if (RUNTIME_MOVED_WORDING.test(errMsg(link))) return true;
+  return false;
+}
+
+/** The platform's transient sentences the pinned SDK's own predicate does not
+ *  name — `isPlatformTransientError` covers the code-update reset, a lost
+ *  connection, the storage-startup reset and the typed `retryable` flag, so
+ *  those are read from the SDK, never from a string table of ours. Two remain:
+ *  the Durable Object reset by a storage operation that did not complete (the
+ *  platform's `Durable Object storage operation exceeded timeout which caused
+ *  object to be reset.` — an assumption about the platform's text, carried
+ *  from the first typing of the catch-all, not in the SDK's source), and the
+ *  Durable Object overloaded. The SDK's retry predicate EXCLUDES the overloaded
+ *  sentence (`isErrorRetryable`: an in-process retry only adds to a queue that
+ *  is full); it is typed transient here because the client's
+ *  `worker-unavailable` is a re-probe after the harness's bounded wait
+ *  (`replacedVerdict`, `PROBE_WAIT_MAX_MS`), never a tight retry, and a full
+ *  queue drains. The platform's sentences alone — a bare `internal error` or
+ *  `overloaded` is a word a git or GitHub failure carries too, and a throw
+ *  typed transient by it would be re-probed for a restore window that never
+ *  clears it. */
+const TRANSIENT_PLATFORM_WORDING = /storage operation|durable object is overloaded/i;
+
+/** The platform's own transient that no route word names — what is left once
+ *  `threadRejectionErr` has answered a reset and a replacement by their words:
+ *  the container's runtime unreachable (`isRuntimeUnreachable`: the SDK's
+ *  connect abort by its `AbortError` name or its exact sentence —
+ *  `RUNTIME_UNREACHABLE_WORDING` is anchored to the whole message, so a
+ *  `TimeoutError`'s `The operation was aborted due to timeout`, the text a
+ *  route's own `AbortSignal.timeout` on a slow GitHub raises, never matches),
+ *  the pinned SDK's own platform-transient predicate (`isPlatformTransientError`:
+ *  the code-update reset, a lost connection, the storage-startup reset, the
+ *  typed `retryable` flag — the SDK's signal, read as it reads it), and the
+ *  remainder sentences above, anywhere in the cause chain. */
+function isUnnamedPlatformTransient(err: unknown): boolean {
+  if (isRuntimeUnreachable(err) || isPlatformTransientError(err)) return true;
+  for (const link of selfAndCauses(err)) {
+    if (link instanceof Error && TRANSIENT_PLATFORM_WORDING.test(link.message)) return true;
+  }
+  return false;
+}
+
+/** Whether a throw no route named is the platform's own transient — the
+ *  Durable Object reset by a deploy (`isControlReset`), the runtime replaced
+ *  under the call (`isRuntimeReplacement`), or the transient no word names
+ *  (`isUnnamedPlatformTransient`) — against a deterministic throw in the route
+ *  itself (a bug, a bad argument). The client reads the answer's `transient`
+ *  field to re-probe the first and judge the second at once (execution.md item
+ *  9), never the words. */
+function isTransientPlatformThrow(err: unknown): boolean {
+  return isControlReset(err) || isRuntimeReplacement(err) || isUnnamedPlatformTransient(err);
+}
+
+/** The 500 for a throw no route named, at whichever catch met it — the fetch
+ *  handler's catch-all, a streamed route's rejection mapper, a route's own
+ *  catch around its body (`prefix`: `attach-failed`, `op-failed`): the words,
+ *  and whether the throw was the platform's transient (`transient`), typed
+ *  here so the client decides by a field and never by which catch met the
+ *  throw. A failure a route DID name — a step that failed, a mirror held — is
+ *  that route's own answer and never comes here. */
+function catchAllErr(err: unknown, prefix?: string): ThreadErr {
+  return unnamedThrowErr(err, isTransientPlatformThrow(err), prefix);
+}
+
+/** The one shape of that 500: the words, behind `prefix` where the route names
+ *  itself, and the verdict `transient`, decided once by the caller — so a
+ *  caller that has already settled the typed predicates (`threadRejectionErr`)
+ *  hands its verdict in instead of walking the cause chain again. */
+function unnamedThrowErr(err: unknown, transient: boolean, prefix?: string): ThreadErr {
+  const words = errMsg(err);
+  return { error: prefix ? `${prefix}: ${words}` : words, status: 500, transient };
+}
+
+type ThreadDataRoute = "/exec" | "/read" | "/write";
+
+/** A thread data-plane route's pending Durable Object call that REJECTED —
+ *  the stub, not the method: the DO reset by a code update before or while the
+ *  method ran, a storage operation that did not complete, the runtime
+ *  unreachable — answered as the DO answers the same fact when it catches it
+ *  inside (`execThreadImpl`, the file methods). A control reset is the DO's
+ *  own word, `control-reset` on a 409: the container and its processes are as
+ *  they were and the command's outcome is unknown, so the client resolves it by
+ *  its own rule and never waits on it as the resident unavailable. A runtime
+ *  replacement is answered by route, as the methods answer it, with the DO's
+ *  own gate (`replacementKnown`) applied as far as it reaches: its first half,
+ *  the SDK vouching the runtime moved (`sdkVouchesRuntimeMoved`), judges the
+ *  moved sentences that survive the stub boundary as text, so it is made here
+ *  by the same rule; its other half, a restore under way (`knowsContainerGone`),
+ *  is the DO's alone and out of reach. So on `/exec` the word is said where
+ *  the SDK vouched and WITHHELD where only the DO could have — that gate's
+ *  unknown branch, the SDK's words on a 409 with no word, for the harness's one
+ *  more command to judge; on `/read` and `/write` the word is said either way
+ *  (`runtimeReplacedErr`, unconditional there on purpose), since it drives the
+ *  client's one re-attach-and-retry and never a verdict. Anything else is the
+ *  typed 500, its `transient` decided here once the typed predicates are
+ *  settled. */
+function threadRejectionErr(err: unknown, route: ThreadDataRoute): ThreadErr {
+  if (isControlReset(err)) return controlResetErr(new ControlResetError("call", err));
+  if (isRuntimeReplacement(err)) {
+    const known = sdkVouchesRuntimeMoved(err);
+    if (route === "/exec" && !known) return { error: errMsg(err), status: 409 };
+    return runtimeReplacedErr(new RuntimeReplacedError("call", err, known));
+  }
+  return unnamedThrowErr(err, isUnnamedPlatformTransient(err));
+}
+
+/** Whether a command's failure is the resident's own Durable Object resetting
+ *  under it (a Worker-code deploy) rather than the container runtime being
+ *  replaced. Checked BEFORE `isRuntimeReplacement` on the thread `run()` path,
+ *  so a DO reset over a still-running container answers `control-reset`, never
+ *  the replaced word. The SDK's predicate already walks the cause
+ *  chain (`selfAndCauses`), so it is called directly on the caught error. */
+function isControlReset(err: unknown): boolean {
+  return isDurableObjectCodeUpdateReset(err);
+}
+
 /** The named ThreadErr every thread route (exec/read/write) answers for a
  *  runtime replacement, so the client can classify it (409 like the other
- *  recoverable thread states; `reason` is the discriminator). */
+ *  recoverable thread states; `reason` is the discriminator). On `/exec` the
+ *  answer goes through `replacedExecAnswer` first: the word only when the
+ *  replacement was known where `run()` classified the failure. On
+ *  `/read` and `/write` the word stays unconditional on purpose: those routes
+ *  reach the same `run()` and can meet the same down wordings, but there the
+ *  word drives the client's re-attach-and-retry (the attach is what wakes a
+ *  container that is asleep or starting), never a verdict — nothing relaunches
+ *  on it, and the harness never reads or writes through them (its seam runs
+ *  every operation as an `/exec` script). Gating them would trade a spare
+ *  re-attach for a read that fails outright while the container starts. */
 function runtimeReplacedErr(err: RuntimeReplacedError): ThreadErr {
   return { error: err.message, status: 409, reason: "runtime-replaced" };
 }
 
+/** The named ThreadErr a DO code-update reset answers with — its own `reason`
+ *  the client keys on, distinct from `runtime-replaced`: the container is
+ *  unchanged and the command's outcome is unknown, so the client re-sends an
+ *  idempotent op or resolves a write by echo (harness-pi item 16), never the
+ *  replaced verdict. */
+function controlResetErr(err: ControlResetError): ThreadErr {
+  return { error: err.message, status: 409, reason: "control-reset" };
+}
+
 /** The container's control port never answered: `exec` rejected with the
  *  DOMException of the SDK's connect abort (`DEFAULT_CONNECT_TIMEOUT_MS`,
  *  30 s), raised inside its wake path — `RuntimeBootstrapProbe.probe` →
@@ -1102,6 +1314,20 @@ interface ThreadBinding {
 interface ThreadErr {
   error: string;
   status: number;
+  /** Beside `state` on a 503 that carries the lifecycle state: the lifecycle
+   *  REASON (`getStatus().reason`), kept apart from `reason`, which is the
+   *  answer's own word (`mirror-busy`, `disk-pressure`, `image-stale`). The
+   *  client reads the pair to decide whether the resident is coming back
+   *  (execution.md item 9); reading `reason` there would take a busy mirror on
+   *  a degraded-but-serviceable resident for a repo failure. */
+  stateReason?: string;
+  /** On the 500 for a throw no route named (`unnamedThrowErr`, through
+   *  `catchAllErr` or `threadRejectionErr`: the fetch handler's catch-all, a
+   *  streamed route's rejection, a route's own catch around its body): whether
+   *  the throw it wrapped was the platform's own transient
+   *  (`isTransientPlatformThrow`) — a re-probe clears it — or a deterministic
+   *  throw in the route, judged at once. */
+  transient?: boolean;
   /** The steps the request ran before it failed (docs/reference/specs/tracing.md item 19):
    *  a failed attach's trace is the one that says which step blew the budget. */
   trace?: ResidentStep[];
@@ -1592,6 +1818,52 @@ export class ResidentDO extends Sandbox<Env> {
    *  survives, so hydration state is always probed from disk. */
   private hydration: Promise<void> | null = null;
 
+  /** Whether this resident knows the container it held is gone
+   *  (docs/reference/specs/resident-repos.md item 43): a restore under way for
+   *  this resident (`restoring`, persisted before any restore work) — its own
+   *  knowledge, reliable — and nothing else.
+   *
+   *  Not the Container base's exit state, not a replayed `onStop`, not the
+   *  platform's `running` flag: each was tried or weighed and separates
+   *  nothing — a roll and an idle sleep leave the same traces, in both
+   *  directions — and the why, with the pinned library's lines, is the
+   *  record's alone (docs/reference/specs/resident-repos.md item 43), so a
+   *  dependency bump rots one place, not three. The harness's own probe
+   *  (`identity`, `alive`) stays the arbiter, so under-saying the word here is
+   *  safe and over-saying it is what orphans a process.
+   *
+   *  A read that fails (the storage under a reset or a restore) is no
+   *  knowledge: the answer falls back to the SDK's words and the harness
+   *  probes, never an unhandled throw where a 409 was due. */
+  private async knowsContainerGone(): Promise<boolean> {
+    try {
+      return (await this.getStatus()).state === "restoring";
+    } catch {
+      return false;
+    }
+  }
+
+  /** The one decision on a failure the SDK classified as a runtime
+   *  replacement, made where it is classified (`run()`): the SDK vouched the
+   *  runtime moved, or a restore is under way for this resident. It gates the
+   *  word on `/exec` and the incarnation mint alike; the per-incarnation memos
+   *  clear before it, unconditionally. */
+  private async replacementKnown(err: unknown): Promise<boolean> {
+    return sdkVouchesRuntimeMoved(err) || (await this.knowsContainerGone());
+  }
+
+  /** The `/exec` answer for a command the SDK failed with a runtime
+   *  replacement (item 43; harness.md item 6): the word — `reason:
+   *  "runtime-replaced"`, the resident's sentence — when the replacement was
+   *  known at the choke point (`RuntimeReplacedError.known`); otherwise what
+   *  the SDK said, with no word and no `reason`, so the harness's one more
+   *  command decides. The command is never re-issued either way: it may have
+   *  started. */
+  private replacedExecAnswer(err: RuntimeReplacedError): ThreadErr {
+    if (err.known) return runtimeReplacedErr(err);
+    return { error: errMsg(err.cause), status: 409 };
+  }
+
   /** The step trace of the request in flight (docs/reference/specs/tracing.md item 19):
    *  `attachThread` and `runOp` each run inside their own collector, so the
    *  commands `runOk` runs and the mirror-lock waits land on the answer that
@@ -1952,6 +2224,12 @@ export class ResidentDO extends Sandbox<Env> {
     try {
       proc = await createExtensionProcessSandbox(this).exec(argv as unknown as SandboxCommand, launch);
     } catch (err) {
+      // A DO code-update reset (our own Worker deploy) BEFORE a runtime
+      // replacement: the container is unchanged, so this is `control-reset`, not
+      // the replaced word — never `swapIncarnation`, and the command's outcome
+      // is unknown. Checked first so a DO reset over a live pi is never
+      // read as a replaced container.
+      if (isControlReset(err)) throw new ControlResetError("spawn", err);
       if (!isRuntimeReplacement(err)) {
         // The control port never answered the SDK's connect (its 30 s abort,
         // raised inside the wake path): no process started and nothing about
@@ -1969,8 +2247,25 @@ export class ResidentDO extends Sandbox<Env> {
       // takes the throw below. It exists so that if a future SDK vouches "never
       // started" we retry then — and only then — without a change here.
       if (!(err instanceof OperationInterruptedError && err.retryable === true)) {
-        this.swapIncarnation(); // the container this incarnation's memos described is gone
-        throw new RuntimeReplacedError("spawn", err);
+        // Two consequences with different safety, split here. The memos
+        // (`hydratedVerdictAt`, `gitSetupDone`, `stageDirsReady`, the deps
+        // flags — the fact that was stale) clear on EVERY replacement the SDK
+        // classified: cheap to re-derive, and the per-incarnation memo block's
+        // invariant is that this one choke point clears them, whatever the
+        // word will say. The incarnation — the lease fencing token, one
+        // isolate paired with one container runtime — is minted only when the
+        // replacement is KNOWN: `isRuntimeReplacement` is the union, and a
+        // transport blip on one request that minted an id would make
+        // `takeMutex` read the live holder's row as `holder-incarnation-gone`
+        // and hand the mirror mutex to a concurrent `/attach` over a running
+        // fetch. (A swap gated whole on `known` was tried and left the memos
+        // stale on a real roll, where `known` is false while the wake rewrites
+        // the base's state — resident-repos item 43.) Only the word is gated,
+        // on the same `known`.
+        this.clearIncarnationMemos();
+        const known = await this.replacementKnown(err);
+        if (known) this.swapIncarnation();
+        throw new RuntimeReplacedError("spawn", err, known);
       }
       console.log(
         `exec: runtime replaced before the process started (SDK says retryable) — retrying once: ${errMsg(err)}`,
@@ -1992,9 +2287,18 @@ export class ResidentDO extends Sandbox<Env> {
         truncated: out.truncated,
       };
     } catch (err) {
+      // A DO code-update reset during collect: the container and the process are
+      // unchanged (only this DO's isolate reset), so the outcome is unknown but
+      // never a replacement — `control-reset`, no `swapIncarnation`.
+      // Checked before the replacement branch below.
+      if (isControlReset(err)) throw new ControlResetError("collect", err);
       if (isRuntimeReplacement(err)) {
-        this.swapIncarnation(); // the container this incarnation's memos described is gone
-        throw new RuntimeReplacedError("collect", err);
+        // The same split as the spawn site above: the memos unconditionally,
+        // the incarnation only when the replacement is known, the word gated.
+        this.clearIncarnationMemos();
+        const known = await this.replacementKnown(err);
+        if (known) this.swapIncarnation();
+        throw new RuntimeReplacedError("collect", err, known);
       }
       if (err instanceof ProcessWaitTimeoutError) {
         // The supervisor should have killed the process at `timeout`; 30 s
@@ -2574,7 +2878,13 @@ export class ResidentDO extends Sandbox<Env> {
       // asks the runtime whether the container the extract wrote to is still
       // there (the incident that named this: a resident went `down` on exactly
       // this, 10 ms before every exec answered "container is not running").
-      const runtimeReplaced = isRuntimeReplacement(err);
+      // A DO code-update reset under a live restore leaves nothing to land (the
+      // restore's SDK call is gone with this isolate), so it shares the
+      // replacement disposition — interrupted, retried onto the container that
+      // comes back. This is the ONLY site that folds the DO reset in;
+      // `isRuntimeReplacement` itself stays immune to it, so `/exec` never
+      // answers the replaced word for a reset over a live container.
+      const runtimeReplaced = isRuntimeReplacement(err) || isControlReset(err);
       const runtimeActive = runtimeReplaced ? false : await this.isRuntimeActive().catch(() => null);
       const disposition = restoreFailureDisposition(errMsg(err), {
         runtimeReplaced,
@@ -3227,6 +3537,14 @@ export class ResidentDO extends Sandbox<Env> {
         await this.recoverFromDiskFull(failure.reason, selfInFlight);
         return { ok: false, reason: failure.reason };
       }
+      // Item 67: a fetch that failed on the MIRROR (its remote gone, its
+      // object store broken) is the resident's own step failing — `fetch-failed`,
+      // counted by the ladder, never serviceable — not GitHub being unreachable.
+      // The first live row found a broken mirror parked as `github-unreachable`.
+      if (fetchFailureIsMirrors(message)) {
+        await this.refreshFailed(failure, selfInFlight);
+        return { ok: false, reason: failure.reason };
+      }
       const reason = `github-unreachable: ${message}`;
       await this.setResidentState("degraded", reason);
       return { ok: false, reason };
@@ -4353,7 +4671,7 @@ export class ResidentDO extends Sandbox<Env> {
       const reason = diskPressureReason({ verdict: final, evicted, kept, spares });
       console.log(`attach ${input.threadKey}: ${reason}`);
       const s = await this.getStatus();
-      return { error: reason, status: 503, state: s.state, reason: DISK_PRESSURE_REASON };
+      return { error: reason, status: 503, state: s.state, stateReason: s.reason, reason: DISK_PRESSURE_REASON };
     }
     const m = final.math;
     console.log(
@@ -4753,6 +5071,7 @@ export class ResidentDO extends Sandbox<Env> {
           error: "image-stale: the container predates the current pool and is restarting; retry shortly",
           status: 503,
           state: "restoring",
+          stateReason: "",
           reason: "image-stale",
         };
       }
@@ -4780,7 +5099,7 @@ export class ResidentDO extends Sandbox<Env> {
         this.attachesInFlight--;
       }
     } catch (err) {
-      return { error: `attach-failed: ${errMsg(err)}`, status: 500 };
+      return catchAllErr(err, "attach-failed");
     }
   }
 
@@ -4792,7 +5111,7 @@ export class ResidentDO extends Sandbox<Env> {
    *  instance's entry gate (the cron's, within one bucket) — an attach never
    *  stops the container itself: it is in flight. */
   private async attachFailed(err: unknown): Promise<ThreadErr> {
-    if (!(err instanceof StepError)) return { error: `attach-failed: ${errMsg(err)}`, status: 500 };
+    if (!(err instanceof StepError)) return catchAllErr(err, "attach-failed");
     const failure = await this.classifyFailure(err.step, err.message);
     if (!failure.diskFull) return { error: `attach-failed at ${err.step}: ${err.message}`, status: 500 };
     console.log(`attach: ${failure.reason}`);
@@ -4816,7 +5135,13 @@ export class ResidentDO extends Sandbox<Env> {
       await this.refreshIfStale(resourceId);
     } catch (err) {
       const s = await this.getStatus();
-      return { error: `not-serviceable: ${errMsg(err)}`, status: 503, state: s.state, reason: s.reason };
+      return {
+        error: `not-serviceable: ${errMsg(err)}`,
+        status: 503,
+        state: s.state,
+        stateReason: s.reason,
+        reason: s.reason,
+      };
     }
     // `resourceId` was read by the caller a moment ago (item 15 of the audit:
     // this used to re-read the same key), the registry record rides in from
@@ -4828,7 +5153,10 @@ export class ResidentDO extends Sandbox<Env> {
     const stored = await this.ctx.storage.get<RepoFacts | ThreadBinding>([FACTS_KEY, threadBindingKey(threadKey)]);
     const facts = stored.get(FACTS_KEY) as RepoFacts | undefined;
     const record = recordFromRoute ?? (await this.registry().getRecord(resource));
-    if (!record || !facts) return { error: "not-serviceable: registry record or repo facts missing", status: 503 };
+    // Typed `reason` beside the words: the client reads the field — a refusal no
+    // wait clears, unlike the restore window's 503s — never the sentence.
+    if (!record || !facts)
+      return { error: "not-serviceable: registry record or repo facts missing", status: 503, reason: "unregistered" };
 
     // The binding's ref wins for the thread's whole life, with one exception
     // (item 16): a thread bound to the repo default for want of a named branch
@@ -4978,9 +5306,9 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, reason: "mirror-busy" };
+        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
       }
-      return { error: `attach-failed: ${errMsg(err)}`, status: 500 };
+      return catchAllErr(err, "attach-failed");
     }
     if (outcome.kind === "none") return { binding: outcome.binding };
     if (outcome.kind === "refuse") {
@@ -5280,7 +5608,7 @@ export class ResidentDO extends Sandbox<Env> {
         return { error: `reuse-refused: ${err.why}`, status: 409, needs: "recreate" };
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, reason: "mirror-busy" };
+        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
       }
       if (err instanceof StepError && err.step === "unknown-ref") {
         return { error: `unknown-ref: ${err.message}`, status: 400 };
@@ -5325,7 +5653,7 @@ export class ResidentDO extends Sandbox<Env> {
       // worktree lock above, so the bot-side fallback can retry.
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, reason: "mirror-busy" };
+        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
       }
       return this.attachFailed(err);
     }
@@ -6163,7 +6491,13 @@ export class ResidentDO extends Sandbox<Env> {
       await this.ensureHydrated();
     } catch (err) {
       const s = await this.getStatus();
-      return { error: `not-serviceable: ${errMsg(err)}`, status: 503, state: s.state, reason: s.reason };
+      return {
+        error: `not-serviceable: ${errMsg(err)}`,
+        status: 503,
+        state: s.state,
+        stateReason: s.reason,
+        reason: s.reason,
+      };
     }
     const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
     if (!binding) {
@@ -6233,11 +6567,33 @@ export class ResidentDO extends Sandbox<Env> {
     return res;
   }
 
+  /** The exec route's one gate for a runtime replacement (item 43): every
+   *  `RuntimeReplacedError` the route meets — the preflight's own probe of the
+   *  worktree, the command itself — is judged by `replacedExecAnswer`, so the
+   *  word reaches the harness only when the SDK vouched the runtime moved or
+   *  the resident knows the container it held is gone. */
   private async execThreadImpl(
     threadKey: string,
     command: string,
     timeoutMs: number,
     env?: Record<string, string>,
+  ): Promise<{ stdout: string; stderr: string; exitCode: number; truncated: boolean } | ThreadErr> {
+    try {
+      return await this.execThreadBody(threadKey, command, timeoutMs, env);
+    } catch (err) {
+      // A DO reset over a live container answers its own word at once — the
+      // container is unchanged, so no `replacedExecAnswer` gate applies.
+      if (err instanceof ControlResetError) return controlResetErr(err);
+      if (err instanceof RuntimeReplacedError) return this.replacedExecAnswer(err);
+      throw err;
+    }
+  }
+
+  private async execThreadBody(
+    threadKey: string,
+    command: string,
+    timeoutMs: number,
+    env?: Record<string, string>,
   ): Promise<{ stdout: string; stderr: string; exitCode: number; truncated: boolean } | ThreadErr> {
     const pre = await this.threadPreflight(threadKey);
     if ("error" in pre) return pre;
@@ -6253,13 +6609,9 @@ export class ResidentDO extends Sandbox<Env> {
         : {}),
     } satisfies ThreadBinding);
 
-    let r: Awaited<ReturnType<ResidentDO["threadRun"]>>;
-    try {
-      r = await this.threadRunCapped(binding.user, binding.worktreePath, command, timeoutMs, EXEC_OUTPUT_CAP, env);
-    } catch (err) {
-      if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
-      throw err;
-    }
+    // A runtime replacement or a control reset under the command propagates to
+    // `execThread`'s gate (`replacedExecAnswer` / `controlResetErr`).
+    const r = await this.threadRunCapped(binding.user, binding.worktreePath, command, timeoutMs, EXEC_OUTPUT_CAP, env);
     const truncated = r.stdout.length > EXEC_OUTPUT_CAP || r.stderr.length > EXEC_OUTPUT_CAP || r.truncated === true;
     const notes: string[] = [];
     if (r.timedOut)
@@ -6311,6 +6663,7 @@ export class ResidentDO extends Sandbox<Env> {
         capBytesFor(READ_CONTENT_CAP),
       );
     } catch (err) {
+      if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
       throw err;
     }
@@ -6359,6 +6712,7 @@ export class ResidentDO extends Sandbox<Env> {
       }
       return { encoding: "base64", content: parts.join(""), size };
     } catch (err) {
+      if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
       throw err;
     }
@@ -6405,6 +6759,7 @@ export class ResidentDO extends Sandbox<Env> {
         DEFAULT_EXEC_TIMEOUT_MS,
       );
     } catch (err) {
+      if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
       const step = err instanceof StepError ? ` at ${err.step}` : "";
       return { error: `write-failed${step}: ${errMsg(err)}`, status: 400 };
@@ -6771,7 +7126,13 @@ export class ResidentDO extends Sandbox<Env> {
       await this.ensureHydrated();
     } catch (err) {
       const s = await this.getStatus();
-      return { error: `not-serviceable: ${errMsg(err)}`, status: 503, state: s.state, reason: s.reason };
+      return {
+        error: `not-serviceable: ${errMsg(err)}`,
+        status: 503,
+        state: s.state,
+        stateReason: s.reason,
+        reason: s.reason,
+      };
     }
     // One storage round trip for the two facts; the registry lookup stays (an
     // op resolves ONLY through the onboard-time command table).
@@ -6779,7 +7140,10 @@ export class ResidentDO extends Sandbox<Env> {
     const resource = (stored.get(RESOURCE_KEY) as string | undefined) ?? "";
     const facts = stored.get(FACTS_KEY) as RepoFacts | undefined;
     const record = await this.registry().getRecord(resource);
-    if (!record || !facts) return { error: "not-serviceable: registry record or repo facts missing", status: 503 };
+    // Typed `reason` beside the words: the client reads the field — a refusal no
+    // wait clears, unlike the restore window's 503s — never the sentence.
+    if (!record || !facts)
+      return { error: "not-serviceable: registry record or repo facts missing", status: 503, reason: "unregistered" };
     const command = record.commands[op];
     if (!command) return { error: `op-unavailable: the command table has no "${op}" entry`, status: 400 };
 
@@ -6875,13 +7239,15 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, reason: "mirror-busy" };
+        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
       }
       if (err instanceof StepError && err.step === "unknown-ref") {
         return { error: `unknown-ref: ${err.message}`, status: 400 };
       }
-      const step = err instanceof StepError ? ` at ${err.step}` : "";
-      return { error: `op-failed${step}: ${errMsg(err)}`, status: 500 };
+      // A step that failed is named and deterministic; a throw no step named is
+      // typed by the one builder every such 500 goes through.
+      if (err instanceof StepError) return { error: `op-failed at ${err.step}: ${errMsg(err)}`, status: 500 };
+      return catchAllErr(err, "op-failed");
     } finally {
       // Disposable means disposable: the checkout dies with the op, pass or
       // fail (best effort — a slept container already destroyed it anyway).
@@ -7035,13 +7401,22 @@ export class ResidentDO extends Sandbox<Env> {
         fates.set(ref, { fate: looked.fate, detail });
       }),
     );
+    // The runs registered on this resident (item 44): a run holds its tree
+    // from its attach to its release, and its process lives in the container
+    // between the bot's calls — the op counters read 0 while its model thinks.
+    const registered = new Set(
+      [...(await this.ctx.storage.list<RunRegistration>({ prefix: RUN_REG_KEY_PREFIX })).values()].map(
+        (r) => r.threadKey,
+      ),
+    );
     for (const binding of live) {
       const isDefaultRef = binding.ref === defaultRef;
       // The default branch is never a finished ref (reclaimDecision keeps it
       // by name), so its fate is never looked up.
       const { fate, detail } = (!isDefaultRef && fates.get(binding.ref)) || { fate: "unknown" as RefFate, detail: "" };
       const busy = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
-      const decision = reclaimDecision({ fate, isDefaultRef, busy });
+      const held = registered.has(binding.threadKey);
+      const decision = reclaimDecision({ fate, isDefaultRef, busy, held });
       if (!decision.reclaim) {
         kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: decision.why });
         continue;
@@ -7058,6 +7433,11 @@ export class ResidentDO extends Sandbox<Env> {
         kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "busy" });
         continue;
       }
+      // A run attached while the tree was measured holds it now (item 44).
+      if ((await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey))) !== undefined) {
+        kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "run-held" });
+        continue;
+      }
       if (!current || current.evicted || current.lastAttachAt !== binding.lastAttachAt) {
         kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "re-attached" });
         continue;
@@ -7411,13 +7791,17 @@ export class ResidentDO extends Sandbox<Env> {
     return { infraStreak: { ...row, rung: infraStreakRung(count) } };
   }
 
-  /** Fault injection for item 67: corrupt the mirror on disk (its objects
-   *  removed), so the next cycle's `fetch` fails in a resident step. The
-   *  snapshots in R2 are untouched — the ladder's recreate restores them.
+  /** Fault injection for item 67: corrupt the mirror on disk so the next
+   *  cycle's `fetch` fails in a resident step. The mirror's `config` goes (its
+   *  `origin` remote with it: `'origin' does not appear to be a git
+   *  repository`, exit 128) — NOT its objects: the wake check (`readyStamp`)
+   *  tests `objects/`, and a missing directory is a disk to restore, which the
+   *  wake did before any step could fail the first time this was tried live.
+   *  The snapshots in R2 are untouched — the ladder's recreate restores them.
    *  Test-only semantics; admin scope. */
   async debugBreakMirror(): Promise<{ broken: boolean; error?: string }> {
     try {
-      await this.runOk(["rm", "-rf", `${MIRROR_DIR}/objects`], "break-mirror");
+      await this.runOk(["rm", "-f", `${MIRROR_DIR}/config`], "break-mirror");
       return { broken: true };
     } catch (err) {
       return { broken: false, error: errMsg(err) };
@@ -7960,7 +8344,7 @@ export default {
             return json({ error: "unknown route" }, 404);
         }
       } catch (err) {
-        return json({ error: errMsg(err) }, 500);
+        return json(catchAllErr(err), 500);
       }
     })();
     root?.end(res.status >= 500 ? "error" : "ok", { httpStatus: res.status });
@@ -8510,7 +8894,7 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
       reason,
     ),
     (result) => result,
-    (err) => ({ error: errMsg(err), status: 500 }),
+    (err) => catchAllErr(err),
   );
 }
 
@@ -8528,7 +8912,7 @@ async function handleAwaitRestore(env: Env, body: Record<string, unknown>): Prom
   return streamHeartbeatJson(
     residentStub(env, resource.resource).awaitRestore(),
     (result) => result,
-    (err) => ({ error: errMsg(err), status: 500 }),
+    (err) => catchAllErr(err),
   );
 }
 
@@ -8606,33 +8990,50 @@ function streamHeartbeatJson<T>(
   return new Response(stream, { headers: { "content-type": "application/json" } });
 }
 
-/** /exec's payload mapping: a result as {stdout, stderr, exitCode, truncated},
- *  a named failure as {error, needs?, reason?, stdout:"", stderr:error, exitCode:127}
- *  (`reason:"runtime-replaced"` is how the client tells a deploy from a dead
- *  exec transport). */
+/** /exec's failure document, in the item-3 dual shape (`error` beside
+ *  `stdout: ""`, `stderr`, `exitCode: 127`) so old and new executors both
+ *  render it. The ThreadErr's fields ride beside the words, as the JSON routes
+ *  carry them: `needs`; the lifecycle pair (`state`, `stateReason`); the
+ *  answer's own word (`reason`, kept independent of `state` — `runtimeReplacedErr()`
+ *  sets `reason: "runtime-replaced"` with NO state, and the client's
+ *  deploy-vs-dead-transport check reads it); the answer's `status`; the
+ *  catch-all's `transient`. The client types a streamed failure by these fields
+ *  (execution.md item 9), and a document that dropped them would make every
+ *  /exec failure a deterministic answer over HTTP 200. One builder for both of
+ *  the stream's paths: a failure the Durable Object named, and a pending
+ *  result that rejected. */
+function execFailureDocument(failure: ThreadErr): object {
+  return {
+    error: failure.error,
+    ...(failure.needs ? { needs: failure.needs } : {}),
+    ...(failure.state ? { state: failure.state } : {}),
+    ...(typeof failure.stateReason === "string" ? { stateReason: failure.stateReason } : {}),
+    ...(failure.reason ? { reason: failure.reason } : {}),
+    status: failure.status,
+    ...(typeof failure.transient === "boolean" ? { transient: failure.transient } : {}),
+    stdout: "",
+    stderr: failure.error,
+    exitCode: 127,
+  };
+}
+
+/** /exec's payload mapping: a result as {stdout, stderr, exitCode, truncated};
+ *  a failure as `execFailureDocument` — the one the Durable Object named, or
+ *  the pending result's rejection answered as the DO would have
+ *  (`threadRejectionErr`: a code-update reset the DO's own `control-reset`
+ *  word, a replacement the SDK's words on a bare 409 — the word withheld on
+ *  /exec alone —, anything else the typed 500), so a rejection streams its
+ *  `status`, its word and its `transient`,
+ *  and the client reads it like the JSON routes' answer, never as a
+ *  deterministic answer over HTTP 200. */
 function streamThreadExec(pending: Promise<Awaited<ReturnType<ResidentDO["execThread"]>>>): Response {
   return streamHeartbeatJson(
     pending,
     (result) =>
       "error" in result
-        ? {
-            error: result.error,
-            ...(result.needs ? { needs: result.needs } : {}),
-            // `reason` must stay independent of `state`: runtimeReplacedErr()
-            // sets reason:"runtime-replaced" with NO state, and the client's
-            // deploy-vs-dead-transport check reads it. Folding these two spreads
-            // back into one silently drops it (no test covers this Worker).
-            ...(result.state ? { state: result.state } : {}),
-            ...(result.reason ? { reason: result.reason } : {}),
-            stdout: "",
-            stderr: result.error,
-            exitCode: 127,
-          }
+        ? execFailureDocument(result)
         : { stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, truncated: result.truncated },
-    (err) => {
-      const msg = errMsg(err);
-      return { error: msg, stdout: "", stderr: msg, exitCode: 127 };
-    },
+    (err) => execFailureDocument(threadRejectionErr(err, "/exec")),
   );
 }
 
@@ -8643,7 +9044,12 @@ async function handleRead(env: Env, body: Record<string, unknown>): Promise<Resp
     return json({ error: "path must be a string relative to the thread worktree" }, 400);
   const encoding = readEncodingOf(body);
   if (typeof encoding !== "string") return json({ error: encoding.error }, 400);
-  const result = await ctx.stub.readThreadFile(ctx.threadKey, body.path, encoding);
+  // A stub that rejects (the DO reset under the call, a storage operation that
+  // did not complete) is answered as the DO answers the same fact inside, never
+  // left to the fetch handler's catch-all.
+  const result = await ctx.stub
+    .readThreadFile(ctx.threadKey, body.path, encoding)
+    .catch((err: unknown) => threadRejectionErr(err, "/read"));
   if ("error" in result) return threadErrResponse(result);
   return json(result);
 }
@@ -8656,7 +9062,9 @@ async function handleWrite(env: Env, body: Record<string, unknown>): Promise<Res
   if (typeof body.content !== "string" || body.content.length > MAX_WRITE_CONTENT) {
     return json({ error: `content must be a string of at most ${MAX_WRITE_CONTENT} chars` }, 400);
   }
-  const result = await ctx.stub.writeThreadFile(ctx.threadKey, body.path, body.content);
+  const result = await ctx.stub
+    .writeThreadFile(ctx.threadKey, body.path, body.content)
+    .catch((err: unknown) => threadRejectionErr(err, "/write"));
   if ("error" in result) return threadErrResponse(result);
   return json(result);
 }
@@ -8730,18 +9138,18 @@ async function handleOp(env: Env, body: Record<string, unknown>, traceparent?: s
 }
 
 /** /op's payload mapping: results pass through; a named error sheds its
- *  transport-only `status` field (the body is the contract, never the code). */
+ *  transport-only `status` field (the body is the contract, never the code); a
+ *  pending op that rejected is the typed 500 (`catchAllErr`, so `transient`
+ *  rides beside the words), its status shed the same way. */
 function streamOp(pending: Promise<Awaited<ReturnType<ResidentDO["runOp"]>>>): Response {
+  const shedStatus = (failure: ThreadErr): object => {
+    const { status: _status, ...rest } = failure;
+    return rest;
+  };
   return streamHeartbeatJson(
     pending,
-    (result) => {
-      if ("error" in result) {
-        const { status: _status, ...rest } = result;
-        return rest;
-      }
-      return result;
-    },
-    (err) => ({ error: errMsg(err) }),
+    (result) => ("error" in result ? shedStatus(result) : result),
+    (err) => shedStatus(catchAllErr(err, "op-failed")),
   );
 }