@coreplane/switchboard 1.243.0 → 1.245.0

package/dist/assets/deploy/cloudflare-resident/worker.ts

@@ -69,6 +69,7 @@
 //      (docs/decisions/0009-residents-second-credential-domain.md).
 import {
   isDurableObjectCodeUpdateReset,
+  isPlatformTransientError,
   OperationInterruptedError,
   ProcessWaitTimeoutError,
   RPCTransportError,
@@ -80,7 +81,7 @@ import { AsyncLocalStorage } from "node:async_hooks";
 import type { DirectoryBackup, SandboxCommand } from "@cloudflare/sandbox";
 import { createExtensionProcessSandbox } from "@cloudflare/sandbox/extensions";
 import { DurableObject } from "cloudflare:workers";
-import { BASH_TIMEOUT_MAX_MS, BASH_TIMEOUT_MS, clampBashTimeout } from "../../src/execution/bashTimeout.js";
+import { BASH_TIMEOUT_MAX_MS, clampBashTimeout } from "../../src/execution/bashTimeout.js";
 import { selectBindingsToPurge } from "../../src/execution/bindingPurge.js";
 import { busyAfterKillReason, planForceDetach } from "../../src/execution/residentDetach.js";
 import { parseReadonly, planReadonlyAttach } from "../../src/execution/residentReadonly.js";
@@ -196,6 +197,7 @@ import {
   type RefreshPlan,
   type RestoreSample,
   type RefreshFailure,
+  fetchFailureIsMirrors,
 } from "../../src/execution/residentRefresh.js";
 import { autoRebuildDecision, isAutoRebuildEligible } from "../../src/execution/residentAutoRebuild.js";
 import {
@@ -321,6 +323,16 @@ import {
 } from "../../src/execution/residentDepsStore.js";
 import { buildId, injectedBuildStamp } from "../../src/deploy/buildStamp.js";
 import { createRefreshInstance, createRefreshInstanceNow, type RefreshInstanceParams } from "./refresh";
+import {
+  ControlResetError,
+  RuntimeReplacedError,
+  controlResetErr,
+  execFailureDocument,
+  runtimeReplacedErr,
+  selfAndCauses,
+  threadErrBuilders,
+  type ThreadErr,
+} from "./threadErr";
 import {
   DEFAULT_EXEC_TIMEOUT_MS,
   DEPS_STEP_OVERHEAD_MS,
@@ -619,9 +631,11 @@ const ATTACH_MUTEX_WAIT_MS = 60_000;
  *  stays far inside the bot's 45-min run budget. Work beyond 20 minutes
  *  belongs in background jobs.
  *
- *  /op runs (test/build) keep the flat 5-minute budget — the deterministic op
- *  path has no caller-supplied knob, so nothing may stretch it. */
-const OP_EXEC_TIMEOUT_MS = BASH_TIMEOUT_MS;
+ *  /op runs (test/build) get the flat 20-minute ceiling — the deterministic op
+ *  path has no caller-supplied knob, and a repo's full suite routinely outlives
+ *  the 5-minute per-command default; the client bounds its /op wait at the same
+ *  ceiling, so the two ends agree. */
+const OP_EXEC_TIMEOUT_MS = BASH_TIMEOUT_MAX_MS;
 /** Sanity bound on /exec's command body — a guard against a runaway caller,
  *  not a working limit: legitimate agent one-liners (heredocs writing test
  *  files, `node -e` scripts, long pipelines) run well past the old 8 000 and
@@ -672,28 +686,10 @@ export function validateEnvNames(vars: Record<string, string>): void {
   }
 }
 
-/** The resident runtime (the Sandbox SDK's control session to the container)
- *  was replaced while a command was in flight — in practice a `wrangler deploy`
- *  swapping this DO's isolate mid-run (which otherwise surfaces as a fake
- *  "OOM" on the run). `phase` says where the SDK failed: `"spawn"`
- *  (the start RPC itself; the SDK never proves the process did NOT start) or
- *  `"collect"` (a `StaleProcessHandleError` on an already-running process). In
- *  both cases the command may have run, so the resident never re-issues it;
- *  the thread routes answer the NAMED `runtime-replaced` error and the client
- *  decides (idempotent read/write retry; exec is handed to the model). */
-class RuntimeReplacedError extends Error {
-  constructor(
-    readonly phase: "spawn" | "collect",
-    readonly cause: unknown,
-  ) {
-    super(
-      `runtime-replaced: the resident runtime was replaced (a deploy) while this command was ${
-        phase === "spawn" ? "starting" : "running"
-      }; its output is lost (${errMsg(cause)})`,
-    );
-    this.name = "RuntimeReplacedError";
-  }
-}
+// `RuntimeReplacedError`, `ControlResetError`, the `ThreadErr` shape and the
+// builders that answer a throw no route named live in ./threadErr.ts, pure
+// over `(err, route)` so plain Node runs them; this entry hands them the
+// predicates only it can supply (`threadErrBuilders`, below the predicates).
 
 /** Every wording the pinned SDK (@cloudflare/sandbox@0.13.0-next.751.1) uses
  *  when the runtime incarnation changed under a call, for the message-based
@@ -711,17 +707,6 @@ class RuntimeReplacedError extends Error {
 // port-ready bound. Both take the message fallback below; neither has a typed
 // class to match.
 
-/** `err` and its `cause` chain, bounded like the SDK's own `selfAndCauses`
- *  walker: the SDK wraps platform errors, so the telling message can sit one or
- *  two links down. */
-function* selfAndCauses(err: unknown): Generator<unknown> {
-  let current = err;
-  for (let depth = 0; depth < 8 && current != null; depth++) {
-    yield current;
-    current = typeof current === "object" ? (current as { cause?: unknown }).cause : undefined;
-  }
-}
-
 /** `OperationInterruptedError` reasons that mean the runtime under the call is
  *  gone and a new one serves the thread: a deploy swapped the isolate
  *  (`runtime_replaced`), the sandbox's lifetime epoch moved
@@ -753,16 +738,25 @@ const RPC_TRANSPORT_LOSS_KINDS = new Set(["peer_closed", "connection_failed", "u
 /** Does this SDK error mean the runtime incarnation changed under us? Typed
  *  checks first (`StaleProcessHandleError`, `RuntimeIdentityInactiveError`, an
  *  `OperationInterruptedError` with one of `RUNTIME_REPLACED_REASONS`, an
- *  `RPCTransportError` with one of `RPC_TRANSPORT_LOSS_KINDS`, the platform's
- *  superseded-isolate reset), then the SDK's message wording — on the error AND
- *  its cause chain — as a belt-and-braces fallback. Anything else (a real spawn
- *  failure, a timeout, a wire-format error) is NOT a runtime replacement. */
+ *  `RPCTransportError` with one of `RPC_TRANSPORT_LOSS_KINDS`), then the SDK's
+ *  message wording — on the error AND its cause chain — as a belt-and-braces
+ *  fallback. Anything else (a real spawn failure, a timeout, a wire-format
+ *  error) is NOT a runtime replacement.
+ *
+ *  Deliberately IMMUNE to the platform's superseded-isolate reset (a DO
+ *  code-update reset): that is this Durable Object resetting over a container it
+ *  did not touch, the container unchanged, so it is never a runtime replacement
+ *  and never swaps the incarnation on the `/exec` path. `run()` answers it as a
+ *  `control-reset` (`isControlReset`), and the restore path folds it in
+ *  explicitly at its own site (`isRuntimeReplacement(err) || isControlReset(err)`)
+ *  — the ONE place a DO reset shares the replacement disposition, since a reset
+ *  under a live restore leaves nothing to land and re-restores onto the container
+ *  that comes back. Keeping it out of this predicate makes that use the only one. */
 function isRuntimeReplacement(err: unknown): boolean {
   if (err instanceof StaleProcessHandleError) return true;
   if (err instanceof RuntimeIdentityInactiveError) return true;
   if (err instanceof OperationInterruptedError && RUNTIME_REPLACED_REASONS.has(err.reason)) return true;
   if (err instanceof RPCTransportError && RPC_TRANSPORT_LOSS_KINDS.has(err.kind)) return true;
-  if (isDurableObjectCodeUpdateReset(err)) return true;
   for (const link of selfAndCauses(err)) if (RUNTIME_REPLACEMENT_WORDING.test(errMsg(link))) return true;
   return false;
 }
@@ -771,32 +765,50 @@ function isRuntimeReplacement(err: unknown): boolean {
  *  — a new incarnation serves the thread, so the container the command was
  *  aimed at is gone whatever else the resident knows: the typed classes for
  *  a stale process handle and an inactive runtime identity, an interruption
- *  whose reason is a replaced runtime, the platform's superseded-isolate
- *  reset, and the SDK's wordings for the same (`RUNTIME_MOVED_WORDING`, on the
- *  error and its cause chain). NOT the words for a container that is merely
- *  down (`STOPPED_CONTAINER_WORDING`: "The container is not running", "Process
- *  supervisor is closed") and NOT a transport lost under the call
- *  (`RPCTransportError`): an asleep or starting container and a network blip
- *  answer those too, so on `/exec` they are the word only when the resident
- *  itself knows the container it held is gone (`replacedExecAnswer`). The
- *  resident's own steps keep the union (`isRuntimeReplacement`): a step
- *  interrupted either way is retried onto the container that comes back. */
+ *  whose reason is a replaced runtime, and the SDK's wordings for the same
+ *  (`RUNTIME_MOVED_WORDING`, on the error and its cause chain). NOT the words
+ *  for a container that is merely down (`STOPPED_CONTAINER_WORDING`: "The
+ *  container is not running", "Process supervisor is closed") and NOT a
+ *  transport lost under the call (`RPCTransportError`): an asleep or starting
+ *  container and a network blip answer those too, so on `/exec` they are the
+ *  word only when the resident itself knows the container it held is gone
+ *  (`replacedExecAnswer`). NOT the platform's superseded-isolate reset either:
+ *  that is this Durable Object resetting over a container it did not touch —
+ *  `run()` intercepts it first, in both phases, as a `control-reset`
+ *  (`isControlReset`), so a DO-reset cause never reaches this predicate, and a
+ *  vouch for it here would say the opposite of that word. The restore path
+ *  keeps the union by naming both explicitly at its site
+ *  (`isRuntimeReplacement(err) || isControlReset(err)`), so a step interrupted
+ *  either way is retried onto the container that comes back — while this
+ *  predicate and `isRuntimeReplacement` stay immune to the reset. */
 function sdkVouchesRuntimeMoved(err: unknown): boolean {
   if (err instanceof StaleProcessHandleError) return true;
   if (err instanceof RuntimeIdentityInactiveError) return true;
   if (err instanceof OperationInterruptedError && RUNTIME_REPLACED_REASONS.has(err.reason)) return true;
-  if (isDurableObjectCodeUpdateReset(err)) return true;
   for (const link of selfAndCauses(err)) if (RUNTIME_MOVED_WORDING.test(errMsg(link))) return true;
   return false;
 }
 
-/** The named ThreadErr every thread route (exec/read/write) answers for a
- *  runtime replacement, so the client can classify it (409 like the other
- *  recoverable thread states; `reason` is the discriminator). On `/exec` the
- *  answer goes through `replacedExecAnswer` first: the word only when the SDK
- *  vouched the runtime moved or the resident knows the container is gone. */
-function runtimeReplacedErr(err: RuntimeReplacedError): ThreadErr {
-  return { error: err.message, status: 409, reason: "runtime-replaced" };
+/** The 500 for a throw no route named and the thread data plane's rejection
+ *  mapping (threadErr.ts holds the rules; threadErr.test.ts runs them under
+ *  plain Node with the SDK's real sentences), built over the predicates only
+ *  this entry can supply: the SDK's reset and platform-transient predicates,
+ *  and the resident's own replacement classifier and vouch. */
+const { catchAllErr, threadRejectionErr } = threadErrBuilders({
+  isControlReset,
+  isRuntimeReplacement,
+  sdkVouchesRuntimeMoved,
+  isPlatformTransientError,
+});
+
+/** Whether a command's failure is the resident's own Durable Object resetting
+ *  under it (a Worker-code deploy) rather than the container runtime being
+ *  replaced. Checked BEFORE `isRuntimeReplacement` on the thread `run()` path,
+ *  so a DO reset over a still-running container answers `control-reset`, never
+ *  the replaced word. The SDK's predicate already walks the cause
+ *  chain (`selfAndCauses`), so it is called directly on the caught error. */
+function isControlReset(err: unknown): boolean {
+  return isDurableObjectCodeUpdateReset(err);
 }
 
 /** The container's control port never answered: `exec` rejected with the
@@ -1122,22 +1134,6 @@ interface ThreadBinding {
   depsKey?: string;
 }
 
-/** Named, RPC-cloneable error shape for the thread data plane. The Worker
- *  maps `status` to the HTTP status; extra fields (`needs`, `state`,
- *  `reason`) ride along into the body. */
-interface ThreadErr {
-  error: string;
-  status: number;
-  /** The steps the request ran before it failed (docs/reference/specs/tracing.md item 19):
-   *  a failed attach's trace is the one that says which step blew the budget. */
-  trace?: ResidentStep[];
-  needs?: string;
-  /** With needs:"ref" — the resident's default branch, so the caller can bind by default. */
-  defaultRef?: string;
-  state?: ResidentState;
-  reason?: string;
-}
-
 interface AttachOk {
   workspace: string;
   ref: string;
@@ -1618,49 +1614,49 @@ export class ResidentDO extends Sandbox<Env> {
    *  survives, so hydration state is always probed from disk. */
   private hydration: Promise<void> | null = null;
 
-  /** The container stop the platform delivered to `onStop` and no command has
-   *  answered since: the resident's own knowledge that the container it held
-   *  is gone (docs/reference/specs/resident-repos.md item 43). In memory on
-   *  purpose — a Durable Object that restarts learns the same fact from the
-   *  restore its next command's wake path starts (`restoring`), and a stop
-   *  recorded in storage would outlive the container that answers now. */
-  private containerStop: { at: number; exitCode: number; reason: string } | undefined;
-
-  /** The platform's signal that the container stopped — the rollout's
-   *  `runtime_signal`, or an exit of its own — recorded before the SDK's
-   *  reconciliation runs, so an `/exec` that meets the stopped container from
-   *  here on answers `runtime-replaced` (`replacedExecAnswer`) rather than the
-   *  binding's bare "The container is not running". The Sandbox base declares
-   *  the hook without parameters; the Container base beneath it passes them. */
-  async onStop(params?: { exitCode: number; reason: string }): Promise<void> {
-    this.containerStop = { at: systemClock(), exitCode: params?.exitCode ?? 0, reason: params?.reason ?? "unknown" };
-    console.log(
-      `container: stopped (exit ${this.containerStop.exitCode}, ${this.containerStop.reason}) — the container this resident held is gone; /exec answers runtime-replaced until a command answers again`,
-    );
-    await super.onStop();
+  /** Whether this resident knows the container it held is gone
+   *  (docs/reference/specs/resident-repos.md item 43): a restore under way for
+   *  this resident (`restoring`, persisted before any restore work) — its own
+   *  knowledge, reliable — and nothing else.
+   *
+   *  Not the Container base's exit state, not a replayed `onStop`, not the
+   *  platform's `running` flag: each was tried or weighed and separates
+   *  nothing — a roll and an idle sleep leave the same traces, in both
+   *  directions — and the why, with the pinned library's lines, is the
+   *  record's alone (docs/reference/specs/resident-repos.md item 43), so a
+   *  dependency bump rots one place, not three. The harness's own probe
+   *  (`identity`, `alive`) stays the arbiter, so under-saying the word here is
+   *  safe and over-saying it is what orphans a process.
+   *
+   *  A read that fails (the storage under a reset or a restore) is no
+   *  knowledge: the answer falls back to the SDK's words and the harness
+   *  probes, never an unhandled throw where a 409 was due. */
+  private async knowsContainerGone(): Promise<boolean> {
+    try {
+      return (await this.getStatus()).state === "restoring";
+    } catch {
+      return false;
+    }
   }
 
-  /** Whether this resident knows the container it held is gone: it saw the
-   *  container stop and no command has answered since, or a restore is under
-   *  way (`restoring`, persisted before any restore work). Never the
-   *  platform's `running` flag — an asleep or starting container answers
-   *  false to it too, and an `/exec` that meets one must say what the SDK
-   *  said so the harness probes rather than relaunch into a container that
-   *  was never replaced. */
-  private async knowsContainerGone(): Promise<boolean> {
-    if (this.containerStop !== undefined) return true;
-    return (await this.getStatus()).state === "restoring";
+  /** The one decision on a failure the SDK classified as a runtime
+   *  replacement, made where it is classified (`run()`): the SDK vouched the
+   *  runtime moved, or a restore is under way for this resident. It gates the
+   *  word on `/exec` and the incarnation mint alike; the per-incarnation memos
+   *  clear before it, unconditionally. */
+  private async replacementKnown(err: unknown): Promise<boolean> {
+    return sdkVouchesRuntimeMoved(err) || (await this.knowsContainerGone());
   }
 
   /** The `/exec` answer for a command the SDK failed with a runtime
    *  replacement (item 43; harness.md item 6): the word — `reason:
-   *  "runtime-replaced"`, the resident's sentence — when the SDK vouched the
-   *  runtime moved or the resident knows the container it held is gone;
-   *  otherwise what the SDK said, with no word and no `reason`, so the
-   *  harness's one more command decides. The command is never re-issued
-   *  either way: it may have started. */
-  private async replacedExecAnswer(err: RuntimeReplacedError): Promise<ThreadErr> {
-    if (sdkVouchesRuntimeMoved(err.cause) || (await this.knowsContainerGone())) return runtimeReplacedErr(err);
+   *  "runtime-replaced"`, the resident's sentence — when the replacement was
+   *  known at the choke point (`RuntimeReplacedError.known`); otherwise what
+   *  the SDK said, with no word and no `reason`, so the harness's one more
+   *  command decides. The command is never re-issued either way: it may have
+   *  started. */
+  private replacedExecAnswer(err: RuntimeReplacedError): ThreadErr {
+    if (err.known) return runtimeReplacedErr(err);
     return { error: errMsg(err.cause), status: 409 };
   }
 
@@ -2024,6 +2020,12 @@ export class ResidentDO extends Sandbox<Env> {
     try {
       proc = await createExtensionProcessSandbox(this).exec(argv as unknown as SandboxCommand, launch);
     } catch (err) {
+      // A DO code-update reset (our own Worker deploy) BEFORE a runtime
+      // replacement: the container is unchanged, so this is `control-reset`, not
+      // the replaced word — never `swapIncarnation`, and the command's outcome
+      // is unknown. Checked first so a DO reset over a live pi is never
+      // read as a replaced container.
+      if (isControlReset(err)) throw new ControlResetError("spawn", err);
       if (!isRuntimeReplacement(err)) {
         // The control port never answered the SDK's connect (its 30 s abort,
         // raised inside the wake path): no process started and nothing about
@@ -2041,8 +2043,25 @@ export class ResidentDO extends Sandbox<Env> {
       // takes the throw below. It exists so that if a future SDK vouches "never
       // started" we retry then — and only then — without a change here.
       if (!(err instanceof OperationInterruptedError && err.retryable === true)) {
-        this.swapIncarnation(); // the container this incarnation's memos described is gone
-        throw new RuntimeReplacedError("spawn", err);
+        // Two consequences with different safety, split here. The memos
+        // (`hydratedVerdictAt`, `gitSetupDone`, `stageDirsReady`, the deps
+        // flags — the fact that was stale) clear on EVERY replacement the SDK
+        // classified: cheap to re-derive, and the per-incarnation memo block's
+        // invariant is that this one choke point clears them, whatever the
+        // word will say. The incarnation — the lease fencing token, one
+        // isolate paired with one container runtime — is minted only when the
+        // replacement is KNOWN: `isRuntimeReplacement` is the union, and a
+        // transport blip on one request that minted an id would make
+        // `takeMutex` read the live holder's row as `holder-incarnation-gone`
+        // and hand the mirror mutex to a concurrent `/attach` over a running
+        // fetch. (A swap gated whole on `known` was tried and left the memos
+        // stale on a real roll, where `known` is false while the wake rewrites
+        // the base's state — resident-repos item 43.) Only the word is gated,
+        // on the same `known`.
+        this.clearIncarnationMemos();
+        const known = await this.replacementKnown(err);
+        if (known) this.swapIncarnation();
+        throw new RuntimeReplacedError("spawn", err, known);
       }
       console.log(
         `exec: runtime replaced before the process started (SDK says retryable) — retrying once: ${errMsg(err)}`,
@@ -2050,9 +2069,7 @@ export class ResidentDO extends Sandbox<Env> {
       proc = await createExtensionProcessSandbox(this).exec(argv as unknown as SandboxCommand, launch);
     }
     // The spawn is the proof the control port answers: a persisted count of
-    // unanswered connects ends here, whatever the command goes on to do — and
-    // a container stop seen before it is history (`knowsContainerGone`).
-    this.containerStop = undefined;
+    // unanswered connects ends here, whatever the command goes on to do.
     await this.clearRuntimeUnreachable();
     try {
       const out = await proc.output({ encoding: "utf8", timeout: timeout + 30_000 });
@@ -2066,9 +2083,18 @@ export class ResidentDO extends Sandbox<Env> {
         truncated: out.truncated,
       };
     } catch (err) {
+      // A DO code-update reset during collect: the container and the process are
+      // unchanged (only this DO's isolate reset), so the outcome is unknown but
+      // never a replacement — `control-reset`, no `swapIncarnation`.
+      // Checked before the replacement branch below.
+      if (isControlReset(err)) throw new ControlResetError("collect", err);
       if (isRuntimeReplacement(err)) {
-        this.swapIncarnation(); // the container this incarnation's memos described is gone
-        throw new RuntimeReplacedError("collect", err);
+        // The same split as the spawn site above: the memos unconditionally,
+        // the incarnation only when the replacement is known, the word gated.
+        this.clearIncarnationMemos();
+        const known = await this.replacementKnown(err);
+        if (known) this.swapIncarnation();
+        throw new RuntimeReplacedError("collect", err, known);
       }
       if (err instanceof ProcessWaitTimeoutError) {
         // The supervisor should have killed the process at `timeout`; 30 s
@@ -2648,7 +2674,13 @@ export class ResidentDO extends Sandbox<Env> {
       // asks the runtime whether the container the extract wrote to is still
       // there (the incident that named this: a resident went `down` on exactly
       // this, 10 ms before every exec answered "container is not running").
-      const runtimeReplaced = isRuntimeReplacement(err);
+      // A DO code-update reset under a live restore leaves nothing to land (the
+      // restore's SDK call is gone with this isolate), so it shares the
+      // replacement disposition — interrupted, retried onto the container that
+      // comes back. This is the ONLY site that folds the DO reset in;
+      // `isRuntimeReplacement` itself stays immune to it, so `/exec` never
+      // answers the replaced word for a reset over a live container.
+      const runtimeReplaced = isRuntimeReplacement(err) || isControlReset(err);
       const runtimeActive = runtimeReplaced ? false : await this.isRuntimeActive().catch(() => null);
       const disposition = restoreFailureDisposition(errMsg(err), {
         runtimeReplaced,
@@ -3301,6 +3333,14 @@ export class ResidentDO extends Sandbox<Env> {
         await this.recoverFromDiskFull(failure.reason, selfInFlight);
         return { ok: false, reason: failure.reason };
       }
+      // Item 67: a fetch that failed on the MIRROR (its remote gone, its
+      // object store broken) is the resident's own step failing — `fetch-failed`,
+      // counted by the ladder, never serviceable — not GitHub being unreachable.
+      // The first live row found a broken mirror parked as `github-unreachable`.
+      if (fetchFailureIsMirrors(message)) {
+        await this.refreshFailed(failure, selfInFlight);
+        return { ok: false, reason: failure.reason };
+      }
       const reason = `github-unreachable: ${message}`;
       await this.setResidentState("degraded", reason);
       return { ok: false, reason };
@@ -4427,7 +4467,7 @@ export class ResidentDO extends Sandbox<Env> {
       const reason = diskPressureReason({ verdict: final, evicted, kept, spares });
       console.log(`attach ${input.threadKey}: ${reason}`);
       const s = await this.getStatus();
-      return { error: reason, status: 503, state: s.state, reason: DISK_PRESSURE_REASON };
+      return { error: reason, status: 503, state: s.state, stateReason: s.reason, reason: DISK_PRESSURE_REASON };
     }
     const m = final.math;
     console.log(
@@ -4827,6 +4867,7 @@ export class ResidentDO extends Sandbox<Env> {
           error: "image-stale: the container predates the current pool and is restarting; retry shortly",
           status: 503,
           state: "restoring",
+          stateReason: "",
           reason: "image-stale",
         };
       }
@@ -4854,7 +4895,7 @@ export class ResidentDO extends Sandbox<Env> {
         this.attachesInFlight--;
       }
     } catch (err) {
-      return { error: `attach-failed: ${errMsg(err)}`, status: 500 };
+      return catchAllErr(err, "attach-failed");
     }
   }
 
@@ -4866,7 +4907,7 @@ export class ResidentDO extends Sandbox<Env> {
    *  instance's entry gate (the cron's, within one bucket) — an attach never
    *  stops the container itself: it is in flight. */
   private async attachFailed(err: unknown): Promise<ThreadErr> {
-    if (!(err instanceof StepError)) return { error: `attach-failed: ${errMsg(err)}`, status: 500 };
+    if (!(err instanceof StepError)) return catchAllErr(err, "attach-failed");
     const failure = await this.classifyFailure(err.step, err.message);
     if (!failure.diskFull) return { error: `attach-failed at ${err.step}: ${err.message}`, status: 500 };
     console.log(`attach: ${failure.reason}`);
@@ -4890,7 +4931,13 @@ export class ResidentDO extends Sandbox<Env> {
       await this.refreshIfStale(resourceId);
     } catch (err) {
       const s = await this.getStatus();
-      return { error: `not-serviceable: ${errMsg(err)}`, status: 503, state: s.state, reason: s.reason };
+      return {
+        error: `not-serviceable: ${errMsg(err)}`,
+        status: 503,
+        state: s.state,
+        stateReason: s.reason,
+        reason: s.reason,
+      };
     }
     // `resourceId` was read by the caller a moment ago (item 15 of the audit:
     // this used to re-read the same key), the registry record rides in from
@@ -4902,7 +4949,10 @@ export class ResidentDO extends Sandbox<Env> {
     const stored = await this.ctx.storage.get<RepoFacts | ThreadBinding>([FACTS_KEY, threadBindingKey(threadKey)]);
     const facts = stored.get(FACTS_KEY) as RepoFacts | undefined;
     const record = recordFromRoute ?? (await this.registry().getRecord(resource));
-    if (!record || !facts) return { error: "not-serviceable: registry record or repo facts missing", status: 503 };
+    // Typed `reason` beside the words: the client reads the field — a refusal no
+    // wait clears, unlike the restore window's 503s — never the sentence.
+    if (!record || !facts)
+      return { error: "not-serviceable: registry record or repo facts missing", status: 503, reason: "unregistered" };
 
     // The binding's ref wins for the thread's whole life, with one exception
     // (item 16): a thread bound to the repo default for want of a named branch
@@ -5052,9 +5102,9 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, reason: "mirror-busy" };
+        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
       }
-      return { error: `attach-failed: ${errMsg(err)}`, status: 500 };
+      return catchAllErr(err, "attach-failed");
     }
     if (outcome.kind === "none") return { binding: outcome.binding };
     if (outcome.kind === "refuse") {
@@ -5354,7 +5404,7 @@ export class ResidentDO extends Sandbox<Env> {
         return { error: `reuse-refused: ${err.why}`, status: 409, needs: "recreate" };
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, reason: "mirror-busy" };
+        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
       }
       if (err instanceof StepError && err.step === "unknown-ref") {
         return { error: `unknown-ref: ${err.message}`, status: 400 };
@@ -5399,7 +5449,7 @@ export class ResidentDO extends Sandbox<Env> {
       // worktree lock above, so the bot-side fallback can retry.
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, reason: "mirror-busy" };
+        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
       }
       return this.attachFailed(err);
     }
@@ -6237,7 +6287,13 @@ export class ResidentDO extends Sandbox<Env> {
       await this.ensureHydrated();
     } catch (err) {
       const s = await this.getStatus();
-      return { error: `not-serviceable: ${errMsg(err)}`, status: 503, state: s.state, reason: s.reason };
+      return {
+        error: `not-serviceable: ${errMsg(err)}`,
+        status: 503,
+        state: s.state,
+        stateReason: s.reason,
+        reason: s.reason,
+      };
     }
     const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
     if (!binding) {
@@ -6321,6 +6377,9 @@ export class ResidentDO extends Sandbox<Env> {
     try {
       return await this.execThreadBody(threadKey, command, timeoutMs, env);
     } catch (err) {
+      // A DO reset over a live container answers its own word at once — the
+      // container is unchanged, so no `replacedExecAnswer` gate applies.
+      if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return this.replacedExecAnswer(err);
       throw err;
     }
@@ -6346,7 +6405,8 @@ export class ResidentDO extends Sandbox<Env> {
         : {}),
     } satisfies ThreadBinding);
 
-    // A runtime replacement under the command propagates to `execThreadImpl`'s gate.
+    // A runtime replacement or a control reset under the command propagates to
+    // `execThread`'s gate (`replacedExecAnswer` / `controlResetErr`).
     const r = await this.threadRunCapped(binding.user, binding.worktreePath, command, timeoutMs, EXEC_OUTPUT_CAP, env);
     const truncated = r.stdout.length > EXEC_OUTPUT_CAP || r.stderr.length > EXEC_OUTPUT_CAP || r.truncated === true;
     const notes: string[] = [];
@@ -6399,6 +6459,7 @@ export class ResidentDO extends Sandbox<Env> {
         capBytesFor(READ_CONTENT_CAP),
       );
     } catch (err) {
+      if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
       throw err;
     }
@@ -6447,6 +6508,7 @@ export class ResidentDO extends Sandbox<Env> {
       }
       return { encoding: "base64", content: parts.join(""), size };
     } catch (err) {
+      if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
       throw err;
     }
@@ -6493,6 +6555,7 @@ export class ResidentDO extends Sandbox<Env> {
         DEFAULT_EXEC_TIMEOUT_MS,
       );
     } catch (err) {
+      if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
       const step = err instanceof StepError ? ` at ${err.step}` : "";
       return { error: `write-failed${step}: ${errMsg(err)}`, status: 400 };
@@ -6859,7 +6922,13 @@ export class ResidentDO extends Sandbox<Env> {
       await this.ensureHydrated();
     } catch (err) {
       const s = await this.getStatus();
-      return { error: `not-serviceable: ${errMsg(err)}`, status: 503, state: s.state, reason: s.reason };
+      return {
+        error: `not-serviceable: ${errMsg(err)}`,
+        status: 503,
+        state: s.state,
+        stateReason: s.reason,
+        reason: s.reason,
+      };
     }
     // One storage round trip for the two facts; the registry lookup stays (an
     // op resolves ONLY through the onboard-time command table).
@@ -6867,7 +6936,10 @@ export class ResidentDO extends Sandbox<Env> {
     const resource = (stored.get(RESOURCE_KEY) as string | undefined) ?? "";
     const facts = stored.get(FACTS_KEY) as RepoFacts | undefined;
     const record = await this.registry().getRecord(resource);
-    if (!record || !facts) return { error: "not-serviceable: registry record or repo facts missing", status: 503 };
+    // Typed `reason` beside the words: the client reads the field — a refusal no
+    // wait clears, unlike the restore window's 503s — never the sentence.
+    if (!record || !facts)
+      return { error: "not-serviceable: registry record or repo facts missing", status: 503, reason: "unregistered" };
     const command = record.commands[op];
     if (!command) return { error: `op-unavailable: the command table has no "${op}" entry`, status: 400 };
 
@@ -6963,13 +7035,15 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, reason: "mirror-busy" };
+        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
       }
       if (err instanceof StepError && err.step === "unknown-ref") {
         return { error: `unknown-ref: ${err.message}`, status: 400 };
       }
-      const step = err instanceof StepError ? ` at ${err.step}` : "";
-      return { error: `op-failed${step}: ${errMsg(err)}`, status: 500 };
+      // A step that failed is named and deterministic; a throw no step named is
+      // typed by the one builder every such 500 goes through.
+      if (err instanceof StepError) return { error: `op-failed at ${err.step}: ${errMsg(err)}`, status: 500 };
+      return catchAllErr(err, "op-failed");
     } finally {
       // Disposable means disposable: the checkout dies with the op, pass or
       // fail (best effort — a slept container already destroyed it anyway).
@@ -7123,13 +7197,22 @@ export class ResidentDO extends Sandbox<Env> {
         fates.set(ref, { fate: looked.fate, detail });
       }),
     );
+    // The runs registered on this resident (item 44): a run holds its tree
+    // from its attach to its release, and its process lives in the container
+    // between the bot's calls — the op counters read 0 while its model thinks.
+    const registered = new Set(
+      [...(await this.ctx.storage.list<RunRegistration>({ prefix: RUN_REG_KEY_PREFIX })).values()].map(
+        (r) => r.threadKey,
+      ),
+    );
     for (const binding of live) {
       const isDefaultRef = binding.ref === defaultRef;
       // The default branch is never a finished ref (reclaimDecision keeps it
       // by name), so its fate is never looked up.
       const { fate, detail } = (!isDefaultRef && fates.get(binding.ref)) || { fate: "unknown" as RefFate, detail: "" };
       const busy = this.threadOpsInFlight.get(binding.threadKey) ?? 0;
-      const decision = reclaimDecision({ fate, isDefaultRef, busy });
+      const held = registered.has(binding.threadKey);
+      const decision = reclaimDecision({ fate, isDefaultRef, busy, held });
       if (!decision.reclaim) {
         kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: decision.why });
         continue;
@@ -7146,6 +7229,11 @@ export class ResidentDO extends Sandbox<Env> {
         kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "busy" });
         continue;
       }
+      // A run attached while the tree was measured holds it now (item 44).
+      if ((await this.ctx.storage.get<RunRegistration>(runRegKey(binding.threadKey))) !== undefined) {
+        kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "run-held" });
+        continue;
+      }
       if (!current || current.evicted || current.lastAttachAt !== binding.lastAttachAt) {
         kept.push({ threadKey: binding.threadKey, ref: binding.ref, why: "re-attached" });
         continue;
@@ -8052,7 +8140,7 @@ export default {
             return json({ error: "unknown route" }, 404);
         }
       } catch (err) {
-        return json({ error: errMsg(err) }, 500);
+        return json(catchAllErr(err), 500);
       }
     })();
     root?.end(res.status >= 500 ? "error" : "ok", { httpStatus: res.status });
@@ -8602,7 +8690,7 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
       reason,
     ),
     (result) => result,
-    (err) => ({ error: errMsg(err), status: 500 }),
+    (err) => catchAllErr(err),
   );
 }
 
@@ -8620,7 +8708,7 @@ async function handleAwaitRestore(env: Env, body: Record<string, unknown>): Prom
   return streamHeartbeatJson(
     residentStub(env, resource.resource).awaitRestore(),
     (result) => result,
-    (err) => ({ error: errMsg(err), status: 500 }),
+    (err) => catchAllErr(err),
   );
 }
 
@@ -8698,33 +8786,23 @@ function streamHeartbeatJson<T>(
   return new Response(stream, { headers: { "content-type": "application/json" } });
 }
 
-/** /exec's payload mapping: a result as {stdout, stderr, exitCode, truncated},
- *  a named failure as {error, needs?, reason?, stdout:"", stderr:error, exitCode:127}
- *  (`reason:"runtime-replaced"` is how the client tells a deploy from a dead
- *  exec transport). */
+/** /exec's payload mapping: a result as {stdout, stderr, exitCode, truncated};
+ *  a failure as `execFailureDocument` — the one the Durable Object named, or
+ *  the pending result's rejection answered as the DO would have
+ *  (`threadRejectionErr`: a code-update reset the DO's own `control-reset`
+ *  word, a replacement the SDK's words on a bare 409 — the word withheld on
+ *  /exec alone —, anything else the typed 500), so a rejection streams its
+ *  `status`, its word and its `transient`,
+ *  and the client reads it like the JSON routes' answer, never as a
+ *  deterministic answer over HTTP 200. */
 function streamThreadExec(pending: Promise<Awaited<ReturnType<ResidentDO["execThread"]>>>): Response {
   return streamHeartbeatJson(
     pending,
     (result) =>
       "error" in result
-        ? {
-            error: result.error,
-            ...(result.needs ? { needs: result.needs } : {}),
-            // `reason` must stay independent of `state`: runtimeReplacedErr()
-            // sets reason:"runtime-replaced" with NO state, and the client's
-            // deploy-vs-dead-transport check reads it. Folding these two spreads
-            // back into one silently drops it (no test covers this Worker).
-            ...(result.state ? { state: result.state } : {}),
-            ...(result.reason ? { reason: result.reason } : {}),
-            stdout: "",
-            stderr: result.error,
-            exitCode: 127,
-          }
+        ? execFailureDocument(result)
         : { stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, truncated: result.truncated },
-    (err) => {
-      const msg = errMsg(err);
-      return { error: msg, stdout: "", stderr: msg, exitCode: 127 };
-    },
+    (err) => execFailureDocument(threadRejectionErr(err, "/exec")),
   );
 }
 
@@ -8735,7 +8813,12 @@ async function handleRead(env: Env, body: Record<string, unknown>): Promise<Resp
     return json({ error: "path must be a string relative to the thread worktree" }, 400);
   const encoding = readEncodingOf(body);
   if (typeof encoding !== "string") return json({ error: encoding.error }, 400);
-  const result = await ctx.stub.readThreadFile(ctx.threadKey, body.path, encoding);
+  // A stub that rejects (the DO reset under the call, a storage operation that
+  // did not complete) is answered as the DO answers the same fact inside, never
+  // left to the fetch handler's catch-all.
+  const result = await ctx.stub
+    .readThreadFile(ctx.threadKey, body.path, encoding)
+    .catch((err: unknown) => threadRejectionErr(err, "/read"));
   if ("error" in result) return threadErrResponse(result);
   return json(result);
 }
@@ -8748,7 +8831,9 @@ async function handleWrite(env: Env, body: Record<string, unknown>): Promise<Res
   if (typeof body.content !== "string" || body.content.length > MAX_WRITE_CONTENT) {
     return json({ error: `content must be a string of at most ${MAX_WRITE_CONTENT} chars` }, 400);
   }
-  const result = await ctx.stub.writeThreadFile(ctx.threadKey, body.path, body.content);
+  const result = await ctx.stub
+    .writeThreadFile(ctx.threadKey, body.path, body.content)
+    .catch((err: unknown) => threadRejectionErr(err, "/write"));
   if ("error" in result) return threadErrResponse(result);
   return json(result);
 }
@@ -8822,18 +8907,18 @@ async function handleOp(env: Env, body: Record<string, unknown>, traceparent?: s
 }
 
 /** /op's payload mapping: results pass through; a named error sheds its
- *  transport-only `status` field (the body is the contract, never the code). */
+ *  transport-only `status` field (the body is the contract, never the code); a
+ *  pending op that rejected is the typed 500 (`catchAllErr`, so `transient`
+ *  rides beside the words), its status shed the same way. */
 function streamOp(pending: Promise<Awaited<ReturnType<ResidentDO["runOp"]>>>): Response {
+  const shedStatus = (failure: ThreadErr): object => {
+    const { status: _status, ...rest } = failure;
+    return rest;
+  };
   return streamHeartbeatJson(
     pending,
-    (result) => {
-      if ("error" in result) {
-        const { status: _status, ...rest } = result;
-        return rest;
-      }
-      return result;
-    },
-    (err) => ({ error: errMsg(err) }),
+    (result) => ("error" in result ? shedStatus(result) : result),
+    (err) => shedStatus(catchAllErr(err, "op-failed")),
   );
 }