@coreplane/switchboard 1.241.0 → 1.243.0

package/dist/assets/deploy/cloudflare-resident/worker.ts

@@ -179,6 +179,7 @@ import {
   isRuntimeUnreachableSignal,
   killStaleBuildProcessesCommand,
   planRefresh,
+  RUNTIME_MOVED_WORDING,
   RUNTIME_REPLACEMENT_WORDING,
   RUNTIME_UNREACHABLE_DOWN_AT,
   runtimeUnreachableReason,
@@ -197,6 +198,14 @@ import {
   type RefreshFailure,
 } from "../../src/execution/residentRefresh.js";
 import { autoRebuildDecision, isAutoRebuildEligible } from "../../src/execution/residentAutoRebuild.js";
+import {
+  INFRA_STREAK_DOWN_AT,
+  infraStreakReason,
+  infraStreakRung,
+  type InfraStreakRow,
+  isRepoCommandStep,
+  parksOnRepeat,
+} from "../../src/execution/residentInfraStreak.js";
 import {
   lifecycleOf,
   parseLifecycle,
@@ -544,6 +553,8 @@ const GITHUB_API_TIMEOUT_MS = 10_000;
  *  idle-park like a warm one; the next attach still refreshes first. */
 const DEGRADED_PARK_AFTER_CYCLES = 3;
 const DEGRADED_STREAK_KEY = "resident:degradedStreak";
+/** Consecutive cycles failed in the resident's OWN steps (item 67): the ladder's count, `InfraStreakRow`. */
+const INFRA_STREAK_KEY = "resident:infraStreak";
 /** Degraded reasons that are not evidence about the repository — both always
  *  carry a `: detail` suffix: the watchdog's `stale-mid-flight: …` (a marker
  *  a dead cycle left behind; a cycle must run) and the wake path's
@@ -561,7 +572,7 @@ const DEGRADED_STREAK_KEY = "resident:degradedStreak";
  *  re-enters the step. A control port that did not answer
  *  (`runtime-unreachable: …`, item 64) is the third: no command ran, the
  *  ladder over its own persisted count owns the recovery. */
-const NON_EVIDENCE_REASON = /^(?:stale-mid-flight|restore-interrupted|runtime-unreachable):/;
+const NON_EVIDENCE_REASON = /^(?:stale-mid-flight|restore-interrupted|runtime-unreachable|infra-streak):/;
 /** When the disk-full recovery last stopped the container (docs/reference/specs/resident-repos.md item 54):
  *  feeds `planDiskFullRecovery`'s cooldown so a working set that refills the
  *  disk is named, not recycled in a loop. */
@@ -756,9 +767,34 @@ function isRuntimeReplacement(err: unknown): boolean {
   return false;
 }
 
+/** Whether the SDK's failure vouches that the runtime under the command MOVED
+ *  — a new incarnation serves the thread, so the container the command was
+ *  aimed at is gone whatever else the resident knows: the typed classes for
+ *  a stale process handle and an inactive runtime identity, an interruption
+ *  whose reason is a replaced runtime, the platform's superseded-isolate
+ *  reset, and the SDK's wordings for the same (`RUNTIME_MOVED_WORDING`, on the
+ *  error and its cause chain). NOT the words for a container that is merely
+ *  down (`STOPPED_CONTAINER_WORDING`: "The container is not running", "Process
+ *  supervisor is closed") and NOT a transport lost under the call
+ *  (`RPCTransportError`): an asleep or starting container and a network blip
+ *  answer those too, so on `/exec` they are the word only when the resident
+ *  itself knows the container it held is gone (`replacedExecAnswer`). The
+ *  resident's own steps keep the union (`isRuntimeReplacement`): a step
+ *  interrupted either way is retried onto the container that comes back. */
+function sdkVouchesRuntimeMoved(err: unknown): boolean {
+  if (err instanceof StaleProcessHandleError) return true;
+  if (err instanceof RuntimeIdentityInactiveError) return true;
+  if (err instanceof OperationInterruptedError && RUNTIME_REPLACED_REASONS.has(err.reason)) return true;
+  if (isDurableObjectCodeUpdateReset(err)) return true;
+  for (const link of selfAndCauses(err)) if (RUNTIME_MOVED_WORDING.test(errMsg(link))) return true;
+  return false;
+}
+
 /** The named ThreadErr every thread route (exec/read/write) answers for a
  *  runtime replacement, so the client can classify it (409 like the other
- *  recoverable thread states; `reason` is the discriminator). */
+ *  recoverable thread states; `reason` is the discriminator). On `/exec` the
+ *  answer goes through `replacedExecAnswer` first: the word only when the SDK
+ *  vouched the runtime moved or the resident knows the container is gone. */
 function runtimeReplacedErr(err: RuntimeReplacedError): ThreadErr {
   return { error: err.message, status: 409, reason: "runtime-replaced" };
 }
@@ -1479,6 +1515,17 @@ const threadBindingKey = (threadKey: string) => `${THREAD_KEY_PREFIX}${threadKey
 
 const parentDir = (p: string): string => p.slice(0, p.lastIndexOf("/"));
 
+/** A run's registration, held from its attach to its release (item 44): a
+ *  harness run's process lives in the container BETWEEN the bot's operator
+ *  calls, so the in-memory op counters read 0 while it runs. Durable rows —
+ *  a fresh isolate still counts the process that survived in the container. */
+const RUN_REG_KEY_PREFIX = "runReg:";
+const runRegKey = (threadKey: string) => `${RUN_REG_KEY_PREFIX}${threadKey}`;
+interface RunRegistration {
+  threadKey: string;
+  registeredAt: string;
+}
+
 /** Deterministic per-thread+ref worktree path. Slugs replace anything outside
  *  [A-Za-z0-9._-]; a threadKey-derived hash suffix keeps two keys that slug
  *  identically (e.g. "a:b" vs "a-b") from colliding on disk. */
@@ -1571,6 +1618,52 @@ export class ResidentDO extends Sandbox<Env> {
    *  survives, so hydration state is always probed from disk. */
   private hydration: Promise<void> | null = null;
 
+  /** The container stop the platform delivered to `onStop` and no command has
+   *  answered since: the resident's own knowledge that the container it held
+   *  is gone (docs/reference/specs/resident-repos.md item 43). In memory on
+   *  purpose — a Durable Object that restarts learns the same fact from the
+   *  restore its next command's wake path starts (`restoring`), and a stop
+   *  recorded in storage would outlive the container that answers now. */
+  private containerStop: { at: number; exitCode: number; reason: string } | undefined;
+
+  /** The platform's signal that the container stopped — the rollout's
+   *  `runtime_signal`, or an exit of its own — recorded before the SDK's
+   *  reconciliation runs, so an `/exec` that meets the stopped container from
+   *  here on answers `runtime-replaced` (`replacedExecAnswer`) rather than the
+   *  binding's bare "The container is not running". The Sandbox base declares
+   *  the hook without parameters; the Container base beneath it passes them. */
+  async onStop(params?: { exitCode: number; reason: string }): Promise<void> {
+    this.containerStop = { at: systemClock(), exitCode: params?.exitCode ?? 0, reason: params?.reason ?? "unknown" };
+    console.log(
+      `container: stopped (exit ${this.containerStop.exitCode}, ${this.containerStop.reason}) — the container this resident held is gone; /exec answers runtime-replaced until a command answers again`,
+    );
+    await super.onStop();
+  }
+
+  /** Whether this resident knows the container it held is gone: it saw the
+   *  container stop and no command has answered since, or a restore is under
+   *  way (`restoring`, persisted before any restore work). Never the
+   *  platform's `running` flag — an asleep or starting container answers
+   *  false to it too, and an `/exec` that meets one must say what the SDK
+   *  said so the harness probes rather than relaunch into a container that
+   *  was never replaced. */
+  private async knowsContainerGone(): Promise<boolean> {
+    if (this.containerStop !== undefined) return true;
+    return (await this.getStatus()).state === "restoring";
+  }
+
+  /** The `/exec` answer for a command the SDK failed with a runtime
+   *  replacement (item 43; harness.md item 6): the word — `reason:
+   *  "runtime-replaced"`, the resident's sentence — when the SDK vouched the
+   *  runtime moved or the resident knows the container it held is gone;
+   *  otherwise what the SDK said, with no word and no `reason`, so the
+   *  harness's one more command decides. The command is never re-issued
+   *  either way: it may have started. */
+  private async replacedExecAnswer(err: RuntimeReplacedError): Promise<ThreadErr> {
+    if (sdkVouchesRuntimeMoved(err.cause) || (await this.knowsContainerGone())) return runtimeReplacedErr(err);
+    return { error: errMsg(err.cause), status: 409 };
+  }
+
   /** The step trace of the request in flight (docs/reference/specs/tracing.md item 19):
    *  `attachThread` and `runOp` each run inside their own collector, so the
    *  commands `runOk` runs and the mirror-lock waits land on the answer that
@@ -1957,7 +2050,9 @@ export class ResidentDO extends Sandbox<Env> {
       proc = await createExtensionProcessSandbox(this).exec(argv as unknown as SandboxCommand, launch);
     }
     // The spawn is the proof the control port answers: a persisted count of
-    // unanswered connects ends here, whatever the command goes on to do.
+    // unanswered connects ends here, whatever the command goes on to do — and
+    // a container stop seen before it is history (`knowsContainerGone`).
+    this.containerStop = undefined;
     await this.clearRuntimeUnreachable();
     try {
       const out = await proc.output({ encoding: "utf8", timeout: timeout + 30_000 });
@@ -3099,10 +3194,12 @@ export class ResidentDO extends Sandbox<Env> {
       }
     }
     let settled = entry.state === "warm";
-    if (entry.state === "degraded" && !isNonEvidenceReason(entry.reason)) {
+    if (entry.state === "degraded" && !isNonEvidenceReason(entry.reason) && parksOnRepeat(entry.reason)) {
       // Count consecutive cycles that found the same REFRESH-PRODUCED degraded
-      // reason (github-unreachable, <step>-failed); a stable streak means
-      // retrying is not going to help and parking is the right cost behavior.
+      // reason that is evidence about the repo or GitHub (github-unreachable,
+      // a repo-command step's <step>-failed); a stable streak means retrying
+      // is not going to help and parking is the right cost behavior. A
+      // resident-step failure never parks: item 67's ladder is its escalation.
       // Any other state resets the streak (below).
       const prev = await this.ctx.storage.get<{ reason: string; count: number }>(DEGRADED_STREAK_KEY);
       const streak =
@@ -3317,6 +3414,8 @@ export class ResidentDO extends Sandbox<Env> {
     // "idle since" and every attach would take the wake-fetch path).
     delete updatedFacts.idleSince;
     await this.ctx.storage.put(FACTS_KEY, updatedFacts);
+    // A completed cycle ends any resident-step streak (item 67).
+    await this.ctx.storage.delete(INFRA_STREAK_KEY);
     await this.setResidentState("warm");
     // Event-triggered reclamation: the prune above already told the
     // mirror which branches died; finished refs give their worktree and
@@ -3370,8 +3469,57 @@ export class ResidentDO extends Sandbox<Env> {
   private async refreshFailed(failure: RefreshFailure, selfInFlight: number): Promise<void> {
     console.log(`refresh: cycle failed — ${failure.reason.slice(0, 400)}`);
     await this.recordRefreshError(failure.reason);
-    await this.setResidentState("degraded", failure.reason); // last snapshot keeps serving
-    if (failure.diskFull) await this.recoverFromDiskFull(failure.reason, selfInFlight);
+    if (failure.diskFull) {
+      await this.setResidentState("degraded", failure.reason);
+      await this.recoverFromDiskFull(failure.reason, selfInFlight);
+      return;
+    }
+    // Item 67: a failure in the repository's own command is evidence about the
+    // repo — degraded, and the gate's park streak decides; it also ends any
+    // resident-step streak, since our machinery got that far. A failure in
+    // the resident's own steps climbs the ladder over the persisted count.
+    if (isRepoCommandStep(failure.step)) {
+      await this.ctx.storage.delete(INFRA_STREAK_KEY);
+      await this.setResidentState("degraded", failure.reason);
+      return;
+    }
+    const row = await this.noteInfraStreak(failure.step);
+    switch (infraStreakRung(row.count)) {
+      case "count":
+        await this.setResidentState("degraded", failure.reason);
+        return;
+      case "recreate":
+        // The disk, not the repo: destroy the container, snapshots kept; the
+        // next cycle's wake restores mirror, checkout and deps from R2.
+        await this.recreateContainer(infraStreakReason(row, "recreate"));
+        return;
+      case "down": {
+        // A recreated container failed the same way: the snapshot or the
+        // resident is what is wrong, and a rebuild from GitHub is the exit —
+        // item 36's transition, the VM destroyed first so provisioning starts
+        // on a fresh one.
+        const reason = infraStreakReason(row, "down");
+        this.swapIncarnation(); // deliberate incarnation swap
+        await this.forgetRuntimeIdentity();
+        await this.destroy().catch((err) => console.log(`infra-streak: destroy failed: ${errMsg(err)}`));
+        await this.ctx.storage.delete(INFRA_STREAK_KEY);
+        await this.goDown(reason);
+        return;
+      }
+    }
+  }
+
+  /** One more cycle failed in the resident's own steps: the count up by one,
+   *  `firstAt` kept, the step and `lastAt` now. */
+  private async noteInfraStreak(step: string): Promise<InfraStreakRow> {
+    const prev = await this.ctx.storage.get<InfraStreakRow>(INFRA_STREAK_KEY);
+    const now = new Date(systemClock()).toISOString();
+    const row: InfraStreakRow = { step, count: (prev?.count ?? 0) + 1, firstAt: prev?.firstAt ?? now, lastAt: now };
+    await this.ctx.storage.put(INFRA_STREAK_KEY, row);
+    console.log(
+      `infra-streak: cycle ${row.count} of ${INFRA_STREAK_DOWN_AT} failed in the resident's own step ${step} (first at ${row.firstAt})`,
+    );
+    return row;
   }
 
   // -- the runtime that never answers (docs/reference/specs/resident-repos.md item 64) ------
@@ -3494,7 +3642,7 @@ export class ResidentDO extends Sandbox<Env> {
    *  restore, for the reason `escalateRuntimeUnreachable` gives. */
   private async recreateContainer(reason: string): Promise<void> {
     console.log(
-      `runtime-unreachable: destroying the container — snapshots kept; the next exec restores from R2 (${reason.slice(0, 200)})`,
+      `recreate: destroying the container — snapshots kept; the next exec restores from R2 (${reason.slice(0, 200)})`,
     );
     this.swapIncarnation(); // deliberate incarnation swap
     await this.forgetRuntimeIdentity();
@@ -4687,7 +4835,7 @@ export class ResidentDO extends Sandbox<Env> {
       // container under it (and isIdle never parks the cycle mid-attach).
       this.attachesInFlight++;
       try {
-        return await this.attachThreadBody(
+        const res = await this.attachThreadBody(
           threadKey,
           refHint,
           readonly,
@@ -4698,6 +4846,10 @@ export class ResidentDO extends Sandbox<Env> {
           record,
           reason,
         );
+        // The run this attach opens is now in flight until its release —
+        // whatever its op counters read between the bot's calls (item 44).
+        if (!("error" in res)) await this.registerRun(threadKey);
+        return res;
       } finally {
         this.attachesInFlight--;
       }
@@ -5135,10 +5287,16 @@ export class ResidentDO extends Sandbox<Env> {
         // second movement is decided on. (Under the mirror mutex — the mirror
         // cannot change in between.)
         const refExists = why === null || (await this.refExists(binding.ref));
+        // The tip is read here, once, and is what the target is judged on and
+        // what the worktree is provisioned at: a caller that named a commit
+        // gets that commit or a `stale-tip` refusal, never whatever the tip
+        // happens to be (item 51).
+        let tipSha = refExists ? await this.readMirrorSha(binding.ref).catch(() => null) : null;
         let target = attachTarget({
           refExists,
           wantSha: want,
           commitInMirror: !refExists && want !== null && (await this.commitInMirror(want)),
+          tipSha,
         });
         if (target.kind === "unknown-ref" && returnable) {
           // The thread's branch — one a rebind moved it onto, or one its own
@@ -5151,10 +5309,13 @@ export class ResidentDO extends Sandbox<Env> {
           binding = back.binding;
           returned = back.returned;
           want = wantShaForBinding({ boundRef: binding.ref, refHint, wantSha });
+          const defaultExists = await this.refExists(binding.ref);
+          tipSha = defaultExists ? await this.readMirrorSha(binding.ref).catch(() => null) : null;
           target = attachTarget({
-            refExists: await this.refExists(binding.ref),
+            refExists: defaultExists,
             wantSha: want,
             commitInMirror: false,
+            tipSha,
           });
         }
         if (target.kind === "unknown-ref") {
@@ -5164,7 +5325,19 @@ export class ResidentDO extends Sandbox<Env> {
               (want !== null ? `, and the mirror does not hold the expected commit ${want.slice(0, 7)} either` : ""),
           );
         }
-        const sha = target.kind === "sha" ? target.sha : await this.readMirrorSha(binding.ref);
+        if (target.kind === "stale-tip") {
+          throw new StepError(
+            "stale-tip",
+            `ref ${JSON.stringify(binding.ref)} is at ${target.tip ? target.tip.slice(0, 7) : "an unreadable tip"} in the mirror${why === null ? " (no fetch was due)" : " even after the fetch"}; the caller expects ${target.want.slice(0, 7)} — the run executes at the commit it asked for, or not on this resident`,
+          );
+        }
+        // A tip the read could not answer is never provisioned at: with no
+        // commit named the target is `ref` without looking at the tip, so the
+        // failed read is named here instead of becoming a null sha.
+        const sha = target.kind === "sha" ? target.sha : tipSha;
+        if (sha === null) {
+          throw new StepError("rev-parse", `the mirror's tip of ${JSON.stringify(binding.ref)} could not be read`);
+        }
         const threadLockKey = await this.lockfileKey(sha);
         const recreated = await this.ensureThreadWorktree(binding, sha, mode.originUrl, mode.modeSwitch, {
           detached: target.kind === "sha",
@@ -5186,6 +5359,13 @@ export class ResidentDO extends Sandbox<Env> {
       if (err instanceof StepError && err.step === "unknown-ref") {
         return { error: `unknown-ref: ${err.message}`, status: 400 };
       }
+      if (err instanceof StepError && err.step === "stale-tip") {
+        // Item 51: not a resident fault and not a caller fault — a fact about
+        // the mirror at this instant. 409 with the state, so the bot's named
+        // fallback runs cold at the commit it asked for.
+        const s = await this.getStatus();
+        return { error: `stale-tip: ${err.message}`, status: 409, state: s.state, reason: "stale-tip" };
+      }
       return this.attachFailed(err);
     }
 
@@ -6127,11 +6307,30 @@ export class ResidentDO extends Sandbox<Env> {
     return res;
   }
 
+  /** The exec route's one gate for a runtime replacement (item 43): every
+   *  `RuntimeReplacedError` the route meets — the preflight's own probe of the
+   *  worktree, the command itself — is judged by `replacedExecAnswer`, so the
+   *  word reaches the harness only when the SDK vouched the runtime moved or
+   *  the resident knows the container it held is gone. */
   private async execThreadImpl(
     threadKey: string,
     command: string,
     timeoutMs: number,
     env?: Record<string, string>,
+  ): Promise<{ stdout: string; stderr: string; exitCode: number; truncated: boolean } | ThreadErr> {
+    try {
+      return await this.execThreadBody(threadKey, command, timeoutMs, env);
+    } catch (err) {
+      if (err instanceof RuntimeReplacedError) return this.replacedExecAnswer(err);
+      throw err;
+    }
+  }
+
+  private async execThreadBody(
+    threadKey: string,
+    command: string,
+    timeoutMs: number,
+    env?: Record<string, string>,
   ): Promise<{ stdout: string; stderr: string; exitCode: number; truncated: boolean } | ThreadErr> {
     const pre = await this.threadPreflight(threadKey);
     if ("error" in pre) return pre;
@@ -6147,13 +6346,8 @@ export class ResidentDO extends Sandbox<Env> {
         : {}),
     } satisfies ThreadBinding);
 
-    let r: Awaited<ReturnType<ResidentDO["threadRun"]>>;
-    try {
-      r = await this.threadRunCapped(binding.user, binding.worktreePath, command, timeoutMs, EXEC_OUTPUT_CAP, env);
-    } catch (err) {
-      if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
-      throw err;
-    }
+    // A runtime replacement under the command propagates to `execThreadImpl`'s gate.
+    const r = await this.threadRunCapped(binding.user, binding.worktreePath, command, timeoutMs, EXEC_OUTPUT_CAP, env);
     const truncated = r.stdout.length > EXEC_OUTPUT_CAP || r.stderr.length > EXEC_OUTPUT_CAP || r.truncated === true;
     const notes: string[] = [];
     if (r.timedOut)
@@ -6361,6 +6555,9 @@ export class ResidentDO extends Sandbox<Env> {
       evictedLeftBehind: tree && "leftBehind" in tree ? tree.leftBehind : undefined,
       evictedUnmeasured: tree && "unmeasured" in tree ? tree.unmeasured : undefined,
     } satisfies ThreadBinding);
+    // The run's registration goes with its binding (item 44): released here,
+    // never on a give-way above — a re-attach has re-registered it anyway.
+    await this.ctx.storage.delete(runRegKey(binding.threadKey));
     if (tree) console.log(`${logCtx}: ${binding.threadKey} evicted (${why}) — ${evictedTreeSentence(tree)}`);
     return true;
   }
@@ -6522,10 +6719,33 @@ export class ResidentDO extends Sandbox<Env> {
     return threadOps + this.opUsersInUse.size + this.attachesInFlight;
   }
   /** In-flight activity for the deploy preflight (GET /status, GET /residents).
-   *  In-memory by nature: a fresh isolate answers 0, which is correct — nothing
-   *  survived to be interrupted. */
+   *  The in-memory counters (a fresh isolate answers 0 for them — nothing of
+   *  THEIRS survived to be interrupted) plus the durable run registrations:
+   *  a harness run's process DOES survive an isolate swap in the container,
+   *  so its registration must too. */
   async getInFlightCount(): Promise<number> {
-    return this.inFlightCount();
+    return this.inFlightCount() + (await this.registeredRunsBeyondOps());
+  }
+  /** A run holds its worktree from `/attach` to its release (`/detach`, the
+   *  sweep, disk pressure — every path ends in `evictBinding`, which clears
+   *  the row). Written by the attach; idempotent, a re-attach refreshes it. */
+  private async registerRun(threadKey: string): Promise<void> {
+    await this.ctx.storage.put(runRegKey(threadKey), {
+      threadKey,
+      registeredAt: new Date(systemClock()).toISOString(),
+    } satisfies RunRegistration);
+  }
+  /** The registrations the op counters do not already see: a thread with an
+   *  op in flight is counted by `runsInFlightCount`, so its registration is
+   *  not counted again. Deliberately NOT part of `inFlightCount()`: the
+   *  container-lifecycle predicates (isIdle, reconcileImage) must not let a
+   *  stale registration pin a container awake — the clean-idle sweep is what
+   *  drains a registration whose release never came. */
+  private async registeredRunsBeyondOps(): Promise<number> {
+    const regs = await this.ctx.storage.list<RunRegistration>({ prefix: RUN_REG_KEY_PREFIX });
+    let n = 0;
+    for (const r of regs.values()) if ((this.threadOpsInFlight.get(r.threadKey) ?? 0) === 0) n++;
+    return n;
   }
   private attachesInFlight = 0;
   /** A refresh cycle past its idle/reconcile gates (fetching, rebuilding, snapshotting). */
@@ -7068,6 +7288,7 @@ export class ResidentDO extends Sandbox<Env> {
       REFRESH_INSTANCE_KEY,
       RUNTIME_UNREACHABLE_KEY,
       AUTO_REBUILDS_KEY,
+      INFRA_STREAK_KEY,
     ]);
     const facts = map.get(FACTS_KEY) as RepoFacts | undefined;
     const snap = map.get(SNAPSHOT_KEY) as SnapshotRecord | undefined;
@@ -7120,6 +7341,9 @@ export class ResidentDO extends Sandbox<Env> {
           recycledUnmeasured: recycledUnmeasured ?? null,
         }),
       );
+    // The runs whose process lives in the container between operator calls
+    // (item 44): what the deploy preflight must see, added to both counts.
+    const registeredRuns = await this.registeredRunsBeyondOps();
     return {
       resource: map.get(RESOURCE_KEY) ?? null,
       state: map.get(STATE_KEY) ?? "down",
@@ -7139,9 +7363,9 @@ export class ResidentDO extends Sandbox<Env> {
         provisionRun: provisionRun.length,
         provisionDeadline: provisionDeadline.length,
       },
-      inFlight: this.inFlightCount(),
+      inFlight: this.inFlightCount() + registeredRuns,
       // The runs alone (no refresh cycle): what the deploy preflight refuses on.
-      runsInFlight: this.runsInFlightCount(),
+      runsInFlight: this.runsInFlightCount() + registeredRuns,
       // Item 22: who holds what, as the rows say — the mirror mutex and the
       // cycle/hydration leases, each judged against this incarnation.
       incarnation: this.incarnation,
@@ -7173,6 +7397,12 @@ export class ResidentDO extends Sandbox<Env> {
       // Item 36: the auto-rebuilds this resident has had (ISO instants; the
       // budget's window is judged over them). Empty after a person's rebuild.
       autoRebuilds: (map.get(AUTO_REBUILDS_KEY) as string[] | undefined) ?? [],
+      // Item 67: consecutive cycles failed in the resident's own steps, with
+      // the rung that count is on; null while cycles complete.
+      infraStreak: (() => {
+        const row = map.get(INFRA_STREAK_KEY) as InfraStreakRow | undefined;
+        return row ? { ...row, rung: infraStreakRung(row.count) } : null;
+      })(),
     };
   }
 
@@ -7255,6 +7485,37 @@ export class ResidentDO extends Sandbox<Env> {
     return { recreated: true, restoreStartedAt };
   }
 
+  /** Fault injection for item 67: write the resident-step streak row at `count`
+   *  (0 deletes it), so the next real failure lands on the rung under test.
+   *  Test-only semantics; admin scope. */
+  async debugSetInfraStreak(count: number): Promise<{ infraStreak: (InfraStreakRow & { rung: string }) | null }> {
+    if (count === 0) {
+      await this.ctx.storage.delete(INFRA_STREAK_KEY);
+      return { infraStreak: null };
+    }
+    const now = new Date(systemClock()).toISOString();
+    const row: InfraStreakRow = { step: "injected", count, firstAt: now, lastAt: now };
+    await this.ctx.storage.put(INFRA_STREAK_KEY, row);
+    return { infraStreak: { ...row, rung: infraStreakRung(count) } };
+  }
+
+  /** Fault injection for item 67: corrupt the mirror on disk so the next
+   *  cycle's `fetch` fails in a resident step. The mirror's `config` goes (its
+   *  `origin` remote with it: `'origin' does not appear to be a git
+   *  repository`, exit 128) — NOT its objects: the wake check (`readyStamp`)
+   *  tests `objects/`, and a missing directory is a disk to restore, which the
+   *  wake did before any step could fail the first time this was tried live.
+   *  The snapshots in R2 are untouched — the ladder's recreate restores them.
+   *  Test-only semantics; admin scope. */
+  async debugBreakMirror(): Promise<{ broken: boolean; error?: string }> {
+    try {
+      await this.runOk(["rm", "-f", `${MIRROR_DIR}/config`], "break-mirror");
+      return { broken: true };
+    } catch (err) {
+      return { broken: false, error: errMsg(err) };
+    }
+  }
+
   /** Fault injection for item 36 without corrupting real R2 objects. Bare: persist
    *  `down` with a rehydration-flavored reason and nothing else, so the next
    *  watchdog pass is the backstop under test. `transition`: go through
@@ -8644,6 +8905,19 @@ async function handleDebug(env: Env, body: Record<string, unknown>): Promise<Res
     }
     case "force-onboarding":
       return json(await stub.debugForceOnboarding());
+    case "infra-streak": {
+      // Fault injection for item 67: set the resident-step streak's count, so
+      // one real failure lands on the rung under test (recreate at 3, down at 5).
+      const count =
+        typeof body.count === "number" && Number.isInteger(body.count) && body.count >= 0 ? body.count : null;
+      if (count === null) return json({ error: "count must be a non-negative integer" }, 400);
+      return json(await stub.debugSetInfraStreak(count));
+    }
+    case "break-mirror":
+      // Fault injection for item 67: remove the mirror's objects, so the next
+      // cycle's fetch fails in a resident step — a corrupt disk, which the
+      // ladder's recreate restores from the snapshot. Test residents only.
+      return json(await stub.debugBreakMirror());
     case "force-down": {
       // Fault injection for item 36; a rehydration-flavored default reason
       // makes it eligible. `transition: true` fires the down through goDown

package/dist/assets/deploy/cloudflare-sandbox/worker.ts

@@ -87,8 +87,10 @@ import {
   SEED_ABANDONED_RESTORE_WAIT_MS,
   seedFixupScript,
   seedMarkerText,
+  type RestorePhases,
   type SandboxSeed,
   type SeedAnswer,
+  type SeedPhases,
   type SeedStep,
 } from "../../src/execution/seedPlan.js";
 import { shellQuote } from "../../src/execution/shellQuote.js";
@@ -450,12 +452,13 @@ export class SwitchboardSandbox extends Sandbox<Env> {
       // marker for another handle.
       await this.runRoot(["sh", "-c", `${unmountAllRestoresScript()}\n${this.seedSweep()}`], 60_000);
       let t = systemClock();
-      await this.restoreSeedInto(seed.checkoutBackupId, SEED_CHECKOUT_DIR, deadline, "checkout");
+      const checkout = await this.restoreSeedInto(seed.checkoutBackupId, SEED_CHECKOUT_DIR, deadline, "checkout");
       steps.restore = systemClock() - t;
+      const phases: SeedPhases = { checkout, deps: null };
       if (seed.depsBackupId) {
         step = "deps";
         t = systemClock();
-        await this.restoreSeedInto(seed.depsBackupId, SEED_DEPS_STAGING_DIR, deadline, "deps");
+        phases.deps = await this.restoreSeedInto(seed.depsBackupId, SEED_DEPS_STAGING_DIR, deadline, "deps");
         steps.deps = systemClock() - t;
       }
       step = "fixup";
@@ -480,9 +483,27 @@ export class SwitchboardSandbox extends Sandbox<Env> {
       );
       const ms = systemClock() - t0;
       console.log(
-        JSON.stringify({ event: "sandbox.seeded", slug: seed.slug, ref: seed.fetchRef ?? seed.ref, sha, steps, ms }),
+        JSON.stringify({
+          event: "sandbox.seeded",
+          slug: seed.slug,
+          ref: seed.fetchRef ?? seed.ref,
+          sha,
+          steps,
+          phases,
+          ms,
+        }),
       );
-      return { seeded: true, cached: false, slug: seed.slug, ref: seed.fetchRef ?? seed.ref, sha, from, steps, ms };
+      return {
+        seeded: true,
+        cached: false,
+        slug: seed.slug,
+        ref: seed.fetchRef ?? seed.ref,
+        sha,
+        from,
+        steps,
+        phases,
+        ms,
+      };
     } catch (err) {
       const shape = thrownShape(err);
       // A half seed never survives either: the run that follows goes cold
@@ -510,7 +531,12 @@ export class SwitchboardSandbox extends Sandbox<Env> {
    *  a staging sibling, the wait is judged by bytes arriving (the SDK's call
    *  takes no timeout) against the seed's one deadline, then the extract
    *  script puts a real tree in place and the target appears last. */
-  private async restoreSeedInto(id: string, targetDir: string, deadlineMs: number, what: string): Promise<void> {
+  private async restoreSeedInto(
+    id: string,
+    targetDir: string,
+    deadlineMs: number,
+    what: string,
+  ): Promise<RestorePhases> {
     const attempt = crypto.randomUUID().slice(0, 8);
     const mountDir = restoreMountDir(targetDir, attempt);
     const backup: DirectoryBackup = { id, dir: mountDir };
@@ -535,11 +561,18 @@ export class SwitchboardSandbox extends Sandbox<Env> {
       const verdict = judgeRestoreProgress({ startedMs, nowMs: systemClock(), samples, deadlineMs });
       if (verdict.verdict !== "wait") throw new Error(`${what} restore ${verdict.verdict}: ${verdict.detail}`);
     }
+    const download = systemClock() - startedMs;
+    const extractStartedMs = systemClock();
     const r = await this.runRoot(
       ["sh", "-c", extractRestoreScript({ mountDir, backupId: id, archivePath: restoreArchivePath(id), targetDir })],
       Math.max(60_000, deadlineMs - systemClock()),
     );
     if (r.exitCode !== 0) throw new Error(`${what} extract exited ${r.exitCode}: ${tail(r.stderr || r.stdout)}`);
+    const extract = systemClock() - extractStartedMs;
+    console.log(
+      JSON.stringify({ event: "sandbox.seed-restore", what, id, download, extract, method: r.stdout.trim() }),
+    );
+    return { download, extract };
   }
 
   /** Restores this object started that have not settled: what a failure

package/dist/assets/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "switchboard",
-  "version": "1.241.0",
+  "version": "1.243.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "switchboard",
-      "version": "1.241.0",
+      "version": "1.243.0",
       "license": "Apache-2.0",
       "workspaces": [
         "web",
@@ -20445,7 +20445,7 @@
     },
     "packages/switchboard": {
       "name": "@coreplane/switchboard",
-      "version": "1.241.0",
+      "version": "1.243.0",
       "license": "Apache-2.0",
       "dependencies": {
         "@earendil-works/pi-ai": "0.85.1",

package/dist/assets/package.json

@@ -1,6 +1,6 @@
 {
   "name": "switchboard",
-  "version": "1.241.0",
+  "version": "1.243.0",
   "private": true,
   "description": "Mention it in Slack and an agent reviews the PR, ships the fix, or answers the question — on the model you choose, with its tools running where you decide.",
   "license": "Apache-2.0",