@coreplane/switchboard 1.254.1 → 1.256.0

package/dist/assets/deploy/cloudflare-resident/worker.ts

@@ -85,6 +85,16 @@ import { DurableObject } from "cloudflare:workers";
 import { BASH_TIMEOUT_MAX_MS, clampBashTimeout } from "../../src/execution/bashTimeout.js";
 import { selectBindingsToPurge } from "../../src/execution/bindingPurge.js";
 import { busyAfterKillReason, planForceDetach } from "../../src/execution/residentDetach.js";
+import {
+  levelPosts,
+  memorySide,
+  mergeOutbox,
+  seatSide,
+  type LevelPost,
+  type LevelSample,
+  type LevelSide,
+  type ResidentLevelsDoc,
+} from "./levels.js";
 import { parseReadonly, planReadonlyAttach } from "../../src/execution/residentReadonly.js";
 import { decideWorktree, parseReuse, type WorktreeFacts } from "../../src/execution/residentReuse.js";
 import {
@@ -333,7 +343,15 @@ import {
 } from "../../src/execution/residentDepsStore.js";
 import { buildId, injectedBuildStamp } from "../../src/deploy/buildStamp.js";
 import { createRefreshInstance, createRefreshInstanceNow, type RefreshInstanceParams } from "./refresh";
-import { drainRefusal, liveDrain, parseDrainRequest, type DrainRecord } from "./drain";
+import {
+  drainRefusal,
+  holdDrain,
+  liftDrain,
+  liveDrain,
+  parseDrainRequest,
+  reportImageCurrent,
+  type DrainRecord,
+} from "./drain";
 import {
   CGROUP_READ_ARGV,
   MEMORY_PRESSURE_REASON,
@@ -636,6 +654,12 @@ const DISK_KEY = "resident:disk";
  *  tick, persisted so the gauges (`/status`, `/residents`, the watchdog line)
  *  read storage only — the watchdog never touches the container. */
 const MEMORY_KEY = "resident:memory";
+/** The last level sample and the crossing outbox (record 0064; `levels.ts`). */
+const LEVELS_LAST_KEY = "resident:levels:last";
+const LEVELS_OUTBOX_KEY = "resident:levels:outbox";
+/** The registry's drain post (record 0064): the fleet drain's set/cleared/expired,
+ *  re-offered on every registry answer until superseded. */
+const DRAIN_OUTBOX_KEY = "drain:outbox";
 /** The lifecycle row (docs/reference/specs/resident-repos.md item 7): `workflow`,
  *  the one scheduler. Kept from the flagged rollout so `/status` and `/debug
  *  info` can say so; an `alarm` value a flip left behind reads `workflow`
@@ -1193,6 +1217,13 @@ interface ThreadBinding {
   depsKey?: string;
 }
 
+/** What one image reconcile decided (`reconcileImage`): the container was
+ *  stopped to restart on the current image (`restarted`), already runs it
+ *  (`current`), is not running so the next start uses it anyway (`inactive`),
+ *  or is busy — an operation, an attach or a registered run in flight — and
+ *  the restart is deferred to the next quiet check (`deferred`). */
+type ImageReconcileResult = "restarted" | "current" | "inactive" | "deferred";
+
 interface AttachOk {
   workspace: string;
   ref: string;
@@ -1411,6 +1442,10 @@ const TEST_OVERRIDES_KEY = "testOverrides";
  *  counts as a slot; it survives the isolate swap a deploy performs, which is
  *  why the record carries its own end. */
 const DRAIN_KEY = "drain";
+/** Resident-DO key (issue 1931): set when a deploy's reconcile could not
+ *  verify this resident's container on the new image; the next reconcile that
+ *  finds it current reports to the registry and clears it. */
+const IMAGE_REPORT_PENDING_KEY = "imageReportPending";
 
 type OnboardResult = { ok: true; record: ResidentRecord } | { ok: false; status: number; error: string };
 
@@ -1535,15 +1570,94 @@ export class ResidentRegistryDO extends DurableObject<Env> {
   }
 
   /** Admin-only by construction (reached solely via POST /drain): replaces
-   *  whatever drain stood — a second deploy's drain extends the first's. */
+   *  whatever drain stood — a second deploy's drain extends the first's. The
+   *  set posts `above` for the plane (record 0064) and arms ONE alarm at
+   *  `until`, so a drain nobody lifts posts its expiry itself. */
   async setDrain(record: DrainRecord): Promise<DrainRecord> {
     await this.ctx.storage.put(DRAIN_KEY, record);
+    await this.pushDrainPost("above");
+    await this.ctx.storage.setAlarm(Date.parse(record.until));
     return record;
   }
 
-  /** Admin-only by construction (POST /undrain): true when a record was there. */
-  async clearDrain(): Promise<boolean> {
-    return this.ctx.storage.delete(DRAIN_KEY);
+  /** Admin-only by construction (POST /undrain): `cleared` when a record was
+   *  there and was lifted — a cleared drain posts `below`, the plane's
+   *  resident-drain window lifts. With holds outstanding (issue 1931: a
+   *  container still to report the deploy's image) the fleet STAYS closed:
+   *  the record stands with `liftAsked` and the last container's report lifts
+   *  it (`reportContainerImageCurrent`); `until` remains the backstop. */
+  async clearDrain(): Promise<{ cleared: boolean; held: string[] }> {
+    const record = liveDrain(await this.ctx.storage.get(DRAIN_KEY), systemClock());
+    if (record !== null) {
+      const lift = liftDrain(record);
+      if (!lift.cleared) {
+        await this.ctx.storage.put(DRAIN_KEY, lift.record);
+        return { cleared: false, held: lift.record.holds ?? [] };
+      }
+    }
+    const had = await this.ctx.storage.delete(DRAIN_KEY);
+    if (had) await this.pushDrainPost("below");
+    return { cleared: had, held: [] };
+  }
+
+  /** The deploy's reconcile could not verify these residents' containers on
+   *  the new image (issue 1931): hold the drain for each — the fleet must not
+   *  reopen onto them until they report. No live drain, nothing to hold. */
+  async holdDrainFor(resources: string[]): Promise<void> {
+    const record = liveDrain(await this.ctx.storage.get(DRAIN_KEY), systemClock());
+    if (record === null || resources.length === 0) return;
+    await this.ctx.storage.put(DRAIN_KEY, holdDrain(record, resources));
+  }
+
+  /** One resident's word that its running container is on the deploy's image:
+   *  its hold drops, and when it was the last hold of a lift already asked the
+   *  drain lifts here — the reopen fires on the last container's report. */
+  async reportContainerImageCurrent(resource: string): Promise<{ lifted: boolean }> {
+    const record = liveDrain(await this.ctx.storage.get(DRAIN_KEY), systemClock());
+    if (record === null) return { lifted: false };
+    const report = reportImageCurrent(record, resource);
+    if (report.record === null) {
+      await this.ctx.storage.delete(DRAIN_KEY);
+      await this.pushDrainPost("below");
+      console.log(`[drain] fleet reopened — ${resource} was the last container to report the deploy's image`);
+      return { lifted: true };
+    }
+    await this.ctx.storage.put(DRAIN_KEY, report.record);
+    return { lifted: false };
+  }
+
+  /** The one alarm, at the drain's `until` (record 0064): a drain past
+   *  its end is nothing (`liveDrain` already reads it so), and the expiry is
+   *  posted `below` like a clear — whoever forgot the drain, the plane's
+   *  window lifts. A drain replaced with a later `until` re-arms via setDrain. */
+  async alarm(): Promise<void> {
+    const now = systemClock();
+    const stored = await this.ctx.storage.get(DRAIN_KEY);
+    if (stored === undefined) return;
+    if (liveDrain(stored, now) === null) {
+      await this.ctx.storage.delete(DRAIN_KEY);
+      await this.pushDrainPost("below");
+    } else {
+      // Replaced with a later end under an already-armed alarm: re-arm at it.
+      await this.ctx.storage.setAlarm(Date.parse((stored as DrainRecord).until));
+    }
+  }
+
+  /** The drain's plane post (name `drain`), superseding the last: the plane
+   *  only needs the current side, and the bot's forward is idempotent. */
+  private async pushDrainPost(side: LevelSide): Promise<void> {
+    await this.ctx.storage.put(DRAIN_OUTBOX_KEY, {
+      name: "drain",
+      side,
+      generation: "",
+      at: new Date(systemClock()).toISOString(),
+    });
+  }
+
+  /** The pending drain post, for the registry's answers (`/drain`, `/undrain`,
+   *  `/residents`) — the bot forwards it to `POST /plane/level`. */
+  async getDrainOutbox(): Promise<LevelPost | null> {
+    return ((await this.ctx.storage.get(DRAIN_OUTBOX_KEY)) as LevelPost | undefined) ?? null;
   }
 }
 
@@ -3366,7 +3480,7 @@ export class ResidentDO extends Sandbox<Env> {
     // RUNNING container keeps the old one, so new Worker code can name pool
     // users the image lacks. Reconcile here (every cycle, cheap) — see
     // reconcileImage — so a rollout self-applies within one refresh.
-    if (await this.reconcileImage("refresh")) {
+    if ((await this.reconcileImage("refresh")) === "restarted") {
       // Container stopping; it restarts on the new image in seconds. The
       // engine's retry re-enters this step thirty seconds on and re-warms the
       // resident within the minute, instead of the next bucket.
@@ -4592,6 +4706,37 @@ export class ResidentDO extends Sandbox<Env> {
     return this.memoryGuard.lastReading ?? (await this.ctx.storage.get<MemoryReading>(MEMORY_KEY)) ?? null;
   }
 
+  /** The resident's levels (record 0064; `levels.ts`): the seat (the
+   *  thread/op user pool, whose exhaustion is the `user-pool-exhausted`
+   *  refusal) and the memory line (the gate's soft side), stamped with this
+   *  incarnation — the container's boot id where one is memoized, else the
+   *  DO incarnation, so a boot or a replaced runtime reads as a new
+   *  generation and re-states both levels. A crossing lands in the outbox
+   *  and is re-offered on every answer until a newer crossing of the same
+   *  name supersedes it; the bot forwards posts to `POST /plane/level`.
+   *  Storage reads and one storage write only — never a container touch, so
+   *  every `/attach`, `/exec` and `/status` answer can carry the document. */
+  async residentLevels(): Promise<ResidentLevelsDoc> {
+    const at = new Date(systemClock()).toISOString();
+    const all = await this.ctx.storage.list<ThreadBinding>({ prefix: THREAD_KEY_PREFIX });
+    const used = new Set([...all.values()].filter((b) => !b.evicted && b.user).map((b) => b.user));
+    for (const u of this.opUsersInUse) used.add(u);
+    const reading = await this.memoryGauge();
+    const generation = this.containerIdMemo ?? this.incarnation;
+    const seat = { side: seatSide(used.size, THREAD_USERS.length), used: used.size, total: THREAD_USERS.length };
+    const memory = { side: memorySide(reading?.percent ?? null), percent: reading?.percent ?? null };
+    const next: LevelSample = { seat: seat.side, memory: memory.side, generation };
+    const prev = (await this.ctx.storage.get<LevelSample>(LEVELS_LAST_KEY)) ?? null;
+    const posts = levelPosts(prev, next, at);
+    let outbox = (await this.ctx.storage.get<LevelPost[]>(LEVELS_OUTBOX_KEY)) ?? [];
+    if (posts.length > 0) {
+      outbox = mergeOutbox(outbox, posts);
+      await this.ctx.storage.put(LEVELS_OUTBOX_KEY, outbox);
+      await this.ctx.storage.put(LEVELS_LAST_KEY, next);
+    }
+    return { seat, memory, generation, at, posts: outbox };
+  }
+
   /** The route gate (item 70): one fresh sample, then the pure verdict over
    *  the last reading. A refusal is the same 503 shape as `mirror-busy`, so
    *  the bot falls back or waits legibly — and a command already running is
@@ -4831,27 +4976,131 @@ export class ResidentDO extends Sandbox<Env> {
   /** Pool users live in the IMAGE (Dockerfile useradd loop) while THREAD_USERS
    *  lives in the Worker. After a deploy that grows the pool, a still-running
    *  container lacks the new users and `install -o workerN` fails. Check the
-   *  last pool user exists; if not and nothing is in flight, stop the container
-   *  so it restarts on the current image (state is DO storage + R2 — the
-   *  disk is a cache). Returns true when a stop was issued. */
-  private async reconcileImage(where: string): Promise<boolean> {
-    if (!(await this.isRuntimeActive().catch(() => false))) return false;
+   *  last pool user exists; if not and nothing is in flight — the in-memory
+   *  counters AND the durable run registrations (item 44): a harness run's
+   *  process lives in the container between the bot's operator calls, so a
+   *  restart decided on the op counters alone stops the container under a live
+   *  run — stop the container so it restarts on the current image (state is DO
+   *  storage + R2 — the disk is a cache). A deferred restart re-checks on
+   *  every later attach and refresh cycle until the resident is quiet; a
+   *  registration whose release never came defers it only until the clean-idle
+   *  sweep drains that registration. Answers `restarted` when a stop was
+   *  issued, else why not. */
+  private async reconcileImage(where: string, force = false): Promise<ImageReconcileResult> {
+    if (!(await this.isRuntimeActive().catch(() => false))) {
+      // An inactive container's next start is on the deployed image by
+      // construction — and a held resident whose container idled out has no
+      // other reporter left: new attaches are refused by the very drain the
+      // hold keeps standing, so without this report the fleet stays closed
+      // until the drain's `until` backstop (issue 1931).
+      await this.reportPendingImageCurrent(where);
+      return "inactive";
+    }
     const last = THREAD_USERS[THREAD_USERS.length - 1];
-    const probe = await this.run(["id", "-u", last]);
-    if (probe.exitCode === 0) return false;
+    // A pending report marker (issue 1931) means a deploy could not verify this
+    // container on its image: the probe shortcut below is a POOL-USER check the
+    // pre-deploy image passes when the pool did not change, so it can never
+    // satisfy the marker — the container is treated as stale until it is cycled
+    // post-deploy and the fresh start reported.
+    const pending = await this.ctx.storage.get<{ resource: string }>(IMAGE_REPORT_PENDING_KEY);
+    if (!force && pending === undefined) {
+      const probe = await this.run(["id", "-u", last]);
+      if (probe.exitCode === 0) return "current";
+    }
+    const stale = force
+      ? "the container predates the deploy"
+      : pending !== undefined
+        ? "a deploy's new-image report is pending, so the running container cannot be trusted current"
+        : `${last} missing in the running container`;
     const busy = this.inFlightCount();
     if (busy > 0) {
+      console.log(`image-stale (${where}): ${stale} but ${busy} operation(s)/attach(es) in flight — deferring restart`);
+      return "deferred";
+    }
+    const registered = await this.registeredRunsBeyondOps();
+    if (registered > 0) {
       console.log(
-        `image-stale (${where}): ${last} missing but ${busy} operation(s)/attach(es) in flight — deferring restart`,
+        `image-stale (${where}): ${stale} but ${registered} run registration(s) live — deferring restart until the resident is quiet`,
       );
-      return false;
+      return "deferred";
     }
-    console.log(
-      `image-stale (${where}): ${last} missing in the running container — stopping so it restarts on the current image`,
-    );
+    console.log(`image-stale (${where}): ${stale} — stopping so it restarts on the current image`);
     this.swapIncarnation(); // deliberate incarnation swap
     await this.stop().catch((err) => console.log(`image-stale: stop failed: ${errMsg(err)}`));
-    return true;
+    if (pending !== undefined) {
+      // The post-deploy cycle just happened: probe the FRESH container — the
+      // SDK boots it for this command on the image the deployed Worker pins,
+      // so a fresh start answering after the deploy is the verified fact
+      // (issue 1931), never the pool-user probe on the old process. A failed
+      // probe keeps the marker: the next reconcile tries the cycle again.
+      const fresh = await this.run(["id", "-u", last])
+        .then((p) => p.exitCode === 0)
+        .catch(() => false);
+      if (fresh) await this.reportPendingImageCurrent(where);
+      else
+        console.log(`image-stale (${where}): the fresh container's probe failed — the new-image report stays pending`);
+    }
+    return "restarted";
+  }
+
+  /** The report a held drain waits for (issue 1931): when the deploy's
+   *  reconcile could not verify this resident's fresh container on the new
+   *  image, a marker stays in storage and the registry holds the drain. The
+   *  fact the report stands on is a fresh container start AFTER the deploy —
+   *  the marker makes `reconcileImage` treat the container as stale until the
+   *  cycle happens, so this is called only behind that fresh start (or on an
+   *  inactive resident, whose next start is on the new image by construction).
+   *  The marker is deleted only after a successful report: a transient failure
+   *  keeps it for the next reconcile, and the drain's `until` is the backstop
+   *  for a report that never lands. */
+  private async reportPendingImageCurrent(where: string): Promise<boolean> {
+    const pending = await this.ctx.storage.get<{ resource: string }>(IMAGE_REPORT_PENDING_KEY);
+    if (pending === undefined) return true;
+    try {
+      const { lifted } = await registryStub(this.env).reportContainerImageCurrent(pending.resource);
+      await this.ctx.storage.delete(IMAGE_REPORT_PENDING_KEY);
+      console.log(
+        `[reconcile] ${pending.resource} reports the deploy's image (${where})${lifted ? " — drain lifted" : ""}`,
+      );
+      return true;
+    } catch (err) {
+      console.log(
+        `[reconcile] ${pending.resource}: the new-image report failed (${where}): ${errMsg(err)} — retried on the next reconcile`,
+      );
+      return false;
+    }
+  }
+
+  /** The deploy's reconcile, inside the drain window (item 69's order: the
+   *  runs in flight end, the swap lands, the containers reconcile, the fleet
+   *  reopens): `POST /reconcile` calls this on every resident after the Worker
+   *  deploy landed and BEFORE the drain is lifted. The pool-user probe cannot
+   *  tell the pre-deploy image from the new one when the pool did not change,
+   *  and the platform replaces the container processes asynchronously after an
+   *  image-changing deploy (issue 1931) — so this path never trusts "current":
+   *  an active, quiet container is always cycled, and the fresh start —
+   *  necessarily on the deploy's image, which landed before this call — is
+   *  probed. `verified` true is the fact the reopen may stand on; anything
+   *  else (deferred, a failed fresh probe) leaves a hold on the drain, cleared
+   *  by this resident's later report (`reportPendingImageCurrent`), never by a
+   *  timer.
+   *
+   *  The hold lands BEFORE the marker: the marker is what lets any concurrent
+   *  attach/refresh reconcile report, and a report that reaches the registry
+   *  before the hold is a no-op — the marker would be consumed and the hold
+   *  added afterwards would wait for a report nothing sends (until the drain's
+   *  `until`). Held first, every report finds its hold. */
+  async reconcileForDeploy(resource: string): Promise<{ result: ImageReconcileResult; verified: boolean }> {
+    await registryStub(this.env).holdDrainFor([resource]);
+    await this.ctx.storage.put(IMAGE_REPORT_PENDING_KEY, { resource });
+    const result = await this.reconcileImage("deploy", true);
+    if (result === "deferred") return { result, verified: false };
+    // Inactive: the inactive early-return reported (the next start is on the
+    // new image by construction). Restarted: the cycle's fresh probe and
+    // report ran inside reconcileImage. Either way the marker gone is the
+    // verification.
+    const verified = (await this.ctx.storage.get(IMAGE_REPORT_PENDING_KEY)) === undefined;
+    return { result, verified };
   }
 
   // -- watchdog (the sparse cron; it re-arms nothing) --------------------------
@@ -5193,7 +5442,7 @@ export class ResidentDO extends Sandbox<Env> {
       const memory = await this.memoryGate("attach", registered);
       if (memory) return memory;
       const resourceId = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
-      if (await this.reconcileImage("attach")) {
+      if ((await this.reconcileImage("attach")) === "restarted") {
         return {
           error: "image-stale: the container predates the current pool and is restarting; retry shortly",
           status: 503,
@@ -8459,6 +8708,7 @@ const ROUTES: Record<string, { scope: Scope; method: string }> = {
   "/rebuild": { scope: "admin", method: "POST" },
   "/drain": { scope: "drain", method: "POST" }, // close the fleet to new runs for a deploy (item 69; admin implied)
   "/undrain": { scope: "drain", method: "POST" }, // reopen it
+  "/reconcile": { scope: "drain", method: "POST" }, // reconcile every container onto the current image, inside the drain window
   "/residents": { scope: "read", method: "GET" }, // admin implied; read-only bearer allowed
   "/debug": { scope: "read", method: "POST" }, // per-op: READ_DEBUG_OPS for read scope, everything for admin
   "/status": { scope: "operator", method: "GET" },
@@ -8563,6 +8813,8 @@ export default {
             return await handleDrain(env, body);
           case "/undrain":
             return await handleUndrain(env);
+          case "/reconcile":
+            return await handleReconcile(env);
           case "/residents":
             return await handleResidents(env);
           case "/debug": {
@@ -8993,15 +9245,52 @@ async function handleDrain(env: Env, body: Record<string, unknown>): Promise<Res
   if (!parsed.ok) return json({ error: parsed.error }, 400);
   const record = await registryStub(env).setDrain(parsed.record);
   console.log(`[drain] fleet closed to new runs by ${record.by} for ${record.reason}: until ${record.until}`);
-  return json({ draining: record });
+  return json({ draining: record, planeOutbox: await registryStub(env).getDrainOutbox() });
 }
 
-/** POST /undrain (admin): reopen the fleet. Idempotent — `cleared` says whether
- *  a drain stood. */
+/** POST /undrain (admin): reopen the fleet. Idempotent — `cleared` says
+ *  whether a drain stood and was lifted. With containers still to report the
+ *  deploy's image (issue 1931) the fleet STAYS closed: `held` names them, the
+ *  record stands with the lift asked, and the last container's report reopens
+ *  the fleet — a fact, never a timer; the drain's `until` is the backstop. */
 async function handleUndrain(env: Env): Promise<Response> {
-  const cleared = await registryStub(env).clearDrain();
-  console.log(`[drain] fleet reopened (${cleared ? "a drain stood" : "no drain stood"})`);
-  return json({ draining: null, cleared });
+  const lift = await registryStub(env).clearDrain();
+  if (!lift.cleared && lift.held.length > 0) {
+    console.log(`[drain] fleet stays closed — containers still to report the deploy's image: ${lift.held.join(", ")}`);
+    const draining = liveDrain(await registryStub(env).getDrain(), systemClock());
+    return json({ draining, cleared: false, held: lift.held, planeOutbox: await registryStub(env).getDrainOutbox() });
+  }
+  console.log(`[drain] fleet reopened (${lift.cleared ? "a drain stood" : "no drain stood"})`);
+  return json({ draining: null, cleared: lift.cleared, planeOutbox: await registryStub(env).getDrainOutbox() });
+}
+
+/** POST /reconcile (drain scope): reconcile every resident's container onto
+ *  the current image — the deploy runner posts it after its Worker deploy
+ *  landed and BEFORE its `/undrain`, so a stale container restarts inside the
+ *  drain window (item 69's order) and never under a run the reopened fleet
+ *  admits. Each resident answers what its reconcile decided and whether its
+ *  fresh container was VERIFIED on the deploy's image (issue 1931); one not
+ *  verified — deferred, a failed fresh probe, an error — leaves a hold on the
+ *  drain, so the fleet reopens only on that container's later report. */
+async function handleReconcile(env: Env): Promise<Response> {
+  const residents = await registryStub(env).list();
+  const settled = await Promise.allSettled(
+    residents.map((record) => residentStub(env, record.resource).reconcileForDeploy(record.resource)),
+  );
+  const reconciled = residents.map((record, i) => {
+    const s = settled[i];
+    return s.status === "fulfilled"
+      ? { resource: record.resource, result: s.value.result, verified: s.value.verified }
+      : { resource: record.resource, result: "error" as const, verified: false, error: errMsg(s.reason) };
+  });
+  // Each resident held ITSELF before its marker (reconcileForDeploy), so no
+  // report can outrun its hold; this pass is the backstop for a resident whose
+  // call rejected before it could — holdDrain deduplicates, so re-holding a
+  // deferred resident changes nothing.
+  const unverified = reconciled.filter((r) => !r.verified).map((r) => r.resource);
+  if (unverified.length > 0) await registryStub(env).holdDrainFor(unverified);
+  console.log(`[reconcile] deploy image reconcile: ${JSON.stringify(reconciled)}`);
+  return json({ reconciled });
 }
 
 async function handleResidents(env: Env): Promise<Response> {
@@ -9060,13 +9349,16 @@ async function handleStatus(env: Env, url: URL): Promise<Response> {
   // deploy gate reads /residents. The registry check rides in the same flight
   // (its 404 is judged first, the probes' results discarded then).
   const stub = residentStub(env, resource.resource);
-  const [record, status, inFlight, refresh, snapshot, memory] = await Promise.all([
+  const [record, status, inFlight, refresh, snapshot, memory, levels] = await Promise.all([
     registryStub(env).getRecord(resource.resource),
     stub.getStatus(),
     stub.getInFlightCount(),
     stub.getRefreshView(),
     stub.snapshotHandle(),
     stub.memoryGauge(),
+    // The levels (record 0064): what the bot forwards to the plane; the
+    // plane's `probe` effect is answered by exactly this read.
+    stub.residentLevels().catch(() => null),
   ]);
   if (!record) return json({ error: `${resource.resource} is not onboarded` }, 404);
   // Item 7: which scheduler drives the refresh cycle and, on the Workflow
@@ -9082,9 +9374,23 @@ async function handleStatus(env: Env, url: URL): Promise<Response> {
     // Item 70: the last memory reading, so the bot and the residents page can
     // show what the resident's own gate is reading.
     memory,
+    // Record 0064, record 0064: the levels and the crossing outbox on every answer.
+    ...(levels ? { levels } : {}),
   });
 }
 
+/** Levels on every data-plane answer (record 0064): the document is read
+ *  after the route's own work so the sample reflects it; a failed sample never
+ *  fails the answer — the bot forwards nothing that call. */
+async function withLevels<T>(
+  stub: ReturnType<typeof residentStub>,
+  pending: Promise<T>,
+): Promise<{ result: T; levels: ResidentLevelsDoc | null }> {
+  const result = await pending;
+  const levels = await stub.residentLevels().catch(() => null);
+  return { result, levels };
+}
+
 // -- thread data plane handlers -----------------------------------------------
 
 /** Shared front half of the thread routes: validate resource + threadKey (P1:
@@ -9161,17 +9467,20 @@ async function handleAttach(env: Env, body: Record<string, unknown>, traceparent
   // response does (`fetch failed` a few minutes in). A refusal
   // carries its `status` in the body; `ResidentExecutor.attach` reads it there.
   return streamHeartbeatJson(
-    ctx.stub.attachThread(
-      ctx.threadKey,
-      refHint,
-      readonly.readonly,
-      want.sha,
-      reuse.reuse,
-      ctx.record,
-      traceparent,
-      reason,
+    withLevels(
+      ctx.stub,
+      ctx.stub.attachThread(
+        ctx.threadKey,
+        refHint,
+        readonly.readonly,
+        want.sha,
+        reuse.reuse,
+        ctx.record,
+        traceparent,
+        reason,
+      ),
     ),
-    (result) => result,
+    ({ result, levels }) => ({ ...(result as object), ...(levels ? { levels } : {}) }),
     (err) => catchAllErr(err),
   );
 }
@@ -9226,7 +9535,9 @@ async function handleExec(env: Env, body: Record<string, unknown>, traceparent?:
   // read from the body alone through the one validated reader the sandbox
   // Worker uses, and handed to the exec's env option, never onto the command.
   const execEnv = envFromRequest({ body });
-  return streamThreadExec(ctx.stub.execThread(ctx.threadKey, body.command, timeoutMs, traceparent, execEnv));
+  return streamThreadExec(
+    withLevels(ctx.stub, ctx.stub.execThread(ctx.threadKey, body.command, timeoutMs, traceparent, execEnv)),
+  );
 }
 
 /** Stream one pending result with the thread-sandbox Worker's heartbeat
@@ -9277,13 +9588,18 @@ function streamHeartbeatJson<T>(
  *  `status`, its word and its `transient`,
  *  and the client reads it like the JSON routes' answer, never as a
  *  deterministic answer over HTTP 200. */
-function streamThreadExec(pending: Promise<Awaited<ReturnType<ResidentDO["execThread"]>>>): Response {
+function streamThreadExec(
+  pending: Promise<{ result: Awaited<ReturnType<ResidentDO["execThread"]>>; levels: ResidentLevelsDoc | null }>,
+): Response {
   return streamHeartbeatJson(
     pending,
-    (result) =>
-      "error" in result
+    ({ result, levels }) => ({
+      ...("error" in result
         ? execFailureDocument(result)
-        : { stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, truncated: result.truncated },
+        : { stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, truncated: result.truncated }),
+      // Record 0064, record 0064: levels on every answer, the crossing outbox included.
+      ...(levels ? { levels } : {}),
+    }),
     (err) => execFailureDocument(threadRejectionErr(err, "/exec")),
   );
 }

package/dist/assets/deploy/cloudflare-sandbox/Dockerfile

@@ -61,7 +61,17 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/* \
   && git config --system credential.helper '!gh auth git-credential' \
   && git config --system user.name "switchboard-bot" \
-  && git config --system user.email "switchboard-bot@users.noreply.github.com"
+  && git config --system user.email "switchboard-bot@switchboard.invalid" \
+  && git config --system core.hooksPath /opt/switchboard/hooks
+
+# The shared agent-trailer hook (deploy/hooks/prepare-commit-msg — this copy is
+# byte-identical, held so by a test: the build context is this directory, so
+# COPY cannot reach the canonical file). The fallback user.email above stays
+# OFF the GitHub domain; the real pairs ride the per-exec environment. The
+# system-level core.hooksPath above replaces repo-local .git/hooks entirely —
+# deliberate: an untrusted checkout's own hooks never run here (a repo-local
+# core.hooksPath still wins) — docs/reference/specs/execution.md item 5.
+COPY --chmod=0755 prepare-commit-msg /opt/switchboard/hooks/prepare-commit-msg
 
 # `docker` resolves to this wrapper (/usr/local/bin precedes /usr/bin on PATH):
 # it starts dockerd in its own session on first use — a plain background

package/dist/assets/deploy/cloudflare-sandbox/prepare-commit-msg

@@ -0,0 +1,17 @@
+#!/bin/sh
+# The agent trailer (docs/reference/specs/execution.md): every commit made in a
+# Switchboard image carries `Co-Authored-By: <bot pair>`, so the agent's hand
+# stays visible even when the author is the requester. The pair is read from
+# GIT_COMMITTER_NAME/GIT_COMMITTER_EMAIL at commit time — the bot fills them
+# per exec — so the image stays installation-agnostic; without them the
+# image's own git identity stands (its fallback address is off the GitHub
+# domain, so it never renders as a GitHub account). Idempotent: a message that
+# already carries this exact trailer is left unchanged; a foreign
+# Co-Authored-By does not stop it (the identity rewrite scrubs those).
+set -e
+msg="$1"
+name="${GIT_COMMITTER_NAME:-$(git config user.name || true)}"
+email="${GIT_COMMITTER_EMAIL:-$(git config user.email || true)}"
+[ -n "$name" ] && [ -n "$email" ] || exit 0
+git interpret-trailers --in-place --if-exists addIfDifferent \
+  --trailer "Co-Authored-By: $name <$email>" "$msg"

package/dist/assets/deploy/cloudflare-sandbox/worker.ts

@@ -57,6 +57,7 @@ import {
 import {
   fleetBusyAnswer,
   fleetBusyExecAnswer,
+  fleetBusyRefusedLine,
   isFleetBusyError,
   isRuntimeBusyError,
   isRuntimeBusySignal,
@@ -213,6 +214,9 @@ export interface ExecAnswer {
 export interface ExecFailure {
   error: string;
   reason?: string;
+  /** The thread's Durable Object id, on a `fleet-busy` answer alone — the bot's
+   *  ending log names which object the platform refused an instance to. */
+  containerId?: string;
   stdout: "";
   stderr: string;
   exitCode: 127;
@@ -668,8 +672,15 @@ export class SwitchboardSandbox extends Sandbox<Env> {
   private execFailure(err: unknown, startedAt: number): ExecFailure {
     const raw = thrownText(thrownShape(err));
     // A full fleet (docs/reference/specs/execution.md item 14): no container
-    // instance for this thread, so nothing started — the executor waits.
-    if (isFleetBusyError(err)) return fleetBusyExecAnswer(raw);
+    // instance for this thread, so nothing started — the executor waits. One
+    // queryable line per refusal, so a log sweep after a capacity incident
+    // can count them without reading cards; the answer carries this object's
+    // id so the bot's ending log can name which object the fleet refused.
+    if (isFleetBusyError(err)) {
+      const container = this.ctx.id.toString();
+      console.log(fleetBusyRefusedLine({ thread: this.ctx.id.name ?? container, container, refusal: raw }));
+      return fleetBusyExecAnswer(raw, container);
+    }
     // The runtime changed under the command (item 9): the process, if it
     // started, is gone with its output. Certain — the SDK said so by type.
     if (isRuntimeReplacement(err)) {
@@ -934,8 +945,14 @@ export default {
       // instance for this thread's Durable Object, so the file op never
       // started — re-sending is safe by construction. Named so the executor
       // waits instead of reading it as a dead sandbox; 503 because that is
-      // what it is.
-      if (isFleetBusyError(err)) return json(fleetBusyAnswer(msg), 503);
+      // what it is. One queryable line per refusal (the same event as the
+      // exec path's), the object's id computed from the thread key it is
+      // named by, since the error crossed the RPC boundary without it.
+      if (isFleetBusyError(err)) {
+        const container = env.Sandbox.idFromName(threadKey).toString();
+        console.log(fleetBusyRefusedLine({ thread: threadKey, container, refusal: msg, route: url.pathname }));
+        return json(fleetBusyAnswer(msg, container), 503);
+      }
       return json({ error: msg }, 500);
     }
   },