@coreplane/switchboard 1.248.0 → 1.249.0

package/dist/assets/deploy/cloudflare-resident/worker.ts

@@ -13,6 +13,7 @@
 //
 // Route surface (JSON in/out; every route below requires a bearer secret):
 //   admin scope     POST /onboard /offboard /reconfigure /rebuild /debug (all ops)
+//   drain scope     POST /drain /undrain (admin implied) — the deploy's bearer, nothing else
 //   read scope      GET /residents   POST /debug ops info|schedules|threads only (admin implied)
 //   operator scope  POST /attach /detach /exec /read /write /op            GET /status (state, reason, inFlight)
 //   unauthenticated GET /healthz (deploy wake ping; touches no DO)
@@ -158,6 +159,7 @@ import {
 } from "../../src/core/schedules.js";
 import type { ResidentLifecycleState } from "../../src/execution/residentState.js";
 import { RestoreWaiters } from "../../src/execution/restoreWaiters.js";
+import { isRuntimeBusySignal, SandboxRuntimeBusyError } from "../../src/execution/sandboxErrors.js";
 import {
   decisivePull,
   effectiveLimits,
@@ -323,11 +325,13 @@ import {
 } from "../../src/execution/residentDepsStore.js";
 import { buildId, injectedBuildStamp } from "../../src/deploy/buildStamp.js";
 import { createRefreshInstance, createRefreshInstanceNow, type RefreshInstanceParams } from "./refresh";
+import { drainRefusal, liveDrain, parseDrainRequest, type DrainRecord } from "./drain";
 import {
   ControlResetError,
   RuntimeReplacedError,
   controlResetErr,
   execFailureDocument,
+  runtimeBusyErr,
   runtimeReplacedErr,
   selfAndCauses,
   threadErrBuilders,
@@ -417,6 +421,10 @@ export interface Env {
    *  (info, schedules, threads) — for dashboards and humans who need to look,
    *  never to change anything. Unset = no read scope exists. */
   RESIDENT_READ_TOKEN?: string;
+  /** Optional drain-only bearer (item 69): POST /drain and /undrain, nothing
+   *  else — what a release deploy holds so it can close the fleet without the
+   *  admin bearer. Unset = only admin can drain. */
+  RESIDENT_DRAIN_TOKEN?: string;
   // GitHub App identity for minting installation tokens inside residents
   // (provisioned via `npm run secrets` from deploy/secrets.manifest.json; when
   // unset, clones/fetches run anonymously —
@@ -843,6 +851,18 @@ function isRuntimeUnreachable(err: unknown): boolean {
   for (const link of selfAndCauses(err)) if (isRuntimeUnreachableSignal(link)) return true;
   return false;
 }
+
+/** Did the platform refuse the connect inside its own accept allowance
+ *  (docs/reference/specs/resident-repos.md item 68; execution.md item 28)? The
+ *  platform's own wording — a plain `Error`, the SDK hands it on unwrapped —
+ *  anywhere in the cause chain; its words blame load, which the platform never
+ *  measured and an idle container has disproved. Asked only of a spawn-phase
+ *  error, after `isRuntimeReplacement`: such a container is neither replaced
+ *  nor silent for good, and a command's own output never gets here. */
+function isRuntimeBusy(err: unknown): boolean {
+  for (const link of selfAndCauses(err)) if (isRuntimeBusySignal(link)) return true;
+  return false;
+}
 /** Trailing slice of one string for an error reason. Command RESULTS are not
  *  described here — `describeStepFailure` owns that, because choosing between
  *  the two streams is what lost a diagnosis (residentStepReport.ts). */
@@ -1347,6 +1367,11 @@ const registryKey = (resource: string) => `${REGISTRY_KEY_PREFIX}${resource}`;
 /** Registry-DO key for the admin test overrides (gc.ts `StoredTestOverrides`).
  *  Deliberately OUTSIDE the `resident:` prefix so it never counts as a slot. */
 const TEST_OVERRIDES_KEY = "testOverrides";
+/** Registry-DO key for the fleet drain (drain.ts; docs/reference/specs/resident-repos.md
+ *  item 69). Outside the `resident:` prefix like the overrides, so it never
+ *  counts as a slot; it survives the isolate swap a deploy performs, which is
+ *  why the record carries its own end. */
+const DRAIN_KEY = "drain";
 
 type OnboardResult = { ok: true; record: ResidentRecord } | { ok: false; status: number; error: string };
 
@@ -1462,6 +1487,25 @@ export class ResidentRegistryDO extends DurableObject<Env> {
   async remove(resource: string): Promise<boolean> {
     return this.ctx.storage.delete(registryKey(resource));
   }
+
+  /** The stored drain record as it is — `liveDrain` (drain.ts) decides at the
+   *  caller's clock whether it is in force; the registry keeps no clock of its
+   *  own so an expired record is read the same by every route. */
+  async getDrain(): Promise<unknown> {
+    return (await this.ctx.storage.get(DRAIN_KEY)) ?? null;
+  }
+
+  /** Admin-only by construction (reached solely via POST /drain): replaces
+   *  whatever drain stood — a second deploy's drain extends the first's. */
+  async setDrain(record: DrainRecord): Promise<DrainRecord> {
+    await this.ctx.storage.put(DRAIN_KEY, record);
+    return record;
+  }
+
+  /** Admin-only by construction (POST /undrain): true when a record was there. */
+  async clearDrain(): Promise<boolean> {
+    return this.ctx.storage.delete(DRAIN_KEY);
+  }
 }
 
 // ---------------------------------------------------------------------------
@@ -2027,6 +2071,14 @@ export class ResidentDO extends Sandbox<Env> {
       // read as a replaced container.
       if (isControlReset(err)) throw new ControlResetError("spawn", err);
       if (!isRuntimeReplacement(err)) {
+        // The container is running but did not accept the SDK's connect inside
+        // the platform's own allowance (item 68): a command already running in
+        // it has its cores. Nothing started, the worktree is as it was, and the
+        // container accepts again in moments — the typed word, for the thread
+        // routes to answer with the wait token; never counted as unreachable.
+        if (isRuntimeBusy(err)) {
+          throw new SandboxRuntimeBusyError({ containerId: this.ctx.id.toString(), cause: errMsg(err) });
+        }
         // The control port never answered the SDK's connect (its 30 s abort,
         // raised inside the wake path): no process started and nothing about
         // the repository is known. Count it in storage — the ladder of item 64
@@ -4861,6 +4913,20 @@ export class ResidentDO extends Sandbox<Env> {
   ): Promise<AttachOk | ThreadErr> {
     try {
       await this.ensureHydrated();
+      // The fleet drain (item 69): a deploy is waiting for the runs in flight
+      // to end, and a NEW run's attach is refused with the record the bot
+      // waits on — a real 503 in the streamed document, read by the client as
+      // `draining`, never as the platform's transient. A run already in flight
+      // — registered from its attach to its release (item 44) — re-attaches
+      // through: a rolled container, an evicted worktree, a resumed run are
+      // the runs the drain waits FOR, and refusing them would hold the fleet
+      // closed on the run it is closed for. Read before the image reconcile so
+      // a refused attach never restarts a container.
+      const drain = await this.fleetDrain();
+      if (drain && !(await this.ctx.storage.get(runRegKey(threadKey)))) {
+        const refusal: ThreadErr & { draining: DrainRecord } = drainRefusal(drain);
+        return refusal;
+      }
       const resourceId = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
       if (await this.reconcileImage("attach")) {
         return {
@@ -4937,6 +5003,7 @@ export class ResidentDO extends Sandbox<Env> {
         state: s.state,
         stateReason: s.reason,
         reason: s.reason,
+        cause: "system",
       };
     }
     // `resourceId` was read by the caller a moment ago (item 15 of the audit:
@@ -4952,7 +5019,12 @@ export class ResidentDO extends Sandbox<Env> {
     // Typed `reason` beside the words: the client reads the field — a refusal no
     // wait clears, unlike the restore window's 503s — never the sentence.
     if (!record || !facts)
-      return { error: "not-serviceable: registry record or repo facts missing", status: 503, reason: "unregistered" };
+      return {
+        error: "not-serviceable: registry record or repo facts missing",
+        status: 503,
+        reason: "unregistered",
+        cause: "system",
+      };
 
     // The binding's ref wins for the thread's whole life, with one exception
     // (item 16): a thread bound to the repo default for want of a named branch
@@ -4981,6 +5053,7 @@ export class ResidentDO extends Sandbox<Env> {
         status: 409,
         needs: "ref",
         defaultRef: facts.defaultRef,
+        cause: "request",
       };
     }
     const worktreePath = prior?.worktreePath ?? (await threadWorktreePath(threadKey, ref));
@@ -5102,7 +5175,14 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
+        return {
+          error: errMsg(err),
+          status: 503,
+          state: s.state,
+          stateReason: s.reason,
+          reason: "mirror-busy",
+          cause: "system",
+        };
       }
       return catchAllErr(err, "attach-failed");
     }
@@ -5404,17 +5484,30 @@ export class ResidentDO extends Sandbox<Env> {
         return { error: `reuse-refused: ${err.why}`, status: 409, needs: "recreate" };
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
+        return {
+          error: errMsg(err),
+          status: 503,
+          state: s.state,
+          stateReason: s.reason,
+          reason: "mirror-busy",
+          cause: "system",
+        };
       }
       if (err instanceof StepError && err.step === "unknown-ref") {
-        return { error: `unknown-ref: ${err.message}`, status: 400 };
+        return { error: `unknown-ref: ${err.message}`, status: 400, cause: "request" };
       }
       if (err instanceof StepError && err.step === "stale-tip") {
         // Item 51: not a resident fault and not a caller fault — a fact about
         // the mirror at this instant. 409 with the state, so the bot's named
         // fallback runs cold at the commit it asked for.
         const s = await this.getStatus();
-        return { error: `stale-tip: ${err.message}`, status: 409, state: s.state, reason: "stale-tip" };
+        return {
+          error: `stale-tip: ${err.message}`,
+          status: 409,
+          state: s.state,
+          reason: "stale-tip",
+          cause: "system",
+        };
       }
       return this.attachFailed(err);
     }
@@ -5449,7 +5542,14 @@ export class ResidentDO extends Sandbox<Env> {
       // worktree lock above, so the bot-side fallback can retry.
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
+        return {
+          error: errMsg(err),
+          status: 503,
+          state: s.state,
+          stateReason: s.reason,
+          reason: "mirror-busy",
+          cause: "system",
+        };
       }
       return this.attachFailed(err);
     }
@@ -6293,6 +6393,7 @@ export class ResidentDO extends Sandbox<Env> {
         state: s.state,
         stateReason: s.reason,
         reason: s.reason,
+        cause: "system",
       };
     }
     const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
@@ -6381,6 +6482,9 @@ export class ResidentDO extends Sandbox<Env> {
       // container is unchanged, so no `replacedExecAnswer` gate applies.
       if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return this.replacedExecAnswer(err);
+      // A refused connect at the command's spawn (item 68): the wait token,
+      // the command never started.
+      if (err instanceof SandboxRuntimeBusyError) return runtimeBusyErr(err);
       throw err;
     }
   }
@@ -6461,6 +6565,7 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
+      if (err instanceof SandboxRuntimeBusyError) return runtimeBusyErr(err);
       throw err;
     }
     if (r.exitCode !== 0 || r.timedOut) return { error: `read-failed: ${describeStepFailure(r)}`, status: 404 };
@@ -6510,6 +6615,7 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
+      if (err instanceof SandboxRuntimeBusyError) return runtimeBusyErr(err);
       throw err;
     }
   }
@@ -6557,6 +6663,7 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
+      if (err instanceof SandboxRuntimeBusyError) return runtimeBusyErr(err);
       const step = err instanceof StepError ? ` at ${err.step}` : "";
       return { error: `write-failed${step}: ${errMsg(err)}`, status: 400 };
     }
@@ -6781,6 +6888,19 @@ export class ResidentDO extends Sandbox<Env> {
     const threadOps = [...this.threadOpsInFlight.values()].reduce((a, n) => a + n, 0);
     return threadOps + this.opUsersInUse.size + this.attachesInFlight;
   }
+  /** The fleet drain in force (item 69), read from the registry at this clock;
+   *  a registry that cannot be read is NO drain: a run must never fail because
+   *  a flag could not be read, and the deploy's own preflight fails closed on
+   *  its side (an unknown fleet refuses the deploy), so the failure lands on
+   *  the deploy, never on the run. Said in the log. */
+  private async fleetDrain(): Promise<DrainRecord | null> {
+    try {
+      return liveDrain(await this.registry().getDrain(), systemClock());
+    } catch (err) {
+      console.warn(`[drain] registry unreadable at attach — treating as no drain: ${errMsg(err)}`);
+      return null;
+    }
+  }
   /** In-flight activity for the deploy preflight (GET /status, GET /residents).
    *  The in-memory counters (a fresh isolate answers 0 for them — nothing of
    *  THEIRS survived to be interrupted) plus the durable run registrations:
@@ -6928,6 +7048,7 @@ export class ResidentDO extends Sandbox<Env> {
         state: s.state,
         stateReason: s.reason,
         reason: s.reason,
+        cause: "system",
       };
     }
     // One storage round trip for the two facts; the registry lookup stays (an
@@ -6939,7 +7060,12 @@ export class ResidentDO extends Sandbox<Env> {
     // Typed `reason` beside the words: the client reads the field — a refusal no
     // wait clears, unlike the restore window's 503s — never the sentence.
     if (!record || !facts)
-      return { error: "not-serviceable: registry record or repo facts missing", status: 503, reason: "unregistered" };
+      return {
+        error: "not-serviceable: registry record or repo facts missing",
+        status: 503,
+        reason: "unregistered",
+        cause: "system",
+      };
     const command = record.commands[op];
     if (!command) return { error: `op-unavailable: the command table has no "${op}" entry`, status: 400 };
 
@@ -7035,10 +7161,17 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
+        return {
+          error: errMsg(err),
+          status: 503,
+          state: s.state,
+          stateReason: s.reason,
+          reason: "mirror-busy",
+          cause: "system",
+        };
       }
       if (err instanceof StepError && err.step === "unknown-ref") {
-        return { error: `unknown-ref: ${err.message}`, status: 400 };
+        return { error: `unknown-ref: ${err.message}`, status: 400, cause: "request" };
       }
       // A step that failed is named and deterministic; a throw no step named is
       // typed by the one builder every such 500 goes through.
@@ -7821,7 +7954,7 @@ function timingSafeEqual(a: string, b: string): boolean {
   return diff === 0;
 }
 
-type Scope = "admin" | "operator" | "read";
+type Scope = "admin" | "operator" | "read" | "drain";
 
 /** Which token a bearer is, or null. Constant-time per comparison; fail closed
  *  on unset/empty secrets. */
@@ -7830,6 +7963,7 @@ function tokenScope(env: Env, token: string | null): Scope | null {
   if (env.RESIDENT_ADMIN_TOKEN && timingSafeEqual(token, env.RESIDENT_ADMIN_TOKEN)) return "admin";
   if (env.RESIDENT_OPERATOR_TOKEN && timingSafeEqual(token, env.RESIDENT_OPERATOR_TOKEN)) return "operator";
   if (env.RESIDENT_READ_TOKEN && timingSafeEqual(token, env.RESIDENT_READ_TOKEN)) return "read";
+  if (env.RESIDENT_DRAIN_TOKEN && timingSafeEqual(token, env.RESIDENT_DRAIN_TOKEN)) return "drain";
   return null;
 }
 
@@ -8009,6 +8143,8 @@ const ROUTES: Record<string, { scope: Scope; method: string }> = {
   "/offboard": { scope: "admin", method: "POST" },
   "/reconfigure": { scope: "admin", method: "POST" },
   "/rebuild": { scope: "admin", method: "POST" },
+  "/drain": { scope: "drain", method: "POST" }, // close the fleet to new runs for a deploy (item 69; admin implied)
+  "/undrain": { scope: "drain", method: "POST" }, // reopen it
   "/residents": { scope: "read", method: "GET" }, // admin implied; read-only bearer allowed
   "/debug": { scope: "read", method: "POST" }, // per-op: READ_DEBUG_OPS for read scope, everything for admin
   "/status": { scope: "operator", method: "GET" },
@@ -8109,6 +8245,10 @@ export default {
             return await handleReconfigure(env, body);
           case "/rebuild":
             return await handleRebuild(env, body);
+          case "/drain":
+            return await handleDrain(env, body);
+          case "/undrain":
+            return await handleUndrain(env);
           case "/residents":
             return await handleResidents(env);
           case "/debug": {
@@ -8239,6 +8379,7 @@ async function handleOnboard(env: Env, body: Record<string, unknown>): Promise<R
             `exact name (GitHub's token API answers the same 422 for both). An org admin adds it under the ` +
             `App's installation settings (Settings → GitHub Apps → Configure → Repository access), ` +
             `then retry (${errMsg(err)})`,
+          cause: "policy",
         },
         403,
       );
@@ -8529,6 +8670,26 @@ async function handleRebuild(env: Env, body: Record<string, unknown>): Promise<R
  *  targets a different DO, so they run concurrently; a failing one degrades
  *  to {error} without touching its neighbors, and the response order follows
  *  the registry list. */
+/** POST /drain (admin): close the fleet to new runs (docs/reference/specs/resident-repos.md
+ *  item 69). The record carries its own end (`until`), so a drain nobody lifts
+ *  ends by itself; a second drain replaces the first. Runs in flight are
+ *  untouched — the deploy that asked waits for them through `/residents`. */
+async function handleDrain(env: Env, body: Record<string, unknown>): Promise<Response> {
+  const parsed = parseDrainRequest(body, systemClock());
+  if (!parsed.ok) return json({ error: parsed.error }, 400);
+  const record = await registryStub(env).setDrain(parsed.record);
+  console.log(`[drain] fleet closed to new runs by ${record.by} for ${record.reason}: until ${record.until}`);
+  return json({ draining: record });
+}
+
+/** POST /undrain (admin): reopen the fleet. Idempotent — `cleared` says whether
+ *  a drain stood. */
+async function handleUndrain(env: Env): Promise<Response> {
+  const cleared = await registryStub(env).clearDrain();
+  console.log(`[drain] fleet reopened (${cleared ? "a drain stood" : "no drain stood"})`);
+  return json({ draining: null, cleared });
+}
+
 async function handleResidents(env: Env): Promise<Response> {
   const residents = await registryStub(env).list();
   const settled = await Promise.allSettled(
@@ -8564,6 +8725,9 @@ async function handleResidents(env: Env): Promise<Response> {
     count: residents.length,
     inFlight,
     inFlightUnknown,
+    // The drain in force, or null (item 69): the deploy runner and the
+    // dashboard read it here; the attach route reads the same record.
+    draining: liveDrain(await registryStub(env).getDrain(), systemClock()),
     residents: enriched,
   });
 }

package/dist/assets/deploy/cloudflare-sandbox/worker.ts

@@ -58,9 +58,14 @@ import {
   fleetBusyAnswer,
   fleetBusyExecAnswer,
   isFleetBusyError,
+  isRuntimeBusyError,
+  isRuntimeBusySignal,
   isRuntimeUnreachableError,
+  runtimeBusyAnswer,
+  runtimeBusyExecAnswer,
   runtimeUnreachableAnswer,
   runtimeUnreachableExecAnswer,
+  SandboxRuntimeBusyError,
   SandboxRuntimeUnreachableError,
   sandboxStartingAnswer,
   sandboxStartingExecAnswer,
@@ -182,6 +187,16 @@ function isRuntimeUnreachable(err: unknown): boolean {
   return false;
 }
 
+/** Did the platform refuse the connect inside its own accept allowance
+ *  (docs/reference/specs/execution.md item 28)? The platform's own wording,
+ *  anywhere in the cause chain; a plain `Error`, so the wording is all there
+ *  is — and its words blame load the platform never measured. Asked only of
+ *  a failure met before a process was started. */
+function isRuntimeBusy(err: unknown): boolean {
+  for (const link of selfAndCauses(err)) if (isRuntimeBusySignal(link)) return true;
+  return false;
+}
+
 /** A finished command, as `/exec` answers it. `durationMs` is the command's
  *  wall time in the sandbox (docs/reference/specs/tracing.md item 19). */
 export interface ExecAnswer {
@@ -194,7 +209,7 @@ export interface ExecAnswer {
 /** A command the sandbox never answered for, in the dual in-body shape
  *  (docs/reference/specs/execution.md item 3): a new executor throws on `error`, an
  *  older one still renders `exit 127: <stderr>`. `reason` names the machine
- *  token when there is one (`fleet-busy`, `runtime-unreachable`). */
+ *  token when there is one (`fleet-busy`, `runtime-busy`, `runtime-unreachable`). */
 export interface ExecFailure {
   error: string;
   reason?: string;
@@ -205,8 +220,8 @@ export interface ExecFailure {
 
 /** A file route's refusal, with the HTTP status the fetch handler answers and,
  *  when the refusal is a named condition the executor waits on (`fleet-busy`,
- *  `runtime-unreachable`), its machine token — the executor reads the token,
- *  never the text, so a refusal without it is a dead sandbox to it. */
+ *  `runtime-busy`, `runtime-unreachable`), its machine token — the executor
+ *  reads the token, never the text, so a refusal without it is a dead sandbox to it. */
 interface FileRefusal {
   error: string;
   status: number;
@@ -321,9 +336,10 @@ export class SwitchboardSandbox extends Sandbox<Env> {
           (cause) => sandboxStartingExecAnswer(cause),
         );
       } catch (err) {
-        // The warm-up's own failure, handed on by the gate: a full fleet or a
-        // silent control port keeps its name; anything else propagates.
-        return this.execFailure(err, startedAt);
+        // The warm-up's own failure, handed on by the gate: a full fleet, a
+        // refused connect or a silent control port keeps its name; anything
+        // else propagates. Nothing ran — the warm-up is a spawn.
+        return this.spawnFailure(err, startedAt);
       }
     });
   }
@@ -343,7 +359,10 @@ export class SwitchboardSandbox extends Sandbox<Env> {
     try {
       proc = await createExtensionProcessSandbox(this).exec(argv, { env: envVars, timeout: backstopMs });
     } catch (err) {
-      return this.execFailure(err, startedAt);
+      // The process was never started: a refused connect is the wait token
+      // here and only here (item 28) — the executor re-sends, and
+      // nothing runs twice.
+      return this.spawnFailure(err, startedAt);
     }
     try {
       const out = await proc.output({
@@ -631,6 +650,20 @@ export class SwitchboardSandbox extends Sandbox<Env> {
     return { stdout: out.stdout, stderr: out.stderr, exitCode: out.timedOut ? 124 : out.exitCode };
   }
 
+  /** A failure met BEFORE a process was started — the spawn, or the gate's
+   *  warm-up: a container that did not accept the connection is named with
+   *  its wait token (docs/reference/specs/execution.md item 28), since
+   *  nothing ran and the identical request is safe to re-send. Every other
+   *  failure is classified as after a start (`execFailure`). A failure of a
+   *  running command's output never comes here: the process exists, and a
+   *  re-send would run it again. */
+  private spawnFailure(err: unknown, startedAt: number): ExecFailure {
+    if (!isFleetBusyError(err) && !isRuntimeReplacement(err) && isRuntimeBusy(err)) {
+      return runtimeBusyExecAnswer(this.runtimeBusy(thrownText(thrownShape(err))).message);
+    }
+    return this.execFailure(err, startedAt);
+  }
+
   /** The named failures, as `/exec` data; anything else is thrown as it came. */
   private execFailure(err: unknown, startedAt: number): ExecFailure {
     const raw = thrownText(thrownShape(err));
@@ -660,15 +693,25 @@ export class SwitchboardSandbox extends Sandbox<Env> {
     });
   }
 
+  /** The typed, named error for a container that did not accept the
+   *  connection (item 28), with this container's id — thrown across the RPC
+   *  boundary to the fetch handler on the file routes, matched by name. */
+  private runtimeBusy(cause: string): SandboxRuntimeBusyError {
+    return new SandboxRuntimeBusyError({ containerId: this.ctx.id.toString(), cause });
+  }
+
   /** A file operation with its runtime failures named for the fetch handler:
-   *  a silent control port becomes the typed error (item 9); a missing file
-   *  is the refusal the route answers 404. The SDK's other errors propagate. */
+   *  a refused connect (item 28) and a silent control port (item 9) become
+   *  the typed errors; a missing file is the refusal the route answers 404.
+   *  The SDK's other errors propagate. A file operation that met either
+   *  never reached the runtime, so the executor's re-send does nothing twice. */
   private async fileOp<T>(op: () => Promise<T>): Promise<T | FileRefusal> {
     try {
       return await op();
     } catch (err) {
       const shape = thrownShape(err);
       if (shape.name === "FileNotFoundError") return { error: `read-failed: ${thrownText(shape)}`, status: 404 };
+      if (!isRuntimeReplacement(err) && isRuntimeBusy(err)) throw this.runtimeBusy(thrownText(shape));
       if (!isRuntimeReplacement(err) && isRuntimeUnreachable(err)) throw this.runtimeUnreachable(thrownText(shape));
       throw err;
     }
@@ -884,6 +927,9 @@ export default {
       // op never reached a runtime, so a 503 the executor's transport retry
       // re-sends, with the named reason and the container in the text.
       if (isRuntimeUnreachableError(err)) return json(runtimeUnreachableAnswer(msg), 503);
+      // The container did not accept the connection (item 28): the file op
+      // never reached it either — a 503 with the wait token.
+      if (isRuntimeBusyError(err)) return json(runtimeBusyAnswer(msg), 503);
       // A full fleet (docs/reference/specs/execution.md item 14): no container
       // instance for this thread's Durable Object, so the file op never
       // started — re-sending is safe by construction. Named so the executor
@@ -972,6 +1018,7 @@ function streamExec(run: () => Promise<ExecAnswer | ExecFailure>, traceparent: s
       const shape = thrownShape(err);
       const raw = thrownText(shape);
       if (isRuntimeUnreachableError(err)) return runtimeUnreachableExecAnswer(raw);
+      if (isRuntimeBusyError(err)) return runtimeBusyExecAnswer(raw);
       if (isFleetBusyError(err)) return fleetBusyExecAnswer(raw);
       // A recycle the Durable Object did not catch by type: by name, or
       // a recycle-shaped text minutes into the attempt (item 9).

package/dist/assets/deploy/secrets.manifest.json

@@ -81,6 +81,12 @@
       "workers": ["resident"],
       "note": "Read-only /residents + debug routes; what the resident deploy preflight needs (CI holds this one). Self-minted."
     },
+    {
+      "name": "RESIDENT_DRAIN_TOKEN",
+      "workers": ["resident"],
+      "optional": true,
+      "note": "Drain-only bearer: POST /drain and /undrain, nothing else (docs/reference/specs/resident-repos.md item 69). What the release deploy holds (CI) so the resident step can close the fleet to new runs and land once the runs in flight end, without the admin bearer. Self-minted; optional — unset, only admin can drain and the deploy waits without one."
+    },
     {
       "name": "R2_ACCESS_KEY_ID",
       "workers": ["resident", "sandbox"],

package/dist/assets/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "switchboard",
-  "version": "1.248.0",
+  "version": "1.249.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "switchboard",
-      "version": "1.248.0",
+      "version": "1.249.0",
       "license": "Apache-2.0",
       "workspaces": [
         "web",
@@ -20445,7 +20445,7 @@
     },
     "packages/switchboard": {
       "name": "@coreplane/switchboard",
-      "version": "1.248.0",
+      "version": "1.249.0",
       "license": "Apache-2.0",
       "dependencies": {
         "@earendil-works/pi-ai": "0.85.1",

package/dist/assets/package.json

@@ -1,6 +1,6 @@
 {
   "name": "switchboard",
-  "version": "1.248.0",
+  "version": "1.249.0",
   "private": true,
   "description": "Mention it in Slack and an agent reviews the PR, ships the fix, or answers the question — on the model you choose, with its tools running where you decide.",
   "license": "Apache-2.0",

package/dist/assets/source.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.248.0",
-  "commit": "e2ba9e4c1b52ac15c841be8ad086b017d4a14ced",
-  "builtAt": "2026-09-18T04:34:16.576Z"
+  "version": "1.249.0",
+  "commit": "1145eea8ffdd6a2bd1ae8f1b26a4919db2caf47d",
+  "builtAt": "2026-09-18T07:16:19.891Z"
 }

package/dist/assets/src/agents/registry.ts

@@ -188,7 +188,7 @@ const UNIT_CONTRACT = `UNIT CONTRACT: when your first user turn carries a \`${CO
 // it to the unit's board issue without a person writing it there. Both coding
 // prompts carry this verbatim, right after the contract paragraph, so the
 // sandbox and resident children read the same rule.
-const UNIT_HANDOFF = `UNIT HANDOFF: when your first user turn carries a \`${CONTRACT_HEADING}\` block, call the submit_handoff tool once, after submit_pr_description and before your final message, with the typed handoff — deviations: where you departed from the unit as written (from, to, why); followUps: what you found and did not do, and where it belongs (what, where); unproven: which of the unit's test scenarios or criteria you could not prove, and why (criterion, why). Switchboard records it on the run and posts it to the unit's board issue, where a person decides each row's disposition; you never edit the plan's ledger yourself. An empty handoff is submitted as three empty lists, never skipped — a missing handoff reads as an unfinished run, not as nothing to say. Without a \`${CONTRACT_HEADING}\` block, do not call it.`;
+const UNIT_HANDOFF = `UNIT HANDOFF: when your first user turn carries a \`${CONTRACT_HEADING}\` block, call the submit_handoff tool once, after submit_pr_description and before your final message, with the typed handoff — deviations: where you departed from the unit as written (from, to, why); followUps: what you found and did not do, and where it belongs (what, where); unproven: which of the unit's test scenarios or criteria you could not prove, and why (criterion, why); landed (optional): what of the unit was already on the base when you began, and the pull request or commit that carries it (what, where) — when the whole unit is already there, push nothing of your own and open no pull request; the handoff's landed rows end the unit done. Switchboard records it on the run and posts it to the unit's board issue, where a person decides each row's disposition; you never edit the plan's ledger yourself. An empty handoff is submitted as three empty lists, never skipped — a missing handoff reads as an unfinished run, not as nothing to say. Without a \`${CONTRACT_HEADING}\` block, do not call it.`;
 
 // What both execution images carry beyond git and the package managers
 // (docs/reference/specs/execution.md item 10), said in one sentence by every
@@ -544,7 +544,7 @@ Your tools work without a workspace: the GitHub tools — \`github_repos\` (the
 
 ${statusCardRule('"Read the issue and its thread", "Post the comment"')} A one-step answer needs no checklist; post one when the request has steps the person would wait on.
 
-You cannot run commands, clone repositories, edit code, or review pull requests, and you cannot search the web. Other Switchboard agents can: for a code change or a pull request tell the user to re-send with \`agent:ship\` (it makes the change, opens the PR and loops review); for a PR review, \`agent:review\`; for a web-research question, \`agent:research\` (e.g. "\`agent:ship in acme/api: fix the failing login test\`", "\`agent:research compare X and Y\`"). Delete an issue only when the user explicitly asked to delete it (closing is an update).`;
+You cannot run commands, clone repositories, edit code, or review pull requests, and you cannot search the web. Other Switchboard agents can, and a plain message reaches them by itself: for a code change or a pull request, say what you found and that the change is not yours to make, and that asking for it in plain words in a new message — "in acme/api: fix the failing login test" — starts the agent that makes the change, opens the PR and loops review; the same for a PR review ("review <PR URL>") and a web-research question ("compare X and Y on the web"). Never hand back a command or an \`agent:…\` line for the person to type: describe the ask in their words. Delete an issue only when the user explicitly asked to delete it (closing is an update).`;
 
 // The explore agent (docs/reference/specs/agent-explore.md): a long, read-only
 // investigation — "run our CI locally and validate the claims", "how long does
@@ -567,9 +567,9 @@ THE DELIVERABLE IS A CLAIM TABLE. Turn the request into the claims it makes or a
 
 TIME. Your budget is up to two hours — less when a boundary or the request's \`budget:\` directive clipped it, which the runtime-config block above says — and the wrap-up warning tells you when to stop starting new checks. A single command is capped at ${BASH_TIMEOUT_MAX_MS / 60_000} minutes (pass the bash tool's \`timeoutMs\`, up to ${BASH_TIMEOUT_MAX_MS} ms, for a long one). A job that needs longer — a full suite, a build, a pipeline run — is started detached and polled across tool calls: \`setsid -f sh -c '<command> > /tmp/job.log 2>&1; echo $? > /tmp/job.exit'\`, then \`tail -n 40 /tmp/job.log\` and \`cat /tmp/job.exit\` on later calls (a plain background job dies with the command that started it; a \`setsid -f\` job outlives it). Batch commands into few tool calls; never explore file by file.
 
-READ-ONLY: NEVER open a pull request, and never commit or push — no branch, no \`gh pr create\`, no PR or issue write of any kind. You hold a read credential and your job is to find out, not to change. If the investigation shows a change is needed, say exactly what and where in your write-up and point the user at \`agent:ship\` (it makes the change, opens the PR and loops review).
+READ-ONLY: NEVER open a pull request, and never commit or push — no branch, no \`gh pr create\`, no PR or issue write of any kind. You hold a read credential and your job is to find out, not to change. If the investigation shows a change is needed, say exactly what and where in your write-up, and that asking for it in plain words in a new message ("in <owner/name>: <the change>") starts the agent that makes the change, opens the PR and loops review — never hand back a command or an \`agent:…\` line to type.
 
-You cannot attach or post files: your whole answer is text. Never say a file is attached or below — name its path in the workspace and describe it (what it shows, its size) instead; a person who needs the file itself asks \`agent:coding\`, which can attach.
+You cannot attach or post files: your whole answer is text. Never say a file is attached or below — name its path in the workspace and describe it (what it shows, its size) instead; a person who needs the file itself asks for it to be attached in a new message, which reaches a preset that can.
 
 ${statusCardRule('"Clone and install", "Time the full suite"')}