@coreplane/switchboard 1.247.0 → 1.249.0

package/dist/assets/deploy/cloudflare-resident/worker.ts

@@ -13,6 +13,7 @@
 //
 // Route surface (JSON in/out; every route below requires a bearer secret):
 //   admin scope     POST /onboard /offboard /reconfigure /rebuild /debug (all ops)
+//   drain scope     POST /drain /undrain (admin implied) — the deploy's bearer, nothing else
 //   read scope      GET /residents   POST /debug ops info|schedules|threads only (admin implied)
 //   operator scope  POST /attach /detach /exec /read /write /op            GET /status (state, reason, inFlight)
 //   unauthenticated GET /healthz (deploy wake ping; touches no DO)
@@ -158,6 +159,7 @@ import {
 } from "../../src/core/schedules.js";
 import type { ResidentLifecycleState } from "../../src/execution/residentState.js";
 import { RestoreWaiters } from "../../src/execution/restoreWaiters.js";
+import { isRuntimeBusySignal, SandboxRuntimeBusyError } from "../../src/execution/sandboxErrors.js";
 import {
   decisivePull,
   effectiveLimits,
@@ -323,11 +325,13 @@ import {
 } from "../../src/execution/residentDepsStore.js";
 import { buildId, injectedBuildStamp } from "../../src/deploy/buildStamp.js";
 import { createRefreshInstance, createRefreshInstanceNow, type RefreshInstanceParams } from "./refresh";
+import { drainRefusal, liveDrain, parseDrainRequest, type DrainRecord } from "./drain";
 import {
   ControlResetError,
   RuntimeReplacedError,
   controlResetErr,
   execFailureDocument,
+  runtimeBusyErr,
   runtimeReplacedErr,
   selfAndCauses,
   threadErrBuilders,
@@ -417,6 +421,10 @@ export interface Env {
    *  (info, schedules, threads) — for dashboards and humans who need to look,
    *  never to change anything. Unset = no read scope exists. */
   RESIDENT_READ_TOKEN?: string;
+  /** Optional drain-only bearer (item 69): POST /drain and /undrain, nothing
+   *  else — what a release deploy holds so it can close the fleet without the
+   *  admin bearer. Unset = only admin can drain. */
+  RESIDENT_DRAIN_TOKEN?: string;
   // GitHub App identity for minting installation tokens inside residents
   // (provisioned via `npm run secrets` from deploy/secrets.manifest.json; when
   // unset, clones/fetches run anonymously —
@@ -843,6 +851,18 @@ function isRuntimeUnreachable(err: unknown): boolean {
   for (const link of selfAndCauses(err)) if (isRuntimeUnreachableSignal(link)) return true;
   return false;
 }
+
+/** Did the platform refuse the connect inside its own accept allowance
+ *  (docs/reference/specs/resident-repos.md item 68; execution.md item 28)? The
+ *  platform's own wording — a plain `Error`, the SDK hands it on unwrapped —
+ *  anywhere in the cause chain; its words blame load, which the platform never
+ *  measured and an idle container has disproved. Asked only of a spawn-phase
+ *  error, after `isRuntimeReplacement`: such a container is neither replaced
+ *  nor silent for good, and a command's own output never gets here. */
+function isRuntimeBusy(err: unknown): boolean {
+  for (const link of selfAndCauses(err)) if (isRuntimeBusySignal(link)) return true;
+  return false;
+}
 /** Trailing slice of one string for an error reason. Command RESULTS are not
  *  described here — `describeStepFailure` owns that, because choosing between
  *  the two streams is what lost a diagnosis (residentStepReport.ts). */
@@ -1347,6 +1367,11 @@ const registryKey = (resource: string) => `${REGISTRY_KEY_PREFIX}${resource}`;
 /** Registry-DO key for the admin test overrides (gc.ts `StoredTestOverrides`).
  *  Deliberately OUTSIDE the `resident:` prefix so it never counts as a slot. */
 const TEST_OVERRIDES_KEY = "testOverrides";
+/** Registry-DO key for the fleet drain (drain.ts; docs/reference/specs/resident-repos.md
+ *  item 69). Outside the `resident:` prefix like the overrides, so it never
+ *  counts as a slot; it survives the isolate swap a deploy performs, which is
+ *  why the record carries its own end. */
+const DRAIN_KEY = "drain";
 
 type OnboardResult = { ok: true; record: ResidentRecord } | { ok: false; status: number; error: string };
 
@@ -1462,6 +1487,25 @@ export class ResidentRegistryDO extends DurableObject<Env> {
   async remove(resource: string): Promise<boolean> {
     return this.ctx.storage.delete(registryKey(resource));
   }
+
+  /** The stored drain record as it is — `liveDrain` (drain.ts) decides at the
+   *  caller's clock whether it is in force; the registry keeps no clock of its
+   *  own so an expired record is read the same by every route. */
+  async getDrain(): Promise<unknown> {
+    return (await this.ctx.storage.get(DRAIN_KEY)) ?? null;
+  }
+
+  /** Admin-only by construction (reached solely via POST /drain): replaces
+   *  whatever drain stood — a second deploy's drain extends the first's. */
+  async setDrain(record: DrainRecord): Promise<DrainRecord> {
+    await this.ctx.storage.put(DRAIN_KEY, record);
+    return record;
+  }
+
+  /** Admin-only by construction (POST /undrain): true when a record was there. */
+  async clearDrain(): Promise<boolean> {
+    return this.ctx.storage.delete(DRAIN_KEY);
+  }
 }
 
 // ---------------------------------------------------------------------------
@@ -2027,6 +2071,14 @@ export class ResidentDO extends Sandbox<Env> {
       // read as a replaced container.
       if (isControlReset(err)) throw new ControlResetError("spawn", err);
       if (!isRuntimeReplacement(err)) {
+        // The container is running but did not accept the SDK's connect inside
+        // the platform's own allowance (item 68): a command already running in
+        // it has its cores. Nothing started, the worktree is as it was, and the
+        // container accepts again in moments — the typed word, for the thread
+        // routes to answer with the wait token; never counted as unreachable.
+        if (isRuntimeBusy(err)) {
+          throw new SandboxRuntimeBusyError({ containerId: this.ctx.id.toString(), cause: errMsg(err) });
+        }
         // The control port never answered the SDK's connect (its 30 s abort,
         // raised inside the wake path): no process started and nothing about
         // the repository is known. Count it in storage — the ladder of item 64
@@ -4861,6 +4913,20 @@ export class ResidentDO extends Sandbox<Env> {
   ): Promise<AttachOk | ThreadErr> {
     try {
       await this.ensureHydrated();
+      // The fleet drain (item 69): a deploy is waiting for the runs in flight
+      // to end, and a NEW run's attach is refused with the record the bot
+      // waits on — a real 503 in the streamed document, read by the client as
+      // `draining`, never as the platform's transient. A run already in flight
+      // — registered from its attach to its release (item 44) — re-attaches
+      // through: a rolled container, an evicted worktree, a resumed run are
+      // the runs the drain waits FOR, and refusing them would hold the fleet
+      // closed on the run it is closed for. Read before the image reconcile so
+      // a refused attach never restarts a container.
+      const drain = await this.fleetDrain();
+      if (drain && !(await this.ctx.storage.get(runRegKey(threadKey)))) {
+        const refusal: ThreadErr & { draining: DrainRecord } = drainRefusal(drain);
+        return refusal;
+      }
       const resourceId = (await this.ctx.storage.get<string>(RESOURCE_KEY)) ?? "";
       if (await this.reconcileImage("attach")) {
         return {
@@ -4937,6 +5003,7 @@ export class ResidentDO extends Sandbox<Env> {
         state: s.state,
         stateReason: s.reason,
         reason: s.reason,
+        cause: "system",
       };
     }
     // `resourceId` was read by the caller a moment ago (item 15 of the audit:
@@ -4952,7 +5019,12 @@ export class ResidentDO extends Sandbox<Env> {
     // Typed `reason` beside the words: the client reads the field — a refusal no
     // wait clears, unlike the restore window's 503s — never the sentence.
     if (!record || !facts)
-      return { error: "not-serviceable: registry record or repo facts missing", status: 503, reason: "unregistered" };
+      return {
+        error: "not-serviceable: registry record or repo facts missing",
+        status: 503,
+        reason: "unregistered",
+        cause: "system",
+      };
 
     // The binding's ref wins for the thread's whole life, with one exception
     // (item 16): a thread bound to the repo default for want of a named branch
@@ -4981,6 +5053,7 @@ export class ResidentDO extends Sandbox<Env> {
         status: 409,
         needs: "ref",
         defaultRef: facts.defaultRef,
+        cause: "request",
       };
     }
     const worktreePath = prior?.worktreePath ?? (await threadWorktreePath(threadKey, ref));
@@ -5102,7 +5175,14 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
+        return {
+          error: errMsg(err),
+          status: 503,
+          state: s.state,
+          stateReason: s.reason,
+          reason: "mirror-busy",
+          cause: "system",
+        };
       }
       return catchAllErr(err, "attach-failed");
     }
@@ -5404,17 +5484,30 @@ export class ResidentDO extends Sandbox<Env> {
         return { error: `reuse-refused: ${err.why}`, status: 409, needs: "recreate" };
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
+        return {
+          error: errMsg(err),
+          status: 503,
+          state: s.state,
+          stateReason: s.reason,
+          reason: "mirror-busy",
+          cause: "system",
+        };
       }
       if (err instanceof StepError && err.step === "unknown-ref") {
-        return { error: `unknown-ref: ${err.message}`, status: 400 };
+        return { error: `unknown-ref: ${err.message}`, status: 400, cause: "request" };
       }
       if (err instanceof StepError && err.step === "stale-tip") {
         // Item 51: not a resident fault and not a caller fault — a fact about
         // the mirror at this instant. 409 with the state, so the bot's named
         // fallback runs cold at the commit it asked for.
         const s = await this.getStatus();
-        return { error: `stale-tip: ${err.message}`, status: 409, state: s.state, reason: "stale-tip" };
+        return {
+          error: `stale-tip: ${err.message}`,
+          status: 409,
+          state: s.state,
+          reason: "stale-tip",
+          cause: "system",
+        };
       }
       return this.attachFailed(err);
     }
@@ -5449,7 +5542,14 @@ export class ResidentDO extends Sandbox<Env> {
       // worktree lock above, so the bot-side fallback can retry.
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
+        return {
+          error: errMsg(err),
+          status: 503,
+          state: s.state,
+          stateReason: s.reason,
+          reason: "mirror-busy",
+          cause: "system",
+        };
       }
       return this.attachFailed(err);
     }
@@ -6293,6 +6393,7 @@ export class ResidentDO extends Sandbox<Env> {
         state: s.state,
         stateReason: s.reason,
         reason: s.reason,
+        cause: "system",
       };
     }
     const binding = await this.ctx.storage.get<ThreadBinding>(threadBindingKey(threadKey));
@@ -6381,6 +6482,9 @@ export class ResidentDO extends Sandbox<Env> {
       // container is unchanged, so no `replacedExecAnswer` gate applies.
       if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return this.replacedExecAnswer(err);
+      // A refused connect at the command's spawn (item 68): the wait token,
+      // the command never started.
+      if (err instanceof SandboxRuntimeBusyError) return runtimeBusyErr(err);
       throw err;
     }
   }
@@ -6461,6 +6565,7 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
+      if (err instanceof SandboxRuntimeBusyError) return runtimeBusyErr(err);
       throw err;
     }
     if (r.exitCode !== 0 || r.timedOut) return { error: `read-failed: ${describeStepFailure(r)}`, status: 404 };
@@ -6510,6 +6615,7 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
+      if (err instanceof SandboxRuntimeBusyError) return runtimeBusyErr(err);
       throw err;
     }
   }
@@ -6557,6 +6663,7 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof ControlResetError) return controlResetErr(err);
       if (err instanceof RuntimeReplacedError) return runtimeReplacedErr(err);
+      if (err instanceof SandboxRuntimeBusyError) return runtimeBusyErr(err);
       const step = err instanceof StepError ? ` at ${err.step}` : "";
       return { error: `write-failed${step}: ${errMsg(err)}`, status: 400 };
     }
@@ -6781,6 +6888,19 @@ export class ResidentDO extends Sandbox<Env> {
     const threadOps = [...this.threadOpsInFlight.values()].reduce((a, n) => a + n, 0);
     return threadOps + this.opUsersInUse.size + this.attachesInFlight;
   }
+  /** The fleet drain in force (item 69), read from the registry at this clock;
+   *  a registry that cannot be read is NO drain: a run must never fail because
+   *  a flag could not be read, and the deploy's own preflight fails closed on
+   *  its side (an unknown fleet refuses the deploy), so the failure lands on
+   *  the deploy, never on the run. Said in the log. */
+  private async fleetDrain(): Promise<DrainRecord | null> {
+    try {
+      return liveDrain(await this.registry().getDrain(), systemClock());
+    } catch (err) {
+      console.warn(`[drain] registry unreadable at attach — treating as no drain: ${errMsg(err)}`);
+      return null;
+    }
+  }
   /** In-flight activity for the deploy preflight (GET /status, GET /residents).
    *  The in-memory counters (a fresh isolate answers 0 for them — nothing of
    *  THEIRS survived to be interrupted) plus the durable run registrations:
@@ -6928,6 +7048,7 @@ export class ResidentDO extends Sandbox<Env> {
         state: s.state,
         stateReason: s.reason,
         reason: s.reason,
+        cause: "system",
       };
     }
     // One storage round trip for the two facts; the registry lookup stays (an
@@ -6939,7 +7060,12 @@ export class ResidentDO extends Sandbox<Env> {
     // Typed `reason` beside the words: the client reads the field — a refusal no
     // wait clears, unlike the restore window's 503s — never the sentence.
     if (!record || !facts)
-      return { error: "not-serviceable: registry record or repo facts missing", status: 503, reason: "unregistered" };
+      return {
+        error: "not-serviceable: registry record or repo facts missing",
+        status: 503,
+        reason: "unregistered",
+        cause: "system",
+      };
     const command = record.commands[op];
     if (!command) return { error: `op-unavailable: the command table has no "${op}" entry`, status: 400 };
 
@@ -7035,10 +7161,17 @@ export class ResidentDO extends Sandbox<Env> {
     } catch (err) {
       if (err instanceof MirrorBusyError) {
         const s = await this.getStatus();
-        return { error: errMsg(err), status: 503, state: s.state, stateReason: s.reason, reason: "mirror-busy" };
+        return {
+          error: errMsg(err),
+          status: 503,
+          state: s.state,
+          stateReason: s.reason,
+          reason: "mirror-busy",
+          cause: "system",
+        };
       }
       if (err instanceof StepError && err.step === "unknown-ref") {
-        return { error: `unknown-ref: ${err.message}`, status: 400 };
+        return { error: `unknown-ref: ${err.message}`, status: 400, cause: "request" };
       }
       // A step that failed is named and deterministic; a throw no step named is
       // typed by the one builder every such 500 goes through.
@@ -7821,7 +7954,7 @@ function timingSafeEqual(a: string, b: string): boolean {
   return diff === 0;
 }
 
-type Scope = "admin" | "operator" | "read";
+type Scope = "admin" | "operator" | "read" | "drain";
 
 /** Which token a bearer is, or null. Constant-time per comparison; fail closed
  *  on unset/empty secrets. */
@@ -7830,6 +7963,7 @@ function tokenScope(env: Env, token: string | null): Scope | null {
   if (env.RESIDENT_ADMIN_TOKEN && timingSafeEqual(token, env.RESIDENT_ADMIN_TOKEN)) return "admin";
   if (env.RESIDENT_OPERATOR_TOKEN && timingSafeEqual(token, env.RESIDENT_OPERATOR_TOKEN)) return "operator";
   if (env.RESIDENT_READ_TOKEN && timingSafeEqual(token, env.RESIDENT_READ_TOKEN)) return "read";
+  if (env.RESIDENT_DRAIN_TOKEN && timingSafeEqual(token, env.RESIDENT_DRAIN_TOKEN)) return "drain";
   return null;
 }
 
@@ -8009,6 +8143,8 @@ const ROUTES: Record<string, { scope: Scope; method: string }> = {
   "/offboard": { scope: "admin", method: "POST" },
   "/reconfigure": { scope: "admin", method: "POST" },
   "/rebuild": { scope: "admin", method: "POST" },
+  "/drain": { scope: "drain", method: "POST" }, // close the fleet to new runs for a deploy (item 69; admin implied)
+  "/undrain": { scope: "drain", method: "POST" }, // reopen it
   "/residents": { scope: "read", method: "GET" }, // admin implied; read-only bearer allowed
   "/debug": { scope: "read", method: "POST" }, // per-op: READ_DEBUG_OPS for read scope, everything for admin
   "/status": { scope: "operator", method: "GET" },
@@ -8109,6 +8245,10 @@ export default {
             return await handleReconfigure(env, body);
           case "/rebuild":
             return await handleRebuild(env, body);
+          case "/drain":
+            return await handleDrain(env, body);
+          case "/undrain":
+            return await handleUndrain(env);
           case "/residents":
             return await handleResidents(env);
           case "/debug": {
@@ -8239,6 +8379,7 @@ async function handleOnboard(env: Env, body: Record<string, unknown>): Promise<R
             `exact name (GitHub's token API answers the same 422 for both). An org admin adds it under the ` +
             `App's installation settings (Settings → GitHub Apps → Configure → Repository access), ` +
             `then retry (${errMsg(err)})`,
+          cause: "policy",
         },
         403,
       );
@@ -8529,6 +8670,26 @@ async function handleRebuild(env: Env, body: Record<string, unknown>): Promise<R
  *  targets a different DO, so they run concurrently; a failing one degrades
  *  to {error} without touching its neighbors, and the response order follows
  *  the registry list. */
+/** POST /drain (admin): close the fleet to new runs (docs/reference/specs/resident-repos.md
+ *  item 69). The record carries its own end (`until`), so a drain nobody lifts
+ *  ends by itself; a second drain replaces the first. Runs in flight are
+ *  untouched — the deploy that asked waits for them through `/residents`. */
+async function handleDrain(env: Env, body: Record<string, unknown>): Promise<Response> {
+  const parsed = parseDrainRequest(body, systemClock());
+  if (!parsed.ok) return json({ error: parsed.error }, 400);
+  const record = await registryStub(env).setDrain(parsed.record);
+  console.log(`[drain] fleet closed to new runs by ${record.by} for ${record.reason}: until ${record.until}`);
+  return json({ draining: record });
+}
+
+/** POST /undrain (admin): reopen the fleet. Idempotent — `cleared` says whether
+ *  a drain stood. */
+async function handleUndrain(env: Env): Promise<Response> {
+  const cleared = await registryStub(env).clearDrain();
+  console.log(`[drain] fleet reopened (${cleared ? "a drain stood" : "no drain stood"})`);
+  return json({ draining: null, cleared });
+}
+
 async function handleResidents(env: Env): Promise<Response> {
   const residents = await registryStub(env).list();
   const settled = await Promise.allSettled(
@@ -8564,6 +8725,9 @@ async function handleResidents(env: Env): Promise<Response> {
     count: residents.length,
     inFlight,
     inFlightUnknown,
+    // The drain in force, or null (item 69): the deploy runner and the
+    // dashboard read it here; the attach route reads the same record.
+    draining: liveDrain(await registryStub(env).getDrain(), systemClock()),
     residents: enriched,
   });
 }

package/dist/assets/deploy/cloudflare-sandbox/worker.ts

@@ -58,9 +58,14 @@ import {
   fleetBusyAnswer,
   fleetBusyExecAnswer,
   isFleetBusyError,
+  isRuntimeBusyError,
+  isRuntimeBusySignal,
   isRuntimeUnreachableError,
+  runtimeBusyAnswer,
+  runtimeBusyExecAnswer,
   runtimeUnreachableAnswer,
   runtimeUnreachableExecAnswer,
+  SandboxRuntimeBusyError,
   SandboxRuntimeUnreachableError,
   sandboxStartingAnswer,
   sandboxStartingExecAnswer,
@@ -182,6 +187,16 @@ function isRuntimeUnreachable(err: unknown): boolean {
   return false;
 }
 
+/** Did the platform refuse the connect inside its own accept allowance
+ *  (docs/reference/specs/execution.md item 28)? The platform's own wording,
+ *  anywhere in the cause chain; a plain `Error`, so the wording is all there
+ *  is — and its words blame load the platform never measured. Asked only of
+ *  a failure met before a process was started. */
+function isRuntimeBusy(err: unknown): boolean {
+  for (const link of selfAndCauses(err)) if (isRuntimeBusySignal(link)) return true;
+  return false;
+}
+
 /** A finished command, as `/exec` answers it. `durationMs` is the command's
  *  wall time in the sandbox (docs/reference/specs/tracing.md item 19). */
 export interface ExecAnswer {
@@ -194,7 +209,7 @@ export interface ExecAnswer {
 /** A command the sandbox never answered for, in the dual in-body shape
  *  (docs/reference/specs/execution.md item 3): a new executor throws on `error`, an
  *  older one still renders `exit 127: <stderr>`. `reason` names the machine
- *  token when there is one (`fleet-busy`, `runtime-unreachable`). */
+ *  token when there is one (`fleet-busy`, `runtime-busy`, `runtime-unreachable`). */
 export interface ExecFailure {
   error: string;
   reason?: string;
@@ -205,8 +220,8 @@ export interface ExecFailure {
 
 /** A file route's refusal, with the HTTP status the fetch handler answers and,
  *  when the refusal is a named condition the executor waits on (`fleet-busy`,
- *  `runtime-unreachable`), its machine token — the executor reads the token,
- *  never the text, so a refusal without it is a dead sandbox to it. */
+ *  `runtime-busy`, `runtime-unreachable`), its machine token — the executor
+ *  reads the token, never the text, so a refusal without it is a dead sandbox to it. */
 interface FileRefusal {
   error: string;
   status: number;
@@ -321,9 +336,10 @@ export class SwitchboardSandbox extends Sandbox<Env> {
           (cause) => sandboxStartingExecAnswer(cause),
         );
       } catch (err) {
-        // The warm-up's own failure, handed on by the gate: a full fleet or a
-        // silent control port keeps its name; anything else propagates.
-        return this.execFailure(err, startedAt);
+        // The warm-up's own failure, handed on by the gate: a full fleet, a
+        // refused connect or a silent control port keeps its name; anything
+        // else propagates. Nothing ran — the warm-up is a spawn.
+        return this.spawnFailure(err, startedAt);
       }
     });
   }
@@ -343,7 +359,10 @@ export class SwitchboardSandbox extends Sandbox<Env> {
     try {
       proc = await createExtensionProcessSandbox(this).exec(argv, { env: envVars, timeout: backstopMs });
     } catch (err) {
-      return this.execFailure(err, startedAt);
+      // The process was never started: a refused connect is the wait token
+      // here and only here (item 28) — the executor re-sends, and
+      // nothing runs twice.
+      return this.spawnFailure(err, startedAt);
     }
     try {
       const out = await proc.output({
@@ -631,6 +650,20 @@ export class SwitchboardSandbox extends Sandbox<Env> {
     return { stdout: out.stdout, stderr: out.stderr, exitCode: out.timedOut ? 124 : out.exitCode };
   }
 
+  /** A failure met BEFORE a process was started — the spawn, or the gate's
+   *  warm-up: a container that did not accept the connection is named with
+   *  its wait token (docs/reference/specs/execution.md item 28), since
+   *  nothing ran and the identical request is safe to re-send. Every other
+   *  failure is classified as after a start (`execFailure`). A failure of a
+   *  running command's output never comes here: the process exists, and a
+   *  re-send would run it again. */
+  private spawnFailure(err: unknown, startedAt: number): ExecFailure {
+    if (!isFleetBusyError(err) && !isRuntimeReplacement(err) && isRuntimeBusy(err)) {
+      return runtimeBusyExecAnswer(this.runtimeBusy(thrownText(thrownShape(err))).message);
+    }
+    return this.execFailure(err, startedAt);
+  }
+
   /** The named failures, as `/exec` data; anything else is thrown as it came. */
   private execFailure(err: unknown, startedAt: number): ExecFailure {
     const raw = thrownText(thrownShape(err));
@@ -660,15 +693,25 @@ export class SwitchboardSandbox extends Sandbox<Env> {
     });
   }
 
+  /** The typed, named error for a container that did not accept the
+   *  connection (item 28), with this container's id — thrown across the RPC
+   *  boundary to the fetch handler on the file routes, matched by name. */
+  private runtimeBusy(cause: string): SandboxRuntimeBusyError {
+    return new SandboxRuntimeBusyError({ containerId: this.ctx.id.toString(), cause });
+  }
+
   /** A file operation with its runtime failures named for the fetch handler:
-   *  a silent control port becomes the typed error (item 9); a missing file
-   *  is the refusal the route answers 404. The SDK's other errors propagate. */
+   *  a refused connect (item 28) and a silent control port (item 9) become
+   *  the typed errors; a missing file is the refusal the route answers 404.
+   *  The SDK's other errors propagate. A file operation that met either
+   *  never reached the runtime, so the executor's re-send does nothing twice. */
   private async fileOp<T>(op: () => Promise<T>): Promise<T | FileRefusal> {
     try {
       return await op();
     } catch (err) {
       const shape = thrownShape(err);
       if (shape.name === "FileNotFoundError") return { error: `read-failed: ${thrownText(shape)}`, status: 404 };
+      if (!isRuntimeReplacement(err) && isRuntimeBusy(err)) throw this.runtimeBusy(thrownText(shape));
       if (!isRuntimeReplacement(err) && isRuntimeUnreachable(err)) throw this.runtimeUnreachable(thrownText(shape));
       throw err;
     }
@@ -884,6 +927,9 @@ export default {
       // op never reached a runtime, so a 503 the executor's transport retry
       // re-sends, with the named reason and the container in the text.
       if (isRuntimeUnreachableError(err)) return json(runtimeUnreachableAnswer(msg), 503);
+      // The container did not accept the connection (item 28): the file op
+      // never reached it either — a 503 with the wait token.
+      if (isRuntimeBusyError(err)) return json(runtimeBusyAnswer(msg), 503);
       // A full fleet (docs/reference/specs/execution.md item 14): no container
       // instance for this thread's Durable Object, so the file op never
       // started — re-sending is safe by construction. Named so the executor
@@ -972,6 +1018,7 @@ function streamExec(run: () => Promise<ExecAnswer | ExecFailure>, traceparent: s
       const shape = thrownShape(err);
       const raw = thrownText(shape);
       if (isRuntimeUnreachableError(err)) return runtimeUnreachableExecAnswer(raw);
+      if (isRuntimeBusyError(err)) return runtimeBusyExecAnswer(raw);
       if (isFleetBusyError(err)) return fleetBusyExecAnswer(raw);
       // A recycle the Durable Object did not catch by type: by name, or
       // a recycle-shaped text minutes into the attempt (item 9).

package/dist/assets/deploy/secrets.manifest.json

@@ -81,6 +81,12 @@
       "workers": ["resident"],
       "note": "Read-only /residents + debug routes; what the resident deploy preflight needs (CI holds this one). Self-minted."
     },
+    {
+      "name": "RESIDENT_DRAIN_TOKEN",
+      "workers": ["resident"],
+      "optional": true,
+      "note": "Drain-only bearer: POST /drain and /undrain, nothing else (docs/reference/specs/resident-repos.md item 69). What the release deploy holds (CI) so the resident step can close the fleet to new runs and land once the runs in flight end, without the admin bearer. Self-minted; optional — unset, only admin can drain and the deploy waits without one."
+    },
     {
       "name": "R2_ACCESS_KEY_ID",
       "workers": ["resident", "sandbox"],

package/dist/assets/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "switchboard",
-  "version": "1.247.0",
+  "version": "1.249.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "switchboard",
-      "version": "1.247.0",
+      "version": "1.249.0",
       "license": "Apache-2.0",
       "workspaces": [
         "web",
@@ -20445,7 +20445,7 @@
     },
     "packages/switchboard": {
       "name": "@coreplane/switchboard",
-      "version": "1.247.0",
+      "version": "1.249.0",
       "license": "Apache-2.0",
       "dependencies": {
         "@earendil-works/pi-ai": "0.85.1",

package/dist/assets/package.json

@@ -1,6 +1,6 @@
 {
   "name": "switchboard",
-  "version": "1.247.0",
+  "version": "1.249.0",
   "private": true,
   "description": "Mention it in Slack and an agent reviews the PR, ships the fix, or answers the question — on the model you choose, with its tools running where you decide.",
   "license": "Apache-2.0",

package/dist/assets/source.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.247.0",
-  "commit": "713b83c9db0b6f3f10ef6d2cc1a46714b9a23c3c",
-  "builtAt": "2026-09-18T03:41:18.440Z"
+  "version": "1.249.0",
+  "commit": "1145eea8ffdd6a2bd1ae8f1b26a4919db2caf47d",
+  "builtAt": "2026-09-18T07:16:19.891Z"
 }