@staix/agent-hub 0.4.0 → 0.5.0

package/src/hub/budget.ts

@@ -76,6 +76,9 @@ export class Budget {
   /** `ahub budget resume`: the owner overrode a pause; readings over the gate are ignored for that peer until then. */
   private readonly ignoreUntil = new Map<PeerId, number>();
   private closed = false;
+  private recoveryHeld = false;
+  private deferred = false;
+  private readonly deferredHard = new Set<PeerId>();
 
   constructor(
     dbPath: string,
@@ -88,11 +91,33 @@ export class Budget {
       reason TEXT NOT NULL, summary TEXT, handed_off INTEGER NOT NULL DEFAULT 0, moved TEXT NOT NULL DEFAULT '[]')`);
   }
 
+  /** Freeze pause/resume and handoff transitions while a controlled restart is fenced. Readings continue to update. */
+  setRecoveryHold(held: boolean): void {
+    this.recoveryHeld = held;
+    if (!held && this.deferred && !this.closed) {
+      this.deferred = false;
+      this.tick();
+      for (const peer of this.readings.keys()) {
+        const hard = this.deferredHard.delete(peer);
+        void this.evaluate(peer, hard).catch((e: Error) => this.closed || this.hooks.notify(`budget: evaluating ${peer} failed: ${e.message}`));
+      }
+    }
+  }
+
+  get recoverySettled(): boolean {
+    return this.pausing.size === 0 && this.handingOff.size === 0;
+  }
+
   record(peer: PeerId): PauseRecord | undefined {
     const r = this.db.query("SELECT * FROM budget_pauses WHERE peer = ?").get(peer) as Record<string, any> | null;
     return r ? { peer: r.peer, since: r.since, resetsAt: r.resets_at, reason: r.reason, summary: r.summary, handedOff: !!r.handed_off, moved: JSON.parse(r.moved) } : undefined;
   }
 
+  /** Recovery integrity input: persisted pause records only, excluding live usage readings and private summaries. */
+  persistedPauseDigestRows(): { peer: PeerId; since: number; resetsAt: number; reason: string }[] {
+    return this.records().map(({ peer, since, resetsAt, reason }) => ({ peer, since, resetsAt, reason }));
+  }
+
   private records(): PauseRecord[] {
     return (this.db.query("SELECT peer FROM budget_pauses").all() as { peer: string }[]).map((r) => this.record(r.peer)!);
   }
@@ -122,6 +147,7 @@ export class Budget {
     const mine = this.readings.get(peer) ?? new Map<string, Reading>();
     this.readings.set(peer, mine);
     for (const w of windows) mine.set(w.id, { ...w, used: Math.max(0, Math.min(1, w.used)), at });
+    if (this.recoveryHeld) { this.deferred = true; if (hard) this.deferredHard.add(peer); return; }
     this.evaluate(peer, hard).catch((e: Error) => this.closed || this.hooks.notify(`budget: evaluating ${peer} failed: ${e.message}`));
   }
 
@@ -141,6 +167,7 @@ export class Budget {
   }
 
   private async evaluate(peer: PeerId, hard: boolean): Promise<void> {
+    if (this.recoveryHeld) { this.deferred = true; if (hard) this.deferredHard.add(peer); return; }
     const fresh = this.fresh(peer);
     const over = fresh.filter((r) => r.used >= this.cfg.gate);
     const open = this.record(peer);
@@ -160,6 +187,7 @@ export class Budget {
       const reason = `${worst.id} window at ${Math.round(worst.used * 100)}% (${worst.source})`;
       // Checkpoint first, pause second: a paused peer receives nothing, and a turn in flight is never cut by the coordinator.
       const summary = hard ? undefined : await this.hooks.requestCheckpoint(peer).catch(() => undefined);
+      if (this.recoveryHeld) { this.deferred = true; return; }
       if (this.closed) return; // the hub is shutting down: nothing is recorded, the next run sees the reading again
       this.db.query("INSERT OR REPLACE INTO budget_pauses (peer, since, resets_at, reason, summary) VALUES (?, ?, ?, ?, ?)").run(peer, this.now(), resetsAt, reason, summary ?? null);
       this.hooks.pause(peer);
@@ -171,10 +199,12 @@ export class Budget {
   }
 
   private async finishHandoff(peer: PeerId, summary: string | undefined): Promise<void> {
-    if (this.closed || this.handingOff.has(peer) || !this.hooks.canHandOff(peer)) return; // retried by the next tick
+    if (this.closed || this.recoveryHeld) { this.deferred = true; return; }
+    if (this.handingOff.has(peer) || !this.hooks.canHandOff(peer)) return; // retried by the next tick
     this.handingOff.add(peer);
     try {
       await this.handOff(peer, summary);
+      if (this.recoveryHeld) { this.deferred = true; return; }
     } finally {
       this.handingOff.delete(peer);
     }
@@ -202,6 +232,7 @@ export class Budget {
 
   /** Called on a timer. Resumes every peer whose window has reset. */
   tick(): void {
+    if (this.recoveryHeld) { this.deferred = true; return; }
     for (const r of this.records()) {
       if (this.now() >= r.resetsAt + 60_000) {
         if (this.hooks.attached(r.peer)) this.resume(r.peer, "the window has reset"); // else: wait, the notice needs somebody to receive it

package/src/hub/bus.ts

@@ -17,6 +17,17 @@ export interface BusOptions {
   /** Optional: rewrite a delivery before it goes out (M6 digest condensation). Must return its input on any failure. */
   condense?: (envs: Envelope[]) => Promise<Envelope[]>;
 }
+
+/** Serializable delivery state used by the controlled restart coordinator. Bodies stay in the private daemon file. */
+export interface BusSnapshot {
+  schemaVersion: 1;
+  queues: Record<string, Envelope[]>;
+  prefaces: Record<string, Envelope>;
+  seen: Envelope[];
+  attempts: Record<string, number>;
+  withdrawn: string[];
+}
+
 export const DEFAULT_BUS: BusOptions = { retryMs: 1000, batchMax: 3, batchMs: 15_000, queueCap: 200 };
 
 const SEEN_CAP = 2048;
@@ -42,6 +53,9 @@ export class Bus {
   /** What each peer was last handed, next to what it stands for: an adapter that reports a failure later hands back the former. */
   private readonly lastDelivery = new Map<PeerId, { out: Envelope[]; originals: Envelope[] }>();
   private readonly attempts = new Map<string, number>(); // `${peer}:${envelope id}` -> failed deliveries
+  private recoveryHeld = false;
+  private steering = 0;
+  private condensing = 0;
 
   constructor(opts: Partial<BusOptions> = {}) {
     this.opts = { ...DEFAULT_BUS, ...opts };
@@ -66,6 +80,57 @@ export class Bus {
     };
   }
 
+  /** Stop new deliveries while a coordinator takes a stable snapshot. Existing adapter turns are left alone. */
+  setRecoveryHold(held: boolean): void {
+    this.recoveryHeld = held;
+    if (!held) for (const id of this.peers.keys()) void this.drain(id);
+  }
+
+  get isRecoveryHeld(): boolean {
+    return this.recoveryHeld;
+  }
+
+  /** Wait until a hold has fenced condensation, steering and queue drains already in flight. */
+  async fenceRecovery(): Promise<void> {
+    this.recoveryHeld = true;
+    while (this.draining.size || this.steering || this.condensing) await Bun.sleep(0);
+  }
+
+  /** A bounded, JSON-safe representation of queued work and retry/dedupe state. */
+  snapshot(): BusSnapshot {
+    return {
+      schemaVersion: 1,
+      queues: Object.fromEntries([...this.queues].map(([id, queue]) => [id, queue.map((e) => ({ ...e, ...(e.to ? { to: [...e.to] } : {}) }))])),
+      prefaces: Object.fromEntries([...this.prefaces].map(([id, e]) => [id, { ...e, ...(e.to ? { to: [...e.to] } : {}) }])),
+      seen: [...this.seen.values()].map((e) => ({ ...e, ...(e.to ? { to: [...e.to] } : {}) })),
+      attempts: Object.fromEntries(this.attempts),
+      withdrawn: [...this.withdrawn],
+    };
+  }
+
+  /** Restore state before peers attach. Queues remain held until the coordinator calls setRecoveryHold(false). */
+  restore(snapshot: BusSnapshot): void {
+    if (snapshot.schemaVersion !== 1) throw new Error("unsupported bus recovery snapshot");
+    this.queues.clear();
+    for (const [id, queue] of Object.entries(snapshot.queues ?? {})) this.queues.set(id, queue.map((e) => ({ ...e, ...(e.to ? { to: [...e.to] } : {}) })));
+    this.prefaces.clear();
+    for (const [id, e] of Object.entries(snapshot.prefaces ?? {})) this.prefaces.set(id, { ...e, ...(e.to ? { to: [...e.to] } : {}) });
+    this.seen.clear();
+    for (const e of snapshot.seen ?? []) this.seen.set(e.id, { ...e, ...(e.to ? { to: [...e.to] } : {}) });
+    this.attempts.clear();
+    for (const [key, n] of Object.entries(snapshot.attempts ?? {})) if (Number.isInteger(n) && n > 0) this.attempts.set(key, n);
+    this.withdrawn.clear();
+    for (const id of snapshot.withdrawn ?? []) this.withdrawn.add(id);
+  }
+
+  queueIds(id: PeerId): string[] {
+    return (this.queues.get(id) ?? []).map((e) => e.id);
+  }
+
+  attemptState(): Record<string, number> {
+    return Object.fromEntries(this.attempts);
+  }
+
   tap(fn: (e: BusEvent) => void): () => void {
     this.taps.add(fn);
     return () => this.taps.delete(fn);
@@ -90,6 +155,13 @@ export class Bus {
 
   /** Context the hub wants the peer to see once: it rides in front of the next delivery instead of costing a turn of its own. */
   preface(id: PeerId, body: string): void {
+    const existing = this.prefaces.get(id);
+    if (existing) {
+      // Recovery and session recall may both contribute context. Keep the original id/hop so a queued
+      // preface remains deduplicated and append the new, separately generated context.
+      this.prefaces.set(id, { ...existing, body: `${existing.body}\n\n${body}` });
+      return;
+    }
     this.prefaces.set(id, newEnvelope(HUB, body, { to: [id], kind: "presence" }));
   }
 
@@ -107,9 +179,10 @@ export class Bus {
     const targets = (env.to ?? [...this.peers.keys()]).filter((id) => id !== env.from && this.peers.has(id));
     for (const id of targets) {
       const peer = this.peers.get(id)!;
-      if (env.priority === "important" && peer.steer && this.stateOf(id) === "busy") {
+      if (!this.recoveryHeld && env.priority === "important" && peer.steer && this.stateOf(id) === "busy") {
         // Not queued while the steer is in flight, or an idle transition would deliver it a second time.
-        peer.steer([env]).catch(() => this.enqueue(id, env, true));
+        this.steering++;
+        peer.steer([env]).catch(() => this.enqueue(id, env, true)).finally(() => { this.steering--; });
       } else this.enqueue(id, env);
     }
     return targets;
@@ -165,7 +238,7 @@ export class Bus {
     try {
       const peer = this.peers.get(id)!;
       const queue = this.queues.get(id)!;
-      while (queue.length && this.stateOf(id) === "idle") {
+      while (!this.recoveryHeld && queue.length && this.stateOf(id) === "idle") {
         const wait = this.wait(queue);
         if (wait > 0) {
           this.arm(id, wait);
@@ -178,8 +251,10 @@ export class Bus {
         // What goes out may be condensed; what comes back on failure is always the originals. An important envelope
         // is why the queue became ready, so it never waits on a model call.
         const mayCondense = this.opts.condense && !delivery.some((e) => e.priority === "important");
+        this.condensing += mayCondense ? 1 : 0;
         const out = mayCondense ? await this.opts.condense!(delivery).catch(() => delivery) : delivery;
-        if (mayCondense && this.stateOf(id) !== "idle") {
+        this.condensing -= mayCondense ? 1 : 0;
+        if (this.recoveryHeld || (mayCondense && this.stateOf(id) !== "idle")) {
           // The peer got busy while the delivery was prepared. Nothing was attempted: back to the head of the queue,
           // without counting against the envelopes.
           if (preface) this.prefaces.set(id, preface);

package/src/hub/child-process.ts

@@ -1,5 +1,16 @@
 import type { ChildProcess } from "node:child_process";
 
+/**
+ * Recovery authority belongs to the short-lived ahub wrapper/launcher. Native
+ * agents and their long-lived children must never inherit it, or their later
+ * lifecycle calls could bypass the machine recovery lock.
+ */
+export function childEnv(source: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv {
+  const env = { ...source };
+  delete env.AGENTHUB_RECOVERY_OPERATION;
+  return env;
+}
+
 /**
  * Stop a child owned by this hub and confirm that it exited. A timeout is an
  * incomplete shutdown, even after SIGKILL, because callers must not reuse a

package/src/hub/control-client.ts

@@ -7,7 +7,7 @@ export function stateDirFor(cwd: string): string {
 }
 
 /** Control WS wire version. 2 = `deliver` carries `envs` (digests); 3 = `tools` role and task messages; 4 = budget messages and `hub_checkpoint`; 5 = `ask`; 6 = console-only `ui` session bootstrap. The plugin is installed apart from the daemon, so they can drift. */
-export const PROTOCOL = 7; // project/instance identity and console-only dashboard forwarding
+export const PROTOCOL = 8; // controlled daemon restart/recovery RPC and project/instance identity
 
 export interface Hello {
   /** `tools`: acts for `peer` (task tools, hub_send) without being a delivery target: the MCP server Kimi and Codex run. */

package/src/hub/daemon.ts

@@ -1,5 +1,5 @@
-import { randomBytes, randomUUID } from "node:crypto";
-import { appendFileSync, chmodSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { createHash, randomBytes, randomUUID } from "node:crypto";
+import { appendFileSync, chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import type { ServerWebSocket } from "bun";
 import { AcpPeer, type PermissionRequest } from "../adapters/acp.ts";
@@ -28,6 +28,7 @@ import { BasePeer, DEFAULT_WATCHDOG_MS } from "./peers.ts";
 import { MemoryClient, workerUrl } from "../memory/client.ts";
 import { VERSION } from "../version.ts";
 import { projectChain, recallFor } from "../memory/recall.ts";
+import { archiveRestartSnapshot, readRestartSnapshot, removeRestartSnapshot, restartPath, writeRestartSnapshot, type RecoveryPhase, type RestartPeerSnapshot, type RestartSnapshot } from "./restart.ts";
 
 export interface HubConfig {
   watchdog_ms: number;
@@ -134,6 +135,10 @@ class WsPeer extends BasePeer {
   async stop(): Promise<void> {
     this.sock?.close();
   }
+
+  recoveryMetadata(): Record<string, unknown> {
+    return { launch: { kind: "claude-channel", peer: this.id } };
+  }
 }
 
 export async function startDaemon(opts: DaemonOptions) {
@@ -181,6 +186,43 @@ export async function startDaemon(opts: DaemonOptions) {
   // The hub's own model calls (digest condensation, task triage) are wired below, once the gateway client exists.
   let inference: Inference | undefined;
   const bus = new Bus({ batchMax: config.batch_max, batchMs: config.batch_ms, queueCap: config.queue_cap, condense: (envs) => inference?.condense(envs) ?? Promise.resolve(envs) });
+  const manualPaused = new Set<PeerId>(); // `ahub pause`: recovery never lifts these
+  let recoveryOperationId: string | undefined;
+  let recoveryPhase: RecoveryPhase | undefined;
+  let recoveryCommitted = false;
+  let recoveryPeerSnapshot: RestartPeerSnapshot[] | undefined;
+  let recoveryLeaseTimer: ReturnType<typeof setTimeout> | undefined;
+  const recoveryActive = () => !!recoveryOperationId && recoveryPhase !== "released";
+  const recoveryOperation = process.env.AGENTHUB_RECOVERY_OPERATION;
+  const restartFilePresent = existsSync(restartPath(opts.stateDir));
+  const restored = recoveryOperation
+    ? readRestartSnapshot(opts.stateDir, { projectRoot: opts.cwd, projectId, operationId: recoveryOperation })
+    : undefined;
+  if ((restartFilePresent && !restored) || (recoveryOperation && !restored)) throw new Error("restart state is unreadable, missing, or does not match this project and recovery operation");
+  if (restored) {
+    bus.restore(restored.bus);
+    for (const peer of restored.manualPaused) { manualPaused.add(peer); bus.pause(peer); }
+    recoveryOperationId = restored.operationId;
+    recoveryPhase = "restored";
+    recoveryPeerSnapshot = restored.peers;
+    bus.setRecoveryHold(true);
+  }
+  const recoveryPeerAllowed = (id: PeerId) => !recoveryActive() || !recoveryPeerSnapshot || recoveryPeerSnapshot.some((peer) => peer.id === id);
+  const armRecoveryLease = () => {
+    clearTimeout(recoveryLeaseTimer);
+    recoveryLeaseTimer = setTimeout(() => {
+      if (!recoveryOperationId || recoveryCommitted || (recoveryPhase !== "preparing" && recoveryPhase !== "prepared")) return;
+      log(`recovery lease expired for ${recoveryOperationId}; hold aborted`);
+      recoveryOperationId = undefined;
+      recoveryPhase = undefined;
+      recoveryPeerSnapshot = undefined;
+      bus.setRecoveryHold(false);
+      budget.setRecoveryHold(false);
+      removeRestartSnapshot(opts.stateDir);
+      writeStatus();
+    }, 10 * 60_000);
+    recoveryLeaseTimer.unref?.();
+  };
   const memory = new MemoryClient(config.memory.worker_url ?? workerUrl(), 2000, log);
   const chain = projectChain(opts.cwd);
   const recalled = new Set<PeerId>(); // once per peer per hub run, however often the peer's session restarts
@@ -227,7 +269,6 @@ export async function startDaemon(opts: DaemonOptions) {
     triage: { classify: (title, detail) => inference?.triage(title, detail) ?? Promise.resolve(undefined), onCampus: () => onCampus() },
   });
   // ---- budget relay -------------------------------------------------------------------------------------------
-  const manualPaused = new Set<PeerId>(); // `ahub pause`: the coordinator never lifts these
   const checkpointWaits = new Map<PeerId, (summary: string | undefined) => void>();
   const PLATFORM: Record<PeerId, string> = { claude: "claude", codex: "codex", kimi: "kimi" };
   const budget = new Budget(join(opts.stateDir, "hub.db"), config.budget, {
@@ -270,6 +311,7 @@ export async function startDaemon(opts: DaemonOptions) {
     },
     notify: (line) => notify(line),
   });
+  budget.setRecoveryHold(recoveryActive());
   const kimiTokens: { at: number; n: number }[] = [];
   startupCleanup.push(() => budget.close());
   let kimiSessionTotal = 0;
@@ -363,6 +405,16 @@ export async function startDaemon(opts: DaemonOptions) {
     }
     throw new Error(`unknown task operation ${op}`);
   }
+  function recoveryTaskPreface(peer: PeerId): void {
+    if (recoveryPhase !== "restored") return;
+    const open = board.list().filter((task) => (task.owner === peer || task.reviewer === peer) && !["approved"].includes(task.state));
+    if (!open.length) return;
+    const lines = open.map((task) => {
+      const view = tasks.publicView(task) as { title?: unknown; detail?: unknown; state?: unknown; owner?: unknown; reviewer?: unknown };
+      return `#${task.id} ${String(view.title ?? "[private]")} (${task.state}, owner ${task.owner ?? "none"}, reviewer ${task.reviewer ?? "none"})${view.detail && view.detail !== "[pii]" ? `\n${String(view.detail).slice(0, 1000)}` : ""}`;
+    });
+    bus.preface(peer, `Controlled restart restored your open task context. Check the board before acting:\n${lines.join("\n\n")}`);
+  }
   const permissions = new Map<string, { push: string; done: (optionId: string | undefined) => void }>();
   let stopping = false;
 
@@ -370,6 +422,45 @@ export async function startDaemon(opts: DaemonOptions) {
     const r = budget.record(id); // one read per peer: status.json is rewritten on every bus event
     return r ? { paused: `budget: ${r.reason}, resets ${new Date(r.resetsAt).toLocaleTimeString()}` } : {};
   };
+  const recoveryReady = () => {
+    if (!recoveryActive() || permissions.size !== 0 || starting.size !== 0 || !budget.recoverySettled || [...bus.peers.values()].some((peer) => peer.state === "busy")) return false;
+    if (!recoveryPeerSnapshot) return true;
+    const current = recoveryPeers();
+    return recoveryPeerSnapshot.every((saved) => {
+      const now = current[saved.id];
+      if (!now) return true; // a detached peer is checked by the coordinator before terminal close
+      if (saved.threadId && now.threadId && saved.threadId !== now.threadId) return false;
+      if (saved.sessionId && now.sessionId && saved.sessionId !== now.sessionId) return false;
+      return true;
+    });
+  };
+  const claudeSessionId = () => {
+    try {
+      const value = JSON.parse(readFileSync(join(opts.stateDir, "claude-session.json"), "utf8"));
+      if (value.instanceId !== instanceId) return undefined;
+      try {
+        const records = JSON.parse(readFileSync(join(opts.stateDir, "terminal-recovery.json"), "utf8"));
+        const current = Array.isArray(records) ? records.find((row) => row?.peer === "claude" && row?.projectRoot === opts.cwd && row?.instanceId === instanceId) : undefined;
+        if (current?.launchId && value.launchId !== current.launchId) return undefined;
+      } catch { /* no managed terminal record: the instance fence is still enforced */ }
+      return typeof value.sessionId === "string" && value.sessionId ? value.sessionId : undefined;
+    } catch { return undefined; }
+  };
+  const recoveryPeers = (): Record<string, RestartPeerSnapshot> => Object.fromEntries([...bus.peers].map(([id, peer]) => {
+    const metadata = peer.recoveryMetadata?.() ?? {};
+    const row: RestartPeerSnapshot = { id, state: peer.state, queueIds: bus.queueIds(id), ...(metadata.launch ? { launch: metadata.launch as Record<string, unknown> } : {}) };
+    if (typeof metadata.threadId === "string") row.threadId = metadata.threadId;
+    const sessionId = id === "claude" ? claudeSessionId() : metadata.sessionId;
+    if (typeof sessionId === "string" && sessionId) row.sessionId = sessionId;
+    return [id, row];
+  }));
+  const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
+  const integrity = () => {
+    const queues = Object.fromEntries(Object.keys(bus.snapshot().queues).sort().map((id) => [id, bus.queueIds(id)]));
+    const boardState = board.list().sort((a, b) => a.id - b.id);
+    const budgetState = budget.persistedPauseDigestRows().sort((a, b) => a.peer.localeCompare(b.peer));
+    return { queues, manualPaused: [...manualPaused].sort(), boardDigest: digest(boardState), budgetDigest: digest(budgetState) };
+  };
   const status = () => ({
     projectId,
     instanceId,
@@ -386,6 +477,7 @@ export async function startDaemon(opts: DaemonOptions) {
     ),
     ...(sidecar ? { switchyard: sidecar.status } : {}),
     tasks: board.counts(),
+    ...(recoveryOperationId ? { recovery: { operationId: recoveryOperationId, phase: recoveryPhase, ready: recoveryReady() } } : {}),
   });
   const writeStatus = () => {
     const file = join(opts.stateDir, "status.json"); // clients parse this on every connect: replace it atomically
@@ -447,6 +539,7 @@ export async function startDaemon(opts: DaemonOptions) {
       const cmd = args.model ? [bin!, "--model", args.model, ...rest] : config.kimi_cmd;
       const kimi = new AcpPeer("kimi", {
         cmd,
+        ...(args.model ? { launchModel: args.model } : {}),
         cwd: opts.cwd,
         watchdogMs: config.watchdog_ms,
         onPermission,
@@ -455,6 +548,7 @@ export async function startDaemon(opts: DaemonOptions) {
         mcpServers: [{ name: "agent-hub", command: "bun", args: ["run", SERVER_JS], env: Object.entries(toolEnv("kimi")).map(([name, value]) => ({ name, value })) }],
         preamble: roleContract("kimi", config.roles),
       });
+      recoveryTaskPreface("kimi");
       await ensurePreface("kimi");
       bus.add(kimi);
       await kimi.start();
@@ -480,6 +574,7 @@ export async function startDaemon(opts: DaemonOptions) {
         watchdogMs: config.watchdog_ms,
         log,
       });
+      recoveryTaskPreface("codex");
       await ensurePreface("codex");
       bus.add(codex);
       await codex.start();
@@ -526,6 +621,7 @@ export async function startDaemon(opts: DaemonOptions) {
         maxSteps: config.local.max_steps,
         log,
       });
+      recoveryTaskPreface("local");
       await ensurePreface("local");
       bus.add(local);
       await local.start();
@@ -611,6 +707,122 @@ export async function startDaemon(opts: DaemonOptions) {
     }
   }
 
+  const recoveryView = () => ({
+    operationId: recoveryOperationId,
+    phase: recoveryPhase,
+    ready: recoveryReady(),
+    pendingApprovals: permissions.size,
+    capabilities: { controlledRestart: true, snapshotSchemaVersion: 1, maxLeaseMs: 10 * 60_000 },
+    blockers: [
+      ...(starting.size ? ["peer startup in progress"] : []),
+      ...(!budget.recoverySettled ? ["budget transition in progress"] : []),
+      ...([...bus.peers].filter(([, peer]) => peer.state === "busy").map(([id]) => `${id} is busy`)),
+      ...(permissions.size ? ["pending approvals"] : []),
+    ],
+    integrity: { current: integrity(), ...(restored?.integrity ? { expected: restored.integrity } : {}) },
+    peers: Object.fromEntries([...(recoveryPeerSnapshot ?? []), ...Object.values(recoveryPeers()).filter((peer) => !(recoveryPeerSnapshot ?? []).some((saved) => saved.id === peer.id))].map((peer) => {
+      const now = recoveryPeers()[peer.id];
+      if (recoveryPhase === "restored" || recoveryPhase === "released") return [peer.id, now ?? { id: peer.id, state: "offline", queueIds: bus.queueIds(peer.id) }];
+      return [peer.id, now ? { ...peer, ...now, queueIds: now.queueIds } : peer];
+    })),
+  });
+  const recoveryError = (error: string) => ({ t: "recovery", ok: false, error });
+  async function recoveryOp(msg: any): Promise<Record<string, unknown>> {
+    if (typeof msg.op !== "string" || !["inspect", "prepare", "commit", "abort", "release"].includes(msg.op)) return recoveryError("unknown recovery operation");
+    if (typeof msg.expectedInstanceId !== "string" || msg.expectedInstanceId !== instanceId) return recoveryError("expected daemon instance does not match");
+    if (msg.op === "inspect" && msg.operationId === undefined) return { t: "recovery", ok: true, recovery: recoveryView() };
+    if (typeof msg.operationId !== "string" || msg.operationId.length < 1 || msg.operationId.length > 128) return recoveryError("operationId is required");
+    const op = msg.operationId as string;
+    if (msg.op === "prepare") {
+      if (recoveryCommitted) return recoveryError("recovery commit is already in progress");
+      if (recoveryPhase === "released" && recoveryOperationId !== op) {
+        recoveryOperationId = undefined;
+        recoveryPhase = undefined;
+        recoveryPeerSnapshot = undefined;
+      }
+      if (recoveryOperationId && recoveryOperationId !== op) return recoveryError("another recovery operation is active");
+      recoveryOperationId = op;
+      if (!recoveryPhase) recoveryPhase = "preparing";
+      armRecoveryLease();
+      recoveryPeerSnapshot ??= Object.values(recoveryPeers());
+      budget.setRecoveryHold(true);
+      bus.setRecoveryHold(true);
+      await bus.fenceRecovery();
+      const blocked = [...bus.peers].filter(([, peer]) => peer.state === "busy").map(([id]) => id);
+      if (!blocked.length && permissions.size === 0 && recoveryReady()) {
+        recoveryPeerSnapshot ??= Object.values(recoveryPeers());
+        recoveryPhase = "prepared";
+      }
+      writeStatus();
+      return { t: "recovery", ok: true, recovery: { ...recoveryView(), ...(blocked.length ? { blockedPeers: blocked } : {}), ...(permissions.size ? { blocked: "pending approvals" } : {}) } };
+    }
+    if (!recoveryOperationId || recoveryOperationId !== op) return msg.op === "inspect"
+      ? { t: "recovery", ok: true, recovery: recoveryView() }
+      : recoveryError("operation does not match this daemon");
+    if (msg.op === "inspect") return { t: "recovery", ok: true, recovery: recoveryView() };
+    if (msg.op === "abort") {
+      if (recoveryCommitted) return recoveryError("a committed recovery cannot be aborted");
+      if (recoveryPhase === "released") return { t: "recovery", ok: true, recovery: recoveryView() };
+      recoveryOperationId = undefined;
+      recoveryPhase = undefined;
+      recoveryPeerSnapshot = undefined;
+      clearTimeout(recoveryLeaseTimer);
+      bus.setRecoveryHold(false);
+      budget.setRecoveryHold(false);
+      writeStatus();
+      return { t: "recovery", ok: true, aborted: true, recovery: recoveryView() };
+    }
+    if (msg.op === "commit") {
+      if ((recoveryPhase !== "prepared" && recoveryPhase !== "preparing") || !recoveryReady()) return recoveryError("recovery is not ready; inspect until peers are idle and approvals are complete");
+      recoveryPhase = "prepared";
+      recoveryPeerSnapshot ??= Object.values(recoveryPeers());
+      const currentPeers = recoveryPeers();
+      const peers = (recoveryPeerSnapshot ?? Object.values(currentPeers)).map((saved) => ({
+        ...saved,
+        queueIds: bus.queueIds(saved.id),
+      }));
+      const snapshot: RestartSnapshot = {
+        schemaVersion: 1,
+        projectRoot: opts.cwd,
+        projectId,
+        sourceInstanceId: instanceId,
+        operationId: op,
+        committedAt: Date.now(),
+        bus: bus.snapshot(),
+        manualPaused: [...manualPaused],
+        peers,
+        integrity: integrity(),
+      };
+      try { writeRestartSnapshot(opts.stateDir, snapshot); } catch { return recoveryError("could not persist restart state"); }
+      recoveryCommitted = true;
+      clearTimeout(recoveryLeaseTimer);
+      writeStatus();
+      // Keep phase prepared in status until the replacement daemon reports restored.
+      setTimeout(() => void stop().catch((error) => log(`recovery shutdown incomplete: ${(error as Error).message}`)), 0);
+      return { t: "recovery", ok: true, committed: true, recovery: { ...recoveryView(), phase: "prepared" } };
+    }
+    if (msg.op === "release") {
+      if (recoveryPhase === "released") return { t: "recovery", ok: true, released: true, recovery: recoveryView() };
+      if (recoveryPhase !== "restored") return recoveryError("recovery is not restored");
+      await bus.fenceRecovery();
+      const current = recoveryPeers();
+      const missing = (recoveryPeerSnapshot ?? []).filter((saved) => saved.state !== "offline" && !manualPaused.has(saved.id) && !budget.record(saved.id) && (!current[saved.id] || current[saved.id]!.state === "offline"));
+      if (missing.length) return recoveryError(`required peers are not attached: ${missing.map((peer) => peer.id).join(", ")}`);
+      if (!recoveryReady()) return recoveryError("required peers or approvals are not ready");
+      if (!restored?.integrity || JSON.stringify(restored.integrity) !== JSON.stringify(integrity())) {
+        return recoveryError("queue, pause, task board or budget integrity changed during recovery");
+      }
+      archiveRestartSnapshot(opts.stateDir, op);
+      recoveryPhase = "released";
+      clearTimeout(recoveryLeaseTimer);
+      bus.setRecoveryHold(false);
+      budget.setRecoveryHold(false);
+      writeStatus();
+      return { t: "recovery", ok: true, released: true, recovery: recoveryView() };
+    }
+    return recoveryError("unknown recovery operation");
+  }
+
   function onMessage(sock: Sock, msg: any): void {
     const c = sock.data;
     const reply = (body: Record<string, unknown>) => { if (sock.readyState === WebSocket.OPEN) sock.send(JSON.stringify({ rid: msg.rid, ...body })); };
@@ -635,6 +847,7 @@ export async function startDaemon(opts: DaemonOptions) {
       } else {
         c.peer = String(msg.peer ?? "claude");
         if (!PEER_ID.test(c.peer) || RESERVED_IDS.has(c.peer)) return sock.close(4403, "peer id is reserved or malformed");
+        if (!recoveryPeerAllowed(c.peer)) return sock.close(4403, "peer id is not part of the recovery roster");
         let peer = bus.peers.get(c.peer);
         if (!peer) bus.add((peer = new WsPeer(c.peer)));
         if (!(peer instanceof WsPeer)) return sock.close(4409, "peer id is taken by a hub-managed adapter");
@@ -648,7 +861,12 @@ export async function startDaemon(opts: DaemonOptions) {
     }
     if (stopping && msg.t !== "status" && msg.t !== "kill") return void reply({ ok: false, error: "hub is stopping" });
     switch (msg.t) {
+      case "recovery":
+        if (c.role !== "console") return void reply(recoveryError("recovery is a console command"));
+        void recoveryOp(msg).then(reply, () => reply(recoveryError("recovery operation failed")));
+        return;
       case "send": {
+        if (recoveryActive()) return void reply({ t: "sent", ok: false, error: "recovery is holding new deliveries" });
         // A human at the console should not wait out the batch window; agents default to status.
         const { priority, body } = parseMarker(String(msg.body ?? ""), c.peer ? "status" : "important");
         if (!body) return void reply({ t: "sent", ok: false, error: "empty body" });
@@ -681,12 +899,15 @@ export async function startDaemon(opts: DaemonOptions) {
         return void reply(uiSnapshot(msg.after ?? 0));
       case "ui_action":
         if (c.role !== "console") return void reply({ ok: false, error: "ui_action is a console command" });
+        if (recoveryActive()) return void reply({ ok: false, error: "recovery is holding mutations" });
         if (msg.instanceId !== instanceId) return void reply({ ok: false, error: "hub restarted; refresh before acting" });
         if (!msg.action || typeof msg.action !== "object" || Array.isArray(msg.action)) return void reply({ ok: false, error: "invalid dashboard action" });
         void uiAction(msg.action).then((result) => reply(result as Record<string, unknown>), () => reply({ ok: false, error: "dashboard action failed; check its inputs" }));
         return;
       case "start":
         if (c.role !== "console") return;
+        if (recoveryActive() && !(recoveryPhase === "restored" && msg.operationId === recoveryOperationId)) return void reply({ t: "started", ok: false, error: "recovery is holding mutations" });
+        if (recoveryActive() && !recoveryPeerAllowed(String(msg.peer))) return void reply({ t: "started", ok: false, error: "peer is not part of the recovery roster" });
         startPeer(String(msg.peer), msg.args ?? {})
           .catch((e: Error) => ({ ok: false, error: e.message }))
           .then((r) => {
@@ -695,6 +916,7 @@ export async function startDaemon(opts: DaemonOptions) {
           });
         return;
       case "task":
+        if (recoveryActive() && recoveryPhase !== "preparing" && String(msg.op) !== "hub_checkpoint") return void reply({ t: "task", ok: false, error: "recovery is holding mutations" });
         taskOp(c.peer ?? USER, String(msg.op), msg.args ?? {}).then(
           (text) => reply({ t: "task", ok: true, text }),
           (e: Error) => reply({ t: "task", ok: false, error: e.message }),
@@ -703,6 +925,7 @@ export async function startDaemon(opts: DaemonOptions) {
       case "pause":
       case "resume": {
         if (c.role !== "console") return;
+        if (recoveryActive()) return void reply({ t: msg.t, ok: false, error: "recovery is holding mutations" });
         return void reply({ t: msg.t, ...holdPeer(msg.t, String(msg.peer)) });
       }
       case "ask":
@@ -741,6 +964,7 @@ export async function startDaemon(opts: DaemonOptions) {
         return;
       case "budget":
         if (c.role !== "console") return;
+        if (recoveryActive()) return void reply({ t: "budget", ok: false, error: "recovery is holding mutations" });
         if (msg.resume) {
           if (!budget.override(String(msg.resume))) return void reply({ t: "budget", ok: false, error: `${msg.resume} is not paused by the budget coordinator` });
         } else if (msg.set) {