@staix/agent-hub 0.3.2 → 0.5.0

package/src/hub/daemon.ts

@@ -1,5 +1,5 @@
-import { randomBytes, randomUUID } from "node:crypto";
-import { appendFileSync, chmodSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { createHash, randomBytes, randomUUID } from "node:crypto";
+import { appendFileSync, chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import type { ServerWebSocket } from "bun";
 import { AcpPeer, type PermissionRequest } from "../adapters/acp.ts";
@@ -26,7 +26,9 @@ import { PROTOCOL, stateDirFor } from "./control-client.ts";
 import { newEnvelope, parseMarker, USER, type Envelope, type PeerId } from "./envelope.ts";
 import { BasePeer, DEFAULT_WATCHDOG_MS } from "./peers.ts";
 import { MemoryClient, workerUrl } from "../memory/client.ts";
+import { VERSION } from "../version.ts";
 import { projectChain, recallFor } from "../memory/recall.ts";
+import { archiveRestartSnapshot, readRestartSnapshot, removeRestartSnapshot, restartPath, writeRestartSnapshot, type RecoveryPhase, type RestartPeerSnapshot, type RestartSnapshot } from "./restart.ts";
 
 export interface HubConfig {
   watchdog_ms: number;
@@ -82,6 +84,8 @@ export function loadConfig(cwd: string): HubConfig {
 }
 
 export interface DaemonOptions {
+  projectId?: string;
+  instanceId?: string;
   cwd: string;
   stateDir: string;
   controlPort: number;
@@ -131,9 +135,18 @@ class WsPeer extends BasePeer {
   async stop(): Promise<void> {
     this.sock?.close();
   }
+
+  recoveryMetadata(): Record<string, unknown> {
+    return { launch: { kind: "claude-channel", peer: this.id } };
+  }
 }
 
 export async function startDaemon(opts: DaemonOptions) {
+  const projectId = opts.projectId ?? randomUUID();
+  const instanceId = opts.instanceId ?? randomUUID();
+  const startupCleanup: (() => void)[] = [];
+  let ready = false;
+  try {
   const config = opts.config ?? loadConfig(opts.cwd);
   mkdirSync(opts.stateDir, { recursive: true });
   const logFile = join(opts.stateDir, "hub.log");
@@ -143,9 +156,73 @@ export async function startDaemon(opts: DaemonOptions) {
   // The file is written only after the port is bound: a second daemon that loses the bind must not clobber it.
   const token = randomBytes(24).toString("hex");
 
+  const server = Bun.serve<Client>({
+    hostname: "127.0.0.1",
+    port: opts.controlPort,
+    fetch(req, srv) {
+      if (req.headers.has("origin")) return new Response("forbidden", { status: 403 });
+      if (new URL(req.url).pathname === "/healthz") return new Response("ok");
+      return srv.upgrade(req, { data: { authed: false } }) ? undefined : new Response("agent-hub control");
+    },
+    websocket: {
+      message(sock, data) {
+        try {
+          onMessage(sock, JSON.parse(String(data)));
+        } catch (e) {
+          log(`bad control message: ${(e as Error).message}`);
+        }
+      },
+      close(sock) {
+        consoles.delete(sock);
+        sock.data.tail?.();
+        const peer = sock.data.peer ? bus.peers.get(sock.data.peer) : undefined;
+        if (peer instanceof WsPeer) peer.detach(sock);
+      },
+    },
+  });
+  startupCleanup.push(() => server.stop(true));
+
+
   // The hub's own model calls (digest condensation, task triage) are wired below, once the gateway client exists.
   let inference: Inference | undefined;
   const bus = new Bus({ batchMax: config.batch_max, batchMs: config.batch_ms, queueCap: config.queue_cap, condense: (envs) => inference?.condense(envs) ?? Promise.resolve(envs) });
+  const manualPaused = new Set<PeerId>(); // `ahub pause`: recovery never lifts these
+  let recoveryOperationId: string | undefined;
+  let recoveryPhase: RecoveryPhase | undefined;
+  let recoveryCommitted = false;
+  let recoveryPeerSnapshot: RestartPeerSnapshot[] | undefined;
+  let recoveryLeaseTimer: ReturnType<typeof setTimeout> | undefined;
+  const recoveryActive = () => !!recoveryOperationId && recoveryPhase !== "released";
+  const recoveryOperation = process.env.AGENTHUB_RECOVERY_OPERATION;
+  const restartFilePresent = existsSync(restartPath(opts.stateDir));
+  const restored = recoveryOperation
+    ? readRestartSnapshot(opts.stateDir, { projectRoot: opts.cwd, projectId, operationId: recoveryOperation })
+    : undefined;
+  if ((restartFilePresent && !restored) || (recoveryOperation && !restored)) throw new Error("restart state is unreadable, missing, or does not match this project and recovery operation");
+  if (restored) {
+    bus.restore(restored.bus);
+    for (const peer of restored.manualPaused) { manualPaused.add(peer); bus.pause(peer); }
+    recoveryOperationId = restored.operationId;
+    recoveryPhase = "restored";
+    recoveryPeerSnapshot = restored.peers;
+    bus.setRecoveryHold(true);
+  }
+  const recoveryPeerAllowed = (id: PeerId) => !recoveryActive() || !recoveryPeerSnapshot || recoveryPeerSnapshot.some((peer) => peer.id === id);
+  const armRecoveryLease = () => {
+    clearTimeout(recoveryLeaseTimer);
+    recoveryLeaseTimer = setTimeout(() => {
+      if (!recoveryOperationId || recoveryCommitted || (recoveryPhase !== "preparing" && recoveryPhase !== "prepared")) return;
+      log(`recovery lease expired for ${recoveryOperationId}; hold aborted`);
+      recoveryOperationId = undefined;
+      recoveryPhase = undefined;
+      recoveryPeerSnapshot = undefined;
+      bus.setRecoveryHold(false);
+      budget.setRecoveryHold(false);
+      removeRestartSnapshot(opts.stateDir);
+      writeStatus();
+    }, 10 * 60_000);
+    recoveryLeaseTimer.unref?.();
+  };
   const memory = new MemoryClient(config.memory.worker_url ?? workerUrl(), 2000, log);
   const chain = projectChain(opts.cwd);
   const recalled = new Set<PeerId>(); // once per peer per hub run, however often the peer's session restarts
@@ -180,6 +257,7 @@ export async function startDaemon(opts: DaemonOptions) {
     for (const c of consoles) if (c.data.tail) c.send(JSON.stringify({ t: "notice", line }));
   };
   const board = new Board(join(opts.stateDir, "hub.db"));
+  startupCleanup.push(() => board.close());
   const tasks = new Tasks({
     board,
     bus,
@@ -191,7 +269,6 @@ export async function startDaemon(opts: DaemonOptions) {
     triage: { classify: (title, detail) => inference?.triage(title, detail) ?? Promise.resolve(undefined), onCampus: () => onCampus() },
   });
   // ---- budget relay -------------------------------------------------------------------------------------------
-  const manualPaused = new Set<PeerId>(); // `ahub pause`: the coordinator never lifts these
   const checkpointWaits = new Map<PeerId, (summary: string | undefined) => void>();
   const PLATFORM: Record<PeerId, string> = { claude: "claude", codex: "codex", kimi: "kimi" };
   const budget = new Budget(join(opts.stateDir, "hub.db"), config.budget, {
@@ -234,7 +311,9 @@ export async function startDaemon(opts: DaemonOptions) {
     },
     notify: (line) => notify(line),
   });
+  budget.setRecoveryHold(recoveryActive());
   const kimiTokens: { at: number; n: number }[] = [];
+  startupCleanup.push(() => budget.close());
   let kimiSessionTotal = 0;
   /** `total` is the session's running count: only what was added since the last update goes into the rolling window. */
   const onKimiTokens = (total: number) => {
@@ -267,6 +346,7 @@ export async function startDaemon(opts: DaemonOptions) {
     }, 5_000),
   ];
   for (const i of intervals) i.unref?.();
+  startupCleanup.push(() => { for (const i of intervals) clearInterval(i); });
 
   /** What the console stream and the log may show: a private envelope (PII task) keeps its body to its recipients. */
   const redact = (e: BusEvent): BusEvent => {
@@ -277,7 +357,7 @@ export async function startDaemon(opts: DaemonOptions) {
     return { ...e, env: { ...env, ...(task ? { refs: { task } } : {}), body: `[private${task ? `: task #${task}, see ahub task show ${task}` : ""}]` } };
   };
   const SERVER_JS = join(import.meta.dir, "..", "..", "plugins", "agent-hub", "server.js");
-  const toolEnv = (peer: PeerId) => ({ AGENTHUB_MODE: "tools", AGENTHUB_PEER_ID: peer, AGENTHUB_STATE_DIR: opts.stateDir });
+  const toolEnv = (peer: PeerId) => ({ AGENTHUB_MODE: "tools", AGENTHUB_PEER_ID: peer, AGENTHUB_STATE_DIR: opts.stateDir, AGENTHUB_PROJECT_DIR: opts.cwd });
 
   /** One entry point for the task tools, whoever calls them: MCP clients, the local worker, the console. */
   async function taskOp(by: PeerId, op: string, a: Record<string, any>, inProcess = false, piiTurn = false): Promise<string> {
@@ -325,6 +405,16 @@ export async function startDaemon(opts: DaemonOptions) {
     }
     throw new Error(`unknown task operation ${op}`);
   }
+  function recoveryTaskPreface(peer: PeerId): void {
+    if (recoveryPhase !== "restored") return;
+    const open = board.list().filter((task) => (task.owner === peer || task.reviewer === peer) && !["approved"].includes(task.state));
+    if (!open.length) return;
+    const lines = open.map((task) => {
+      const view = tasks.publicView(task) as { title?: unknown; detail?: unknown; state?: unknown; owner?: unknown; reviewer?: unknown };
+      return `#${task.id} ${String(view.title ?? "[private]")} (${task.state}, owner ${task.owner ?? "none"}, reviewer ${task.reviewer ?? "none"})${view.detail && view.detail !== "[pii]" ? `\n${String(view.detail).slice(0, 1000)}` : ""}`;
+    });
+    bus.preface(peer, `Controlled restart restored your open task context. Check the board before acting:\n${lines.join("\n\n")}`);
+  }
   const permissions = new Map<string, { push: string; done: (optionId: string | undefined) => void }>();
   let stopping = false;
 
@@ -332,7 +422,51 @@ export async function startDaemon(opts: DaemonOptions) {
     const r = budget.record(id); // one read per peer: status.json is rewritten on every bus event
     return r ? { paused: `budget: ${r.reason}, resets ${new Date(r.resetsAt).toLocaleTimeString()}` } : {};
   };
+  const recoveryReady = () => {
+    if (!recoveryActive() || permissions.size !== 0 || starting.size !== 0 || !budget.recoverySettled || [...bus.peers.values()].some((peer) => peer.state === "busy")) return false;
+    if (!recoveryPeerSnapshot) return true;
+    const current = recoveryPeers();
+    return recoveryPeerSnapshot.every((saved) => {
+      const now = current[saved.id];
+      if (!now) return true; // a detached peer is checked by the coordinator before terminal close
+      if (saved.threadId && now.threadId && saved.threadId !== now.threadId) return false;
+      if (saved.sessionId && now.sessionId && saved.sessionId !== now.sessionId) return false;
+      return true;
+    });
+  };
+  const claudeSessionId = () => {
+    try {
+      const value = JSON.parse(readFileSync(join(opts.stateDir, "claude-session.json"), "utf8"));
+      if (value.instanceId !== instanceId) return undefined;
+      try {
+        const records = JSON.parse(readFileSync(join(opts.stateDir, "terminal-recovery.json"), "utf8"));
+        const current = Array.isArray(records) ? records.find((row) => row?.peer === "claude" && row?.projectRoot === opts.cwd && row?.instanceId === instanceId) : undefined;
+        if (current?.launchId && value.launchId !== current.launchId) return undefined;
+      } catch { /* no managed terminal record: the instance fence is still enforced */ }
+      return typeof value.sessionId === "string" && value.sessionId ? value.sessionId : undefined;
+    } catch { return undefined; }
+  };
+  const recoveryPeers = (): Record<string, RestartPeerSnapshot> => Object.fromEntries([...bus.peers].map(([id, peer]) => {
+    const metadata = peer.recoveryMetadata?.() ?? {};
+    const row: RestartPeerSnapshot = { id, state: peer.state, queueIds: bus.queueIds(id), ...(metadata.launch ? { launch: metadata.launch as Record<string, unknown> } : {}) };
+    if (typeof metadata.threadId === "string") row.threadId = metadata.threadId;
+    const sessionId = id === "claude" ? claudeSessionId() : metadata.sessionId;
+    if (typeof sessionId === "string" && sessionId) row.sessionId = sessionId;
+    return [id, row];
+  }));
+  const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
+  const integrity = () => {
+    const queues = Object.fromEntries(Object.keys(bus.snapshot().queues).sort().map((id) => [id, bus.queueIds(id)]));
+    const boardState = board.list().sort((a, b) => a.id - b.id);
+    const budgetState = budget.persistedPauseDigestRows().sort((a, b) => a.peer.localeCompare(b.peer));
+    return { queues, manualPaused: [...manualPaused].sort(), boardDigest: digest(boardState), budgetDigest: digest(budgetState) };
+  };
   const status = () => ({
+    projectId,
+    instanceId,
+    version: VERSION,
+    protocol: PROTOCOL,
+    stopping,
     pid: process.pid,
     cwd: opts.cwd,
     controlPort: server.port,
@@ -343,19 +477,18 @@ export async function startDaemon(opts: DaemonOptions) {
     ),
     ...(sidecar ? { switchyard: sidecar.status } : {}),
     tasks: board.counts(),
+    ...(recoveryOperationId ? { recovery: { operationId: recoveryOperationId, phase: recoveryPhase, ready: recoveryReady() } } : {}),
   });
   const writeStatus = () => {
     const file = join(opts.stateDir, "status.json"); // clients parse this on every connect: replace it atomically
-    writeFileSync(`${file}.tmp`, `${JSON.stringify(status(), null, 2)}\n`);
-    renameSync(`${file}.tmp`, file);
+    writeFileSync(`${file}.${instanceId}.tmp`, `${JSON.stringify(status(), null, 2)}\n`);
+    renameSync(`${file}.${instanceId}.tmp`, file);
   };
 
   bus.tap((e) => {
     e = redact(e);
-    if (dashboard) {
-      uiEvents.push({ seq: ++uiSequence, event: e });
-      if (uiEvents.length > 200) uiEvents.shift();
-    }
+    uiEvents.push({ seq: ++uiSequence, event: e });
+    if (uiEvents.length > 200) uiEvents.shift();
     if (e.t === "state") log(`state ${e.peer} -> ${e.state}`);
     else if (e.t === "undeliverable") log(`UNDELIVERABLE to ${e.peer} after retries: ${e.env.id} from ${e.env.from}`);
     else if (e.t === "overflow") log(`OVERFLOW ${e.peer}: dropped ${e.env.id} from ${e.env.from}`);
@@ -364,6 +497,7 @@ export async function startDaemon(opts: DaemonOptions) {
   });
 
   async function onPermission(req: PermissionRequest): Promise<string | undefined> {
+    if (stopping) return undefined;
     if (opts.unattended) return req.options.find((o) => o.kind === "allow_once")?.optionId;
     const id = randomUUID().slice(0, 8);
     // The title is written by an agent and read by the person approving it: escape sequences and carriage returns
@@ -388,6 +522,7 @@ export async function startDaemon(opts: DaemonOptions) {
   // One start per peer at a time: a second `ahub codex` must not tear down an adapter that is still coming up.
   const starting = new Map<string, Promise<Record<string, unknown>>>();
   function startPeer(peer: string, args: { model?: string; route?: string }): Promise<Record<string, unknown>> {
+    if (stopping) return Promise.resolve({ ok: false, error: "hub is stopping" });
     const running = starting.get(peer) ?? startPeerOnce(peer, args).finally(() => starting.delete(peer));
     starting.set(peer, running);
     return running;
@@ -404,6 +539,7 @@ export async function startDaemon(opts: DaemonOptions) {
       const cmd = args.model ? [bin!, "--model", args.model, ...rest] : config.kimi_cmd;
       const kimi = new AcpPeer("kimi", {
         cmd,
+        ...(args.model ? { launchModel: args.model } : {}),
         cwd: opts.cwd,
         watchdogMs: config.watchdog_ms,
         onPermission,
@@ -412,6 +548,7 @@ export async function startDaemon(opts: DaemonOptions) {
         mcpServers: [{ name: "agent-hub", command: "bun", args: ["run", SERVER_JS], env: Object.entries(toolEnv("kimi")).map(([name, value]) => ({ name, value })) }],
         preamble: roleContract("kimi", config.roles),
       });
+      recoveryTaskPreface("kimi");
       await ensurePreface("kimi");
       bus.add(kimi);
       await kimi.start();
@@ -437,6 +574,7 @@ export async function startDaemon(opts: DaemonOptions) {
         watchdogMs: config.watchdog_ms,
         log,
       });
+      recoveryTaskPreface("codex");
       await ensurePreface("codex");
       bus.add(codex);
       await codex.start();
@@ -450,7 +588,7 @@ export async function startDaemon(opts: DaemonOptions) {
       // The sidecar serves the routes it was generated from: a changed routing.toml needs a new one.
       const routingKey = JSON.stringify([routing.targets, routing.routes]);
       if (sidecar && routingKey !== sidecarRouting) {
-        sidecar.stop();
+        await sidecar.stop();
         sidecar = undefined;
       }
       if (route && opts.switchyardPort) {
@@ -483,6 +621,7 @@ export async function startDaemon(opts: DaemonOptions) {
         maxSteps: config.local.max_steps,
         log,
       });
+      recoveryTaskPreface("local");
       await ensurePreface("local");
       bus.add(local);
       await local.start();
@@ -507,6 +646,8 @@ export async function startDaemon(opts: DaemonOptions) {
   function uiSnapshot(after: number) {
     return {
       ok: true,
+      projectId,
+      instanceId,
       status: { peers: status().peers },
       tasks: board.list().map((task) => {
         const view = tasks.publicView(task);
@@ -566,9 +707,125 @@ export async function startDaemon(opts: DaemonOptions) {
     }
   }
 
+  const recoveryView = () => ({
+    operationId: recoveryOperationId,
+    phase: recoveryPhase,
+    ready: recoveryReady(),
+    pendingApprovals: permissions.size,
+    capabilities: { controlledRestart: true, snapshotSchemaVersion: 1, maxLeaseMs: 10 * 60_000 },
+    blockers: [
+      ...(starting.size ? ["peer startup in progress"] : []),
+      ...(!budget.recoverySettled ? ["budget transition in progress"] : []),
+      ...([...bus.peers].filter(([, peer]) => peer.state === "busy").map(([id]) => `${id} is busy`)),
+      ...(permissions.size ? ["pending approvals"] : []),
+    ],
+    integrity: { current: integrity(), ...(restored?.integrity ? { expected: restored.integrity } : {}) },
+    peers: Object.fromEntries([...(recoveryPeerSnapshot ?? []), ...Object.values(recoveryPeers()).filter((peer) => !(recoveryPeerSnapshot ?? []).some((saved) => saved.id === peer.id))].map((peer) => {
+      const now = recoveryPeers()[peer.id];
+      if (recoveryPhase === "restored" || recoveryPhase === "released") return [peer.id, now ?? { id: peer.id, state: "offline", queueIds: bus.queueIds(peer.id) }];
+      return [peer.id, now ? { ...peer, ...now, queueIds: now.queueIds } : peer];
+    })),
+  });
+  const recoveryError = (error: string) => ({ t: "recovery", ok: false, error });
+  async function recoveryOp(msg: any): Promise<Record<string, unknown>> {
+    if (typeof msg.op !== "string" || !["inspect", "prepare", "commit", "abort", "release"].includes(msg.op)) return recoveryError("unknown recovery operation");
+    if (typeof msg.expectedInstanceId !== "string" || msg.expectedInstanceId !== instanceId) return recoveryError("expected daemon instance does not match");
+    if (msg.op === "inspect" && msg.operationId === undefined) return { t: "recovery", ok: true, recovery: recoveryView() };
+    if (typeof msg.operationId !== "string" || msg.operationId.length < 1 || msg.operationId.length > 128) return recoveryError("operationId is required");
+    const op = msg.operationId as string;
+    if (msg.op === "prepare") {
+      if (recoveryCommitted) return recoveryError("recovery commit is already in progress");
+      if (recoveryPhase === "released" && recoveryOperationId !== op) {
+        recoveryOperationId = undefined;
+        recoveryPhase = undefined;
+        recoveryPeerSnapshot = undefined;
+      }
+      if (recoveryOperationId && recoveryOperationId !== op) return recoveryError("another recovery operation is active");
+      recoveryOperationId = op;
+      if (!recoveryPhase) recoveryPhase = "preparing";
+      armRecoveryLease();
+      recoveryPeerSnapshot ??= Object.values(recoveryPeers());
+      budget.setRecoveryHold(true);
+      bus.setRecoveryHold(true);
+      await bus.fenceRecovery();
+      const blocked = [...bus.peers].filter(([, peer]) => peer.state === "busy").map(([id]) => id);
+      if (!blocked.length && permissions.size === 0 && recoveryReady()) {
+        recoveryPeerSnapshot ??= Object.values(recoveryPeers());
+        recoveryPhase = "prepared";
+      }
+      writeStatus();
+      return { t: "recovery", ok: true, recovery: { ...recoveryView(), ...(blocked.length ? { blockedPeers: blocked } : {}), ...(permissions.size ? { blocked: "pending approvals" } : {}) } };
+    }
+    if (!recoveryOperationId || recoveryOperationId !== op) return msg.op === "inspect"
+      ? { t: "recovery", ok: true, recovery: recoveryView() }
+      : recoveryError("operation does not match this daemon");
+    if (msg.op === "inspect") return { t: "recovery", ok: true, recovery: recoveryView() };
+    if (msg.op === "abort") {
+      if (recoveryCommitted) return recoveryError("a committed recovery cannot be aborted");
+      if (recoveryPhase === "released") return { t: "recovery", ok: true, recovery: recoveryView() };
+      recoveryOperationId = undefined;
+      recoveryPhase = undefined;
+      recoveryPeerSnapshot = undefined;
+      clearTimeout(recoveryLeaseTimer);
+      bus.setRecoveryHold(false);
+      budget.setRecoveryHold(false);
+      writeStatus();
+      return { t: "recovery", ok: true, aborted: true, recovery: recoveryView() };
+    }
+    if (msg.op === "commit") {
+      if ((recoveryPhase !== "prepared" && recoveryPhase !== "preparing") || !recoveryReady()) return recoveryError("recovery is not ready; inspect until peers are idle and approvals are complete");
+      recoveryPhase = "prepared";
+      recoveryPeerSnapshot ??= Object.values(recoveryPeers());
+      const currentPeers = recoveryPeers();
+      const peers = (recoveryPeerSnapshot ?? Object.values(currentPeers)).map((saved) => ({
+        ...saved,
+        queueIds: bus.queueIds(saved.id),
+      }));
+      const snapshot: RestartSnapshot = {
+        schemaVersion: 1,
+        projectRoot: opts.cwd,
+        projectId,
+        sourceInstanceId: instanceId,
+        operationId: op,
+        committedAt: Date.now(),
+        bus: bus.snapshot(),
+        manualPaused: [...manualPaused],
+        peers,
+        integrity: integrity(),
+      };
+      try { writeRestartSnapshot(opts.stateDir, snapshot); } catch { return recoveryError("could not persist restart state"); }
+      recoveryCommitted = true;
+      clearTimeout(recoveryLeaseTimer);
+      writeStatus();
+      // Keep phase prepared in status until the replacement daemon reports restored.
+      setTimeout(() => void stop().catch((error) => log(`recovery shutdown incomplete: ${(error as Error).message}`)), 0);
+      return { t: "recovery", ok: true, committed: true, recovery: { ...recoveryView(), phase: "prepared" } };
+    }
+    if (msg.op === "release") {
+      if (recoveryPhase === "released") return { t: "recovery", ok: true, released: true, recovery: recoveryView() };
+      if (recoveryPhase !== "restored") return recoveryError("recovery is not restored");
+      await bus.fenceRecovery();
+      const current = recoveryPeers();
+      const missing = (recoveryPeerSnapshot ?? []).filter((saved) => saved.state !== "offline" && !manualPaused.has(saved.id) && !budget.record(saved.id) && (!current[saved.id] || current[saved.id]!.state === "offline"));
+      if (missing.length) return recoveryError(`required peers are not attached: ${missing.map((peer) => peer.id).join(", ")}`);
+      if (!recoveryReady()) return recoveryError("required peers or approvals are not ready");
+      if (!restored?.integrity || JSON.stringify(restored.integrity) !== JSON.stringify(integrity())) {
+        return recoveryError("queue, pause, task board or budget integrity changed during recovery");
+      }
+      archiveRestartSnapshot(opts.stateDir, op);
+      recoveryPhase = "released";
+      clearTimeout(recoveryLeaseTimer);
+      bus.setRecoveryHold(false);
+      budget.setRecoveryHold(false);
+      writeStatus();
+      return { t: "recovery", ok: true, released: true, recovery: recoveryView() };
+    }
+    return recoveryError("unknown recovery operation");
+  }
+
   function onMessage(sock: Sock, msg: any): void {
     const c = sock.data;
-    const reply = (body: Record<string, unknown>) => sock.send(JSON.stringify({ rid: msg.rid, ...body }));
+    const reply = (body: Record<string, unknown>) => { if (sock.readyState === WebSocket.OPEN) sock.send(JSON.stringify({ rid: msg.rid, ...body })); };
     if (!c.authed) {
       if (msg.t !== "hello" || msg.token !== token) return sock.close(4401, "bad token");
       if (msg.v !== PROTOCOL) {
@@ -576,8 +833,12 @@ export async function startDaemon(opts: DaemonOptions) {
         log(`refused ${msg.role} ${msg.peer ?? ""}: wire version ${msg.v ?? 1}, hub speaks ${PROTOCOL} (claude plugin update agent-hub@agent-hub)`);
         return sock.close(4426, `wire version mismatch: hub speaks ${PROTOCOL}; update the agent-hub plugin`);
       }
+      if ((msg.projectId && msg.projectId !== projectId) || (msg.instanceId && msg.instanceId !== instanceId) ||
+          (msg.projectRoot && msg.projectRoot !== opts.cwd)) return sock.close(4404, "project or instance mismatch");
+      if (!["peer", "tools", "console"].includes(msg.role)) return sock.close(4403, "invalid client role");
+      if (stopping && msg.role !== "console") return sock.close(1013, "hub is stopping");
       c.authed = true;
-      c.role = msg.role === "peer" || msg.role === "tools" ? msg.role : "console";
+      c.role = msg.role;
       if (c.role === "console") consoles.add(sock);
       else if (c.role === "tools") {
         // Acts for a peer the hub manages itself (kimi, codex): may send and use the board as that peer, is never a delivery target.
@@ -586,6 +847,7 @@ export async function startDaemon(opts: DaemonOptions) {
       } else {
         c.peer = String(msg.peer ?? "claude");
         if (!PEER_ID.test(c.peer) || RESERVED_IDS.has(c.peer)) return sock.close(4403, "peer id is reserved or malformed");
+        if (!recoveryPeerAllowed(c.peer)) return sock.close(4403, "peer id is not part of the recovery roster");
         let peer = bus.peers.get(c.peer);
         if (!peer) bus.add((peer = new WsPeer(c.peer)));
         if (!(peer instanceof WsPeer)) return sock.close(4409, "peer id is taken by a hub-managed adapter");
@@ -595,10 +857,16 @@ export async function startDaemon(opts: DaemonOptions) {
           if (sock.readyState === WebSocket.OPEN) ws.attach(sock);
         });
       }
-      return void reply({ t: "welcome" });
+      return void reply({ t: "welcome", projectId, instanceId, cwd: opts.cwd, protocol: PROTOCOL });
     }
+    if (stopping && msg.t !== "status" && msg.t !== "kill") return void reply({ ok: false, error: "hub is stopping" });
     switch (msg.t) {
+      case "recovery":
+        if (c.role !== "console") return void reply(recoveryError("recovery is a console command"));
+        void recoveryOp(msg).then(reply, () => reply(recoveryError("recovery operation failed")));
+        return;
       case "send": {
+        if (recoveryActive()) return void reply({ t: "sent", ok: false, error: "recovery is holding new deliveries" });
         // A human at the console should not wait out the batch window; agents default to status.
         const { priority, body } = parseMarker(String(msg.body ?? ""), c.peer ? "status" : "important");
         if (!body) return void reply({ t: "sent", ok: false, error: "empty body" });
@@ -625,8 +893,21 @@ export async function startDaemon(opts: DaemonOptions) {
         }
       case "status":
         return void reply({ t: "status", status: status() });
+      case "ui_snapshot":
+        if (c.role !== "console") return void reply({ ok: false, error: "ui_snapshot is a console command" });
+        if (!Number.isSafeInteger(msg.after ?? 0) || (msg.after ?? 0) < 0) return void reply({ ok: false, error: "invalid cursor" });
+        return void reply(uiSnapshot(msg.after ?? 0));
+      case "ui_action":
+        if (c.role !== "console") return void reply({ ok: false, error: "ui_action is a console command" });
+        if (recoveryActive()) return void reply({ ok: false, error: "recovery is holding mutations" });
+        if (msg.instanceId !== instanceId) return void reply({ ok: false, error: "hub restarted; refresh before acting" });
+        if (!msg.action || typeof msg.action !== "object" || Array.isArray(msg.action)) return void reply({ ok: false, error: "invalid dashboard action" });
+        void uiAction(msg.action).then((result) => reply(result as Record<string, unknown>), () => reply({ ok: false, error: "dashboard action failed; check its inputs" }));
+        return;
       case "start":
         if (c.role !== "console") return;
+        if (recoveryActive() && !(recoveryPhase === "restored" && msg.operationId === recoveryOperationId)) return void reply({ t: "started", ok: false, error: "recovery is holding mutations" });
+        if (recoveryActive() && !recoveryPeerAllowed(String(msg.peer))) return void reply({ t: "started", ok: false, error: "peer is not part of the recovery roster" });
         startPeer(String(msg.peer), msg.args ?? {})
           .catch((e: Error) => ({ ok: false, error: e.message }))
           .then((r) => {
@@ -635,6 +916,7 @@ export async function startDaemon(opts: DaemonOptions) {
           });
         return;
       case "task":
+        if (recoveryActive() && recoveryPhase !== "preparing" && String(msg.op) !== "hub_checkpoint") return void reply({ t: "task", ok: false, error: "recovery is holding mutations" });
         taskOp(c.peer ?? USER, String(msg.op), msg.args ?? {}).then(
           (text) => reply({ t: "task", ok: true, text }),
           (e: Error) => reply({ t: "task", ok: false, error: e.message }),
@@ -643,6 +925,7 @@ export async function startDaemon(opts: DaemonOptions) {
       case "pause":
       case "resume": {
         if (c.role !== "console") return;
+        if (recoveryActive()) return void reply({ t: msg.t, ok: false, error: "recovery is holding mutations" });
         return void reply({ t: msg.t, ...holdPeer(msg.t, String(msg.peer)) });
       }
       case "ask":
@@ -681,6 +964,7 @@ export async function startDaemon(opts: DaemonOptions) {
         return;
       case "budget":
         if (c.role !== "console") return;
+        if (recoveryActive()) return void reply({ t: "budget", ok: false, error: "recovery is holding mutations" });
         if (msg.resume) {
           if (!budget.override(String(msg.resume))) return void reply({ t: "budget", ok: false, error: `${msg.resume} is not paused by the budget coordinator` });
         } else if (msg.set) {
@@ -694,7 +978,11 @@ export async function startDaemon(opts: DaemonOptions) {
         if (c.role === "console") permissions.get(String(msg.id))?.done(msg.option ? String(msg.option) : undefined);
         return;
       case "kill":
-        if (c.role === "console") void stop();
+        if (c.role !== "console") return void reply({ ok: false, error: "kill is a console command" });
+        if (msg.instanceId !== undefined && msg.instanceId !== instanceId) return void reply({ ok: false, error: "hub restarted; refresh before stopping" });
+        reply({ t: "stopping", ok: true, instanceId });
+        // Let the acknowledgement flush before closing the control listener.
+        setTimeout(() => void stop().catch((error) => log(`shutdown incomplete: ${(error as Error).message}`)), 0);
         return;
       default:
         // A newer CLI talking to an older hub must get an answer, not wait forever.
@@ -702,45 +990,34 @@ export async function startDaemon(opts: DaemonOptions) {
     }
   }
 
-  const server = Bun.serve<Client>({
-    hostname: "127.0.0.1",
-    port: opts.controlPort,
-    fetch(req, srv) {
-      if (req.headers.has("origin")) return new Response("forbidden", { status: 403 });
-      if (new URL(req.url).pathname === "/healthz") return new Response("ok");
-      return srv.upgrade(req, { data: { authed: false } }) ? undefined : new Response("agent-hub control");
-    },
-    websocket: {
-      message(sock, data) {
-        try {
-          onMessage(sock, JSON.parse(String(data)));
-        } catch (e) {
-          log(`bad control message: ${(e as Error).message}`);
-        }
-      },
-      close(sock) {
-        consoles.delete(sock);
-        sock.data.tail?.();
-        const peer = sock.data.peer ? bus.peers.get(sock.data.peer) : undefined;
-        if (peer instanceof WsPeer) peer.detach(sock);
-      },
-    },
-  });
 
-  async function stop(): Promise<void> {
-    if (stopping) return;
+  let shutdown: Promise<void> | undefined;
+  function stop(): Promise<void> {
+    if (shutdown) return shutdown;
+    shutdown = stopOnce().catch((error) => { shutdown = undefined; throw error; });
+    return shutdown;
+  }
+  async function stopOnce(): Promise<void> {
     stopping = true;
     log("hub stopping");
+    writeStatus();
     dashboard?.stop();
-    for (const p of permissions.values()) p.done(undefined);
-    await Promise.allSettled([...bus.peers.values()].map((p) => p.stop()));
-    sidecar?.stop();
     for (const i of intervals) clearInterval(i);
     for (const done of checkpointWaits.values()) done(undefined);
+    for (const p of permissions.values()) p.done(undefined);
+    // A peer can still be inside memory recall or its native handshake when kill arrives.
+    // Drain those starts before taking the final owned-process snapshot.
+    await Promise.allSettled([...starting.values()]);
+    const exits = await Promise.allSettled([...bus.peers.values()].map((p) => p.stop()).concat(sidecar ? [sidecar.stop()] : []));
+    const failed = exits.find((r) => r.status === "rejected");
+    if (failed?.status === "rejected") throw failed.reason;
     budget.close();
     board.close();
     server.stop(true);
-    for (const f of ["hub.pid", "status.json", "control-token"]) rmSync(join(opts.stateDir, f), { force: true });
+    try {
+      const current = JSON.parse(readFileSync(join(opts.stateDir, "status.json"), "utf8"));
+      if (current.instanceId === instanceId) for (const f of ["hub.pid", "status.json", "control-token"]) rmSync(join(opts.stateDir, f), { force: true });
+    } catch { /* another owner or no published state: never remove it */ }
     onStop?.();
   }
   let onStop: (() => void) | undefined;
@@ -752,5 +1029,9 @@ export async function startDaemon(opts: DaemonOptions) {
   writeFileSync(join(opts.stateDir, "hub.pid"), `${process.pid}\n`);
   writeStatus();
   log(`${RUN_START}${process.pid} control=127.0.0.1:${server.port} cwd=${opts.cwd}`);
+  ready = true;
   return { bus, token, port: server.port as number, stop, stopped: new Promise<void>((r) => (onStop = r)) };
+  } finally {
+    if (!ready) for (const cleanup of startupCleanup.reverse()) { try { cleanup(); } catch { /* preserve startup error */ } }
+  }
 }