@staix/agent-hub 0.3.2 → 0.4.0

package/src/hub/daemon.ts

@@ -26,6 +26,7 @@ import { PROTOCOL, stateDirFor } from "./control-client.ts";
 import { newEnvelope, parseMarker, USER, type Envelope, type PeerId } from "./envelope.ts";
 import { BasePeer, DEFAULT_WATCHDOG_MS } from "./peers.ts";
 import { MemoryClient, workerUrl } from "../memory/client.ts";
+import { VERSION } from "../version.ts";
 import { projectChain, recallFor } from "../memory/recall.ts";
 
 export interface HubConfig {
@@ -82,6 +83,8 @@ export function loadConfig(cwd: string): HubConfig {
 }
 
 export interface DaemonOptions {
+  projectId?: string;
+  instanceId?: string;
   cwd: string;
   stateDir: string;
   controlPort: number;
@@ -134,6 +137,11 @@ class WsPeer extends BasePeer {
 }
 
 export async function startDaemon(opts: DaemonOptions) {
+  const projectId = opts.projectId ?? randomUUID();
+  const instanceId = opts.instanceId ?? randomUUID();
+  const startupCleanup: (() => void)[] = [];
+  let ready = false;
+  try {
   const config = opts.config ?? loadConfig(opts.cwd);
   mkdirSync(opts.stateDir, { recursive: true });
   const logFile = join(opts.stateDir, "hub.log");
@@ -143,6 +151,33 @@ export async function startDaemon(opts: DaemonOptions) {
   // The file is written only after the port is bound: a second daemon that loses the bind must not clobber it.
   const token = randomBytes(24).toString("hex");
 
+  const server = Bun.serve<Client>({
+    hostname: "127.0.0.1",
+    port: opts.controlPort,
+    fetch(req, srv) {
+      if (req.headers.has("origin")) return new Response("forbidden", { status: 403 });
+      if (new URL(req.url).pathname === "/healthz") return new Response("ok");
+      return srv.upgrade(req, { data: { authed: false } }) ? undefined : new Response("agent-hub control");
+    },
+    websocket: {
+      message(sock, data) {
+        try {
+          onMessage(sock, JSON.parse(String(data)));
+        } catch (e) {
+          log(`bad control message: ${(e as Error).message}`);
+        }
+      },
+      close(sock) {
+        consoles.delete(sock);
+        sock.data.tail?.();
+        const peer = sock.data.peer ? bus.peers.get(sock.data.peer) : undefined;
+        if (peer instanceof WsPeer) peer.detach(sock);
+      },
+    },
+  });
+  startupCleanup.push(() => server.stop(true));
+
+
   // The hub's own model calls (digest condensation, task triage) are wired below, once the gateway client exists.
   let inference: Inference | undefined;
   const bus = new Bus({ batchMax: config.batch_max, batchMs: config.batch_ms, queueCap: config.queue_cap, condense: (envs) => inference?.condense(envs) ?? Promise.resolve(envs) });
@@ -180,6 +215,7 @@ export async function startDaemon(opts: DaemonOptions) {
     for (const c of consoles) if (c.data.tail) c.send(JSON.stringify({ t: "notice", line }));
   };
   const board = new Board(join(opts.stateDir, "hub.db"));
+  startupCleanup.push(() => board.close());
   const tasks = new Tasks({
     board,
     bus,
@@ -235,6 +271,7 @@ export async function startDaemon(opts: DaemonOptions) {
     notify: (line) => notify(line),
   });
   const kimiTokens: { at: number; n: number }[] = [];
+  startupCleanup.push(() => budget.close());
   let kimiSessionTotal = 0;
   /** `total` is the session's running count: only what was added since the last update goes into the rolling window. */
   const onKimiTokens = (total: number) => {
@@ -267,6 +304,7 @@ export async function startDaemon(opts: DaemonOptions) {
     }, 5_000),
   ];
   for (const i of intervals) i.unref?.();
+  startupCleanup.push(() => { for (const i of intervals) clearInterval(i); });
 
   /** What the console stream and the log may show: a private envelope (PII task) keeps its body to its recipients. */
   const redact = (e: BusEvent): BusEvent => {
@@ -277,7 +315,7 @@ export async function startDaemon(opts: DaemonOptions) {
     return { ...e, env: { ...env, ...(task ? { refs: { task } } : {}), body: `[private${task ? `: task #${task}, see ahub task show ${task}` : ""}]` } };
   };
   const SERVER_JS = join(import.meta.dir, "..", "..", "plugins", "agent-hub", "server.js");
-  const toolEnv = (peer: PeerId) => ({ AGENTHUB_MODE: "tools", AGENTHUB_PEER_ID: peer, AGENTHUB_STATE_DIR: opts.stateDir });
+  const toolEnv = (peer: PeerId) => ({ AGENTHUB_MODE: "tools", AGENTHUB_PEER_ID: peer, AGENTHUB_STATE_DIR: opts.stateDir, AGENTHUB_PROJECT_DIR: opts.cwd });
 
   /** One entry point for the task tools, whoever calls them: MCP clients, the local worker, the console. */
   async function taskOp(by: PeerId, op: string, a: Record<string, any>, inProcess = false, piiTurn = false): Promise<string> {
@@ -333,6 +371,11 @@ export async function startDaemon(opts: DaemonOptions) {
     return r ? { paused: `budget: ${r.reason}, resets ${new Date(r.resetsAt).toLocaleTimeString()}` } : {};
   };
   const status = () => ({
+    projectId,
+    instanceId,
+    version: VERSION,
+    protocol: PROTOCOL,
+    stopping,
     pid: process.pid,
     cwd: opts.cwd,
     controlPort: server.port,
@@ -346,16 +389,14 @@ export async function startDaemon(opts: DaemonOptions) {
   });
   const writeStatus = () => {
     const file = join(opts.stateDir, "status.json"); // clients parse this on every connect: replace it atomically
-    writeFileSync(`${file}.tmp`, `${JSON.stringify(status(), null, 2)}\n`);
-    renameSync(`${file}.tmp`, file);
+    writeFileSync(`${file}.${instanceId}.tmp`, `${JSON.stringify(status(), null, 2)}\n`);
+    renameSync(`${file}.${instanceId}.tmp`, file);
   };
 
   bus.tap((e) => {
     e = redact(e);
-    if (dashboard) {
-      uiEvents.push({ seq: ++uiSequence, event: e });
-      if (uiEvents.length > 200) uiEvents.shift();
-    }
+    uiEvents.push({ seq: ++uiSequence, event: e });
+    if (uiEvents.length > 200) uiEvents.shift();
     if (e.t === "state") log(`state ${e.peer} -> ${e.state}`);
     else if (e.t === "undeliverable") log(`UNDELIVERABLE to ${e.peer} after retries: ${e.env.id} from ${e.env.from}`);
     else if (e.t === "overflow") log(`OVERFLOW ${e.peer}: dropped ${e.env.id} from ${e.env.from}`);
@@ -364,6 +405,7 @@ export async function startDaemon(opts: DaemonOptions) {
   });
 
   async function onPermission(req: PermissionRequest): Promise<string | undefined> {
+    if (stopping) return undefined;
     if (opts.unattended) return req.options.find((o) => o.kind === "allow_once")?.optionId;
     const id = randomUUID().slice(0, 8);
     // The title is written by an agent and read by the person approving it: escape sequences and carriage returns
@@ -388,6 +430,7 @@ export async function startDaemon(opts: DaemonOptions) {
   // One start per peer at a time: a second `ahub codex` must not tear down an adapter that is still coming up.
   const starting = new Map<string, Promise<Record<string, unknown>>>();
   function startPeer(peer: string, args: { model?: string; route?: string }): Promise<Record<string, unknown>> {
+    if (stopping) return Promise.resolve({ ok: false, error: "hub is stopping" });
     const running = starting.get(peer) ?? startPeerOnce(peer, args).finally(() => starting.delete(peer));
     starting.set(peer, running);
     return running;
@@ -450,7 +493,7 @@ export async function startDaemon(opts: DaemonOptions) {
       // The sidecar serves the routes it was generated from: a changed routing.toml needs a new one.
       const routingKey = JSON.stringify([routing.targets, routing.routes]);
       if (sidecar && routingKey !== sidecarRouting) {
-        sidecar.stop();
+        await sidecar.stop();
         sidecar = undefined;
       }
       if (route && opts.switchyardPort) {
@@ -507,6 +550,8 @@ export async function startDaemon(opts: DaemonOptions) {
   function uiSnapshot(after: number) {
     return {
       ok: true,
+      projectId,
+      instanceId,
       status: { peers: status().peers },
       tasks: board.list().map((task) => {
         const view = tasks.publicView(task);
@@ -568,7 +613,7 @@ export async function startDaemon(opts: DaemonOptions) {
 
   function onMessage(sock: Sock, msg: any): void {
     const c = sock.data;
-    const reply = (body: Record<string, unknown>) => sock.send(JSON.stringify({ rid: msg.rid, ...body }));
+    const reply = (body: Record<string, unknown>) => { if (sock.readyState === WebSocket.OPEN) sock.send(JSON.stringify({ rid: msg.rid, ...body })); };
     if (!c.authed) {
       if (msg.t !== "hello" || msg.token !== token) return sock.close(4401, "bad token");
       if (msg.v !== PROTOCOL) {
@@ -576,8 +621,12 @@ export async function startDaemon(opts: DaemonOptions) {
         log(`refused ${msg.role} ${msg.peer ?? ""}: wire version ${msg.v ?? 1}, hub speaks ${PROTOCOL} (claude plugin update agent-hub@agent-hub)`);
         return sock.close(4426, `wire version mismatch: hub speaks ${PROTOCOL}; update the agent-hub plugin`);
       }
+      if ((msg.projectId && msg.projectId !== projectId) || (msg.instanceId && msg.instanceId !== instanceId) ||
+          (msg.projectRoot && msg.projectRoot !== opts.cwd)) return sock.close(4404, "project or instance mismatch");
+      if (!["peer", "tools", "console"].includes(msg.role)) return sock.close(4403, "invalid client role");
+      if (stopping && msg.role !== "console") return sock.close(1013, "hub is stopping");
       c.authed = true;
-      c.role = msg.role === "peer" || msg.role === "tools" ? msg.role : "console";
+      c.role = msg.role;
       if (c.role === "console") consoles.add(sock);
       else if (c.role === "tools") {
         // Acts for a peer the hub manages itself (kimi, codex): may send and use the board as that peer, is never a delivery target.
@@ -595,8 +644,9 @@ export async function startDaemon(opts: DaemonOptions) {
           if (sock.readyState === WebSocket.OPEN) ws.attach(sock);
         });
       }
-      return void reply({ t: "welcome" });
+      return void reply({ t: "welcome", projectId, instanceId, cwd: opts.cwd, protocol: PROTOCOL });
     }
+    if (stopping && msg.t !== "status" && msg.t !== "kill") return void reply({ ok: false, error: "hub is stopping" });
     switch (msg.t) {
       case "send": {
         // A human at the console should not wait out the batch window; agents default to status.
@@ -625,6 +675,16 @@ export async function startDaemon(opts: DaemonOptions) {
         }
       case "status":
         return void reply({ t: "status", status: status() });
+      case "ui_snapshot":
+        if (c.role !== "console") return void reply({ ok: false, error: "ui_snapshot is a console command" });
+        if (!Number.isSafeInteger(msg.after ?? 0) || (msg.after ?? 0) < 0) return void reply({ ok: false, error: "invalid cursor" });
+        return void reply(uiSnapshot(msg.after ?? 0));
+      case "ui_action":
+        if (c.role !== "console") return void reply({ ok: false, error: "ui_action is a console command" });
+        if (msg.instanceId !== instanceId) return void reply({ ok: false, error: "hub restarted; refresh before acting" });
+        if (!msg.action || typeof msg.action !== "object" || Array.isArray(msg.action)) return void reply({ ok: false, error: "invalid dashboard action" });
+        void uiAction(msg.action).then((result) => reply(result as Record<string, unknown>), () => reply({ ok: false, error: "dashboard action failed; check its inputs" }));
+        return;
       case "start":
         if (c.role !== "console") return;
         startPeer(String(msg.peer), msg.args ?? {})
@@ -694,7 +754,11 @@ export async function startDaemon(opts: DaemonOptions) {
         if (c.role === "console") permissions.get(String(msg.id))?.done(msg.option ? String(msg.option) : undefined);
         return;
       case "kill":
-        if (c.role === "console") void stop();
+        if (c.role !== "console") return void reply({ ok: false, error: "kill is a console command" });
+        if (msg.instanceId !== undefined && msg.instanceId !== instanceId) return void reply({ ok: false, error: "hub restarted; refresh before stopping" });
+        reply({ t: "stopping", ok: true, instanceId });
+        // Let the acknowledgement flush before closing the control listener.
+        setTimeout(() => void stop().catch((error) => log(`shutdown incomplete: ${(error as Error).message}`)), 0);
         return;
       default:
         // A newer CLI talking to an older hub must get an answer, not wait forever.
@@ -702,45 +766,34 @@ export async function startDaemon(opts: DaemonOptions) {
     }
   }
 
-  const server = Bun.serve<Client>({
-    hostname: "127.0.0.1",
-    port: opts.controlPort,
-    fetch(req, srv) {
-      if (req.headers.has("origin")) return new Response("forbidden", { status: 403 });
-      if (new URL(req.url).pathname === "/healthz") return new Response("ok");
-      return srv.upgrade(req, { data: { authed: false } }) ? undefined : new Response("agent-hub control");
-    },
-    websocket: {
-      message(sock, data) {
-        try {
-          onMessage(sock, JSON.parse(String(data)));
-        } catch (e) {
-          log(`bad control message: ${(e as Error).message}`);
-        }
-      },
-      close(sock) {
-        consoles.delete(sock);
-        sock.data.tail?.();
-        const peer = sock.data.peer ? bus.peers.get(sock.data.peer) : undefined;
-        if (peer instanceof WsPeer) peer.detach(sock);
-      },
-    },
-  });
 
-  async function stop(): Promise<void> {
-    if (stopping) return;
+  let shutdown: Promise<void> | undefined;
+  function stop(): Promise<void> {
+    if (shutdown) return shutdown;
+    shutdown = stopOnce().catch((error) => { shutdown = undefined; throw error; });
+    return shutdown;
+  }
+  async function stopOnce(): Promise<void> {
     stopping = true;
     log("hub stopping");
+    writeStatus();
     dashboard?.stop();
-    for (const p of permissions.values()) p.done(undefined);
-    await Promise.allSettled([...bus.peers.values()].map((p) => p.stop()));
-    sidecar?.stop();
     for (const i of intervals) clearInterval(i);
     for (const done of checkpointWaits.values()) done(undefined);
+    for (const p of permissions.values()) p.done(undefined);
+    // A peer can still be inside memory recall or its native handshake when kill arrives.
+    // Drain those starts before taking the final owned-process snapshot.
+    await Promise.allSettled([...starting.values()]);
+    const exits = await Promise.allSettled([...bus.peers.values()].map((p) => p.stop()).concat(sidecar ? [sidecar.stop()] : []));
+    const failed = exits.find((r) => r.status === "rejected");
+    if (failed?.status === "rejected") throw failed.reason;
     budget.close();
     board.close();
     server.stop(true);
-    for (const f of ["hub.pid", "status.json", "control-token"]) rmSync(join(opts.stateDir, f), { force: true });
+    try {
+      const current = JSON.parse(readFileSync(join(opts.stateDir, "status.json"), "utf8"));
+      if (current.instanceId === instanceId) for (const f of ["hub.pid", "status.json", "control-token"]) rmSync(join(opts.stateDir, f), { force: true });
+    } catch { /* another owner or no published state: never remove it */ }
     onStop?.();
   }
   let onStop: (() => void) | undefined;
@@ -752,5 +805,9 @@ export async function startDaemon(opts: DaemonOptions) {
   writeFileSync(join(opts.stateDir, "hub.pid"), `${process.pid}\n`);
   writeStatus();
   log(`${RUN_START}${process.pid} control=127.0.0.1:${server.port} cwd=${opts.cwd}`);
+  ready = true;
   return { bus, token, port: server.port as number, stop, stopped: new Promise<void>((r) => (onStop = r)) };
+  } finally {
+    if (!ready) for (const cleanup of startupCleanup.reverse()) { try { cleanup(); } catch { /* preserve startup error */ } }
+  }
 }

package/src/hub/lifecycle.ts

@@ -0,0 +1,181 @@
+import { spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { createServer, type Server } from "node:net";
+import { ControlClient, PROTOCOL, readControl } from "./control-client.ts";
+import { startDaemon } from "./daemon.ts";
+import { Registry, type Project } from "./registry.ts";
+import { CODEX_APP, CODEX_PROXY, CONTROL, SWITCHYARD } from "./ports.ts";
+
+export type ProjectInspection = {
+  state: "running" | "stopped" | "stopping" | "unavailable" | "incompatible" | "missing" | "starting";
+  status?: any;
+  error?: string;
+};
+
+function processAlive(pid: unknown): boolean | undefined {
+  if (typeof pid !== "number" || !Number.isSafeInteger(pid) || pid <= 0) return undefined;
+  try { process.kill(pid, 0); return true; }
+  catch (error) { return (error as NodeJS.ErrnoException).code === "ESRCH" ? false : undefined; }
+}
+
+/** A port or PID alone is never evidence that the selected project's hub is alive. */
+export async function inspectProject(project: Project): Promise<ProjectInspection> {
+  if (!existsSync(project.root)) return { state: "missing", error: "project directory is missing" };
+  const control = readControl(project.stateDir);
+  if (control) {
+    if (processAlive(control.pid) === false) return { state: "stopped" };
+    if (control.protocol !== undefined && control.protocol !== PROTOCOL) {
+      return { state: "incompatible", error: `hub protocol ${control.protocol}; stop with its matching CLI before upgrading` };
+    }
+    if ((control.projectId && control.projectId !== project.id) || control.cwd !== project.root) {
+      return { state: "unavailable", error: "state directory belongs to a different project" };
+    }
+    let client: ControlClient | undefined;
+    try {
+      // Older manifests omit identity. Authenticate first, then require the full current contract.
+      client = await ControlClient.connect(project.stateDir, { role: "console", projectRoot: project.root,
+        ...(control.projectId ? { projectId: project.id } : {}), ...(control.instanceId ? { instanceId: control.instanceId } : {}) });
+      const response = await client.request({ t: "status" }, 3000);
+      if (!response.status || response.status.projectId !== project.id || !response.status.instanceId || response.status.cwd !== project.root) {
+        return { state: "unavailable", error: "authenticated hub identity does not match registration" };
+      }
+      if (project.instanceId && project.instanceId !== response.status.instanceId && processAlive(project.pid) !== false) {
+        return { state: "unavailable", error: "daemon instance differs from the registry claim" };
+      }
+      return response.status.stopping
+        ? { state: "stopping", status: response.status, error: "hub shutdown is pending; wait before starting it again" }
+        : { state: "running", status: response.status };
+    } catch (error) {
+      return { state: (error as { code?: number }).code === 4426 ? "incompatible" : "unavailable", error: (error as Error).message };
+    } finally { client?.close(); }
+  }
+  if (project.instanceId && processAlive(project.pid) !== false) return { state: "starting", error: "daemon owns a startup or shutdown claim" };
+  try {
+    const pid = Number(readFileSync(join(project.stateDir, "hub.pid"), "utf8").trim());
+    if (processAlive(pid) !== false) return { state: "unavailable", error: "state has a live or uncertain owner but no usable control connection" };
+  } catch { /* no legacy PID file */ }
+  return { state: "stopped" };
+}
+
+/** Probe the full stride's used ports before publishing a daemon; close only our own probes. */
+async function available(base: number): Promise<boolean> {
+  const servers: Server[] = [];
+  try {
+    for (const offset of [CONTROL, CODEX_APP, CODEX_PROXY, SWITCHYARD]) {
+      const server = createServer();
+      servers.push(server);
+      await new Promise<void>((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(base + offset, "127.0.0.1", () => resolve());
+      });
+    }
+    return true;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "EADDRINUSE") return false;
+    throw error;
+  } finally {
+    await Promise.all(servers.map((server) => new Promise<void>((resolve) => {
+      if (!server.listening) return resolve();
+      server.close(() => resolve());
+    })));
+  }
+}
+
+/** Called only in the detached daemon process. A competing starter exits without touching state. */
+export async function runProjectDaemon(project: Project, unattended = false): Promise<void> {
+  const registry = new Registry();
+  const instanceId = randomUUID();
+  let claimed = false;
+  let daemon: Awaited<ReturnType<typeof startDaemon>> | undefined;
+  try {
+    const before = await inspectProject(registry.get(project.id) ?? project);
+    if (before.state === "running" || before.state === "starting") return;
+    if (before.state !== "stopped") throw new Error(before.error ?? before.state);
+    claimed = registry.claim(project.id, instanceId, process.pid);
+    if (!claimed) return;
+    let base = registry.get(project.id)!.basePort;
+    for (let attempt = 0; attempt < 100; attempt++) {
+      if (await available(base)) {
+        try {
+          daemon = await startDaemon({ cwd: project.root, stateDir: project.stateDir, projectId: project.id, instanceId,
+            controlPort: base + CONTROL, codexAppPort: base + CODEX_APP, codexProxyPort: base + CODEX_PROXY,
+            switchyardPort: base + SWITCHYARD, unattended });
+          break;
+        } catch (error) {
+          if ((error as NodeJS.ErrnoException).code !== "EADDRINUSE") throw error;
+        }
+      }
+      base = registry.reallocate(project.id, instanceId);
+    }
+    if (!daemon) throw new Error("no free hub port range after 100 attempts");
+    const stop = () => { void daemon!.stop().catch((error) => console.error(`shutdown incomplete: ${(error as Error).message}`)); };
+    process.on("SIGINT", stop);
+    process.on("SIGTERM", stop);
+    try { await daemon.stopped; }
+    finally { process.off("SIGINT", stop); process.off("SIGTERM", stop); }
+  } finally {
+    if (claimed) registry.release(project.id, instanceId);
+    registry.close();
+  }
+}
+
+export async function startProject(project: Project, options: { unattended?: boolean; env?: NodeJS.ProcessEnv } = {}): Promise<any> {
+  const registry = new Registry();
+  try {
+    const current = registry.get(project.id);
+    if (!current || current.root !== project.root || current.stateDir !== project.stateDir) throw new Error("project registration changed; refresh before starting");
+    const before = await inspectProject(current);
+    if (before.state === "running") return before.status;
+    if (before.state !== "stopped" && before.state !== "starting") throw new Error(before.error ?? before.state);
+    if (!existsSync(join(project.root, ".agenthub", "config.json"))) throw new Error("project is not initialized; run ahub init there first");
+    if (before.state === "stopped") {
+      mkdirSync(project.stateDir, { recursive: true });
+      const marker = join(project.stateDir, "project.json");
+      const temporary = `${marker}.${randomUUID()}.tmp`;
+      writeFileSync(temporary, JSON.stringify({ root: project.root, projectId: project.id }), { mode: 0o600 });
+      renameSync(temporary, marker);
+      const log = openSync(join(project.stateDir, "hub.log"), "a");
+      try {
+        const child = spawn(process.execPath, [join(import.meta.dir, "../cli/main.ts"), "--project", project.root, "daemon",
+          ...(options.unattended ? ["--unattended"] : [])], {
+          cwd: project.root, detached: true, stdio: ["ignore", log, log],
+          env: { ...(options.env ?? process.env), AGENTHUB_STATE_DIR: project.stateDir, AGENTHUB_PROJECT_DIR: project.root,
+            AGENTHUB_UNATTENDED: options.unattended ? "1" : "0" },
+        });
+        child.on("error", () => {}); // bounded readiness below reports failure with the project's log path
+        child.unref();
+      } finally { closeSync(log); }
+    }
+    const deadline = Date.now() + 12_000;
+    while (Date.now() < deadline) {
+      const observed = await inspectProject(registry.get(project.id) ?? project);
+      if (observed.state === "running") return observed.status;
+      if (observed.state === "incompatible" || observed.state === "missing") throw new Error(observed.error ?? observed.state);
+      await Bun.sleep(100);
+    }
+    throw new Error(`hub readiness not confirmed; inspect ${join(project.stateDir, "hub.log")} before retrying`);
+  } finally { registry.close(); }
+}
+
+export async function stopProject(project: Project, expectedInstance?: string): Promise<void> {
+  const inspection = await inspectProject(project);
+  if (inspection.state === "stopped") return;
+  if (inspection.state !== "running" && inspection.state !== "stopping") throw new Error(inspection.error ?? "hub ownership is not verified; not stopping it");
+  const instanceId = inspection.status.instanceId as string;
+  if (expectedInstance && expectedInstance !== instanceId) throw new Error("hub restarted; refresh before stopping");
+  const client = await ControlClient.connect(project.stateDir, { role: "console", projectId: project.id, projectRoot: project.root, instanceId });
+  try {
+    const result = await client.request({ t: "kill", instanceId }, 3000);
+    if (!result.ok) throw new Error(result.error ?? "hub did not acknowledge shutdown");
+  } finally { client.close(); }
+  const deadline = Date.now() + 10_000;
+  while (Date.now() < deadline) {
+    const current = readControl(project.stateDir);
+    if (!current) return; // daemon removes its manifest only after owned children stop
+    if (current.instanceId !== instanceId) throw new Error("a new hub instance started; it was not stopped");
+    await Bun.sleep(100);
+  }
+  throw new Error("shutdown is still pending; state and ownership were retained");
+}