@staix/agent-hub 0.12.21 → 0.12.23

package/src/cli/upgrade-interactive.ts

@@ -0,0 +1,297 @@
+import { recoveryArgv } from "../hub/recovery-store.ts";
+import { cancellableWait, nextChoices, publicOperation, type Inspection, type PlannedProject, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
+
+/**
+ * #272: the interactive screens of `ahub upgrade`, `ahub restart` and bare `ahub recovery`. They add no way to change
+ * an operation: every action is an existing command (`recovery resume|abort|dispose`, `reset`) run for the person
+ * through `io.run`, so each command's own refusals decide. The one thing a screen does itself is stop a runner that
+ * only waits, and only one whose claim is verified (`runnerStoppable`).
+ */
+
+/** The terminal a screen talks to; tests script it. */
+export interface ScreenIO {
+  /** One trimmed line from the person; null when the input ended or they pressed Ctrl+C, which no prompt takes for text. */
+  ask(question: string): Promise<string | null>;
+  out(line: string): void;
+  /** Run an ahub entry with the terminal attached (argv after `bun`); resolves with its exit code. */
+  run(argv: string[]): Promise<number>;
+  sleep(ms: number): Promise<void>;
+  /** True from the person's Ctrl+C on: the screens leave, and nothing is stopped. */
+  interrupted(): boolean;
+  /** True once when a line was entered while nothing was asked (during a follow); that line is dropped. */
+  typed(): boolean;
+}
+
+/** What the screens read and start. */
+export interface UpgradeHost {
+  plan(): Promise<UpgradePlan>;
+  /** Check the reviewed plan again and start it as a detached operation; `interrupted` is asked last, before anything is created. */
+  apply(plan: UpgradePlan, interrupted: () => boolean): Promise<RecoveryOperation>;
+  /** End one attached agent now: close a TUI agent's terminal, or ask the hub to stop a headless one. One line of outcome. */
+  endPeer(planned: PlannedProject, peer: RecoveryPeer): Promise<string>;
+  /** The operation that holds the machine's recovery lock. */
+  lock(): string | undefined;
+  read(id: string): RecoveryOperation;
+  runner(id: string): number | "unknown" | undefined;
+  live(op: RecoveryOperation): Promise<Record<string, Inspection | undefined>>;
+  /** Whether the runner's claim carries a process signature that still matches: the only kind a screen may stop. */
+  runnerStoppable(id: string): boolean;
+  /** Stop that verified runner and wait until it is gone; false when one still holds the operation. */
+  stopRunner(id: string): Promise<boolean>;
+  /** This release's CLI entry, for `reset`. */
+  entry: string;
+  /** This CLI's version. */
+  version: string;
+}
+
+const yes = async (io: ScreenIO, question: string): Promise<boolean> => /^y(es)?$/i.test((await io.ask(question)) ?? "");
+/** The way back after Ctrl+C: the operation's own coordinator, since the installed `ahub` may still be the older release (#215). */
+const left = (op: { id: string; sourceRoot?: string }): string => `left: the runner keeps working; \`bun ${recoveryArgv(op, "status")[0]} recovery\` shows the operation and what can be done`;
+
+/**
+ * How an attached peer runs, which decides how it can be ended: a TUI agent in a terminal the plan could bind, a
+ * headless agent the hub owns, or `unmanaged` (shown as `own`): a session the hub did not launch, or one it launched
+ * whose terminal the plan could not bind (a person ends that one where it runs).
+ */
+export function peerKind(planned: PlannedProject, peer: RecoveryPeer): "tui" | "headless" | "unmanaged" | "offline" {
+  if (peer.state === "offline") return "offline";
+  if ((planned.terminals as { peer: string }[]).some((t) => t.peer === peer.id)) return "tui";
+  if (peer.id === "claude" || peer.id === "codex" || (peer.id === "pi" && peer.args?.mode === "tui")) return "unmanaged";
+  return "headless";
+}
+
+const STEPS: Record<string, string> = {
+  stage: "staging the release", prepare: "holding deliveries and waiting until the hub is quiet", reprepare: "preparing the source again after an expired hold",
+  commit: "closing terminals and stopping the old hub", restart: "restarting a target that stopped", start: "starting the new hub", restore: "restoring Codex and Pi sessions",
+  "install-plugin": "installing the Claude plugin", "restore-claude": "restoring the Claude session and verifying the hub", release: "releasing held deliveries",
+  "refresh-manager": "refreshing the dashboard manager", "install-global": "installing the global CLI", completed: "completed", cancelled: "cancelled",
+};
+
+/** A receipt step (`commit:<project>`) in a person's words; a step this release does not know is shown as it is. */
+export function stepLabel(step: string): string {
+  const at = step.indexOf(":"), name = at < 0 ? step : step.slice(0, at);
+  return STEPS[name] ? `${at < 0 ? "" : `${step.slice(at + 1)}: `}${STEPS[name]}` : step;
+}
+
+function peerAction(planned: PlannedProject, peer: RecoveryPeer): string {
+  if (peer.state === "offline") return "offline, left as it is";
+  if (planned.reconnectOnly?.includes(peer.id)) return "reconnects by itself (unmanaged session; its terminal is left alone)";
+  if (planned.freshStart?.includes(peer.id)) return "restarts as a new session (no turn to lose)";
+  const terminal = (planned.terminals as { peer: string; handle: string }[]).find((t) => t.peer === peer.id);
+  if (terminal) return `resumes its session in a new terminal (replaces ${terminal.handle})`;
+  return peer.id === "pi" ? "restarts headless on its recorded session" : "restarts headless as a new session";
+}
+
+const KINDS = { tui: "TUI", headless: "headless", unmanaged: "own", offline: "-" } as const;
+
+/** What the source says keeps it from being quiet; an older hub names only its busy peers. */
+const inProgress = (source: Inspection): string[] => source.recovery?.waiting ?? source.peers.filter((p) => p.state === "busy").map((p) => `${p.id} is busy`);
+
+export function planLines(plan: UpgradePlan, current: string): string[] {
+  const lines = [plan.kind === "upgrade" ? `upgrade to ${plan.version} (coordinator ${current})` : `restart on ${plan.version}`];
+  for (const p of plan.projects) {
+    lines.push("", `${p.project.id}  ${p.project.root}  hub ${p.source.version ?? "unknown"} (${p.source.state})`);
+    const width = Math.max(0, ...p.source.peers.map((peer) => peer.id.length));
+    for (const peer of p.source.peers) lines.push(`  ${peer.id.padEnd(width)}  ${peer.state.padEnd(7)}  ${KINDS[peerKind(p, peer)].padEnd(8)}  ${peerAction(p, peer)}`);
+    const waiting = inProgress(p.source);
+    if (waiting.length) lines.push(`  in progress: ${waiting.join(", ")} (apply waits up to 10 minutes for it, then leaves this hub running)`);
+    for (const blocker of p.blockers) lines.push(`  blocker: ${blocker}`);
+  }
+  if (plan.blockers.length) lines.push("");
+  for (const blocker of plan.blockers) lines.push(`blocker: ${blocker}`);
+  return lines;
+}
+
+const ago = (ms: number): string => ms < 90_000 ? `${Math.max(0, Math.round(ms / 1000))}s` : ms < 5_400_000 ? `${Math.round(ms / 60_000)}m` : `${Math.round(ms / 3_600_000)}h`;
+
+export function operationLines(op: RecoveryOperation, runner: number | "unknown" | undefined, now: number): string[] {
+  const held = runner === "unknown" ? "; whether a runner holds it cannot be read" : runner ? `; runner ${runner} is working` : op.phase === "running" ? "; no runner holds it (the last one stopped mid-step)" : "";
+  const lines = [`${op.plan.kind} to ${op.plan.version}: operation ${op.id}`, `  ${op.phase} at "${stepLabel(op.step)}", updated ${ago(now - op.updatedAt)} ago${held}`];
+  for (const p of op.projects) {
+    const effects = Object.entries(p.terminals).map(([key, value]) => value === "pending" || value === "failed" ? `${key} ${value}` : key);
+    lines.push(`  ${p.id}  ${p.phase}${effects.length ? `  (${effects.join(", ")})` : ""}${p.restarts?.length ? `  restarts ${p.restarts.length}` : ""}`);
+  }
+  // The error ends with the commands status would print; the menu below offers them instead.
+  if (op.error) lines.push(`  error: ${op.error.split("; next actions: ")[0]}`);
+  return lines;
+}
+
+/**
+ * Print steps and readiness waits until the operation completes, blocks or loses its runner ("open"), the person
+ * enters a line (also "open": the menu), or presses Ctrl+C ("left"). `scheduled` is the receipt's `updatedAt` from
+ * before a runner was scheduled (resume, a fresh session): until that runner writes, the receipt still shows the state
+ * it was scheduled from, which is not the outcome.
+ */
+export async function follow(host: UpgradeHost, io: ScreenIO, id: string, scheduled?: number): Promise<"completed" | "open" | "left"> {
+  let step = "", waiting = "", unowned = 0, tick = 0;
+  for (;; tick++) {
+    const op = host.read(id);
+    if (io.interrupted()) { io.out(left(op)); return "left"; }
+    if (io.typed()) return "open";
+    const unwritten = scheduled !== undefined && op.updatedAt === scheduled;
+    if (!unwritten) {
+      scheduled = undefined;
+      if (op.step !== step) { step = op.step; waiting = ""; tick = 0; io.out(`  ${stepLabel(step)}`); }
+      if (op.phase === "completed") return "completed";
+      if (op.phase === "cancelled" || op.phase === "blocked") return "open";
+    }
+    // A runner claims and writes a moment after it is spawned: only 5 s without either reads as gone.
+    unowned = host.runner(id) === undefined || unwritten ? unowned + 1 : 0;
+    if (unowned > 20) return "open";
+    if (!unwritten && /^(re)?prepare:/.test(step) && tick % 8 === 0) {
+      const live = await host.live(op).catch(() => ({} as Record<string, Inspection | undefined>));
+      // A hub that is not ready and names no cause (one older than #272 lists only busy peers and approvals) still waits.
+      const held = Object.values(live).some((i) => i?.recovery?.operationId === id && i.recovery.ready === false);
+      const now = [...new Set(Object.values(live).flatMap((i) => i?.recovery?.waiting ?? []))].join(", ") || (held ? "the hub is not quiet yet (it names no cause: a completion check, a task command or a Pi call may be in flight)" : "");
+      if (now && now !== waiting) io.out(`    waiting for: ${now}`);
+      waiting = now;
+    }
+    await io.sleep(250);
+  }
+}
+
+/**
+ * End attached agents before the upgrade, by kind or by name: an ended agent is offline in the next plan, so it is
+ * neither waited for nor restored. True when at least one was asked to end.
+ */
+export async function endAgents(host: UpgradeHost, io: ScreenIO, endable: { planned: PlannedProject; peer: RecoveryPeer }[]): Promise<boolean> {
+  type Agent = (typeof endable)[number];
+  // An upgrade plans every running project and each has its own claude or codex: with several, a name carries its project.
+  const several = new Set(endable.map((e) => e.planned.project.id)).size > 1;
+  const full = (e: Agent) => `${e.planned.project.id}/${e.peer.id}`, label = (e: Agent) => several ? full(e) : e.peer.id;
+  const of = (kind: string) => endable.filter((e) => peerKind(e.planned, e.peer) === kind);
+  const tui = of("tui"), headless = of("headless");
+  const choices = [...(tui.length ? [`[t] the TUI agents (${tui.map(label).join(", ")})`] : []), ...(headless.length ? [`[h] the headless agents (${headless.map(label).join(", ")})`] : []), "or names separated by spaces", "[Enter] none"];
+  const words = new Set(((await io.ask(`End which agents? ${choices.join("  ")}: `)) ?? "").toLowerCase().split(/\s+/).filter(Boolean));
+  const picked = new Set<Agent>(), unknown: string[] = [], ambiguous: string[] = [];
+  for (const word of words) {
+    const named = word === "t" ? tui : word === "h" ? headless : endable.filter((e) => word === full(e) || word === e.peer.id);
+    if (word === "t" || word === "h" || named.length === 1) for (const e of named) picked.add(e);
+    else (named.length ? ambiguous : unknown).push(word);
+  }
+  if (unknown.length) io.out(`not an agent that can be ended here: ${unknown.join(", ")}`);
+  if (ambiguous.length) io.out(`${ambiguous.join(", ")}: attached in several projects; name the one to end with its project (${endable.filter((e) => ambiguous.includes(e.peer.id)).map(full).join(", ")})`);
+  const chosen = endable.filter((e) => picked.has(e));
+  if (!chosen.length) { io.out("no agent was ended"); return false; }
+  if (!(await yes(io, `End ${chosen.map(label).join(", ")} now? A TUI agent's terminal is closed, a turn in progress is cut, and none of them is restored by the upgrade. [y/N] `))) { io.out("no agent was ended"); return false; }
+  let asked = 0;
+  for (const e of chosen) {
+    if (io.interrupted()) { io.out(`interrupted: ${chosen.length - asked} of ${chosen.length} left alone (${chosen.slice(asked).map(label).join(", ")})`); break; }
+    io.out(`  ${await host.endPeer(e.planned, e.peer)}`);
+    asked++;
+  }
+  return asked > 0;
+}
+
+/** `ahub reset` for one project: the scope, its dry run, a confirmation, then the reset itself. */
+export async function resetFlow(host: UpgradeHost, io: ScreenIO, projects: { id: string; root: string }[]): Promise<void> {
+  if (host.lock()) return io.out("an operation holds the recovery lock; cancel or end it before a reset");
+  let project = projects[0];
+  if (projects.length > 1) {
+    projects.forEach((p, i) => io.out(`  ${i + 1}  ${p.id}  ${p.root}`));
+    project = projects[Number(await io.ask("Reset which project? [number, Enter: none] ")) - 1];
+  }
+  if (!project) return io.out("nothing was reset");
+  const scope = ((await io.ask("[r] runtime reset (deliveries, holds, pauses, session pointers)  [a] full reset (archive the state directory)  [Enter] none: ")) ?? "").toLowerCase();
+  if (scope !== "r" && scope !== "a") return io.out("nothing was reset");
+  const reset = [host.entry, "--project", project.root, "reset", ...(scope === "a" ? ["--all"] : [])];
+  if ((await io.run(reset)) !== 0) return; // the dry run: it lists and changes nothing
+  if (!(await yes(io, `Apply this reset of ${project.id}? [y/N] `))) return io.out("nothing was reset");
+  await io.run([...reset, "--yes"]);
+}
+
+/** One open operation: what it did, and the choices `status` would name, run on the spot. */
+export async function operationScreen(host: UpgradeHost, io: ScreenIO, id: string): Promise<"completed" | "ended" | "quit"> {
+  for (let acted = false, reset = false; ;) {
+    if (io.interrupted()) return "quit";
+    const op = host.read(id), runner = host.runner(id);
+    if (op.phase === "completed") { io.out(`${op.plan.kind} to ${op.plan.version} completed`); return "completed"; }
+    if (op.phase === "cancelled") {
+      io.out(`operation ${id} is ${op.disposition ? "ended (abandoned, not completed)" : "cancelled"}; the recovery lock is free`);
+      // Ended from this screen: the reset the person may have come for is one key away.
+      if (acted && !reset && await yes(io, "Reset a project's hub now? [y/N] ")) await resetFlow(host, io, op.plan.projects.map((p) => p.project));
+      return "ended";
+    }
+    const live = runner ? {} : await host.live(op);
+    io.out("");
+    for (const line of operationLines(op, runner, Date.now())) io.out(line);
+    const run = async (action: "resume" | "abort" | "dispose", flags: string[] = []) => { acted = true; return (await io.run(recoveryArgv(op, action, flags))) === 0; };
+    // A runner scheduled by a command has not written yet: follow from the receipt as that command left it (resume
+    // writes nothing, a fresh-session choice records itself first).
+    const followed = async (): Promise<boolean> => (await follow(host, io, id, host.read(id).updatedAt)) === "left";
+    const end = async (): Promise<boolean> => {
+      if (!(await yes(io, "End this operation? Its own targets are stopped and the upgrade is abandoned, not completed. [y/N] "))) return false;
+      const reason = await io.ask("Reason for the audit [Enter: ended from the upgrade screen]: ");
+      if (reason === null) return false; // Ctrl+C or the end of input is never a reason
+      return run("dispose", ["--stop-and-archive", "--reason", reason.slice(0, 500) || "ended from the upgrade screen"]);
+    };
+    // An act returns true when the person left (Ctrl+C in a follow).
+    const menu = new Map<string, { label: string; act: () => Promise<boolean | void> }>();
+    if (typeof runner === "number") {
+      menu.set("w", { label: "follow its progress", act: async () => (await follow(host, io, id)) === "left" });
+      if (cancellableWait(op) && host.runnerStoppable(id)) menu.set("c", { label: "cancel: stop the waiting runner and cancel (nothing was closed or stopped)", act: async () => {
+        // Decided again on the receipt as it is now: the runner may have moved on while the menu was read.
+        if (!cancellableWait(host.read(id)) || !host.runnerStoppable(id)) return io.out("the runner moved on; nothing was stopped or cancelled");
+        if (await host.stopRunner(id)) await run("abort");
+        else io.out("the runner could not be stopped; nothing was cancelled");
+      } });
+      else if (cancellableWait(op)) io.out("  cancel is not offered: this runner's claim cannot be verified (an older coordinator's, or unreadable); a live runner gives up its wait after 10 minutes");
+    }
+    for (const choice of nextChoices(op, runner, live)) {
+      if (choice.kind === "wait") { if (typeof runner !== "number") io.out(`  ${choice.text}`); }
+      else if (choice.kind === "resume") menu.set("r", { label: `resume${op.error ? " (after the step the error names)" : ""}`, act: async () => (await run("resume")) && followed() });
+      else if (choice.kind === "abort") menu.set("c", { label: "cancel (no runtime was stopped; nothing to roll back)", act: async () => { await run("abort"); } });
+      else if (choice.kind === "fresh") menu.set(menu.has("f") ? `f-${choice.peer}` : "f", { label: `start ${choice.peer} as a new session (its conversation is recorded as lost)`, act: async () => {
+        const reason = await io.ask(`Reason for losing ${choice.peer}'s conversation [Enter: go back]: `);
+        return !!reason && (await run("dispose", ["--fresh-session", choice.peer!, "--reason", reason.slice(0, 500)])) && followed();
+      } });
+      else {
+        menu.set("e", { label: "end: stop and archive (the upgrade is abandoned, not completed)", act: async () => { await end(); } });
+        menu.set("x", { label: "end, then reset a project's hub", act: async () => { if (await end()) { reset = true; await resetFlow(host, io, op.plan.projects.map((p) => p.project)); } } });
+      }
+    }
+    menu.set("s", { label: "refresh", act: async () => {} });
+    menu.set("j", { label: "receipt as JSON", act: async () => io.out(JSON.stringify(publicOperation(op, runner, live), null, 2)) });
+    for (const [key, item] of menu) io.out(`  [${key}] ${item.label}`);
+    io.out("  [q] quit (the operation stays as it is)");
+    const answer = await io.ask("> ");
+    if (answer === null || answer.toLowerCase() === "q") return "quit";
+    if ((await menu.get(answer.toLowerCase())?.act()) === true) return "quit";
+  }
+}
+
+/** The plan, reviewed with the person: apply and follow, or an open operation's screen first. */
+export async function planScreen(host: UpgradeHost, io: ScreenIO): Promise<void> {
+  for (let touched = false; ;) {
+    if (io.interrupted()) return;
+    const owner = host.lock();
+    if (owner) {
+      if ((await operationScreen(host, io, owner)) !== "ended") return;
+      if (!(await yes(io, "Plan again? [y/N] "))) return;
+      continue;
+    }
+    const plan = await host.plan();
+    if (io.interrupted()) return;
+    io.out("");
+    for (const line of planLines(plan, host.version)) io.out(line);
+    const blocked = plan.blockers.length > 0 || plan.projects.some((p) => p.blockers.length > 0);
+    if (!plan.projects.length) io.out(plan.kind === "upgrade" ? `no hub is running, so nothing is carried over: install with \`bun add -g @staix/agent-hub@${plan.version}\`, then \`ahub setup\`` : "this project's hub is not running: `ahub up` starts it");
+    const endable = plan.projects.flatMap((p) => p.source.peers.filter((peer) => ["tui", "headless"].includes(peerKind(p, peer))).map((peer) => ({ planned: p, peer })));
+    const keys = [...(blocked ? [] : ["[a] apply"]), "[r] refresh", ...(endable.length ? ["[k] end agents"] : []), "[j] plan as JSON", ...(plan.projects.length ? ["[x] reset a project's hub"] : []), "[q] quit"];
+    const answer = (await io.ask(`${blocked ? "Blocked: take the next action each blocker names, then refresh.\n" : ""}${keys.join("  ")}: `))?.toLowerCase() ?? "q";
+    if (answer === "q") return touched ? undefined : io.out("nothing was changed");
+    if (answer === "j") io.out(JSON.stringify(plan, null, 2));
+    else if (answer === "x" && plan.projects.length) { touched = true; await resetFlow(host, io, plan.projects.map((p) => p.project)); }
+    else if (answer === "k" && endable.length) touched = (await endAgents(host, io, endable)) || touched;
+    else if (answer === "a" && !blocked) {
+      touched = true;
+      let op: RecoveryOperation;
+      try { op = await host.apply(plan, () => io.interrupted()); } catch (error) { io.out(`ahub: ${(error as Error).message}`); continue; }
+      io.out(`operation ${op.id} started; Enter opens its menu, Ctrl+C leaves, and neither stops the ${plan.kind}`);
+      const end = await follow(host, io, op.id);
+      if (end === "completed") return io.out(`${plan.kind} to ${plan.version} completed`);
+      if (end === "left") return;
+      // Blocked, cancelled or left to its runner: the lock check above opens its screen, or plans again once it is free.
+    }
+  }
+}

package/src/cli/upgrade-runtime.ts

@@ -1,3 +1,4 @@
+import { isDatabaseBusy, isDatabaseBusyReply, RecoveryDatabaseBusy } from "../hub/recovery-error.ts";
 import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync } from "node:fs";
 import { homedir } from "node:os";
 import { basename, join, resolve } from "node:path";
@@ -8,10 +9,11 @@ import type { Project } from "../hub/registry.ts";
 import { hubHome } from "../hub/project.ts";
 import { packageDigest, registryRelease, runCommand, stageRelease, verifyPackage, type RunCommand } from "./recovery-package.ts";
 import { inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
-import { FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type UpgradePlan } from "./upgrade.ts";
+import { FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
 import { readEvents } from "../hub/events.ts";
 import { refreshManager } from "../hub/manager.ts";
 import { abandonRestartSnapshot, readRestartSnapshot, waiveRecoveryPeers } from "../hub/restart.ts";
+import { recoveryLock } from "../hub/recovery-store.ts";
 
 /** An unmanaged Claude's plugin retries with a backoff of at most 30 s; three of those bound the reconnect wait (#206). */
 const RECONNECT_WAIT_MS = 90_000;
@@ -82,21 +84,79 @@ export function restoredTerminalArgv(entrypoint: string, projectRoot: string, bi
     ...(binding.sessionFile ? ["--session-file", binding.sessionFile] : ["--session-id", binding.sessionId])];
 }
 
+const RECOVERY_CONTROL_TIMEOUT_MS = 30_000;
+
 async function rpc(project: Project, message: Record<string, unknown>, protocol = PROTOCOL): Promise<any> {
-  const client = await ControlClient.connect(project.stateDir, { role: "console", projectRoot: project.root, projectId: project.id }, 30_000, protocol);
+  const client = await ControlClient.connect(project.stateDir, { role: "console", projectRoot: project.root, projectId: project.id }, RECOVERY_CONTROL_TIMEOUT_MS, protocol);
   try {
-    const reply = await client.request(message, 30_000);
-    if (reply.ok === false) throw new Error(reply.error ?? "recovery control request failed");
+    const reply = await client.request(message, RECOVERY_CONTROL_TIMEOUT_MS);
+    if (reply.ok === false) {
+      // An older hub has only its exact SQLite message; preserve compatibility without trusting arbitrary text.
+      if (isDatabaseBusyReply(reply)) throw new RecoveryDatabaseBusy();
+      throw new Error(reply.error ?? "recovery control request failed");
+    }
     return reply;
   } finally { client.close(); }
 }
 
+/**
+ * #272: end one attached agent before an upgrade, for the person who chose it on the plan screen. The plan is what was
+ * drawn, so all of it is read again and any difference ends nothing: no operation holds the lock, the hub runs as the
+ * instance the plan read with the peer attached, and for a TUI agent the attached session is the one the plan bound and
+ * binds again to a terminal with the plan's handle and incarnation. `closeTerminal` itself compares only the terminal's
+ * handle, incarnation, worktree and root, never who runs in it. A headless agent is the hub's own process, so the hub
+ * is asked to stop it, which a hub older than that request cannot do. `deps` are a test's fakes.
+ */
+export async function endPlannedPeer(planned: PlannedProject, peer: RecoveryPeer, deps: { run?: RunCommand; inspect?: (project: Project) => Promise<Inspection>; lock?: () => string | undefined } = {}): Promise<string> {
+  const { run = runCommand, inspect = inspectRecovery, lock = recoveryLock } = deps;
+  const name = `${planned.project.id}/${peer.id}`;
+  const binding = (planned.terminals as TerminalBinding[]).find((t) => t.peer === peer.id);
+  const refused = (why: string) => `${name}: not ended (${why})`;
+  let cut = "", current: TerminalBinding | undefined;
+  try {
+    const owner = lock();
+    if (owner) return refused(`recovery operation ${owner} holds the lock`);
+    const hub = await inspect(planned.project);
+    if (hub.state !== "running" || hub.instanceId !== planned.source.instanceId) return refused(`its hub reads as ${hub.state === "running" ? `instance ${hub.instanceId}, not the ${planned.source.instanceId} of the plan` : hub.state}`);
+    const now = hub.peers.find((p) => p.id === peer.id && p.state !== "offline");
+    if (!now) return refused("it is no longer attached to its hub");
+    // The hub holds what a peer that goes offline mid-turn was working on (`uncertain` in bus.ts).
+    if (now.state === "busy") cut = "; its turn was cut: a delivery it had not settled is held as needs_review, and its queue stays held until ahub queue resolve";
+    if (binding) {
+      if ((binding.peer === "codex" ? now.threadId : now.sessionId) !== binding.sessionId) return refused(`the session attached now is not the one the plan bound to terminal ${binding.handle}`);
+      // The same question the plan asked (`makeUpgradePlan`), for this one peer.
+      const session = binding.peer === "pi" ? { sessionId: binding.sessionId, ...(now.sessionFile ? { sessionFile: now.sessionFile } : {}), ...(now.args?.backend ? { backend: now.args.backend } : {}), ...(now.args?.model ? { model: now.args.model } : {}) } : binding.sessionId;
+      const found = await inspectTerminals(planned.project.root, { [binding.peer]: session }, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: planned.source.instanceId });
+      current = found.byPeer[binding.peer];
+      if (found.manualRequired || !current) return refused(`its terminal cannot be bound again: ${found.blockers[0]?.message ?? "no terminal was found"}`);
+      if (current.handle !== binding.handle || current.incarnationId !== binding.incarnationId) return refused(`its session is now bound to terminal ${current.handle} (${current.incarnationId}), not the plan's ${binding.handle} (${binding.incarnationId})`);
+    }
+    // The reads above take a second or more: an operation that took the lock meanwhile still stops this.
+    const taken = lock();
+    if (taken) return refused(`recovery operation ${taken} holds the lock`);
+  } catch (error) { return refused((error as Error).message); } // a lock or a registry that cannot be read throws
+  if (current) {
+    const result = await closeTerminal(current, 0, terminalOptions(run));
+    return result.closed ? `${name}: terminal ${current.handle} closed${cut}` : `${name}: its terminal was not closed (${result.blockers[0]?.message ?? "unknown reason"}); end it in that terminal`;
+  }
+  try {
+    await rpc(planned.project, { t: "peer_stop", peer: peer.id }, planned.source.protocol ?? PROTOCOL);
+    return `${name}: stopped${cut}`;
+  } catch (error) {
+    const message = (error as Error).message;
+    return /does not know "peer_stop"/.test(message)
+      ? `${name}: hub ${planned.source.version ?? "of this version"} cannot end a headless agent by itself; it stops with the old hub at the upgrade and the new hub starts it again (ahub stop ${peer.id} ends it then)`
+      : `${name}: not stopped (${message})`;
+  }
+}
+
 export async function inspectRecovery(project: Project): Promise<Inspection> {
   // #226: the claim is judged by the registry as it is now, never by the plan's frozen copy, which holds the source's
   // pid from planning time (after a reboot another process may hold it). A registry that cannot be read throws: the
   // caller reads that as not inspected, never as stopped.
   const claim = registeredProjects().find((p) => p.id === project.id);
-  const base = await inspectProject({ ...project, instanceId: claim?.instanceId ?? null, pid: claim?.pid ?? null, pidSignature: claim?.pidSignature ?? null });
+  const base = await inspectProject({ ...project, instanceId: claim?.instanceId ?? null, pid: claim?.pid ?? null, pidSignature: claim?.pidSignature ?? null }, RECOVERY_CONTROL_TIMEOUT_MS);
+  if (base.failure?.code === "database_busy") throw new RecoveryDatabaseBusy();
   const control = readControl(project.stateDir);
   const sourceProtocol = control?.protocol;
   const legacySupported = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]);
@@ -109,11 +169,16 @@ export async function inspectRecovery(project: Project): Promise<Inspection> {
     ...(control?.instanceId ? { instanceId: control.instanceId } : {}), ...(control?.protocol ? { protocol: control.protocol } : {}), ...(snapshot ? { snapshot } : {}),
   };
   let status = base.status;
-  if (!status && legacySupported) {
+  if (!status && control && sourceProtocol === PROTOCOL) return {
+    state: "unavailable", peers: [], blockers: ["runtime changed during recovery inspection"],
+    ...(control.instanceId ? { instanceId: control.instanceId } : {}), protocol: control.protocol,
+  };
+  if (!status && legacySupported && sourceProtocol !== PROTOCOL) {
     try {
       const readback = await rpc(project, { t: "status" }, sourceProtocol);
       status = readback.status;
-    } catch {
+    } catch (error) {
+      if (isDatabaseBusy(error)) throw error;
       // server.stop() runs before state-file removal, so a stopping hub briefly refuses
       // connections with its manifest still on disk. Never infer stopped from a connect
       // failure: callers poll again and see "stopped" once the manifest is gone.
@@ -124,15 +189,18 @@ export async function inspectRecovery(project: Project): Promise<Inspection> {
   if (!status || !control) return { state: "unavailable", peers: [], blockers: ["authenticated source status unavailable"] };
   let response: any;
   try { response = await rpc(project, { t: "recovery", op: "inspect", expectedInstanceId: status.instanceId }, sourceProtocol ?? PROTOCOL); }
-  catch {
+  catch (error) {
+    if (isDatabaseBusy(error)) throw error;
     // A stop can complete between authenticated status and the second read. Never infer stopped
     // from a connection failure: the coordinator will inspect the ownership manifest again.
     return { state: "unavailable", instanceId: status.instanceId, version: status.version, protocol: status.protocol,
       peers: [], blockers: ["runtime changed during recovery inspection"] };
   }
   const peers = Object.values(response.recovery?.peers ?? {}) as any[];
+  // #272: the hub's own words for what it waits on; shown to the person, never decided on.
+  const waiting = (Array.isArray(response.recovery?.blockers) ? response.recovery.blockers : []).filter((b: unknown): b is string => typeof b === "string").slice(0, 20).map((b: string) => b.slice(0, 200));
   return { state: "running", instanceId: status.instanceId, version: status.version, protocol: status.protocol,
-    recovery: { operationId: response.recovery?.operationId, phase: response.recovery?.phase, ready: response.recovery?.ready }, peers: peers.map((peer) => ({ id: peer.id, state: peer.state,
+    recovery: { operationId: response.recovery?.operationId, phase: response.recovery?.phase, ready: response.recovery?.ready, ...(waiting.length ? { waiting } : {}) }, peers: peers.map((peer) => ({ id: peer.id, state: peer.state,
       ...(peer.threadId ? { threadId: peer.threadId } : {}), ...(peer.sessionId ? { sessionId: peer.sessionId } : {}),
       ...(typeof (peer.sessionFile ?? peer.launch?.sessionFile) === "string" ? { sessionFile: peer.sessionFile ?? peer.launch.sessionFile } : {}),
       ...(peer.launch ? { args: Object.fromEntries(Object.entries(peer.launch).filter(([k, v]) => ["model", "route", "sessionFile", "mode", "backend"].includes(k) && typeof v === "string")) as Record<string, string> } : {}) })), blockers: [] };
@@ -449,6 +517,9 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
           if (current && current.state !== "offline") continue;
           const args = [peer.id];
           for (const key of ["mode", "backend", "model", "route"]) if (typeof peer.args?.[key] === "string") args.push(`--${key}`, peer.args[key]!);
+          // #269: `ahub pi` without a mode now takes Pi's start mode (tui by default). This Pi ran headless (a terminal
+          // one is filtered out above), so a snapshot that recorded no mode still restores it headless.
+          if (peer.id === "pi" && typeof peer.args?.mode !== "string") args.push("--mode", "headless");
           if (peer.id === "pi") {
             const sessionFile = peer.sessionFile ?? peer.args?.sessionFile;
             if (typeof sessionFile === "string") args.push("--session-file", sessionFile);