@staix/agent-hub 0.12.20 → 0.12.22

package/src/cli/upgrade-runtime.ts

@@ -8,10 +8,11 @@ import type { Project } from "../hub/registry.ts";
 import { hubHome } from "../hub/project.ts";
 import { packageDigest, registryRelease, runCommand, stageRelease, verifyPackage, type RunCommand } from "./recovery-package.ts";
 import { inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
-import { FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type UpgradePlan } from "./upgrade.ts";
+import { FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
 import { readEvents } from "../hub/events.ts";
 import { refreshManager } from "../hub/manager.ts";
 import { abandonRestartSnapshot, readRestartSnapshot, waiveRecoveryPeers } from "../hub/restart.ts";
+import { recoveryLock } from "../hub/recovery-store.ts";
 
 /** An unmanaged Claude's plugin retries with a backoff of at most 30 s; three of those bound the reconnect wait (#206). */
 const RECONNECT_WAIT_MS = 90_000;
@@ -91,8 +92,63 @@ async function rpc(project: Project, message: Record<string, unknown>, protocol
   } finally { client.close(); }
 }
 
+/**
+ * #272: end one attached agent before an upgrade, for the person who chose it on the plan screen. The plan is what was
+ * drawn, so all of it is read again and any difference ends nothing: no operation holds the lock, the hub runs as the
+ * instance the plan read with the peer attached, and for a TUI agent the attached session is the one the plan bound and
+ * binds again to a terminal with the plan's handle and incarnation. `closeTerminal` itself compares only the terminal's
+ * handle, incarnation, worktree and root, never who runs in it. A headless agent is the hub's own process, so the hub
+ * is asked to stop it, which a hub older than that request cannot do. `deps` are a test's fakes.
+ */
+export async function endPlannedPeer(planned: PlannedProject, peer: RecoveryPeer, deps: { run?: RunCommand; inspect?: (project: Project) => Promise<Inspection>; lock?: () => string | undefined } = {}): Promise<string> {
+  const { run = runCommand, inspect = inspectRecovery, lock = recoveryLock } = deps;
+  const name = `${planned.project.id}/${peer.id}`;
+  const binding = (planned.terminals as TerminalBinding[]).find((t) => t.peer === peer.id);
+  const refused = (why: string) => `${name}: not ended (${why})`;
+  let cut = "", current: TerminalBinding | undefined;
+  try {
+    const owner = lock();
+    if (owner) return refused(`recovery operation ${owner} holds the lock`);
+    const hub = await inspect(planned.project);
+    if (hub.state !== "running" || hub.instanceId !== planned.source.instanceId) return refused(`its hub reads as ${hub.state === "running" ? `instance ${hub.instanceId}, not the ${planned.source.instanceId} of the plan` : hub.state}`);
+    const now = hub.peers.find((p) => p.id === peer.id && p.state !== "offline");
+    if (!now) return refused("it is no longer attached to its hub");
+    // The hub holds what a peer that goes offline mid-turn was working on (`uncertain` in bus.ts).
+    if (now.state === "busy") cut = "; its turn was cut: a delivery it had not settled is held as needs_review, and its queue stays held until ahub queue resolve";
+    if (binding) {
+      if ((binding.peer === "codex" ? now.threadId : now.sessionId) !== binding.sessionId) return refused(`the session attached now is not the one the plan bound to terminal ${binding.handle}`);
+      // The same question the plan asked (`makeUpgradePlan`), for this one peer.
+      const session = binding.peer === "pi" ? { sessionId: binding.sessionId, ...(now.sessionFile ? { sessionFile: now.sessionFile } : {}), ...(now.args?.backend ? { backend: now.args.backend } : {}), ...(now.args?.model ? { model: now.args.model } : {}) } : binding.sessionId;
+      const found = await inspectTerminals(planned.project.root, { [binding.peer]: session }, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: planned.source.instanceId });
+      current = found.byPeer[binding.peer];
+      if (found.manualRequired || !current) return refused(`its terminal cannot be bound again: ${found.blockers[0]?.message ?? "no terminal was found"}`);
+      if (current.handle !== binding.handle || current.incarnationId !== binding.incarnationId) return refused(`its session is now bound to terminal ${current.handle} (${current.incarnationId}), not the plan's ${binding.handle} (${binding.incarnationId})`);
+    }
+    // The reads above take a second or more: an operation that took the lock meanwhile still stops this.
+    const taken = lock();
+    if (taken) return refused(`recovery operation ${taken} holds the lock`);
+  } catch (error) { return refused((error as Error).message); } // a lock or a registry that cannot be read throws
+  if (current) {
+    const result = await closeTerminal(current, 0, terminalOptions(run));
+    return result.closed ? `${name}: terminal ${current.handle} closed${cut}` : `${name}: its terminal was not closed (${result.blockers[0]?.message ?? "unknown reason"}); end it in that terminal`;
+  }
+  try {
+    await rpc(planned.project, { t: "peer_stop", peer: peer.id }, planned.source.protocol ?? PROTOCOL);
+    return `${name}: stopped${cut}`;
+  } catch (error) {
+    const message = (error as Error).message;
+    return /does not know "peer_stop"/.test(message)
+      ? `${name}: hub ${planned.source.version ?? "of this version"} cannot end a headless agent by itself; it stops with the old hub at the upgrade and the new hub starts it again (ahub stop ${peer.id} ends it then)`
+      : `${name}: not stopped (${message})`;
+  }
+}
+
 export async function inspectRecovery(project: Project): Promise<Inspection> {
-  const base = await inspectProject(project);
+  // #226: the claim is judged by the registry as it is now, never by the plan's frozen copy, which holds the source's
+  // pid from planning time (after a reboot another process may hold it). A registry that cannot be read throws: the
+  // caller reads that as not inspected, never as stopped.
+  const claim = registeredProjects().find((p) => p.id === project.id);
+  const base = await inspectProject({ ...project, instanceId: claim?.instanceId ?? null, pid: claim?.pid ?? null, pidSignature: claim?.pidSignature ?? null });
   const control = readControl(project.stateDir);
   const sourceProtocol = control?.protocol;
   const legacySupported = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]);
@@ -127,8 +183,10 @@ export async function inspectRecovery(project: Project): Promise<Inspection> {
       peers: [], blockers: ["runtime changed during recovery inspection"] };
   }
   const peers = Object.values(response.recovery?.peers ?? {}) as any[];
+  // #272: the hub's own words for what it waits on; shown to the person, never decided on.
+  const waiting = (Array.isArray(response.recovery?.blockers) ? response.recovery.blockers : []).filter((b: unknown): b is string => typeof b === "string").slice(0, 20).map((b: string) => b.slice(0, 200));
   return { state: "running", instanceId: status.instanceId, version: status.version, protocol: status.protocol,
-    recovery: { operationId: response.recovery?.operationId, phase: response.recovery?.phase, ready: response.recovery?.ready }, peers: peers.map((peer) => ({ id: peer.id, state: peer.state,
+    recovery: { operationId: response.recovery?.operationId, phase: response.recovery?.phase, ready: response.recovery?.ready, ...(waiting.length ? { waiting } : {}) }, peers: peers.map((peer) => ({ id: peer.id, state: peer.state,
       ...(peer.threadId ? { threadId: peer.threadId } : {}), ...(peer.sessionId ? { sessionId: peer.sessionId } : {}),
       ...(typeof (peer.sessionFile ?? peer.launch?.sessionFile) === "string" ? { sessionFile: peer.sessionFile ?? peer.launch.sessionFile } : {}),
       ...(peer.launch ? { args: Object.fromEntries(Object.entries(peer.launch).filter(([k, v]) => ["model", "route", "sessionFile", "mode", "backend"].includes(k) && typeof v === "string")) as Record<string, string> } : {}) })), blockers: [] };
@@ -175,7 +233,7 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
         if (recorded?.state === "unknown") {
           // An unreadable launcher is never gone (#215): whether the session is managed cannot be told.
           delete peer.sessionId;
-          blockers.push(`claude: ${recorded.record ? `the launcher recorded in terminal ${recorded.record.handle}` : "the launcher record (terminal-recovery.json)"} cannot be read, so whether the attached session is managed is unknown; manual-required; next action: make it readable (or end that session and close its terminal), then make a new plan`);
+          blockers.push(`claude: ${recorded.record ? `the launcher recorded in terminal ${recorded.record.handle}` : recorded.invalidRow ? "a launch record in terminal-recovery.json that may be its launcher's" : "the launcher record (terminal-recovery.json)"} cannot be read, so whether the attached session is managed is unknown; manual-required; next action: make it readable (or end that session and close its terminal), then make a new plan`);
           continue;
         }
         const launcher = recorded?.state === "live" ? recorded.record! : undefined;
@@ -305,7 +363,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
    * expected instance can report "gone"; an unreadable target or launcher is unknown and blocks, never gone.
    */
   const peerEvidence = async (planned: PlannedProject, progress: ProjectProgress, peer: TerminalBinding["peer"]):
-      Promise<{ state: "live"; session?: string; handle?: string } | { state: "gone" } | { state: "unknown"; why: string; step?: string }> => {
+      Promise<{ state: "live"; session?: string; handle?: string; attached?: true } | { state: "gone" } | { state: "unknown"; why: string; step?: string }> => {
     const target = await inspectRecovery(planned.project);
     if (target.state !== "running" || target.instanceId !== progress.instanceId) {
       // Only a hub that may still answer is waited for; a stopped, missing or other one is settled by the runner's list.
@@ -315,14 +373,44 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
     const attached = target.peers.find((p) => p.id === peer && p.state !== "offline");
     const session = peer === "codex" ? attached?.threadId : attached?.sessionId;
     if (session) return { state: "live", session };
-    const launch = await launcherOf(peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId });
-    if (launch?.state === "live") return { state: "live", handle: launch.record!.handle };
-    if (launch?.state === "unknown") {
-      return launch.record ? { state: "unknown", why: `its launcher in terminal ${launch.record.handle} cannot be read`, step: `wait until it attaches, or end it and close terminal ${launch.record.handle}` }
-        : { state: "unknown", why: "the launcher records cannot be read", step: `inspect ${recordPath(planned.project.stateDir)} and move it aside (sessions it recorded then count as unmanaged)` };
+    // #225: the launch records of every instance this operation started for the project, the current one first: a
+    // launcher a dead instance recorded may still run (read before the check below, so a block names its terminal).
+    for (const instanceId of [progress.instanceId, ...(progress.restarts ?? []).map((r) => r.instanceId)]) {
+      if (!instanceId) continue;
+      const launch = await launcherOf(peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId });
+      if (launch?.state === "live") return { state: "live", handle: launch.record!.handle, ...(attached ? { attached: true as const } : {}) };
+      if (launch?.state === "unknown") {
+        return launch.record ? { state: "unknown", why: `its launcher in terminal ${launch.record.handle} cannot be read`, step: `wait until it attaches, or end it and close terminal ${launch.record.handle}` }
+          : launch.invalidRow ? { state: "unknown", why: "a launch record that may be its launcher's cannot be evaluated", step: `inspect ${recordPath(planned.project.stateDir)} and fix that row, or end that launcher and remove its row` }
+          : { state: "unknown", why: "the launcher records cannot be read", step: `inspect ${recordPath(planned.project.stateDir)} and move it aside (sessions it recorded then count as unmanaged)` };
+      }
     }
+    // #225: attached with no provable session id (a Claude of a dead instance whose plugin reconnected, or one that has
+    // not reported its id yet) is a session of unknown identity, never "nothing attached".
+    if (attached) return { state: "unknown", why: `a ${peer} session is attached to the target without a session id`, step: `wait until it reports one, or end that ${peer} session` };
     return { state: "gone" };
   };
+  /**
+   * #225: the terminal of a done restoration of a target instance that died is closed under the close receipt rules
+   * (`closedRetired:<peer>` `pending` until Orca no longer lists it; an unreadable or truncated inventory is unknown and
+   * changes nothing) or found absent, before its peer is relaunched against the new instance.
+   */
+  const closeRetired = async (progress: ProjectProgress, peer: string, save: () => void): Promise<void> => {
+    const retired = progress.terminals[`retired:${peer}`] as TerminalBinding | undefined;
+    const key = `closedRetired:${peer}`;
+    if (!retired || typeof retired !== "object" || progress.terminals[key] === true) return;
+    const seen = await listed(retired);
+    if (progress.terminals[key] === "pending" || seen !== "listed") {
+      if (seen !== "absent") throw new Error(closeUnsettled(retired, seen));
+      progress.terminals[key] = true; save(); return;
+    }
+    const idle = await waitForIdle(retired, 120_000, terminalOptions(run));
+    if (!idle.satisfied) throw new Error(`${peer}: terminal ${retired.handle} of the stopped target's session is not verified idle; finish or cancel its turn, or close that terminal by hand, first`);
+    progress.terminals[key] = "pending"; save();
+    const result = await closeTerminal(retired, 0, terminalOptions(run));
+    if (result.manualRequired) throw new Error(`${peer}: closing terminal ${retired.handle} of the stopped target's session was not verified; manual-required`);
+    progress.terminals[key] = true; save();
+  };
   const attachedId = async (planned: PlannedProject, peer: string, op: RecoveryOperation): Promise<string> => {
     for (const deadline = now() + RECONNECT_WAIT_MS; ;) {
       const current = (await inspectRecovery(planned.project)).peers.find((p) => p.id === peer && p.state !== "offline");
@@ -423,17 +511,30 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
           await command(op, args, planned.project);
         }
       }
-      for (const original of planned.terminals as TerminalBinding[]) {
-        if ((original.peer === "claude") !== (group === "claude")) continue;
-        const key = `restored:${original.peer}`;
+      for (const plannedBinding of planned.terminals as TerminalBinding[]) {
+        if ((plannedBinding.peer === "claude") !== (group === "claude")) continue;
+        const key = `restored:${plannedBinding.peer}`;
+        // #225: after a restart, the stopped target's terminal for this peer goes first; a session it had accepted
+        // instead of the planned one (fresh choice, fresh start, zero-turn Claude) is relaunched by that id, which the
+        // operation-fenced waiver already covers.
+        await closeRetired(progress, plannedBinding.peer, save);
+        const retired = progress.terminals[`retired:${plannedBinding.peer}`] as TerminalBinding | undefined;
+        const accepted = retired && typeof retired === "object" && retired.sessionId !== plannedBinding.sessionId ? retired.sessionId : undefined;
+        const original = accepted ? { ...plannedBinding, sessionId: accepted } : plannedBinding;
         // A new session is accepted only by the operator's choice (#215 dispose --fresh-session) or a planned fresh
         // start of a Codex thread with nothing to lose that still has no rollout; an unreadable store is neither.
         const chosen = !!progress.fresh?.[original.peer];
         let read: ReturnType<typeof codexTranscript> | undefined;
-        const transcript = () => original.peer !== "codex" || chosen ? "found" : (read ??= codexTranscript(original));
-        const fresh = () => chosen || (!!planned.freshStart?.includes(original.peer) && transcript() === "missing");
+        const transcript = () => original.peer !== "codex" || (chosen && !accepted) ? "found" : (read ??= codexTranscript(original));
+        // An accepted session starts new again only when it has nothing to resume: a Codex thread that never got a
+        // rollout, or a Claude session with no transcript (`--resume` could never succeed, #21).
+        const fresh = () => accepted
+          ? (original.peer === "codex" ? (chosen || !!planned.freshStart?.includes("codex")) && transcript() === "missing" : original.peer === "claude" && !claudeTranscriptExists(original))
+          : chosen || (!!planned.freshStart?.includes(original.peer) && transcript() === "missing");
         // The runner ends this with the choices `status` shows (nextActions); resume comes first and launches a failed
         // peer again, which helps once its cause (a transient Orca create failure, a fixed store) is gone.
+        // The waiver's audit label: the operator's choice, a Claude with nothing to resume, or the plan's Codex fresh start.
+        const waiver = () => progress.fresh?.[original.peer] ? "fresh-session" : original.peer === "claude" ? "zero-turn" : "fresh-start";
         const notRestored = (why: string) => new Error(fresh()
           ? `${original.peer}: ${why}; read that terminal in Orca for the launcher's error and fix its cause`
           : `${original.peer}: ${why}; session ${original.sessionId} was not restored; resume launches it again once the cause is fixed`);
@@ -442,7 +543,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
           ? `${original.peer}: whether a ${original.peer} session or launcher is live cannot be told (${evidence.why}); nothing was recorded or created${evidence.step ? `; ${evidence.step} first` : ""}`
           : evidence.session
             ? `${original.peer}: session ${evidence.session} is attached instead of ${original.sessionId}; no terminal was created; end that ${original.peer} session and close its terminal first`
-            : `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`);
+            : `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session ${evidence.attached ? "with an id " : ""}attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`);
         const receipt = progress.terminals[key];
         if (receipt && receipt !== "pending" && receipt !== "failed") {
           progress.terminals[key] = await revalidateTerminal(planned, progress, receipt as TerminalBinding, true); save();
@@ -456,14 +557,16 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
           if (attached !== original.sessionId && !fresh() && !(original.peer === "claude" && !claudeTranscriptExists(original))) {
             throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${attached} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
           }
-          // The original came back after all: a fresh session chosen for it lost nothing.
-          if (attached === original.sessionId && progress.fresh?.[original.peer]) {
+          // The planned session came back after all: a fresh session chosen for it lost nothing. Compared with the plan,
+          // never with a session accepted after a restart (#225), whose choice must stay on record.
+          if (attached === plannedBinding.sessionId && progress.fresh?.[original.peer]) {
             delete progress.fresh[original.peer];
             if (!Object.keys(progress.fresh).length) delete progress.fresh;
           }
           // Any other accepted session needs the target to waive the saved id, or release would wait for it forever;
           // a launch writes the waiver first, but this must not rely on that.
-          if (attached !== original.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: progress.fresh?.[original.peer] ? "fresh-session" : fresh() ? "fresh-start" : "zero-turn" });
+          // The planned session needs no waiver: the target's snapshot already expects it.
+          if (attached !== original.sessionId && attached !== plannedBinding.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
           progress.terminals[key] = await revalidateTerminal(planned, progress, { ...original, sessionId: attached }, false); save();
         };
         // #215: a pending or failed receipt is settled by what is live now (the receipt table in the recovery spec).
@@ -494,7 +597,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
         if (fresh()) {
           if (!targetReadsWaivers(op)) throw new Error(`${original.peer}: target ${op.plan.version} predates recovery waivers, so it cannot accept a new session`);
           // The restored daemon must accept the new session instead of the recorded one.
-          waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: progress.fresh?.[original.peer] ? "fresh-session" : "fresh-start" });
+          waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
         }
         const entrypoint = join(op.targetRoot!, "src/cli/main.js");
         const argv = restoredTerminalArgv(entrypoint, planned.project.root, original, fresh());

package/src/cli/upgrade.ts

@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite";
 import { createHash, randomUUID } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
-import type { Project } from "../hub/registry.ts";
+import { PROJECT_COLUMNS, projectFromRow, type Project } from "../hub/registry.ts";
 import { hubHome } from "../hub/project.ts";
 import { acquireRecoveryLock, claimRunner, coordinatorCurrent, readOperation, recoveryCommand, recoveryLock, releaseRecoveryLock, writeOperation } from "../hub/recovery-store.ts";
 
@@ -20,7 +20,8 @@ export interface Inspection {
   version?: string;
   protocol?: number;
   peers: RecoveryPeer[];
-  recovery?: { operationId?: string; phase?: string; ready?: boolean };
+  /** `waiting` (#272): what the running hub says keeps it from being quiet (busy peers, pending approvals); display only. */
+  recovery?: { operationId?: string; phase?: string; ready?: boolean; waiting?: string[] };
   blockers: string[];
   /** #215: for a stopped runtime, the operation whose committed restart snapshot its state dir holds (unreleased). */
   snapshot?: string;
@@ -56,9 +57,18 @@ export interface ProjectProgress {
   fresh?: Record<string, { lost: string; reason: string; at: number }>;
   /** #215: set before the commit request is sent; from then on the source may have committed (abort is not offered). */
   commitSent?: boolean;
+  /**
+   * #225: each restart of a target that stopped after it started, recorded before the new one starts: the dead
+   * instance and when. Status and audit read it; restore reads the launch records of every instance listed here.
+   */
+  restarts?: { instanceId: string; at: number }[];
 }
 export interface RecoveryOperation {
-  schema: 1;
+  /**
+   * #227: 2 marks a stop-and-archive recorded on an operation whose coordinator predates #215. Every older runner and
+   * abort refuses a schema other than 1 before it takes the lock or claims the runner, so none can advance it.
+   */
+  schema: 1 | 2;
   id: string;
   plan: UpgradePlan;
   createdAt: number;
@@ -94,6 +104,37 @@ export function targetReadsWaivers(op: { targetRoot?: string }): boolean {
   try { return !!op.targetRoot && readFileSync(join(op.targetRoot, "src/hub/restart.ts"), "utf8").includes("export function readRecoveryWaivers"); } catch { return false; }
 }
 
+/**
+ * #225: whether an operation's coordinator restarts a target that stopped after it started. Resume runs the operation's
+ * own runner, so an older one keeps refusing and `next` must not offer resume for that row.
+ * ponytail: a text sniff of its upgrade.ts, like targetReadsWaivers and coordinatorCurrent; a rename reads as "cannot
+ * restart" (the safe side). Upgrade path: the capability list those two name.
+ */
+export function coordinatorRestarts(op: { sourceRoot?: string }): boolean {
+  if (!op.sourceRoot) return true;
+  try { return readFileSync(join(op.sourceRoot, "src/cli/upgrade.ts"), "utf8").includes("export function recordRestart"); } catch { return false; }
+}
+
+/**
+ * #225: record the restart of a target that stopped after it started, before the new one starts: the dead instance and
+ * the time stay for status and audit. Each done `restored:<peer>` binding of the dead instance is retired
+ * (`retired:<peer>`): restore closes its terminal (or finds it absent) before that peer is relaunched, and never
+ * revalidates it against the new instance. `closed:<peer>`, `commitSent`, `fresh`, waivers and the install flags stay.
+ * The project goes back to `stopped`, whose path starts the target from this operation's snapshot.
+ */
+export function recordRestart(progress: ProjectProgress, at: number): void {
+  (progress.restarts ??= []).push({ instanceId: progress.instanceId ?? "unknown", at });
+  for (const [key, value] of Object.entries(progress.terminals)) {
+    if (!key.startsWith("restored:") || !value || typeof value !== "object") continue;
+    const peer = key.slice("restored:".length);
+    progress.terminals[`retired:${peer}`] = value;
+    delete progress.terminals[`closedRetired:${peer}`]; // a newer binding to close: its own receipt starts over
+    delete progress.terminals[key];
+  }
+  delete progress.instanceId;
+  progress.phase = "stopped";
+}
+
 /** Effects anywhere in the operation: a project past `prepared` or any terminal receipt. Abort needs none. */
 export const hasEffects = (op: RecoveryOperation) => op.projects.some((p) => !["pending", "prepared"].includes(p.phase) || Object.keys(p.terminals).length > 0);
 /**
@@ -144,9 +185,9 @@ export function resumeBlocked(op: RecoveryOperation, live: Record<string, Inspec
     if (state.state !== "stopped") continue;
     if (p.phase === "pending") return `${p.id}: the source stopped before it was prepared`;
     if (p.phase === "prepared" && !p.commitSent && !old) return `${p.id}: the source stopped while prepared, with no commit request`;
-    // ponytail: a target that dies after it started is not restarted; that needs its peers relaunched against the new
-    // instance (CodexPeer is created only by start, launcher records are keyed by instance). Follow-up issue.
-    if (p.phase === "started" || p.phase === "peers-restored") return `${p.id}: the target stopped after it started; restarting it is not supported in this release`;
+    // #225: a target that stopped after it started is restarted from this operation's snapshot (the line below), by a
+    // coordinator that can; an older one's runner refuses it.
+    if ((p.phase === "started" || p.phase === "peers-restored") && !coordinatorRestarts(op)) return `${p.id}: the target stopped after it started; this operation's coordinator cannot restart it`;
     if (state.snapshot !== op.id) return `${p.id}: its hub stopped without this operation's restart snapshot`;
   }
   return undefined;
@@ -162,32 +203,48 @@ export function disposeRefusal(op: RecoveryOperation, live: Record<string, Inspe
   return unsettled ? { project: unsettled.id, reads: live[unsettled.id]?.state ?? "not inspected" } : undefined;
 }
 
+/** #272: one entry of `next`. `text` is the line status prints; a screen runs `kind` through `recoveryArgv`. */
+export interface NextChoice {
+  kind: "resume" | "abort" | "fresh" | "stop" | "wait";
+  /** `fresh` only: the peer that starts as a new session. */
+  peer?: string;
+  text: string;
+}
+
 /**
  * #215: what the person can do now, in the receipt table's order: resume (it also launches a failed peer again),
  * abort where it can succeed, a fresh session for a failed Codex or Claude restoration, and stop-and-archive last.
- * `status` and every error that names choices use this one list.
+ * `status`, every error that names choices and the interactive screen (#272) use this one list.
  */
-export function nextActions(receipt: RecoveryOperation, runner?: number | "unknown", live: Record<string, Inspection | undefined> = {}): string[] {
+export function nextChoices(receipt: RecoveryOperation, runner?: number | "unknown", live: Record<string, Inspection | undefined> = {}): NextChoice[] {
   // A receipt read without its project lists (never written so by a coordinator) still gets the general choices.
   const op: RecoveryOperation = receipt.projects ? receipt : { ...receipt, projects: [], plan: { ...receipt.plan, projects: receipt.plan?.projects ?? [] } };
   if (op.phase === "completed" || op.phase === "cancelled") return [];
   // claimRunner refuses an owner it cannot read, so resume, abort and dispose are all refused until the record reads.
-  if (runner === "unknown") return [`${recoveryCommand(op, "status")} again: whether a runner holds the operation could not be read (resume, abort and dispose are refused until it can)`];
-  if (runner) return [`wait: runner ${runner} is working; ${recoveryCommand(op, "status")}`];
+  if (runner === "unknown") return [{ kind: "wait", text: `${recoveryCommand(op, "status")} again: whether a runner holds the operation could not be read (resume, abort and dispose are refused until it can)` }];
+  if (runner) return [{ kind: "wait", text: `wait: runner ${runner} is working; ${recoveryCommand(op, "status")}` }];
   const unsettled = disposeRefusal(op, live);
-  const stop = unsettled ? `wait until ${unsettled.project}'s hub settles, then ${recoveryCommand(op, "status")}` : recoveryCommand(op, "dispose", STOP);
-  if (op.disposition) return [unsettled ? stop : `rerun ${stop}`];
+  const stop: NextChoice = unsettled ? { kind: "wait", text: `wait until ${unsettled.project}'s hub settles, then ${recoveryCommand(op, "status")}` } : { kind: "stop", text: recoveryCommand(op, "dispose", STOP) };
+  if (op.disposition) return [unsettled ? stop : { kind: "stop", text: `rerun ${stop.text}` }];
   const failed = [...new Set(op.projects.flatMap((p) => failedPeers(op, p, live)))];
   const old = !!op.sourceRoot && !coordinatorCurrent(op.sourceRoot);
   const abortable = !abortRefusal(op, live);
   const stuck = resumeBlocked(op, live);
   return [
-    ...(stuck ? [] : [`${recoveryCommand(op, "resume")}${op.error ? " (after the next action in error)" : ""}${old ? ` (runs the coordinator that started this operation, which cannot re-prepare an expired hold: if it reports "source is no longer prepared", use ${abortable ? "abort or " : ""}stop-and-archive)` : ""}`]),
-    ...(abortable ? [recoveryCommand(op, "abort")] : []),
-    ...(targetReadsWaivers(op) && !stuck ? failed.map((peer) => recoveryCommand(op, "dispose", `--fresh-session ${peer} --reason <text>`)) : []),
+    ...(stuck ? [] : [{ kind: "resume" as const, text: `${recoveryCommand(op, "resume")}${op.error ? " (after the next action in error)" : ""}${old ? ` (runs the coordinator that started this operation, which cannot re-prepare an expired hold: if it reports "source is no longer prepared", use ${abortable ? "abort or " : ""}stop-and-archive)` : ""}` }]),
+    ...(abortable ? [{ kind: "abort" as const, text: recoveryCommand(op, "abort") }] : []),
+    ...(targetReadsWaivers(op) && !stuck ? failed.map((peer) => ({ kind: "fresh" as const, peer, text: recoveryCommand(op, "dispose", `--fresh-session ${peer} --reason <text>`) })) : []),
     stop,
   ];
 }
+export const nextActions = (receipt: RecoveryOperation, runner?: number | "unknown", live: Record<string, Inspection | undefined> = {}): string[] => nextChoices(receipt, runner, live).map((c) => c.text);
+
+/**
+ * #272: a runner that only waits for its source to get quiet can be stopped and the operation aborted: nothing was
+ * closed, stopped or asked to commit. Abort decides again on what is live, so a runner that moved on meanwhile is
+ * refused there, never here.
+ */
+export const cancellableWait = (op: RecoveryOperation): boolean => op.phase === "running" && op.step.startsWith("prepare:") && !op.disposition && !hasEffects(op) && !op.projects.some((p) => p.commitSent);
 
 /**
  * "next actions: ..." from nextActions, with the inspections the caller holds. Every error that lists choices ends with
@@ -231,7 +288,9 @@ export function registeredProjects(home = hubHome()): Project[] {
   if (!existsSync(path)) return [];
   const db = new Database(path, { readonly: true });
   try {
-    return db.query("SELECT id, root, state_dir as stateDir, base_port as basePort, instance_id as instanceId, pid FROM projects ORDER BY root").all() as Project[];
+    // #226: a registry no current release opened yet has no signature column, and this read never migrates.
+    const signed = (db.query("PRAGMA table_info(projects)").all() as { name: string }[]).some((c) => c.name === "pid_signature");
+    return (db.query(`SELECT ${PROJECT_COLUMNS}${signed ? ", pid_signature" : ""} FROM projects ORDER BY root`).all() as Record<string, unknown>[]).map(projectFromRow);
   } finally { db.close(); }
 }
 
@@ -256,8 +315,21 @@ export function createOperation(plan: UpgradePlan, sourceRoot: string, home = hu
   return op;
 }
 
+/**
+ * #227: a coordinator from before #215 never reads `disposition`, and an older global `ahub recovery resume` runs it.
+ * Every older validateReceipt refuses a schema other than 1 before the lock and the runner claim, so schema 2 keeps it
+ * from advancing an operation being abandoned. A #215 coordinator's runner refuses the disposition itself and keeps
+ * schema 1, so a 0.12.20 CLI can still rerun its dispose. True when the receipt changed (a disposition recorded at 1
+ * by 0.12.20 is moved to 2 by the next command of this release that holds the runner claim).
+ */
+function abandoned(op: RecoveryOperation): boolean {
+  if (!op.disposition || op.schema !== 1 || !op.sourceRoot || coordinatorCurrent(op.sourceRoot)) return false;
+  op.schema = 2;
+  return true;
+}
+
 function validateReceipt(id: string, op: RecoveryOperation): void {
-  if (op.schema !== 1 || op.id !== id) throw new Error("unsupported operation receipt");
+  if (!(op.schema === 1 || (op.schema === 2 && op.disposition)) || op.id !== id) throw new Error("unsupported operation receipt");
   const { fingerprint, ...reviewed } = op.plan;
   if (fingerprint !== planFingerprint(reviewed) || op.projects.length !== op.plan.projects.length ||
       op.projects.some((p, i) => p.id !== op.plan.projects[i]?.project.id)) throw new Error("reviewed operation plan or project scope changed");
@@ -302,8 +374,12 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
   } catch (error) { releaseRunner(); throw error; }
   const save = () => { op.updatedAt = driver.now(); writeOperation(id, op, home); };
   const step = (value: string) => { op.step = value; save(); };
+  // #225: one restart per project per resume, so a target that keeps dying never spins unattended.
+  const restarted = new Set<string>();
+  const stoppedAgain = (progress: ProjectProgress) => `the target stopped again after restart ${progress.restarts?.length ?? 0} in this resume`;
   const identity = (observed: Inspection, planned: PlannedProject, progress: ProjectProgress) => {
-    if (observed.state === "stopped") throw new Error(progress.phase !== "stopped" ? `${planned.project.id}: the target stopped after it started (a crash or a reboot); restarting it is not supported in this release`
+    if (observed.state === "stopped") throw new Error(restarted.has(progress.id) ? `${planned.project.id}: ${stoppedAgain(progress)}; read its hub.log before resuming again`
+      : progress.phase !== "stopped" ? `${planned.project.id}: the target stopped after it started (a crash or a reboot)${observed.snapshot === id ? "; resume restarts it from this operation's snapshot" : " and this operation's restart snapshot is gone, so nothing can start it"}`
       : `${planned.project.id}: the target is stopped${observed.snapshot === id ? "" : " and this operation's restart snapshot is gone, so nothing can start it"}`);
     if (observed.state === "missing") throw new Error(`${planned.project.id}: the project directory is missing`);
     if (observed.state !== "running") throw new Error(`${planned.project.id}: the target reads as ${observed.state}; wait until it answers`);
@@ -381,6 +457,15 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
     }
     for (let i = 0; i < op.projects.length; i++) {
       const progress = op.projects[i]!, planned = op.plan.projects[i]!, project = planned.project;
+      if (progress.phase === "started" || progress.phase === "peers-restored") {
+        // #225: a target that stopped after it started, with this operation's snapshot, starts again from it.
+        const live = await driver.inspect(project);
+        if (live.state === "stopped") {
+          if (live.snapshot !== id) throw new Error(`${project.id}: the target stopped after it started and this operation's restart snapshot is gone, so nothing can start it`);
+          step(`restart:${project.id}`);
+          recordRestart(progress, driver.now()); restarted.add(progress.id); save();
+        }
+      }
       if (progress.phase === "verified" || progress.phase === "peers-restored") continue;
       if (progress.phase === "pending") {
         step(`prepare:${project.id}`);
@@ -486,7 +571,8 @@ export function publicOperation(op: RecoveryOperation, runnerPid?: number | "unk
     runner: runnerPid === "unknown" ? { state: "unknown" } : runnerPid ? { state: "running", pid: runnerPid } : { state: "none" },
     ...(op.phase === "running" && !runnerPid ? { stale: "the receipt says running but no runner holds the operation: its last runner stopped mid-step" } : {}),
     projects: op.projects.map((p) => ({ id: p.id, phase: p.phase, ...(Object.keys(p.terminals).length ? { effects: Object.fromEntries(Object.entries(p.terminals).map(([key, value]) => [key, effect(value)])) } : {}),
-      ...(p.fresh ? { lostContinuity: Object.fromEntries(Object.entries(p.fresh).map(([peer, f]) => [peer, f.lost])) } : {}) })),
+      ...(p.fresh ? { lostContinuity: Object.fromEntries(Object.entries(p.fresh).map(([peer, f]) => [peer, f.lost])) } : {}),
+      ...(p.restarts?.length ? { restarts: p.restarts.length } : {}) })),
     pluginInstalled: !!op.pluginInstalled, globalInstalled: !!op.globalInstalled,
     ...(!targetReadsWaivers(op) && op.projects.some((p) => failedPeers(op, p).length) ? { freshSession: `not offered: target ${op.plan.version} cannot read recovery waivers, so it could not release a new session` } : {}),
     ...(op.error ? { error: op.error } : {}), ...(op.disposition ? { disposition: op.disposition } : {}), ...(op.audit ? { audit: op.audit } : {}), next };
@@ -507,6 +593,7 @@ export async function abortRecovery(id: string, driver: RecoveryDriver, home = h
       if (recoveryLock(home) === id) releaseRecoveryLock(id, home);
       return;
     }
+    if (abandoned(op)) writeOperation(id, op, home);
     const { fingerprint, ...body } = op.plan;
     if (fingerprint !== planFingerprint(body)) throw new Error("operation plan changed");
     const live = await liveProjects(op, driver.inspect);
@@ -543,6 +630,7 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
       if (recoveryLock(home) === id) releaseRecoveryLock(id, home);
       throw new Error(`operation is already ${op.phase}`);
     }
+    if (abandoned(op)) writeOperation(id, op, home);
     const at = driver.now();
     // Read once, as status and the runner do: every choice and refusal below is decided on it.
     const live = await liveProjects(op, driver.inspect);
@@ -596,6 +684,7 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
     }
     // Each outcome is recorded as it happens, so a disposition that stops partway stays true and resume refuses it.
     const disposition = (op.disposition ??= { choice: "stop-and-archive", at, projects: {} });
+    abandoned(op); // #227: in the same write as the disposition
     (op.audit ??= []).push({ at, action: "stop-and-archive", reason, projects: acts.map((a) => a.planned.project.id) });
     // Recorded before the first act: a crash inside a stop leaves a disposition that resume refuses to override.
     op.updatedAt = driver.now(); writeOperation(id, op, home);