@staix/agent-hub 0.12.20 → 0.12.21

package/src/cli/terminal-recovery.ts

@@ -2,7 +2,7 @@ import { isAbsolute, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { existsSync, mkdirSync, readFileSync } from "node:fs";
 import { randomUUID } from "node:crypto";
-import { processSignature } from "../pi/process-signature.ts";
+import { processLiveness, processSignature } from "../pi/process-signature.ts";
 import { atomicPrivateJSON } from "../hub/recovery-store.ts";
 
 /** The only terminal commands used by this adapter. Keep this list in sync with Orca's public CLI. */
@@ -130,6 +130,12 @@ export interface TerminalRecoveryOptions {
   instanceId?: string;
   /** Injectable process-start identity used to reject stale or reused launcher PIDs. */
   processIdentity?: ProcessIdentity;
+  /**
+   * #228: the identity as releases up to 0.12.12 computed it (unpinned `ps`, the reader's TZ and locale). It defaults
+   * to that read only when `processIdentity` is the default, so an injected identity is never second-guessed by a
+   * real `ps` read of the same pid.
+   */
+  legacyIdentity?: ProcessIdentity;
 }
 
 export interface RecordedTerminalLaunch {
@@ -233,6 +239,7 @@ function normalizeOptions(options?: CommandRunner | TerminalRecoveryOptions): Re
     stateDir: config.stateDir ?? "",
     instanceId: config.instanceId ?? "",
     processIdentity: config.processIdentity ?? defaultProcessIdentity,
+    legacyIdentity: config.legacyIdentity ?? (config.processIdentity ? () => undefined : unpinnedProcessIdentity),
   };
 }
 
@@ -272,22 +279,40 @@ export function recordPath(stateDir: string): string {
   return join(stateDir, RECORD_FILE);
 }
 
-/** #215: "unreadable" (a corrupt or unreadable file) says nothing about launchers, so it must never read as "none". */
-function launchRecords(stateDir: string): RecordedTerminalLaunch[] | "unreadable" {
-  if (!stateDir || !existsSync(recordPath(stateDir))) return [];
+function validRecord(item: unknown): item is RecordedTerminalLaunch {
+  const row = object(item);
+  return typeof row.peer === "string" && typeof row.projectRoot === "string" && typeof row.instanceId === "string" && typeof row.handle === "string" && typeof row.incarnationId === "string" && typeof row.worktreeId === "string" && typeof row.launcherPid === "number" && typeof row.launcherSignature === "string" && row.launcherSignature.length > 0 && typeof row.launchId === "string" && row.launchId.length > 0;
+}
+
+/**
+ * #215: "unreadable" (a corrupt or unreadable file) says nothing about launchers, so it must never read as "none".
+ * #228: a row that fails validation is kept apart (`invalid`), never dropped: whatever it would answer is unknown, and
+ * `all` is the file as read, so a writer puts every such row back unchanged.
+ */
+type LaunchRecords = { rows: RecordedTerminalLaunch[]; invalid: unknown[]; all: unknown[] };
+function launchRecords(stateDir: string): LaunchRecords | "unreadable" {
+  const none = { rows: [], invalid: [], all: [] };
+  if (!stateDir || !existsSync(recordPath(stateDir))) return none;
   try {
     const value: unknown = JSON.parse(readFileSync(recordPath(stateDir), "utf8"));
     if (!Array.isArray(value)) return "unreadable";
-    return value.filter((item): item is RecordedTerminalLaunch => {
-      const row = object(item);
-      return typeof row.peer === "string" && typeof row.projectRoot === "string" && typeof row.instanceId === "string" && typeof row.handle === "string" && typeof row.incarnationId === "string" && typeof row.worktreeId === "string" && typeof row.launcherPid === "number" && typeof row.launcherSignature === "string" && row.launcherSignature.length > 0 && typeof row.launchId === "string" && row.launchId.length > 0;
-    });
+    return { rows: value.filter(validRecord), invalid: value.filter((item) => !validRecord(item)), all: value };
   } catch (error) {
-    return (error as NodeJS.ErrnoException).code === "ENOENT" ? [] : "unreadable";
+    return (error as NodeJS.ErrnoException).code === "ENOENT" ? none : "unreadable";
   }
 }
 
-function writeLaunchRecords(stateDir: string, rows: RecordedTerminalLaunch[]): void {
+/**
+ * #228: whether a row that cannot be evaluated may answer a question about this peer, project and instance: it does
+ * unless one of its fields is a readable string that names another one (a row for another peer or an older instance).
+ */
+function concerns(raw: unknown, query: { peer?: string; projectRoot: string; instanceId: string }): boolean {
+  const row = object(raw);
+  const other = (value: unknown, wanted?: string) => wanted !== undefined && typeof value === "string" && value !== wanted;
+  return !other(row.peer, query.peer) && !other(row.projectRoot, query.projectRoot) && !other(row.instanceId, query.instanceId);
+}
+
+function writeLaunchRecords(stateDir: string, rows: unknown[]): void {
   mkdirSync(stateDir, { recursive: true });
   atomicPrivateJSON(recordPath(stateDir), rows); // temp + rename: a reader never sees half a file
 }
@@ -297,30 +322,50 @@ function writeLaunchRecords(stateDir: string, rows: RecordedTerminalLaunch[]): v
  * process holds it now; unknown when the pid exists but its identity cannot be read. Unknown is never gone.
  */
 export type LauncherState = "live" | "gone" | "unknown";
-async function launcherState(row: RecordedTerminalLaunch, identity: ProcessIdentity): Promise<LauncherState> {
-  let signature: string | undefined;
-  try { signature = await identity(row.launcherPid); } catch { /* unreadable: decided by the pid probe below */ }
-  if (typeof signature === "string" && signature.length > 0) return signature === row.launcherSignature ? "live" : "gone";
-  try { process.kill(row.launcherPid, 0); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return "gone"; }
-  return "unknown";
+async function launcherState(row: RecordedTerminalLaunch, identity: ProcessIdentity, legacy: ProcessIdentity): Promise<LauncherState> {
+  let current: string | undefined;
+  try { current = await identity(row.launcherPid) || undefined; } catch { /* unreadable: decided by the pid probe */ }
+  // #228: a launcher recorded by 0.12.12 or older hashed `ps` in its own TZ and locale (#177 pinned it in 0.12.13). Read
+  // the same unpinned way in this reader's environment, a match is that launcher; anything else stays gone (pid reuse).
+  if (current && current !== row.launcherSignature) {
+    let old: string | undefined;
+    try { old = await legacy(row.launcherPid) || undefined; } catch { /* no legacy reading: the pinned one decides */ }
+    if (old === row.launcherSignature) return "live";
+  }
+  return processLiveness(row.launcherPid, row.launcherSignature, () => current);
 }
 
-/** This instance's launch records whose launcher is live, and those whose launcher cannot be identified (never gone). */
-async function recordsByState(rows: RecordedTerminalLaunch[], projectRoot: string, instanceId: string, identity: ProcessIdentity): Promise<{ live: RecordedTerminalLaunch[]; unknown: RecordedTerminalLaunch[] }> {
-  if (!instanceId) return { live: [], unknown: [] };
-  const candidates = rows.filter((item) => item.projectRoot === projectRoot && item.instanceId === instanceId);
-  const states = await Promise.all(candidates.map((item) => launcherState(item, identity)));
-  return { live: candidates.filter((_, i) => states[i] === "live"), unknown: candidates.filter((_, i) => states[i] === "unknown") };
+/** The launcher signature as releases up to 0.12.12 computed it: `ps` in the caller's own TZ and locale, unpinned. */
+function unpinnedProcessIdentity(pid: number): string | undefined {
+  try {
+    const result = Bun.spawnSync(["ps", "-p", String(pid), "-o", "lstart=,comm="], { stdout: "pipe", stderr: "pipe" });
+    const text = result.exitCode === 0 ? result.stdout.toString().trim() : "";
+    return text ? new Bun.CryptoHasher("sha256").update(text).digest("hex") : undefined;
+  } catch { return undefined; }
+}
+
+/**
+ * This instance's launch records whose launcher is live, those whose launcher cannot be identified (never gone), and
+ * the rows that cannot be evaluated and may be this project's and instance's (#228: unknown for their peer).
+ */
+async function recordsByState(records: LaunchRecords, projectRoot: string, instanceId: string, identity: ProcessIdentity, legacy: ProcessIdentity): Promise<{ live: RecordedTerminalLaunch[]; unknown: RecordedTerminalLaunch[]; invalid: unknown[] }> {
+  if (!instanceId) return { live: [], unknown: [], invalid: [] };
+  const candidates = records.rows.filter((item) => item.projectRoot === projectRoot && item.instanceId === instanceId);
+  const states = await Promise.all(candidates.map((item) => launcherState(item, identity, legacy)));
+  return { live: candidates.filter((_, i) => states[i] === "live"), unknown: candidates.filter((_, i) => states[i] === "unknown"),
+    invalid: records.invalid.filter((raw) => concerns(raw, { projectRoot, instanceId })) };
 }
 
 /** #215: the launcher recorded for `peer` on this instance (one row per peer and instance) and its state; managed means live. */
-export async function launcherOf(peer: TerminalPeer, projectRoot: string, options?: TerminalRecoveryOptions): Promise<{ record?: RecordedTerminalLaunch; state: LauncherState } | undefined> {
+export async function launcherOf(peer: TerminalPeer, projectRoot: string, options?: TerminalRecoveryOptions): Promise<{ record?: RecordedTerminalLaunch; state: LauncherState; invalidRow?: true } | undefined> {
   const config = normalizeOptions(options);
   if (!config.stateDir || !config.instanceId) return undefined;
-  const rows = launchRecords(config.stateDir);
-  if (rows === "unreadable") return { state: "unknown" };
-  const record = rows.find((row) => row.peer === peer && row.projectRoot === projectRoot && row.instanceId === config.instanceId);
-  return record ? { record, state: await launcherState(record, config.processIdentity) } : undefined;
+  const records = launchRecords(config.stateDir);
+  if (records === "unreadable") return { state: "unknown" };
+  // #228: a row that cannot be evaluated and may be this launcher's is unknown, never "no launcher".
+  if (records.invalid.some((raw) => concerns(raw, { peer, projectRoot, instanceId: config.instanceId }))) return { state: "unknown", invalidRow: true };
+  const record = records.rows.find((row) => row.peer === peer && row.projectRoot === projectRoot && row.instanceId === config.instanceId);
+  return record ? { record, state: await launcherState(record, config.processIdentity, config.legacyIdentity) } : undefined;
 }
 
 /**
@@ -353,7 +398,8 @@ export async function recordTerminalLaunch(peer: TerminalPeer, projectRoot: stri
   if (existing === "unreadable") {
     throw new OrcaCommandError(blocker("command-error", `${recordPath(stateDir)} cannot be read, so recording this launch would erase every other launcher's record; next action: inspect that file and move it aside (recovery then treats sessions it recorded as unmanaged), then launch again`, peer, handle));
   }
-  const rows = existing.filter((item) => !(item.peer === peer && item.instanceId === instanceId));
+  // #228: every row that cannot be evaluated goes back unchanged; only this peer's valid row for this instance is replaced.
+  const rows = existing.all.filter((item) => !(validRecord(item) && item.peer === peer && item.instanceId === instanceId));
   rows.push(row);
   writeLaunchRecords(stateDir, rows);
   return row;
@@ -482,7 +528,7 @@ export async function inspectTerminals(projectRoot: string, sessions: Partial<Re
   // CODEX_HOME or CLAUDE_CONFIG_DIR: an unreadable file blocks the binding, never reads as "no records".
   const rows = launchRecords(config.stateDir);
   if (rows === "unreadable") return { bindings, byPeer, blockers: [blocker("command-error", `${recordPath(config.stateDir)} cannot be read, so no terminal can be bound to its launch; inspect or move it aside, then plan or resume again`)], manualRequired: true };
-  const { live: records, unknown } = await recordsByState(rows, projectRoot, config.instanceId, config.processIdentity);
+  const { live: records, unknown, invalid } = await recordsByState(rows, projectRoot, config.instanceId, config.processIdentity, config.legacyIdentity);
 
   let listed: Record<string, unknown>[];
   try {
@@ -500,6 +546,10 @@ export async function inspectTerminals(projectRoot: string, sessions: Partial<Re
       blockers.push(blocker("ownership-unknown", `the ${peer} launcher recorded for terminal ${unread.handle} (pid ${unread.launcherPid}) cannot be identified, so no terminal can be bound to its launch; wait until it exits or end it, then plan or resume again`, peer, unread.handle));
       continue;
     }
+    if (invalid.some((raw) => concerns(raw, { peer, projectRoot, instanceId: config.instanceId }))) {
+      blockers.push(blocker("ownership-unknown", `${recordPath(config.stateDir)} holds a launch record that cannot be evaluated and may be this ${peer} launcher's, so no terminal can be bound to its launch; next action: inspect that row and fix it, or end that launcher and remove its row, then plan or resume again`, peer));
+      continue;
+    }
     const candidates = listed.filter((terminal) => rootMatches(terminal.worktreePath ?? terminal.projectRoot, projectRoot));
     const peerCandidates: Record<string, unknown>[] = [];
     for (const listedTerminal of candidates) {
@@ -639,8 +689,10 @@ export async function createTerminal(binding: TerminalBinding, options?: Command
     const launches = async () => {
       const rows = launchRecords(config.stateDir);
       if (rows === "unreadable") return { live: undefined, gone: false, mayRun: true }; // unknown, never gone
-      const possible = rows.filter((item) => item.instanceId === config.instanceId && item.peer === binding.peer && item.projectRoot === binding.projectRoot && item.handle === createdHandle && item.worktreeId === replacement.worktreeId && item.incarnationId === replacement.incarnationId);
-      const states = await Promise.all(possible.map(async (item) => ({ item, state: await launcherState(item, config.processIdentity) })));
+      // #228: a row that cannot be evaluated and may be this launch's is unknown too.
+      if (rows.invalid.some((raw) => concerns(raw, { peer: binding.peer, projectRoot: binding.projectRoot, instanceId: config.instanceId }))) return { live: undefined, gone: false, mayRun: true };
+      const possible = rows.rows.filter((item) => item.instanceId === config.instanceId && item.peer === binding.peer && item.projectRoot === binding.projectRoot && item.handle === createdHandle && item.worktreeId === replacement.worktreeId && item.incarnationId === replacement.incarnationId);
+      const states = await Promise.all(possible.map(async (item) => ({ item, state: await launcherState(item, config.processIdentity, config.legacyIdentity) })));
       return { live: states.find((entry) => entry.state === "live")?.item, gone: states.length > 0 && states.every((entry) => entry.state === "gone"), mayRun: states.some((entry) => entry.state !== "gone") };
     };
     // #215: Orca types the command into a login shell that outlives it, so the terminal never exits with the launcher.

package/src/cli/upgrade-runtime.ts

@@ -92,7 +92,11 @@ async function rpc(project: Project, message: Record<string, unknown>, protocol
 }
 
 export async function inspectRecovery(project: Project): Promise<Inspection> {
-  const base = await inspectProject(project);
+  // #226: the claim is judged by the registry as it is now, never by the plan's frozen copy, which holds the source's
+  // pid from planning time (after a reboot another process may hold it). A registry that cannot be read throws: the
+  // caller reads that as not inspected, never as stopped.
+  const claim = registeredProjects().find((p) => p.id === project.id);
+  const base = await inspectProject({ ...project, instanceId: claim?.instanceId ?? null, pid: claim?.pid ?? null, pidSignature: claim?.pidSignature ?? null });
   const control = readControl(project.stateDir);
   const sourceProtocol = control?.protocol;
   const legacySupported = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]);
@@ -175,7 +179,7 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
         if (recorded?.state === "unknown") {
           // An unreadable launcher is never gone (#215): whether the session is managed cannot be told.
           delete peer.sessionId;
-          blockers.push(`claude: ${recorded.record ? `the launcher recorded in terminal ${recorded.record.handle}` : "the launcher record (terminal-recovery.json)"} cannot be read, so whether the attached session is managed is unknown; manual-required; next action: make it readable (or end that session and close its terminal), then make a new plan`);
+          blockers.push(`claude: ${recorded.record ? `the launcher recorded in terminal ${recorded.record.handle}` : recorded.invalidRow ? "a launch record in terminal-recovery.json that may be its launcher's" : "the launcher record (terminal-recovery.json)"} cannot be read, so whether the attached session is managed is unknown; manual-required; next action: make it readable (or end that session and close its terminal), then make a new plan`);
           continue;
         }
         const launcher = recorded?.state === "live" ? recorded.record! : undefined;
@@ -305,7 +309,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
    * expected instance can report "gone"; an unreadable target or launcher is unknown and blocks, never gone.
    */
   const peerEvidence = async (planned: PlannedProject, progress: ProjectProgress, peer: TerminalBinding["peer"]):
-      Promise<{ state: "live"; session?: string; handle?: string } | { state: "gone" } | { state: "unknown"; why: string; step?: string }> => {
+      Promise<{ state: "live"; session?: string; handle?: string; attached?: true } | { state: "gone" } | { state: "unknown"; why: string; step?: string }> => {
     const target = await inspectRecovery(planned.project);
     if (target.state !== "running" || target.instanceId !== progress.instanceId) {
       // Only a hub that may still answer is waited for; a stopped, missing or other one is settled by the runner's list.
@@ -315,14 +319,44 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
     const attached = target.peers.find((p) => p.id === peer && p.state !== "offline");
     const session = peer === "codex" ? attached?.threadId : attached?.sessionId;
     if (session) return { state: "live", session };
-    const launch = await launcherOf(peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId });
-    if (launch?.state === "live") return { state: "live", handle: launch.record!.handle };
-    if (launch?.state === "unknown") {
-      return launch.record ? { state: "unknown", why: `its launcher in terminal ${launch.record.handle} cannot be read`, step: `wait until it attaches, or end it and close terminal ${launch.record.handle}` }
-        : { state: "unknown", why: "the launcher records cannot be read", step: `inspect ${recordPath(planned.project.stateDir)} and move it aside (sessions it recorded then count as unmanaged)` };
+    // #225: the launch records of every instance this operation started for the project, the current one first: a
+    // launcher a dead instance recorded may still run (read before the check below, so a block names its terminal).
+    for (const instanceId of [progress.instanceId, ...(progress.restarts ?? []).map((r) => r.instanceId)]) {
+      if (!instanceId) continue;
+      const launch = await launcherOf(peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId });
+      if (launch?.state === "live") return { state: "live", handle: launch.record!.handle, ...(attached ? { attached: true as const } : {}) };
+      if (launch?.state === "unknown") {
+        return launch.record ? { state: "unknown", why: `its launcher in terminal ${launch.record.handle} cannot be read`, step: `wait until it attaches, or end it and close terminal ${launch.record.handle}` }
+          : launch.invalidRow ? { state: "unknown", why: "a launch record that may be its launcher's cannot be evaluated", step: `inspect ${recordPath(planned.project.stateDir)} and fix that row, or end that launcher and remove its row` }
+          : { state: "unknown", why: "the launcher records cannot be read", step: `inspect ${recordPath(planned.project.stateDir)} and move it aside (sessions it recorded then count as unmanaged)` };
+      }
     }
+    // #225: attached with no provable session id (a Claude of a dead instance whose plugin reconnected, or one that has
+    // not reported its id yet) is a session of unknown identity, never "nothing attached".
+    if (attached) return { state: "unknown", why: `a ${peer} session is attached to the target without a session id`, step: `wait until it reports one, or end that ${peer} session` };
     return { state: "gone" };
   };
+  /**
+   * #225: the terminal of a done restoration of a target instance that died is closed under the close receipt rules
+   * (`closedRetired:<peer>` `pending` until Orca no longer lists it; an unreadable or truncated inventory is unknown and
+   * changes nothing) or found absent, before its peer is relaunched against the new instance.
+   */
+  const closeRetired = async (progress: ProjectProgress, peer: string, save: () => void): Promise<void> => {
+    const retired = progress.terminals[`retired:${peer}`] as TerminalBinding | undefined;
+    const key = `closedRetired:${peer}`;
+    if (!retired || typeof retired !== "object" || progress.terminals[key] === true) return;
+    const seen = await listed(retired);
+    if (progress.terminals[key] === "pending" || seen !== "listed") {
+      if (seen !== "absent") throw new Error(closeUnsettled(retired, seen));
+      progress.terminals[key] = true; save(); return;
+    }
+    const idle = await waitForIdle(retired, 120_000, terminalOptions(run));
+    if (!idle.satisfied) throw new Error(`${peer}: terminal ${retired.handle} of the stopped target's session is not verified idle; finish or cancel its turn, or close that terminal by hand, first`);
+    progress.terminals[key] = "pending"; save();
+    const result = await closeTerminal(retired, 0, terminalOptions(run));
+    if (result.manualRequired) throw new Error(`${peer}: closing terminal ${retired.handle} of the stopped target's session was not verified; manual-required`);
+    progress.terminals[key] = true; save();
+  };
   const attachedId = async (planned: PlannedProject, peer: string, op: RecoveryOperation): Promise<string> => {
     for (const deadline = now() + RECONNECT_WAIT_MS; ;) {
       const current = (await inspectRecovery(planned.project)).peers.find((p) => p.id === peer && p.state !== "offline");
@@ -423,17 +457,30 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
           await command(op, args, planned.project);
         }
       }
-      for (const original of planned.terminals as TerminalBinding[]) {
-        if ((original.peer === "claude") !== (group === "claude")) continue;
-        const key = `restored:${original.peer}`;
+      for (const plannedBinding of planned.terminals as TerminalBinding[]) {
+        if ((plannedBinding.peer === "claude") !== (group === "claude")) continue;
+        const key = `restored:${plannedBinding.peer}`;
+        // #225: after a restart, the stopped target's terminal for this peer goes first; a session it had accepted
+        // instead of the planned one (fresh choice, fresh start, zero-turn Claude) is relaunched by that id, which the
+        // operation-fenced waiver already covers.
+        await closeRetired(progress, plannedBinding.peer, save);
+        const retired = progress.terminals[`retired:${plannedBinding.peer}`] as TerminalBinding | undefined;
+        const accepted = retired && typeof retired === "object" && retired.sessionId !== plannedBinding.sessionId ? retired.sessionId : undefined;
+        const original = accepted ? { ...plannedBinding, sessionId: accepted } : plannedBinding;
         // A new session is accepted only by the operator's choice (#215 dispose --fresh-session) or a planned fresh
         // start of a Codex thread with nothing to lose that still has no rollout; an unreadable store is neither.
         const chosen = !!progress.fresh?.[original.peer];
         let read: ReturnType<typeof codexTranscript> | undefined;
-        const transcript = () => original.peer !== "codex" || chosen ? "found" : (read ??= codexTranscript(original));
-        const fresh = () => chosen || (!!planned.freshStart?.includes(original.peer) && transcript() === "missing");
+        const transcript = () => original.peer !== "codex" || (chosen && !accepted) ? "found" : (read ??= codexTranscript(original));
+        // An accepted session starts new again only when it has nothing to resume: a Codex thread that never got a
+        // rollout, or a Claude session with no transcript (`--resume` could never succeed, #21).
+        const fresh = () => accepted
+          ? (original.peer === "codex" ? (chosen || !!planned.freshStart?.includes("codex")) && transcript() === "missing" : original.peer === "claude" && !claudeTranscriptExists(original))
+          : chosen || (!!planned.freshStart?.includes(original.peer) && transcript() === "missing");
         // The runner ends this with the choices `status` shows (nextActions); resume comes first and launches a failed
         // peer again, which helps once its cause (a transient Orca create failure, a fixed store) is gone.
+        // The waiver's audit label: the operator's choice, a Claude with nothing to resume, or the plan's Codex fresh start.
+        const waiver = () => progress.fresh?.[original.peer] ? "fresh-session" : original.peer === "claude" ? "zero-turn" : "fresh-start";
         const notRestored = (why: string) => new Error(fresh()
           ? `${original.peer}: ${why}; read that terminal in Orca for the launcher's error and fix its cause`
           : `${original.peer}: ${why}; session ${original.sessionId} was not restored; resume launches it again once the cause is fixed`);
@@ -442,7 +489,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
           ? `${original.peer}: whether a ${original.peer} session or launcher is live cannot be told (${evidence.why}); nothing was recorded or created${evidence.step ? `; ${evidence.step} first` : ""}`
           : evidence.session
             ? `${original.peer}: session ${evidence.session} is attached instead of ${original.sessionId}; no terminal was created; end that ${original.peer} session and close its terminal first`
-            : `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`);
+            : `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session ${evidence.attached ? "with an id " : ""}attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`);
         const receipt = progress.terminals[key];
         if (receipt && receipt !== "pending" && receipt !== "failed") {
           progress.terminals[key] = await revalidateTerminal(planned, progress, receipt as TerminalBinding, true); save();
@@ -456,14 +503,16 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
           if (attached !== original.sessionId && !fresh() && !(original.peer === "claude" && !claudeTranscriptExists(original))) {
             throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${attached} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
           }
-          // The original came back after all: a fresh session chosen for it lost nothing.
-          if (attached === original.sessionId && progress.fresh?.[original.peer]) {
+          // The planned session came back after all: a fresh session chosen for it lost nothing. Compared with the plan,
+          // never with a session accepted after a restart (#225), whose choice must stay on record.
+          if (attached === plannedBinding.sessionId && progress.fresh?.[original.peer]) {
             delete progress.fresh[original.peer];
             if (!Object.keys(progress.fresh).length) delete progress.fresh;
           }
           // Any other accepted session needs the target to waive the saved id, or release would wait for it forever;
           // a launch writes the waiver first, but this must not rely on that.
-          if (attached !== original.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: progress.fresh?.[original.peer] ? "fresh-session" : fresh() ? "fresh-start" : "zero-turn" });
+          // The planned session needs no waiver: the target's snapshot already expects it.
+          if (attached !== original.sessionId && attached !== plannedBinding.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
           progress.terminals[key] = await revalidateTerminal(planned, progress, { ...original, sessionId: attached }, false); save();
         };
         // #215: a pending or failed receipt is settled by what is live now (the receipt table in the recovery spec).
@@ -494,7 +543,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
         if (fresh()) {
           if (!targetReadsWaivers(op)) throw new Error(`${original.peer}: target ${op.plan.version} predates recovery waivers, so it cannot accept a new session`);
           // The restored daemon must accept the new session instead of the recorded one.
-          waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: progress.fresh?.[original.peer] ? "fresh-session" : "fresh-start" });
+          waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
         }
         const entrypoint = join(op.targetRoot!, "src/cli/main.js");
         const argv = restoredTerminalArgv(entrypoint, planned.project.root, original, fresh());

package/src/cli/upgrade.ts

@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite";
 import { createHash, randomUUID } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
-import type { Project } from "../hub/registry.ts";
+import { PROJECT_COLUMNS, projectFromRow, type Project } from "../hub/registry.ts";
 import { hubHome } from "../hub/project.ts";
 import { acquireRecoveryLock, claimRunner, coordinatorCurrent, readOperation, recoveryCommand, recoveryLock, releaseRecoveryLock, writeOperation } from "../hub/recovery-store.ts";
 
@@ -56,9 +56,18 @@ export interface ProjectProgress {
   fresh?: Record<string, { lost: string; reason: string; at: number }>;
   /** #215: set before the commit request is sent; from then on the source may have committed (abort is not offered). */
   commitSent?: boolean;
+  /**
+   * #225: each restart of a target that stopped after it started, recorded before the new one starts: the dead
+   * instance and when. Status and audit read it; restore reads the launch records of every instance listed here.
+   */
+  restarts?: { instanceId: string; at: number }[];
 }
 export interface RecoveryOperation {
-  schema: 1;
+  /**
+   * #227: 2 marks a stop-and-archive recorded on an operation whose coordinator predates #215. Every older runner and
+   * abort refuses a schema other than 1 before it takes the lock or claims the runner, so none can advance it.
+   */
+  schema: 1 | 2;
   id: string;
   plan: UpgradePlan;
   createdAt: number;
@@ -94,6 +103,37 @@ export function targetReadsWaivers(op: { targetRoot?: string }): boolean {
   try { return !!op.targetRoot && readFileSync(join(op.targetRoot, "src/hub/restart.ts"), "utf8").includes("export function readRecoveryWaivers"); } catch { return false; }
 }
 
+/**
+ * #225: whether an operation's coordinator restarts a target that stopped after it started. Resume runs the operation's
+ * own runner, so an older one keeps refusing and `next` must not offer resume for that row.
+ * ponytail: a text sniff of its upgrade.ts, like targetReadsWaivers and coordinatorCurrent; a rename reads as "cannot
+ * restart" (the safe side). Upgrade path: the capability list those two name.
+ */
+export function coordinatorRestarts(op: { sourceRoot?: string }): boolean {
+  if (!op.sourceRoot) return true;
+  try { return readFileSync(join(op.sourceRoot, "src/cli/upgrade.ts"), "utf8").includes("export function recordRestart"); } catch { return false; }
+}
+
+/**
+ * #225: record the restart of a target that stopped after it started, before the new one starts: the dead instance and
+ * the time stay for status and audit. Each done `restored:<peer>` binding of the dead instance is retired
+ * (`retired:<peer>`): restore closes its terminal (or finds it absent) before that peer is relaunched, and never
+ * revalidates it against the new instance. `closed:<peer>`, `commitSent`, `fresh`, waivers and the install flags stay.
+ * The project goes back to `stopped`, whose path starts the target from this operation's snapshot.
+ */
+export function recordRestart(progress: ProjectProgress, at: number): void {
+  (progress.restarts ??= []).push({ instanceId: progress.instanceId ?? "unknown", at });
+  for (const [key, value] of Object.entries(progress.terminals)) {
+    if (!key.startsWith("restored:") || !value || typeof value !== "object") continue;
+    const peer = key.slice("restored:".length);
+    progress.terminals[`retired:${peer}`] = value;
+    delete progress.terminals[`closedRetired:${peer}`]; // a newer binding to close: its own receipt starts over
+    delete progress.terminals[key];
+  }
+  delete progress.instanceId;
+  progress.phase = "stopped";
+}
+
 /** Effects anywhere in the operation: a project past `prepared` or any terminal receipt. Abort needs none. */
 export const hasEffects = (op: RecoveryOperation) => op.projects.some((p) => !["pending", "prepared"].includes(p.phase) || Object.keys(p.terminals).length > 0);
 /**
@@ -144,9 +184,9 @@ export function resumeBlocked(op: RecoveryOperation, live: Record<string, Inspec
     if (state.state !== "stopped") continue;
     if (p.phase === "pending") return `${p.id}: the source stopped before it was prepared`;
     if (p.phase === "prepared" && !p.commitSent && !old) return `${p.id}: the source stopped while prepared, with no commit request`;
-    // ponytail: a target that dies after it started is not restarted; that needs its peers relaunched against the new
-    // instance (CodexPeer is created only by start, launcher records are keyed by instance). Follow-up issue.
-    if (p.phase === "started" || p.phase === "peers-restored") return `${p.id}: the target stopped after it started; restarting it is not supported in this release`;
+    // #225: a target that stopped after it started is restarted from this operation's snapshot (the line below), by a
+    // coordinator that can; an older one's runner refuses it.
+    if ((p.phase === "started" || p.phase === "peers-restored") && !coordinatorRestarts(op)) return `${p.id}: the target stopped after it started; this operation's coordinator cannot restart it`;
     if (state.snapshot !== op.id) return `${p.id}: its hub stopped without this operation's restart snapshot`;
   }
   return undefined;
@@ -231,7 +271,9 @@ export function registeredProjects(home = hubHome()): Project[] {
   if (!existsSync(path)) return [];
   const db = new Database(path, { readonly: true });
   try {
-    return db.query("SELECT id, root, state_dir as stateDir, base_port as basePort, instance_id as instanceId, pid FROM projects ORDER BY root").all() as Project[];
+    // #226: a registry no current release opened yet has no signature column, and this read never migrates.
+    const signed = (db.query("PRAGMA table_info(projects)").all() as { name: string }[]).some((c) => c.name === "pid_signature");
+    return (db.query(`SELECT ${PROJECT_COLUMNS}${signed ? ", pid_signature" : ""} FROM projects ORDER BY root`).all() as Record<string, unknown>[]).map(projectFromRow);
   } finally { db.close(); }
 }
 
@@ -256,8 +298,21 @@ export function createOperation(plan: UpgradePlan, sourceRoot: string, home = hu
   return op;
 }
 
+/**
+ * #227: a coordinator from before #215 never reads `disposition`, and an older global `ahub recovery resume` runs it.
+ * Every older validateReceipt refuses a schema other than 1 before the lock and the runner claim, so schema 2 keeps it
+ * from advancing an operation being abandoned. A #215 coordinator's runner refuses the disposition itself and keeps
+ * schema 1, so a 0.12.20 CLI can still rerun its dispose. True when the receipt changed (a disposition recorded at 1
+ * by 0.12.20 is moved to 2 by the next command of this release that holds the runner claim).
+ */
+function abandoned(op: RecoveryOperation): boolean {
+  if (!op.disposition || op.schema !== 1 || !op.sourceRoot || coordinatorCurrent(op.sourceRoot)) return false;
+  op.schema = 2;
+  return true;
+}
+
 function validateReceipt(id: string, op: RecoveryOperation): void {
-  if (op.schema !== 1 || op.id !== id) throw new Error("unsupported operation receipt");
+  if (!(op.schema === 1 || (op.schema === 2 && op.disposition)) || op.id !== id) throw new Error("unsupported operation receipt");
   const { fingerprint, ...reviewed } = op.plan;
   if (fingerprint !== planFingerprint(reviewed) || op.projects.length !== op.plan.projects.length ||
       op.projects.some((p, i) => p.id !== op.plan.projects[i]?.project.id)) throw new Error("reviewed operation plan or project scope changed");
@@ -302,8 +357,12 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
   } catch (error) { releaseRunner(); throw error; }
   const save = () => { op.updatedAt = driver.now(); writeOperation(id, op, home); };
   const step = (value: string) => { op.step = value; save(); };
+  // #225: one restart per project per resume, so a target that keeps dying never spins unattended.
+  const restarted = new Set<string>();
+  const stoppedAgain = (progress: ProjectProgress) => `the target stopped again after restart ${progress.restarts?.length ?? 0} in this resume`;
   const identity = (observed: Inspection, planned: PlannedProject, progress: ProjectProgress) => {
-    if (observed.state === "stopped") throw new Error(progress.phase !== "stopped" ? `${planned.project.id}: the target stopped after it started (a crash or a reboot); restarting it is not supported in this release`
+    if (observed.state === "stopped") throw new Error(restarted.has(progress.id) ? `${planned.project.id}: ${stoppedAgain(progress)}; read its hub.log before resuming again`
+      : progress.phase !== "stopped" ? `${planned.project.id}: the target stopped after it started (a crash or a reboot)${observed.snapshot === id ? "; resume restarts it from this operation's snapshot" : " and this operation's restart snapshot is gone, so nothing can start it"}`
       : `${planned.project.id}: the target is stopped${observed.snapshot === id ? "" : " and this operation's restart snapshot is gone, so nothing can start it"}`);
     if (observed.state === "missing") throw new Error(`${planned.project.id}: the project directory is missing`);
     if (observed.state !== "running") throw new Error(`${planned.project.id}: the target reads as ${observed.state}; wait until it answers`);
@@ -381,6 +440,15 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
     }
     for (let i = 0; i < op.projects.length; i++) {
       const progress = op.projects[i]!, planned = op.plan.projects[i]!, project = planned.project;
+      if (progress.phase === "started" || progress.phase === "peers-restored") {
+        // #225: a target that stopped after it started, with this operation's snapshot, starts again from it.
+        const live = await driver.inspect(project);
+        if (live.state === "stopped") {
+          if (live.snapshot !== id) throw new Error(`${project.id}: the target stopped after it started and this operation's restart snapshot is gone, so nothing can start it`);
+          step(`restart:${project.id}`);
+          recordRestart(progress, driver.now()); restarted.add(progress.id); save();
+        }
+      }
       if (progress.phase === "verified" || progress.phase === "peers-restored") continue;
       if (progress.phase === "pending") {
         step(`prepare:${project.id}`);
@@ -486,7 +554,8 @@ export function publicOperation(op: RecoveryOperation, runnerPid?: number | "unk
     runner: runnerPid === "unknown" ? { state: "unknown" } : runnerPid ? { state: "running", pid: runnerPid } : { state: "none" },
     ...(op.phase === "running" && !runnerPid ? { stale: "the receipt says running but no runner holds the operation: its last runner stopped mid-step" } : {}),
     projects: op.projects.map((p) => ({ id: p.id, phase: p.phase, ...(Object.keys(p.terminals).length ? { effects: Object.fromEntries(Object.entries(p.terminals).map(([key, value]) => [key, effect(value)])) } : {}),
-      ...(p.fresh ? { lostContinuity: Object.fromEntries(Object.entries(p.fresh).map(([peer, f]) => [peer, f.lost])) } : {}) })),
+      ...(p.fresh ? { lostContinuity: Object.fromEntries(Object.entries(p.fresh).map(([peer, f]) => [peer, f.lost])) } : {}),
+      ...(p.restarts?.length ? { restarts: p.restarts.length } : {}) })),
     pluginInstalled: !!op.pluginInstalled, globalInstalled: !!op.globalInstalled,
     ...(!targetReadsWaivers(op) && op.projects.some((p) => failedPeers(op, p).length) ? { freshSession: `not offered: target ${op.plan.version} cannot read recovery waivers, so it could not release a new session` } : {}),
     ...(op.error ? { error: op.error } : {}), ...(op.disposition ? { disposition: op.disposition } : {}), ...(op.audit ? { audit: op.audit } : {}), next };
@@ -507,6 +576,7 @@ export async function abortRecovery(id: string, driver: RecoveryDriver, home = h
       if (recoveryLock(home) === id) releaseRecoveryLock(id, home);
       return;
     }
+    if (abandoned(op)) writeOperation(id, op, home);
     const { fingerprint, ...body } = op.plan;
     if (fingerprint !== planFingerprint(body)) throw new Error("operation plan changed");
     const live = await liveProjects(op, driver.inspect);
@@ -543,6 +613,7 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
       if (recoveryLock(home) === id) releaseRecoveryLock(id, home);
       throw new Error(`operation is already ${op.phase}`);
     }
+    if (abandoned(op)) writeOperation(id, op, home);
     const at = driver.now();
     // Read once, as status and the runner do: every choice and refusal below is decided on it.
     const live = await liveProjects(op, driver.inspect);
@@ -596,6 +667,7 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
     }
     // Each outcome is recorded as it happens, so a disposition that stops partway stays true and resume refuses it.
     const disposition = (op.disposition ??= { choice: "stop-and-archive", at, projects: {} });
+    abandoned(op); // #227: in the same write as the disposition
     (op.audit ??= []).push({ at, action: "stop-and-archive", reason, projects: acts.map((a) => a.planned.project.id) });
     // Recorded before the first act: a crash inside a stop leaves a disposition that resume refuses to override.
     op.updatedAt = driver.now(); writeOperation(id, op, home);