@staix/agent-hub 0.12.30 → 0.12.32

package/src/cli/upgrade.ts

@@ -1,3 +1,6 @@
+import { LIVE_TEST_PERSON_REFUSAL } from "./identity.ts";
+import { checkedSessionPath } from "./recovery-session-file.ts";
+import { canChooseFresh, decodeRestoreReceipt, restoreReceiptStatus } from "./upgrade-restore-state.ts";
 import { databaseBusySite, isDatabaseBusy, RecoveryDatabaseBusy } from "../hub/recovery-error.ts";
 import { Database } from "bun:sqlite";
 import { createHash, randomUUID } from "node:crypto";
@@ -6,6 +9,9 @@ import { join } from "node:path";
 import { REGISTRY_DB_BUSY_TIMEOUT_MS, PROJECT_COLUMNS, projectFromRow, type Project } from "../hub/registry.ts";
 import { hubHome } from "../hub/project.ts";
 import { acquireRecoveryLock, claimRunner, coordinatorCurrent, readOperation, recoveryCommand, recoveryLock, releaseRecoveryLock, writeOperation } from "../hub/recovery-store.ts";
+import { resolveAcceptedBinding, piArchiveExit, recordedPeerGuidance, type RecoveryNativePeer } from "./recovery-guidance.ts";
+import type { TerminalBinding } from "./terminal-recovery.ts";
+import { RecoveryPresentationError, type RecoveryPresentationCause, type RecoveryWaitTiming } from "./upgrade-presentation-data.ts";
 
 export interface RecoveryPeer {
   id: string;
@@ -53,6 +59,10 @@ export interface ProjectProgress {
   phase: "pending" | "prepared" | "stopped" | "started" | "peers-restored" | "verified";
   instanceId?: string;
   terminals: Record<string, unknown>;
+  /** One durable repair allowance per peer, retained after an attempt to prevent a second relaunch. */
+  reattachments?: Partial<Record<RecoveryNativePeer, ReattachmentMarker>>;
+  /** Recorded-session-only continuity, independent of a target incarnation's repair attempt allowance. */
+  repairedSessions?: Partial<Record<RecoveryNativePeer, true>>;
   /** #414: last bounded native-attachment timeout, metadata only; never terminal output or task text. */
   attachmentTimeouts?: Record<string, {
     at: number;
@@ -63,7 +73,7 @@ export interface ProjectProgress {
   }>;
   releaseRequested?: boolean;
   /** #215: peers the operator moved to a fresh session, with the session or thread id whose continuity is lost. */
-  fresh?: Record<string, { lost: string; reason: string; at: number }>;
+  fresh?: Record<string, { lost: string; reason: string; at: number; repaired?: true }>;
   /** #215: set before the commit request is sent; from then on the source may have committed (abort is not offered). */
   commitSent?: boolean;
   /**
@@ -72,6 +82,12 @@ export interface ProjectProgress {
    */
   restarts?: { instanceId: string; at: number }[];
 }
+export interface ReattachmentMarker {
+  state: "ready" | "attempted";
+  at: number;
+  cause: "detached after restore";
+  binding: TerminalBinding;
+}
 export interface RecoveryOperation {
   /**
    * #227: 2 marks a stop-and-archive recorded on an operation whose coordinator predates #215. Every older runner and
@@ -87,14 +103,24 @@ export interface RecoveryOperation {
   sourceRoot: string;
   targetRoot?: string;
   targetDigest?: string;
+  /** Verified capability evidence bound to targetRoot/targetDigest; absence is legacy unknown, not an empty list. */
+  targetFeatures?: readonly string[];
   projects: ProjectProgress[];
   pluginInstalled?: boolean;
   globalInstalled?: boolean;
   error?: string;
+  /** Presentation observations only: old receipts without these fields have unknown timings and cause. */
+  waits?: Record<string, RecoveryWaitTiming>;
+  presentationCause?: RecoveryPresentationCause;
   failure?: { code: "database_busy"; retryable: true; site?: string[] };
+  /** A typed verify outcome, never inferred from an error message or a peer's current state. */
+  blockedReattachment?: { projectId: string; peer: RecoveryNativePeer };
+  /** Positive typed store observation; disposal rechecks it before accepting a repaired-session loss. */
+  missingRecordedSession?: { projectId: string; peer: RecoveryNativePeer; sessionId: string; kind: "rollout" | "transcript"; checkedPath?: string };
+  freshOverride?: { peer: RecoveryNativePeer; evidence: NonNullable<RecoveryOperation["missingRecordedSession"]>; checked: "missing"; at: number };
   /** #215: every human disposition, in order. Operator reasons only; never task or message text. */
   audit?: { at: number; action: "fresh-session" | "stop-and-archive"; reason: string; peer?: string; projects: string[] }[];
-  disposition?: { choice: "stop-and-archive"; at: number; projects: Record<string, string> };
+  disposition?: { choice: "stop-and-archive"; at: number; projects: Record<string, string>; continuity?: Record<string, "not-archived" | "unavailable"> };
   /** #215: a refusal no resume can get past was recorded (FinalRefusal); `next` no longer offers resume. */
   final?: boolean;
 }
@@ -102,6 +128,16 @@ export interface RecoveryOperation {
 /** #215: a refusal that holds for every resume of the operation (its staged target or preserved source is fixed). */
 export class FinalRefusal extends Error {}
 
+export class ReattachmentRequired extends Error {
+  constructor(public readonly projectId: string, public readonly peer: RecoveryNativePeer, message: string, public readonly presentationError?: RecoveryPresentationError) { super(message); }
+}
+
+export class MissingRecordedSession extends Error {
+  readonly code = "recorded_session_missing";
+  constructor(readonly evidence: NonNullable<RecoveryOperation["missingRecordedSession"]>) {
+    super(`${evidence.peer}: recorded session ${evidence.sessionId} has no resumable transcript (positively missing ${evidence.kind}${evidence.checkedPath ? ` checked at ${JSON.stringify(evidence.checkedPath)}` : ""}); choose dispose --fresh-session ${evidence.peer} --reason <text> to accept its loss, or stop-and-archive`);
+  }
+}
 export { recoveryCommand };
 const STOP = "--stop-and-archive --reason <text>";
 /**
@@ -127,7 +163,7 @@ export function coordinatorRestarts(op: { sourceRoot?: string }): boolean {
 
 /**
  * #225: record the restart of a target that stopped after it started, before the new one starts: the dead instance and
- * the time stay for status and audit. Each done `restored:<peer>` binding of the dead instance is retired
+ * the time stay for status and audit. Each done or waiting `restored:<peer>` binding of the dead instance is retired
  * (`retired:<peer>`): restore closes its terminal (or finds it absent) before that peer is relaunched, and never
  * revalidates it against the new instance. `closed:<peer>`, `commitSent`, `fresh`, waivers and the install flags stay.
  * The project goes back to `stopped`, whose path starts the target from this operation's snapshot.
@@ -135,12 +171,19 @@ export function coordinatorRestarts(op: { sourceRoot?: string }): boolean {
 export function recordRestart(progress: ProjectProgress, at: number): void {
   (progress.restarts ??= []).push({ instanceId: progress.instanceId ?? "unknown", at });
   for (const [key, value] of Object.entries(progress.terminals)) {
-    if (!key.startsWith("restored:") || !value || typeof value !== "object") continue;
+    if (!key.startsWith("restored:")) continue;
+    const receipt = decodeRestoreReceipt(value);
+    if (receipt?.state !== "done" && receipt?.state !== "waiting") continue;
     const peer = key.slice("restored:".length);
-    progress.terminals[`retired:${peer}`] = value;
+    progress.terminals[`retired:${peer}`] = receipt.binding;
     delete progress.terminals[`closedRetired:${peer}`]; // a newer binding to close: its own receipt starts over
     delete progress.terminals[key];
+    // Older receipts may have only the attempt marker. Retiring that binding must not retire its continuity rule.
+    if (Object.hasOwn(progress.reattachments ?? {}, peer)) (progress.repairedSessions ??= {})[peer as RecoveryNativePeer] = true;
+    // The allowance belongs to the target whose exact binding was retired, not every later #225 restart.
+    if (progress.reattachments) delete progress.reattachments[peer as RecoveryNativePeer];
   }
+  if (progress.reattachments && !Object.keys(progress.reattachments).length) delete progress.reattachments;
   delete progress.instanceId;
   progress.phase = "stopped";
 }
@@ -215,7 +258,7 @@ export function disposeRefusal(op: RecoveryOperation, live: Record<string, Inspe
 
 /** #272: one entry of `next`. `text` is the line status prints; a screen runs `kind` through `recoveryArgv`. */
 export interface NextChoice {
-  kind: "resume" | "abort" | "fresh" | "stop" | "wait";
+  kind: "resume" | "abort" | "fresh" | "stop" | "wait" | "reattach";
   /** `fresh` only: the peer that starts as a new session. */
   peer?: string;
   text: string;
@@ -240,9 +283,13 @@ export function nextChoices(receipt: RecoveryOperation, runner?: number | "unkno
   const old = !!op.sourceRoot && !coordinatorCurrent(op.sourceRoot);
   const abortable = !abortRefusal(op, live);
   const stuck = resumeBlocked(op, live);
+  const selected = op.phase === "blocked" ? op.blockedReattachment : undefined;
+  const repair = selected && op.projects.find(project => project.id === selected.projectId)?.reattachments?.[selected.peer];
+  const reattach = selected && repair && (repair.state === "ready" || repairWaiting(op, selected.projectId, selected.peer));
   return [
     ...(stuck ? [] : [{ kind: "resume" as const, text: `${recoveryCommand(op, "resume")}${op.error ? " (after the next action in error)" : ""}${old ? ` (runs the coordinator that started this operation, which cannot re-prepare an expired hold: if it reports "source is no longer prepared", use ${abortable ? "abort or " : ""}stop-and-archive)` : ""}` }]),
     ...(abortable ? [{ kind: "abort" as const, text: recoveryCommand(op, "abort") }] : []),
+    ...(selected && reattach ? [{ kind: "reattach" as const, peer: selected.peer, text: recoveryCommand(op, "reattach", selected.peer) }] : []),
     ...(targetReadsWaivers(op) && !stuck ? failed.map((peer) => ({ kind: "fresh" as const, peer, text: recoveryCommand(op, "dispose", `--fresh-session ${peer} --reason <text>`) })) : []),
     stop,
   ];
@@ -273,6 +320,13 @@ export async function liveProjects(op: RecoveryOperation, inspect: (project: Pro
   return Object.fromEntries(await Promise.all(open.map(async (p) => [p.project.id, await inspect(p.project).catch(() => undefined)] as const)));
 }
 
+function missingRepair(op: RecoveryOperation, progress: ProjectProgress, peer: string): boolean {
+  const evidence = op.missingRecordedSession;
+  if (!evidence || evidence.projectId !== progress.id || evidence.peer !== peer || (!progress.repairedSessions?.[evidence.peer] && !Object.hasOwn(progress.reattachments ?? {}, peer))) return false;
+  const planned = op.plan.projects.find(project => project.project.id === progress.id);
+  if (!planned) return false;
+  try { return resolveAcceptedBinding(planned, progress, evidence.peer)?.sessionId === evidence.sessionId; } catch { return false; }
+}
 /**
  * Peers of a project whose restoration failed and for which a fresh session is a real choice: not Pi (a restored hub
  * resumes its recorded session), not one already chosen, and not a planned fresh start (nothing to lose; resume
@@ -280,14 +334,32 @@ export async function liveProjects(op: RecoveryOperation, inspect: (project: Pro
  */
 function failedPeers(op: RecoveryOperation, progress: ProjectProgress, live: Record<string, Inspection | undefined> = {}): string[] {
   const planned = op.plan.projects.find((p) => p.project.id === progress.id);
-  return Object.entries(progress.terminals).filter(([key, value]) => key.startsWith("restored:") && value === "failed").map(([key]) => key.slice("restored:".length))
-    .filter((peer) => peer !== "pi" && !progress.fresh?.[peer] && !planned?.freshStart?.includes(peer) && !plannedAttached(op, progress, peer, live));
+  return Object.entries(progress.terminals).filter(([key, value]) => key.startsWith("restored:") && canChooseFresh(value)).map(([key]) => key.slice("restored:".length))
+    .filter((peer) => peer !== "pi" && ((!Object.hasOwn(progress.reattachments ?? {}, peer) && !Object.hasOwn(progress.repairedSessions ?? {}, peer)) || missingRepair(op, progress, peer)) && !progress.fresh?.[peer] && (!planned?.freshStart?.includes(peer) || missingRepair(op, progress, peer)) && !plannedAttached(op, progress, peer, live));
+}
+
+/** Read-only scope classification; it confers no delegation authority. */
+export function repairedFreshSession(op: RecoveryOperation, peer: string): boolean | undefined {
+  try {
+    validateReceipt(op.id, op);
+    if (!op.plan.projects.some(planned => planned.source.peers.some(row => row.id === peer) ||
+        planned.terminals.some(value => !!value && typeof value === "object" && (value as { peer?: unknown }).peer === peer))) return undefined;
+    return op.projects.some(progress => failedPeers(op, progress).includes(peer) && missingRepair(op, progress, peer));
+  } catch { return undefined; } // Unknown scope never becomes an ordinary delegated disposition.
+}
+
+function repairWaiting(op: RecoveryOperation, projectId: string, peer: RecoveryNativePeer): boolean {
+  try { return decodeRestoreReceipt(op.projects.find(project => project.id === projectId)?.terminals[`restored:${peer}`])?.state === "waiting"; }
+  catch { return false; }
 }
 
 /** #215: the inspection shows the planned session attached for `peer`, so resume records it as restored: nothing is lost. */
 function plannedAttached(op: RecoveryOperation, progress: ProjectProgress, peer: string, live: Record<string, Inspection | undefined>): boolean {
   const state = live[progress.id];
-  const binding = (op.plan.projects.find((p) => p.project.id === progress.id)?.terminals as { peer: string; sessionId: string }[] | undefined)?.find((t) => t.peer === peer);
+  const planned = op.plan.projects.find((p) => p.project.id === progress.id);
+  const binding = planned && Object.hasOwn(progress.repairedSessions ?? {}, peer)
+    ? resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)
+    : (planned?.terminals as { peer: string; sessionId: string }[] | undefined)?.find((t) => t.peer === peer);
   const current = state?.state === "running" ? state.peers.find((p) => p.id === peer && p.state !== "offline") : undefined;
   return !!binding && (peer === "codex" ? current?.threadId : current?.sessionId) === binding.sessionId;
 }
@@ -341,29 +413,84 @@ function abandoned(op: RecoveryOperation): boolean {
 
 function validateReceipt(id: string, op: RecoveryOperation): void {
   if (!(op.schema === 1 || (op.schema === 2 && op.disposition)) || op.id !== id) throw new Error("unsupported operation receipt");
+  if (op.targetFeatures !== undefined && (!Array.isArray(op.targetFeatures) || !op.targetFeatures.every(feature => typeof feature === "string" && feature.length > 0) || !op.targetRoot || !op.targetDigest)) throw new Error("invalid target features");
   const { fingerprint, ...reviewed } = op.plan;
   if (fingerprint !== planFingerprint(reviewed) || op.projects.length !== op.plan.projects.length ||
       op.projects.some((p, i) => p.id !== op.plan.projects[i]?.project.id)) throw new Error("reviewed operation plan or project scope changed");
+  const object = (value: unknown): value is Record<string, unknown> => !!value && typeof value === "object" && !Array.isArray(value);
+  for (const [index, progress] of op.projects.entries()) {
+    if (progress.repairedSessions !== undefined) {
+      if (!object(progress.repairedSessions)) throw new Error("invalid repaired session provenance");
+      for (const [peer, protectedSession] of Object.entries(progress.repairedSessions)) {
+        const restored = decodeRestoreReceipt(progress.terminals[`restored:${peer}`]);
+        const bindings = [progress.reattachments?.[peer as RecoveryNativePeer]?.binding,
+          restored?.state === "done" || restored?.state === "waiting" ? restored.binding : undefined,
+          progress.terminals[`retired:${peer}`]];
+        if (protectedSession !== true || !bindings.some(binding => {
+          if (binding === undefined) return false;
+          const receipt = decodeRestoreReceipt({ state: "done", binding });
+          return receipt?.state === "done" && receipt.binding.peer === peer && receipt.binding.projectRoot === op.plan.projects[index]?.project.root;
+        })) throw new Error("repaired session provenance does not match its retained peer and project");
+      }
+    }
+    for (const [peer, choice] of Object.entries(progress.fresh ?? {})) {
+      if (choice.repaired === undefined) continue;
+      const planned = op.plan.projects[index]!;
+      if (choice.repaired !== true || !progress.repairedSessions?.[peer as RecoveryNativePeer] ||
+          resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)?.sessionId !== choice.lost) throw new Error("invalid one-use repaired-session choice");
+    }
+    if (progress.reattachments === undefined) continue;
+    if (!object(progress.reattachments)) throw new Error("invalid reattachment markers");
+    for (const [peer, marker] of Object.entries(progress.reattachments)) {
+      if (!["codex", "pi", "claude"].includes(peer) || !object(marker) || Object.keys(marker).some(key => !["state", "at", "cause", "binding"].includes(key)) ||
+          !["ready", "attempted"].includes(String(marker.state)) || marker.cause !== "detached after restore" || typeof marker.at !== "number" || !Number.isFinite(marker.at) || marker.at < 0) throw new Error("invalid reattachment marker");
+      const receipt = decodeRestoreReceipt({ state: "done", binding: marker.binding });
+      if (receipt?.state !== "done" || receipt.binding.peer !== peer || receipt.binding.projectRoot !== op.plan.projects[index]?.project.root) throw new Error("reattachment binding does not match its peer and project");
+    }
+  }
+  if (op.missingRecordedSession !== undefined) {
+    const evidence = op.missingRecordedSession;
+    if (!object(evidence) || Object.keys(evidence).some(key => !["projectId", "peer", "sessionId", "kind", "checkedPath"].includes(key)) ||
+        typeof evidence.projectId !== "string" || typeof evidence.peer !== "string" || typeof evidence.sessionId !== "string" ||
+        evidence.checkedPath !== undefined && (typeof evidence.checkedPath !== "string" || !evidence.checkedPath) ||
+        !["rollout", "transcript"].includes(evidence.kind) || !op.projects.some(progress => missingRepair(op, progress, evidence.peer))) throw new Error("invalid recorded-session missing evidence");
+  }
+  if (op.freshOverride !== undefined) {
+    const override = op.freshOverride, evidence = override.evidence;
+    if (!object(override) || Object.keys(override).some(key => !["peer", "evidence", "checked", "at"].includes(key)) || !object(evidence) ||
+        Object.keys(evidence).some(key => !["projectId", "peer", "sessionId", "kind", "checkedPath"].includes(key)) || override.checked !== "missing" ||
+        !Number.isFinite(override.at) || override.at < 0 || override.peer !== evidence.peer || !["rollout", "transcript"].includes(evidence.kind) ||
+        !op.audit?.some(entry => entry.action === "fresh-session" && entry.peer === override.peer && entry.at === override.at && entry.projects.includes(evidence.projectId))) throw new Error("invalid fresh override evidence");
+  }
+  if (op.blockedReattachment !== undefined) {
+    const selected = op.blockedReattachment;
+    if (!object(selected) || Object.keys(selected).some(key => !["projectId", "peer"].includes(key)) || typeof selected.projectId !== "string" || !["codex", "pi", "claude"].includes(String(selected.peer)) ||
+        !op.projects.some(project => project.id === selected.projectId && Object.hasOwn(project.reattachments ?? {}, selected.peer))) throw new Error("invalid blocked reattachment selector");
+  }
 }
 
 export interface RecoveryDriver {
-  stage(op: RecoveryOperation): Promise<{ root: string; digest: string }>;
-  inspect(project: Project): Promise<Inspection>;
-  prepare(project: Project, operation: string, instance: string): Promise<void>;
+  stage(op: RecoveryOperation): Promise<{ root: string; digest: string; targetFeatures?: readonly string[] }>;
+  inspect(project: Project, targetFeatures?: readonly string[]): Promise<Inspection>;
+  prepare(project: Project, operation: string, instance: string, targetFeatures?: readonly string[]): Promise<void>;
   abort(project: Project, operation: string, instance: string): Promise<void>;
-  commit(project: Project, operation: string, instance: string): Promise<void>;
+  commit(project: Project, operation: string, instance: string, targetFeatures?: readonly string[]): Promise<void>;
   closeTerminals(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, save: () => void): Promise<void>;
   start(project: Project, op: RecoveryOperation): Promise<void>;
-  restore(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, group: "native" | "claude", save: () => void): Promise<void>;
+  restore(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, group: "native" | "claude", save: () => void, onlyPeer?: RecoveryNativePeer): Promise<void>;
   installPlugin(op: RecoveryOperation): Promise<void>;
   installGlobal(op: RecoveryOperation): Promise<void>;
   refreshManager?(op: RecoveryOperation): Promise<void>;
   release(project: Project, operation: string, instance: string): Promise<void>;
-  verify(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation): Promise<void>;
+  verify(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, save?: () => void): Promise<void>;
+  /** Read-only positive store evidence; absence of the port means unknown. */
+  recordedSessionState?(binding: TerminalBinding, kind: "rollout" | "transcript"): Promise<"found" | "missing" | "unknown">;
   /** #215 stop-and-archive: stop this operation's unreleased target (when given) and archive its committed snapshot. */
-  stopAndArchive(project: Project, op: RecoveryOperation, targetInstance?: string): Promise<void>;
+  stopAndArchive(project: Project, op: RecoveryOperation, targetInstance?: string): Promise<void | { continuity: "not-archived" | "unavailable" }>;
   sleep(ms: number): Promise<void>;
   now(): number;
+  /** Trusted per-invocation delegation, supplied only by the CLI's identity preflight. */
+  readonly liveTestPeer?: string;
 }
 
 /** Resumable state machine. Driver actions are preceded by receipts and followed by live readback. */
@@ -383,6 +510,7 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
       return op;
     }
   } catch (error) { releaseRunner(); throw error; }
+  let targetFeatures: readonly string[] = [];
   const save = () => { op.updatedAt = driver.now(); writeOperation(id, op, home); };
   const step = (value: string) => { op.step = value; save(); };
   // #225: one restart per project per resume, so a target that keeps dying never spins unattended.
@@ -421,9 +549,20 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
   };
   // Prepare, or after an expired lease re-prepare, the planned source and wait until it is quiet.
   const prepareSource = async (planned: PlannedProject, progress: ProjectProgress, again = false) => {
+    const saveTiming = () => {
+      try { save(); }
+      catch { console.warn("Recovery timing observation could not be saved; recovery continues"); }
+    };
     const project = planned.project, instance = planned.source.instanceId!;
-    await driver.prepare(project, id, instance);
-    const deadline = driver.now() + idleTimeoutMs;
+    await driver.prepare(project, id, instance, targetFeatures);
+    const startedAt = driver.now(), deadline = startedAt + idleTimeoutMs;
+    const waitStep = op.step;
+    if (waitStep) {
+      (op.waits ??= {})[waitStep] = { waitStartedAt: new Date(startedAt).toISOString(), deadlineAt: new Date(deadline).toISOString() };
+      saveTiming();
+    }
+    let waitFailed = false;
+    try {
     for (;;) {
       const live = await driver.inspect(project);
       if (live.instanceId !== instance) throw new Error(`${project.id}: source daemon changed during preparation`);
@@ -439,26 +578,43 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
       }
       await driver.sleep(250);
     }
+    } catch (error) { waitFailed = true; throw error; }
+    finally {
+      const finish = () => {
+        if (!waitStep) return;
+        op.waits![waitStep] = { ...op.waits![waitStep]!, waitEndedAt: new Date(driver.now()).toISOString() };
+        saveTiming();
+      };
+      // A best-effort end observation must not replace the error that stopped the actual wait.
+      if (waitFailed) { try { finish(); } catch { /* original wait failure remains authoritative */ } }
+      else finish();
+    }
   };
   try {
     if (op.disposition) {
       // The stop-and-archive in progress keeps its own cause; resume never runs an operation being abandoned.
       op.error ??= `a stop-and-archive of this operation is partway; ${nextActionsText(op, await liveProjects(op, driver.inspect))}`;
       delete op.failure;
+      delete op.presentationCause;
       op.phase = "blocked"; save();
       return op;
     }
-    op.phase = "running"; delete op.error; delete op.failure; save();
+    op.phase = "running"; delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause;
+    delete op.waits; // Timing is the current attempt's observation, never a prior attempt's countdown.
+    save();
     step("stage");
     const target = await driver.stage(op);
     if (op.targetDigest && (target.root !== op.targetRoot || target.digest !== op.targetDigest)) throw new FinalRefusal("staged target changed since operation started");
-    op.targetRoot = target.root; op.targetDigest = target.digest; save();
+    targetFeatures = target.targetFeatures ?? op.targetFeatures ?? [];
+    op.targetRoot = target.root; op.targetDigest = target.digest;
+    if (target.targetFeatures !== undefined) op.targetFeatures = [...target.targetFeatures];
+    save();
 
-    // Check every untouched source before stopping even the first project.
+    // Pending sources receive the verified target features before holds or terminal/runtime effects.
     for (let i = 0; i < op.projects.length; i++) {
       const progress = op.projects[i]!, planned = op.plan.projects[i]!;
       if (progress.phase !== "pending") continue;
-      const live = await driver.inspect(planned.project);
+      const live = await driver.inspect(planned.project, targetFeatures);
       if (!["running", "stopped", "missing"].includes(live.state)) throw new Error(`${planned.project.id}: the source reads as ${live.state}; wait until it answers`);
       if (live.state !== "running" || live.instanceId !== planned.source.instanceId || live.version !== planned.source.version) {
         throw new Error(`${planned.project.id}: source runtime changed`);
@@ -486,7 +642,14 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
       }
       if (progress.phase === "prepared") {
         step(`commit:${project.id}`);
-        let live = await driver.inspect(project);
+        let live: Inspection;
+        try { live = await driver.inspect(project); }
+        catch (error) {
+          if (!progress.commitSent || !isDatabaseBusy(error)) throw error;
+          live = { state: "unavailable", peers: [], blockers: ["database_busy"] };
+        }
+        const resumedShutdown = progress.commitSent && live.state === "unavailable";
+        let awaitShutdown = resumedShutdown;
         if (live.state === "running" && live.instanceId === planned.source.instanceId) {
           if (live.recovery?.operationId !== id || !live.recovery.ready) {
             const other = live.recovery?.operationId;
@@ -503,8 +666,22 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
           await driver.closeTerminals(planned, progress, op, save);
           // Durable, unlike `step`: from here the source may have committed, which status and abort must not forget.
           progress.commitSent = true; save();
-          await driver.commit(project, id, planned.source.instanceId!);
+          await driver.commit(project, id, planned.source.instanceId!, targetFeatures);
+          awaitShutdown = true;
+        } else if (live.state === "running") {
+          throw new Error(`${project.id}: the source daemon was replaced by instance ${live.instanceId ?? "unknown"}; refusing to prepare, close or stop it (ending this operation leaves that daemon running)`);
+        } else if (live.state === "stopped" && !progress.commitSent && (!op.sourceRoot || coordinatorCurrent(op.sourceRoot))) {
+          // #215: this coordinator records a commit request before sending it, so a stopped source with none crashed while
+          // prepared: no snapshot was committed and there is nothing to start from. The phase stays `prepared`.
+          throw new Error(`${project.id}: the source stopped while prepared and no commit was requested, so there is nothing to restore`);
+        }
+        if (awaitShutdown) {
           const deadline = driver.now() + 30_000;
+          // An already-sent commit can be between its final reply and the source closing its control server.
+          if (resumedShutdown) {
+            if (live.instanceId && live.instanceId !== planned.source.instanceId) throw new Error(`${project.id}: another daemon started during shutdown`);
+            await driver.sleep(100);
+          }
           do {
             try { live = await driver.inspect(project); }
             catch (error) {
@@ -517,12 +694,6 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
             if (live.instanceId && live.instanceId !== planned.source.instanceId) throw new Error(`${project.id}: another daemon started during shutdown`);
             await driver.sleep(100);
           } while (driver.now() < deadline);
-        } else if (live.state === "running") {
-          throw new Error(`${project.id}: the source daemon was replaced by instance ${live.instanceId ?? "unknown"}; refusing to prepare, close or stop it (ending this operation leaves that daemon running)`);
-        } else if (live.state === "stopped" && !progress.commitSent && (!op.sourceRoot || coordinatorCurrent(op.sourceRoot))) {
-          // #215: this coordinator records a commit request before sending it, so a stopped source with none crashed while
-          // prepared: no snapshot was committed and there is nothing to start from. The phase stays `prepared`.
-          throw new Error(`${project.id}: the source stopped while prepared and no commit was requested, so there is nothing to restore`);
         }
         if (live.state === "missing") throw new Error(`${project.id}: the project directory is missing`);
         if (live.state !== "stopped") throw new Error(`${project.id}: old shutdown is not verified (the source reads as ${live.state}); not starting a second daemon; check ${recoveryCommand(op, "status")} once it answers`);
@@ -561,7 +732,7 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
       }
       step(`restore-claude:${planned.project.id}`);
       await driver.restore(planned, progress, op, "claude", save);
-      await driver.verify(planned, progress, op);
+      await driver.verify(planned, progress, op, save);
       step(`release:${planned.project.id}`);
       progress.releaseRequested = true; save();
       await driver.release(planned.project, id, progress.instanceId!);
@@ -574,27 +745,106 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
     releaseRecoveryLock(id, home); op.phase = "completed"; step("completed");
   } catch (error) {
     // #215: every runner error ends with the choices status shows, read from every open source, never only this one.
+    const presentationError = error instanceof RecoveryPresentationError ? error
+      : error instanceof ReattachmentRequired && error.presentationError instanceof RecoveryPresentationError ? error.presentationError : undefined;
+    if (presentationError) op.presentationCause = presentationError.presentationCause;
+    else delete op.presentationCause;
+    if (error instanceof MissingRecordedSession) op.missingRecordedSession = error.evidence;
+    if (error instanceof ReattachmentRequired) op.blockedReattachment = { projectId: error.projectId, peer: error.peer };
+    else delete op.blockedReattachment;
     if (isDatabaseBusy(error)) { op.failure = { code: "database_busy", retryable: true, site: databaseBusySite(error) }; error = new RecoveryDatabaseBusy(); }
     if (error instanceof FinalRefusal) op.final = true;
+    op.phase = "blocked";
     const next = nextActionsText(op, await liveProjects(op, driver.inspect));
-    op.phase = "blocked"; op.error = `${error instanceof Error ? error.message : "recovery failed"}; ${next}`; save();
+    op.error = `${error instanceof Error ? error.message : "recovery failed"}; ${next}`; save();
   } finally { releaseRunner(); }
   return op;
 }
 
 /** Receipts and runner state only (#215): ids, phases and effects, never task or message text. */
 export function publicOperation(op: RecoveryOperation, runnerPid?: number | "unknown", live: Record<string, Inspection | undefined> = {}) {
-  const effect = (value: unknown) => value === "pending" || value === "failed" ? value : "done";
-  const next = nextActions(op, runnerPid, live);
+  const effect = (key: string, value: unknown) => key.startsWith("restored:") ? restoreReceiptStatus(value) : value === "pending" || value === "failed" ? value : "done";
+  const choices = nextChoices(op, runnerPid, live), next = choices.map(choice => choice.text);
   return { id: op.id, phase: op.phase, step: op.step, version: op.plan.version, updatedAt: op.updatedAt,
     runner: runnerPid === "unknown" ? { state: "unknown" } : runnerPid ? { state: "running", pid: runnerPid } : { state: "none" },
     ...(op.phase === "running" && !runnerPid ? { stale: "the receipt says running but no runner holds the operation: its last runner stopped mid-step" } : {}),
-    projects: op.projects.map((p) => ({ id: p.id, phase: p.phase, ...(Object.keys(p.terminals).length ? { effects: Object.fromEntries(Object.entries(p.terminals).map(([key, value]) => [key, effect(value)])) } : {}),
+    projects: op.projects.map((p) => ({ id: p.id, phase: p.phase, ...(Object.keys(p.terminals).length ? { effects: Object.fromEntries(Object.entries(p.terminals).map(([key, value]) => [key, effect(key, value)])) } : {}),
+      ...(op.plan.projects.find(planned => planned.project.id === p.id) ? { recordedSessions: recordedPeerGuidance(op.plan.projects.find(planned => planned.project.id === p.id)!, p, {
+        ...(op.phase === "blocked" && !op.disposition ? { stopAndArchiveCommand: recoveryCommand(op, "dispose", STOP) } : {}),
+        reattachCommand: peer => op.phase === "blocked" && !op.disposition && !op.final && op.targetRoot && op.targetDigest && p.instanceId && ["started", "peers-restored"].includes(p.phase) &&
+          op.blockedReattachment?.projectId === p.id && op.blockedReattachment.peer === peer && choices.some(choice => choice.kind === "reattach" && choice.peer === peer)
+          ? recoveryCommand(op, "reattach", peer) : undefined,
+      }) } : {}),
       ...(p.fresh ? { lostContinuity: Object.fromEntries(Object.entries(p.fresh).map(([peer, f]) => [peer, f.lost])) } : {}),
       ...(p.restarts?.length ? { restarts: p.restarts.length } : {}) })),
     pluginInstalled: !!op.pluginInstalled, globalInstalled: !!op.globalInstalled,
     ...(!targetReadsWaivers(op) && op.projects.some((p) => failedPeers(op, p).length) ? { freshSession: `not offered: target ${op.plan.version} cannot read recovery waivers, so it could not release a new session` } : {}),
-    ...(op.error ? { error: op.error } : {}), ...(op.failure ? { failure: op.failure } : {}), ...(op.disposition ? { disposition: op.disposition } : {}), ...(op.audit ? { audit: op.audit } : {}), next };
+    ...(op.error ? { error: op.error } : {}), ...(op.failure ? { failure: op.failure } : {}), ...(op.blockedReattachment ? { blockedReattachment: op.blockedReattachment } : {}), ...(op.disposition ? { disposition: op.disposition } : {}), ...(op.audit ? { audit: op.audit } : {}), ...(op.freshOverride ? { freshOverride: op.freshOverride } : {}), ...(op.missingRecordedSession ? { missingRecordedSession: op.missingRecordedSession } : {}),
+    ...(op.waits ? { waits: op.waits } : {}), ...(op.presentationCause ? { presentationCause: op.presentationCause } : {}), next };
+}
+
+/**
+ * Person-command entry: restore one recorded conversation under this operation's existing hold.
+ * The driver settles and records the selected native identity; this entry never performs global verification,
+ * release, installation or completion. A later normal resume owns those steps.
+ */
+export async function reattachRecovery(id: string, peer: string, driver: RecoveryDriver, home = hubHome()): Promise<RecoveryOperation> {
+  let op = readOperation<RecoveryOperation>(id, home);
+  validateReceipt(id, op);
+  const selection = () => {
+    if (!["codex", "pi", "claude"].includes(peer) || op.phase !== "blocked" || op.disposition || op.final || op.blockedReattachment?.peer !== peer) throw new Error("reattach is available only for the peer named by a blocked reattachment receipt");
+    const progress = op.projects.find(project => project.id === op.blockedReattachment!.projectId);
+    const planned = op.plan.projects.find(project => project.project.id === progress?.id);
+    const native = peer as RecoveryNativePeer;
+    const marker = progress?.reattachments?.[native];
+    if (!progress || !planned || !marker || !["started", "peers-restored"].includes(progress.phase) || !progress.instanceId || !op.targetRoot || !op.targetDigest ||
+        (marker.state !== "ready" && !repairWaiting(op, progress.id, native))) throw new Error("recorded reattachment is unavailable or its single relaunch allowance was already used; resume to inspect the saved receipt");
+    return { progress, planned, marker, native };
+  };
+  selection(); // Finished or unrelated operations must not acquire a new lock merely to refuse.
+  acquireRecoveryLock(id, home);
+  const releaseRunner = claimRunner(id, home);
+  let attempted = false;
+  const save = () => { op.updatedAt = driver.now(); writeOperation(id, op, home); };
+  try {
+    op = readOperation<RecoveryOperation>(id, home);
+    validateReceipt(id, op);
+    const { progress, planned, marker, native } = selection();
+    const expected = structuredClone(marker.binding);
+    const target = async () => {
+      const observed = await driver.inspect(planned.project);
+      if (observed.state !== "running" || observed.instanceId !== progress.instanceId || observed.version !== op.plan.version || observed.recovery?.operationId !== id || observed.recovery.phase === "released") throw new Error("reattach target changed or is not held by this operation; no replacement runtime is accepted");
+    };
+    await target();
+    attempted = true;
+    await driver.restore(planned, progress, op, native === "claude" ? "claude" : "native", save, native);
+    // A done receipt is written by the runtime only after authenticated attachment and exact settlement.
+    // Inspection peers may be unavailable; do not invent an attachment from a test or a stale status projection.
+    const restored = decodeRestoreReceipt(progress.terminals[`restored:${native}`]);
+    if (restored?.state !== "done" || restored.binding.peer !== native || restored.binding.projectRoot !== expected.projectRoot ||
+        restored.binding.sessionId !== expected.sessionId || restored.binding.sessionFile !== expected.sessionFile) throw new Error(`${native}: recorded session reattachment was not settled with its exact identity`);
+    await target();
+    delete op.blockedReattachment; delete op.failure; delete op.presentationCause;
+    op.phase = "blocked";
+    op.error = `${native}: recorded session reattached; resume to finish verification, release and any remaining installation; ${recoveryCommand(op, "resume")}`;
+    save();
+    return op;
+  } catch (error) {
+    if (attempted) {
+      const presentationError = error instanceof RecoveryPresentationError ? error
+        : error instanceof ReattachmentRequired && error.presentationError instanceof RecoveryPresentationError ? error.presentationError : undefined;
+      if (presentationError) op.presentationCause = presentationError.presentationCause;
+      else delete op.presentationCause;
+      if (error instanceof MissingRecordedSession) op.missingRecordedSession = error.evidence;
+      if (error instanceof ReattachmentRequired) op.blockedReattachment = { projectId: error.projectId, peer: error.peer };
+      else delete op.blockedReattachment;
+      if (isDatabaseBusy(error)) { op.failure = { code: "database_busy", retryable: true, site: databaseBusySite(error) }; error = new RecoveryDatabaseBusy(); }
+      op.phase = "blocked";
+      op.error = `${error instanceof Error ? error.message : "reattachment is unconfirmed"}; ${nextActionsText(op, await liveProjects(op, driver.inspect))}`;
+      save();
+    }
+    throw error;
+  } finally { releaseRunner(); }
 }
 
 /** Escape a blocked preflight without abandoning a stopped runtime or an uncertain terminal mutation. */
@@ -624,7 +874,7 @@ export async function abortRecovery(id: string, driver: RecoveryDriver, home = h
       const state = live[planned.project.id];
       if (state?.recovery?.operationId === id && state.recovery.phase !== "released") await driver.abort(planned.project, id, planned.source.instanceId!);
     }
-    delete op.error; delete op.failure;
+    delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause;
     op.phase = "cancelled"; op.step = "cancelled"; op.updatedAt = driver.now();
     writeOperation(id, op, home); releaseRecoveryLock(id, home);
   } catch (error) {
@@ -639,15 +889,18 @@ export async function abortRecovery(id: string, driver: RecoveryDriver, home = h
  * every runtime this operation held or started is verified released or stopped.
  */
 export async function disposeRecovery(id: string, choice: { fresh: string } | { stop: true }, reason: string, driver: RecoveryDriver, home = hubHome()): Promise<RecoveryOperation> {
-  let op = readOperation<RecoveryOperation>(id, home);
-  validateReceipt(id, op);
+  let op: RecoveryOperation;
+  try { op = readOperation<RecoveryOperation>(id, home); validateReceipt(id, op); }
+  catch (error) { if (driver.liveTestPeer && "fresh" in choice) throw new Error(LIVE_TEST_PERSON_REFUSAL); throw error; }
+  if (driver.liveTestPeer && "fresh" in choice && repairedFreshSession(op, choice.fresh) !== false) throw new Error(LIVE_TEST_PERSON_REFUSAL);
   // A finished operation owns no lock: never take one only to refuse.
   if (op.phase === "completed" || op.phase === "cancelled") throw new Error(`operation is already ${op.phase}`);
   acquireRecoveryLock(id, home);
   const releaseRunner = claimRunner(id, home);
   try {
-    op = readOperation<RecoveryOperation>(id, home);
-    validateReceipt(id, op);
+    try { op = readOperation<RecoveryOperation>(id, home); validateReceipt(id, op); }
+    catch (error) { if (driver.liveTestPeer && "fresh" in choice) throw new Error(LIVE_TEST_PERSON_REFUSAL); throw error; }
+    if (driver.liveTestPeer && "fresh" in choice && repairedFreshSession(op, choice.fresh) !== false) throw new Error(LIVE_TEST_PERSON_REFUSAL);
     if (op.phase === "completed" || op.phase === "cancelled") {
       // Another runner finished it while this one waited: leave no lock behind, as resume and abort do.
       if (recoveryLock(home) === id) releaseRecoveryLock(id, home);
@@ -660,7 +913,7 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
     const next = nextActionsText(op, live);
     if ("fresh" in choice) {
       // `ahub pi` sends no fresh flag, and a restored hub refills a Pi start from its recorded resume (daemon startPeer).
-      if (choice.fresh === "pi") throw new Error(`pi: --fresh-session is not supported: a restored hub resumes Pi's recorded session, so a fresh one cannot be guaranteed; ${next}`);
+      if (choice.fresh === "pi") throw new Error(`pi: --fresh-session is not supported: a restored hub resumes Pi's recorded session, so a fresh one cannot be guaranteed; ${op.plan.projects.map(planned => piArchiveExit(planned, recoveryCommand(op, "dispose", STOP))).join("; ")}; ${next}`);
       if (op.disposition) throw new Error(`a stop-and-archive of this operation is partway; ${next}`);
       if (!targetReadsWaivers(op)) throw new Error(`--fresh-session is not available: target ${op.plan.version} cannot read recovery waivers, so it could not release a new session; ${next}`);
       // The new session is launched by resume, so it helps only where resume can get past what is live.
@@ -669,19 +922,39 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
       const peer = choice.fresh;
       // Per project: a planned fresh start has nothing to lose, so only the other failed projects take the choice.
       const failed = op.projects.filter((p) => failedPeers(op, p, live).includes(peer));
-      if (!failed.length && op.projects.some((p) => p.terminals[`restored:${peer}`] === "failed" && op.plan.projects.find((planned) => planned.project.id === p.id)?.freshStart?.includes(peer))) {
+      if (!failed.length && op.projects.some((p) => canChooseFresh(p.terminals[`restored:${peer}`]) && op.plan.projects.find((planned) => planned.project.id === p.id)?.freshStart?.includes(peer))) {
         throw new Error(`${peer}: its plan already restarts it as a new session (no rollout and no turn, nothing to lose), so there is no conversation to record as lost; ${next}`);
       }
-      if (!failed.length && op.projects.some((p) => p.terminals[`restored:${peer}`] === "failed" && plannedAttached(op, p, peer, live))) {
+      if (!failed.length && op.projects.some((p) => canChooseFresh(p.terminals[`restored:${peer}`]) && plannedAttached(op, p, peer, live))) {
         throw new Error(`${peer}: its planned session is attached again, so nothing was lost; resume records it as restored; ${next}`);
       }
       if (!failed.length) throw new Error(`${peer}: no failed restoration of it is open to a fresh session (none failed, or one is already chosen); --fresh-session applies only then (${recoveryCommand(op, "status")})`);
+      // Collect every proof before changing any project, so one uncertain read cannot leave a partial choice.
+      const selected = [];
       for (const progress of failed) {
-        const binding = (op.plan.projects.find((p) => p.project.id === progress.id)!.terminals as { peer: string; sessionId: string }[]).find((t) => t.peer === peer)!;
-        (progress.fresh ??= {})[peer] = { lost: binding.sessionId, reason, at };
+        const repaired = missingRepair(op, progress, peer);
+        const planned = op.plan.projects.find((p) => p.project.id === progress.id)!;
+        const binding = repaired ? resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)!
+          : (planned.terminals as TerminalBinding[]).find((t) => t.peer === peer)!;
+        const evidence = repaired ? { ...op.missingRecordedSession!, checkedPath: checkedSessionPath(binding, op.missingRecordedSession!.kind) } : undefined;
+        if (evidence && (op.missingRecordedSession!.checkedPath !== undefined && op.missingRecordedSession!.checkedPath !== evidence.checkedPath ||
+            await driver.recordedSessionState?.(binding, evidence.kind) !== "missing")) throw new Error(`${peer}: recorded conversation is not positively missing at ${JSON.stringify(evidence.checkedPath)}; no fresh-session choice was saved`);
+        selected.push({ progress, binding, repaired, evidence });
+      }
+      for (const { progress, binding, repaired, evidence } of selected) {
+        (progress.fresh ??= {})[peer] = { lost: binding.sessionId, reason, at, ...(repaired ? { repaired: true as const } : {}) };
+        if (repaired) {
+          (progress.repairedSessions ??= {})[peer as RecoveryNativePeer] = true;
+          op.freshOverride = { peer: peer as RecoveryNativePeer, evidence: evidence!, checked: "missing", at };
+        }
+        if (repaired && progress.reattachments?.[peer as RecoveryNativePeer]) {
+          progress.terminals[`retired:${peer}`] = binding;
+          delete progress.terminals[`closedRetired:${peer}`];
+          delete progress.reattachments[peer as RecoveryNativePeer];
+        }
       }
       (op.audit ??= []).push({ at, action: "fresh-session", reason, peer, projects: failed.map((p) => p.id) });
-      delete op.error; delete op.failure;
+      delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause; delete op.missingRecordedSession;
       op.updatedAt = at; writeOperation(id, op, home);
       return op;
     }
@@ -707,7 +980,9 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
       acts.push({ planned, ...(state ? { live: state } : {}), act });
     }
     // Each outcome is recorded as it happens, so a disposition that stops partway stays true and resume refuses it.
-    const disposition = (op.disposition ??= { choice: "stop-and-archive", at, projects: {} });
+    const disposition: NonNullable<RecoveryOperation["disposition"]> = (op.disposition ??= { choice: "stop-and-archive", at, projects: {} });
+    delete op.blockedReattachment;
+    delete op.presentationCause;
     abandoned(op); // #227: in the same write as the disposition
     (op.audit ??= []).push({ at, action: "stop-and-archive", reason, projects: acts.map((a) => a.planned.project.id) });
     // Recorded before the first act: a crash inside a stop leaves a disposition that resume refuses to override.
@@ -717,7 +992,8 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
       for (const { planned, live, act } of acts) {
         if (act.startsWith("source hold")) await driver.abort(planned.project, id, planned.source.instanceId!);
         else if (act === "stopped" || act === "target stopped") {
-          await driver.stopAndArchive(planned.project, op, act === "target stopped" ? live!.instanceId : undefined);
+          const outcome = await driver.stopAndArchive(planned.project, op, act === "target stopped" ? live!.instanceId : undefined);
+          if (outcome && outcome.continuity) (disposition.continuity ??= {})[planned.project.id] = outcome.continuity;
           if ((await driver.inspect(planned.project)).state !== "stopped") throw new Error(`${planned.project.id}: shutdown of instance ${live!.instanceId ?? "unknown"} is not verified`);
         }
         disposition.projects[planned.project.id] ??= act;