@staix/agent-hub 0.12.30 → 0.12.31

package/src/cli/upgrade-runtime.ts

@@ -15,9 +15,16 @@ import { readEvents } from "../hub/events.ts";
 import { refreshManager } from "../hub/manager.ts";
 import { abandonRestartSnapshot, readRestartSnapshot, waiveRecoveryPeers } from "../hub/restart.ts";
 import { recoveryLock } from "../hub/recovery-store.ts";
+import { decodeRestoreReceipt, restoreTransition, type RestoreReceipt, type RestoreEvidence, type RestoreTransition } from "./upgrade-restore-state.ts";
 
 /** An unmanaged Claude's plugin retries with a backoff of at most 30 s; three of those bound the reconnect wait (#206). */
-const RECONNECT_WAIT_MS = 90_000;
+const UNMANAGED_RECONNECT_WAIT_MS = 90_000;
+/** Managed TUI startup can include resume and MCP handshakes; match createTerminal's ten-minute readiness ceiling. */
+const RECONNECT_WAIT_MS = 600_000;
+class AttachmentLauncherGone extends Error {
+  constructor(readonly transition: RestoreTransition) { super("chosen attachment launcher exited"); }
+}
+const attachedBinding = (binding: TerminalBinding, sessionId: string): TerminalBinding => ({ ...binding, sessionId });
 
 export const PACKAGE_ROOT = resolve(import.meta.dir, "../..");
 const orcaExecutable = () => process.env.ORCA_CLI_COMMAND || (process.env.ORCA_DEV_REPO_ROOT ? "orca-dev" : process.platform === "linux" && !process.env.ORCA_TERMINAL_HANDLE ? "orca-ide" : "orca");
@@ -151,7 +158,7 @@ export async function endPlannedPeer(planned: PlannedProject, peer: RecoveryPeer
   }
 }
 
-export async function inspectRecovery(project: Project): Promise<Inspection> {
+export async function inspectRecovery(project: Project, targetFeatures?: readonly string[]): Promise<Inspection> {
   // #226: the claim is judged by the registry as it is now, never by the plan's frozen copy, which holds the source's
   // pid from planning time (after a reboot another process may hold it). A registry that cannot be read throws: the
   // caller reads that as not inspected, never as stopped.
@@ -189,9 +196,9 @@ export async function inspectRecovery(project: Project): Promise<Inspection> {
   }
   if (!status || !control) return { state: "unavailable", peers: [], blockers: ["authenticated source status unavailable"] };
   let response: any;
-  try { response = await rpc(project, { t: "recovery", op: "inspect", expectedInstanceId: status.instanceId }, sourceProtocol ?? PROTOCOL); }
+  try { response = await rpc(project, { t: "recovery", op: "inspect", expectedInstanceId: status.instanceId, ...(targetFeatures === undefined ? {} : { targetFeatures }) }, sourceProtocol ?? PROTOCOL); }
   catch (error) {
-    if (isDatabaseBusy(error)) throw error;
+    if (isDatabaseBusy(error) || targetFeatures !== undefined) throw error;
     // A stop can complete between authenticated status and the second read. Never infer stopped
     // from a connection failure: the coordinator will inspect the ownership manifest again.
     return { state: "unavailable", instanceId: status.instanceId, version: status.version, protocol: status.protocol,
@@ -352,10 +359,10 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
     if (result.code !== 0) throw new Error(`target command ${args[0]} failed; inspect recovery state before retrying`);
     return result;
   };
-  const control = (project: Project, op: string, id: string, instance: string) => {
+  const control = (project: Project, op: string, id: string, instance: string, targetFeatures?: readonly string[]) => {
     const sourceProtocol = readControl(project.stateDir)?.protocol;
     const protocol = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]) ? sourceProtocol : PROTOCOL;
-    return rpc(project, { t: "recovery", op, operationId: id, expectedInstanceId: instance }, protocol).then(() => {});
+    return rpc(project, { t: "recovery", op, operationId: id, expectedInstanceId: instance, ...(targetFeatures === undefined ? {} : { targetFeatures }) }, protocol).then(() => {});
   };
   const revalidateTerminal = async (planned: PlannedProject, progress: ProjectProgress, saved: TerminalBinding, exact: boolean): Promise<TerminalBinding> => {
     const options = { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId };
@@ -387,7 +394,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
    * expected instance can report "gone"; an unreadable target or launcher is unknown and blocks, never gone.
    */
   const peerEvidence = async (planned: PlannedProject, progress: ProjectProgress, peer: TerminalBinding["peer"]):
-      Promise<{ state: "live"; session?: string; handle?: string; attached?: true } | { state: "gone" } | { state: "unknown"; why: string; step?: string }> => {
+      Promise<{ state: "live"; session?: string; handle?: string; attached?: true; terminal?: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId"> } | { state: "gone"; terminal?: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId"> } | { state: "unknown"; why: string; step?: string; idless?: true }> => {
     const target = await inspectRecovery(planned.project);
     if (target.state !== "running" || target.instanceId !== progress.instanceId) {
       // Only a hub that may still answer is waited for; a stopped, missing or other one is settled by the runner's list.
@@ -397,22 +404,31 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
     const attached = target.peers.find((p) => p.id === peer && p.state !== "offline");
     const session = peer === "codex" ? attached?.threadId : attached?.sessionId;
     if (session) return { state: "live", session };
+    let endedTerminal: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId"> | undefined;
     // #225: the launch records of every instance this operation started for the project, the current one first: a
     // launcher a dead instance recorded may still run (read before the check below, so a block names its terminal).
     for (const instanceId of [progress.instanceId, ...(progress.restarts ?? []).map((r) => r.instanceId)]) {
       if (!instanceId) continue;
       const launch = await launcherOf(peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId });
-      if (launch?.state === "live") return { state: "live", handle: launch.record!.handle, ...(attached ? { attached: true as const } : {}) };
+      if (launch?.state === "live") {
+        const record = launch.record!;
+        return { state: "live", handle: record.handle, ...(attached ? { attached: true as const } : {}),
+          ...(instanceId === progress.instanceId ? { terminal: { handle: record.handle, incarnationId: record.incarnationId, worktreeId: record.worktreeId } } : {}) };
+      }
       if (launch?.state === "unknown") {
         return launch.record ? { state: "unknown", why: `its launcher in terminal ${launch.record.handle} cannot be read`, step: `wait until it attaches, or end it and close terminal ${launch.record.handle}` }
           : launch.invalidRow ? { state: "unknown", why: "a launch record that may be its launcher's cannot be evaluated", step: `inspect ${recordPath(planned.project.stateDir)} and fix that row, or end that launcher and remove its row` }
           : { state: "unknown", why: "the launcher records cannot be read", step: `inspect ${recordPath(planned.project.stateDir)} and move it aside (sessions it recorded then count as unmanaged)` };
       }
+      if (instanceId === progress.instanceId && launch?.state === "gone" && launch.record) {
+        const record = launch.record;
+        endedTerminal = { handle: record.handle, incarnationId: record.incarnationId, worktreeId: record.worktreeId };
+      }
     }
     // #225: attached with no provable session id (a Claude of a dead instance whose plugin reconnected, or one that has
     // not reported its id yet) is a session of unknown identity, never "nothing attached".
-    if (attached) return { state: "unknown", why: `a ${peer} session is attached to the target without a session id`, step: `wait until it reports one, or end that ${peer} session` };
-    return { state: "gone" };
+    if (attached) return { state: "unknown", idless: true, why: `a ${peer} session is attached to the target without a session id`, step: `wait until it reports one, or end that ${peer} session` };
+    return { state: "gone", ...(endedTerminal ? { terminal: endedTerminal } : {}) };
   };
   /**
    * #225: the terminal of a done restoration of a target instance that died is closed under the close receipt rules
@@ -436,13 +452,16 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
     progress.terminals[key] = true; save();
   };
   /** #414: an idle terminal is not an authenticated peer. Keep its receipt while waiting, including on resume. */
-  const awaitAttachment = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, binding: TerminalBinding, save: () => void, expectedSession: string | null = binding.sessionId): Promise<string> => {
+  const awaitAttachment = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, waiting: Extract<RestoreReceipt, { state: "waiting" }>, save: () => void, expectedSession: string | null = waiting.binding.sessionId): Promise<string> => {
+    const binding = waiting.binding;
+    const matchesTerminal = (terminal: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId">) => terminal.handle === binding.handle && terminal.incarnationId === binding.incarnationId && terminal.worktreeId === binding.worktreeId;
     const deadline = now() + RECONNECT_WAIT_MS;
     let live: Inspection;
     for (;;) {
       live = await inspectRecovery(planned.project);
-      if (live.state === "running" && (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id)) throw new Error(`${binding.peer}: target identity changed while awaiting attachment`);
-      const peer = live.peers.find((p) => p.id === binding.peer);
+      if (live.state !== "running") throw new Error(`${binding.peer}: target hub reads as ${live.state} while awaiting attachment; saved terminal retained`);
+      if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw new Error(`${binding.peer}: target identity changed while awaiting attachment`);
+      const peer = live.peers.find((p) => p.id === binding.peer && p.state !== "offline");
       const session = binding.peer === "codex" ? peer?.threadId : peer?.sessionId;
       const changed = !!session && expectedSession !== null && session !== expectedSession;
       // #64: only the original zero-turn Claude may change identity without an operator choice. Re-read persistence
@@ -450,10 +469,38 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
       const zeroTurn = changed && binding.peer === "claude" && (planned.terminals as TerminalBinding[]).some((p) => p.peer === "claude" && p.sessionId === binding.sessionId) && !claudeTranscriptExists(binding);
       if (changed && !zeroTurn) throw new Error(`${binding.peer}: a different conversation attached while awaiting ${binding.sessionId}`);
       if (now() > deadline) break;
-      if (live.state === "running" && peer && ["idle", "busy", "paused"].includes(peer.state) && session) {
+      const launcher = await launcherOf(binding.peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId });
+      const sameTerminal = launcher?.record && matchesTerminal(launcher.record);
+      let launchState: RestoreEvidence["launcher"] = !launcher ? "no-record-yet" : launcher.state === "unknown" || !launcher.record ? "unknown"
+        : sameTerminal ? launcher.state : launcher.state === "gone" ? "no-record-yet" : "replaced";
+      let olderHandle: string | undefined;
+      if ((launchState === "gone" || launchState === "no-record-yet") && !session) {
+        // An exact exit is not proof while an older launcher or an id-less session may still run.
+        const evidence = await peerEvidence(planned, progress, binding.peer);
+        if (evidence.state === "gone" && evidence.terminal && matchesTerminal(evidence.terminal)) launchState = "gone";
+        else if (evidence.state === "unknown") {
+          if (!(launchState === "no-record-yet" && evidence.idless)) launchState = "unknown";
+        } else if (evidence.state === "live" && !evidence.session) {
+          if (evidence.terminal) launchState = matchesTerminal(evidence.terminal) ? "live" : "replaced";
+          else { launchState = "older-instance"; olderHandle = evidence.handle; }
+        } else if (evidence.state === "live" && evidence.session) {
+          if (now() >= deadline) break;
+          await sleep(Math.min(1000, deadline - now()));
+          continue;
+        }
+      }
+      const decision = restoreTransition(waiting, { target: "running", launcher: launchState,
+        session: session && peer && ["idle", "busy", "paused"].includes(peer.state) ? changed ? "other" : "planned" : "none", allowOther: zeroTurn || expectedSession === null, binding });
+      if (decision.action === "settle" && session) {
         if (zeroTurn) waiveRecoveryPeers(planned.project.stateDir, op.id, { claude: "zero-turn" });
         return session;
       }
+      if (decision.action === "fail" || decision.action === "create") throw new AttachmentLauncherGone(decision);
+      if (decision.action === "block") {
+        if (launchState === "replaced") throw new Error(`${binding.peer}: saved terminal identity changed while awaiting attachment`);
+        if (launchState === "older-instance") throw new Error(`${binding.peer}: an older target's launcher in terminal ${olderHandle ?? "unknown"} may still run; saved terminal retained; no terminal was created`);
+        throw new Error(`${binding.peer}: the chosen restoration launcher's identity cannot be verified (${launchState}); saved terminal retained; no terminal was created`);
+      }
       if (now() >= deadline) break;
       await sleep(Math.min(1000, deadline - now()));
     }
@@ -468,7 +515,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
       }
     } catch { /* unavailable metadata stays unknown; raw output/errors never enter the receipt */ }
     const peer = live.peers.find((p) => p.id === binding.peer);
-    const session = binding.peer === "codex" ? peer?.threadId : peer?.sessionId;
+    const session = peer?.state === "offline" ? undefined : binding.peer === "codex" ? peer?.threadId : peer?.sessionId;
     (progress.attachmentTimeouts ??= {})[binding.peer] = { at: now(), terminal,
       hub: { state: live.state, ...(live.instanceId ? { instanceId: live.instanceId } : {}), ...(peer ? { peerState: peer.state } : {}), ...(session ? { sessionId: session } : {}) },
       lastHubConnectAttempt: "unknown" };
@@ -485,15 +532,23 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
       if (liveTestPeer && (!existsSync(identityPath) || !readFileSync(identityPath, "utf8").includes("export const LIVE_TEST_SUPPORTED = true;"))) {
         throw new FinalRefusal("target does not support live-test delegation; choose a supporting target before retrying; runtimes unchanged");
       }
-      // #215: the protocol and waiver checks are properties of these bytes. Once they passed (the runner recorded the
-      // digest), a resume does not check again, so a transient probe failure cannot refuse an operation for good;
-      // changed bytes are the runner's final "staged target changed".
-      if (op.targetDigest && op.targetDigest === target.digest && op.targetRoot === target.root) return target;
-      const protocol = await run([process.execPath, "-e", `import { PROTOCOL } from ${JSON.stringify(join(target.root, "src/hub/control-client.ts"))}; console.log(PROTOCOL)`]);
-      const read = protocol.code === 0 ? Number.parseInt(protocol.stdout.trim(), 10) : NaN;
-      // A probe that did not answer (killed, a null exit code maps to 1) says nothing about the target: retry.
-      if (!Number.isInteger(read)) throw new Error(`the staged target's control protocol could not be read (probe exit ${protocol.code}); staged package retained, runtimes unchanged; resume again once it can run`);
-      if (read !== PROTOCOL) throw new FinalRefusal("target protocol requires a newer coordinator; staged package retained, runtimes unchanged");
+      // Reuse capability proof only after verifying the cached root and digest still identify these exact bytes.
+      const cached = op.targetDigest === target.digest && op.targetRoot === target.root;
+      if (cached && op.targetFeatures !== undefined) return { ...target, targetFeatures: [...op.targetFeatures] };
+      // A legacy cached operation already past commit needs no capability proof to finish restoration (#215).
+      // A prepared commitSent receipt remains uncertain and must still obtain proof before a possible commit.
+      if (cached && op.projects?.every(project => ["stopped", "started", "peers-restored", "verified"].includes(project.phase))) return target;
+      // Legacy uncommitted receipts acquire verified feature evidence before source compatibility checks.
+      const probe = await run([process.execPath, "-e", `import * as target from ${JSON.stringify(join(target.root, "src/hub/control-client.ts"))}; console.log(JSON.stringify({protocol: target.PROTOCOL, features: target.DAEMON_FEATURES ?? []}))`]);
+      let metadata: { protocol?: unknown; features?: unknown } | undefined;
+      try { if (probe.code === 0) metadata = JSON.parse(probe.stdout); } catch { /* Unknown metadata refuses before effects. */ }
+      if (!metadata || typeof metadata !== "object" || !Number.isInteger(metadata.protocol)) throw new Error(`the staged target's control protocol could not be read (probe exit ${probe.code}); staged package retained, runtimes unchanged; resume again once it can run`);
+      if (!Array.isArray(metadata.features) || !metadata.features.every((feature: unknown) => typeof feature === "string" && feature.length > 0)) throw new Error("the staged target features could not be read; staged package retained, runtimes unchanged");
+      const targetFeatures: readonly string[] = [...metadata.features];
+      if (metadata.protocol !== PROTOCOL) throw new FinalRefusal("target protocol requires a newer coordinator; staged package retained, runtimes unchanged");
+      const verified = { ...target, targetFeatures };
+      // Existing peer/waiver compatibility checks remain properties of the already verified bytes.
+      if (op.targetDigest && op.targetDigest === target.digest && op.targetRoot === target.root) return verified;
       if (op.plan.projects.some(p => p.source.peers.some(peer => peer.id === "qwen" && peer.state !== "offline"))) {
         const catalog = join(target.root, "src/hub/peer-catalog.ts");
         const peers = existsSync(catalog) ? /export const OPTIONAL_PEERS\s*=\s*\[([^\]]*)\]/.exec(readFileSync(catalog, "utf8"))?.[1] : undefined;
@@ -508,11 +563,11 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
       if (op.plan.projects.some((p) => p.reconnectOnly?.length || p.freshStart?.length) && !targetReadsWaivers({ targetRoot: target.root })) {
         throw new FinalRefusal(`target ${op.plan.version} predates recovery waivers, so its hub could never accept a reconnect-only Claude or a fresh Codex start; staged package retained, runtimes unchanged; choose a newer target once this operation is cancelled or ended`);
       }
-      return target;
+      return verified;
     },
-    prepare: (p, id, instance) => control(p, "prepare", id, instance),
+    prepare: (p, id, instance, features) => control(p, "prepare", id, instance, features),
     abort: (p, id, instance) => control(p, "abort", id, instance),
-    commit: (p, id, instance) => control(p, "commit", id, instance),
+    commit: (p, id, instance, features) => control(p, "commit", id, instance, features),
     release: (p, id, instance) => control(p, "release", id, instance),
     closeTerminals: async (planned, progress, op, save) => {
       for (const binding of planned.terminals as TerminalBinding[]) {
@@ -627,22 +682,35 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
           : evidence.session
             ? `${original.peer}: session ${evidence.session} is attached instead of ${original.sessionId}; no terminal was created; end that ${original.peer} session and close its terminal first`
             : `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session ${evidence.attached ? "with an id " : ""}attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`);
-        const receipt = progress.terminals[key];
-        if (receipt && receipt !== "pending" && receipt !== "failed") {
-          const binding = receipt as TerminalBinding;
-          const sessionId = await awaitAttachment(planned, progress, op, binding, save, fresh() && binding.sessionId === original.sessionId ? null : binding.sessionId);
-          progress.terminals[key] = { ...binding, sessionId }; save();
-          progress.terminals[key] = await revalidateTerminal(planned, progress, progress.terminals[key] as TerminalBinding, true); save();
-          continue;
+        const decide = (receipt: RestoreReceipt, evidence: Awaited<ReturnType<typeof peerEvidence>>) => restoreTransition(receipt, {
+          target: "running", launcher: evidence.state === "unknown" ? "unknown" : evidence.state === "gone" ? "gone" : evidence.terminal ? "live" : "older-instance",
+          session: evidence.state === "live" && evidence.session ? evidence.session === original.sessionId ? "planned" : "other" : "none",
+          allowOther: fresh() || (original.peer === "claude" && !claudeTranscriptExists(original)),
+          binding: evidence.state === "live" && evidence.terminal ? { ...original, ...evidence.terminal } : original,
+        });
+        const receipt = decodeRestoreReceipt(progress.terminals[key]);
+        function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }> & { origin: "pending" }, expected: string | null): Promise<string>;
+        function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined>;
+        async function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined> {
+          try { return await awaitAttachment(planned, progress, op, waiting, save, expected); }
+          catch (error) {
+            if (!(error instanceof AttachmentLauncherGone)) throw error;
+            progress.terminals[key] = error.transition.next; save();
+            // The table decides whether this failed wait blocks or is eligible for a new launch.
+            if (error.transition.action === "create") return undefined;
+            throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
+          }
         }
         // #215: the pending row of the receipt table, used wherever a pending launch meets an attached session.
-        const settle = async (attached: string) => {
+        const settle = async (attached: string, recovered?: TerminalBinding) => {
           // #21: a Claude session that never got a first turn has no transcript, so `claude --resume S` can never
           // succeed and accepting its fresh attach loses nothing. Otherwise only the planned id, or a new session
           // the operator or the plan accepted, counts as restored.
           if (attached !== original.sessionId && !fresh() && !(original.peer === "claude" && !claudeTranscriptExists(original))) {
             throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${attached} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
           }
+          // Validate the proposed transition before committing continuity/waiver effects or a done receipt.
+          const verified = await revalidateTerminal(planned, progress, attachedBinding(recovered ?? original, attached), !!recovered);
           // The planned session came back after all: a fresh session chosen for it lost nothing. Compared with the plan,
           // never with a session accepted after a restart (#225), whose choice must stay on record.
           if (attached === plannedBinding.sessionId && progress.fresh?.[original.peer]) {
@@ -653,31 +721,58 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
           // a launch writes the waiver first, but this must not rely on that.
           // The planned session needs no waiver: the target's snapshot already expects it.
           if (attached !== original.sessionId && attached !== plannedBinding.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
-          progress.terminals[key] = await revalidateTerminal(planned, progress, { ...original, sessionId: attached }, false); save();
+          progress.terminals[key] = { state: "done", binding: verified } satisfies RestoreReceipt; save();
         };
-        // #215: a pending or failed receipt is settled by what is live now (the receipt table in the recovery spec).
-        if (receipt === "pending" || receipt === "failed") {
+        if (receipt?.state === "done") {
+          // A completed restoration is never a waiting launch. A closed terminal blocks exact revalidation;
+          // it cannot erase continuity or create a fresh-session choice (PROBE-B).
+          const observed = await peerEvidence(planned, progress, original.peer);
+          const decision = restoreTransition(receipt, { target: "running", launcher: "no-record-yet", session: observed.state === "live" && observed.session === receipt.binding.sessionId ? "planned" : observed.state === "live" && observed.session ? "other" : "none", allowOther: false });
+          if (decision.action === "block") {
+            const condition = observed.state === "unknown" ? `cannot be verified (${observed.why})`
+              : observed.state === "live" && observed.session ? `session ${observed.session} is attached instead of ${receipt.binding.sessionId}`
+              : observed.state === "live" ? `is not attached; its launcher in terminal ${observed.handle ?? "unknown"} may still run` : "is not attached";
+            throw new Error(`${original.peer}: completed restoration ${condition}; saved done receipt retained; resume its recorded conversation`);
+          }
+          progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, receipt.binding, true) } satisfies RestoreReceipt; save();
+          continue;
+        }
+        if (receipt?.state === "waiting") {
+          const binding = receipt.binding;
+          const sessionId = await awaitBound(receipt, fresh() && binding.sessionId === original.sessionId ? null : binding.sessionId);
+          if (sessionId !== undefined) { await settle(sessionId, binding); continue; }
+        }
+        if (receipt?.state === "pending" || receipt?.state === "failed") {
           const evidence = await peerEvidence(planned, progress, original.peer);
-          if (evidence.state === "live" && evidence.session) { await settle(evidence.session); continue; }
-          if (evidence.state !== "gone") throw blocked(evidence);
-          if (receipt === "pending") {
-            progress.terminals[key] = "failed"; save();
+          const decision = decide(receipt, evidence);
+          if (decision.action === "settle" && evidence.state === "live" && evidence.session) { await settle(evidence.session); continue; }
+          if (decision.action === "poll" && decision.next.state === "waiting") {
+            const waiting = decision.next;
+            progress.terminals[key] = waiting; save();
+            const session = await awaitBound(waiting, fresh() ? null : original.sessionId);
+            if (session !== undefined) { await settle(session, waiting.binding); continue; }
+          } else if (decision.action === "fail") {
+            progress.terminals[key] = decision.next; save();
             throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
+          } else if (decision.action === "block") {
+            if (evidence.state === "live" && evidence.session) {
+              throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${evidence.session} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
+            }
+            if (evidence.state !== "gone") throw blocked(evidence);
+            throw new Error(`${original.peer}: restoration evidence is not actionable`);
           }
-          // failed, nothing live: launched again below; the receipt stays `failed` until `pending` replaces it, so a
-          // refusal on the way keeps the failed-restoration choices
         }
         // Never create beside a live launch, nor on evidence that could not be read: read now, not at restore start.
         // The planned session already attached is the restoration (the table's row for it), not a reason to block.
         const evidence = await peerEvidence(planned, progress, original.peer);
         if (evidence.state === "live" && evidence.session === original.sessionId) {
-          progress.terminals[key] = await revalidateTerminal(planned, progress, original, false); save();
+          progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, original, false) } satisfies RestoreReceipt; save();
           continue;
         }
         if (evidence.state !== "gone") throw blocked(evidence);
         if (transcript() === "unknown") throw new Error(`${unresumable(original)}; no terminal was created; make that store readable first`);
         if (transcript() === "missing" && !fresh()) {
-          progress.terminals[key] = "failed"; save();
+          progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
           throw notRestored(unresumable(original).slice("codex: ".length));
         }
         if (fresh()) {
@@ -691,7 +786,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
         const launch = { ...original.launch, packageEntrypoint: entrypoint, argv,
           command: ["env", ...Object.entries(assignments).map(([k, v]) => `${k}=${v}`), ...argv].map(shellQuote).join(" ") };
         const binding = { ...original, launch, launchMetadata: launch };
-        progress.terminals[key] = "pending"; save();
+        progress.terminals[key] = { state: "pending" } satisfies RestoreReceipt; save();
         const restored = await createTerminal(binding, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId }, undefined, fresh());
         const exited = restored.blockers.find((b) => b.code === "launcher-exited");
         if (exited) {
@@ -699,25 +794,34 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
           // nothing live makes it failed; an attached session is settled as the pending row settles it; anything else
           // stays pending.
           const after = await peerEvidence(planned, progress, original.peer);
-          if (after.state === "live" && after.session) { await settle(after.session); continue; }
+          const decision = decide({ state: "pending" }, after);
+          if (decision.action === "settle" && after.state === "live" && after.session) { await settle(after.session); continue; }
+          if (decision.action === "poll" && decision.next.state === "waiting") {
+            progress.terminals[key] = decision.next; save();
+            const session = await awaitBound(decision.next, fresh() ? null : original.sessionId);
+            if (session !== undefined) { await settle(session, decision.next.binding); continue; }
+          }
+          if (decision.action === "fail") {
+            progress.terminals[key] = decision.next; save();
+            throw notRestored(exited.message);
+          }
           if (after.state !== "gone") throw blocked(after);
-          progress.terminals[key] = "failed"; save();
-          throw notRestored(exited.message);
         }
         if (restored.manualRequired || !restored.newBinding) {
           const handle = restored.binding?.handle ?? restored.blockers[0]?.handle;
           throw new Error(`${original.peer}: original session restoration needs manual verification${restored.blockers[0] ? ` (${restored.blockers[0].message})` : ""}; wait until it attaches, or end that launch and close its terminal${handle ? ` ${handle}` : ""} first`);
         }
         // The daemon's id is authoritative: a fresh session is recorded under the id it reports once attached.
-        progress.terminals[key] = restored.newBinding; save();
-        const sessionId = await awaitAttachment(planned, progress, op, restored.newBinding, save, fresh() ? null : restored.newBinding.sessionId);
-        progress.terminals[key] = { ...restored.newBinding, sessionId }; save();
+        const awaiting = { state: "waiting", origin: "pending", binding: restored.newBinding } satisfies RestoreReceipt;
+        progress.terminals[key] = awaiting; save();
+        const sessionId = await awaitBound(awaiting, fresh() ? null : restored.newBinding.sessionId);
+        await settle(sessionId, awaiting.binding);
       }
       if (group === "claude") for (const id of planned.reconnectOnly ?? []) {
         // #206: nothing is launched for an unmanaged session; its plugin reconnects by itself, within a bound.
-        const deadline = now() + RECONNECT_WAIT_MS;
+        const deadline = now() + UNMANAGED_RECONNECT_WAIT_MS;
         while (!(await inspectRecovery(planned.project)).peers.some((p) => p.id === id && p.state !== "offline")) {
-          if (now() >= deadline) throw new Error(`${id}: the unmanaged session did not reconnect within ${RECONNECT_WAIT_MS / 1000} s; if that Claude session was closed, start Claude in the project again (its plugin reconnects) first`);
+          if (now() >= deadline) throw new Error(`${id}: the unmanaged session did not reconnect within ${UNMANAGED_RECONNECT_WAIT_MS / 1000} s; if that Claude session was closed, start Claude in the project again (its plugin reconnects) first`);
           await sleep(1000);
         }
       }
@@ -771,9 +875,9 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
         if (old.id === "pi" && (peer.sessionId !== old.sessionId || (old.sessionFile && peer.sessionFile !== old.sessionFile))) throw new Error("Pi resumed a different conversation");
       }
       for (const original of planned.terminals as TerminalBinding[]) {
-        const saved = progress.terminals[`restored:${original.peer}`];
-        if (!saved || typeof saved === "string") throw new Error(`${original.peer}: restored terminal outcome is not recorded; manual-required`);
-        await revalidateTerminal(planned, progress, saved as TerminalBinding, true);
+        const saved = decodeRestoreReceipt(progress.terminals[`restored:${original.peer}`]);
+        if (saved?.state !== "done") throw new Error(`${original.peer}: restored terminal outcome is not recorded; manual-required`);
+        await revalidateTerminal(planned, progress, saved.binding, true);
       }
       const snapshot = await rpc(planned.project, { t: "ui_snapshot", after: 0 });
       if (snapshot.ok === false) throw new Error("dashboard snapshot readback failed");

package/src/cli/upgrade.ts

@@ -1,3 +1,4 @@
+import { canChooseFresh, decodeRestoreReceipt, restoreReceiptStatus } from "./upgrade-restore-state.ts";
 import { databaseBusySite, isDatabaseBusy, RecoveryDatabaseBusy } from "../hub/recovery-error.ts";
 import { Database } from "bun:sqlite";
 import { createHash, randomUUID } from "node:crypto";
@@ -87,6 +88,8 @@ export interface RecoveryOperation {
   sourceRoot: string;
   targetRoot?: string;
   targetDigest?: string;
+  /** Verified capability evidence bound to targetRoot/targetDigest; absence is legacy unknown, not an empty list. */
+  targetFeatures?: readonly string[];
   projects: ProjectProgress[];
   pluginInstalled?: boolean;
   globalInstalled?: boolean;
@@ -127,7 +130,7 @@ export function coordinatorRestarts(op: { sourceRoot?: string }): boolean {
 
 /**
  * #225: record the restart of a target that stopped after it started, before the new one starts: the dead instance and
- * the time stay for status and audit. Each done `restored:<peer>` binding of the dead instance is retired
+ * the time stay for status and audit. Each done or waiting `restored:<peer>` binding of the dead instance is retired
  * (`retired:<peer>`): restore closes its terminal (or finds it absent) before that peer is relaunched, and never
  * revalidates it against the new instance. `closed:<peer>`, `commitSent`, `fresh`, waivers and the install flags stay.
  * The project goes back to `stopped`, whose path starts the target from this operation's snapshot.
@@ -135,9 +138,11 @@ export function coordinatorRestarts(op: { sourceRoot?: string }): boolean {
 export function recordRestart(progress: ProjectProgress, at: number): void {
   (progress.restarts ??= []).push({ instanceId: progress.instanceId ?? "unknown", at });
   for (const [key, value] of Object.entries(progress.terminals)) {
-    if (!key.startsWith("restored:") || !value || typeof value !== "object") continue;
+    if (!key.startsWith("restored:")) continue;
+    const receipt = decodeRestoreReceipt(value);
+    if (receipt?.state !== "done" && receipt?.state !== "waiting") continue;
     const peer = key.slice("restored:".length);
-    progress.terminals[`retired:${peer}`] = value;
+    progress.terminals[`retired:${peer}`] = receipt.binding;
     delete progress.terminals[`closedRetired:${peer}`]; // a newer binding to close: its own receipt starts over
     delete progress.terminals[key];
   }
@@ -280,7 +285,7 @@ export async function liveProjects(op: RecoveryOperation, inspect: (project: Pro
  */
 function failedPeers(op: RecoveryOperation, progress: ProjectProgress, live: Record<string, Inspection | undefined> = {}): string[] {
   const planned = op.plan.projects.find((p) => p.project.id === progress.id);
-  return Object.entries(progress.terminals).filter(([key, value]) => key.startsWith("restored:") && value === "failed").map(([key]) => key.slice("restored:".length))
+  return Object.entries(progress.terminals).filter(([key, value]) => key.startsWith("restored:") && canChooseFresh(value)).map(([key]) => key.slice("restored:".length))
     .filter((peer) => peer !== "pi" && !progress.fresh?.[peer] && !planned?.freshStart?.includes(peer) && !plannedAttached(op, progress, peer, live));
 }
 
@@ -341,17 +346,18 @@ function abandoned(op: RecoveryOperation): boolean {
 
 function validateReceipt(id: string, op: RecoveryOperation): void {
   if (!(op.schema === 1 || (op.schema === 2 && op.disposition)) || op.id !== id) throw new Error("unsupported operation receipt");
+  if (op.targetFeatures !== undefined && (!Array.isArray(op.targetFeatures) || !op.targetFeatures.every(feature => typeof feature === "string" && feature.length > 0) || !op.targetRoot || !op.targetDigest)) throw new Error("invalid target features");
   const { fingerprint, ...reviewed } = op.plan;
   if (fingerprint !== planFingerprint(reviewed) || op.projects.length !== op.plan.projects.length ||
       op.projects.some((p, i) => p.id !== op.plan.projects[i]?.project.id)) throw new Error("reviewed operation plan or project scope changed");
 }
 
 export interface RecoveryDriver {
-  stage(op: RecoveryOperation): Promise<{ root: string; digest: string }>;
-  inspect(project: Project): Promise<Inspection>;
-  prepare(project: Project, operation: string, instance: string): Promise<void>;
+  stage(op: RecoveryOperation): Promise<{ root: string; digest: string; targetFeatures?: readonly string[] }>;
+  inspect(project: Project, targetFeatures?: readonly string[]): Promise<Inspection>;
+  prepare(project: Project, operation: string, instance: string, targetFeatures?: readonly string[]): Promise<void>;
   abort(project: Project, operation: string, instance: string): Promise<void>;
-  commit(project: Project, operation: string, instance: string): Promise<void>;
+  commit(project: Project, operation: string, instance: string, targetFeatures?: readonly string[]): Promise<void>;
   closeTerminals(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, save: () => void): Promise<void>;
   start(project: Project, op: RecoveryOperation): Promise<void>;
   restore(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, group: "native" | "claude", save: () => void): Promise<void>;
@@ -383,6 +389,7 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
       return op;
     }
   } catch (error) { releaseRunner(); throw error; }
+  let targetFeatures: readonly string[] = [];
   const save = () => { op.updatedAt = driver.now(); writeOperation(id, op, home); };
   const step = (value: string) => { op.step = value; save(); };
   // #225: one restart per project per resume, so a target that keeps dying never spins unattended.
@@ -422,7 +429,7 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
   // Prepare, or after an expired lease re-prepare, the planned source and wait until it is quiet.
   const prepareSource = async (planned: PlannedProject, progress: ProjectProgress, again = false) => {
     const project = planned.project, instance = planned.source.instanceId!;
-    await driver.prepare(project, id, instance);
+    await driver.prepare(project, id, instance, targetFeatures);
     const deadline = driver.now() + idleTimeoutMs;
     for (;;) {
       const live = await driver.inspect(project);
@@ -452,13 +459,16 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
     step("stage");
     const target = await driver.stage(op);
     if (op.targetDigest && (target.root !== op.targetRoot || target.digest !== op.targetDigest)) throw new FinalRefusal("staged target changed since operation started");
-    op.targetRoot = target.root; op.targetDigest = target.digest; save();
+    targetFeatures = target.targetFeatures ?? op.targetFeatures ?? [];
+    op.targetRoot = target.root; op.targetDigest = target.digest;
+    if (target.targetFeatures !== undefined) op.targetFeatures = [...target.targetFeatures];
+    save();
 
-    // Check every untouched source before stopping even the first project.
+    // Pending sources receive the verified target features before holds or terminal/runtime effects.
     for (let i = 0; i < op.projects.length; i++) {
       const progress = op.projects[i]!, planned = op.plan.projects[i]!;
       if (progress.phase !== "pending") continue;
-      const live = await driver.inspect(planned.project);
+      const live = await driver.inspect(planned.project, targetFeatures);
       if (!["running", "stopped", "missing"].includes(live.state)) throw new Error(`${planned.project.id}: the source reads as ${live.state}; wait until it answers`);
       if (live.state !== "running" || live.instanceId !== planned.source.instanceId || live.version !== planned.source.version) {
         throw new Error(`${planned.project.id}: source runtime changed`);
@@ -486,7 +496,14 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
       }
       if (progress.phase === "prepared") {
         step(`commit:${project.id}`);
-        let live = await driver.inspect(project);
+        let live: Inspection;
+        try { live = await driver.inspect(project); }
+        catch (error) {
+          if (!progress.commitSent || !isDatabaseBusy(error)) throw error;
+          live = { state: "unavailable", peers: [], blockers: ["database_busy"] };
+        }
+        const resumedShutdown = progress.commitSent && live.state === "unavailable";
+        let awaitShutdown = resumedShutdown;
         if (live.state === "running" && live.instanceId === planned.source.instanceId) {
           if (live.recovery?.operationId !== id || !live.recovery.ready) {
             const other = live.recovery?.operationId;
@@ -503,8 +520,22 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
           await driver.closeTerminals(planned, progress, op, save);
           // Durable, unlike `step`: from here the source may have committed, which status and abort must not forget.
           progress.commitSent = true; save();
-          await driver.commit(project, id, planned.source.instanceId!);
+          await driver.commit(project, id, planned.source.instanceId!, targetFeatures);
+          awaitShutdown = true;
+        } else if (live.state === "running") {
+          throw new Error(`${project.id}: the source daemon was replaced by instance ${live.instanceId ?? "unknown"}; refusing to prepare, close or stop it (ending this operation leaves that daemon running)`);
+        } else if (live.state === "stopped" && !progress.commitSent && (!op.sourceRoot || coordinatorCurrent(op.sourceRoot))) {
+          // #215: this coordinator records a commit request before sending it, so a stopped source with none crashed while
+          // prepared: no snapshot was committed and there is nothing to start from. The phase stays `prepared`.
+          throw new Error(`${project.id}: the source stopped while prepared and no commit was requested, so there is nothing to restore`);
+        }
+        if (awaitShutdown) {
           const deadline = driver.now() + 30_000;
+          // An already-sent commit can be between its final reply and the source closing its control server.
+          if (resumedShutdown) {
+            if (live.instanceId && live.instanceId !== planned.source.instanceId) throw new Error(`${project.id}: another daemon started during shutdown`);
+            await driver.sleep(100);
+          }
           do {
             try { live = await driver.inspect(project); }
             catch (error) {
@@ -517,12 +548,6 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
             if (live.instanceId && live.instanceId !== planned.source.instanceId) throw new Error(`${project.id}: another daemon started during shutdown`);
             await driver.sleep(100);
           } while (driver.now() < deadline);
-        } else if (live.state === "running") {
-          throw new Error(`${project.id}: the source daemon was replaced by instance ${live.instanceId ?? "unknown"}; refusing to prepare, close or stop it (ending this operation leaves that daemon running)`);
-        } else if (live.state === "stopped" && !progress.commitSent && (!op.sourceRoot || coordinatorCurrent(op.sourceRoot))) {
-          // #215: this coordinator records a commit request before sending it, so a stopped source with none crashed while
-          // prepared: no snapshot was committed and there is nothing to start from. The phase stays `prepared`.
-          throw new Error(`${project.id}: the source stopped while prepared and no commit was requested, so there is nothing to restore`);
         }
         if (live.state === "missing") throw new Error(`${project.id}: the project directory is missing`);
         if (live.state !== "stopped") throw new Error(`${project.id}: old shutdown is not verified (the source reads as ${live.state}); not starting a second daemon; check ${recoveryCommand(op, "status")} once it answers`);
@@ -584,12 +609,12 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
 
 /** Receipts and runner state only (#215): ids, phases and effects, never task or message text. */
 export function publicOperation(op: RecoveryOperation, runnerPid?: number | "unknown", live: Record<string, Inspection | undefined> = {}) {
-  const effect = (value: unknown) => value === "pending" || value === "failed" ? value : "done";
+  const effect = (key: string, value: unknown) => key.startsWith("restored:") ? restoreReceiptStatus(value) : value === "pending" || value === "failed" ? value : "done";
   const next = nextActions(op, runnerPid, live);
   return { id: op.id, phase: op.phase, step: op.step, version: op.plan.version, updatedAt: op.updatedAt,
     runner: runnerPid === "unknown" ? { state: "unknown" } : runnerPid ? { state: "running", pid: runnerPid } : { state: "none" },
     ...(op.phase === "running" && !runnerPid ? { stale: "the receipt says running but no runner holds the operation: its last runner stopped mid-step" } : {}),
-    projects: op.projects.map((p) => ({ id: p.id, phase: p.phase, ...(Object.keys(p.terminals).length ? { effects: Object.fromEntries(Object.entries(p.terminals).map(([key, value]) => [key, effect(value)])) } : {}),
+    projects: op.projects.map((p) => ({ id: p.id, phase: p.phase, ...(Object.keys(p.terminals).length ? { effects: Object.fromEntries(Object.entries(p.terminals).map(([key, value]) => [key, effect(key, value)])) } : {}),
       ...(p.fresh ? { lostContinuity: Object.fromEntries(Object.entries(p.fresh).map(([peer, f]) => [peer, f.lost])) } : {}),
       ...(p.restarts?.length ? { restarts: p.restarts.length } : {}) })),
     pluginInstalled: !!op.pluginInstalled, globalInstalled: !!op.globalInstalled,
@@ -669,10 +694,10 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
       const peer = choice.fresh;
       // Per project: a planned fresh start has nothing to lose, so only the other failed projects take the choice.
       const failed = op.projects.filter((p) => failedPeers(op, p, live).includes(peer));
-      if (!failed.length && op.projects.some((p) => p.terminals[`restored:${peer}`] === "failed" && op.plan.projects.find((planned) => planned.project.id === p.id)?.freshStart?.includes(peer))) {
+      if (!failed.length && op.projects.some((p) => canChooseFresh(p.terminals[`restored:${peer}`]) && op.plan.projects.find((planned) => planned.project.id === p.id)?.freshStart?.includes(peer))) {
         throw new Error(`${peer}: its plan already restarts it as a new session (no rollout and no turn, nothing to lose), so there is no conversation to record as lost; ${next}`);
       }
-      if (!failed.length && op.projects.some((p) => p.terminals[`restored:${peer}`] === "failed" && plannedAttached(op, p, peer, live))) {
+      if (!failed.length && op.projects.some((p) => canChooseFresh(p.terminals[`restored:${peer}`]) && plannedAttached(op, p, peer, live))) {
         throw new Error(`${peer}: its planned session is attached again, so nothing was lost; resume records it as restored; ${next}`);
       }
       if (!failed.length) throw new Error(`${peer}: no failed restoration of it is open to a fresh session (none failed, or one is already chosen); --fresh-session applies only then (${recoveryCommand(op, "status")})`);