@staix/agent-hub 0.12.31 → 0.12.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/CHANGELOG.md +13 -1
  2. package/README.md +13 -7
  3. package/docs/agent-notes/adapters.md +7 -4
  4. package/docs/agent-notes/bus.md +1 -0
  5. package/docs/agent-notes/daemon.md +10 -7
  6. package/docs/agent-notes/local-worker.md +1 -1
  7. package/docs/agent-notes/tasks.md +3 -2
  8. package/docs/agent-notes/tests.md +1 -0
  9. package/docs/cooperbench.md +1 -1
  10. package/docs/gsd-collaboration-analysis.md +1 -1
  11. package/docs/memory.md +1 -1
  12. package/docs/operations.md +114 -22
  13. package/docs/quickstart.md +10 -6
  14. package/docs/security.md +12 -8
  15. package/docs/smoke.md +34 -0
  16. package/docs/specs/2026-09-19-agent-hub-design.md +204 -0
  17. package/docs/specs/2026-09-20-upgrade-recovery-design.md +191 -0
  18. package/docs/verification/2026-10-18-0.12.32.md +54 -0
  19. package/docs/verified.json +20 -15
  20. package/package.json +2 -2
  21. package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
  22. package/plugins/agent-hub/server.js +1296 -26
  23. package/src/adapters/acp.ts +5 -6
  24. package/src/adapters/definitions/builtin-metadata.ts +121 -0
  25. package/src/adapters/definitions/kimi-native.ts +4 -0
  26. package/src/adapters/definitions/qwen-native.ts +24 -0
  27. package/src/adapters/definitions/qwen.ts +71 -0
  28. package/src/adapters/pi.ts +6 -2
  29. package/src/adapters/registry-types.ts +174 -0
  30. package/src/adapters/registry.ts +36 -0
  31. package/src/cli/console-state.ts +74 -5
  32. package/src/cli/console.ts +35 -2
  33. package/src/cli/help.ts +3 -1
  34. package/src/cli/identity.ts +13 -3
  35. package/src/cli/launch.ts +1 -6
  36. package/src/cli/main.ts +200 -39
  37. package/src/cli/output.ts +2 -0
  38. package/src/cli/preview.ts +8 -5
  39. package/src/cli/recovery-guidance.ts +137 -0
  40. package/src/cli/recovery-presentation-options.ts +8 -0
  41. package/src/cli/recovery-session-file.ts +18 -0
  42. package/src/cli/terminal-recovery.ts +48 -0
  43. package/src/cli/upgrade-interactive.ts +129 -75
  44. package/src/cli/upgrade-labels.ts +30 -0
  45. package/src/cli/upgrade-presentation-data.ts +41 -0
  46. package/src/cli/upgrade-presentation-flags.ts +31 -0
  47. package/src/cli/upgrade-presentation.ts +217 -0
  48. package/src/cli/upgrade-restore-state.ts +6 -2
  49. package/src/cli/upgrade-runtime.ts +345 -194
  50. package/src/cli/upgrade.ts +276 -25
  51. package/src/hub/bus.ts +11 -0
  52. package/src/hub/cohorts.ts +2 -2
  53. package/src/hub/config-apply.ts +242 -0
  54. package/src/hub/config-reload.ts +120 -0
  55. package/src/hub/config-trust.ts +11 -2
  56. package/src/hub/crash.ts +8 -6
  57. package/src/hub/daemon.ts +559 -170
  58. package/src/hub/events.ts +2 -2
  59. package/src/hub/manager.ts +1 -1
  60. package/src/hub/peer-catalog.ts +10 -13
  61. package/src/hub/peers.ts +12 -2
  62. package/src/hub/permission-mode.ts +5 -3
  63. package/src/hub/recovery-continuity.ts +152 -0
  64. package/src/hub/recovery-store.ts +10 -4
  65. package/src/hub/sdlc.ts +5 -1
  66. package/src/hub/settings.ts +50 -36
  67. package/src/hub/settled-roles.ts +35 -0
  68. package/src/hub/supervision.ts +3 -2
  69. package/src/hub/tasks.ts +28 -8
  70. package/src/hub/ui.ts +1 -1
  71. package/src/memory/brief.ts +6 -5
  72. package/src/pi/session.ts +23 -4
  73. package/src/ui/index.html +23 -7
@@ -1,21 +1,25 @@
1
+ import { checkedSessionPath, recordedFileState } from "./recovery-session-file.ts";
2
+ import { resolveAcceptedBinding } from "./recovery-guidance.ts";
3
+ import { definitionOf } from "../adapters/registry.ts";
1
4
  import { qwenSessionId } from "../hub/crash.ts";
2
5
  import { isDatabaseBusy, isDatabaseBusyReply, RecoveryDatabaseBusy } from "../hub/recovery-error.ts";
3
6
  import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync } from "node:fs";
4
- import { homedir } from "node:os";
5
7
  import { basename, join, resolve } from "node:path";
6
- import { randomUUID } from "node:crypto";
8
+ import { createHash, randomUUID } from "node:crypto";
7
9
  import { ControlClient, PROTOCOL, RECOVERY_SOURCE_PROTOCOLS, readControl } from "../hub/control-client.ts";
8
10
  import { awaitStopped, inspectProject } from "../hub/lifecycle.ts";
9
11
  import type { Project } from "../hub/registry.ts";
10
12
  import { hubHome } from "../hub/project.ts";
11
13
  import { packageDigest, preserveRuntimeDependencies, registryRelease, runCommand, stageRelease, verifyPackage, type RunCommand } from "./recovery-package.ts";
12
- import { inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
13
- import { FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
14
+ import { inspectSavedTerminal, inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
15
+ import { ReattachmentRequired, MissingRecordedSession, FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
14
16
  import { readEvents } from "../hub/events.ts";
15
17
  import { refreshManager } from "../hub/manager.ts";
16
18
  import { abandonRestartSnapshot, readRestartSnapshot, waiveRecoveryPeers } from "../hub/restart.ts";
19
+ import { readRecoveryContinuity, retireRecoveryContinuity, writeRecoveryContinuity } from "../hub/recovery-continuity.ts";
17
20
  import { recoveryLock } from "../hub/recovery-store.ts";
18
21
  import { decodeRestoreReceipt, restoreTransition, type RestoreReceipt, type RestoreEvidence, type RestoreTransition } from "./upgrade-restore-state.ts";
22
+ import { RecoveryPreservationError, RecoveryPresentationError, type RecoveryNativeCause } from "./upgrade-presentation-data.ts";
19
23
 
20
24
  /** An unmanaged Claude's plugin retries with a backoff of at most 30 s; three of those bound the reconnect wait (#206). */
21
25
  const UNMANAGED_RECONNECT_WAIT_MS = 90_000;
@@ -34,11 +38,10 @@ const orcaExecutable = () => process.env.ORCA_CLI_COMMAND || (process.env.ORCA_D
34
38
  * dir, creating the project directory only with the first persisted turn; the slug replaces
35
39
  * every non-alphanumeric character of the project root with '-'.
36
40
  */
37
- function claudeTranscriptExists(binding: TerminalBinding): boolean {
38
- const config = binding.launch.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude");
39
- const slug = binding.projectRoot.replace(/[^a-zA-Z0-9]/g, "-");
40
- return existsSync(join(config, "projects", slug, `${binding.sessionId}.jsonl`));
41
+ function claudeTranscriptState(binding: TerminalBinding): "found" | "missing" | "unknown" {
42
+ return recordedFileState(checkedSessionPath(binding, "transcript"));
41
43
  }
44
+ function claudeTranscriptExists(binding: TerminalBinding): boolean { return claudeTranscriptState(binding) === "found"; }
42
45
 
43
46
  /**
44
47
  * #215: `codex resume <id>` reads the thread's rollout from the store its launcher uses (the captured CODEX_HOME, else
@@ -46,7 +49,7 @@ function claudeTranscriptExists(binding: TerminalBinding): boolean {
46
49
  * ponytail: a `.jsonl` under sessions/ whose name carries the thread id is the evidence; ask Codex itself once it offers
47
50
  * a supported resumability query, or when its store layout changes.
48
51
  */
49
- const codexSessions = (binding: TerminalBinding) => join(binding.launch.env.CODEX_HOME ?? join(homedir(), ".codex"), "sessions");
52
+ const codexSessions = (binding: TerminalBinding) => checkedSessionPath(binding, "rollout");
50
53
  function codexTranscript(binding: TerminalBinding): "found" | "missing" | "unknown" {
51
54
  try { return readdirSync(codexSessions(binding), { recursive: true }).some((name) => String(name).endsWith(".jsonl") && basename(String(name)).includes(binding.sessionId)) ? "found" : "missing"; }
52
55
  catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "missing" : "unknown"; } // no store has no rollout; an unreadable one says nothing
@@ -246,6 +249,7 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
246
249
  const reconnectOnly: string[] = [];
247
250
  for (const peer of source.peers) {
248
251
  if (peer.state === "offline") continue;
252
+ const adapter = definitionOf(peer.id);
249
253
  if (peer.id === "kimi" && peer.args?.mode === "tui") {
250
254
  blockers.push("kimi: its pull-only terminal has no automatic restore; end it in that terminal, then make a new recovery plan (its session is not resumed by the hub)");
251
255
  continue;
@@ -282,12 +286,13 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
282
286
  const session = peer.id === "codex" ? peer.threadId : peer.sessionId;
283
287
  if (!session) blockers.push(`${peer.id}: original conversation ID is unknown; manual-required; next action: reconnect the original native session and make a new recovery plan`);
284
288
  else sessions[peer.id] = session;
285
- } else if (peer.id === "qwen" || peer.id === "jcode") {
286
- if (!qwenSessionId(peer.sessionId)) blockers.push(`${peer.id}: saved ACP session identity unavailable; end the peer and make a new recovery plan`);
289
+ } else if (adapter?.session.restoreOnStart || peer.id === "jcode") {
290
+ const session = adapter?.session.crashIdentity ? adapter.session.crashIdentity(peer).sessionId : qwenSessionId(peer.sessionId);
291
+ if (!session) blockers.push(`${peer.id}: saved ACP session identity unavailable; end the peer and make a new recovery plan`);
287
292
  } else if (peer.id === "pi" && peer.args?.mode === "tui") {
288
293
  if (!peer.sessionId) blockers.push("pi: original session ID is unknown; manual-required; next action: reconnect the original Pi session and make a new recovery plan");
289
294
  else sessions.pi = { sessionId: peer.sessionId, ...(peer.sessionFile ? { sessionFile: peer.sessionFile } : {}), ...(peer.args.backend ? { backend: peer.args.backend } : {}), ...(peer.args.model ? { model: peer.args.model } : {}) };
290
- } else if (peer.id !== "kimi" && peer.id !== "qwen" && peer.id !== "local" && peer.id !== "pi") blockers.push(`${peer.id}: no automatic recovery adapter`);
295
+ } else if (peer.id !== "kimi" && peer.id !== "local" && peer.id !== "pi") blockers.push(`${peer.id}: no automatic recovery adapter`);
291
296
  }
292
297
  const terminals = await inspectTerminals(project.root, sessions, {
293
298
  ...terminalOptions(run), stateDir: project.stateDir, instanceId: source.instanceId,
@@ -364,20 +369,23 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
364
369
  const protocol = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]) ? sourceProtocol : PROTOCOL;
365
370
  return rpc(project, { t: "recovery", op, operationId: id, expectedInstanceId: instance, ...(targetFeatures === undefined ? {} : { targetFeatures }) }, protocol).then(() => {});
366
371
  };
372
+ const nativeFailure = (planned: PlannedProject, peer: string, reason: RecoveryNativeCause["reason"], message: string,
373
+ details: Omit<RecoveryNativeCause, "kind" | "reason" | "projectId" | "peer"> = {}) =>
374
+ new RecoveryPresentationError(message, { kind: "native_recovery", reason, projectId: planned.project.id, peer, ...details });
367
375
  const revalidateTerminal = async (planned: PlannedProject, progress: ProjectProgress, saved: TerminalBinding, exact: boolean): Promise<TerminalBinding> => {
368
376
  const options = { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId };
369
- const found = await inspectTerminals(planned.project.root, { [saved.peer]: saved.sessionId }, options);
377
+ const found = await inspectTerminals(planned.project.root, { [saved.peer]: { sessionId: saved.sessionId, sessionFile: saved.sessionFile, backend: saved.backend, model: saved.model } }, options);
370
378
  const current = found.byPeer[saved.peer];
371
379
  if (found.manualRequired || !current || (exact && (current.handle !== saved.handle || current.incarnationId !== saved.incarnationId || current.worktreeId !== saved.worktreeId || current.projectRoot !== saved.projectRoot))) {
372
- throw new Error(`${saved.peer}: saved terminal identity changed or is not verified; manual-required`);
380
+ throw nativeFailure(planned, saved.peer, "terminal_unverified", `${saved.peer}: saved terminal identity changed or is not verified; manual-required`, { handle: saved.handle });
373
381
  }
374
382
  const idle = await waitForIdle(exact ? saved : current, 120_000, options);
375
- if (!idle.satisfied) throw new Error(`${saved.peer}: terminal is not idle; manual-required`);
383
+ if (!idle.satisfied) throw nativeFailure(planned, saved.peer, "terminal_unverified", `${saved.peer}: terminal is not idle; manual-required`, { handle: saved.handle });
376
384
  const observed = await inspectRecovery(planned.project);
377
- if (observed.instanceId !== progress.instanceId) throw new Error(`${saved.peer}: daemon instance changed during terminal revalidation`);
385
+ if (observed.instanceId !== progress.instanceId) throw nativeFailure(planned, saved.peer, "target_unavailable", `${saved.peer}: daemon instance changed during terminal revalidation`, { handle: saved.handle, targetState: observed.state });
378
386
  const peer = observed.peers.find((item) => item.id === saved.peer);
379
387
  const session = saved.peer === "codex" ? peer?.threadId : peer?.sessionId;
380
- if (session !== saved.sessionId) throw new Error(`${saved.peer}: daemon session changed during terminal revalidation; manual-required`);
388
+ if (session !== saved.sessionId || saved.peer === "pi" && progress.reattachments?.pi && saved.sessionFile && peer?.sessionFile !== saved.sessionFile) throw nativeFailure(planned, saved.peer, "session_changed", `${saved.peer}: daemon session changed during terminal revalidation; manual-required`, { handle: saved.handle, expectedSession: saved.sessionId, ...(session === undefined ? {} : { observedSession: session }) });
381
389
  return current;
382
390
  };
383
391
  /** Whether Orca's inventory lists the terminal; a failed, not-ok, truncated or non-JSON answer shows nothing. */
@@ -430,6 +438,31 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
430
438
  if (attached) return { state: "unknown", idless: true, why: `a ${peer} session is attached to the target without a session id`, step: `wait until it reports one, or end that ${peer} session` };
431
439
  return { state: "gone", ...(endedTerminal ? { terminal: endedTerminal } : {}) };
432
440
  };
441
+ /** Only independently confirmed terminal loss plus absent sessions and launchers can arm a repair. */
442
+ const detached = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation,
443
+ binding: TerminalBinding, save?: () => void): Promise<void> => {
444
+ const evidence = await peerEvidence(planned, progress, binding.peer);
445
+ if (evidence.state !== "gone") return;
446
+ const terminal = await inspectSavedTerminal(binding, terminalOptions(run));
447
+ const decision = restoreTransition({ state: "done", binding }, {
448
+ target: "running", launcher: "gone", session: "none", allowOther: false,
449
+ terminal: terminal.state === "missing" || terminal.state === "orphaned" ? "gone" : terminal.state === "present" ? "present" : terminal.state === "changed" ? "changed" : "unknown",
450
+ });
451
+ if (decision.action !== "reattach") return;
452
+ const current = await inspectRecovery(planned.project);
453
+ if (current.state !== "running" || current.instanceId !== progress.instanceId || current.version !== op.plan.version || current.recovery?.operationId !== op.id ||
454
+ (await peerEvidence(planned, progress, binding.peer)).state !== "gone") return;
455
+ const existing = progress.reattachments?.[binding.peer];
456
+ const why = `${binding.peer}: detached after restore; saved terminal ${binding.handle} is ${terminal.state}; recorded session ${binding.sessionId}${binding.sessionFile ? ` (${binding.sessionFile})` : ""}`;
457
+ if (existing) throw new ReattachmentRequired(planned.project.id, binding.peer, `${why}; the one reattachment was already attempted; no second terminal will be created`);
458
+ if (!save) throw new Error(`${why}; a durable coordinator save is required before reattachment`);
459
+ (progress.reattachments ??= {})[binding.peer] = { state: "ready", at: now(), cause: "detached after restore", binding };
460
+ (progress.repairedSessions ??= {})[binding.peer] = true;
461
+ progress.terminals[`restored:${binding.peer}`] = decision.next;
462
+ progress.phase = "started";
463
+ save();
464
+ throw new ReattachmentRequired(planned.project.id, binding.peer, `${why}; resume or use the operation's person-only reattach command`);
465
+ };
433
466
  /**
434
467
  * #225: the terminal of a done restoration of a target instance that died is closed under the close receipt rules
435
468
  * (`closedRetired:<peer>` `pending` until Orca no longer lists it; an unreadable or truncated inventory is unknown and
@@ -453,21 +486,32 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
453
486
  };
454
487
  /** #414: an idle terminal is not an authenticated peer. Keep its receipt while waiting, including on resume. */
455
488
  const awaitAttachment = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, waiting: Extract<RestoreReceipt, { state: "waiting" }>, save: () => void, expectedSession: string | null = waiting.binding.sessionId): Promise<string> => {
489
+ const saveTiming = () => {
490
+ try { save(); }
491
+ catch { console.warn("Recovery timing observation could not be saved; recovery continues"); }
492
+ };
456
493
  const binding = waiting.binding;
457
494
  const matchesTerminal = (terminal: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId">) => terminal.handle === binding.handle && terminal.incarnationId === binding.incarnationId && terminal.worktreeId === binding.worktreeId;
458
- const deadline = now() + RECONNECT_WAIT_MS;
495
+ const startedAt = now(), deadline = startedAt + RECONNECT_WAIT_MS;
496
+ const waitStep = op.step;
497
+ if (waitStep) {
498
+ (op.waits ??= {})[waitStep] = { waitStartedAt: new Date(startedAt).toISOString(), deadlineAt: new Date(deadline).toISOString() };
499
+ saveTiming();
500
+ }
501
+ let waitFailed = false;
502
+ try {
459
503
  let live: Inspection;
460
504
  for (;;) {
461
505
  live = await inspectRecovery(planned.project);
462
- if (live.state !== "running") throw new Error(`${binding.peer}: target hub reads as ${live.state} while awaiting attachment; saved terminal retained`);
463
- if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw new Error(`${binding.peer}: target identity changed while awaiting attachment`);
506
+ if (live.state !== "running") throw nativeFailure(planned, binding.peer, "target_unavailable", `${binding.peer}: target hub reads as ${live.state} while awaiting attachment; saved terminal retained`, { handle: binding.handle, targetState: live.state });
507
+ if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw nativeFailure(planned, binding.peer, "target_unavailable", `${binding.peer}: target identity changed while awaiting attachment`, { handle: binding.handle, targetState: live.state });
464
508
  const peer = live.peers.find((p) => p.id === binding.peer && p.state !== "offline");
465
509
  const session = binding.peer === "codex" ? peer?.threadId : peer?.sessionId;
466
510
  const changed = !!session && expectedSession !== null && session !== expectedSession;
467
511
  // #64: only the original zero-turn Claude may change identity without an operator choice. Re-read persistence
468
512
  // at attachment, not at the plan or wait start; an already accepted replacement is bound to its own id.
469
- const zeroTurn = changed && binding.peer === "claude" && (planned.terminals as TerminalBinding[]).some((p) => p.peer === "claude" && p.sessionId === binding.sessionId) && !claudeTranscriptExists(binding);
470
- if (changed && !zeroTurn) throw new Error(`${binding.peer}: a different conversation attached while awaiting ${binding.sessionId}`);
513
+ const zeroTurn = changed && !progress.reattachments?.[binding.peer] && !progress.repairedSessions?.[binding.peer] && binding.peer === "claude" && (planned.terminals as TerminalBinding[]).some((p) => p.peer === "claude" && p.sessionId === binding.sessionId) && !claudeTranscriptExists(binding);
514
+ if (changed && !zeroTurn) throw nativeFailure(planned, binding.peer, "session_changed", `${binding.peer}: a different conversation attached while awaiting ${binding.sessionId}`, { handle: binding.handle, expectedSession: expectedSession!, observedSession: session! });
471
515
  if (now() > deadline) break;
472
516
  const launcher = await launcherOf(binding.peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId });
473
517
  const sameTerminal = launcher?.record && matchesTerminal(launcher.record);
@@ -497,9 +541,9 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
497
541
  }
498
542
  if (decision.action === "fail" || decision.action === "create") throw new AttachmentLauncherGone(decision);
499
543
  if (decision.action === "block") {
500
- if (launchState === "replaced") throw new Error(`${binding.peer}: saved terminal identity changed while awaiting attachment`);
501
- if (launchState === "older-instance") throw new Error(`${binding.peer}: an older target's launcher in terminal ${olderHandle ?? "unknown"} may still run; saved terminal retained; no terminal was created`);
502
- throw new Error(`${binding.peer}: the chosen restoration launcher's identity cannot be verified (${launchState}); saved terminal retained; no terminal was created`);
544
+ if (launchState === "replaced") throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: saved terminal identity changed while awaiting attachment`, { handle: binding.handle, launcherState: launchState });
545
+ if (launchState === "older-instance") throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: an older target's launcher in terminal ${olderHandle ?? "unknown"} may still run; saved terminal retained; no terminal was created`, { ...(olderHandle ? { handle: olderHandle } : {}), launcherState: launchState });
546
+ throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: the chosen restoration launcher's identity cannot be verified (${launchState}); saved terminal retained; no terminal was created`, { handle: binding.handle, launcherState: launchState });
503
547
  }
504
548
  if (now() >= deadline) break;
505
549
  await sleep(Math.min(1000, deadline - now()));
@@ -520,10 +564,23 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
520
564
  hub: { state: live.state, ...(live.instanceId ? { instanceId: live.instanceId } : {}), ...(peer ? { peerState: peer.state } : {}), ...(session ? { sessionId: session } : {}) },
521
565
  lastHubConnectAttempt: "unknown" };
522
566
  save();
523
- throw new Error(`${binding.peer}: attachment not verified within ${RECONNECT_WAIT_MS / 1000} s; terminal ${binding.handle} retained; resume after it attaches (native-start diagnostic recorded; last hub connect attempt unknown)`);
567
+ throw nativeFailure(planned, binding.peer, "attachment_timeout", `${binding.peer}: attachment not verified within ${RECONNECT_WAIT_MS / 1000} s; terminal ${binding.handle} retained; resume after it attaches (native-start diagnostic recorded; last hub connect attempt unknown)`, { handle: binding.handle, timeoutMs: RECONNECT_WAIT_MS });
568
+ } catch (error) { waitFailed = true; throw error; }
569
+ finally {
570
+ const finish = () => {
571
+ if (!waitStep) return;
572
+ op.waits![waitStep] = { ...op.waits![waitStep]!, waitEndedAt: new Date(now()).toISOString() };
573
+ saveTiming();
574
+ };
575
+ // Metadata storage must not hide the attachment failure or change its recovery outcome.
576
+ if (waitFailed) { try { finish(); } catch { /* original attachment failure remains authoritative */ } }
577
+ else finish();
578
+ }
524
579
  };
525
580
  return {
581
+ ...(liveTestPeer === undefined ? {} : { liveTestPeer }),
526
582
  now, sleep, inspect: inspectRecovery,
583
+ recordedSessionState: async (binding, kind) => kind === "rollout" ? codexTranscript(binding) : claudeTranscriptState(binding),
527
584
  stage: async (op) => {
528
585
  if (packageDigest(op.sourceRoot, { ownDependencies: true }) !== op.plan.sourceDigest) throw new FinalRefusal("preserved source changed");
529
586
  const target = op.plan.kind === "restart" ? { root: op.sourceRoot, digest: verifyPackage(op.sourceRoot, op.plan.version, { ownDependencies: true }) }
@@ -549,7 +606,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
549
606
  const verified = { ...target, targetFeatures };
550
607
  // Existing peer/waiver compatibility checks remain properties of the already verified bytes.
551
608
  if (op.targetDigest && op.targetDigest === target.digest && op.targetRoot === target.root) return verified;
552
- if (op.plan.projects.some(p => p.source.peers.some(peer => peer.id === "qwen" && peer.state !== "offline"))) {
609
+ if (op.plan.projects.some(p => p.source.peers.some(peer => definitionOf(peer.id)?.session.restoreOnStart && peer.state !== "offline"))) {
553
610
  const catalog = join(target.root, "src/hub/peer-catalog.ts");
554
611
  const peers = existsSync(catalog) ? /export const OPTIONAL_PEERS\s*=\s*\[([^\]]*)\]/.exec(readFileSync(catalog, "utf8"))?.[1] : undefined;
555
612
  if (!peers || !/["']qwen["']/.test(peers)) throw new FinalRefusal("target does not support Qwen; end Qwen and make a new plan, or choose a supporting target; runtimes unchanged");
@@ -568,7 +625,15 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
568
625
  prepare: (p, id, instance, features) => control(p, "prepare", id, instance, features),
569
626
  abort: (p, id, instance) => control(p, "abort", id, instance),
570
627
  commit: (p, id, instance, features) => control(p, "commit", id, instance, features),
571
- release: (p, id, instance) => control(p, "release", id, instance),
628
+ release: async (p, id, instance) => {
629
+ // Snapshot before the request: a later writer must never be retired by this older release completion.
630
+ const previous = readRecoveryContinuity(p.stateDir, { projectId: p.id, projectRoot: p.root });
631
+ await control(p, "release", id, instance);
632
+ if (previous.state === "ready" && previous.bookmark.operationId !== id) {
633
+ try { retireRecoveryContinuity(p.stateDir, { projectId: p.id, projectRoot: p.root, operationId: previous.bookmark.operationId, digest: previous.digest }, "recovery_released"); }
634
+ catch { console.error("Pi continuity retirement is unconfirmed after successful recovery release; inspect the retained bookmark before another automatic start"); }
635
+ }
636
+ },
572
637
  closeTerminals: async (planned, progress, op, save) => {
573
638
  for (const binding of planned.terminals as TerminalBinding[]) {
574
639
  const key = `closed:${binding.peer}`;
@@ -618,10 +683,10 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
618
683
  if (reconnect.length) waiveRecoveryPeers(p.stateDir, op.id, Object.fromEntries(reconnect.map((peer) => [peer, "reconnect-only"])));
619
684
  await command(op, ["up"], p);
620
685
  },
621
- restore: async (planned, progress, op, group, save) => {
686
+ restore: async (planned, progress, op, group, save, onlyPeer) => {
622
687
  const live = await inspectRecovery(planned.project);
623
688
  if (live.instanceId !== progress.instanceId) throw new Error("target daemon changed before peer restore");
624
- if (group === "native") {
689
+ if (group === "native" && !onlyPeer) {
625
690
  for (const peer of planned.source.peers.filter((p) => ["kimi", "qwen", "jcode", "local", "pi"].includes(p.id) && p.state !== "offline" && !(p.id === "pi" && p.args?.mode === "tui"))) {
626
691
  const current = live.peers.find((p) => p.id === peer.id);
627
692
  if (current && current.state !== "offline") continue;
@@ -638,7 +703,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
638
703
  // #269: `ahub pi` without a mode now takes Pi's start mode (tui by default). This Pi ran headless (a terminal
639
704
  // one is filtered out above), so a snapshot that recorded no mode still restores it headless.
640
705
  if (peer.id === "pi" && typeof peer.args?.mode !== "string") args.push("--mode", "headless");
641
- if (peer.id === "qwen" && peer.sessionId) args.push("--session-id", peer.sessionId);
706
+ if (definitionOf(peer.id)?.session.restoreOnStart && peer.sessionId) args.push("--session-id", peer.sessionId);
642
707
  if (peer.id === "jcode" && peer.sessionId) args.push("--session-id", peer.sessionId);
643
708
  if (peer.id === "jcode" && peer.args?.providerProfile) args.push("--provider-profile", peer.args.providerProfile);
644
709
  if (peer.id === "pi") {
@@ -649,179 +714,221 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
649
714
  await command(op, args, planned.project);
650
715
  }
651
716
  }
717
+ let claudeGroup = false;
652
718
  for (const plannedBinding of planned.terminals as TerminalBinding[]) {
653
- if ((plannedBinding.peer === "claude") !== (group === "claude")) continue;
719
+ if ((plannedBinding.peer === "claude") !== (claudeGroup = group === "claude") || onlyPeer && plannedBinding.peer !== onlyPeer) continue;
654
720
  const key = `restored:${plannedBinding.peer}`;
655
- // #225: after a restart, the stopped target's terminal for this peer goes first; a session it had accepted
656
- // instead of the planned one (fresh choice, fresh start, zero-turn Claude) is relaunched by that id, which the
657
- // operation-fenced waiver already covers.
658
- await closeRetired(progress, plannedBinding.peer, save);
659
- const retired = progress.terminals[`retired:${plannedBinding.peer}`] as TerminalBinding | undefined;
660
- const accepted = retired && typeof retired === "object" && retired.sessionId !== plannedBinding.sessionId ? retired.sessionId : undefined;
661
- const original = accepted ? { ...plannedBinding, sessionId: accepted } : plannedBinding;
662
- // A new session is accepted only by the operator's choice (#215 dispose --fresh-session) or a planned fresh
663
- // start of a Codex thread with nothing to lose that still has no rollout; an unreadable store is neither.
664
- const chosen = !!progress.fresh?.[original.peer];
665
- let read: ReturnType<typeof codexTranscript> | undefined;
666
- const transcript = () => original.peer !== "codex" || (chosen && !accepted) ? "found" : (read ??= codexTranscript(original));
667
- // An accepted session starts new again only when it has nothing to resume: a Codex thread that never got a
668
- // rollout, or a Claude session with no transcript (`--resume` could never succeed, #21).
669
- const fresh = () => accepted
670
- ? (original.peer === "codex" ? (chosen || !!planned.freshStart?.includes("codex")) && transcript() === "missing" : original.peer === "claude" && !claudeTranscriptExists(original))
671
- : chosen || (!!planned.freshStart?.includes(original.peer) && transcript() === "missing");
672
- // The runner ends this with the choices `status` shows (nextActions); resume comes first and launches a failed
673
- // peer again, which helps once its cause (a transient Orca create failure, a fixed store) is gone.
674
- // The waiver's audit label: the operator's choice, a Claude with nothing to resume, or the plan's Codex fresh start.
675
- const waiver = () => progress.fresh?.[original.peer] ? "fresh-session" : original.peer === "claude" ? "zero-turn" : "fresh-start";
676
- const notRestored = (why: string) => new Error(fresh()
677
- ? `${original.peer}: ${why}; read that terminal in Orca for the launcher's error and fix its cause`
678
- : `${original.peer}: ${why}; session ${original.sessionId} was not restored; resume launches it again once the cause is fixed`);
679
- // Anything live or unknown blocks a create and keeps the receipt: never decide on evidence that could not be read.
680
- const blocked = (evidence: Exclude<Awaited<ReturnType<typeof peerEvidence>>, { state: "gone" }>) => new Error(evidence.state === "unknown"
681
- ? `${original.peer}: whether a ${original.peer} session or launcher is live cannot be told (${evidence.why}); nothing was recorded or created${evidence.step ? `; ${evidence.step} first` : ""}`
682
- : evidence.session
683
- ? `${original.peer}: session ${evidence.session} is attached instead of ${original.sessionId}; no terminal was created; end that ${original.peer} session and close its terminal first`
684
- : `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session ${evidence.attached ? "with an id " : ""}attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`);
685
- const decide = (receipt: RestoreReceipt, evidence: Awaited<ReturnType<typeof peerEvidence>>) => restoreTransition(receipt, {
686
- target: "running", launcher: evidence.state === "unknown" ? "unknown" : evidence.state === "gone" ? "gone" : evidence.terminal ? "live" : "older-instance",
687
- session: evidence.state === "live" && evidence.session ? evidence.session === original.sessionId ? "planned" : "other" : "none",
688
- allowOther: fresh() || (original.peer === "claude" && !claudeTranscriptExists(original)),
689
- binding: evidence.state === "live" && evidence.terminal ? { ...original, ...evidence.terminal } : original,
690
- });
691
- const receipt = decodeRestoreReceipt(progress.terminals[key]);
692
- function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }> & { origin: "pending" }, expected: string | null): Promise<string>;
693
- function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined>;
694
- async function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined> {
695
- try { return await awaitAttachment(planned, progress, op, waiting, save, expected); }
696
- catch (error) {
697
- if (!(error instanceof AttachmentLauncherGone)) throw error;
698
- progress.terminals[key] = error.transition.next; save();
699
- // The table decides whether this failed wait blocks or is eligible for a new launch.
700
- if (error.transition.action === "create") return undefined;
701
- throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
721
+ try {
722
+ // #225: after a restart, the stopped target's terminal for this peer goes first; a session it had accepted
723
+ // instead of the planned one (fresh choice, fresh start, zero-turn Claude) is relaunched by that id, which the
724
+ // operation-fenced waiver already covers.
725
+ await closeRetired(progress, plannedBinding.peer, save);
726
+ const retired = progress.terminals[`retired:${plannedBinding.peer}`] as TerminalBinding | undefined;
727
+ const accepted = retired && typeof retired === "object" && retired.sessionId !== plannedBinding.sessionId ? retired.sessionId : undefined;
728
+ const repair = progress.reattachments?.[plannedBinding.peer];
729
+ const recordedOnly = !!repair || progress.repairedSessions?.[plannedBinding.peer] === true && !progress.fresh?.[plannedBinding.peer]?.repaired;
730
+ const retiredReceipt = recordedOnly && retired !== undefined ? decodeRestoreReceipt({ state: "done", binding: retired }) : undefined;
731
+ const recordedBinding = retiredReceipt?.state === "done" && retiredReceipt.binding.peer === plannedBinding.peer && retiredReceipt.binding.projectRoot === planned.project.root ? retiredReceipt.binding : undefined;
732
+ const original = repair?.binding ?? recordedBinding ?? (accepted ? { ...plannedBinding, sessionId: accepted } : plannedBinding);
733
+ // A new session is accepted only by the operator's choice (#215 dispose --fresh-session) or a planned fresh
734
+ // start of a Codex thread with nothing to lose that still has no rollout; an unreadable store is neither.
735
+ const chosen = !recordedOnly && !!progress.fresh?.[original.peer];
736
+ let read: ReturnType<typeof codexTranscript> | undefined;
737
+ const transcript = () => original.peer !== "codex" || (chosen && !accepted) ? "found" : (read ??= codexTranscript(original));
738
+ // An accepted session starts new again only when it has nothing to resume: a Codex thread that never got a
739
+ // rollout, or a Claude session with no transcript (`--resume` could never succeed, #21).
740
+ const fresh = () => recordedOnly ? false : accepted
741
+ ? (original.peer === "codex" ? (chosen || !!planned.freshStart?.includes("codex")) && transcript() === "missing" : original.peer === "claude" && !claudeTranscriptExists(original))
742
+ : chosen || (!!planned.freshStart?.includes(original.peer) && transcript() === "missing");
743
+ // The runner ends this with the choices `status` shows (nextActions); resume comes first and launches a failed
744
+ // peer again, which helps once its cause (a transient Orca create failure, a fixed store) is gone.
745
+ // The waiver's audit label: the operator's choice, a Claude with nothing to resume, or the plan's Codex fresh start.
746
+ const waiver = () => progress.fresh?.[original.peer] ? "fresh-session" : original.peer === "claude" ? "zero-turn" : "fresh-start";
747
+ const notRestored = (why: string) => new Error(fresh()
748
+ ? `${original.peer}: ${why}; read that terminal in Orca for the launcher's error and fix its cause`
749
+ : `${original.peer}: ${why}; session ${original.sessionId} was not restored; resume launches it again once the cause is fixed`);
750
+ // Anything live or unknown blocks a create and keeps the receipt: never decide on evidence that could not be read.
751
+ const blocked = (evidence: Exclude<Awaited<ReturnType<typeof peerEvidence>>, { state: "gone" }>) => evidence.state === "unknown"
752
+ ? new Error(`${original.peer}: whether a ${original.peer} session or launcher is live cannot be told (${evidence.why}); nothing was recorded or created${evidence.step ? `; ${evidence.step} first` : ""}`)
753
+ : nativeFailure(planned, original.peer, evidence.session ? "session_changed" : "launcher_unverified", evidence.session
754
+ ? `${original.peer}: session ${evidence.session} is attached instead of ${original.sessionId}; no terminal was created; end that ${original.peer} session and close its terminal first`
755
+ : `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session ${evidence.attached ? "with an id " : ""}attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`,
756
+ evidence.session
757
+ ? { expectedSession: original.sessionId, observedSession: evidence.session }
758
+ : { ...(evidence.handle ? { handle: evidence.handle } : {}), launcherState: evidence.terminal ? "live" : "older-instance" });
759
+ const decide = (receipt: RestoreReceipt, evidence: Awaited<ReturnType<typeof peerEvidence>>) => restoreTransition(receipt, {
760
+ target: "running", launcher: evidence.state === "unknown" ? "unknown" : evidence.state === "gone" ? "gone" : evidence.terminal ? "live" : "older-instance",
761
+ session: evidence.state === "live" && evidence.session ? evidence.session === original.sessionId ? "planned" : "other" : "none",
762
+ allowOther: fresh() || (!recordedOnly && original.peer === "claude" && !claudeTranscriptExists(original)),
763
+ binding: evidence.state === "live" && evidence.terminal ? { ...original, ...evidence.terminal } : original,
764
+ });
765
+ const receipt = decodeRestoreReceipt(progress.terminals[key]);
766
+ function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }> & { origin: "pending" }, expected: string | null): Promise<string>;
767
+ function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined>;
768
+ async function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined> {
769
+ try { return await awaitAttachment(planned, progress, op, waiting, save, expected); }
770
+ catch (error) {
771
+ if (!(error instanceof AttachmentLauncherGone)) throw error;
772
+ progress.terminals[key] = error.transition.next; save();
773
+ // The table decides whether this failed wait blocks or is eligible for a new launch.
774
+ if (error.transition.action === "create") return undefined;
775
+ throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
776
+ }
702
777
  }
703
- }
704
- // #215: the pending row of the receipt table, used wherever a pending launch meets an attached session.
705
- const settle = async (attached: string, recovered?: TerminalBinding) => {
706
- // #21: a Claude session that never got a first turn has no transcript, so `claude --resume S` can never
707
- // succeed and accepting its fresh attach loses nothing. Otherwise only the planned id, or a new session
708
- // the operator or the plan accepted, counts as restored.
709
- if (attached !== original.sessionId && !fresh() && !(original.peer === "claude" && !claudeTranscriptExists(original))) {
710
- throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${attached} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
778
+ // #215: the pending row of the receipt table, used wherever a pending launch meets an attached session.
779
+ const settle = async (attached: string, recovered?: TerminalBinding) => {
780
+ // #21: a Claude session that never got a first turn has no transcript, so `claude --resume S` can never
781
+ // succeed and accepting its fresh attach loses nothing. Otherwise only the planned id, or a new session
782
+ // the operator or the plan accepted, counts as restored.
783
+ if (attached !== original.sessionId && !fresh() && !(!recordedOnly && original.peer === "claude" && !claudeTranscriptExists(original))) {
784
+ throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${attached} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
785
+ }
786
+ // Validate the proposed transition before committing continuity/waiver effects or a done receipt.
787
+ const verified = await revalidateTerminal(planned, progress, attachedBinding(recovered ?? original, attached), !!recovered);
788
+ // The planned session came back after all: a fresh session chosen for it lost nothing. Compared with the plan,
789
+ // never with a session accepted after a restart (#225), whose choice must stay on record.
790
+ if (attached === plannedBinding.sessionId && progress.fresh?.[original.peer]?.lost === plannedBinding.sessionId) {
791
+ delete progress.fresh[original.peer];
792
+ if (!Object.keys(progress.fresh).length) delete progress.fresh;
793
+ }
794
+ // Any other accepted session needs the target to waive the saved id, or release would wait for it forever;
795
+ // a launch writes the waiver first, but this must not rely on that.
796
+ // The planned session needs no waiver: the target's snapshot already expects it.
797
+ if (attached !== original.sessionId && attached !== plannedBinding.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
798
+ if (repair) repair.state = "attempted"; // a verified spontaneous reattachment also closes this repair allowance
799
+ // The person accepted one loss, not a permanent exemption for later repairs of the replacement.
800
+ if (progress.fresh?.[original.peer]?.repaired) delete progress.fresh[original.peer]!.repaired;
801
+ progress.terminals[key] = { state: "done", binding: verified } satisfies RestoreReceipt; save();
802
+ };
803
+ if (receipt?.state === "done") {
804
+ // A completed restoration is never a waiting launch. A closed terminal blocks exact revalidation;
805
+ // it cannot erase continuity or create a fresh-session choice (PROBE-B).
806
+ await detached(planned, progress, op, receipt.binding, save);
807
+ const observed = await peerEvidence(planned, progress, original.peer);
808
+ const decision = restoreTransition(receipt, { target: "running", launcher: "no-record-yet", session: observed.state === "live" && observed.session === receipt.binding.sessionId ? "planned" : observed.state === "live" && observed.session ? "other" : "none", allowOther: false });
809
+ if (decision.action === "block") {
810
+ const condition = observed.state === "unknown" ? `cannot be verified (${observed.why})`
811
+ : observed.state === "live" && observed.session ? `session ${observed.session} is attached instead of ${receipt.binding.sessionId}`
812
+ : observed.state === "live" ? `is not attached; its launcher in terminal ${observed.handle ?? "unknown"} may still run` : "is not attached";
813
+ throw new Error(`${original.peer}: completed restoration ${condition}; saved done receipt retained; resume its recorded conversation`);
814
+ }
815
+ progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, receipt.binding, true) } satisfies RestoreReceipt; save();
816
+ continue;
711
817
  }
712
- // Validate the proposed transition before committing continuity/waiver effects or a done receipt.
713
- const verified = await revalidateTerminal(planned, progress, attachedBinding(recovered ?? original, attached), !!recovered);
714
- // The planned session came back after all: a fresh session chosen for it lost nothing. Compared with the plan,
715
- // never with a session accepted after a restart (#225), whose choice must stay on record.
716
- if (attached === plannedBinding.sessionId && progress.fresh?.[original.peer]) {
717
- delete progress.fresh[original.peer];
718
- if (!Object.keys(progress.fresh).length) delete progress.fresh;
818
+ if (receipt?.state === "waiting") {
819
+ const binding = receipt.binding;
820
+ const sessionId = await awaitBound(receipt, fresh() && binding.sessionId === original.sessionId ? null : binding.sessionId);
821
+ if (sessionId !== undefined) { await settle(sessionId, binding); continue; }
719
822
  }
720
- // Any other accepted session needs the target to waive the saved id, or release would wait for it forever;
721
- // a launch writes the waiver first, but this must not rely on that.
722
- // The planned session needs no waiver: the target's snapshot already expects it.
723
- if (attached !== original.sessionId && attached !== plannedBinding.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
724
- progress.terminals[key] = { state: "done", binding: verified } satisfies RestoreReceipt; save();
725
- };
726
- if (receipt?.state === "done") {
727
- // A completed restoration is never a waiting launch. A closed terminal blocks exact revalidation;
728
- // it cannot erase continuity or create a fresh-session choice (PROBE-B).
729
- const observed = await peerEvidence(planned, progress, original.peer);
730
- const decision = restoreTransition(receipt, { target: "running", launcher: "no-record-yet", session: observed.state === "live" && observed.session === receipt.binding.sessionId ? "planned" : observed.state === "live" && observed.session ? "other" : "none", allowOther: false });
731
- if (decision.action === "block") {
732
- const condition = observed.state === "unknown" ? `cannot be verified (${observed.why})`
733
- : observed.state === "live" && observed.session ? `session ${observed.session} is attached instead of ${receipt.binding.sessionId}`
734
- : observed.state === "live" ? `is not attached; its launcher in terminal ${observed.handle ?? "unknown"} may still run` : "is not attached";
735
- throw new Error(`${original.peer}: completed restoration ${condition}; saved done receipt retained; resume its recorded conversation`);
823
+ if (receipt?.state === "pending" || receipt?.state === "failed") {
824
+ const evidence = await peerEvidence(planned, progress, original.peer);
825
+ const decision = decide(receipt, evidence);
826
+ if (decision.action === "settle" && evidence.state === "live" && evidence.session) { await settle(evidence.session); continue; }
827
+ if (decision.action === "poll" && decision.next.state === "waiting") {
828
+ const waiting = decision.next;
829
+ progress.terminals[key] = waiting; save();
830
+ const session = await awaitBound(waiting, fresh() ? null : original.sessionId);
831
+ if (session !== undefined) { await settle(session, waiting.binding); continue; }
832
+ } else if (decision.action === "fail") {
833
+ progress.terminals[key] = decision.next; save();
834
+ throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
835
+ } else if (decision.action === "block") {
836
+ if (evidence.state === "live" && evidence.session) {
837
+ throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${evidence.session} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
838
+ }
839
+ if (evidence.state !== "gone") throw blocked(evidence);
840
+ throw new Error(`${original.peer}: restoration evidence is not actionable`);
841
+ }
736
842
  }
737
- progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, receipt.binding, true) } satisfies RestoreReceipt; save();
738
- continue;
739
- }
740
- if (receipt?.state === "waiting") {
741
- const binding = receipt.binding;
742
- const sessionId = await awaitBound(receipt, fresh() && binding.sessionId === original.sessionId ? null : binding.sessionId);
743
- if (sessionId !== undefined) { await settle(sessionId, binding); continue; }
744
- }
745
- if (receipt?.state === "pending" || receipt?.state === "failed") {
843
+ // Never create beside a live launch, nor on evidence that could not be read: read now, not at restore start.
844
+ // The planned session already attached is the restoration (the table's row for it), not a reason to block.
746
845
  const evidence = await peerEvidence(planned, progress, original.peer);
747
- const decision = decide(receipt, evidence);
748
- if (decision.action === "settle" && evidence.state === "live" && evidence.session) { await settle(evidence.session); continue; }
749
- if (decision.action === "poll" && decision.next.state === "waiting") {
750
- const waiting = decision.next;
751
- progress.terminals[key] = waiting; save();
752
- const session = await awaitBound(waiting, fresh() ? null : original.sessionId);
753
- if (session !== undefined) { await settle(session, waiting.binding); continue; }
754
- } else if (decision.action === "fail") {
755
- progress.terminals[key] = decision.next; save();
756
- throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
757
- } else if (decision.action === "block") {
758
- if (evidence.state === "live" && evidence.session) {
759
- throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${evidence.session} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
846
+ if (evidence.state === "live" && evidence.session === original.sessionId) {
847
+ if (repair) await settle(evidence.session);
848
+ else { progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, original, false) } satisfies RestoreReceipt; save(); }
849
+ continue;
850
+ }
851
+ if (evidence.state !== "gone") throw blocked(evidence);
852
+ if (transcript() === "unknown") throw new Error(`${unresumable(original)}; no terminal was created; make that store readable first`);
853
+ if (transcript() === "missing" && !fresh()) {
854
+ progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
855
+ if (recordedOnly) throw new MissingRecordedSession({ projectId: planned.project.id, peer: original.peer, sessionId: original.sessionId, kind: "rollout", checkedPath: checkedSessionPath(original, "rollout") });
856
+ throw notRestored(unresumable(original).slice("codex: ".length));
857
+ }
858
+ if (fresh()) {
859
+ if (!targetReadsWaivers(op)) throw new Error(`${original.peer}: target ${op.plan.version} predates recovery waivers, so it cannot accept a new session`);
860
+ // The restored daemon must accept the new session instead of the recorded one.
861
+ waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
862
+ }
863
+ if (repair) {
864
+ if (repair.state === "attempted") throw new ReattachmentRequired(planned.project.id, original.peer, `${original.peer}: recorded-session reattachment already attempted; no second terminal will be created`);
865
+ repair.state = "attempted"; save(); // durable before any terminal creation, including an uncertain failure
866
+ }
867
+ const entrypoint = join(op.targetRoot!, "src/cli/main.js");
868
+ const argv = restoredTerminalArgv(entrypoint, planned.project.root, original, fresh());
869
+ const assignments = { ...original.launch.env, AGENTHUB_HOME: hubHome(), AGENTHUB_RECOVERY_OPERATION: op.id };
870
+ const launch = { ...original.launch, packageEntrypoint: entrypoint, argv,
871
+ command: ["env", ...Object.entries(assignments).map(([k, v]) => `${k}=${v}`), ...argv].map(shellQuote).join(" ") };
872
+ const binding = { ...original, launch, launchMetadata: launch };
873
+ progress.terminals[key] = { state: "pending" } satisfies RestoreReceipt; save();
874
+ const restored = await createTerminal(binding, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId }, undefined, fresh());
875
+ const exited = restored.blockers.find((b) => b.code === "launcher-exited");
876
+ if (exited) {
877
+ // #215: the launcher's exit alone is not the peer's state (a target that died or flapped also ends it): only
878
+ // nothing live makes it failed; an attached session is settled as the pending row settles it; anything else
879
+ // stays pending.
880
+ const after = await peerEvidence(planned, progress, original.peer);
881
+ const decision = decide({ state: "pending" }, after);
882
+ if (decision.action === "settle" && after.state === "live" && after.session) { await settle(after.session); continue; }
883
+ if (decision.action === "poll" && decision.next.state === "waiting") {
884
+ progress.terminals[key] = decision.next; save();
885
+ const session = await awaitBound(decision.next, fresh() ? null : original.sessionId);
886
+ if (session !== undefined) { await settle(session, decision.next.binding); continue; }
760
887
  }
761
- if (evidence.state !== "gone") throw blocked(evidence);
762
- throw new Error(`${original.peer}: restoration evidence is not actionable`);
888
+ if (decision.action === "fail") {
889
+ progress.terminals[key] = decision.next; save();
890
+ throw notRestored(exited.message);
891
+ }
892
+ if (after.state !== "gone") throw blocked(after);
763
893
  }
764
- }
765
- // Never create beside a live launch, nor on evidence that could not be read: read now, not at restore start.
766
- // The planned session already attached is the restoration (the table's row for it), not a reason to block.
767
- const evidence = await peerEvidence(planned, progress, original.peer);
768
- if (evidence.state === "live" && evidence.session === original.sessionId) {
769
- progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, original, false) } satisfies RestoreReceipt; save();
770
- continue;
771
- }
772
- if (evidence.state !== "gone") throw blocked(evidence);
773
- if (transcript() === "unknown") throw new Error(`${unresumable(original)}; no terminal was created; make that store readable first`);
774
- if (transcript() === "missing" && !fresh()) {
775
- progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
776
- throw notRestored(unresumable(original).slice("codex: ".length));
777
- }
778
- if (fresh()) {
779
- if (!targetReadsWaivers(op)) throw new Error(`${original.peer}: target ${op.plan.version} predates recovery waivers, so it cannot accept a new session`);
780
- // The restored daemon must accept the new session instead of the recorded one.
781
- waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
782
- }
783
- const entrypoint = join(op.targetRoot!, "src/cli/main.js");
784
- const argv = restoredTerminalArgv(entrypoint, planned.project.root, original, fresh());
785
- const assignments = { ...original.launch.env, AGENTHUB_HOME: hubHome(), AGENTHUB_RECOVERY_OPERATION: op.id };
786
- const launch = { ...original.launch, packageEntrypoint: entrypoint, argv,
787
- command: ["env", ...Object.entries(assignments).map(([k, v]) => `${k}=${v}`), ...argv].map(shellQuote).join(" ") };
788
- const binding = { ...original, launch, launchMetadata: launch };
789
- progress.terminals[key] = { state: "pending" } satisfies RestoreReceipt; save();
790
- const restored = await createTerminal(binding, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId }, undefined, fresh());
791
- const exited = restored.blockers.find((b) => b.code === "launcher-exited");
792
- if (exited) {
793
- // #215: the launcher's exit alone is not the peer's state (a target that died or flapped also ends it): only
794
- // nothing live makes it failed; an attached session is settled as the pending row settles it; anything else
795
- // stays pending.
796
- const after = await peerEvidence(planned, progress, original.peer);
797
- const decision = decide({ state: "pending" }, after);
798
- if (decision.action === "settle" && after.state === "live" && after.session) { await settle(after.session); continue; }
799
- if (decision.action === "poll" && decision.next.state === "waiting") {
800
- progress.terminals[key] = decision.next; save();
801
- const session = await awaitBound(decision.next, fresh() ? null : original.sessionId);
802
- if (session !== undefined) { await settle(session, decision.next.binding); continue; }
894
+ if (restored.manualRequired || !restored.newBinding) {
895
+ const handle = restored.binding?.handle ?? restored.blockers[0]?.handle;
896
+ throw new Error(`${original.peer}: original session restoration needs manual verification${restored.blockers[0] ? ` (${restored.blockers[0].message})` : ""}; wait until it attaches, or end that launch and close its terminal${handle ? ` ${handle}` : ""} first`);
803
897
  }
804
- if (decision.action === "fail") {
805
- progress.terminals[key] = decision.next; save();
806
- throw notRestored(exited.message);
898
+ // The daemon's id is authoritative: a fresh session is recorded under the id it reports once attached.
899
+ const awaiting = { state: "waiting", origin: "pending", binding: restored.newBinding } satisfies RestoreReceipt;
900
+ progress.terminals[key] = awaiting; save();
901
+ const sessionId = await awaitBound(awaiting, fresh() ? null : restored.newBinding.sessionId);
902
+ await settle(sessionId, awaiting.binding);
903
+ } catch (error) {
904
+ if (error instanceof MissingRecordedSession || isDatabaseBusy(error)) throw error;
905
+ // A failed recorded-only Claude launch may prove that its transcript is gone. No error text is evidence.
906
+ if (claudeGroup && progress.repairedSessions?.[plannedBinding.peer] && !progress.fresh?.[plannedBinding.peer]?.repaired) {
907
+ const chosenReceipt = decodeRestoreReceipt(progress.terminals[key]);
908
+ // A bare pending/failed row can hide an uncertain create. Only a durable exact chosen binding
909
+ // can become the terminal that an explicit fresh choice must close before another create.
910
+ const binding = chosenReceipt?.state === "waiting" ? chosenReceipt.binding : undefined;
911
+ if (binding && claudeTranscriptState(binding) === "missing" && (await peerEvidence(planned, progress, plannedBinding.peer)).state === "gone") {
912
+ // Retain the exact chosen terminal so an explicit later fresh choice closes it before creating again.
913
+ progress.terminals[`retired:${plannedBinding.peer}`] = binding;
914
+ delete progress.terminals[`closedRetired:${plannedBinding.peer}`];
915
+ const marker = progress.reattachments?.[plannedBinding.peer];
916
+ if (marker) marker.binding = binding;
917
+ progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
918
+ throw new MissingRecordedSession({ projectId: planned.project.id, peer: plannedBinding.peer, sessionId: binding.sessionId, kind: "transcript", checkedPath: checkedSessionPath(binding, "transcript") });
919
+ }
807
920
  }
808
- if (after.state !== "gone") throw blocked(after);
809
- }
810
- if (restored.manualRequired || !restored.newBinding) {
811
- const handle = restored.binding?.handle ?? restored.blockers[0]?.handle;
812
- throw new Error(`${original.peer}: original session restoration needs manual verification${restored.blockers[0] ? ` (${restored.blockers[0].message})` : ""}; wait until it attaches, or end that launch and close its terminal${handle ? ` ${handle}` : ""} first`);
921
+ if (error instanceof ReattachmentRequired || isDatabaseBusy(error)) throw error;
922
+ if (progress.reattachments?.[plannedBinding.peer]) throw new ReattachmentRequired(planned.project.id, plannedBinding.peer,
923
+ error instanceof Error ? error.message : `${plannedBinding.peer}: recorded-session reattachment is unconfirmed`, error instanceof RecoveryPresentationError ? error : undefined);
924
+ throw error;
813
925
  }
814
- // The daemon's id is authoritative: a fresh session is recorded under the id it reports once attached.
815
- const awaiting = { state: "waiting", origin: "pending", binding: restored.newBinding } satisfies RestoreReceipt;
816
- progress.terminals[key] = awaiting; save();
817
- const sessionId = await awaitBound(awaiting, fresh() ? null : restored.newBinding.sessionId);
818
- await settle(sessionId, awaiting.binding);
819
926
  }
820
- if (group === "claude") for (const id of planned.reconnectOnly ?? []) {
927
+ if (group === "claude" && !onlyPeer) for (const id of planned.reconnectOnly ?? []) {
821
928
  // #206: nothing is launched for an unmanaged session; its plugin reconnects by itself, within a bound.
822
929
  const deadline = now() + UNMANAGED_RECONNECT_WAIT_MS;
823
930
  while (!(await inspectRecovery(planned.project)).peers.some((p) => p.id === id && p.state !== "offline")) {
824
- if (now() >= deadline) throw new Error(`${id}: the unmanaged session did not reconnect within ${UNMANAGED_RECONNECT_WAIT_MS / 1000} s; if that Claude session was closed, start Claude in the project again (its plugin reconnects) first`);
931
+ if (now() >= deadline) throw nativeFailure(planned, id, "attachment_timeout", `${id}: the unmanaged session did not reconnect within ${UNMANAGED_RECONNECT_WAIT_MS / 1000} s; if that Claude session was closed, start Claude in the project again (its plugin reconnects) first`, { timeoutMs: UNMANAGED_RECONNECT_WAIT_MS });
825
932
  await sleep(1000);
826
933
  }
827
934
  }
@@ -855,24 +962,61 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
855
962
  await awaitStopped(project, instance);
856
963
  }
857
964
  abandonRestartSnapshot(project.stateDir, op.id);
965
+ const planned = op.plan.projects?.find(item => item.project.id === project.id);
966
+ const progress = op.projects?.find(item => item.id === project.id);
967
+ const source = planned?.source.peers.find(peer => peer.id === "pi" && peer.state !== "offline");
968
+ if (source && progress) {
969
+ // abandonRestartSnapshot is deliberately a no-op for another operation or an unreadable snapshot.
970
+ // A continuity bookmark needs positive archival evidence, including when disposal resumes after a crash.
971
+ const archive = join(project.stateDir, `restart.abandoned.${createHash("sha256").update(op.id).digest("hex")}.json`);
972
+ let archived: unknown;
973
+ try { archived = JSON.parse(readFileSync(archive, "utf8")); } catch { return { continuity: "not-archived" }; }
974
+ if (!archived || typeof archived !== "object" || (archived as { operationId?: unknown }).operationId !== op.id) return { continuity: "not-archived" };
975
+ try {
976
+ const binding = planned ? resolveAcceptedBinding(planned, progress, "pi") : undefined;
977
+ const sessionId = binding?.sessionId ?? source.sessionId;
978
+ if (!sessionId) throw new Error("Pi recorded session unavailable; archive continuity bookmark was not written");
979
+ const backend = binding?.backend ?? source.args?.backend;
980
+ writeRecoveryContinuity(project.stateDir, { schema: 1, operationId: op.id, projectId: project.id, projectRoot: project.root, at: now(),
981
+ pi: { sessionId, sessionFile: binding?.sessionFile ?? source.sessionFile ?? source.args?.sessionFile,
982
+ ...(backend === "auto" || backend === "dgx" || backend === "mlx" ? { backend } : {}), model: binding?.model ?? source.args?.model } });
983
+ } catch { return { continuity: "unavailable" }; } // optional metadata must not wedge stop-and-archive
984
+ }
858
985
  },
859
986
  refreshManager: async (op) => { await refreshManager({ home: hubHome(), cli: join(op.targetRoot!, "src/cli/main.ts") }); },
860
- verify: async (planned, progress, op) => {
987
+ verify: async (planned, progress, op, save) => {
861
988
  const live = await inspectRecovery(planned.project);
862
989
  if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw new Error("target identity verification failed");
863
990
  for (const old of planned.source.peers.filter((p) => p.state !== "offline")) {
864
991
  const peer = live.peers.find((p) => p.id === old.id);
865
- if (!peer || !["idle", "busy", "paused"].includes(peer.state)) throw new Error(`${old.id}: peer reattachment not verified`);
866
- if (progress.fresh?.[old.id] || planned.freshStart?.includes(old.id)) continue; // #215: a new session was accepted (recorded)
867
- if (old.id === "codex" && peer.threadId !== old.threadId) throw new Error("Codex resumed a different conversation");
992
+ if (!peer || !["idle", "busy", "paused"].includes(peer.state)) {
993
+ const receipt = decodeRestoreReceipt(progress.terminals[`restored:${old.id}`]);
994
+ if (receipt?.state === "done") await detached(planned, progress, op, receipt.binding, save);
995
+ throw new Error(`${old.id}: peer reattachment not verified`);
996
+ }
997
+ const accepted = decodeRestoreReceipt(progress.terminals[`restored:${old.id}`]);
998
+ const repair = progress.reattachments?.[old.id as "codex" | "pi" | "claude"];
999
+ const recordedOnly = !!repair || progress.repairedSessions?.[old.id as keyof NonNullable<ProjectProgress["repairedSessions"]>] === true && !progress.fresh?.[old.id]?.repaired;
1000
+ const retiredValue = progress.terminals[`retired:${old.id}`];
1001
+ const retired = retiredValue === undefined ? undefined : decodeRestoreReceipt({ state: "done", binding: retiredValue });
1002
+ // A #225 restart clears the old target's repair fence, never its accepted conversation.
1003
+ const retained = retired?.state === "done" && accepted?.state === "done" && retired.binding.peer === old.id && retired.binding.projectRoot === planned.project.root && retired.binding.sessionId === accepted.binding.sessionId;
1004
+ if ((recordedOnly || retained) && accepted?.state === "done") {
1005
+ const session = old.id === "codex" ? peer.threadId : peer.sessionId;
1006
+ if (session !== accepted.binding.sessionId || repair && accepted.binding.sessionId !== repair.binding.sessionId ||
1007
+ old.id === "pi" && (recordedOnly || old.sessionFile) && accepted.binding.sessionFile && peer.sessionFile !== accepted.binding.sessionFile) throw nativeFailure(planned, old.id, "session_changed", `${old.id}: reattached a different recorded conversation`, { handle: accepted.binding.handle, expectedSession: accepted.binding.sessionId, ...(session === undefined ? {} : { observedSession: session }) });
1008
+ continue;
1009
+ }
1010
+ if (!recordedOnly && (progress.fresh?.[old.id] || planned.freshStart?.includes(old.id))) continue; // #215: a new session was accepted (recorded)
1011
+ if (old.id === "codex" && peer.threadId !== old.threadId) throw nativeFailure(planned, old.id, "session_changed", "Codex resumed a different conversation", { ...(old.threadId ? { expectedSession: old.threadId } : {}), ...(peer.threadId ? { observedSession: peer.threadId } : {}) });
868
1012
  if (old.id === "claude" && peer.sessionId !== old.sessionId) {
869
1013
  // #64: the restore gate already accepted a fresh session when the original never
870
1014
  // persisted a transcript. Re-derive persistence from the binding exactly like that
871
1015
  // gate did, so a first turn landing after the plan was made keeps the check strict.
872
1016
  const binding = (planned.terminals as TerminalBinding[]).find((t) => t.peer === "claude");
873
- if (!binding || binding.sessionId !== old.sessionId || claudeTranscriptExists(binding)) throw new Error("Claude resumed a different conversation");
1017
+ if (recordedOnly || !binding || binding.sessionId !== old.sessionId || claudeTranscriptExists(binding)) throw nativeFailure(planned, old.id, "session_changed", "Claude resumed a different conversation", { ...(old.sessionId ? { expectedSession: old.sessionId } : {}), ...(peer.sessionId ? { observedSession: peer.sessionId } : {}) });
874
1018
  }
875
- if (old.id === "pi" && (peer.sessionId !== old.sessionId || (old.sessionFile && peer.sessionFile !== old.sessionFile))) throw new Error("Pi resumed a different conversation");
1019
+ if (old.id === "pi" && (peer.sessionId !== old.sessionId || (old.sessionFile && peer.sessionFile !== old.sessionFile))) throw nativeFailure(planned, old.id, "session_changed", "Pi resumed a different conversation", { ...(old.sessionId ? { expectedSession: old.sessionId } : {}), ...(peer.sessionId ? { observedSession: peer.sessionId } : {}) });
876
1020
  }
877
1021
  for (const original of planned.terminals as TerminalBinding[]) {
878
1022
  const saved = decodeRestoreReceipt(progress.terminals[`restored:${original.peer}`]);
@@ -883,6 +1027,13 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
883
1027
  if (snapshot.ok === false) throw new Error("dashboard snapshot readback failed");
884
1028
  const recovery = await rpc(planned.project, { t: "recovery", op: "inspect", expectedInstanceId: progress.instanceId });
885
1029
  const integrity = recovery.recovery?.integrity;
1030
+ if (integrity?.expected && integrity.current) {
1031
+ for (const check of ["queues", "manualPaused", "boardDigest", "budgetDigest"] as const) {
1032
+ if (JSON.stringify(integrity.current[check]) !== JSON.stringify(integrity.expected[check])) {
1033
+ throw new RecoveryPreservationError(check, planned.project.id);
1034
+ }
1035
+ }
1036
+ }
886
1037
  if (!integrity?.expected || JSON.stringify(integrity.current) !== JSON.stringify(integrity.expected)) {
887
1038
  throw new Error("queue, manual pause, task board or budget preservation was not verified");
888
1039
  }