@staix/agent-hub 0.12.31 → 0.12.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +20 -1
  2. package/README.md +14 -7
  3. package/docs/agent-notes/adapters.md +7 -4
  4. package/docs/agent-notes/bus.md +3 -0
  5. package/docs/agent-notes/daemon.md +13 -8
  6. package/docs/agent-notes/local-worker.md +1 -1
  7. package/docs/agent-notes/tasks.md +4 -3
  8. package/docs/agent-notes/tests.md +1 -0
  9. package/docs/cooperbench.md +1 -1
  10. package/docs/gsd-collaboration-analysis.md +1 -1
  11. package/docs/memory.md +1 -1
  12. package/docs/operations.md +141 -25
  13. package/docs/quickstart.md +10 -6
  14. package/docs/security.md +18 -9
  15. package/docs/smoke.md +34 -0
  16. package/docs/specs/2026-09-19-agent-hub-design.md +204 -0
  17. package/docs/specs/2026-09-20-upgrade-recovery-design.md +211 -0
  18. package/docs/verification/2026-10-18-0.12.32.md +54 -0
  19. package/docs/verification/2026-10-18-0.12.33.md +42 -0
  20. package/docs/verified.json +24 -16
  21. package/package.json +2 -2
  22. package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
  23. package/plugins/agent-hub/server.js +1296 -26
  24. package/src/adapters/acp.ts +5 -6
  25. package/src/adapters/definitions/builtin-metadata.ts +121 -0
  26. package/src/adapters/definitions/kimi-native.ts +4 -0
  27. package/src/adapters/definitions/qwen-native.ts +24 -0
  28. package/src/adapters/definitions/qwen.ts +71 -0
  29. package/src/adapters/pi.ts +6 -2
  30. package/src/adapters/registry-types.ts +174 -0
  31. package/src/adapters/registry.ts +36 -0
  32. package/src/cli/console-state.ts +74 -5
  33. package/src/cli/console.ts +35 -2
  34. package/src/cli/help.ts +3 -1
  35. package/src/cli/identity.ts +13 -3
  36. package/src/cli/launch.ts +1 -6
  37. package/src/cli/main.ts +200 -39
  38. package/src/cli/output.ts +2 -0
  39. package/src/cli/preview.ts +8 -5
  40. package/src/cli/recovery-guidance.ts +137 -0
  41. package/src/cli/recovery-presentation-options.ts +8 -0
  42. package/src/cli/recovery-session-file.ts +18 -0
  43. package/src/cli/reset.ts +13 -1
  44. package/src/cli/terminal-recovery.ts +48 -0
  45. package/src/cli/upgrade-interactive.ts +129 -75
  46. package/src/cli/upgrade-labels.ts +30 -0
  47. package/src/cli/upgrade-presentation-data.ts +41 -0
  48. package/src/cli/upgrade-presentation-flags.ts +31 -0
  49. package/src/cli/upgrade-presentation.ts +227 -0
  50. package/src/cli/upgrade-restore-state.ts +6 -2
  51. package/src/cli/upgrade-runtime.ts +360 -199
  52. package/src/cli/upgrade.ts +278 -25
  53. package/src/hub/bus.ts +144 -14
  54. package/src/hub/cohorts.ts +2 -2
  55. package/src/hub/config-apply.ts +242 -0
  56. package/src/hub/config-reload.ts +120 -0
  57. package/src/hub/config-trust.ts +11 -2
  58. package/src/hub/control-client.ts +1 -1
  59. package/src/hub/crash.ts +8 -6
  60. package/src/hub/daemon.ts +577 -179
  61. package/src/hub/deferred-hub.ts +60 -0
  62. package/src/hub/delivery-journal.ts +6 -0
  63. package/src/hub/events.ts +2 -2
  64. package/src/hub/manager.ts +1 -1
  65. package/src/hub/peer-catalog.ts +10 -13
  66. package/src/hub/peers.ts +12 -2
  67. package/src/hub/permission-mode.ts +5 -3
  68. package/src/hub/recovery-continuity.ts +152 -0
  69. package/src/hub/recovery-store.ts +10 -4
  70. package/src/hub/restart.ts +12 -1
  71. package/src/hub/sdlc.ts +5 -1
  72. package/src/hub/settings.ts +50 -36
  73. package/src/hub/settled-roles.ts +35 -0
  74. package/src/hub/supervision.ts +3 -2
  75. package/src/hub/tasks.ts +29 -9
  76. package/src/hub/ui.ts +1 -1
  77. package/src/memory/brief.ts +6 -5
  78. package/src/pi/session.ts +23 -4
  79. package/src/ui/index.html +23 -7
@@ -1,3 +1,5 @@
1
+ import { LIVE_TEST_PERSON_REFUSAL } from "./identity.ts";
2
+ import { checkedSessionPath } from "./recovery-session-file.ts";
1
3
  import { canChooseFresh, decodeRestoreReceipt, restoreReceiptStatus } from "./upgrade-restore-state.ts";
2
4
  import { databaseBusySite, isDatabaseBusy, RecoveryDatabaseBusy } from "../hub/recovery-error.ts";
3
5
  import { Database } from "bun:sqlite";
@@ -7,6 +9,9 @@ import { join } from "node:path";
7
9
  import { REGISTRY_DB_BUSY_TIMEOUT_MS, PROJECT_COLUMNS, projectFromRow, type Project } from "../hub/registry.ts";
8
10
  import { hubHome } from "../hub/project.ts";
9
11
  import { acquireRecoveryLock, claimRunner, coordinatorCurrent, readOperation, recoveryCommand, recoveryLock, releaseRecoveryLock, writeOperation } from "../hub/recovery-store.ts";
12
+ import { resolveAcceptedBinding, piArchiveExit, recordedPeerGuidance, type RecoveryNativePeer } from "./recovery-guidance.ts";
13
+ import type { TerminalBinding } from "./terminal-recovery.ts";
14
+ import { RecoveryPresentationError, type RecoveryPresentationCause, type RecoveryWaitTiming } from "./upgrade-presentation-data.ts";
10
15
 
11
16
  export interface RecoveryPeer {
12
17
  id: string;
@@ -27,6 +32,8 @@ export interface Inspection {
27
32
  blockers: string[];
28
33
  /** #215: for a stopped runtime, the operation whose committed restart snapshot its state dir holds (unreleased). */
29
34
  snapshot?: string;
35
+ /** The runtime may be positively stopped while its restart snapshot remains unreadable; never claim ownership. */
36
+ snapshotError?: { code: "recovery_deferred_hub"; message: string };
30
37
  }
31
38
  export interface PlannedProject {
32
39
  project: Project;
@@ -54,6 +61,10 @@ export interface ProjectProgress {
54
61
  phase: "pending" | "prepared" | "stopped" | "started" | "peers-restored" | "verified";
55
62
  instanceId?: string;
56
63
  terminals: Record<string, unknown>;
64
+ /** One durable repair allowance per peer, retained after an attempt to prevent a second relaunch. */
65
+ reattachments?: Partial<Record<RecoveryNativePeer, ReattachmentMarker>>;
66
+ /** Recorded-session-only continuity, independent of a target incarnation's repair attempt allowance. */
67
+ repairedSessions?: Partial<Record<RecoveryNativePeer, true>>;
57
68
  /** #414: last bounded native-attachment timeout, metadata only; never terminal output or task text. */
58
69
  attachmentTimeouts?: Record<string, {
59
70
  at: number;
@@ -64,7 +75,7 @@ export interface ProjectProgress {
64
75
  }>;
65
76
  releaseRequested?: boolean;
66
77
  /** #215: peers the operator moved to a fresh session, with the session or thread id whose continuity is lost. */
67
- fresh?: Record<string, { lost: string; reason: string; at: number }>;
78
+ fresh?: Record<string, { lost: string; reason: string; at: number; repaired?: true }>;
68
79
  /** #215: set before the commit request is sent; from then on the source may have committed (abort is not offered). */
69
80
  commitSent?: boolean;
70
81
  /**
@@ -73,6 +84,12 @@ export interface ProjectProgress {
73
84
  */
74
85
  restarts?: { instanceId: string; at: number }[];
75
86
  }
87
+ export interface ReattachmentMarker {
88
+ state: "ready" | "attempted";
89
+ at: number;
90
+ cause: "detached after restore";
91
+ binding: TerminalBinding;
92
+ }
76
93
  export interface RecoveryOperation {
77
94
  /**
78
95
  * #227: 2 marks a stop-and-archive recorded on an operation whose coordinator predates #215. Every older runner and
@@ -94,10 +111,18 @@ export interface RecoveryOperation {
94
111
  pluginInstalled?: boolean;
95
112
  globalInstalled?: boolean;
96
113
  error?: string;
114
+ /** Presentation observations only: old receipts without these fields have unknown timings and cause. */
115
+ waits?: Record<string, RecoveryWaitTiming>;
116
+ presentationCause?: RecoveryPresentationCause;
97
117
  failure?: { code: "database_busy"; retryable: true; site?: string[] };
118
+ /** A typed verify outcome, never inferred from an error message or a peer's current state. */
119
+ blockedReattachment?: { projectId: string; peer: RecoveryNativePeer };
120
+ /** Positive typed store observation; disposal rechecks it before accepting a repaired-session loss. */
121
+ missingRecordedSession?: { projectId: string; peer: RecoveryNativePeer; sessionId: string; kind: "rollout" | "transcript"; checkedPath?: string };
122
+ freshOverride?: { peer: RecoveryNativePeer; evidence: NonNullable<RecoveryOperation["missingRecordedSession"]>; checked: "missing"; at: number };
98
123
  /** #215: every human disposition, in order. Operator reasons only; never task or message text. */
99
124
  audit?: { at: number; action: "fresh-session" | "stop-and-archive"; reason: string; peer?: string; projects: string[] }[];
100
- disposition?: { choice: "stop-and-archive"; at: number; projects: Record<string, string> };
125
+ disposition?: { choice: "stop-and-archive"; at: number; projects: Record<string, string>; continuity?: Record<string, "not-archived" | "unavailable"> };
101
126
  /** #215: a refusal no resume can get past was recorded (FinalRefusal); `next` no longer offers resume. */
102
127
  final?: boolean;
103
128
  }
@@ -105,6 +130,16 @@ export interface RecoveryOperation {
105
130
  /** #215: a refusal that holds for every resume of the operation (its staged target or preserved source is fixed). */
106
131
  export class FinalRefusal extends Error {}
107
132
 
133
+ export class ReattachmentRequired extends Error {
134
+ constructor(public readonly projectId: string, public readonly peer: RecoveryNativePeer, message: string, public readonly presentationError?: RecoveryPresentationError) { super(message); }
135
+ }
136
+
137
+ export class MissingRecordedSession extends Error {
138
+ readonly code = "recorded_session_missing";
139
+ constructor(readonly evidence: NonNullable<RecoveryOperation["missingRecordedSession"]>) {
140
+ super(`${evidence.peer}: recorded session ${evidence.sessionId} has no resumable transcript (positively missing ${evidence.kind}${evidence.checkedPath ? ` checked at ${JSON.stringify(evidence.checkedPath)}` : ""}); choose dispose --fresh-session ${evidence.peer} --reason <text> to accept its loss, or stop-and-archive`);
141
+ }
142
+ }
108
143
  export { recoveryCommand };
109
144
  const STOP = "--stop-and-archive --reason <text>";
110
145
  /**
@@ -145,7 +180,12 @@ export function recordRestart(progress: ProjectProgress, at: number): void {
145
180
  progress.terminals[`retired:${peer}`] = receipt.binding;
146
181
  delete progress.terminals[`closedRetired:${peer}`]; // a newer binding to close: its own receipt starts over
147
182
  delete progress.terminals[key];
183
+ // Older receipts may have only the attempt marker. Retiring that binding must not retire its continuity rule.
184
+ if (Object.hasOwn(progress.reattachments ?? {}, peer)) (progress.repairedSessions ??= {})[peer as RecoveryNativePeer] = true;
185
+ // The allowance belongs to the target whose exact binding was retired, not every later #225 restart.
186
+ if (progress.reattachments) delete progress.reattachments[peer as RecoveryNativePeer];
148
187
  }
188
+ if (progress.reattachments && !Object.keys(progress.reattachments).length) delete progress.reattachments;
149
189
  delete progress.instanceId;
150
190
  progress.phase = "stopped";
151
191
  }
@@ -220,7 +260,7 @@ export function disposeRefusal(op: RecoveryOperation, live: Record<string, Inspe
220
260
 
221
261
  /** #272: one entry of `next`. `text` is the line status prints; a screen runs `kind` through `recoveryArgv`. */
222
262
  export interface NextChoice {
223
- kind: "resume" | "abort" | "fresh" | "stop" | "wait";
263
+ kind: "resume" | "abort" | "fresh" | "stop" | "wait" | "reattach";
224
264
  /** `fresh` only: the peer that starts as a new session. */
225
265
  peer?: string;
226
266
  text: string;
@@ -245,9 +285,13 @@ export function nextChoices(receipt: RecoveryOperation, runner?: number | "unkno
245
285
  const old = !!op.sourceRoot && !coordinatorCurrent(op.sourceRoot);
246
286
  const abortable = !abortRefusal(op, live);
247
287
  const stuck = resumeBlocked(op, live);
288
+ const selected = op.phase === "blocked" ? op.blockedReattachment : undefined;
289
+ const repair = selected && op.projects.find(project => project.id === selected.projectId)?.reattachments?.[selected.peer];
290
+ const reattach = selected && repair && (repair.state === "ready" || repairWaiting(op, selected.projectId, selected.peer));
248
291
  return [
249
292
  ...(stuck ? [] : [{ kind: "resume" as const, text: `${recoveryCommand(op, "resume")}${op.error ? " (after the next action in error)" : ""}${old ? ` (runs the coordinator that started this operation, which cannot re-prepare an expired hold: if it reports "source is no longer prepared", use ${abortable ? "abort or " : ""}stop-and-archive)` : ""}` }]),
250
293
  ...(abortable ? [{ kind: "abort" as const, text: recoveryCommand(op, "abort") }] : []),
294
+ ...(selected && reattach ? [{ kind: "reattach" as const, peer: selected.peer, text: recoveryCommand(op, "reattach", selected.peer) }] : []),
251
295
  ...(targetReadsWaivers(op) && !stuck ? failed.map((peer) => ({ kind: "fresh" as const, peer, text: recoveryCommand(op, "dispose", `--fresh-session ${peer} --reason <text>`) })) : []),
252
296
  stop,
253
297
  ];
@@ -278,6 +322,13 @@ export async function liveProjects(op: RecoveryOperation, inspect: (project: Pro
278
322
  return Object.fromEntries(await Promise.all(open.map(async (p) => [p.project.id, await inspect(p.project).catch(() => undefined)] as const)));
279
323
  }
280
324
 
325
+ function missingRepair(op: RecoveryOperation, progress: ProjectProgress, peer: string): boolean {
326
+ const evidence = op.missingRecordedSession;
327
+ if (!evidence || evidence.projectId !== progress.id || evidence.peer !== peer || (!progress.repairedSessions?.[evidence.peer] && !Object.hasOwn(progress.reattachments ?? {}, peer))) return false;
328
+ const planned = op.plan.projects.find(project => project.project.id === progress.id);
329
+ if (!planned) return false;
330
+ try { return resolveAcceptedBinding(planned, progress, evidence.peer)?.sessionId === evidence.sessionId; } catch { return false; }
331
+ }
281
332
  /**
282
333
  * Peers of a project whose restoration failed and for which a fresh session is a real choice: not Pi (a restored hub
283
334
  * resumes its recorded session), not one already chosen, and not a planned fresh start (nothing to lose; resume
@@ -286,13 +337,31 @@ export async function liveProjects(op: RecoveryOperation, inspect: (project: Pro
286
337
  function failedPeers(op: RecoveryOperation, progress: ProjectProgress, live: Record<string, Inspection | undefined> = {}): string[] {
287
338
  const planned = op.plan.projects.find((p) => p.project.id === progress.id);
288
339
  return Object.entries(progress.terminals).filter(([key, value]) => key.startsWith("restored:") && canChooseFresh(value)).map(([key]) => key.slice("restored:".length))
289
- .filter((peer) => peer !== "pi" && !progress.fresh?.[peer] && !planned?.freshStart?.includes(peer) && !plannedAttached(op, progress, peer, live));
340
+ .filter((peer) => peer !== "pi" && ((!Object.hasOwn(progress.reattachments ?? {}, peer) && !Object.hasOwn(progress.repairedSessions ?? {}, peer)) || missingRepair(op, progress, peer)) && !progress.fresh?.[peer] && (!planned?.freshStart?.includes(peer) || missingRepair(op, progress, peer)) && !plannedAttached(op, progress, peer, live));
341
+ }
342
+
343
+ /** Read-only scope classification; it confers no delegation authority. */
344
+ export function repairedFreshSession(op: RecoveryOperation, peer: string): boolean | undefined {
345
+ try {
346
+ validateReceipt(op.id, op);
347
+ if (!op.plan.projects.some(planned => planned.source.peers.some(row => row.id === peer) ||
348
+ planned.terminals.some(value => !!value && typeof value === "object" && (value as { peer?: unknown }).peer === peer))) return undefined;
349
+ return op.projects.some(progress => failedPeers(op, progress).includes(peer) && missingRepair(op, progress, peer));
350
+ } catch { return undefined; } // Unknown scope never becomes an ordinary delegated disposition.
351
+ }
352
+
353
+ function repairWaiting(op: RecoveryOperation, projectId: string, peer: RecoveryNativePeer): boolean {
354
+ try { return decodeRestoreReceipt(op.projects.find(project => project.id === projectId)?.terminals[`restored:${peer}`])?.state === "waiting"; }
355
+ catch { return false; }
290
356
  }
291
357
 
292
358
  /** #215: the inspection shows the planned session attached for `peer`, so resume records it as restored: nothing is lost. */
293
359
  function plannedAttached(op: RecoveryOperation, progress: ProjectProgress, peer: string, live: Record<string, Inspection | undefined>): boolean {
294
360
  const state = live[progress.id];
295
- const binding = (op.plan.projects.find((p) => p.project.id === progress.id)?.terminals as { peer: string; sessionId: string }[] | undefined)?.find((t) => t.peer === peer);
361
+ const planned = op.plan.projects.find((p) => p.project.id === progress.id);
362
+ const binding = planned && Object.hasOwn(progress.repairedSessions ?? {}, peer)
363
+ ? resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)
364
+ : (planned?.terminals as { peer: string; sessionId: string }[] | undefined)?.find((t) => t.peer === peer);
296
365
  const current = state?.state === "running" ? state.peers.find((p) => p.id === peer && p.state !== "offline") : undefined;
297
366
  return !!binding && (peer === "codex" ? current?.threadId : current?.sessionId) === binding.sessionId;
298
367
  }
@@ -350,6 +419,56 @@ function validateReceipt(id: string, op: RecoveryOperation): void {
350
419
  const { fingerprint, ...reviewed } = op.plan;
351
420
  if (fingerprint !== planFingerprint(reviewed) || op.projects.length !== op.plan.projects.length ||
352
421
  op.projects.some((p, i) => p.id !== op.plan.projects[i]?.project.id)) throw new Error("reviewed operation plan or project scope changed");
422
+ const object = (value: unknown): value is Record<string, unknown> => !!value && typeof value === "object" && !Array.isArray(value);
423
+ for (const [index, progress] of op.projects.entries()) {
424
+ if (progress.repairedSessions !== undefined) {
425
+ if (!object(progress.repairedSessions)) throw new Error("invalid repaired session provenance");
426
+ for (const [peer, protectedSession] of Object.entries(progress.repairedSessions)) {
427
+ const restored = decodeRestoreReceipt(progress.terminals[`restored:${peer}`]);
428
+ const bindings = [progress.reattachments?.[peer as RecoveryNativePeer]?.binding,
429
+ restored?.state === "done" || restored?.state === "waiting" ? restored.binding : undefined,
430
+ progress.terminals[`retired:${peer}`]];
431
+ if (protectedSession !== true || !bindings.some(binding => {
432
+ if (binding === undefined) return false;
433
+ const receipt = decodeRestoreReceipt({ state: "done", binding });
434
+ return receipt?.state === "done" && receipt.binding.peer === peer && receipt.binding.projectRoot === op.plan.projects[index]?.project.root;
435
+ })) throw new Error("repaired session provenance does not match its retained peer and project");
436
+ }
437
+ }
438
+ for (const [peer, choice] of Object.entries(progress.fresh ?? {})) {
439
+ if (choice.repaired === undefined) continue;
440
+ const planned = op.plan.projects[index]!;
441
+ if (choice.repaired !== true || !progress.repairedSessions?.[peer as RecoveryNativePeer] ||
442
+ resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)?.sessionId !== choice.lost) throw new Error("invalid one-use repaired-session choice");
443
+ }
444
+ if (progress.reattachments === undefined) continue;
445
+ if (!object(progress.reattachments)) throw new Error("invalid reattachment markers");
446
+ for (const [peer, marker] of Object.entries(progress.reattachments)) {
447
+ if (!["codex", "pi", "claude"].includes(peer) || !object(marker) || Object.keys(marker).some(key => !["state", "at", "cause", "binding"].includes(key)) ||
448
+ !["ready", "attempted"].includes(String(marker.state)) || marker.cause !== "detached after restore" || typeof marker.at !== "number" || !Number.isFinite(marker.at) || marker.at < 0) throw new Error("invalid reattachment marker");
449
+ const receipt = decodeRestoreReceipt({ state: "done", binding: marker.binding });
450
+ if (receipt?.state !== "done" || receipt.binding.peer !== peer || receipt.binding.projectRoot !== op.plan.projects[index]?.project.root) throw new Error("reattachment binding does not match its peer and project");
451
+ }
452
+ }
453
+ if (op.missingRecordedSession !== undefined) {
454
+ const evidence = op.missingRecordedSession;
455
+ if (!object(evidence) || Object.keys(evidence).some(key => !["projectId", "peer", "sessionId", "kind", "checkedPath"].includes(key)) ||
456
+ typeof evidence.projectId !== "string" || typeof evidence.peer !== "string" || typeof evidence.sessionId !== "string" ||
457
+ evidence.checkedPath !== undefined && (typeof evidence.checkedPath !== "string" || !evidence.checkedPath) ||
458
+ !["rollout", "transcript"].includes(evidence.kind) || !op.projects.some(progress => missingRepair(op, progress, evidence.peer))) throw new Error("invalid recorded-session missing evidence");
459
+ }
460
+ if (op.freshOverride !== undefined) {
461
+ const override = op.freshOverride, evidence = override.evidence;
462
+ if (!object(override) || Object.keys(override).some(key => !["peer", "evidence", "checked", "at"].includes(key)) || !object(evidence) ||
463
+ Object.keys(evidence).some(key => !["projectId", "peer", "sessionId", "kind", "checkedPath"].includes(key)) || override.checked !== "missing" ||
464
+ !Number.isFinite(override.at) || override.at < 0 || override.peer !== evidence.peer || !["rollout", "transcript"].includes(evidence.kind) ||
465
+ !op.audit?.some(entry => entry.action === "fresh-session" && entry.peer === override.peer && entry.at === override.at && entry.projects.includes(evidence.projectId))) throw new Error("invalid fresh override evidence");
466
+ }
467
+ if (op.blockedReattachment !== undefined) {
468
+ const selected = op.blockedReattachment;
469
+ if (!object(selected) || Object.keys(selected).some(key => !["projectId", "peer"].includes(key)) || typeof selected.projectId !== "string" || !["codex", "pi", "claude"].includes(String(selected.peer)) ||
470
+ !op.projects.some(project => project.id === selected.projectId && Object.hasOwn(project.reattachments ?? {}, selected.peer))) throw new Error("invalid blocked reattachment selector");
471
+ }
353
472
  }
354
473
 
355
474
  export interface RecoveryDriver {
@@ -360,16 +479,20 @@ export interface RecoveryDriver {
360
479
  commit(project: Project, operation: string, instance: string, targetFeatures?: readonly string[]): Promise<void>;
361
480
  closeTerminals(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, save: () => void): Promise<void>;
362
481
  start(project: Project, op: RecoveryOperation): Promise<void>;
363
- restore(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, group: "native" | "claude", save: () => void): Promise<void>;
482
+ restore(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, group: "native" | "claude", save: () => void, onlyPeer?: RecoveryNativePeer): Promise<void>;
364
483
  installPlugin(op: RecoveryOperation): Promise<void>;
365
484
  installGlobal(op: RecoveryOperation): Promise<void>;
366
485
  refreshManager?(op: RecoveryOperation): Promise<void>;
367
486
  release(project: Project, operation: string, instance: string): Promise<void>;
368
- verify(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation): Promise<void>;
487
+ verify(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, save?: () => void): Promise<void>;
488
+ /** Read-only positive store evidence; absence of the port means unknown. */
489
+ recordedSessionState?(binding: TerminalBinding, kind: "rollout" | "transcript"): Promise<"found" | "missing" | "unknown">;
369
490
  /** #215 stop-and-archive: stop this operation's unreleased target (when given) and archive its committed snapshot. */
370
- stopAndArchive(project: Project, op: RecoveryOperation, targetInstance?: string): Promise<void>;
491
+ stopAndArchive(project: Project, op: RecoveryOperation, targetInstance?: string): Promise<void | { continuity: "not-archived" | "unavailable" }>;
371
492
  sleep(ms: number): Promise<void>;
372
493
  now(): number;
494
+ /** Trusted per-invocation delegation, supplied only by the CLI's identity preflight. */
495
+ readonly liveTestPeer?: string;
373
496
  }
374
497
 
375
498
  /** Resumable state machine. Driver actions are preceded by receipts and followed by live readback. */
@@ -428,9 +551,20 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
428
551
  };
429
552
  // Prepare, or after an expired lease re-prepare, the planned source and wait until it is quiet.
430
553
  const prepareSource = async (planned: PlannedProject, progress: ProjectProgress, again = false) => {
554
+ const saveTiming = () => {
555
+ try { save(); }
556
+ catch { console.warn("Recovery timing observation could not be saved; recovery continues"); }
557
+ };
431
558
  const project = planned.project, instance = planned.source.instanceId!;
432
559
  await driver.prepare(project, id, instance, targetFeatures);
433
- const deadline = driver.now() + idleTimeoutMs;
560
+ const startedAt = driver.now(), deadline = startedAt + idleTimeoutMs;
561
+ const waitStep = op.step;
562
+ if (waitStep) {
563
+ (op.waits ??= {})[waitStep] = { waitStartedAt: new Date(startedAt).toISOString(), deadlineAt: new Date(deadline).toISOString() };
564
+ saveTiming();
565
+ }
566
+ let waitFailed = false;
567
+ try {
434
568
  for (;;) {
435
569
  const live = await driver.inspect(project);
436
570
  if (live.instanceId !== instance) throw new Error(`${project.id}: source daemon changed during preparation`);
@@ -446,16 +580,30 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
446
580
  }
447
581
  await driver.sleep(250);
448
582
  }
583
+ } catch (error) { waitFailed = true; throw error; }
584
+ finally {
585
+ const finish = () => {
586
+ if (!waitStep) return;
587
+ op.waits![waitStep] = { ...op.waits![waitStep]!, waitEndedAt: new Date(driver.now()).toISOString() };
588
+ saveTiming();
589
+ };
590
+ // A best-effort end observation must not replace the error that stopped the actual wait.
591
+ if (waitFailed) { try { finish(); } catch { /* original wait failure remains authoritative */ } }
592
+ else finish();
593
+ }
449
594
  };
450
595
  try {
451
596
  if (op.disposition) {
452
597
  // The stop-and-archive in progress keeps its own cause; resume never runs an operation being abandoned.
453
598
  op.error ??= `a stop-and-archive of this operation is partway; ${nextActionsText(op, await liveProjects(op, driver.inspect))}`;
454
599
  delete op.failure;
600
+ delete op.presentationCause;
455
601
  op.phase = "blocked"; save();
456
602
  return op;
457
603
  }
458
- op.phase = "running"; delete op.error; delete op.failure; save();
604
+ op.phase = "running"; delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause;
605
+ delete op.waits; // Timing is the current attempt's observation, never a prior attempt's countdown.
606
+ save();
459
607
  step("stage");
460
608
  const target = await driver.stage(op);
461
609
  if (op.targetDigest && (target.root !== op.targetRoot || target.digest !== op.targetDigest)) throw new FinalRefusal("staged target changed since operation started");
@@ -586,7 +734,7 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
586
734
  }
587
735
  step(`restore-claude:${planned.project.id}`);
588
736
  await driver.restore(planned, progress, op, "claude", save);
589
- await driver.verify(planned, progress, op);
737
+ await driver.verify(planned, progress, op, save);
590
738
  step(`release:${planned.project.id}`);
591
739
  progress.releaseRequested = true; save();
592
740
  await driver.release(planned.project, id, progress.instanceId!);
@@ -599,10 +747,18 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
599
747
  releaseRecoveryLock(id, home); op.phase = "completed"; step("completed");
600
748
  } catch (error) {
601
749
  // #215: every runner error ends with the choices status shows, read from every open source, never only this one.
750
+ const presentationError = error instanceof RecoveryPresentationError ? error
751
+ : error instanceof ReattachmentRequired && error.presentationError instanceof RecoveryPresentationError ? error.presentationError : undefined;
752
+ if (presentationError) op.presentationCause = presentationError.presentationCause;
753
+ else delete op.presentationCause;
754
+ if (error instanceof MissingRecordedSession) op.missingRecordedSession = error.evidence;
755
+ if (error instanceof ReattachmentRequired) op.blockedReattachment = { projectId: error.projectId, peer: error.peer };
756
+ else delete op.blockedReattachment;
602
757
  if (isDatabaseBusy(error)) { op.failure = { code: "database_busy", retryable: true, site: databaseBusySite(error) }; error = new RecoveryDatabaseBusy(); }
603
758
  if (error instanceof FinalRefusal) op.final = true;
759
+ op.phase = "blocked";
604
760
  const next = nextActionsText(op, await liveProjects(op, driver.inspect));
605
- op.phase = "blocked"; op.error = `${error instanceof Error ? error.message : "recovery failed"}; ${next}`; save();
761
+ op.error = `${error instanceof Error ? error.message : "recovery failed"}; ${next}`; save();
606
762
  } finally { releaseRunner(); }
607
763
  return op;
608
764
  }
@@ -610,16 +766,87 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
610
766
  /** Receipts and runner state only (#215): ids, phases and effects, never task or message text. */
611
767
  export function publicOperation(op: RecoveryOperation, runnerPid?: number | "unknown", live: Record<string, Inspection | undefined> = {}) {
612
768
  const effect = (key: string, value: unknown) => key.startsWith("restored:") ? restoreReceiptStatus(value) : value === "pending" || value === "failed" ? value : "done";
613
- const next = nextActions(op, runnerPid, live);
769
+ const choices = nextChoices(op, runnerPid, live), next = choices.map(choice => choice.text);
614
770
  return { id: op.id, phase: op.phase, step: op.step, version: op.plan.version, updatedAt: op.updatedAt,
615
771
  runner: runnerPid === "unknown" ? { state: "unknown" } : runnerPid ? { state: "running", pid: runnerPid } : { state: "none" },
616
772
  ...(op.phase === "running" && !runnerPid ? { stale: "the receipt says running but no runner holds the operation: its last runner stopped mid-step" } : {}),
617
773
  projects: op.projects.map((p) => ({ id: p.id, phase: p.phase, ...(Object.keys(p.terminals).length ? { effects: Object.fromEntries(Object.entries(p.terminals).map(([key, value]) => [key, effect(key, value)])) } : {}),
774
+ ...(op.plan.projects.find(planned => planned.project.id === p.id) ? { recordedSessions: recordedPeerGuidance(op.plan.projects.find(planned => planned.project.id === p.id)!, p, {
775
+ ...(op.phase === "blocked" && !op.disposition ? { stopAndArchiveCommand: recoveryCommand(op, "dispose", STOP) } : {}),
776
+ reattachCommand: peer => op.phase === "blocked" && !op.disposition && !op.final && op.targetRoot && op.targetDigest && p.instanceId && ["started", "peers-restored"].includes(p.phase) &&
777
+ op.blockedReattachment?.projectId === p.id && op.blockedReattachment.peer === peer && choices.some(choice => choice.kind === "reattach" && choice.peer === peer)
778
+ ? recoveryCommand(op, "reattach", peer) : undefined,
779
+ }) } : {}),
618
780
  ...(p.fresh ? { lostContinuity: Object.fromEntries(Object.entries(p.fresh).map(([peer, f]) => [peer, f.lost])) } : {}),
619
781
  ...(p.restarts?.length ? { restarts: p.restarts.length } : {}) })),
620
782
  pluginInstalled: !!op.pluginInstalled, globalInstalled: !!op.globalInstalled,
621
783
  ...(!targetReadsWaivers(op) && op.projects.some((p) => failedPeers(op, p).length) ? { freshSession: `not offered: target ${op.plan.version} cannot read recovery waivers, so it could not release a new session` } : {}),
622
- ...(op.error ? { error: op.error } : {}), ...(op.failure ? { failure: op.failure } : {}), ...(op.disposition ? { disposition: op.disposition } : {}), ...(op.audit ? { audit: op.audit } : {}), next };
784
+ ...(op.error ? { error: op.error } : {}), ...(op.failure ? { failure: op.failure } : {}), ...(op.blockedReattachment ? { blockedReattachment: op.blockedReattachment } : {}), ...(op.disposition ? { disposition: op.disposition } : {}), ...(op.audit ? { audit: op.audit } : {}), ...(op.freshOverride ? { freshOverride: op.freshOverride } : {}), ...(op.missingRecordedSession ? { missingRecordedSession: op.missingRecordedSession } : {}),
785
+ ...(op.waits ? { waits: op.waits } : {}), ...(op.presentationCause ? { presentationCause: op.presentationCause } : {}), next };
786
+ }
787
+
788
+ /**
789
+ * Person-command entry: restore one recorded conversation under this operation's existing hold.
790
+ * The driver settles and records the selected native identity; this entry never performs global verification,
791
+ * release, installation or completion. A later normal resume owns those steps.
792
+ */
793
+ export async function reattachRecovery(id: string, peer: string, driver: RecoveryDriver, home = hubHome()): Promise<RecoveryOperation> {
794
+ let op = readOperation<RecoveryOperation>(id, home);
795
+ validateReceipt(id, op);
796
+ const selection = () => {
797
+ if (!["codex", "pi", "claude"].includes(peer) || op.phase !== "blocked" || op.disposition || op.final || op.blockedReattachment?.peer !== peer) throw new Error("reattach is available only for the peer named by a blocked reattachment receipt");
798
+ const progress = op.projects.find(project => project.id === op.blockedReattachment!.projectId);
799
+ const planned = op.plan.projects.find(project => project.project.id === progress?.id);
800
+ const native = peer as RecoveryNativePeer;
801
+ const marker = progress?.reattachments?.[native];
802
+ if (!progress || !planned || !marker || !["started", "peers-restored"].includes(progress.phase) || !progress.instanceId || !op.targetRoot || !op.targetDigest ||
803
+ (marker.state !== "ready" && !repairWaiting(op, progress.id, native))) throw new Error("recorded reattachment is unavailable or its single relaunch allowance was already used; resume to inspect the saved receipt");
804
+ return { progress, planned, marker, native };
805
+ };
806
+ selection(); // Finished or unrelated operations must not acquire a new lock merely to refuse.
807
+ acquireRecoveryLock(id, home);
808
+ const releaseRunner = claimRunner(id, home);
809
+ let attempted = false;
810
+ const save = () => { op.updatedAt = driver.now(); writeOperation(id, op, home); };
811
+ try {
812
+ op = readOperation<RecoveryOperation>(id, home);
813
+ validateReceipt(id, op);
814
+ const { progress, planned, marker, native } = selection();
815
+ const expected = structuredClone(marker.binding);
816
+ const target = async () => {
817
+ const observed = await driver.inspect(planned.project);
818
+ if (observed.state !== "running" || observed.instanceId !== progress.instanceId || observed.version !== op.plan.version || observed.recovery?.operationId !== id || observed.recovery.phase === "released") throw new Error("reattach target changed or is not held by this operation; no replacement runtime is accepted");
819
+ };
820
+ await target();
821
+ attempted = true;
822
+ await driver.restore(planned, progress, op, native === "claude" ? "claude" : "native", save, native);
823
+ // A done receipt is written by the runtime only after authenticated attachment and exact settlement.
824
+ // Inspection peers may be unavailable; do not invent an attachment from a test or a stale status projection.
825
+ const restored = decodeRestoreReceipt(progress.terminals[`restored:${native}`]);
826
+ if (restored?.state !== "done" || restored.binding.peer !== native || restored.binding.projectRoot !== expected.projectRoot ||
827
+ restored.binding.sessionId !== expected.sessionId || restored.binding.sessionFile !== expected.sessionFile) throw new Error(`${native}: recorded session reattachment was not settled with its exact identity`);
828
+ await target();
829
+ delete op.blockedReattachment; delete op.failure; delete op.presentationCause;
830
+ op.phase = "blocked";
831
+ op.error = `${native}: recorded session reattached; resume to finish verification, release and any remaining installation; ${recoveryCommand(op, "resume")}`;
832
+ save();
833
+ return op;
834
+ } catch (error) {
835
+ if (attempted) {
836
+ const presentationError = error instanceof RecoveryPresentationError ? error
837
+ : error instanceof ReattachmentRequired && error.presentationError instanceof RecoveryPresentationError ? error.presentationError : undefined;
838
+ if (presentationError) op.presentationCause = presentationError.presentationCause;
839
+ else delete op.presentationCause;
840
+ if (error instanceof MissingRecordedSession) op.missingRecordedSession = error.evidence;
841
+ if (error instanceof ReattachmentRequired) op.blockedReattachment = { projectId: error.projectId, peer: error.peer };
842
+ else delete op.blockedReattachment;
843
+ if (isDatabaseBusy(error)) { op.failure = { code: "database_busy", retryable: true, site: databaseBusySite(error) }; error = new RecoveryDatabaseBusy(); }
844
+ op.phase = "blocked";
845
+ op.error = `${error instanceof Error ? error.message : "reattachment is unconfirmed"}; ${nextActionsText(op, await liveProjects(op, driver.inspect))}`;
846
+ save();
847
+ }
848
+ throw error;
849
+ } finally { releaseRunner(); }
623
850
  }
624
851
 
625
852
  /** Escape a blocked preflight without abandoning a stopped runtime or an uncertain terminal mutation. */
@@ -649,7 +876,7 @@ export async function abortRecovery(id: string, driver: RecoveryDriver, home = h
649
876
  const state = live[planned.project.id];
650
877
  if (state?.recovery?.operationId === id && state.recovery.phase !== "released") await driver.abort(planned.project, id, planned.source.instanceId!);
651
878
  }
652
- delete op.error; delete op.failure;
879
+ delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause;
653
880
  op.phase = "cancelled"; op.step = "cancelled"; op.updatedAt = driver.now();
654
881
  writeOperation(id, op, home); releaseRecoveryLock(id, home);
655
882
  } catch (error) {
@@ -664,15 +891,18 @@ export async function abortRecovery(id: string, driver: RecoveryDriver, home = h
664
891
  * every runtime this operation held or started is verified released or stopped.
665
892
  */
666
893
  export async function disposeRecovery(id: string, choice: { fresh: string } | { stop: true }, reason: string, driver: RecoveryDriver, home = hubHome()): Promise<RecoveryOperation> {
667
- let op = readOperation<RecoveryOperation>(id, home);
668
- validateReceipt(id, op);
894
+ let op: RecoveryOperation;
895
+ try { op = readOperation<RecoveryOperation>(id, home); validateReceipt(id, op); }
896
+ catch (error) { if (driver.liveTestPeer && "fresh" in choice) throw new Error(LIVE_TEST_PERSON_REFUSAL); throw error; }
897
+ if (driver.liveTestPeer && "fresh" in choice && repairedFreshSession(op, choice.fresh) !== false) throw new Error(LIVE_TEST_PERSON_REFUSAL);
669
898
  // A finished operation owns no lock: never take one only to refuse.
670
899
  if (op.phase === "completed" || op.phase === "cancelled") throw new Error(`operation is already ${op.phase}`);
671
900
  acquireRecoveryLock(id, home);
672
901
  const releaseRunner = claimRunner(id, home);
673
902
  try {
674
- op = readOperation<RecoveryOperation>(id, home);
675
- validateReceipt(id, op);
903
+ try { op = readOperation<RecoveryOperation>(id, home); validateReceipt(id, op); }
904
+ catch (error) { if (driver.liveTestPeer && "fresh" in choice) throw new Error(LIVE_TEST_PERSON_REFUSAL); throw error; }
905
+ if (driver.liveTestPeer && "fresh" in choice && repairedFreshSession(op, choice.fresh) !== false) throw new Error(LIVE_TEST_PERSON_REFUSAL);
676
906
  if (op.phase === "completed" || op.phase === "cancelled") {
677
907
  // Another runner finished it while this one waited: leave no lock behind, as resume and abort do.
678
908
  if (recoveryLock(home) === id) releaseRecoveryLock(id, home);
@@ -685,7 +915,7 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
685
915
  const next = nextActionsText(op, live);
686
916
  if ("fresh" in choice) {
687
917
  // `ahub pi` sends no fresh flag, and a restored hub refills a Pi start from its recorded resume (daemon startPeer).
688
- if (choice.fresh === "pi") throw new Error(`pi: --fresh-session is not supported: a restored hub resumes Pi's recorded session, so a fresh one cannot be guaranteed; ${next}`);
918
+ if (choice.fresh === "pi") throw new Error(`pi: --fresh-session is not supported: a restored hub resumes Pi's recorded session, so a fresh one cannot be guaranteed; ${op.plan.projects.map(planned => piArchiveExit(planned, recoveryCommand(op, "dispose", STOP))).join("; ")}; ${next}`);
689
919
  if (op.disposition) throw new Error(`a stop-and-archive of this operation is partway; ${next}`);
690
920
  if (!targetReadsWaivers(op)) throw new Error(`--fresh-session is not available: target ${op.plan.version} cannot read recovery waivers, so it could not release a new session; ${next}`);
691
921
  // The new session is launched by resume, so it helps only where resume can get past what is live.
@@ -701,12 +931,32 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
701
931
  throw new Error(`${peer}: its planned session is attached again, so nothing was lost; resume records it as restored; ${next}`);
702
932
  }
703
933
  if (!failed.length) throw new Error(`${peer}: no failed restoration of it is open to a fresh session (none failed, or one is already chosen); --fresh-session applies only then (${recoveryCommand(op, "status")})`);
934
+ // Collect every proof before changing any project, so one uncertain read cannot leave a partial choice.
935
+ const selected = [];
704
936
  for (const progress of failed) {
705
- const binding = (op.plan.projects.find((p) => p.project.id === progress.id)!.terminals as { peer: string; sessionId: string }[]).find((t) => t.peer === peer)!;
706
- (progress.fresh ??= {})[peer] = { lost: binding.sessionId, reason, at };
937
+ const repaired = missingRepair(op, progress, peer);
938
+ const planned = op.plan.projects.find((p) => p.project.id === progress.id)!;
939
+ const binding = repaired ? resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)!
940
+ : (planned.terminals as TerminalBinding[]).find((t) => t.peer === peer)!;
941
+ const evidence = repaired ? { ...op.missingRecordedSession!, checkedPath: checkedSessionPath(binding, op.missingRecordedSession!.kind) } : undefined;
942
+ if (evidence && (op.missingRecordedSession!.checkedPath !== undefined && op.missingRecordedSession!.checkedPath !== evidence.checkedPath ||
943
+ await driver.recordedSessionState?.(binding, evidence.kind) !== "missing")) throw new Error(`${peer}: recorded conversation is not positively missing at ${JSON.stringify(evidence.checkedPath)}; no fresh-session choice was saved`);
944
+ selected.push({ progress, binding, repaired, evidence });
945
+ }
946
+ for (const { progress, binding, repaired, evidence } of selected) {
947
+ (progress.fresh ??= {})[peer] = { lost: binding.sessionId, reason, at, ...(repaired ? { repaired: true as const } : {}) };
948
+ if (repaired) {
949
+ (progress.repairedSessions ??= {})[peer as RecoveryNativePeer] = true;
950
+ op.freshOverride = { peer: peer as RecoveryNativePeer, evidence: evidence!, checked: "missing", at };
951
+ }
952
+ if (repaired && progress.reattachments?.[peer as RecoveryNativePeer]) {
953
+ progress.terminals[`retired:${peer}`] = binding;
954
+ delete progress.terminals[`closedRetired:${peer}`];
955
+ delete progress.reattachments[peer as RecoveryNativePeer];
956
+ }
707
957
  }
708
958
  (op.audit ??= []).push({ at, action: "fresh-session", reason, peer, projects: failed.map((p) => p.id) });
709
- delete op.error; delete op.failure;
959
+ delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause; delete op.missingRecordedSession;
710
960
  op.updatedAt = at; writeOperation(id, op, home);
711
961
  return op;
712
962
  }
@@ -732,7 +982,9 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
732
982
  acts.push({ planned, ...(state ? { live: state } : {}), act });
733
983
  }
734
984
  // Each outcome is recorded as it happens, so a disposition that stops partway stays true and resume refuses it.
735
- const disposition = (op.disposition ??= { choice: "stop-and-archive", at, projects: {} });
985
+ const disposition: NonNullable<RecoveryOperation["disposition"]> = (op.disposition ??= { choice: "stop-and-archive", at, projects: {} });
986
+ delete op.blockedReattachment;
987
+ delete op.presentationCause;
736
988
  abandoned(op); // #227: in the same write as the disposition
737
989
  (op.audit ??= []).push({ at, action: "stop-and-archive", reason, projects: acts.map((a) => a.planned.project.id) });
738
990
  // Recorded before the first act: a crash inside a stop leaves a disposition that resume refuses to override.
@@ -742,7 +994,8 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
742
994
  for (const { planned, live, act } of acts) {
743
995
  if (act.startsWith("source hold")) await driver.abort(planned.project, id, planned.source.instanceId!);
744
996
  else if (act === "stopped" || act === "target stopped") {
745
- await driver.stopAndArchive(planned.project, op, act === "target stopped" ? live!.instanceId : undefined);
997
+ const outcome = await driver.stopAndArchive(planned.project, op, act === "target stopped" ? live!.instanceId : undefined);
998
+ if (outcome && outcome.continuity) (disposition.continuity ??= {})[planned.project.id] = outcome.continuity;
746
999
  if ((await driver.inspect(planned.project)).state !== "stopped") throw new Error(`${planned.project.id}: shutdown of instance ${live!.instanceId ?? "unknown"} is not verified`);
747
1000
  }
748
1001
  disposition.projects[planned.project.id] ??= act;