@staix/agent-hub 0.12.31 → 0.12.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -1
- package/README.md +14 -7
- package/docs/agent-notes/adapters.md +7 -4
- package/docs/agent-notes/bus.md +3 -0
- package/docs/agent-notes/daemon.md +13 -8
- package/docs/agent-notes/local-worker.md +1 -1
- package/docs/agent-notes/tasks.md +4 -3
- package/docs/agent-notes/tests.md +1 -0
- package/docs/cooperbench.md +1 -1
- package/docs/gsd-collaboration-analysis.md +1 -1
- package/docs/memory.md +1 -1
- package/docs/operations.md +141 -25
- package/docs/quickstart.md +10 -6
- package/docs/security.md +18 -9
- package/docs/smoke.md +34 -0
- package/docs/specs/2026-09-19-agent-hub-design.md +204 -0
- package/docs/specs/2026-09-20-upgrade-recovery-design.md +211 -0
- package/docs/verification/2026-10-18-0.12.32.md +54 -0
- package/docs/verification/2026-10-18-0.12.33.md +42 -0
- package/docs/verified.json +24 -16
- package/package.json +2 -2
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +1296 -26
- package/src/adapters/acp.ts +5 -6
- package/src/adapters/definitions/builtin-metadata.ts +121 -0
- package/src/adapters/definitions/kimi-native.ts +4 -0
- package/src/adapters/definitions/qwen-native.ts +24 -0
- package/src/adapters/definitions/qwen.ts +71 -0
- package/src/adapters/pi.ts +6 -2
- package/src/adapters/registry-types.ts +174 -0
- package/src/adapters/registry.ts +36 -0
- package/src/cli/console-state.ts +74 -5
- package/src/cli/console.ts +35 -2
- package/src/cli/help.ts +3 -1
- package/src/cli/identity.ts +13 -3
- package/src/cli/launch.ts +1 -6
- package/src/cli/main.ts +200 -39
- package/src/cli/output.ts +2 -0
- package/src/cli/preview.ts +8 -5
- package/src/cli/recovery-guidance.ts +137 -0
- package/src/cli/recovery-presentation-options.ts +8 -0
- package/src/cli/recovery-session-file.ts +18 -0
- package/src/cli/reset.ts +13 -1
- package/src/cli/terminal-recovery.ts +48 -0
- package/src/cli/upgrade-interactive.ts +129 -75
- package/src/cli/upgrade-labels.ts +30 -0
- package/src/cli/upgrade-presentation-data.ts +41 -0
- package/src/cli/upgrade-presentation-flags.ts +31 -0
- package/src/cli/upgrade-presentation.ts +227 -0
- package/src/cli/upgrade-restore-state.ts +6 -2
- package/src/cli/upgrade-runtime.ts +360 -199
- package/src/cli/upgrade.ts +278 -25
- package/src/hub/bus.ts +144 -14
- package/src/hub/cohorts.ts +2 -2
- package/src/hub/config-apply.ts +242 -0
- package/src/hub/config-reload.ts +120 -0
- package/src/hub/config-trust.ts +11 -2
- package/src/hub/control-client.ts +1 -1
- package/src/hub/crash.ts +8 -6
- package/src/hub/daemon.ts +577 -179
- package/src/hub/deferred-hub.ts +60 -0
- package/src/hub/delivery-journal.ts +6 -0
- package/src/hub/events.ts +2 -2
- package/src/hub/manager.ts +1 -1
- package/src/hub/peer-catalog.ts +10 -13
- package/src/hub/peers.ts +12 -2
- package/src/hub/permission-mode.ts +5 -3
- package/src/hub/recovery-continuity.ts +152 -0
- package/src/hub/recovery-store.ts +10 -4
- package/src/hub/restart.ts +12 -1
- package/src/hub/sdlc.ts +5 -1
- package/src/hub/settings.ts +50 -36
- package/src/hub/settled-roles.ts +35 -0
- package/src/hub/supervision.ts +3 -2
- package/src/hub/tasks.ts +29 -9
- package/src/hub/ui.ts +1 -1
- package/src/memory/brief.ts +6 -5
- package/src/pi/session.ts +23 -4
- package/src/ui/index.html +23 -7
package/src/cli/upgrade.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { LIVE_TEST_PERSON_REFUSAL } from "./identity.ts";
|
|
2
|
+
import { checkedSessionPath } from "./recovery-session-file.ts";
|
|
1
3
|
import { canChooseFresh, decodeRestoreReceipt, restoreReceiptStatus } from "./upgrade-restore-state.ts";
|
|
2
4
|
import { databaseBusySite, isDatabaseBusy, RecoveryDatabaseBusy } from "../hub/recovery-error.ts";
|
|
3
5
|
import { Database } from "bun:sqlite";
|
|
@@ -7,6 +9,9 @@ import { join } from "node:path";
|
|
|
7
9
|
import { REGISTRY_DB_BUSY_TIMEOUT_MS, PROJECT_COLUMNS, projectFromRow, type Project } from "../hub/registry.ts";
|
|
8
10
|
import { hubHome } from "../hub/project.ts";
|
|
9
11
|
import { acquireRecoveryLock, claimRunner, coordinatorCurrent, readOperation, recoveryCommand, recoveryLock, releaseRecoveryLock, writeOperation } from "../hub/recovery-store.ts";
|
|
12
|
+
import { resolveAcceptedBinding, piArchiveExit, recordedPeerGuidance, type RecoveryNativePeer } from "./recovery-guidance.ts";
|
|
13
|
+
import type { TerminalBinding } from "./terminal-recovery.ts";
|
|
14
|
+
import { RecoveryPresentationError, type RecoveryPresentationCause, type RecoveryWaitTiming } from "./upgrade-presentation-data.ts";
|
|
10
15
|
|
|
11
16
|
export interface RecoveryPeer {
|
|
12
17
|
id: string;
|
|
@@ -27,6 +32,8 @@ export interface Inspection {
|
|
|
27
32
|
blockers: string[];
|
|
28
33
|
/** #215: for a stopped runtime, the operation whose committed restart snapshot its state dir holds (unreleased). */
|
|
29
34
|
snapshot?: string;
|
|
35
|
+
/** The runtime may be positively stopped while its restart snapshot remains unreadable; never claim ownership. */
|
|
36
|
+
snapshotError?: { code: "recovery_deferred_hub"; message: string };
|
|
30
37
|
}
|
|
31
38
|
export interface PlannedProject {
|
|
32
39
|
project: Project;
|
|
@@ -54,6 +61,10 @@ export interface ProjectProgress {
|
|
|
54
61
|
phase: "pending" | "prepared" | "stopped" | "started" | "peers-restored" | "verified";
|
|
55
62
|
instanceId?: string;
|
|
56
63
|
terminals: Record<string, unknown>;
|
|
64
|
+
/** One durable repair allowance per peer, retained after an attempt to prevent a second relaunch. */
|
|
65
|
+
reattachments?: Partial<Record<RecoveryNativePeer, ReattachmentMarker>>;
|
|
66
|
+
/** Recorded-session-only continuity, independent of a target incarnation's repair attempt allowance. */
|
|
67
|
+
repairedSessions?: Partial<Record<RecoveryNativePeer, true>>;
|
|
57
68
|
/** #414: last bounded native-attachment timeout, metadata only; never terminal output or task text. */
|
|
58
69
|
attachmentTimeouts?: Record<string, {
|
|
59
70
|
at: number;
|
|
@@ -64,7 +75,7 @@ export interface ProjectProgress {
|
|
|
64
75
|
}>;
|
|
65
76
|
releaseRequested?: boolean;
|
|
66
77
|
/** #215: peers the operator moved to a fresh session, with the session or thread id whose continuity is lost. */
|
|
67
|
-
fresh?: Record<string, { lost: string; reason: string; at: number }>;
|
|
78
|
+
fresh?: Record<string, { lost: string; reason: string; at: number; repaired?: true }>;
|
|
68
79
|
/** #215: set before the commit request is sent; from then on the source may have committed (abort is not offered). */
|
|
69
80
|
commitSent?: boolean;
|
|
70
81
|
/**
|
|
@@ -73,6 +84,12 @@ export interface ProjectProgress {
|
|
|
73
84
|
*/
|
|
74
85
|
restarts?: { instanceId: string; at: number }[];
|
|
75
86
|
}
|
|
87
|
+
export interface ReattachmentMarker {
|
|
88
|
+
state: "ready" | "attempted";
|
|
89
|
+
at: number;
|
|
90
|
+
cause: "detached after restore";
|
|
91
|
+
binding: TerminalBinding;
|
|
92
|
+
}
|
|
76
93
|
export interface RecoveryOperation {
|
|
77
94
|
/**
|
|
78
95
|
* #227: 2 marks a stop-and-archive recorded on an operation whose coordinator predates #215. Every older runner and
|
|
@@ -94,10 +111,18 @@ export interface RecoveryOperation {
|
|
|
94
111
|
pluginInstalled?: boolean;
|
|
95
112
|
globalInstalled?: boolean;
|
|
96
113
|
error?: string;
|
|
114
|
+
/** Presentation observations only: old receipts without these fields have unknown timings and cause. */
|
|
115
|
+
waits?: Record<string, RecoveryWaitTiming>;
|
|
116
|
+
presentationCause?: RecoveryPresentationCause;
|
|
97
117
|
failure?: { code: "database_busy"; retryable: true; site?: string[] };
|
|
118
|
+
/** A typed verify outcome, never inferred from an error message or a peer's current state. */
|
|
119
|
+
blockedReattachment?: { projectId: string; peer: RecoveryNativePeer };
|
|
120
|
+
/** Positive typed store observation; disposal rechecks it before accepting a repaired-session loss. */
|
|
121
|
+
missingRecordedSession?: { projectId: string; peer: RecoveryNativePeer; sessionId: string; kind: "rollout" | "transcript"; checkedPath?: string };
|
|
122
|
+
freshOverride?: { peer: RecoveryNativePeer; evidence: NonNullable<RecoveryOperation["missingRecordedSession"]>; checked: "missing"; at: number };
|
|
98
123
|
/** #215: every human disposition, in order. Operator reasons only; never task or message text. */
|
|
99
124
|
audit?: { at: number; action: "fresh-session" | "stop-and-archive"; reason: string; peer?: string; projects: string[] }[];
|
|
100
|
-
disposition?: { choice: "stop-and-archive"; at: number; projects: Record<string, string> };
|
|
125
|
+
disposition?: { choice: "stop-and-archive"; at: number; projects: Record<string, string>; continuity?: Record<string, "not-archived" | "unavailable"> };
|
|
101
126
|
/** #215: a refusal no resume can get past was recorded (FinalRefusal); `next` no longer offers resume. */
|
|
102
127
|
final?: boolean;
|
|
103
128
|
}
|
|
@@ -105,6 +130,16 @@ export interface RecoveryOperation {
|
|
|
105
130
|
/** #215: a refusal that holds for every resume of the operation (its staged target or preserved source is fixed). */
|
|
106
131
|
export class FinalRefusal extends Error {}
|
|
107
132
|
|
|
133
|
+
export class ReattachmentRequired extends Error {
|
|
134
|
+
constructor(public readonly projectId: string, public readonly peer: RecoveryNativePeer, message: string, public readonly presentationError?: RecoveryPresentationError) { super(message); }
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export class MissingRecordedSession extends Error {
|
|
138
|
+
readonly code = "recorded_session_missing";
|
|
139
|
+
constructor(readonly evidence: NonNullable<RecoveryOperation["missingRecordedSession"]>) {
|
|
140
|
+
super(`${evidence.peer}: recorded session ${evidence.sessionId} has no resumable transcript (positively missing ${evidence.kind}${evidence.checkedPath ? ` checked at ${JSON.stringify(evidence.checkedPath)}` : ""}); choose dispose --fresh-session ${evidence.peer} --reason <text> to accept its loss, or stop-and-archive`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
108
143
|
export { recoveryCommand };
|
|
109
144
|
const STOP = "--stop-and-archive --reason <text>";
|
|
110
145
|
/**
|
|
@@ -145,7 +180,12 @@ export function recordRestart(progress: ProjectProgress, at: number): void {
|
|
|
145
180
|
progress.terminals[`retired:${peer}`] = receipt.binding;
|
|
146
181
|
delete progress.terminals[`closedRetired:${peer}`]; // a newer binding to close: its own receipt starts over
|
|
147
182
|
delete progress.terminals[key];
|
|
183
|
+
// Older receipts may have only the attempt marker. Retiring that binding must not retire its continuity rule.
|
|
184
|
+
if (Object.hasOwn(progress.reattachments ?? {}, peer)) (progress.repairedSessions ??= {})[peer as RecoveryNativePeer] = true;
|
|
185
|
+
// The allowance belongs to the target whose exact binding was retired, not every later #225 restart.
|
|
186
|
+
if (progress.reattachments) delete progress.reattachments[peer as RecoveryNativePeer];
|
|
148
187
|
}
|
|
188
|
+
if (progress.reattachments && !Object.keys(progress.reattachments).length) delete progress.reattachments;
|
|
149
189
|
delete progress.instanceId;
|
|
150
190
|
progress.phase = "stopped";
|
|
151
191
|
}
|
|
@@ -220,7 +260,7 @@ export function disposeRefusal(op: RecoveryOperation, live: Record<string, Inspe
|
|
|
220
260
|
|
|
221
261
|
/** #272: one entry of `next`. `text` is the line status prints; a screen runs `kind` through `recoveryArgv`. */
|
|
222
262
|
export interface NextChoice {
|
|
223
|
-
kind: "resume" | "abort" | "fresh" | "stop" | "wait";
|
|
263
|
+
kind: "resume" | "abort" | "fresh" | "stop" | "wait" | "reattach";
|
|
224
264
|
/** `fresh` only: the peer that starts as a new session. */
|
|
225
265
|
peer?: string;
|
|
226
266
|
text: string;
|
|
@@ -245,9 +285,13 @@ export function nextChoices(receipt: RecoveryOperation, runner?: number | "unkno
|
|
|
245
285
|
const old = !!op.sourceRoot && !coordinatorCurrent(op.sourceRoot);
|
|
246
286
|
const abortable = !abortRefusal(op, live);
|
|
247
287
|
const stuck = resumeBlocked(op, live);
|
|
288
|
+
const selected = op.phase === "blocked" ? op.blockedReattachment : undefined;
|
|
289
|
+
const repair = selected && op.projects.find(project => project.id === selected.projectId)?.reattachments?.[selected.peer];
|
|
290
|
+
const reattach = selected && repair && (repair.state === "ready" || repairWaiting(op, selected.projectId, selected.peer));
|
|
248
291
|
return [
|
|
249
292
|
...(stuck ? [] : [{ kind: "resume" as const, text: `${recoveryCommand(op, "resume")}${op.error ? " (after the next action in error)" : ""}${old ? ` (runs the coordinator that started this operation, which cannot re-prepare an expired hold: if it reports "source is no longer prepared", use ${abortable ? "abort or " : ""}stop-and-archive)` : ""}` }]),
|
|
250
293
|
...(abortable ? [{ kind: "abort" as const, text: recoveryCommand(op, "abort") }] : []),
|
|
294
|
+
...(selected && reattach ? [{ kind: "reattach" as const, peer: selected.peer, text: recoveryCommand(op, "reattach", selected.peer) }] : []),
|
|
251
295
|
...(targetReadsWaivers(op) && !stuck ? failed.map((peer) => ({ kind: "fresh" as const, peer, text: recoveryCommand(op, "dispose", `--fresh-session ${peer} --reason <text>`) })) : []),
|
|
252
296
|
stop,
|
|
253
297
|
];
|
|
@@ -278,6 +322,13 @@ export async function liveProjects(op: RecoveryOperation, inspect: (project: Pro
|
|
|
278
322
|
return Object.fromEntries(await Promise.all(open.map(async (p) => [p.project.id, await inspect(p.project).catch(() => undefined)] as const)));
|
|
279
323
|
}
|
|
280
324
|
|
|
325
|
+
function missingRepair(op: RecoveryOperation, progress: ProjectProgress, peer: string): boolean {
|
|
326
|
+
const evidence = op.missingRecordedSession;
|
|
327
|
+
if (!evidence || evidence.projectId !== progress.id || evidence.peer !== peer || (!progress.repairedSessions?.[evidence.peer] && !Object.hasOwn(progress.reattachments ?? {}, peer))) return false;
|
|
328
|
+
const planned = op.plan.projects.find(project => project.project.id === progress.id);
|
|
329
|
+
if (!planned) return false;
|
|
330
|
+
try { return resolveAcceptedBinding(planned, progress, evidence.peer)?.sessionId === evidence.sessionId; } catch { return false; }
|
|
331
|
+
}
|
|
281
332
|
/**
|
|
282
333
|
* Peers of a project whose restoration failed and for which a fresh session is a real choice: not Pi (a restored hub
|
|
283
334
|
* resumes its recorded session), not one already chosen, and not a planned fresh start (nothing to lose; resume
|
|
@@ -286,13 +337,31 @@ export async function liveProjects(op: RecoveryOperation, inspect: (project: Pro
|
|
|
286
337
|
function failedPeers(op: RecoveryOperation, progress: ProjectProgress, live: Record<string, Inspection | undefined> = {}): string[] {
|
|
287
338
|
const planned = op.plan.projects.find((p) => p.project.id === progress.id);
|
|
288
339
|
return Object.entries(progress.terminals).filter(([key, value]) => key.startsWith("restored:") && canChooseFresh(value)).map(([key]) => key.slice("restored:".length))
|
|
289
|
-
.filter((peer) => peer !== "pi" && !progress.fresh?.[peer] && !planned?.freshStart?.includes(peer) && !plannedAttached(op, progress, peer, live));
|
|
340
|
+
.filter((peer) => peer !== "pi" && ((!Object.hasOwn(progress.reattachments ?? {}, peer) && !Object.hasOwn(progress.repairedSessions ?? {}, peer)) || missingRepair(op, progress, peer)) && !progress.fresh?.[peer] && (!planned?.freshStart?.includes(peer) || missingRepair(op, progress, peer)) && !plannedAttached(op, progress, peer, live));
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/** Read-only scope classification; it confers no delegation authority. */
|
|
344
|
+
export function repairedFreshSession(op: RecoveryOperation, peer: string): boolean | undefined {
|
|
345
|
+
try {
|
|
346
|
+
validateReceipt(op.id, op);
|
|
347
|
+
if (!op.plan.projects.some(planned => planned.source.peers.some(row => row.id === peer) ||
|
|
348
|
+
planned.terminals.some(value => !!value && typeof value === "object" && (value as { peer?: unknown }).peer === peer))) return undefined;
|
|
349
|
+
return op.projects.some(progress => failedPeers(op, progress).includes(peer) && missingRepair(op, progress, peer));
|
|
350
|
+
} catch { return undefined; } // Unknown scope never becomes an ordinary delegated disposition.
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
function repairWaiting(op: RecoveryOperation, projectId: string, peer: RecoveryNativePeer): boolean {
|
|
354
|
+
try { return decodeRestoreReceipt(op.projects.find(project => project.id === projectId)?.terminals[`restored:${peer}`])?.state === "waiting"; }
|
|
355
|
+
catch { return false; }
|
|
290
356
|
}
|
|
291
357
|
|
|
292
358
|
/** #215: the inspection shows the planned session attached for `peer`, so resume records it as restored: nothing is lost. */
|
|
293
359
|
function plannedAttached(op: RecoveryOperation, progress: ProjectProgress, peer: string, live: Record<string, Inspection | undefined>): boolean {
|
|
294
360
|
const state = live[progress.id];
|
|
295
|
-
const
|
|
361
|
+
const planned = op.plan.projects.find((p) => p.project.id === progress.id);
|
|
362
|
+
const binding = planned && Object.hasOwn(progress.repairedSessions ?? {}, peer)
|
|
363
|
+
? resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)
|
|
364
|
+
: (planned?.terminals as { peer: string; sessionId: string }[] | undefined)?.find((t) => t.peer === peer);
|
|
296
365
|
const current = state?.state === "running" ? state.peers.find((p) => p.id === peer && p.state !== "offline") : undefined;
|
|
297
366
|
return !!binding && (peer === "codex" ? current?.threadId : current?.sessionId) === binding.sessionId;
|
|
298
367
|
}
|
|
@@ -350,6 +419,56 @@ function validateReceipt(id: string, op: RecoveryOperation): void {
|
|
|
350
419
|
const { fingerprint, ...reviewed } = op.plan;
|
|
351
420
|
if (fingerprint !== planFingerprint(reviewed) || op.projects.length !== op.plan.projects.length ||
|
|
352
421
|
op.projects.some((p, i) => p.id !== op.plan.projects[i]?.project.id)) throw new Error("reviewed operation plan or project scope changed");
|
|
422
|
+
const object = (value: unknown): value is Record<string, unknown> => !!value && typeof value === "object" && !Array.isArray(value);
|
|
423
|
+
for (const [index, progress] of op.projects.entries()) {
|
|
424
|
+
if (progress.repairedSessions !== undefined) {
|
|
425
|
+
if (!object(progress.repairedSessions)) throw new Error("invalid repaired session provenance");
|
|
426
|
+
for (const [peer, protectedSession] of Object.entries(progress.repairedSessions)) {
|
|
427
|
+
const restored = decodeRestoreReceipt(progress.terminals[`restored:${peer}`]);
|
|
428
|
+
const bindings = [progress.reattachments?.[peer as RecoveryNativePeer]?.binding,
|
|
429
|
+
restored?.state === "done" || restored?.state === "waiting" ? restored.binding : undefined,
|
|
430
|
+
progress.terminals[`retired:${peer}`]];
|
|
431
|
+
if (protectedSession !== true || !bindings.some(binding => {
|
|
432
|
+
if (binding === undefined) return false;
|
|
433
|
+
const receipt = decodeRestoreReceipt({ state: "done", binding });
|
|
434
|
+
return receipt?.state === "done" && receipt.binding.peer === peer && receipt.binding.projectRoot === op.plan.projects[index]?.project.root;
|
|
435
|
+
})) throw new Error("repaired session provenance does not match its retained peer and project");
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
for (const [peer, choice] of Object.entries(progress.fresh ?? {})) {
|
|
439
|
+
if (choice.repaired === undefined) continue;
|
|
440
|
+
const planned = op.plan.projects[index]!;
|
|
441
|
+
if (choice.repaired !== true || !progress.repairedSessions?.[peer as RecoveryNativePeer] ||
|
|
442
|
+
resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)?.sessionId !== choice.lost) throw new Error("invalid one-use repaired-session choice");
|
|
443
|
+
}
|
|
444
|
+
if (progress.reattachments === undefined) continue;
|
|
445
|
+
if (!object(progress.reattachments)) throw new Error("invalid reattachment markers");
|
|
446
|
+
for (const [peer, marker] of Object.entries(progress.reattachments)) {
|
|
447
|
+
if (!["codex", "pi", "claude"].includes(peer) || !object(marker) || Object.keys(marker).some(key => !["state", "at", "cause", "binding"].includes(key)) ||
|
|
448
|
+
!["ready", "attempted"].includes(String(marker.state)) || marker.cause !== "detached after restore" || typeof marker.at !== "number" || !Number.isFinite(marker.at) || marker.at < 0) throw new Error("invalid reattachment marker");
|
|
449
|
+
const receipt = decodeRestoreReceipt({ state: "done", binding: marker.binding });
|
|
450
|
+
if (receipt?.state !== "done" || receipt.binding.peer !== peer || receipt.binding.projectRoot !== op.plan.projects[index]?.project.root) throw new Error("reattachment binding does not match its peer and project");
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
if (op.missingRecordedSession !== undefined) {
|
|
454
|
+
const evidence = op.missingRecordedSession;
|
|
455
|
+
if (!object(evidence) || Object.keys(evidence).some(key => !["projectId", "peer", "sessionId", "kind", "checkedPath"].includes(key)) ||
|
|
456
|
+
typeof evidence.projectId !== "string" || typeof evidence.peer !== "string" || typeof evidence.sessionId !== "string" ||
|
|
457
|
+
evidence.checkedPath !== undefined && (typeof evidence.checkedPath !== "string" || !evidence.checkedPath) ||
|
|
458
|
+
!["rollout", "transcript"].includes(evidence.kind) || !op.projects.some(progress => missingRepair(op, progress, evidence.peer))) throw new Error("invalid recorded-session missing evidence");
|
|
459
|
+
}
|
|
460
|
+
if (op.freshOverride !== undefined) {
|
|
461
|
+
const override = op.freshOverride, evidence = override.evidence;
|
|
462
|
+
if (!object(override) || Object.keys(override).some(key => !["peer", "evidence", "checked", "at"].includes(key)) || !object(evidence) ||
|
|
463
|
+
Object.keys(evidence).some(key => !["projectId", "peer", "sessionId", "kind", "checkedPath"].includes(key)) || override.checked !== "missing" ||
|
|
464
|
+
!Number.isFinite(override.at) || override.at < 0 || override.peer !== evidence.peer || !["rollout", "transcript"].includes(evidence.kind) ||
|
|
465
|
+
!op.audit?.some(entry => entry.action === "fresh-session" && entry.peer === override.peer && entry.at === override.at && entry.projects.includes(evidence.projectId))) throw new Error("invalid fresh override evidence");
|
|
466
|
+
}
|
|
467
|
+
if (op.blockedReattachment !== undefined) {
|
|
468
|
+
const selected = op.blockedReattachment;
|
|
469
|
+
if (!object(selected) || Object.keys(selected).some(key => !["projectId", "peer"].includes(key)) || typeof selected.projectId !== "string" || !["codex", "pi", "claude"].includes(String(selected.peer)) ||
|
|
470
|
+
!op.projects.some(project => project.id === selected.projectId && Object.hasOwn(project.reattachments ?? {}, selected.peer))) throw new Error("invalid blocked reattachment selector");
|
|
471
|
+
}
|
|
353
472
|
}
|
|
354
473
|
|
|
355
474
|
export interface RecoveryDriver {
|
|
@@ -360,16 +479,20 @@ export interface RecoveryDriver {
|
|
|
360
479
|
commit(project: Project, operation: string, instance: string, targetFeatures?: readonly string[]): Promise<void>;
|
|
361
480
|
closeTerminals(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, save: () => void): Promise<void>;
|
|
362
481
|
start(project: Project, op: RecoveryOperation): Promise<void>;
|
|
363
|
-
restore(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, group: "native" | "claude", save: () => void): Promise<void>;
|
|
482
|
+
restore(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, group: "native" | "claude", save: () => void, onlyPeer?: RecoveryNativePeer): Promise<void>;
|
|
364
483
|
installPlugin(op: RecoveryOperation): Promise<void>;
|
|
365
484
|
installGlobal(op: RecoveryOperation): Promise<void>;
|
|
366
485
|
refreshManager?(op: RecoveryOperation): Promise<void>;
|
|
367
486
|
release(project: Project, operation: string, instance: string): Promise<void>;
|
|
368
|
-
verify(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation): Promise<void>;
|
|
487
|
+
verify(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, save?: () => void): Promise<void>;
|
|
488
|
+
/** Read-only positive store evidence; absence of the port means unknown. */
|
|
489
|
+
recordedSessionState?(binding: TerminalBinding, kind: "rollout" | "transcript"): Promise<"found" | "missing" | "unknown">;
|
|
369
490
|
/** #215 stop-and-archive: stop this operation's unreleased target (when given) and archive its committed snapshot. */
|
|
370
|
-
stopAndArchive(project: Project, op: RecoveryOperation, targetInstance?: string): Promise<void>;
|
|
491
|
+
stopAndArchive(project: Project, op: RecoveryOperation, targetInstance?: string): Promise<void | { continuity: "not-archived" | "unavailable" }>;
|
|
371
492
|
sleep(ms: number): Promise<void>;
|
|
372
493
|
now(): number;
|
|
494
|
+
/** Trusted per-invocation delegation, supplied only by the CLI's identity preflight. */
|
|
495
|
+
readonly liveTestPeer?: string;
|
|
373
496
|
}
|
|
374
497
|
|
|
375
498
|
/** Resumable state machine. Driver actions are preceded by receipts and followed by live readback. */
|
|
@@ -428,9 +551,20 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
|
|
|
428
551
|
};
|
|
429
552
|
// Prepare, or after an expired lease re-prepare, the planned source and wait until it is quiet.
|
|
430
553
|
const prepareSource = async (planned: PlannedProject, progress: ProjectProgress, again = false) => {
|
|
554
|
+
const saveTiming = () => {
|
|
555
|
+
try { save(); }
|
|
556
|
+
catch { console.warn("Recovery timing observation could not be saved; recovery continues"); }
|
|
557
|
+
};
|
|
431
558
|
const project = planned.project, instance = planned.source.instanceId!;
|
|
432
559
|
await driver.prepare(project, id, instance, targetFeatures);
|
|
433
|
-
const
|
|
560
|
+
const startedAt = driver.now(), deadline = startedAt + idleTimeoutMs;
|
|
561
|
+
const waitStep = op.step;
|
|
562
|
+
if (waitStep) {
|
|
563
|
+
(op.waits ??= {})[waitStep] = { waitStartedAt: new Date(startedAt).toISOString(), deadlineAt: new Date(deadline).toISOString() };
|
|
564
|
+
saveTiming();
|
|
565
|
+
}
|
|
566
|
+
let waitFailed = false;
|
|
567
|
+
try {
|
|
434
568
|
for (;;) {
|
|
435
569
|
const live = await driver.inspect(project);
|
|
436
570
|
if (live.instanceId !== instance) throw new Error(`${project.id}: source daemon changed during preparation`);
|
|
@@ -446,16 +580,30 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
|
|
|
446
580
|
}
|
|
447
581
|
await driver.sleep(250);
|
|
448
582
|
}
|
|
583
|
+
} catch (error) { waitFailed = true; throw error; }
|
|
584
|
+
finally {
|
|
585
|
+
const finish = () => {
|
|
586
|
+
if (!waitStep) return;
|
|
587
|
+
op.waits![waitStep] = { ...op.waits![waitStep]!, waitEndedAt: new Date(driver.now()).toISOString() };
|
|
588
|
+
saveTiming();
|
|
589
|
+
};
|
|
590
|
+
// A best-effort end observation must not replace the error that stopped the actual wait.
|
|
591
|
+
if (waitFailed) { try { finish(); } catch { /* original wait failure remains authoritative */ } }
|
|
592
|
+
else finish();
|
|
593
|
+
}
|
|
449
594
|
};
|
|
450
595
|
try {
|
|
451
596
|
if (op.disposition) {
|
|
452
597
|
// The stop-and-archive in progress keeps its own cause; resume never runs an operation being abandoned.
|
|
453
598
|
op.error ??= `a stop-and-archive of this operation is partway; ${nextActionsText(op, await liveProjects(op, driver.inspect))}`;
|
|
454
599
|
delete op.failure;
|
|
600
|
+
delete op.presentationCause;
|
|
455
601
|
op.phase = "blocked"; save();
|
|
456
602
|
return op;
|
|
457
603
|
}
|
|
458
|
-
op.phase = "running"; delete op.error; delete op.failure;
|
|
604
|
+
op.phase = "running"; delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause;
|
|
605
|
+
delete op.waits; // Timing is the current attempt's observation, never a prior attempt's countdown.
|
|
606
|
+
save();
|
|
459
607
|
step("stage");
|
|
460
608
|
const target = await driver.stage(op);
|
|
461
609
|
if (op.targetDigest && (target.root !== op.targetRoot || target.digest !== op.targetDigest)) throw new FinalRefusal("staged target changed since operation started");
|
|
@@ -586,7 +734,7 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
|
|
|
586
734
|
}
|
|
587
735
|
step(`restore-claude:${planned.project.id}`);
|
|
588
736
|
await driver.restore(planned, progress, op, "claude", save);
|
|
589
|
-
await driver.verify(planned, progress, op);
|
|
737
|
+
await driver.verify(planned, progress, op, save);
|
|
590
738
|
step(`release:${planned.project.id}`);
|
|
591
739
|
progress.releaseRequested = true; save();
|
|
592
740
|
await driver.release(planned.project, id, progress.instanceId!);
|
|
@@ -599,10 +747,18 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
|
|
|
599
747
|
releaseRecoveryLock(id, home); op.phase = "completed"; step("completed");
|
|
600
748
|
} catch (error) {
|
|
601
749
|
// #215: every runner error ends with the choices status shows, read from every open source, never only this one.
|
|
750
|
+
const presentationError = error instanceof RecoveryPresentationError ? error
|
|
751
|
+
: error instanceof ReattachmentRequired && error.presentationError instanceof RecoveryPresentationError ? error.presentationError : undefined;
|
|
752
|
+
if (presentationError) op.presentationCause = presentationError.presentationCause;
|
|
753
|
+
else delete op.presentationCause;
|
|
754
|
+
if (error instanceof MissingRecordedSession) op.missingRecordedSession = error.evidence;
|
|
755
|
+
if (error instanceof ReattachmentRequired) op.blockedReattachment = { projectId: error.projectId, peer: error.peer };
|
|
756
|
+
else delete op.blockedReattachment;
|
|
602
757
|
if (isDatabaseBusy(error)) { op.failure = { code: "database_busy", retryable: true, site: databaseBusySite(error) }; error = new RecoveryDatabaseBusy(); }
|
|
603
758
|
if (error instanceof FinalRefusal) op.final = true;
|
|
759
|
+
op.phase = "blocked";
|
|
604
760
|
const next = nextActionsText(op, await liveProjects(op, driver.inspect));
|
|
605
|
-
op.
|
|
761
|
+
op.error = `${error instanceof Error ? error.message : "recovery failed"}; ${next}`; save();
|
|
606
762
|
} finally { releaseRunner(); }
|
|
607
763
|
return op;
|
|
608
764
|
}
|
|
@@ -610,16 +766,87 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
|
|
|
610
766
|
/** Receipts and runner state only (#215): ids, phases and effects, never task or message text. */
|
|
611
767
|
export function publicOperation(op: RecoveryOperation, runnerPid?: number | "unknown", live: Record<string, Inspection | undefined> = {}) {
|
|
612
768
|
const effect = (key: string, value: unknown) => key.startsWith("restored:") ? restoreReceiptStatus(value) : value === "pending" || value === "failed" ? value : "done";
|
|
613
|
-
const
|
|
769
|
+
const choices = nextChoices(op, runnerPid, live), next = choices.map(choice => choice.text);
|
|
614
770
|
return { id: op.id, phase: op.phase, step: op.step, version: op.plan.version, updatedAt: op.updatedAt,
|
|
615
771
|
runner: runnerPid === "unknown" ? { state: "unknown" } : runnerPid ? { state: "running", pid: runnerPid } : { state: "none" },
|
|
616
772
|
...(op.phase === "running" && !runnerPid ? { stale: "the receipt says running but no runner holds the operation: its last runner stopped mid-step" } : {}),
|
|
617
773
|
projects: op.projects.map((p) => ({ id: p.id, phase: p.phase, ...(Object.keys(p.terminals).length ? { effects: Object.fromEntries(Object.entries(p.terminals).map(([key, value]) => [key, effect(key, value)])) } : {}),
|
|
774
|
+
...(op.plan.projects.find(planned => planned.project.id === p.id) ? { recordedSessions: recordedPeerGuidance(op.plan.projects.find(planned => planned.project.id === p.id)!, p, {
|
|
775
|
+
...(op.phase === "blocked" && !op.disposition ? { stopAndArchiveCommand: recoveryCommand(op, "dispose", STOP) } : {}),
|
|
776
|
+
reattachCommand: peer => op.phase === "blocked" && !op.disposition && !op.final && op.targetRoot && op.targetDigest && p.instanceId && ["started", "peers-restored"].includes(p.phase) &&
|
|
777
|
+
op.blockedReattachment?.projectId === p.id && op.blockedReattachment.peer === peer && choices.some(choice => choice.kind === "reattach" && choice.peer === peer)
|
|
778
|
+
? recoveryCommand(op, "reattach", peer) : undefined,
|
|
779
|
+
}) } : {}),
|
|
618
780
|
...(p.fresh ? { lostContinuity: Object.fromEntries(Object.entries(p.fresh).map(([peer, f]) => [peer, f.lost])) } : {}),
|
|
619
781
|
...(p.restarts?.length ? { restarts: p.restarts.length } : {}) })),
|
|
620
782
|
pluginInstalled: !!op.pluginInstalled, globalInstalled: !!op.globalInstalled,
|
|
621
783
|
...(!targetReadsWaivers(op) && op.projects.some((p) => failedPeers(op, p).length) ? { freshSession: `not offered: target ${op.plan.version} cannot read recovery waivers, so it could not release a new session` } : {}),
|
|
622
|
-
...(op.error ? { error: op.error } : {}), ...(op.failure ? { failure: op.failure } : {}), ...(op.disposition ? { disposition: op.disposition } : {}), ...(op.audit ? { audit: op.audit } : {}),
|
|
784
|
+
...(op.error ? { error: op.error } : {}), ...(op.failure ? { failure: op.failure } : {}), ...(op.blockedReattachment ? { blockedReattachment: op.blockedReattachment } : {}), ...(op.disposition ? { disposition: op.disposition } : {}), ...(op.audit ? { audit: op.audit } : {}), ...(op.freshOverride ? { freshOverride: op.freshOverride } : {}), ...(op.missingRecordedSession ? { missingRecordedSession: op.missingRecordedSession } : {}),
|
|
785
|
+
...(op.waits ? { waits: op.waits } : {}), ...(op.presentationCause ? { presentationCause: op.presentationCause } : {}), next };
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* Person-command entry: restore one recorded conversation under this operation's existing hold.
|
|
790
|
+
* The driver settles and records the selected native identity; this entry never performs global verification,
|
|
791
|
+
* release, installation or completion. A later normal resume owns those steps.
|
|
792
|
+
*/
|
|
793
|
+
export async function reattachRecovery(id: string, peer: string, driver: RecoveryDriver, home = hubHome()): Promise<RecoveryOperation> {
|
|
794
|
+
let op = readOperation<RecoveryOperation>(id, home);
|
|
795
|
+
validateReceipt(id, op);
|
|
796
|
+
const selection = () => {
|
|
797
|
+
if (!["codex", "pi", "claude"].includes(peer) || op.phase !== "blocked" || op.disposition || op.final || op.blockedReattachment?.peer !== peer) throw new Error("reattach is available only for the peer named by a blocked reattachment receipt");
|
|
798
|
+
const progress = op.projects.find(project => project.id === op.blockedReattachment!.projectId);
|
|
799
|
+
const planned = op.plan.projects.find(project => project.project.id === progress?.id);
|
|
800
|
+
const native = peer as RecoveryNativePeer;
|
|
801
|
+
const marker = progress?.reattachments?.[native];
|
|
802
|
+
if (!progress || !planned || !marker || !["started", "peers-restored"].includes(progress.phase) || !progress.instanceId || !op.targetRoot || !op.targetDigest ||
|
|
803
|
+
(marker.state !== "ready" && !repairWaiting(op, progress.id, native))) throw new Error("recorded reattachment is unavailable or its single relaunch allowance was already used; resume to inspect the saved receipt");
|
|
804
|
+
return { progress, planned, marker, native };
|
|
805
|
+
};
|
|
806
|
+
selection(); // Finished or unrelated operations must not acquire a new lock merely to refuse.
|
|
807
|
+
acquireRecoveryLock(id, home);
|
|
808
|
+
const releaseRunner = claimRunner(id, home);
|
|
809
|
+
let attempted = false;
|
|
810
|
+
const save = () => { op.updatedAt = driver.now(); writeOperation(id, op, home); };
|
|
811
|
+
try {
|
|
812
|
+
op = readOperation<RecoveryOperation>(id, home);
|
|
813
|
+
validateReceipt(id, op);
|
|
814
|
+
const { progress, planned, marker, native } = selection();
|
|
815
|
+
const expected = structuredClone(marker.binding);
|
|
816
|
+
const target = async () => {
|
|
817
|
+
const observed = await driver.inspect(planned.project);
|
|
818
|
+
if (observed.state !== "running" || observed.instanceId !== progress.instanceId || observed.version !== op.plan.version || observed.recovery?.operationId !== id || observed.recovery.phase === "released") throw new Error("reattach target changed or is not held by this operation; no replacement runtime is accepted");
|
|
819
|
+
};
|
|
820
|
+
await target();
|
|
821
|
+
attempted = true;
|
|
822
|
+
await driver.restore(planned, progress, op, native === "claude" ? "claude" : "native", save, native);
|
|
823
|
+
// A done receipt is written by the runtime only after authenticated attachment and exact settlement.
|
|
824
|
+
// Inspection peers may be unavailable; do not invent an attachment from a test or a stale status projection.
|
|
825
|
+
const restored = decodeRestoreReceipt(progress.terminals[`restored:${native}`]);
|
|
826
|
+
if (restored?.state !== "done" || restored.binding.peer !== native || restored.binding.projectRoot !== expected.projectRoot ||
|
|
827
|
+
restored.binding.sessionId !== expected.sessionId || restored.binding.sessionFile !== expected.sessionFile) throw new Error(`${native}: recorded session reattachment was not settled with its exact identity`);
|
|
828
|
+
await target();
|
|
829
|
+
delete op.blockedReattachment; delete op.failure; delete op.presentationCause;
|
|
830
|
+
op.phase = "blocked";
|
|
831
|
+
op.error = `${native}: recorded session reattached; resume to finish verification, release and any remaining installation; ${recoveryCommand(op, "resume")}`;
|
|
832
|
+
save();
|
|
833
|
+
return op;
|
|
834
|
+
} catch (error) {
|
|
835
|
+
if (attempted) {
|
|
836
|
+
const presentationError = error instanceof RecoveryPresentationError ? error
|
|
837
|
+
: error instanceof ReattachmentRequired && error.presentationError instanceof RecoveryPresentationError ? error.presentationError : undefined;
|
|
838
|
+
if (presentationError) op.presentationCause = presentationError.presentationCause;
|
|
839
|
+
else delete op.presentationCause;
|
|
840
|
+
if (error instanceof MissingRecordedSession) op.missingRecordedSession = error.evidence;
|
|
841
|
+
if (error instanceof ReattachmentRequired) op.blockedReattachment = { projectId: error.projectId, peer: error.peer };
|
|
842
|
+
else delete op.blockedReattachment;
|
|
843
|
+
if (isDatabaseBusy(error)) { op.failure = { code: "database_busy", retryable: true, site: databaseBusySite(error) }; error = new RecoveryDatabaseBusy(); }
|
|
844
|
+
op.phase = "blocked";
|
|
845
|
+
op.error = `${error instanceof Error ? error.message : "reattachment is unconfirmed"}; ${nextActionsText(op, await liveProjects(op, driver.inspect))}`;
|
|
846
|
+
save();
|
|
847
|
+
}
|
|
848
|
+
throw error;
|
|
849
|
+
} finally { releaseRunner(); }
|
|
623
850
|
}
|
|
624
851
|
|
|
625
852
|
/** Escape a blocked preflight without abandoning a stopped runtime or an uncertain terminal mutation. */
|
|
@@ -649,7 +876,7 @@ export async function abortRecovery(id: string, driver: RecoveryDriver, home = h
|
|
|
649
876
|
const state = live[planned.project.id];
|
|
650
877
|
if (state?.recovery?.operationId === id && state.recovery.phase !== "released") await driver.abort(planned.project, id, planned.source.instanceId!);
|
|
651
878
|
}
|
|
652
|
-
delete op.error; delete op.failure;
|
|
879
|
+
delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause;
|
|
653
880
|
op.phase = "cancelled"; op.step = "cancelled"; op.updatedAt = driver.now();
|
|
654
881
|
writeOperation(id, op, home); releaseRecoveryLock(id, home);
|
|
655
882
|
} catch (error) {
|
|
@@ -664,15 +891,18 @@ export async function abortRecovery(id: string, driver: RecoveryDriver, home = h
|
|
|
664
891
|
* every runtime this operation held or started is verified released or stopped.
|
|
665
892
|
*/
|
|
666
893
|
export async function disposeRecovery(id: string, choice: { fresh: string } | { stop: true }, reason: string, driver: RecoveryDriver, home = hubHome()): Promise<RecoveryOperation> {
|
|
667
|
-
let op
|
|
668
|
-
validateReceipt(id, op);
|
|
894
|
+
let op: RecoveryOperation;
|
|
895
|
+
try { op = readOperation<RecoveryOperation>(id, home); validateReceipt(id, op); }
|
|
896
|
+
catch (error) { if (driver.liveTestPeer && "fresh" in choice) throw new Error(LIVE_TEST_PERSON_REFUSAL); throw error; }
|
|
897
|
+
if (driver.liveTestPeer && "fresh" in choice && repairedFreshSession(op, choice.fresh) !== false) throw new Error(LIVE_TEST_PERSON_REFUSAL);
|
|
669
898
|
// A finished operation owns no lock: never take one only to refuse.
|
|
670
899
|
if (op.phase === "completed" || op.phase === "cancelled") throw new Error(`operation is already ${op.phase}`);
|
|
671
900
|
acquireRecoveryLock(id, home);
|
|
672
901
|
const releaseRunner = claimRunner(id, home);
|
|
673
902
|
try {
|
|
674
|
-
op = readOperation<RecoveryOperation>(id, home);
|
|
675
|
-
|
|
903
|
+
try { op = readOperation<RecoveryOperation>(id, home); validateReceipt(id, op); }
|
|
904
|
+
catch (error) { if (driver.liveTestPeer && "fresh" in choice) throw new Error(LIVE_TEST_PERSON_REFUSAL); throw error; }
|
|
905
|
+
if (driver.liveTestPeer && "fresh" in choice && repairedFreshSession(op, choice.fresh) !== false) throw new Error(LIVE_TEST_PERSON_REFUSAL);
|
|
676
906
|
if (op.phase === "completed" || op.phase === "cancelled") {
|
|
677
907
|
// Another runner finished it while this one waited: leave no lock behind, as resume and abort do.
|
|
678
908
|
if (recoveryLock(home) === id) releaseRecoveryLock(id, home);
|
|
@@ -685,7 +915,7 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
|
|
|
685
915
|
const next = nextActionsText(op, live);
|
|
686
916
|
if ("fresh" in choice) {
|
|
687
917
|
// `ahub pi` sends no fresh flag, and a restored hub refills a Pi start from its recorded resume (daemon startPeer).
|
|
688
|
-
if (choice.fresh === "pi") throw new Error(`pi: --fresh-session is not supported: a restored hub resumes Pi's recorded session, so a fresh one cannot be guaranteed; ${next}`);
|
|
918
|
+
if (choice.fresh === "pi") throw new Error(`pi: --fresh-session is not supported: a restored hub resumes Pi's recorded session, so a fresh one cannot be guaranteed; ${op.plan.projects.map(planned => piArchiveExit(planned, recoveryCommand(op, "dispose", STOP))).join("; ")}; ${next}`);
|
|
689
919
|
if (op.disposition) throw new Error(`a stop-and-archive of this operation is partway; ${next}`);
|
|
690
920
|
if (!targetReadsWaivers(op)) throw new Error(`--fresh-session is not available: target ${op.plan.version} cannot read recovery waivers, so it could not release a new session; ${next}`);
|
|
691
921
|
// The new session is launched by resume, so it helps only where resume can get past what is live.
|
|
@@ -701,12 +931,32 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
|
|
|
701
931
|
throw new Error(`${peer}: its planned session is attached again, so nothing was lost; resume records it as restored; ${next}`);
|
|
702
932
|
}
|
|
703
933
|
if (!failed.length) throw new Error(`${peer}: no failed restoration of it is open to a fresh session (none failed, or one is already chosen); --fresh-session applies only then (${recoveryCommand(op, "status")})`);
|
|
934
|
+
// Collect every proof before changing any project, so one uncertain read cannot leave a partial choice.
|
|
935
|
+
const selected = [];
|
|
704
936
|
for (const progress of failed) {
|
|
705
|
-
const
|
|
706
|
-
|
|
937
|
+
const repaired = missingRepair(op, progress, peer);
|
|
938
|
+
const planned = op.plan.projects.find((p) => p.project.id === progress.id)!;
|
|
939
|
+
const binding = repaired ? resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)!
|
|
940
|
+
: (planned.terminals as TerminalBinding[]).find((t) => t.peer === peer)!;
|
|
941
|
+
const evidence = repaired ? { ...op.missingRecordedSession!, checkedPath: checkedSessionPath(binding, op.missingRecordedSession!.kind) } : undefined;
|
|
942
|
+
if (evidence && (op.missingRecordedSession!.checkedPath !== undefined && op.missingRecordedSession!.checkedPath !== evidence.checkedPath ||
|
|
943
|
+
await driver.recordedSessionState?.(binding, evidence.kind) !== "missing")) throw new Error(`${peer}: recorded conversation is not positively missing at ${JSON.stringify(evidence.checkedPath)}; no fresh-session choice was saved`);
|
|
944
|
+
selected.push({ progress, binding, repaired, evidence });
|
|
945
|
+
}
|
|
946
|
+
for (const { progress, binding, repaired, evidence } of selected) {
|
|
947
|
+
(progress.fresh ??= {})[peer] = { lost: binding.sessionId, reason, at, ...(repaired ? { repaired: true as const } : {}) };
|
|
948
|
+
if (repaired) {
|
|
949
|
+
(progress.repairedSessions ??= {})[peer as RecoveryNativePeer] = true;
|
|
950
|
+
op.freshOverride = { peer: peer as RecoveryNativePeer, evidence: evidence!, checked: "missing", at };
|
|
951
|
+
}
|
|
952
|
+
if (repaired && progress.reattachments?.[peer as RecoveryNativePeer]) {
|
|
953
|
+
progress.terminals[`retired:${peer}`] = binding;
|
|
954
|
+
delete progress.terminals[`closedRetired:${peer}`];
|
|
955
|
+
delete progress.reattachments[peer as RecoveryNativePeer];
|
|
956
|
+
}
|
|
707
957
|
}
|
|
708
958
|
(op.audit ??= []).push({ at, action: "fresh-session", reason, peer, projects: failed.map((p) => p.id) });
|
|
709
|
-
delete op.error; delete op.failure;
|
|
959
|
+
delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause; delete op.missingRecordedSession;
|
|
710
960
|
op.updatedAt = at; writeOperation(id, op, home);
|
|
711
961
|
return op;
|
|
712
962
|
}
|
|
@@ -732,7 +982,9 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
|
|
|
732
982
|
acts.push({ planned, ...(state ? { live: state } : {}), act });
|
|
733
983
|
}
|
|
734
984
|
// Each outcome is recorded as it happens, so a disposition that stops partway stays true and resume refuses it.
|
|
735
|
-
const disposition = (op.disposition ??= { choice: "stop-and-archive", at, projects: {} });
|
|
985
|
+
const disposition: NonNullable<RecoveryOperation["disposition"]> = (op.disposition ??= { choice: "stop-and-archive", at, projects: {} });
|
|
986
|
+
delete op.blockedReattachment;
|
|
987
|
+
delete op.presentationCause;
|
|
736
988
|
abandoned(op); // #227: in the same write as the disposition
|
|
737
989
|
(op.audit ??= []).push({ at, action: "stop-and-archive", reason, projects: acts.map((a) => a.planned.project.id) });
|
|
738
990
|
// Recorded before the first act: a crash inside a stop leaves a disposition that resume refuses to override.
|
|
@@ -742,7 +994,8 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
|
|
|
742
994
|
for (const { planned, live, act } of acts) {
|
|
743
995
|
if (act.startsWith("source hold")) await driver.abort(planned.project, id, planned.source.instanceId!);
|
|
744
996
|
else if (act === "stopped" || act === "target stopped") {
|
|
745
|
-
await driver.stopAndArchive(planned.project, op, act === "target stopped" ? live!.instanceId : undefined);
|
|
997
|
+
const outcome = await driver.stopAndArchive(planned.project, op, act === "target stopped" ? live!.instanceId : undefined);
|
|
998
|
+
if (outcome && outcome.continuity) (disposition.continuity ??= {})[planned.project.id] = outcome.continuity;
|
|
746
999
|
if ((await driver.inspect(planned.project)).state !== "stopped") throw new Error(`${planned.project.id}: shutdown of instance ${live!.instanceId ?? "unknown"} is not verified`);
|
|
747
1000
|
}
|
|
748
1001
|
disposition.projects[planned.project.id] ??= act;
|