@staix/agent-hub 0.12.31 → 0.12.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -1
- package/README.md +13 -7
- package/docs/agent-notes/adapters.md +7 -4
- package/docs/agent-notes/bus.md +1 -0
- package/docs/agent-notes/daemon.md +10 -7
- package/docs/agent-notes/local-worker.md +1 -1
- package/docs/agent-notes/tasks.md +3 -2
- package/docs/agent-notes/tests.md +1 -0
- package/docs/cooperbench.md +1 -1
- package/docs/gsd-collaboration-analysis.md +1 -1
- package/docs/memory.md +1 -1
- package/docs/operations.md +114 -22
- package/docs/quickstart.md +10 -6
- package/docs/security.md +12 -8
- package/docs/smoke.md +34 -0
- package/docs/specs/2026-09-19-agent-hub-design.md +204 -0
- package/docs/specs/2026-09-20-upgrade-recovery-design.md +191 -0
- package/docs/verification/2026-10-18-0.12.32.md +54 -0
- package/docs/verified.json +20 -15
- package/package.json +2 -2
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +1296 -26
- package/src/adapters/acp.ts +5 -6
- package/src/adapters/definitions/builtin-metadata.ts +121 -0
- package/src/adapters/definitions/kimi-native.ts +4 -0
- package/src/adapters/definitions/qwen-native.ts +24 -0
- package/src/adapters/definitions/qwen.ts +71 -0
- package/src/adapters/pi.ts +6 -2
- package/src/adapters/registry-types.ts +174 -0
- package/src/adapters/registry.ts +36 -0
- package/src/cli/console-state.ts +74 -5
- package/src/cli/console.ts +35 -2
- package/src/cli/help.ts +3 -1
- package/src/cli/identity.ts +13 -3
- package/src/cli/launch.ts +1 -6
- package/src/cli/main.ts +200 -39
- package/src/cli/output.ts +2 -0
- package/src/cli/preview.ts +8 -5
- package/src/cli/recovery-guidance.ts +137 -0
- package/src/cli/recovery-presentation-options.ts +8 -0
- package/src/cli/recovery-session-file.ts +18 -0
- package/src/cli/terminal-recovery.ts +48 -0
- package/src/cli/upgrade-interactive.ts +129 -75
- package/src/cli/upgrade-labels.ts +30 -0
- package/src/cli/upgrade-presentation-data.ts +41 -0
- package/src/cli/upgrade-presentation-flags.ts +31 -0
- package/src/cli/upgrade-presentation.ts +217 -0
- package/src/cli/upgrade-restore-state.ts +6 -2
- package/src/cli/upgrade-runtime.ts +345 -194
- package/src/cli/upgrade.ts +276 -25
- package/src/hub/bus.ts +11 -0
- package/src/hub/cohorts.ts +2 -2
- package/src/hub/config-apply.ts +242 -0
- package/src/hub/config-reload.ts +120 -0
- package/src/hub/config-trust.ts +11 -2
- package/src/hub/crash.ts +8 -6
- package/src/hub/daemon.ts +559 -170
- package/src/hub/events.ts +2 -2
- package/src/hub/manager.ts +1 -1
- package/src/hub/peer-catalog.ts +10 -13
- package/src/hub/peers.ts +12 -2
- package/src/hub/permission-mode.ts +5 -3
- package/src/hub/recovery-continuity.ts +152 -0
- package/src/hub/recovery-store.ts +10 -4
- package/src/hub/sdlc.ts +5 -1
- package/src/hub/settings.ts +50 -36
- package/src/hub/settled-roles.ts +35 -0
- package/src/hub/supervision.ts +3 -2
- package/src/hub/tasks.ts +28 -8
- package/src/hub/ui.ts +1 -1
- package/src/memory/brief.ts +6 -5
- package/src/pi/session.ts +23 -4
- package/src/ui/index.html +23 -7
package/src/cli/upgrade.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { LIVE_TEST_PERSON_REFUSAL } from "./identity.ts";
|
|
2
|
+
import { checkedSessionPath } from "./recovery-session-file.ts";
|
|
1
3
|
import { canChooseFresh, decodeRestoreReceipt, restoreReceiptStatus } from "./upgrade-restore-state.ts";
|
|
2
4
|
import { databaseBusySite, isDatabaseBusy, RecoveryDatabaseBusy } from "../hub/recovery-error.ts";
|
|
3
5
|
import { Database } from "bun:sqlite";
|
|
@@ -7,6 +9,9 @@ import { join } from "node:path";
|
|
|
7
9
|
import { REGISTRY_DB_BUSY_TIMEOUT_MS, PROJECT_COLUMNS, projectFromRow, type Project } from "../hub/registry.ts";
|
|
8
10
|
import { hubHome } from "../hub/project.ts";
|
|
9
11
|
import { acquireRecoveryLock, claimRunner, coordinatorCurrent, readOperation, recoveryCommand, recoveryLock, releaseRecoveryLock, writeOperation } from "../hub/recovery-store.ts";
|
|
12
|
+
import { resolveAcceptedBinding, piArchiveExit, recordedPeerGuidance, type RecoveryNativePeer } from "./recovery-guidance.ts";
|
|
13
|
+
import type { TerminalBinding } from "./terminal-recovery.ts";
|
|
14
|
+
import { RecoveryPresentationError, type RecoveryPresentationCause, type RecoveryWaitTiming } from "./upgrade-presentation-data.ts";
|
|
10
15
|
|
|
11
16
|
export interface RecoveryPeer {
|
|
12
17
|
id: string;
|
|
@@ -54,6 +59,10 @@ export interface ProjectProgress {
|
|
|
54
59
|
phase: "pending" | "prepared" | "stopped" | "started" | "peers-restored" | "verified";
|
|
55
60
|
instanceId?: string;
|
|
56
61
|
terminals: Record<string, unknown>;
|
|
62
|
+
/** One durable repair allowance per peer, retained after an attempt to prevent a second relaunch. */
|
|
63
|
+
reattachments?: Partial<Record<RecoveryNativePeer, ReattachmentMarker>>;
|
|
64
|
+
/** Recorded-session-only continuity, independent of a target incarnation's repair attempt allowance. */
|
|
65
|
+
repairedSessions?: Partial<Record<RecoveryNativePeer, true>>;
|
|
57
66
|
/** #414: last bounded native-attachment timeout, metadata only; never terminal output or task text. */
|
|
58
67
|
attachmentTimeouts?: Record<string, {
|
|
59
68
|
at: number;
|
|
@@ -64,7 +73,7 @@ export interface ProjectProgress {
|
|
|
64
73
|
}>;
|
|
65
74
|
releaseRequested?: boolean;
|
|
66
75
|
/** #215: peers the operator moved to a fresh session, with the session or thread id whose continuity is lost. */
|
|
67
|
-
fresh?: Record<string, { lost: string; reason: string; at: number }>;
|
|
76
|
+
fresh?: Record<string, { lost: string; reason: string; at: number; repaired?: true }>;
|
|
68
77
|
/** #215: set before the commit request is sent; from then on the source may have committed (abort is not offered). */
|
|
69
78
|
commitSent?: boolean;
|
|
70
79
|
/**
|
|
@@ -73,6 +82,12 @@ export interface ProjectProgress {
|
|
|
73
82
|
*/
|
|
74
83
|
restarts?: { instanceId: string; at: number }[];
|
|
75
84
|
}
|
|
85
|
+
export interface ReattachmentMarker {
|
|
86
|
+
state: "ready" | "attempted";
|
|
87
|
+
at: number;
|
|
88
|
+
cause: "detached after restore";
|
|
89
|
+
binding: TerminalBinding;
|
|
90
|
+
}
|
|
76
91
|
export interface RecoveryOperation {
|
|
77
92
|
/**
|
|
78
93
|
* #227: 2 marks a stop-and-archive recorded on an operation whose coordinator predates #215. Every older runner and
|
|
@@ -94,10 +109,18 @@ export interface RecoveryOperation {
|
|
|
94
109
|
pluginInstalled?: boolean;
|
|
95
110
|
globalInstalled?: boolean;
|
|
96
111
|
error?: string;
|
|
112
|
+
/** Presentation observations only: old receipts without these fields have unknown timings and cause. */
|
|
113
|
+
waits?: Record<string, RecoveryWaitTiming>;
|
|
114
|
+
presentationCause?: RecoveryPresentationCause;
|
|
97
115
|
failure?: { code: "database_busy"; retryable: true; site?: string[] };
|
|
116
|
+
/** A typed verify outcome, never inferred from an error message or a peer's current state. */
|
|
117
|
+
blockedReattachment?: { projectId: string; peer: RecoveryNativePeer };
|
|
118
|
+
/** Positive typed store observation; disposal rechecks it before accepting a repaired-session loss. */
|
|
119
|
+
missingRecordedSession?: { projectId: string; peer: RecoveryNativePeer; sessionId: string; kind: "rollout" | "transcript"; checkedPath?: string };
|
|
120
|
+
freshOverride?: { peer: RecoveryNativePeer; evidence: NonNullable<RecoveryOperation["missingRecordedSession"]>; checked: "missing"; at: number };
|
|
98
121
|
/** #215: every human disposition, in order. Operator reasons only; never task or message text. */
|
|
99
122
|
audit?: { at: number; action: "fresh-session" | "stop-and-archive"; reason: string; peer?: string; projects: string[] }[];
|
|
100
|
-
disposition?: { choice: "stop-and-archive"; at: number; projects: Record<string, string> };
|
|
123
|
+
disposition?: { choice: "stop-and-archive"; at: number; projects: Record<string, string>; continuity?: Record<string, "not-archived" | "unavailable"> };
|
|
101
124
|
/** #215: a refusal no resume can get past was recorded (FinalRefusal); `next` no longer offers resume. */
|
|
102
125
|
final?: boolean;
|
|
103
126
|
}
|
|
@@ -105,6 +128,16 @@ export interface RecoveryOperation {
|
|
|
105
128
|
/** #215: a refusal that holds for every resume of the operation (its staged target or preserved source is fixed). */
|
|
106
129
|
export class FinalRefusal extends Error {}
|
|
107
130
|
|
|
131
|
+
export class ReattachmentRequired extends Error {
|
|
132
|
+
constructor(public readonly projectId: string, public readonly peer: RecoveryNativePeer, message: string, public readonly presentationError?: RecoveryPresentationError) { super(message); }
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export class MissingRecordedSession extends Error {
|
|
136
|
+
readonly code = "recorded_session_missing";
|
|
137
|
+
constructor(readonly evidence: NonNullable<RecoveryOperation["missingRecordedSession"]>) {
|
|
138
|
+
super(`${evidence.peer}: recorded session ${evidence.sessionId} has no resumable transcript (positively missing ${evidence.kind}${evidence.checkedPath ? ` checked at ${JSON.stringify(evidence.checkedPath)}` : ""}); choose dispose --fresh-session ${evidence.peer} --reason <text> to accept its loss, or stop-and-archive`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
108
141
|
export { recoveryCommand };
|
|
109
142
|
const STOP = "--stop-and-archive --reason <text>";
|
|
110
143
|
/**
|
|
@@ -145,7 +178,12 @@ export function recordRestart(progress: ProjectProgress, at: number): void {
|
|
|
145
178
|
progress.terminals[`retired:${peer}`] = receipt.binding;
|
|
146
179
|
delete progress.terminals[`closedRetired:${peer}`]; // a newer binding to close: its own receipt starts over
|
|
147
180
|
delete progress.terminals[key];
|
|
181
|
+
// Older receipts may have only the attempt marker. Retiring that binding must not retire its continuity rule.
|
|
182
|
+
if (Object.hasOwn(progress.reattachments ?? {}, peer)) (progress.repairedSessions ??= {})[peer as RecoveryNativePeer] = true;
|
|
183
|
+
// The allowance belongs to the target whose exact binding was retired, not every later #225 restart.
|
|
184
|
+
if (progress.reattachments) delete progress.reattachments[peer as RecoveryNativePeer];
|
|
148
185
|
}
|
|
186
|
+
if (progress.reattachments && !Object.keys(progress.reattachments).length) delete progress.reattachments;
|
|
149
187
|
delete progress.instanceId;
|
|
150
188
|
progress.phase = "stopped";
|
|
151
189
|
}
|
|
@@ -220,7 +258,7 @@ export function disposeRefusal(op: RecoveryOperation, live: Record<string, Inspe
|
|
|
220
258
|
|
|
221
259
|
/** #272: one entry of `next`. `text` is the line status prints; a screen runs `kind` through `recoveryArgv`. */
|
|
222
260
|
export interface NextChoice {
|
|
223
|
-
kind: "resume" | "abort" | "fresh" | "stop" | "wait";
|
|
261
|
+
kind: "resume" | "abort" | "fresh" | "stop" | "wait" | "reattach";
|
|
224
262
|
/** `fresh` only: the peer that starts as a new session. */
|
|
225
263
|
peer?: string;
|
|
226
264
|
text: string;
|
|
@@ -245,9 +283,13 @@ export function nextChoices(receipt: RecoveryOperation, runner?: number | "unkno
|
|
|
245
283
|
const old = !!op.sourceRoot && !coordinatorCurrent(op.sourceRoot);
|
|
246
284
|
const abortable = !abortRefusal(op, live);
|
|
247
285
|
const stuck = resumeBlocked(op, live);
|
|
286
|
+
const selected = op.phase === "blocked" ? op.blockedReattachment : undefined;
|
|
287
|
+
const repair = selected && op.projects.find(project => project.id === selected.projectId)?.reattachments?.[selected.peer];
|
|
288
|
+
const reattach = selected && repair && (repair.state === "ready" || repairWaiting(op, selected.projectId, selected.peer));
|
|
248
289
|
return [
|
|
249
290
|
...(stuck ? [] : [{ kind: "resume" as const, text: `${recoveryCommand(op, "resume")}${op.error ? " (after the next action in error)" : ""}${old ? ` (runs the coordinator that started this operation, which cannot re-prepare an expired hold: if it reports "source is no longer prepared", use ${abortable ? "abort or " : ""}stop-and-archive)` : ""}` }]),
|
|
250
291
|
...(abortable ? [{ kind: "abort" as const, text: recoveryCommand(op, "abort") }] : []),
|
|
292
|
+
...(selected && reattach ? [{ kind: "reattach" as const, peer: selected.peer, text: recoveryCommand(op, "reattach", selected.peer) }] : []),
|
|
251
293
|
...(targetReadsWaivers(op) && !stuck ? failed.map((peer) => ({ kind: "fresh" as const, peer, text: recoveryCommand(op, "dispose", `--fresh-session ${peer} --reason <text>`) })) : []),
|
|
252
294
|
stop,
|
|
253
295
|
];
|
|
@@ -278,6 +320,13 @@ export async function liveProjects(op: RecoveryOperation, inspect: (project: Pro
|
|
|
278
320
|
return Object.fromEntries(await Promise.all(open.map(async (p) => [p.project.id, await inspect(p.project).catch(() => undefined)] as const)));
|
|
279
321
|
}
|
|
280
322
|
|
|
323
|
+
function missingRepair(op: RecoveryOperation, progress: ProjectProgress, peer: string): boolean {
|
|
324
|
+
const evidence = op.missingRecordedSession;
|
|
325
|
+
if (!evidence || evidence.projectId !== progress.id || evidence.peer !== peer || (!progress.repairedSessions?.[evidence.peer] && !Object.hasOwn(progress.reattachments ?? {}, peer))) return false;
|
|
326
|
+
const planned = op.plan.projects.find(project => project.project.id === progress.id);
|
|
327
|
+
if (!planned) return false;
|
|
328
|
+
try { return resolveAcceptedBinding(planned, progress, evidence.peer)?.sessionId === evidence.sessionId; } catch { return false; }
|
|
329
|
+
}
|
|
281
330
|
/**
|
|
282
331
|
* Peers of a project whose restoration failed and for which a fresh session is a real choice: not Pi (a restored hub
|
|
283
332
|
* resumes its recorded session), not one already chosen, and not a planned fresh start (nothing to lose; resume
|
|
@@ -286,13 +335,31 @@ export async function liveProjects(op: RecoveryOperation, inspect: (project: Pro
|
|
|
286
335
|
function failedPeers(op: RecoveryOperation, progress: ProjectProgress, live: Record<string, Inspection | undefined> = {}): string[] {
|
|
287
336
|
const planned = op.plan.projects.find((p) => p.project.id === progress.id);
|
|
288
337
|
return Object.entries(progress.terminals).filter(([key, value]) => key.startsWith("restored:") && canChooseFresh(value)).map(([key]) => key.slice("restored:".length))
|
|
289
|
-
.filter((peer) => peer !== "pi" && !progress.fresh?.[peer] && !planned?.freshStart?.includes(peer) && !plannedAttached(op, progress, peer, live));
|
|
338
|
+
.filter((peer) => peer !== "pi" && ((!Object.hasOwn(progress.reattachments ?? {}, peer) && !Object.hasOwn(progress.repairedSessions ?? {}, peer)) || missingRepair(op, progress, peer)) && !progress.fresh?.[peer] && (!planned?.freshStart?.includes(peer) || missingRepair(op, progress, peer)) && !plannedAttached(op, progress, peer, live));
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/** Read-only scope classification; it confers no delegation authority. */
|
|
342
|
+
export function repairedFreshSession(op: RecoveryOperation, peer: string): boolean | undefined {
|
|
343
|
+
try {
|
|
344
|
+
validateReceipt(op.id, op);
|
|
345
|
+
if (!op.plan.projects.some(planned => planned.source.peers.some(row => row.id === peer) ||
|
|
346
|
+
planned.terminals.some(value => !!value && typeof value === "object" && (value as { peer?: unknown }).peer === peer))) return undefined;
|
|
347
|
+
return op.projects.some(progress => failedPeers(op, progress).includes(peer) && missingRepair(op, progress, peer));
|
|
348
|
+
} catch { return undefined; } // Unknown scope never becomes an ordinary delegated disposition.
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
function repairWaiting(op: RecoveryOperation, projectId: string, peer: RecoveryNativePeer): boolean {
|
|
352
|
+
try { return decodeRestoreReceipt(op.projects.find(project => project.id === projectId)?.terminals[`restored:${peer}`])?.state === "waiting"; }
|
|
353
|
+
catch { return false; }
|
|
290
354
|
}
|
|
291
355
|
|
|
292
356
|
/** #215: the inspection shows the planned session attached for `peer`, so resume records it as restored: nothing is lost. */
|
|
293
357
|
function plannedAttached(op: RecoveryOperation, progress: ProjectProgress, peer: string, live: Record<string, Inspection | undefined>): boolean {
|
|
294
358
|
const state = live[progress.id];
|
|
295
|
-
const
|
|
359
|
+
const planned = op.plan.projects.find((p) => p.project.id === progress.id);
|
|
360
|
+
const binding = planned && Object.hasOwn(progress.repairedSessions ?? {}, peer)
|
|
361
|
+
? resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)
|
|
362
|
+
: (planned?.terminals as { peer: string; sessionId: string }[] | undefined)?.find((t) => t.peer === peer);
|
|
296
363
|
const current = state?.state === "running" ? state.peers.find((p) => p.id === peer && p.state !== "offline") : undefined;
|
|
297
364
|
return !!binding && (peer === "codex" ? current?.threadId : current?.sessionId) === binding.sessionId;
|
|
298
365
|
}
|
|
@@ -350,6 +417,56 @@ function validateReceipt(id: string, op: RecoveryOperation): void {
|
|
|
350
417
|
const { fingerprint, ...reviewed } = op.plan;
|
|
351
418
|
if (fingerprint !== planFingerprint(reviewed) || op.projects.length !== op.plan.projects.length ||
|
|
352
419
|
op.projects.some((p, i) => p.id !== op.plan.projects[i]?.project.id)) throw new Error("reviewed operation plan or project scope changed");
|
|
420
|
+
const object = (value: unknown): value is Record<string, unknown> => !!value && typeof value === "object" && !Array.isArray(value);
|
|
421
|
+
for (const [index, progress] of op.projects.entries()) {
|
|
422
|
+
if (progress.repairedSessions !== undefined) {
|
|
423
|
+
if (!object(progress.repairedSessions)) throw new Error("invalid repaired session provenance");
|
|
424
|
+
for (const [peer, protectedSession] of Object.entries(progress.repairedSessions)) {
|
|
425
|
+
const restored = decodeRestoreReceipt(progress.terminals[`restored:${peer}`]);
|
|
426
|
+
const bindings = [progress.reattachments?.[peer as RecoveryNativePeer]?.binding,
|
|
427
|
+
restored?.state === "done" || restored?.state === "waiting" ? restored.binding : undefined,
|
|
428
|
+
progress.terminals[`retired:${peer}`]];
|
|
429
|
+
if (protectedSession !== true || !bindings.some(binding => {
|
|
430
|
+
if (binding === undefined) return false;
|
|
431
|
+
const receipt = decodeRestoreReceipt({ state: "done", binding });
|
|
432
|
+
return receipt?.state === "done" && receipt.binding.peer === peer && receipt.binding.projectRoot === op.plan.projects[index]?.project.root;
|
|
433
|
+
})) throw new Error("repaired session provenance does not match its retained peer and project");
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
for (const [peer, choice] of Object.entries(progress.fresh ?? {})) {
|
|
437
|
+
if (choice.repaired === undefined) continue;
|
|
438
|
+
const planned = op.plan.projects[index]!;
|
|
439
|
+
if (choice.repaired !== true || !progress.repairedSessions?.[peer as RecoveryNativePeer] ||
|
|
440
|
+
resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)?.sessionId !== choice.lost) throw new Error("invalid one-use repaired-session choice");
|
|
441
|
+
}
|
|
442
|
+
if (progress.reattachments === undefined) continue;
|
|
443
|
+
if (!object(progress.reattachments)) throw new Error("invalid reattachment markers");
|
|
444
|
+
for (const [peer, marker] of Object.entries(progress.reattachments)) {
|
|
445
|
+
if (!["codex", "pi", "claude"].includes(peer) || !object(marker) || Object.keys(marker).some(key => !["state", "at", "cause", "binding"].includes(key)) ||
|
|
446
|
+
!["ready", "attempted"].includes(String(marker.state)) || marker.cause !== "detached after restore" || typeof marker.at !== "number" || !Number.isFinite(marker.at) || marker.at < 0) throw new Error("invalid reattachment marker");
|
|
447
|
+
const receipt = decodeRestoreReceipt({ state: "done", binding: marker.binding });
|
|
448
|
+
if (receipt?.state !== "done" || receipt.binding.peer !== peer || receipt.binding.projectRoot !== op.plan.projects[index]?.project.root) throw new Error("reattachment binding does not match its peer and project");
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
if (op.missingRecordedSession !== undefined) {
|
|
452
|
+
const evidence = op.missingRecordedSession;
|
|
453
|
+
if (!object(evidence) || Object.keys(evidence).some(key => !["projectId", "peer", "sessionId", "kind", "checkedPath"].includes(key)) ||
|
|
454
|
+
typeof evidence.projectId !== "string" || typeof evidence.peer !== "string" || typeof evidence.sessionId !== "string" ||
|
|
455
|
+
evidence.checkedPath !== undefined && (typeof evidence.checkedPath !== "string" || !evidence.checkedPath) ||
|
|
456
|
+
!["rollout", "transcript"].includes(evidence.kind) || !op.projects.some(progress => missingRepair(op, progress, evidence.peer))) throw new Error("invalid recorded-session missing evidence");
|
|
457
|
+
}
|
|
458
|
+
if (op.freshOverride !== undefined) {
|
|
459
|
+
const override = op.freshOverride, evidence = override.evidence;
|
|
460
|
+
if (!object(override) || Object.keys(override).some(key => !["peer", "evidence", "checked", "at"].includes(key)) || !object(evidence) ||
|
|
461
|
+
Object.keys(evidence).some(key => !["projectId", "peer", "sessionId", "kind", "checkedPath"].includes(key)) || override.checked !== "missing" ||
|
|
462
|
+
!Number.isFinite(override.at) || override.at < 0 || override.peer !== evidence.peer || !["rollout", "transcript"].includes(evidence.kind) ||
|
|
463
|
+
!op.audit?.some(entry => entry.action === "fresh-session" && entry.peer === override.peer && entry.at === override.at && entry.projects.includes(evidence.projectId))) throw new Error("invalid fresh override evidence");
|
|
464
|
+
}
|
|
465
|
+
if (op.blockedReattachment !== undefined) {
|
|
466
|
+
const selected = op.blockedReattachment;
|
|
467
|
+
if (!object(selected) || Object.keys(selected).some(key => !["projectId", "peer"].includes(key)) || typeof selected.projectId !== "string" || !["codex", "pi", "claude"].includes(String(selected.peer)) ||
|
|
468
|
+
!op.projects.some(project => project.id === selected.projectId && Object.hasOwn(project.reattachments ?? {}, selected.peer))) throw new Error("invalid blocked reattachment selector");
|
|
469
|
+
}
|
|
353
470
|
}
|
|
354
471
|
|
|
355
472
|
export interface RecoveryDriver {
|
|
@@ -360,16 +477,20 @@ export interface RecoveryDriver {
|
|
|
360
477
|
commit(project: Project, operation: string, instance: string, targetFeatures?: readonly string[]): Promise<void>;
|
|
361
478
|
closeTerminals(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, save: () => void): Promise<void>;
|
|
362
479
|
start(project: Project, op: RecoveryOperation): Promise<void>;
|
|
363
|
-
restore(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, group: "native" | "claude", save: () => void): Promise<void>;
|
|
480
|
+
restore(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, group: "native" | "claude", save: () => void, onlyPeer?: RecoveryNativePeer): Promise<void>;
|
|
364
481
|
installPlugin(op: RecoveryOperation): Promise<void>;
|
|
365
482
|
installGlobal(op: RecoveryOperation): Promise<void>;
|
|
366
483
|
refreshManager?(op: RecoveryOperation): Promise<void>;
|
|
367
484
|
release(project: Project, operation: string, instance: string): Promise<void>;
|
|
368
|
-
verify(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation): Promise<void>;
|
|
485
|
+
verify(planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, save?: () => void): Promise<void>;
|
|
486
|
+
/** Read-only positive store evidence; absence of the port means unknown. */
|
|
487
|
+
recordedSessionState?(binding: TerminalBinding, kind: "rollout" | "transcript"): Promise<"found" | "missing" | "unknown">;
|
|
369
488
|
/** #215 stop-and-archive: stop this operation's unreleased target (when given) and archive its committed snapshot. */
|
|
370
|
-
stopAndArchive(project: Project, op: RecoveryOperation, targetInstance?: string): Promise<void>;
|
|
489
|
+
stopAndArchive(project: Project, op: RecoveryOperation, targetInstance?: string): Promise<void | { continuity: "not-archived" | "unavailable" }>;
|
|
371
490
|
sleep(ms: number): Promise<void>;
|
|
372
491
|
now(): number;
|
|
492
|
+
/** Trusted per-invocation delegation, supplied only by the CLI's identity preflight. */
|
|
493
|
+
readonly liveTestPeer?: string;
|
|
373
494
|
}
|
|
374
495
|
|
|
375
496
|
/** Resumable state machine. Driver actions are preceded by receipts and followed by live readback. */
|
|
@@ -428,9 +549,20 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
|
|
|
428
549
|
};
|
|
429
550
|
// Prepare, or after an expired lease re-prepare, the planned source and wait until it is quiet.
|
|
430
551
|
const prepareSource = async (planned: PlannedProject, progress: ProjectProgress, again = false) => {
|
|
552
|
+
const saveTiming = () => {
|
|
553
|
+
try { save(); }
|
|
554
|
+
catch { console.warn("Recovery timing observation could not be saved; recovery continues"); }
|
|
555
|
+
};
|
|
431
556
|
const project = planned.project, instance = planned.source.instanceId!;
|
|
432
557
|
await driver.prepare(project, id, instance, targetFeatures);
|
|
433
|
-
const
|
|
558
|
+
const startedAt = driver.now(), deadline = startedAt + idleTimeoutMs;
|
|
559
|
+
const waitStep = op.step;
|
|
560
|
+
if (waitStep) {
|
|
561
|
+
(op.waits ??= {})[waitStep] = { waitStartedAt: new Date(startedAt).toISOString(), deadlineAt: new Date(deadline).toISOString() };
|
|
562
|
+
saveTiming();
|
|
563
|
+
}
|
|
564
|
+
let waitFailed = false;
|
|
565
|
+
try {
|
|
434
566
|
for (;;) {
|
|
435
567
|
const live = await driver.inspect(project);
|
|
436
568
|
if (live.instanceId !== instance) throw new Error(`${project.id}: source daemon changed during preparation`);
|
|
@@ -446,16 +578,30 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
|
|
|
446
578
|
}
|
|
447
579
|
await driver.sleep(250);
|
|
448
580
|
}
|
|
581
|
+
} catch (error) { waitFailed = true; throw error; }
|
|
582
|
+
finally {
|
|
583
|
+
const finish = () => {
|
|
584
|
+
if (!waitStep) return;
|
|
585
|
+
op.waits![waitStep] = { ...op.waits![waitStep]!, waitEndedAt: new Date(driver.now()).toISOString() };
|
|
586
|
+
saveTiming();
|
|
587
|
+
};
|
|
588
|
+
// A best-effort end observation must not replace the error that stopped the actual wait.
|
|
589
|
+
if (waitFailed) { try { finish(); } catch { /* original wait failure remains authoritative */ } }
|
|
590
|
+
else finish();
|
|
591
|
+
}
|
|
449
592
|
};
|
|
450
593
|
try {
|
|
451
594
|
if (op.disposition) {
|
|
452
595
|
// The stop-and-archive in progress keeps its own cause; resume never runs an operation being abandoned.
|
|
453
596
|
op.error ??= `a stop-and-archive of this operation is partway; ${nextActionsText(op, await liveProjects(op, driver.inspect))}`;
|
|
454
597
|
delete op.failure;
|
|
598
|
+
delete op.presentationCause;
|
|
455
599
|
op.phase = "blocked"; save();
|
|
456
600
|
return op;
|
|
457
601
|
}
|
|
458
|
-
op.phase = "running"; delete op.error; delete op.failure;
|
|
602
|
+
op.phase = "running"; delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause;
|
|
603
|
+
delete op.waits; // Timing is the current attempt's observation, never a prior attempt's countdown.
|
|
604
|
+
save();
|
|
459
605
|
step("stage");
|
|
460
606
|
const target = await driver.stage(op);
|
|
461
607
|
if (op.targetDigest && (target.root !== op.targetRoot || target.digest !== op.targetDigest)) throw new FinalRefusal("staged target changed since operation started");
|
|
@@ -586,7 +732,7 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
|
|
|
586
732
|
}
|
|
587
733
|
step(`restore-claude:${planned.project.id}`);
|
|
588
734
|
await driver.restore(planned, progress, op, "claude", save);
|
|
589
|
-
await driver.verify(planned, progress, op);
|
|
735
|
+
await driver.verify(planned, progress, op, save);
|
|
590
736
|
step(`release:${planned.project.id}`);
|
|
591
737
|
progress.releaseRequested = true; save();
|
|
592
738
|
await driver.release(planned.project, id, progress.instanceId!);
|
|
@@ -599,10 +745,18 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
|
|
|
599
745
|
releaseRecoveryLock(id, home); op.phase = "completed"; step("completed");
|
|
600
746
|
} catch (error) {
|
|
601
747
|
// #215: every runner error ends with the choices status shows, read from every open source, never only this one.
|
|
748
|
+
const presentationError = error instanceof RecoveryPresentationError ? error
|
|
749
|
+
: error instanceof ReattachmentRequired && error.presentationError instanceof RecoveryPresentationError ? error.presentationError : undefined;
|
|
750
|
+
if (presentationError) op.presentationCause = presentationError.presentationCause;
|
|
751
|
+
else delete op.presentationCause;
|
|
752
|
+
if (error instanceof MissingRecordedSession) op.missingRecordedSession = error.evidence;
|
|
753
|
+
if (error instanceof ReattachmentRequired) op.blockedReattachment = { projectId: error.projectId, peer: error.peer };
|
|
754
|
+
else delete op.blockedReattachment;
|
|
602
755
|
if (isDatabaseBusy(error)) { op.failure = { code: "database_busy", retryable: true, site: databaseBusySite(error) }; error = new RecoveryDatabaseBusy(); }
|
|
603
756
|
if (error instanceof FinalRefusal) op.final = true;
|
|
757
|
+
op.phase = "blocked";
|
|
604
758
|
const next = nextActionsText(op, await liveProjects(op, driver.inspect));
|
|
605
|
-
op.
|
|
759
|
+
op.error = `${error instanceof Error ? error.message : "recovery failed"}; ${next}`; save();
|
|
606
760
|
} finally { releaseRunner(); }
|
|
607
761
|
return op;
|
|
608
762
|
}
|
|
@@ -610,16 +764,87 @@ export async function runRecovery(id: string, driver: RecoveryDriver, home = hub
|
|
|
610
764
|
/** Receipts and runner state only (#215): ids, phases and effects, never task or message text. */
|
|
611
765
|
export function publicOperation(op: RecoveryOperation, runnerPid?: number | "unknown", live: Record<string, Inspection | undefined> = {}) {
|
|
612
766
|
const effect = (key: string, value: unknown) => key.startsWith("restored:") ? restoreReceiptStatus(value) : value === "pending" || value === "failed" ? value : "done";
|
|
613
|
-
const
|
|
767
|
+
const choices = nextChoices(op, runnerPid, live), next = choices.map(choice => choice.text);
|
|
614
768
|
return { id: op.id, phase: op.phase, step: op.step, version: op.plan.version, updatedAt: op.updatedAt,
|
|
615
769
|
runner: runnerPid === "unknown" ? { state: "unknown" } : runnerPid ? { state: "running", pid: runnerPid } : { state: "none" },
|
|
616
770
|
...(op.phase === "running" && !runnerPid ? { stale: "the receipt says running but no runner holds the operation: its last runner stopped mid-step" } : {}),
|
|
617
771
|
projects: op.projects.map((p) => ({ id: p.id, phase: p.phase, ...(Object.keys(p.terminals).length ? { effects: Object.fromEntries(Object.entries(p.terminals).map(([key, value]) => [key, effect(key, value)])) } : {}),
|
|
772
|
+
...(op.plan.projects.find(planned => planned.project.id === p.id) ? { recordedSessions: recordedPeerGuidance(op.plan.projects.find(planned => planned.project.id === p.id)!, p, {
|
|
773
|
+
...(op.phase === "blocked" && !op.disposition ? { stopAndArchiveCommand: recoveryCommand(op, "dispose", STOP) } : {}),
|
|
774
|
+
reattachCommand: peer => op.phase === "blocked" && !op.disposition && !op.final && op.targetRoot && op.targetDigest && p.instanceId && ["started", "peers-restored"].includes(p.phase) &&
|
|
775
|
+
op.blockedReattachment?.projectId === p.id && op.blockedReattachment.peer === peer && choices.some(choice => choice.kind === "reattach" && choice.peer === peer)
|
|
776
|
+
? recoveryCommand(op, "reattach", peer) : undefined,
|
|
777
|
+
}) } : {}),
|
|
618
778
|
...(p.fresh ? { lostContinuity: Object.fromEntries(Object.entries(p.fresh).map(([peer, f]) => [peer, f.lost])) } : {}),
|
|
619
779
|
...(p.restarts?.length ? { restarts: p.restarts.length } : {}) })),
|
|
620
780
|
pluginInstalled: !!op.pluginInstalled, globalInstalled: !!op.globalInstalled,
|
|
621
781
|
...(!targetReadsWaivers(op) && op.projects.some((p) => failedPeers(op, p).length) ? { freshSession: `not offered: target ${op.plan.version} cannot read recovery waivers, so it could not release a new session` } : {}),
|
|
622
|
-
...(op.error ? { error: op.error } : {}), ...(op.failure ? { failure: op.failure } : {}), ...(op.disposition ? { disposition: op.disposition } : {}), ...(op.audit ? { audit: op.audit } : {}),
|
|
782
|
+
...(op.error ? { error: op.error } : {}), ...(op.failure ? { failure: op.failure } : {}), ...(op.blockedReattachment ? { blockedReattachment: op.blockedReattachment } : {}), ...(op.disposition ? { disposition: op.disposition } : {}), ...(op.audit ? { audit: op.audit } : {}), ...(op.freshOverride ? { freshOverride: op.freshOverride } : {}), ...(op.missingRecordedSession ? { missingRecordedSession: op.missingRecordedSession } : {}),
|
|
783
|
+
...(op.waits ? { waits: op.waits } : {}), ...(op.presentationCause ? { presentationCause: op.presentationCause } : {}), next };
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
/**
|
|
787
|
+
* Person-command entry: restore one recorded conversation under this operation's existing hold.
|
|
788
|
+
* The driver settles and records the selected native identity; this entry never performs global verification,
|
|
789
|
+
* release, installation or completion. A later normal resume owns those steps.
|
|
790
|
+
*/
|
|
791
|
+
export async function reattachRecovery(id: string, peer: string, driver: RecoveryDriver, home = hubHome()): Promise<RecoveryOperation> {
|
|
792
|
+
let op = readOperation<RecoveryOperation>(id, home);
|
|
793
|
+
validateReceipt(id, op);
|
|
794
|
+
const selection = () => {
|
|
795
|
+
if (!["codex", "pi", "claude"].includes(peer) || op.phase !== "blocked" || op.disposition || op.final || op.blockedReattachment?.peer !== peer) throw new Error("reattach is available only for the peer named by a blocked reattachment receipt");
|
|
796
|
+
const progress = op.projects.find(project => project.id === op.blockedReattachment!.projectId);
|
|
797
|
+
const planned = op.plan.projects.find(project => project.project.id === progress?.id);
|
|
798
|
+
const native = peer as RecoveryNativePeer;
|
|
799
|
+
const marker = progress?.reattachments?.[native];
|
|
800
|
+
if (!progress || !planned || !marker || !["started", "peers-restored"].includes(progress.phase) || !progress.instanceId || !op.targetRoot || !op.targetDigest ||
|
|
801
|
+
(marker.state !== "ready" && !repairWaiting(op, progress.id, native))) throw new Error("recorded reattachment is unavailable or its single relaunch allowance was already used; resume to inspect the saved receipt");
|
|
802
|
+
return { progress, planned, marker, native };
|
|
803
|
+
};
|
|
804
|
+
selection(); // Finished or unrelated operations must not acquire a new lock merely to refuse.
|
|
805
|
+
acquireRecoveryLock(id, home);
|
|
806
|
+
const releaseRunner = claimRunner(id, home);
|
|
807
|
+
let attempted = false;
|
|
808
|
+
const save = () => { op.updatedAt = driver.now(); writeOperation(id, op, home); };
|
|
809
|
+
try {
|
|
810
|
+
op = readOperation<RecoveryOperation>(id, home);
|
|
811
|
+
validateReceipt(id, op);
|
|
812
|
+
const { progress, planned, marker, native } = selection();
|
|
813
|
+
const expected = structuredClone(marker.binding);
|
|
814
|
+
const target = async () => {
|
|
815
|
+
const observed = await driver.inspect(planned.project);
|
|
816
|
+
if (observed.state !== "running" || observed.instanceId !== progress.instanceId || observed.version !== op.plan.version || observed.recovery?.operationId !== id || observed.recovery.phase === "released") throw new Error("reattach target changed or is not held by this operation; no replacement runtime is accepted");
|
|
817
|
+
};
|
|
818
|
+
await target();
|
|
819
|
+
attempted = true;
|
|
820
|
+
await driver.restore(planned, progress, op, native === "claude" ? "claude" : "native", save, native);
|
|
821
|
+
// A done receipt is written by the runtime only after authenticated attachment and exact settlement.
|
|
822
|
+
// Inspection peers may be unavailable; do not invent an attachment from a test or a stale status projection.
|
|
823
|
+
const restored = decodeRestoreReceipt(progress.terminals[`restored:${native}`]);
|
|
824
|
+
if (restored?.state !== "done" || restored.binding.peer !== native || restored.binding.projectRoot !== expected.projectRoot ||
|
|
825
|
+
restored.binding.sessionId !== expected.sessionId || restored.binding.sessionFile !== expected.sessionFile) throw new Error(`${native}: recorded session reattachment was not settled with its exact identity`);
|
|
826
|
+
await target();
|
|
827
|
+
delete op.blockedReattachment; delete op.failure; delete op.presentationCause;
|
|
828
|
+
op.phase = "blocked";
|
|
829
|
+
op.error = `${native}: recorded session reattached; resume to finish verification, release and any remaining installation; ${recoveryCommand(op, "resume")}`;
|
|
830
|
+
save();
|
|
831
|
+
return op;
|
|
832
|
+
} catch (error) {
|
|
833
|
+
if (attempted) {
|
|
834
|
+
const presentationError = error instanceof RecoveryPresentationError ? error
|
|
835
|
+
: error instanceof ReattachmentRequired && error.presentationError instanceof RecoveryPresentationError ? error.presentationError : undefined;
|
|
836
|
+
if (presentationError) op.presentationCause = presentationError.presentationCause;
|
|
837
|
+
else delete op.presentationCause;
|
|
838
|
+
if (error instanceof MissingRecordedSession) op.missingRecordedSession = error.evidence;
|
|
839
|
+
if (error instanceof ReattachmentRequired) op.blockedReattachment = { projectId: error.projectId, peer: error.peer };
|
|
840
|
+
else delete op.blockedReattachment;
|
|
841
|
+
if (isDatabaseBusy(error)) { op.failure = { code: "database_busy", retryable: true, site: databaseBusySite(error) }; error = new RecoveryDatabaseBusy(); }
|
|
842
|
+
op.phase = "blocked";
|
|
843
|
+
op.error = `${error instanceof Error ? error.message : "reattachment is unconfirmed"}; ${nextActionsText(op, await liveProjects(op, driver.inspect))}`;
|
|
844
|
+
save();
|
|
845
|
+
}
|
|
846
|
+
throw error;
|
|
847
|
+
} finally { releaseRunner(); }
|
|
623
848
|
}
|
|
624
849
|
|
|
625
850
|
/** Escape a blocked preflight without abandoning a stopped runtime or an uncertain terminal mutation. */
|
|
@@ -649,7 +874,7 @@ export async function abortRecovery(id: string, driver: RecoveryDriver, home = h
|
|
|
649
874
|
const state = live[planned.project.id];
|
|
650
875
|
if (state?.recovery?.operationId === id && state.recovery.phase !== "released") await driver.abort(planned.project, id, planned.source.instanceId!);
|
|
651
876
|
}
|
|
652
|
-
delete op.error; delete op.failure;
|
|
877
|
+
delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause;
|
|
653
878
|
op.phase = "cancelled"; op.step = "cancelled"; op.updatedAt = driver.now();
|
|
654
879
|
writeOperation(id, op, home); releaseRecoveryLock(id, home);
|
|
655
880
|
} catch (error) {
|
|
@@ -664,15 +889,18 @@ export async function abortRecovery(id: string, driver: RecoveryDriver, home = h
|
|
|
664
889
|
* every runtime this operation held or started is verified released or stopped.
|
|
665
890
|
*/
|
|
666
891
|
export async function disposeRecovery(id: string, choice: { fresh: string } | { stop: true }, reason: string, driver: RecoveryDriver, home = hubHome()): Promise<RecoveryOperation> {
|
|
667
|
-
let op
|
|
668
|
-
validateReceipt(id, op);
|
|
892
|
+
let op: RecoveryOperation;
|
|
893
|
+
try { op = readOperation<RecoveryOperation>(id, home); validateReceipt(id, op); }
|
|
894
|
+
catch (error) { if (driver.liveTestPeer && "fresh" in choice) throw new Error(LIVE_TEST_PERSON_REFUSAL); throw error; }
|
|
895
|
+
if (driver.liveTestPeer && "fresh" in choice && repairedFreshSession(op, choice.fresh) !== false) throw new Error(LIVE_TEST_PERSON_REFUSAL);
|
|
669
896
|
// A finished operation owns no lock: never take one only to refuse.
|
|
670
897
|
if (op.phase === "completed" || op.phase === "cancelled") throw new Error(`operation is already ${op.phase}`);
|
|
671
898
|
acquireRecoveryLock(id, home);
|
|
672
899
|
const releaseRunner = claimRunner(id, home);
|
|
673
900
|
try {
|
|
674
|
-
op = readOperation<RecoveryOperation>(id, home);
|
|
675
|
-
|
|
901
|
+
try { op = readOperation<RecoveryOperation>(id, home); validateReceipt(id, op); }
|
|
902
|
+
catch (error) { if (driver.liveTestPeer && "fresh" in choice) throw new Error(LIVE_TEST_PERSON_REFUSAL); throw error; }
|
|
903
|
+
if (driver.liveTestPeer && "fresh" in choice && repairedFreshSession(op, choice.fresh) !== false) throw new Error(LIVE_TEST_PERSON_REFUSAL);
|
|
676
904
|
if (op.phase === "completed" || op.phase === "cancelled") {
|
|
677
905
|
// Another runner finished it while this one waited: leave no lock behind, as resume and abort do.
|
|
678
906
|
if (recoveryLock(home) === id) releaseRecoveryLock(id, home);
|
|
@@ -685,7 +913,7 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
|
|
|
685
913
|
const next = nextActionsText(op, live);
|
|
686
914
|
if ("fresh" in choice) {
|
|
687
915
|
// `ahub pi` sends no fresh flag, and a restored hub refills a Pi start from its recorded resume (daemon startPeer).
|
|
688
|
-
if (choice.fresh === "pi") throw new Error(`pi: --fresh-session is not supported: a restored hub resumes Pi's recorded session, so a fresh one cannot be guaranteed; ${next}`);
|
|
916
|
+
if (choice.fresh === "pi") throw new Error(`pi: --fresh-session is not supported: a restored hub resumes Pi's recorded session, so a fresh one cannot be guaranteed; ${op.plan.projects.map(planned => piArchiveExit(planned, recoveryCommand(op, "dispose", STOP))).join("; ")}; ${next}`);
|
|
689
917
|
if (op.disposition) throw new Error(`a stop-and-archive of this operation is partway; ${next}`);
|
|
690
918
|
if (!targetReadsWaivers(op)) throw new Error(`--fresh-session is not available: target ${op.plan.version} cannot read recovery waivers, so it could not release a new session; ${next}`);
|
|
691
919
|
// The new session is launched by resume, so it helps only where resume can get past what is live.
|
|
@@ -701,12 +929,32 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
|
|
|
701
929
|
throw new Error(`${peer}: its planned session is attached again, so nothing was lost; resume records it as restored; ${next}`);
|
|
702
930
|
}
|
|
703
931
|
if (!failed.length) throw new Error(`${peer}: no failed restoration of it is open to a fresh session (none failed, or one is already chosen); --fresh-session applies only then (${recoveryCommand(op, "status")})`);
|
|
932
|
+
// Collect every proof before changing any project, so one uncertain read cannot leave a partial choice.
|
|
933
|
+
const selected = [];
|
|
704
934
|
for (const progress of failed) {
|
|
705
|
-
const
|
|
706
|
-
|
|
935
|
+
const repaired = missingRepair(op, progress, peer);
|
|
936
|
+
const planned = op.plan.projects.find((p) => p.project.id === progress.id)!;
|
|
937
|
+
const binding = repaired ? resolveAcceptedBinding(planned, progress, peer as RecoveryNativePeer)!
|
|
938
|
+
: (planned.terminals as TerminalBinding[]).find((t) => t.peer === peer)!;
|
|
939
|
+
const evidence = repaired ? { ...op.missingRecordedSession!, checkedPath: checkedSessionPath(binding, op.missingRecordedSession!.kind) } : undefined;
|
|
940
|
+
if (evidence && (op.missingRecordedSession!.checkedPath !== undefined && op.missingRecordedSession!.checkedPath !== evidence.checkedPath ||
|
|
941
|
+
await driver.recordedSessionState?.(binding, evidence.kind) !== "missing")) throw new Error(`${peer}: recorded conversation is not positively missing at ${JSON.stringify(evidence.checkedPath)}; no fresh-session choice was saved`);
|
|
942
|
+
selected.push({ progress, binding, repaired, evidence });
|
|
943
|
+
}
|
|
944
|
+
for (const { progress, binding, repaired, evidence } of selected) {
|
|
945
|
+
(progress.fresh ??= {})[peer] = { lost: binding.sessionId, reason, at, ...(repaired ? { repaired: true as const } : {}) };
|
|
946
|
+
if (repaired) {
|
|
947
|
+
(progress.repairedSessions ??= {})[peer as RecoveryNativePeer] = true;
|
|
948
|
+
op.freshOverride = { peer: peer as RecoveryNativePeer, evidence: evidence!, checked: "missing", at };
|
|
949
|
+
}
|
|
950
|
+
if (repaired && progress.reattachments?.[peer as RecoveryNativePeer]) {
|
|
951
|
+
progress.terminals[`retired:${peer}`] = binding;
|
|
952
|
+
delete progress.terminals[`closedRetired:${peer}`];
|
|
953
|
+
delete progress.reattachments[peer as RecoveryNativePeer];
|
|
954
|
+
}
|
|
707
955
|
}
|
|
708
956
|
(op.audit ??= []).push({ at, action: "fresh-session", reason, peer, projects: failed.map((p) => p.id) });
|
|
709
|
-
delete op.error; delete op.failure;
|
|
957
|
+
delete op.error; delete op.failure; delete op.blockedReattachment; delete op.presentationCause; delete op.missingRecordedSession;
|
|
710
958
|
op.updatedAt = at; writeOperation(id, op, home);
|
|
711
959
|
return op;
|
|
712
960
|
}
|
|
@@ -732,7 +980,9 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
|
|
|
732
980
|
acts.push({ planned, ...(state ? { live: state } : {}), act });
|
|
733
981
|
}
|
|
734
982
|
// Each outcome is recorded as it happens, so a disposition that stops partway stays true and resume refuses it.
|
|
735
|
-
const disposition = (op.disposition ??= { choice: "stop-and-archive", at, projects: {} });
|
|
983
|
+
const disposition: NonNullable<RecoveryOperation["disposition"]> = (op.disposition ??= { choice: "stop-and-archive", at, projects: {} });
|
|
984
|
+
delete op.blockedReattachment;
|
|
985
|
+
delete op.presentationCause;
|
|
736
986
|
abandoned(op); // #227: in the same write as the disposition
|
|
737
987
|
(op.audit ??= []).push({ at, action: "stop-and-archive", reason, projects: acts.map((a) => a.planned.project.id) });
|
|
738
988
|
// Recorded before the first act: a crash inside a stop leaves a disposition that resume refuses to override.
|
|
@@ -742,7 +992,8 @@ export async function disposeRecovery(id: string, choice: { fresh: string } | {
|
|
|
742
992
|
for (const { planned, live, act } of acts) {
|
|
743
993
|
if (act.startsWith("source hold")) await driver.abort(planned.project, id, planned.source.instanceId!);
|
|
744
994
|
else if (act === "stopped" || act === "target stopped") {
|
|
745
|
-
await driver.stopAndArchive(planned.project, op, act === "target stopped" ? live!.instanceId : undefined);
|
|
995
|
+
const outcome = await driver.stopAndArchive(planned.project, op, act === "target stopped" ? live!.instanceId : undefined);
|
|
996
|
+
if (outcome && outcome.continuity) (disposition.continuity ??= {})[planned.project.id] = outcome.continuity;
|
|
746
997
|
if ((await driver.inspect(planned.project)).state !== "stopped") throw new Error(`${planned.project.id}: shutdown of instance ${live!.instanceId ?? "unknown"} is not verified`);
|
|
747
998
|
}
|
|
748
999
|
disposition.projects[planned.project.id] ??= act;
|
package/src/hub/bus.ts
CHANGED
|
@@ -162,6 +162,17 @@ export class Bus {
|
|
|
162
162
|
}
|
|
163
163
|
}
|
|
164
164
|
|
|
165
|
+
/** Apply future batching and admission limits without disturbing queued work or its existing deadline. */
|
|
166
|
+
configureScheduling(config: Pick<BusOptions, "batchMax" | "batchMs" | "queueCap">): void {
|
|
167
|
+
const { batchMax, batchMs, queueCap } = config;
|
|
168
|
+
if (!Number.isSafeInteger(batchMax) || batchMax < 1 || batchMax > 2_147_483_647 ||
|
|
169
|
+
!Number.isFinite(batchMs) || batchMs < 0 || batchMs > 2_147_483_647 ||
|
|
170
|
+
!Number.isSafeInteger(queueCap) || queueCap < 1 || queueCap > 2_147_483_647) {
|
|
171
|
+
throw new Error("invalid bus scheduling configuration");
|
|
172
|
+
}
|
|
173
|
+
Object.assign(this.opts, { batchMax, batchMs, queueCap });
|
|
174
|
+
}
|
|
175
|
+
|
|
165
176
|
add(peer: PeerAdapter): void {
|
|
166
177
|
this.peers.set(peer.id, peer);
|
|
167
178
|
if (!this.queues.has(peer.id)) this.queues.set(peer.id, []);
|
package/src/hub/cohorts.ts
CHANGED
|
@@ -102,7 +102,7 @@ export class Cohorts {
|
|
|
102
102
|
* in, records owner changes (each bumps the revision and voids that member's intent), and lifts a silent cohort that
|
|
103
103
|
* an owner without verified facts joins. Returns what happened, for the notices.
|
|
104
104
|
*/
|
|
105
|
-
join(task: Task, others: Task[], gen: (t: Task) => number, handed: (t: Task) => number = () => Date.now()): { cohort: Cohort; formed: boolean; lifted: boolean } | undefined {
|
|
105
|
+
join(task: Task, others: Task[], gen: (t: Task) => number, handed: (t: Task) => number = () => Date.now(), allowSilence = true): { cohort: Cohort; formed: boolean; lifted: boolean } | undefined {
|
|
106
106
|
this.gc();
|
|
107
107
|
const all = [task, ...others].filter((t) => t.owner);
|
|
108
108
|
const found = [...new Set(all.map((t) => this.of(t.id)).filter((c): c is Cohort => !!c))];
|
|
@@ -133,7 +133,7 @@ export class Cohorts {
|
|
|
133
133
|
}
|
|
134
134
|
if (changed) cohort.revision++;
|
|
135
135
|
const owners = [...new Set([...cohort.members.values()].map((m) => m.owner))];
|
|
136
|
-
if (formed) cohort.silent = this.d.silence(owners);
|
|
136
|
+
if (formed) cohort.silent = allowSilence && this.d.silence(owners);
|
|
137
137
|
else if (cohort.silent && changed && !this.d.silence(owners)) cohort.silent = false;
|
|
138
138
|
// Lifted: a silent cohort (or a silent one merged into this) is no longer silent, and its members must hear it.
|
|
139
139
|
return { cohort, formed, lifted: wasSilent && !cohort.silent };
|