@staix/agent-hub 0.12.31 → 0.12.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -1
- package/README.md +14 -7
- package/docs/agent-notes/adapters.md +7 -4
- package/docs/agent-notes/bus.md +3 -0
- package/docs/agent-notes/daemon.md +13 -8
- package/docs/agent-notes/local-worker.md +1 -1
- package/docs/agent-notes/tasks.md +4 -3
- package/docs/agent-notes/tests.md +1 -0
- package/docs/cooperbench.md +1 -1
- package/docs/gsd-collaboration-analysis.md +1 -1
- package/docs/memory.md +1 -1
- package/docs/operations.md +141 -25
- package/docs/quickstart.md +10 -6
- package/docs/security.md +18 -9
- package/docs/smoke.md +34 -0
- package/docs/specs/2026-09-19-agent-hub-design.md +204 -0
- package/docs/specs/2026-09-20-upgrade-recovery-design.md +211 -0
- package/docs/verification/2026-10-18-0.12.32.md +54 -0
- package/docs/verification/2026-10-18-0.12.33.md +42 -0
- package/docs/verified.json +24 -16
- package/package.json +2 -2
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +1296 -26
- package/src/adapters/acp.ts +5 -6
- package/src/adapters/definitions/builtin-metadata.ts +121 -0
- package/src/adapters/definitions/kimi-native.ts +4 -0
- package/src/adapters/definitions/qwen-native.ts +24 -0
- package/src/adapters/definitions/qwen.ts +71 -0
- package/src/adapters/pi.ts +6 -2
- package/src/adapters/registry-types.ts +174 -0
- package/src/adapters/registry.ts +36 -0
- package/src/cli/console-state.ts +74 -5
- package/src/cli/console.ts +35 -2
- package/src/cli/help.ts +3 -1
- package/src/cli/identity.ts +13 -3
- package/src/cli/launch.ts +1 -6
- package/src/cli/main.ts +200 -39
- package/src/cli/output.ts +2 -0
- package/src/cli/preview.ts +8 -5
- package/src/cli/recovery-guidance.ts +137 -0
- package/src/cli/recovery-presentation-options.ts +8 -0
- package/src/cli/recovery-session-file.ts +18 -0
- package/src/cli/reset.ts +13 -1
- package/src/cli/terminal-recovery.ts +48 -0
- package/src/cli/upgrade-interactive.ts +129 -75
- package/src/cli/upgrade-labels.ts +30 -0
- package/src/cli/upgrade-presentation-data.ts +41 -0
- package/src/cli/upgrade-presentation-flags.ts +31 -0
- package/src/cli/upgrade-presentation.ts +227 -0
- package/src/cli/upgrade-restore-state.ts +6 -2
- package/src/cli/upgrade-runtime.ts +360 -199
- package/src/cli/upgrade.ts +278 -25
- package/src/hub/bus.ts +144 -14
- package/src/hub/cohorts.ts +2 -2
- package/src/hub/config-apply.ts +242 -0
- package/src/hub/config-reload.ts +120 -0
- package/src/hub/config-trust.ts +11 -2
- package/src/hub/control-client.ts +1 -1
- package/src/hub/crash.ts +8 -6
- package/src/hub/daemon.ts +577 -179
- package/src/hub/deferred-hub.ts +60 -0
- package/src/hub/delivery-journal.ts +6 -0
- package/src/hub/events.ts +2 -2
- package/src/hub/manager.ts +1 -1
- package/src/hub/peer-catalog.ts +10 -13
- package/src/hub/peers.ts +12 -2
- package/src/hub/permission-mode.ts +5 -3
- package/src/hub/recovery-continuity.ts +152 -0
- package/src/hub/recovery-store.ts +10 -4
- package/src/hub/restart.ts +12 -1
- package/src/hub/sdlc.ts +5 -1
- package/src/hub/settings.ts +50 -36
- package/src/hub/settled-roles.ts +35 -0
- package/src/hub/supervision.ts +3 -2
- package/src/hub/tasks.ts +29 -9
- package/src/hub/ui.ts +1 -1
- package/src/memory/brief.ts +6 -5
- package/src/pi/session.ts +23 -4
- package/src/ui/index.html +23 -7
|
@@ -1,21 +1,26 @@
|
|
|
1
|
+
import { checkedSessionPath, recordedFileState } from "./recovery-session-file.ts";
|
|
2
|
+
import { resolveAcceptedBinding } from "./recovery-guidance.ts";
|
|
3
|
+
import { definitionOf } from "../adapters/registry.ts";
|
|
1
4
|
import { qwenSessionId } from "../hub/crash.ts";
|
|
2
5
|
import { isDatabaseBusy, isDatabaseBusyReply, RecoveryDatabaseBusy } from "../hub/recovery-error.ts";
|
|
3
6
|
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync } from "node:fs";
|
|
4
|
-
import { homedir } from "node:os";
|
|
5
7
|
import { basename, join, resolve } from "node:path";
|
|
6
|
-
import { randomUUID } from "node:crypto";
|
|
8
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
7
9
|
import { ControlClient, PROTOCOL, RECOVERY_SOURCE_PROTOCOLS, readControl } from "../hub/control-client.ts";
|
|
8
10
|
import { awaitStopped, inspectProject } from "../hub/lifecycle.ts";
|
|
9
11
|
import type { Project } from "../hub/registry.ts";
|
|
10
12
|
import { hubHome } from "../hub/project.ts";
|
|
11
13
|
import { packageDigest, preserveRuntimeDependencies, registryRelease, runCommand, stageRelease, verifyPackage, type RunCommand } from "./recovery-package.ts";
|
|
12
|
-
import { inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
|
|
13
|
-
import { FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
|
|
14
|
+
import { inspectSavedTerminal, inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
|
|
15
|
+
import { ReattachmentRequired, MissingRecordedSession, FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
|
|
14
16
|
import { readEvents } from "../hub/events.ts";
|
|
15
17
|
import { refreshManager } from "../hub/manager.ts";
|
|
16
18
|
import { abandonRestartSnapshot, readRestartSnapshot, waiveRecoveryPeers } from "../hub/restart.ts";
|
|
19
|
+
import { readRecoveryContinuity, retireRecoveryContinuity, writeRecoveryContinuity } from "../hub/recovery-continuity.ts";
|
|
17
20
|
import { recoveryLock } from "../hub/recovery-store.ts";
|
|
18
21
|
import { decodeRestoreReceipt, restoreTransition, type RestoreReceipt, type RestoreEvidence, type RestoreTransition } from "./upgrade-restore-state.ts";
|
|
22
|
+
import { RecoveryDeferredHubError } from "../hub/deferred-hub.ts";
|
|
23
|
+
import { RecoveryPreservationError, RecoveryPresentationError, type RecoveryNativeCause } from "./upgrade-presentation-data.ts";
|
|
19
24
|
|
|
20
25
|
/** An unmanaged Claude's plugin retries with a backoff of at most 30 s; three of those bound the reconnect wait (#206). */
|
|
21
26
|
const UNMANAGED_RECONNECT_WAIT_MS = 90_000;
|
|
@@ -34,11 +39,10 @@ const orcaExecutable = () => process.env.ORCA_CLI_COMMAND || (process.env.ORCA_D
|
|
|
34
39
|
* dir, creating the project directory only with the first persisted turn; the slug replaces
|
|
35
40
|
* every non-alphanumeric character of the project root with '-'.
|
|
36
41
|
*/
|
|
37
|
-
function
|
|
38
|
-
|
|
39
|
-
const slug = binding.projectRoot.replace(/[^a-zA-Z0-9]/g, "-");
|
|
40
|
-
return existsSync(join(config, "projects", slug, `${binding.sessionId}.jsonl`));
|
|
42
|
+
function claudeTranscriptState(binding: TerminalBinding): "found" | "missing" | "unknown" {
|
|
43
|
+
return recordedFileState(checkedSessionPath(binding, "transcript"));
|
|
41
44
|
}
|
|
45
|
+
function claudeTranscriptExists(binding: TerminalBinding): boolean { return claudeTranscriptState(binding) === "found"; }
|
|
42
46
|
|
|
43
47
|
/**
|
|
44
48
|
* #215: `codex resume <id>` reads the thread's rollout from the store its launcher uses (the captured CODEX_HOME, else
|
|
@@ -46,7 +50,7 @@ function claudeTranscriptExists(binding: TerminalBinding): boolean {
|
|
|
46
50
|
* ponytail: a `.jsonl` under sessions/ whose name carries the thread id is the evidence; ask Codex itself once it offers
|
|
47
51
|
* a supported resumability query, or when its store layout changes.
|
|
48
52
|
*/
|
|
49
|
-
const codexSessions = (binding: TerminalBinding) =>
|
|
53
|
+
const codexSessions = (binding: TerminalBinding) => checkedSessionPath(binding, "rollout");
|
|
50
54
|
function codexTranscript(binding: TerminalBinding): "found" | "missing" | "unknown" {
|
|
51
55
|
try { return readdirSync(codexSessions(binding), { recursive: true }).some((name) => String(name).endsWith(".jsonl") && basename(String(name)).includes(binding.sessionId)) ? "found" : "missing"; }
|
|
52
56
|
catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "missing" : "unknown"; } // no store has no rollout; an unreadable one says nothing
|
|
@@ -169,12 +173,21 @@ export async function inspectRecovery(project: Project, targetFeatures?: readonl
|
|
|
169
173
|
const sourceProtocol = control?.protocol;
|
|
170
174
|
const legacySupported = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]);
|
|
171
175
|
// A stopped runtime reports whose unreleased restart snapshot it holds: only that one can start it again (#215).
|
|
172
|
-
|
|
176
|
+
let snapshot: string | undefined, snapshotError: Inspection["snapshotError"];
|
|
177
|
+
if (base.state === "stopped") {
|
|
178
|
+
try { snapshot = readRestartSnapshot(project.stateDir, { projectRoot: project.root, projectId: project.id })?.operationId; }
|
|
179
|
+
catch (error) {
|
|
180
|
+
if (!(error instanceof RecoveryDeferredHubError)) throw error;
|
|
181
|
+
// Corruption blocks restore, but cannot erase independently confirmed shutdown or the person's archive exit.
|
|
182
|
+
// Keep diagnostics static: the rejected snapshot may contain forged envelope fields.
|
|
183
|
+
snapshotError = { code: "recovery_deferred_hub", message: "restart snapshot deferred hub notices are invalid; restore is blocked" };
|
|
184
|
+
}
|
|
185
|
+
}
|
|
173
186
|
// A dead pid or a missing project directory is final whatever protocol its manifest names: probing it would read
|
|
174
187
|
// "unavailable" forever and wedge abort, resume and stop-and-archive (#215).
|
|
175
188
|
if (base.state === "stopped" || base.state === "missing" || (base.state !== "running" && !legacySupported)) return {
|
|
176
|
-
state: base.state, peers: [], blockers: base.state === "stopped" ? [] : [base.state === "incompatible" ? "manual-bootstrap-required: source lacks the recovery contract; use its matching CLI" : base.error ?? base.state],
|
|
177
|
-
...(control?.instanceId ? { instanceId: control.instanceId } : {}), ...(control?.protocol ? { protocol: control.protocol } : {}), ...(snapshot ? { snapshot } : {}),
|
|
189
|
+
state: base.state, peers: [], blockers: base.state === "stopped" ? snapshotError ? [snapshotError.message] : [] : [base.state === "incompatible" ? "manual-bootstrap-required: source lacks the recovery contract; use its matching CLI" : base.error ?? base.state],
|
|
190
|
+
...(control?.instanceId ? { instanceId: control.instanceId } : {}), ...(control?.protocol ? { protocol: control.protocol } : {}), ...(snapshot ? { snapshot } : {}), ...(snapshotError ? { snapshotError } : {}),
|
|
178
191
|
};
|
|
179
192
|
let status = base.status;
|
|
180
193
|
if (!status && control && sourceProtocol === PROTOCOL) return {
|
|
@@ -246,6 +259,7 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
|
|
|
246
259
|
const reconnectOnly: string[] = [];
|
|
247
260
|
for (const peer of source.peers) {
|
|
248
261
|
if (peer.state === "offline") continue;
|
|
262
|
+
const adapter = definitionOf(peer.id);
|
|
249
263
|
if (peer.id === "kimi" && peer.args?.mode === "tui") {
|
|
250
264
|
blockers.push("kimi: its pull-only terminal has no automatic restore; end it in that terminal, then make a new recovery plan (its session is not resumed by the hub)");
|
|
251
265
|
continue;
|
|
@@ -282,12 +296,13 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
|
|
|
282
296
|
const session = peer.id === "codex" ? peer.threadId : peer.sessionId;
|
|
283
297
|
if (!session) blockers.push(`${peer.id}: original conversation ID is unknown; manual-required; next action: reconnect the original native session and make a new recovery plan`);
|
|
284
298
|
else sessions[peer.id] = session;
|
|
285
|
-
} else if (
|
|
286
|
-
|
|
299
|
+
} else if (adapter?.session.restoreOnStart || peer.id === "jcode") {
|
|
300
|
+
const session = adapter?.session.crashIdentity ? adapter.session.crashIdentity(peer).sessionId : qwenSessionId(peer.sessionId);
|
|
301
|
+
if (!session) blockers.push(`${peer.id}: saved ACP session identity unavailable; end the peer and make a new recovery plan`);
|
|
287
302
|
} else if (peer.id === "pi" && peer.args?.mode === "tui") {
|
|
288
303
|
if (!peer.sessionId) blockers.push("pi: original session ID is unknown; manual-required; next action: reconnect the original Pi session and make a new recovery plan");
|
|
289
304
|
else sessions.pi = { sessionId: peer.sessionId, ...(peer.sessionFile ? { sessionFile: peer.sessionFile } : {}), ...(peer.args.backend ? { backend: peer.args.backend } : {}), ...(peer.args.model ? { model: peer.args.model } : {}) };
|
|
290
|
-
} else if (peer.id !== "kimi" && peer.id !== "
|
|
305
|
+
} else if (peer.id !== "kimi" && peer.id !== "local" && peer.id !== "pi") blockers.push(`${peer.id}: no automatic recovery adapter`);
|
|
291
306
|
}
|
|
292
307
|
const terminals = await inspectTerminals(project.root, sessions, {
|
|
293
308
|
...terminalOptions(run), stateDir: project.stateDir, instanceId: source.instanceId,
|
|
@@ -364,20 +379,23 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
364
379
|
const protocol = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]) ? sourceProtocol : PROTOCOL;
|
|
365
380
|
return rpc(project, { t: "recovery", op, operationId: id, expectedInstanceId: instance, ...(targetFeatures === undefined ? {} : { targetFeatures }) }, protocol).then(() => {});
|
|
366
381
|
};
|
|
382
|
+
const nativeFailure = (planned: PlannedProject, peer: string, reason: RecoveryNativeCause["reason"], message: string,
|
|
383
|
+
details: Omit<RecoveryNativeCause, "kind" | "reason" | "projectId" | "peer"> = {}) =>
|
|
384
|
+
new RecoveryPresentationError(message, { kind: "native_recovery", reason, projectId: planned.project.id, peer, ...details });
|
|
367
385
|
const revalidateTerminal = async (planned: PlannedProject, progress: ProjectProgress, saved: TerminalBinding, exact: boolean): Promise<TerminalBinding> => {
|
|
368
386
|
const options = { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId };
|
|
369
|
-
const found = await inspectTerminals(planned.project.root, { [saved.peer]: saved.sessionId }, options);
|
|
387
|
+
const found = await inspectTerminals(planned.project.root, { [saved.peer]: { sessionId: saved.sessionId, sessionFile: saved.sessionFile, backend: saved.backend, model: saved.model } }, options);
|
|
370
388
|
const current = found.byPeer[saved.peer];
|
|
371
389
|
if (found.manualRequired || !current || (exact && (current.handle !== saved.handle || current.incarnationId !== saved.incarnationId || current.worktreeId !== saved.worktreeId || current.projectRoot !== saved.projectRoot))) {
|
|
372
|
-
throw
|
|
390
|
+
throw nativeFailure(planned, saved.peer, "terminal_unverified", `${saved.peer}: saved terminal identity changed or is not verified; manual-required`, { handle: saved.handle });
|
|
373
391
|
}
|
|
374
392
|
const idle = await waitForIdle(exact ? saved : current, 120_000, options);
|
|
375
|
-
if (!idle.satisfied) throw
|
|
393
|
+
if (!idle.satisfied) throw nativeFailure(planned, saved.peer, "terminal_unverified", `${saved.peer}: terminal is not idle; manual-required`, { handle: saved.handle });
|
|
376
394
|
const observed = await inspectRecovery(planned.project);
|
|
377
|
-
if (observed.instanceId !== progress.instanceId) throw
|
|
395
|
+
if (observed.instanceId !== progress.instanceId) throw nativeFailure(planned, saved.peer, "target_unavailable", `${saved.peer}: daemon instance changed during terminal revalidation`, { handle: saved.handle, targetState: observed.state });
|
|
378
396
|
const peer = observed.peers.find((item) => item.id === saved.peer);
|
|
379
397
|
const session = saved.peer === "codex" ? peer?.threadId : peer?.sessionId;
|
|
380
|
-
if (session !== saved.sessionId) throw
|
|
398
|
+
if (session !== saved.sessionId || saved.peer === "pi" && progress.reattachments?.pi && saved.sessionFile && peer?.sessionFile !== saved.sessionFile) throw nativeFailure(planned, saved.peer, "session_changed", `${saved.peer}: daemon session changed during terminal revalidation; manual-required`, { handle: saved.handle, expectedSession: saved.sessionId, ...(session === undefined ? {} : { observedSession: session }) });
|
|
381
399
|
return current;
|
|
382
400
|
};
|
|
383
401
|
/** Whether Orca's inventory lists the terminal; a failed, not-ok, truncated or non-JSON answer shows nothing. */
|
|
@@ -430,6 +448,31 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
430
448
|
if (attached) return { state: "unknown", idless: true, why: `a ${peer} session is attached to the target without a session id`, step: `wait until it reports one, or end that ${peer} session` };
|
|
431
449
|
return { state: "gone", ...(endedTerminal ? { terminal: endedTerminal } : {}) };
|
|
432
450
|
};
|
|
451
|
+
/** Only independently confirmed terminal loss plus absent sessions and launchers can arm a repair. */
|
|
452
|
+
const detached = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation,
|
|
453
|
+
binding: TerminalBinding, save?: () => void): Promise<void> => {
|
|
454
|
+
const evidence = await peerEvidence(planned, progress, binding.peer);
|
|
455
|
+
if (evidence.state !== "gone") return;
|
|
456
|
+
const terminal = await inspectSavedTerminal(binding, terminalOptions(run));
|
|
457
|
+
const decision = restoreTransition({ state: "done", binding }, {
|
|
458
|
+
target: "running", launcher: "gone", session: "none", allowOther: false,
|
|
459
|
+
terminal: terminal.state === "missing" || terminal.state === "orphaned" ? "gone" : terminal.state === "present" ? "present" : terminal.state === "changed" ? "changed" : "unknown",
|
|
460
|
+
});
|
|
461
|
+
if (decision.action !== "reattach") return;
|
|
462
|
+
const current = await inspectRecovery(planned.project);
|
|
463
|
+
if (current.state !== "running" || current.instanceId !== progress.instanceId || current.version !== op.plan.version || current.recovery?.operationId !== op.id ||
|
|
464
|
+
(await peerEvidence(planned, progress, binding.peer)).state !== "gone") return;
|
|
465
|
+
const existing = progress.reattachments?.[binding.peer];
|
|
466
|
+
const why = `${binding.peer}: detached after restore; saved terminal ${binding.handle} is ${terminal.state}; recorded session ${binding.sessionId}${binding.sessionFile ? ` (${binding.sessionFile})` : ""}`;
|
|
467
|
+
if (existing) throw new ReattachmentRequired(planned.project.id, binding.peer, `${why}; the one reattachment was already attempted; no second terminal will be created`);
|
|
468
|
+
if (!save) throw new Error(`${why}; a durable coordinator save is required before reattachment`);
|
|
469
|
+
(progress.reattachments ??= {})[binding.peer] = { state: "ready", at: now(), cause: "detached after restore", binding };
|
|
470
|
+
(progress.repairedSessions ??= {})[binding.peer] = true;
|
|
471
|
+
progress.terminals[`restored:${binding.peer}`] = decision.next;
|
|
472
|
+
progress.phase = "started";
|
|
473
|
+
save();
|
|
474
|
+
throw new ReattachmentRequired(planned.project.id, binding.peer, `${why}; resume or use the operation's person-only reattach command`);
|
|
475
|
+
};
|
|
433
476
|
/**
|
|
434
477
|
* #225: the terminal of a done restoration of a target instance that died is closed under the close receipt rules
|
|
435
478
|
* (`closedRetired:<peer>` `pending` until Orca no longer lists it; an unreadable or truncated inventory is unknown and
|
|
@@ -453,21 +496,32 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
453
496
|
};
|
|
454
497
|
/** #414: an idle terminal is not an authenticated peer. Keep its receipt while waiting, including on resume. */
|
|
455
498
|
const awaitAttachment = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, waiting: Extract<RestoreReceipt, { state: "waiting" }>, save: () => void, expectedSession: string | null = waiting.binding.sessionId): Promise<string> => {
|
|
499
|
+
const saveTiming = () => {
|
|
500
|
+
try { save(); }
|
|
501
|
+
catch { console.warn("Recovery timing observation could not be saved; recovery continues"); }
|
|
502
|
+
};
|
|
456
503
|
const binding = waiting.binding;
|
|
457
504
|
const matchesTerminal = (terminal: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId">) => terminal.handle === binding.handle && terminal.incarnationId === binding.incarnationId && terminal.worktreeId === binding.worktreeId;
|
|
458
|
-
const
|
|
505
|
+
const startedAt = now(), deadline = startedAt + RECONNECT_WAIT_MS;
|
|
506
|
+
const waitStep = op.step;
|
|
507
|
+
if (waitStep) {
|
|
508
|
+
(op.waits ??= {})[waitStep] = { waitStartedAt: new Date(startedAt).toISOString(), deadlineAt: new Date(deadline).toISOString() };
|
|
509
|
+
saveTiming();
|
|
510
|
+
}
|
|
511
|
+
let waitFailed = false;
|
|
512
|
+
try {
|
|
459
513
|
let live: Inspection;
|
|
460
514
|
for (;;) {
|
|
461
515
|
live = await inspectRecovery(planned.project);
|
|
462
|
-
if (live.state !== "running") throw
|
|
463
|
-
if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw
|
|
516
|
+
if (live.state !== "running") throw nativeFailure(planned, binding.peer, "target_unavailable", `${binding.peer}: target hub reads as ${live.state} while awaiting attachment; saved terminal retained`, { handle: binding.handle, targetState: live.state });
|
|
517
|
+
if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw nativeFailure(planned, binding.peer, "target_unavailable", `${binding.peer}: target identity changed while awaiting attachment`, { handle: binding.handle, targetState: live.state });
|
|
464
518
|
const peer = live.peers.find((p) => p.id === binding.peer && p.state !== "offline");
|
|
465
519
|
const session = binding.peer === "codex" ? peer?.threadId : peer?.sessionId;
|
|
466
520
|
const changed = !!session && expectedSession !== null && session !== expectedSession;
|
|
467
521
|
// #64: only the original zero-turn Claude may change identity without an operator choice. Re-read persistence
|
|
468
522
|
// at attachment, not at the plan or wait start; an already accepted replacement is bound to its own id.
|
|
469
|
-
const zeroTurn = changed && binding.peer === "claude" && (planned.terminals as TerminalBinding[]).some((p) => p.peer === "claude" && p.sessionId === binding.sessionId) && !claudeTranscriptExists(binding);
|
|
470
|
-
if (changed && !zeroTurn) throw
|
|
523
|
+
const zeroTurn = changed && !progress.reattachments?.[binding.peer] && !progress.repairedSessions?.[binding.peer] && binding.peer === "claude" && (planned.terminals as TerminalBinding[]).some((p) => p.peer === "claude" && p.sessionId === binding.sessionId) && !claudeTranscriptExists(binding);
|
|
524
|
+
if (changed && !zeroTurn) throw nativeFailure(planned, binding.peer, "session_changed", `${binding.peer}: a different conversation attached while awaiting ${binding.sessionId}`, { handle: binding.handle, expectedSession: expectedSession!, observedSession: session! });
|
|
471
525
|
if (now() > deadline) break;
|
|
472
526
|
const launcher = await launcherOf(binding.peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId });
|
|
473
527
|
const sameTerminal = launcher?.record && matchesTerminal(launcher.record);
|
|
@@ -497,9 +551,9 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
497
551
|
}
|
|
498
552
|
if (decision.action === "fail" || decision.action === "create") throw new AttachmentLauncherGone(decision);
|
|
499
553
|
if (decision.action === "block") {
|
|
500
|
-
if (launchState === "replaced") throw
|
|
501
|
-
if (launchState === "older-instance") throw
|
|
502
|
-
throw
|
|
554
|
+
if (launchState === "replaced") throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: saved terminal identity changed while awaiting attachment`, { handle: binding.handle, launcherState: launchState });
|
|
555
|
+
if (launchState === "older-instance") throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: an older target's launcher in terminal ${olderHandle ?? "unknown"} may still run; saved terminal retained; no terminal was created`, { ...(olderHandle ? { handle: olderHandle } : {}), launcherState: launchState });
|
|
556
|
+
throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: the chosen restoration launcher's identity cannot be verified (${launchState}); saved terminal retained; no terminal was created`, { handle: binding.handle, launcherState: launchState });
|
|
503
557
|
}
|
|
504
558
|
if (now() >= deadline) break;
|
|
505
559
|
await sleep(Math.min(1000, deadline - now()));
|
|
@@ -520,10 +574,23 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
520
574
|
hub: { state: live.state, ...(live.instanceId ? { instanceId: live.instanceId } : {}), ...(peer ? { peerState: peer.state } : {}), ...(session ? { sessionId: session } : {}) },
|
|
521
575
|
lastHubConnectAttempt: "unknown" };
|
|
522
576
|
save();
|
|
523
|
-
throw
|
|
577
|
+
throw nativeFailure(planned, binding.peer, "attachment_timeout", `${binding.peer}: attachment not verified within ${RECONNECT_WAIT_MS / 1000} s; terminal ${binding.handle} retained; resume after it attaches (native-start diagnostic recorded; last hub connect attempt unknown)`, { handle: binding.handle, timeoutMs: RECONNECT_WAIT_MS });
|
|
578
|
+
} catch (error) { waitFailed = true; throw error; }
|
|
579
|
+
finally {
|
|
580
|
+
const finish = () => {
|
|
581
|
+
if (!waitStep) return;
|
|
582
|
+
op.waits![waitStep] = { ...op.waits![waitStep]!, waitEndedAt: new Date(now()).toISOString() };
|
|
583
|
+
saveTiming();
|
|
584
|
+
};
|
|
585
|
+
// Metadata storage must not hide the attachment failure or change its recovery outcome.
|
|
586
|
+
if (waitFailed) { try { finish(); } catch { /* original attachment failure remains authoritative */ } }
|
|
587
|
+
else finish();
|
|
588
|
+
}
|
|
524
589
|
};
|
|
525
590
|
return {
|
|
591
|
+
...(liveTestPeer === undefined ? {} : { liveTestPeer }),
|
|
526
592
|
now, sleep, inspect: inspectRecovery,
|
|
593
|
+
recordedSessionState: async (binding, kind) => kind === "rollout" ? codexTranscript(binding) : claudeTranscriptState(binding),
|
|
527
594
|
stage: async (op) => {
|
|
528
595
|
if (packageDigest(op.sourceRoot, { ownDependencies: true }) !== op.plan.sourceDigest) throw new FinalRefusal("preserved source changed");
|
|
529
596
|
const target = op.plan.kind === "restart" ? { root: op.sourceRoot, digest: verifyPackage(op.sourceRoot, op.plan.version, { ownDependencies: true }) }
|
|
@@ -549,7 +616,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
549
616
|
const verified = { ...target, targetFeatures };
|
|
550
617
|
// Existing peer/waiver compatibility checks remain properties of the already verified bytes.
|
|
551
618
|
if (op.targetDigest && op.targetDigest === target.digest && op.targetRoot === target.root) return verified;
|
|
552
|
-
if (op.plan.projects.some(p => p.source.peers.some(peer => peer.id
|
|
619
|
+
if (op.plan.projects.some(p => p.source.peers.some(peer => definitionOf(peer.id)?.session.restoreOnStart && peer.state !== "offline"))) {
|
|
553
620
|
const catalog = join(target.root, "src/hub/peer-catalog.ts");
|
|
554
621
|
const peers = existsSync(catalog) ? /export const OPTIONAL_PEERS\s*=\s*\[([^\]]*)\]/.exec(readFileSync(catalog, "utf8"))?.[1] : undefined;
|
|
555
622
|
if (!peers || !/["']qwen["']/.test(peers)) throw new FinalRefusal("target does not support Qwen; end Qwen and make a new plan, or choose a supporting target; runtimes unchanged");
|
|
@@ -568,7 +635,15 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
568
635
|
prepare: (p, id, instance, features) => control(p, "prepare", id, instance, features),
|
|
569
636
|
abort: (p, id, instance) => control(p, "abort", id, instance),
|
|
570
637
|
commit: (p, id, instance, features) => control(p, "commit", id, instance, features),
|
|
571
|
-
release: (p, id, instance) =>
|
|
638
|
+
release: async (p, id, instance) => {
|
|
639
|
+
// Snapshot before the request: a later writer must never be retired by this older release completion.
|
|
640
|
+
const previous = readRecoveryContinuity(p.stateDir, { projectId: p.id, projectRoot: p.root });
|
|
641
|
+
await control(p, "release", id, instance);
|
|
642
|
+
if (previous.state === "ready" && previous.bookmark.operationId !== id) {
|
|
643
|
+
try { retireRecoveryContinuity(p.stateDir, { projectId: p.id, projectRoot: p.root, operationId: previous.bookmark.operationId, digest: previous.digest }, "recovery_released"); }
|
|
644
|
+
catch { console.error("Pi continuity retirement is unconfirmed after successful recovery release; inspect the retained bookmark before another automatic start"); }
|
|
645
|
+
}
|
|
646
|
+
},
|
|
572
647
|
closeTerminals: async (planned, progress, op, save) => {
|
|
573
648
|
for (const binding of planned.terminals as TerminalBinding[]) {
|
|
574
649
|
const key = `closed:${binding.peer}`;
|
|
@@ -601,8 +676,8 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
601
676
|
throw new Error(`source preparation expired or changed while waiting for the terminal; no terminal was closed`);
|
|
602
677
|
}
|
|
603
678
|
// Refresh the same prepared lease immediately before the terminal effect. Preserve
|
|
604
|
-
// the original peer roster
|
|
605
|
-
await control(planned.project, "prepare", op.id, planned.source.instanceId
|
|
679
|
+
// the original peer roster and verified target features, including after a resume.
|
|
680
|
+
await control(planned.project, "prepare", op.id, planned.source.instanceId!, op.targetFeatures);
|
|
606
681
|
progress.terminals[key] = "pending"; save();
|
|
607
682
|
const result = await closeTerminal(binding, 0, terminalOptions(run));
|
|
608
683
|
if (result.manualRequired) throw new Error(`${binding.peer}: terminal close not verified; manual-required`);
|
|
@@ -618,10 +693,10 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
618
693
|
if (reconnect.length) waiveRecoveryPeers(p.stateDir, op.id, Object.fromEntries(reconnect.map((peer) => [peer, "reconnect-only"])));
|
|
619
694
|
await command(op, ["up"], p);
|
|
620
695
|
},
|
|
621
|
-
restore: async (planned, progress, op, group, save) => {
|
|
696
|
+
restore: async (planned, progress, op, group, save, onlyPeer) => {
|
|
622
697
|
const live = await inspectRecovery(planned.project);
|
|
623
698
|
if (live.instanceId !== progress.instanceId) throw new Error("target daemon changed before peer restore");
|
|
624
|
-
if (group === "native") {
|
|
699
|
+
if (group === "native" && !onlyPeer) {
|
|
625
700
|
for (const peer of planned.source.peers.filter((p) => ["kimi", "qwen", "jcode", "local", "pi"].includes(p.id) && p.state !== "offline" && !(p.id === "pi" && p.args?.mode === "tui"))) {
|
|
626
701
|
const current = live.peers.find((p) => p.id === peer.id);
|
|
627
702
|
if (current && current.state !== "offline") continue;
|
|
@@ -638,7 +713,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
638
713
|
// #269: `ahub pi` without a mode now takes Pi's start mode (tui by default). This Pi ran headless (a terminal
|
|
639
714
|
// one is filtered out above), so a snapshot that recorded no mode still restores it headless.
|
|
640
715
|
if (peer.id === "pi" && typeof peer.args?.mode !== "string") args.push("--mode", "headless");
|
|
641
|
-
if (peer.id
|
|
716
|
+
if (definitionOf(peer.id)?.session.restoreOnStart && peer.sessionId) args.push("--session-id", peer.sessionId);
|
|
642
717
|
if (peer.id === "jcode" && peer.sessionId) args.push("--session-id", peer.sessionId);
|
|
643
718
|
if (peer.id === "jcode" && peer.args?.providerProfile) args.push("--provider-profile", peer.args.providerProfile);
|
|
644
719
|
if (peer.id === "pi") {
|
|
@@ -649,179 +724,221 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
649
724
|
await command(op, args, planned.project);
|
|
650
725
|
}
|
|
651
726
|
}
|
|
727
|
+
let claudeGroup = false;
|
|
652
728
|
for (const plannedBinding of planned.terminals as TerminalBinding[]) {
|
|
653
|
-
if ((plannedBinding.peer === "claude") !== (group === "claude")) continue;
|
|
729
|
+
if ((plannedBinding.peer === "claude") !== (claudeGroup = group === "claude") || onlyPeer && plannedBinding.peer !== onlyPeer) continue;
|
|
654
730
|
const key = `restored:${plannedBinding.peer}`;
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
731
|
+
try {
|
|
732
|
+
// #225: after a restart, the stopped target's terminal for this peer goes first; a session it had accepted
|
|
733
|
+
// instead of the planned one (fresh choice, fresh start, zero-turn Claude) is relaunched by that id, which the
|
|
734
|
+
// operation-fenced waiver already covers.
|
|
735
|
+
await closeRetired(progress, plannedBinding.peer, save);
|
|
736
|
+
const retired = progress.terminals[`retired:${plannedBinding.peer}`] as TerminalBinding | undefined;
|
|
737
|
+
const accepted = retired && typeof retired === "object" && retired.sessionId !== plannedBinding.sessionId ? retired.sessionId : undefined;
|
|
738
|
+
const repair = progress.reattachments?.[plannedBinding.peer];
|
|
739
|
+
const recordedOnly = !!repair || progress.repairedSessions?.[plannedBinding.peer] === true && !progress.fresh?.[plannedBinding.peer]?.repaired;
|
|
740
|
+
const retiredReceipt = recordedOnly && retired !== undefined ? decodeRestoreReceipt({ state: "done", binding: retired }) : undefined;
|
|
741
|
+
const recordedBinding = retiredReceipt?.state === "done" && retiredReceipt.binding.peer === plannedBinding.peer && retiredReceipt.binding.projectRoot === planned.project.root ? retiredReceipt.binding : undefined;
|
|
742
|
+
const original = repair?.binding ?? recordedBinding ?? (accepted ? { ...plannedBinding, sessionId: accepted } : plannedBinding);
|
|
743
|
+
// A new session is accepted only by the operator's choice (#215 dispose --fresh-session) or a planned fresh
|
|
744
|
+
// start of a Codex thread with nothing to lose that still has no rollout; an unreadable store is neither.
|
|
745
|
+
const chosen = !recordedOnly && !!progress.fresh?.[original.peer];
|
|
746
|
+
let read: ReturnType<typeof codexTranscript> | undefined;
|
|
747
|
+
const transcript = () => original.peer !== "codex" || (chosen && !accepted) ? "found" : (read ??= codexTranscript(original));
|
|
748
|
+
// An accepted session starts new again only when it has nothing to resume: a Codex thread that never got a
|
|
749
|
+
// rollout, or a Claude session with no transcript (`--resume` could never succeed, #21).
|
|
750
|
+
const fresh = () => recordedOnly ? false : accepted
|
|
751
|
+
? (original.peer === "codex" ? (chosen || !!planned.freshStart?.includes("codex")) && transcript() === "missing" : original.peer === "claude" && !claudeTranscriptExists(original))
|
|
752
|
+
: chosen || (!!planned.freshStart?.includes(original.peer) && transcript() === "missing");
|
|
753
|
+
// The runner ends this with the choices `status` shows (nextActions); resume comes first and launches a failed
|
|
754
|
+
// peer again, which helps once its cause (a transient Orca create failure, a fixed store) is gone.
|
|
755
|
+
// The waiver's audit label: the operator's choice, a Claude with nothing to resume, or the plan's Codex fresh start.
|
|
756
|
+
const waiver = () => progress.fresh?.[original.peer] ? "fresh-session" : original.peer === "claude" ? "zero-turn" : "fresh-start";
|
|
757
|
+
const notRestored = (why: string) => new Error(fresh()
|
|
758
|
+
? `${original.peer}: ${why}; read that terminal in Orca for the launcher's error and fix its cause`
|
|
759
|
+
: `${original.peer}: ${why}; session ${original.sessionId} was not restored; resume launches it again once the cause is fixed`);
|
|
760
|
+
// Anything live or unknown blocks a create and keeps the receipt: never decide on evidence that could not be read.
|
|
761
|
+
const blocked = (evidence: Exclude<Awaited<ReturnType<typeof peerEvidence>>, { state: "gone" }>) => evidence.state === "unknown"
|
|
762
|
+
? new Error(`${original.peer}: whether a ${original.peer} session or launcher is live cannot be told (${evidence.why}); nothing was recorded or created${evidence.step ? `; ${evidence.step} first` : ""}`)
|
|
763
|
+
: nativeFailure(planned, original.peer, evidence.session ? "session_changed" : "launcher_unverified", evidence.session
|
|
764
|
+
? `${original.peer}: session ${evidence.session} is attached instead of ${original.sessionId}; no terminal was created; end that ${original.peer} session and close its terminal first`
|
|
765
|
+
: `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session ${evidence.attached ? "with an id " : ""}attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`,
|
|
766
|
+
evidence.session
|
|
767
|
+
? { expectedSession: original.sessionId, observedSession: evidence.session }
|
|
768
|
+
: { ...(evidence.handle ? { handle: evidence.handle } : {}), launcherState: evidence.terminal ? "live" : "older-instance" });
|
|
769
|
+
const decide = (receipt: RestoreReceipt, evidence: Awaited<ReturnType<typeof peerEvidence>>) => restoreTransition(receipt, {
|
|
770
|
+
target: "running", launcher: evidence.state === "unknown" ? "unknown" : evidence.state === "gone" ? "gone" : evidence.terminal ? "live" : "older-instance",
|
|
771
|
+
session: evidence.state === "live" && evidence.session ? evidence.session === original.sessionId ? "planned" : "other" : "none",
|
|
772
|
+
allowOther: fresh() || (!recordedOnly && original.peer === "claude" && !claudeTranscriptExists(original)),
|
|
773
|
+
binding: evidence.state === "live" && evidence.terminal ? { ...original, ...evidence.terminal } : original,
|
|
774
|
+
});
|
|
775
|
+
const receipt = decodeRestoreReceipt(progress.terminals[key]);
|
|
776
|
+
function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }> & { origin: "pending" }, expected: string | null): Promise<string>;
|
|
777
|
+
function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined>;
|
|
778
|
+
async function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined> {
|
|
779
|
+
try { return await awaitAttachment(planned, progress, op, waiting, save, expected); }
|
|
780
|
+
catch (error) {
|
|
781
|
+
if (!(error instanceof AttachmentLauncherGone)) throw error;
|
|
782
|
+
progress.terminals[key] = error.transition.next; save();
|
|
783
|
+
// The table decides whether this failed wait blocks or is eligible for a new launch.
|
|
784
|
+
if (error.transition.action === "create") return undefined;
|
|
785
|
+
throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
|
|
786
|
+
}
|
|
702
787
|
}
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
788
|
+
// #215: the pending row of the receipt table, used wherever a pending launch meets an attached session.
|
|
789
|
+
const settle = async (attached: string, recovered?: TerminalBinding) => {
|
|
790
|
+
// #21: a Claude session that never got a first turn has no transcript, so `claude --resume S` can never
|
|
791
|
+
// succeed and accepting its fresh attach loses nothing. Otherwise only the planned id, or a new session
|
|
792
|
+
// the operator or the plan accepted, counts as restored.
|
|
793
|
+
if (attached !== original.sessionId && !fresh() && !(!recordedOnly && original.peer === "claude" && !claudeTranscriptExists(original))) {
|
|
794
|
+
throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${attached} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
|
|
795
|
+
}
|
|
796
|
+
// Validate the proposed transition before committing continuity/waiver effects or a done receipt.
|
|
797
|
+
const verified = await revalidateTerminal(planned, progress, attachedBinding(recovered ?? original, attached), !!recovered);
|
|
798
|
+
// The planned session came back after all: a fresh session chosen for it lost nothing. Compared with the plan,
|
|
799
|
+
// never with a session accepted after a restart (#225), whose choice must stay on record.
|
|
800
|
+
if (attached === plannedBinding.sessionId && progress.fresh?.[original.peer]?.lost === plannedBinding.sessionId) {
|
|
801
|
+
delete progress.fresh[original.peer];
|
|
802
|
+
if (!Object.keys(progress.fresh).length) delete progress.fresh;
|
|
803
|
+
}
|
|
804
|
+
// Any other accepted session needs the target to waive the saved id, or release would wait for it forever;
|
|
805
|
+
// a launch writes the waiver first, but this must not rely on that.
|
|
806
|
+
// The planned session needs no waiver: the target's snapshot already expects it.
|
|
807
|
+
if (attached !== original.sessionId && attached !== plannedBinding.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
|
|
808
|
+
if (repair) repair.state = "attempted"; // a verified spontaneous reattachment also closes this repair allowance
|
|
809
|
+
// The person accepted one loss, not a permanent exemption for later repairs of the replacement.
|
|
810
|
+
if (progress.fresh?.[original.peer]?.repaired) delete progress.fresh[original.peer]!.repaired;
|
|
811
|
+
progress.terminals[key] = { state: "done", binding: verified } satisfies RestoreReceipt; save();
|
|
812
|
+
};
|
|
813
|
+
if (receipt?.state === "done") {
|
|
814
|
+
// A completed restoration is never a waiting launch. A closed terminal blocks exact revalidation;
|
|
815
|
+
// it cannot erase continuity or create a fresh-session choice (PROBE-B).
|
|
816
|
+
await detached(planned, progress, op, receipt.binding, save);
|
|
817
|
+
const observed = await peerEvidence(planned, progress, original.peer);
|
|
818
|
+
const decision = restoreTransition(receipt, { target: "running", launcher: "no-record-yet", session: observed.state === "live" && observed.session === receipt.binding.sessionId ? "planned" : observed.state === "live" && observed.session ? "other" : "none", allowOther: false });
|
|
819
|
+
if (decision.action === "block") {
|
|
820
|
+
const condition = observed.state === "unknown" ? `cannot be verified (${observed.why})`
|
|
821
|
+
: observed.state === "live" && observed.session ? `session ${observed.session} is attached instead of ${receipt.binding.sessionId}`
|
|
822
|
+
: observed.state === "live" ? `is not attached; its launcher in terminal ${observed.handle ?? "unknown"} may still run` : "is not attached";
|
|
823
|
+
throw new Error(`${original.peer}: completed restoration ${condition}; saved done receipt retained; resume its recorded conversation`);
|
|
824
|
+
}
|
|
825
|
+
progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, receipt.binding, true) } satisfies RestoreReceipt; save();
|
|
826
|
+
continue;
|
|
711
827
|
}
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
if (attached === plannedBinding.sessionId && progress.fresh?.[original.peer]) {
|
|
717
|
-
delete progress.fresh[original.peer];
|
|
718
|
-
if (!Object.keys(progress.fresh).length) delete progress.fresh;
|
|
828
|
+
if (receipt?.state === "waiting") {
|
|
829
|
+
const binding = receipt.binding;
|
|
830
|
+
const sessionId = await awaitBound(receipt, fresh() && binding.sessionId === original.sessionId ? null : binding.sessionId);
|
|
831
|
+
if (sessionId !== undefined) { await settle(sessionId, binding); continue; }
|
|
719
832
|
}
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
833
|
+
if (receipt?.state === "pending" || receipt?.state === "failed") {
|
|
834
|
+
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
835
|
+
const decision = decide(receipt, evidence);
|
|
836
|
+
if (decision.action === "settle" && evidence.state === "live" && evidence.session) { await settle(evidence.session); continue; }
|
|
837
|
+
if (decision.action === "poll" && decision.next.state === "waiting") {
|
|
838
|
+
const waiting = decision.next;
|
|
839
|
+
progress.terminals[key] = waiting; save();
|
|
840
|
+
const session = await awaitBound(waiting, fresh() ? null : original.sessionId);
|
|
841
|
+
if (session !== undefined) { await settle(session, waiting.binding); continue; }
|
|
842
|
+
} else if (decision.action === "fail") {
|
|
843
|
+
progress.terminals[key] = decision.next; save();
|
|
844
|
+
throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
|
|
845
|
+
} else if (decision.action === "block") {
|
|
846
|
+
if (evidence.state === "live" && evidence.session) {
|
|
847
|
+
throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${evidence.session} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
|
|
848
|
+
}
|
|
849
|
+
if (evidence.state !== "gone") throw blocked(evidence);
|
|
850
|
+
throw new Error(`${original.peer}: restoration evidence is not actionable`);
|
|
851
|
+
}
|
|
736
852
|
}
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
}
|
|
740
|
-
if (receipt?.state === "waiting") {
|
|
741
|
-
const binding = receipt.binding;
|
|
742
|
-
const sessionId = await awaitBound(receipt, fresh() && binding.sessionId === original.sessionId ? null : binding.sessionId);
|
|
743
|
-
if (sessionId !== undefined) { await settle(sessionId, binding); continue; }
|
|
744
|
-
}
|
|
745
|
-
if (receipt?.state === "pending" || receipt?.state === "failed") {
|
|
853
|
+
// Never create beside a live launch, nor on evidence that could not be read: read now, not at restore start.
|
|
854
|
+
// The planned session already attached is the restoration (the table's row for it), not a reason to block.
|
|
746
855
|
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
progress.terminals[key] =
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
856
|
+
if (evidence.state === "live" && evidence.session === original.sessionId) {
|
|
857
|
+
if (repair) await settle(evidence.session);
|
|
858
|
+
else { progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, original, false) } satisfies RestoreReceipt; save(); }
|
|
859
|
+
continue;
|
|
860
|
+
}
|
|
861
|
+
if (evidence.state !== "gone") throw blocked(evidence);
|
|
862
|
+
if (transcript() === "unknown") throw new Error(`${unresumable(original)}; no terminal was created; make that store readable first`);
|
|
863
|
+
if (transcript() === "missing" && !fresh()) {
|
|
864
|
+
progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
|
|
865
|
+
if (recordedOnly) throw new MissingRecordedSession({ projectId: planned.project.id, peer: original.peer, sessionId: original.sessionId, kind: "rollout", checkedPath: checkedSessionPath(original, "rollout") });
|
|
866
|
+
throw notRestored(unresumable(original).slice("codex: ".length));
|
|
867
|
+
}
|
|
868
|
+
if (fresh()) {
|
|
869
|
+
if (!targetReadsWaivers(op)) throw new Error(`${original.peer}: target ${op.plan.version} predates recovery waivers, so it cannot accept a new session`);
|
|
870
|
+
// The restored daemon must accept the new session instead of the recorded one.
|
|
871
|
+
waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
|
|
872
|
+
}
|
|
873
|
+
if (repair) {
|
|
874
|
+
if (repair.state === "attempted") throw new ReattachmentRequired(planned.project.id, original.peer, `${original.peer}: recorded-session reattachment already attempted; no second terminal will be created`);
|
|
875
|
+
repair.state = "attempted"; save(); // durable before any terminal creation, including an uncertain failure
|
|
876
|
+
}
|
|
877
|
+
const entrypoint = join(op.targetRoot!, "src/cli/main.js");
|
|
878
|
+
const argv = restoredTerminalArgv(entrypoint, planned.project.root, original, fresh());
|
|
879
|
+
const assignments = { ...original.launch.env, AGENTHUB_HOME: hubHome(), AGENTHUB_RECOVERY_OPERATION: op.id };
|
|
880
|
+
const launch = { ...original.launch, packageEntrypoint: entrypoint, argv,
|
|
881
|
+
command: ["env", ...Object.entries(assignments).map(([k, v]) => `${k}=${v}`), ...argv].map(shellQuote).join(" ") };
|
|
882
|
+
const binding = { ...original, launch, launchMetadata: launch };
|
|
883
|
+
progress.terminals[key] = { state: "pending" } satisfies RestoreReceipt; save();
|
|
884
|
+
const restored = await createTerminal(binding, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId }, undefined, fresh());
|
|
885
|
+
const exited = restored.blockers.find((b) => b.code === "launcher-exited");
|
|
886
|
+
if (exited) {
|
|
887
|
+
// #215: the launcher's exit alone is not the peer's state (a target that died or flapped also ends it): only
|
|
888
|
+
// nothing live makes it failed; an attached session is settled as the pending row settles it; anything else
|
|
889
|
+
// stays pending.
|
|
890
|
+
const after = await peerEvidence(planned, progress, original.peer);
|
|
891
|
+
const decision = decide({ state: "pending" }, after);
|
|
892
|
+
if (decision.action === "settle" && after.state === "live" && after.session) { await settle(after.session); continue; }
|
|
893
|
+
if (decision.action === "poll" && decision.next.state === "waiting") {
|
|
894
|
+
progress.terminals[key] = decision.next; save();
|
|
895
|
+
const session = await awaitBound(decision.next, fresh() ? null : original.sessionId);
|
|
896
|
+
if (session !== undefined) { await settle(session, decision.next.binding); continue; }
|
|
897
|
+
}
|
|
898
|
+
if (decision.action === "fail") {
|
|
899
|
+
progress.terminals[key] = decision.next; save();
|
|
900
|
+
throw notRestored(exited.message);
|
|
760
901
|
}
|
|
761
|
-
if (
|
|
762
|
-
throw new Error(`${original.peer}: restoration evidence is not actionable`);
|
|
902
|
+
if (after.state !== "gone") throw blocked(after);
|
|
763
903
|
}
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
768
|
-
if (evidence.state === "live" && evidence.session === original.sessionId) {
|
|
769
|
-
progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, original, false) } satisfies RestoreReceipt; save();
|
|
770
|
-
continue;
|
|
771
|
-
}
|
|
772
|
-
if (evidence.state !== "gone") throw blocked(evidence);
|
|
773
|
-
if (transcript() === "unknown") throw new Error(`${unresumable(original)}; no terminal was created; make that store readable first`);
|
|
774
|
-
if (transcript() === "missing" && !fresh()) {
|
|
775
|
-
progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
|
|
776
|
-
throw notRestored(unresumable(original).slice("codex: ".length));
|
|
777
|
-
}
|
|
778
|
-
if (fresh()) {
|
|
779
|
-
if (!targetReadsWaivers(op)) throw new Error(`${original.peer}: target ${op.plan.version} predates recovery waivers, so it cannot accept a new session`);
|
|
780
|
-
// The restored daemon must accept the new session instead of the recorded one.
|
|
781
|
-
waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
|
|
782
|
-
}
|
|
783
|
-
const entrypoint = join(op.targetRoot!, "src/cli/main.js");
|
|
784
|
-
const argv = restoredTerminalArgv(entrypoint, planned.project.root, original, fresh());
|
|
785
|
-
const assignments = { ...original.launch.env, AGENTHUB_HOME: hubHome(), AGENTHUB_RECOVERY_OPERATION: op.id };
|
|
786
|
-
const launch = { ...original.launch, packageEntrypoint: entrypoint, argv,
|
|
787
|
-
command: ["env", ...Object.entries(assignments).map(([k, v]) => `${k}=${v}`), ...argv].map(shellQuote).join(" ") };
|
|
788
|
-
const binding = { ...original, launch, launchMetadata: launch };
|
|
789
|
-
progress.terminals[key] = { state: "pending" } satisfies RestoreReceipt; save();
|
|
790
|
-
const restored = await createTerminal(binding, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId }, undefined, fresh());
|
|
791
|
-
const exited = restored.blockers.find((b) => b.code === "launcher-exited");
|
|
792
|
-
if (exited) {
|
|
793
|
-
// #215: the launcher's exit alone is not the peer's state (a target that died or flapped also ends it): only
|
|
794
|
-
// nothing live makes it failed; an attached session is settled as the pending row settles it; anything else
|
|
795
|
-
// stays pending.
|
|
796
|
-
const after = await peerEvidence(planned, progress, original.peer);
|
|
797
|
-
const decision = decide({ state: "pending" }, after);
|
|
798
|
-
if (decision.action === "settle" && after.state === "live" && after.session) { await settle(after.session); continue; }
|
|
799
|
-
if (decision.action === "poll" && decision.next.state === "waiting") {
|
|
800
|
-
progress.terminals[key] = decision.next; save();
|
|
801
|
-
const session = await awaitBound(decision.next, fresh() ? null : original.sessionId);
|
|
802
|
-
if (session !== undefined) { await settle(session, decision.next.binding); continue; }
|
|
904
|
+
if (restored.manualRequired || !restored.newBinding) {
|
|
905
|
+
const handle = restored.binding?.handle ?? restored.blockers[0]?.handle;
|
|
906
|
+
throw new Error(`${original.peer}: original session restoration needs manual verification${restored.blockers[0] ? ` (${restored.blockers[0].message})` : ""}; wait until it attaches, or end that launch and close its terminal${handle ? ` ${handle}` : ""} first`);
|
|
803
907
|
}
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
908
|
+
// The daemon's id is authoritative: a fresh session is recorded under the id it reports once attached.
|
|
909
|
+
const awaiting = { state: "waiting", origin: "pending", binding: restored.newBinding } satisfies RestoreReceipt;
|
|
910
|
+
progress.terminals[key] = awaiting; save();
|
|
911
|
+
const sessionId = await awaitBound(awaiting, fresh() ? null : restored.newBinding.sessionId);
|
|
912
|
+
await settle(sessionId, awaiting.binding);
|
|
913
|
+
} catch (error) {
|
|
914
|
+
if (error instanceof MissingRecordedSession || isDatabaseBusy(error)) throw error;
|
|
915
|
+
// A failed recorded-only Claude launch may prove that its transcript is gone. No error text is evidence.
|
|
916
|
+
if (claudeGroup && progress.repairedSessions?.[plannedBinding.peer] && !progress.fresh?.[plannedBinding.peer]?.repaired) {
|
|
917
|
+
const chosenReceipt = decodeRestoreReceipt(progress.terminals[key]);
|
|
918
|
+
// A bare pending/failed row can hide an uncertain create. Only a durable exact chosen binding
|
|
919
|
+
// can become the terminal that an explicit fresh choice must close before another create.
|
|
920
|
+
const binding = chosenReceipt?.state === "waiting" ? chosenReceipt.binding : undefined;
|
|
921
|
+
if (binding && claudeTranscriptState(binding) === "missing" && (await peerEvidence(planned, progress, plannedBinding.peer)).state === "gone") {
|
|
922
|
+
// Retain the exact chosen terminal so an explicit later fresh choice closes it before creating again.
|
|
923
|
+
progress.terminals[`retired:${plannedBinding.peer}`] = binding;
|
|
924
|
+
delete progress.terminals[`closedRetired:${plannedBinding.peer}`];
|
|
925
|
+
const marker = progress.reattachments?.[plannedBinding.peer];
|
|
926
|
+
if (marker) marker.binding = binding;
|
|
927
|
+
progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
|
|
928
|
+
throw new MissingRecordedSession({ projectId: planned.project.id, peer: plannedBinding.peer, sessionId: binding.sessionId, kind: "transcript", checkedPath: checkedSessionPath(binding, "transcript") });
|
|
929
|
+
}
|
|
807
930
|
}
|
|
808
|
-
if (
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
throw new Error(`${original.peer}: original session restoration needs manual verification${restored.blockers[0] ? ` (${restored.blockers[0].message})` : ""}; wait until it attaches, or end that launch and close its terminal${handle ? ` ${handle}` : ""} first`);
|
|
931
|
+
if (error instanceof ReattachmentRequired || isDatabaseBusy(error)) throw error;
|
|
932
|
+
if (progress.reattachments?.[plannedBinding.peer]) throw new ReattachmentRequired(planned.project.id, plannedBinding.peer,
|
|
933
|
+
error instanceof Error ? error.message : `${plannedBinding.peer}: recorded-session reattachment is unconfirmed`, error instanceof RecoveryPresentationError ? error : undefined);
|
|
934
|
+
throw error;
|
|
813
935
|
}
|
|
814
|
-
// The daemon's id is authoritative: a fresh session is recorded under the id it reports once attached.
|
|
815
|
-
const awaiting = { state: "waiting", origin: "pending", binding: restored.newBinding } satisfies RestoreReceipt;
|
|
816
|
-
progress.terminals[key] = awaiting; save();
|
|
817
|
-
const sessionId = await awaitBound(awaiting, fresh() ? null : restored.newBinding.sessionId);
|
|
818
|
-
await settle(sessionId, awaiting.binding);
|
|
819
936
|
}
|
|
820
|
-
if (group === "claude") for (const id of planned.reconnectOnly ?? []) {
|
|
937
|
+
if (group === "claude" && !onlyPeer) for (const id of planned.reconnectOnly ?? []) {
|
|
821
938
|
// #206: nothing is launched for an unmanaged session; its plugin reconnects by itself, within a bound.
|
|
822
939
|
const deadline = now() + UNMANAGED_RECONNECT_WAIT_MS;
|
|
823
940
|
while (!(await inspectRecovery(planned.project)).peers.some((p) => p.id === id && p.state !== "offline")) {
|
|
824
|
-
if (now() >= deadline) throw
|
|
941
|
+
if (now() >= deadline) throw nativeFailure(planned, id, "attachment_timeout", `${id}: the unmanaged session did not reconnect within ${UNMANAGED_RECONNECT_WAIT_MS / 1000} s; if that Claude session was closed, start Claude in the project again (its plugin reconnects) first`, { timeoutMs: UNMANAGED_RECONNECT_WAIT_MS });
|
|
825
942
|
await sleep(1000);
|
|
826
943
|
}
|
|
827
944
|
}
|
|
@@ -855,24 +972,61 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
855
972
|
await awaitStopped(project, instance);
|
|
856
973
|
}
|
|
857
974
|
abandonRestartSnapshot(project.stateDir, op.id);
|
|
975
|
+
const planned = op.plan.projects?.find(item => item.project.id === project.id);
|
|
976
|
+
const progress = op.projects?.find(item => item.id === project.id);
|
|
977
|
+
const source = planned?.source.peers.find(peer => peer.id === "pi" && peer.state !== "offline");
|
|
978
|
+
if (source && progress) {
|
|
979
|
+
// abandonRestartSnapshot is deliberately a no-op for another operation or an unreadable snapshot.
|
|
980
|
+
// A continuity bookmark needs positive archival evidence, including when disposal resumes after a crash.
|
|
981
|
+
const archive = join(project.stateDir, `restart.abandoned.${createHash("sha256").update(op.id).digest("hex")}.json`);
|
|
982
|
+
let archived: unknown;
|
|
983
|
+
try { archived = JSON.parse(readFileSync(archive, "utf8")); } catch { return { continuity: "not-archived" }; }
|
|
984
|
+
if (!archived || typeof archived !== "object" || (archived as { operationId?: unknown }).operationId !== op.id) return { continuity: "not-archived" };
|
|
985
|
+
try {
|
|
986
|
+
const binding = planned ? resolveAcceptedBinding(planned, progress, "pi") : undefined;
|
|
987
|
+
const sessionId = binding?.sessionId ?? source.sessionId;
|
|
988
|
+
if (!sessionId) throw new Error("Pi recorded session unavailable; archive continuity bookmark was not written");
|
|
989
|
+
const backend = binding?.backend ?? source.args?.backend;
|
|
990
|
+
writeRecoveryContinuity(project.stateDir, { schema: 1, operationId: op.id, projectId: project.id, projectRoot: project.root, at: now(),
|
|
991
|
+
pi: { sessionId, sessionFile: binding?.sessionFile ?? source.sessionFile ?? source.args?.sessionFile,
|
|
992
|
+
...(backend === "auto" || backend === "dgx" || backend === "mlx" ? { backend } : {}), model: binding?.model ?? source.args?.model } });
|
|
993
|
+
} catch { return { continuity: "unavailable" }; } // optional metadata must not wedge stop-and-archive
|
|
994
|
+
}
|
|
858
995
|
},
|
|
859
996
|
refreshManager: async (op) => { await refreshManager({ home: hubHome(), cli: join(op.targetRoot!, "src/cli/main.ts") }); },
|
|
860
|
-
verify: async (planned, progress, op) => {
|
|
997
|
+
verify: async (planned, progress, op, save) => {
|
|
861
998
|
const live = await inspectRecovery(planned.project);
|
|
862
999
|
if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw new Error("target identity verification failed");
|
|
863
1000
|
for (const old of planned.source.peers.filter((p) => p.state !== "offline")) {
|
|
864
1001
|
const peer = live.peers.find((p) => p.id === old.id);
|
|
865
|
-
if (!peer || !["idle", "busy", "paused"].includes(peer.state))
|
|
866
|
-
|
|
867
|
-
|
|
1002
|
+
if (!peer || !["idle", "busy", "paused"].includes(peer.state)) {
|
|
1003
|
+
const receipt = decodeRestoreReceipt(progress.terminals[`restored:${old.id}`]);
|
|
1004
|
+
if (receipt?.state === "done") await detached(planned, progress, op, receipt.binding, save);
|
|
1005
|
+
throw new Error(`${old.id}: peer reattachment not verified`);
|
|
1006
|
+
}
|
|
1007
|
+
const accepted = decodeRestoreReceipt(progress.terminals[`restored:${old.id}`]);
|
|
1008
|
+
const repair = progress.reattachments?.[old.id as "codex" | "pi" | "claude"];
|
|
1009
|
+
const recordedOnly = !!repair || progress.repairedSessions?.[old.id as keyof NonNullable<ProjectProgress["repairedSessions"]>] === true && !progress.fresh?.[old.id]?.repaired;
|
|
1010
|
+
const retiredValue = progress.terminals[`retired:${old.id}`];
|
|
1011
|
+
const retired = retiredValue === undefined ? undefined : decodeRestoreReceipt({ state: "done", binding: retiredValue });
|
|
1012
|
+
// A #225 restart clears the old target's repair fence, never its accepted conversation.
|
|
1013
|
+
const retained = retired?.state === "done" && accepted?.state === "done" && retired.binding.peer === old.id && retired.binding.projectRoot === planned.project.root && retired.binding.sessionId === accepted.binding.sessionId;
|
|
1014
|
+
if ((recordedOnly || retained) && accepted?.state === "done") {
|
|
1015
|
+
const session = old.id === "codex" ? peer.threadId : peer.sessionId;
|
|
1016
|
+
if (session !== accepted.binding.sessionId || repair && accepted.binding.sessionId !== repair.binding.sessionId ||
|
|
1017
|
+
old.id === "pi" && (recordedOnly || old.sessionFile) && accepted.binding.sessionFile && peer.sessionFile !== accepted.binding.sessionFile) throw nativeFailure(planned, old.id, "session_changed", `${old.id}: reattached a different recorded conversation`, { handle: accepted.binding.handle, expectedSession: accepted.binding.sessionId, ...(session === undefined ? {} : { observedSession: session }) });
|
|
1018
|
+
continue;
|
|
1019
|
+
}
|
|
1020
|
+
if (!recordedOnly && (progress.fresh?.[old.id] || planned.freshStart?.includes(old.id))) continue; // #215: a new session was accepted (recorded)
|
|
1021
|
+
if (old.id === "codex" && peer.threadId !== old.threadId) throw nativeFailure(planned, old.id, "session_changed", "Codex resumed a different conversation", { ...(old.threadId ? { expectedSession: old.threadId } : {}), ...(peer.threadId ? { observedSession: peer.threadId } : {}) });
|
|
868
1022
|
if (old.id === "claude" && peer.sessionId !== old.sessionId) {
|
|
869
1023
|
// #64: the restore gate already accepted a fresh session when the original never
|
|
870
1024
|
// persisted a transcript. Re-derive persistence from the binding exactly like that
|
|
871
1025
|
// gate did, so a first turn landing after the plan was made keeps the check strict.
|
|
872
1026
|
const binding = (planned.terminals as TerminalBinding[]).find((t) => t.peer === "claude");
|
|
873
|
-
if (!binding || binding.sessionId !== old.sessionId || claudeTranscriptExists(binding)) throw
|
|
1027
|
+
if (recordedOnly || !binding || binding.sessionId !== old.sessionId || claudeTranscriptExists(binding)) throw nativeFailure(planned, old.id, "session_changed", "Claude resumed a different conversation", { ...(old.sessionId ? { expectedSession: old.sessionId } : {}), ...(peer.sessionId ? { observedSession: peer.sessionId } : {}) });
|
|
874
1028
|
}
|
|
875
|
-
if (old.id === "pi" && (peer.sessionId !== old.sessionId || (old.sessionFile && peer.sessionFile !== old.sessionFile))) throw
|
|
1029
|
+
if (old.id === "pi" && (peer.sessionId !== old.sessionId || (old.sessionFile && peer.sessionFile !== old.sessionFile))) throw nativeFailure(planned, old.id, "session_changed", "Pi resumed a different conversation", { ...(old.sessionId ? { expectedSession: old.sessionId } : {}), ...(peer.sessionId ? { observedSession: peer.sessionId } : {}) });
|
|
876
1030
|
}
|
|
877
1031
|
for (const original of planned.terminals as TerminalBinding[]) {
|
|
878
1032
|
const saved = decodeRestoreReceipt(progress.terminals[`restored:${original.peer}`]);
|
|
@@ -883,6 +1037,13 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
883
1037
|
if (snapshot.ok === false) throw new Error("dashboard snapshot readback failed");
|
|
884
1038
|
const recovery = await rpc(planned.project, { t: "recovery", op: "inspect", expectedInstanceId: progress.instanceId });
|
|
885
1039
|
const integrity = recovery.recovery?.integrity;
|
|
1040
|
+
if (integrity?.expected && integrity.current) {
|
|
1041
|
+
for (const check of ["queues", "manualPaused", "boardDigest", "budgetDigest"] as const) {
|
|
1042
|
+
if (JSON.stringify(integrity.current[check]) !== JSON.stringify(integrity.expected[check])) {
|
|
1043
|
+
throw new RecoveryPreservationError(check, planned.project.id);
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
886
1047
|
if (!integrity?.expected || JSON.stringify(integrity.current) !== JSON.stringify(integrity.expected)) {
|
|
887
1048
|
throw new Error("queue, manual pause, task board or budget preservation was not verified");
|
|
888
1049
|
}
|