@staix/agent-hub 0.12.31 → 0.12.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -1
- package/README.md +13 -7
- package/docs/agent-notes/adapters.md +7 -4
- package/docs/agent-notes/bus.md +1 -0
- package/docs/agent-notes/daemon.md +10 -7
- package/docs/agent-notes/local-worker.md +1 -1
- package/docs/agent-notes/tasks.md +3 -2
- package/docs/agent-notes/tests.md +1 -0
- package/docs/cooperbench.md +1 -1
- package/docs/gsd-collaboration-analysis.md +1 -1
- package/docs/memory.md +1 -1
- package/docs/operations.md +114 -22
- package/docs/quickstart.md +10 -6
- package/docs/security.md +12 -8
- package/docs/smoke.md +34 -0
- package/docs/specs/2026-09-19-agent-hub-design.md +204 -0
- package/docs/specs/2026-09-20-upgrade-recovery-design.md +191 -0
- package/docs/verification/2026-10-18-0.12.32.md +54 -0
- package/docs/verified.json +20 -15
- package/package.json +2 -2
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +1296 -26
- package/src/adapters/acp.ts +5 -6
- package/src/adapters/definitions/builtin-metadata.ts +121 -0
- package/src/adapters/definitions/kimi-native.ts +4 -0
- package/src/adapters/definitions/qwen-native.ts +24 -0
- package/src/adapters/definitions/qwen.ts +71 -0
- package/src/adapters/pi.ts +6 -2
- package/src/adapters/registry-types.ts +174 -0
- package/src/adapters/registry.ts +36 -0
- package/src/cli/console-state.ts +74 -5
- package/src/cli/console.ts +35 -2
- package/src/cli/help.ts +3 -1
- package/src/cli/identity.ts +13 -3
- package/src/cli/launch.ts +1 -6
- package/src/cli/main.ts +200 -39
- package/src/cli/output.ts +2 -0
- package/src/cli/preview.ts +8 -5
- package/src/cli/recovery-guidance.ts +137 -0
- package/src/cli/recovery-presentation-options.ts +8 -0
- package/src/cli/recovery-session-file.ts +18 -0
- package/src/cli/terminal-recovery.ts +48 -0
- package/src/cli/upgrade-interactive.ts +129 -75
- package/src/cli/upgrade-labels.ts +30 -0
- package/src/cli/upgrade-presentation-data.ts +41 -0
- package/src/cli/upgrade-presentation-flags.ts +31 -0
- package/src/cli/upgrade-presentation.ts +217 -0
- package/src/cli/upgrade-restore-state.ts +6 -2
- package/src/cli/upgrade-runtime.ts +345 -194
- package/src/cli/upgrade.ts +276 -25
- package/src/hub/bus.ts +11 -0
- package/src/hub/cohorts.ts +2 -2
- package/src/hub/config-apply.ts +242 -0
- package/src/hub/config-reload.ts +120 -0
- package/src/hub/config-trust.ts +11 -2
- package/src/hub/crash.ts +8 -6
- package/src/hub/daemon.ts +559 -170
- package/src/hub/events.ts +2 -2
- package/src/hub/manager.ts +1 -1
- package/src/hub/peer-catalog.ts +10 -13
- package/src/hub/peers.ts +12 -2
- package/src/hub/permission-mode.ts +5 -3
- package/src/hub/recovery-continuity.ts +152 -0
- package/src/hub/recovery-store.ts +10 -4
- package/src/hub/sdlc.ts +5 -1
- package/src/hub/settings.ts +50 -36
- package/src/hub/settled-roles.ts +35 -0
- package/src/hub/supervision.ts +3 -2
- package/src/hub/tasks.ts +28 -8
- package/src/hub/ui.ts +1 -1
- package/src/memory/brief.ts +6 -5
- package/src/pi/session.ts +23 -4
- package/src/ui/index.html +23 -7
|
@@ -1,21 +1,25 @@
|
|
|
1
|
+
import { checkedSessionPath, recordedFileState } from "./recovery-session-file.ts";
|
|
2
|
+
import { resolveAcceptedBinding } from "./recovery-guidance.ts";
|
|
3
|
+
import { definitionOf } from "../adapters/registry.ts";
|
|
1
4
|
import { qwenSessionId } from "../hub/crash.ts";
|
|
2
5
|
import { isDatabaseBusy, isDatabaseBusyReply, RecoveryDatabaseBusy } from "../hub/recovery-error.ts";
|
|
3
6
|
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync } from "node:fs";
|
|
4
|
-
import { homedir } from "node:os";
|
|
5
7
|
import { basename, join, resolve } from "node:path";
|
|
6
|
-
import { randomUUID } from "node:crypto";
|
|
8
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
7
9
|
import { ControlClient, PROTOCOL, RECOVERY_SOURCE_PROTOCOLS, readControl } from "../hub/control-client.ts";
|
|
8
10
|
import { awaitStopped, inspectProject } from "../hub/lifecycle.ts";
|
|
9
11
|
import type { Project } from "../hub/registry.ts";
|
|
10
12
|
import { hubHome } from "../hub/project.ts";
|
|
11
13
|
import { packageDigest, preserveRuntimeDependencies, registryRelease, runCommand, stageRelease, verifyPackage, type RunCommand } from "./recovery-package.ts";
|
|
12
|
-
import { inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
|
|
13
|
-
import { FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
|
|
14
|
+
import { inspectSavedTerminal, inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
|
|
15
|
+
import { ReattachmentRequired, MissingRecordedSession, FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
|
|
14
16
|
import { readEvents } from "../hub/events.ts";
|
|
15
17
|
import { refreshManager } from "../hub/manager.ts";
|
|
16
18
|
import { abandonRestartSnapshot, readRestartSnapshot, waiveRecoveryPeers } from "../hub/restart.ts";
|
|
19
|
+
import { readRecoveryContinuity, retireRecoveryContinuity, writeRecoveryContinuity } from "../hub/recovery-continuity.ts";
|
|
17
20
|
import { recoveryLock } from "../hub/recovery-store.ts";
|
|
18
21
|
import { decodeRestoreReceipt, restoreTransition, type RestoreReceipt, type RestoreEvidence, type RestoreTransition } from "./upgrade-restore-state.ts";
|
|
22
|
+
import { RecoveryPreservationError, RecoveryPresentationError, type RecoveryNativeCause } from "./upgrade-presentation-data.ts";
|
|
19
23
|
|
|
20
24
|
/** An unmanaged Claude's plugin retries with a backoff of at most 30 s; three of those bound the reconnect wait (#206). */
|
|
21
25
|
const UNMANAGED_RECONNECT_WAIT_MS = 90_000;
|
|
@@ -34,11 +38,10 @@ const orcaExecutable = () => process.env.ORCA_CLI_COMMAND || (process.env.ORCA_D
|
|
|
34
38
|
* dir, creating the project directory only with the first persisted turn; the slug replaces
|
|
35
39
|
* every non-alphanumeric character of the project root with '-'.
|
|
36
40
|
*/
|
|
37
|
-
function
|
|
38
|
-
|
|
39
|
-
const slug = binding.projectRoot.replace(/[^a-zA-Z0-9]/g, "-");
|
|
40
|
-
return existsSync(join(config, "projects", slug, `${binding.sessionId}.jsonl`));
|
|
41
|
+
function claudeTranscriptState(binding: TerminalBinding): "found" | "missing" | "unknown" {
|
|
42
|
+
return recordedFileState(checkedSessionPath(binding, "transcript"));
|
|
41
43
|
}
|
|
44
|
+
function claudeTranscriptExists(binding: TerminalBinding): boolean { return claudeTranscriptState(binding) === "found"; }
|
|
42
45
|
|
|
43
46
|
/**
|
|
44
47
|
* #215: `codex resume <id>` reads the thread's rollout from the store its launcher uses (the captured CODEX_HOME, else
|
|
@@ -46,7 +49,7 @@ function claudeTranscriptExists(binding: TerminalBinding): boolean {
|
|
|
46
49
|
* ponytail: a `.jsonl` under sessions/ whose name carries the thread id is the evidence; ask Codex itself once it offers
|
|
47
50
|
* a supported resumability query, or when its store layout changes.
|
|
48
51
|
*/
|
|
49
|
-
const codexSessions = (binding: TerminalBinding) =>
|
|
52
|
+
const codexSessions = (binding: TerminalBinding) => checkedSessionPath(binding, "rollout");
|
|
50
53
|
function codexTranscript(binding: TerminalBinding): "found" | "missing" | "unknown" {
|
|
51
54
|
try { return readdirSync(codexSessions(binding), { recursive: true }).some((name) => String(name).endsWith(".jsonl") && basename(String(name)).includes(binding.sessionId)) ? "found" : "missing"; }
|
|
52
55
|
catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "missing" : "unknown"; } // no store has no rollout; an unreadable one says nothing
|
|
@@ -246,6 +249,7 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
|
|
|
246
249
|
const reconnectOnly: string[] = [];
|
|
247
250
|
for (const peer of source.peers) {
|
|
248
251
|
if (peer.state === "offline") continue;
|
|
252
|
+
const adapter = definitionOf(peer.id);
|
|
249
253
|
if (peer.id === "kimi" && peer.args?.mode === "tui") {
|
|
250
254
|
blockers.push("kimi: its pull-only terminal has no automatic restore; end it in that terminal, then make a new recovery plan (its session is not resumed by the hub)");
|
|
251
255
|
continue;
|
|
@@ -282,12 +286,13 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
|
|
|
282
286
|
const session = peer.id === "codex" ? peer.threadId : peer.sessionId;
|
|
283
287
|
if (!session) blockers.push(`${peer.id}: original conversation ID is unknown; manual-required; next action: reconnect the original native session and make a new recovery plan`);
|
|
284
288
|
else sessions[peer.id] = session;
|
|
285
|
-
} else if (
|
|
286
|
-
|
|
289
|
+
} else if (adapter?.session.restoreOnStart || peer.id === "jcode") {
|
|
290
|
+
const session = adapter?.session.crashIdentity ? adapter.session.crashIdentity(peer).sessionId : qwenSessionId(peer.sessionId);
|
|
291
|
+
if (!session) blockers.push(`${peer.id}: saved ACP session identity unavailable; end the peer and make a new recovery plan`);
|
|
287
292
|
} else if (peer.id === "pi" && peer.args?.mode === "tui") {
|
|
288
293
|
if (!peer.sessionId) blockers.push("pi: original session ID is unknown; manual-required; next action: reconnect the original Pi session and make a new recovery plan");
|
|
289
294
|
else sessions.pi = { sessionId: peer.sessionId, ...(peer.sessionFile ? { sessionFile: peer.sessionFile } : {}), ...(peer.args.backend ? { backend: peer.args.backend } : {}), ...(peer.args.model ? { model: peer.args.model } : {}) };
|
|
290
|
-
} else if (peer.id !== "kimi" && peer.id !== "
|
|
295
|
+
} else if (peer.id !== "kimi" && peer.id !== "local" && peer.id !== "pi") blockers.push(`${peer.id}: no automatic recovery adapter`);
|
|
291
296
|
}
|
|
292
297
|
const terminals = await inspectTerminals(project.root, sessions, {
|
|
293
298
|
...terminalOptions(run), stateDir: project.stateDir, instanceId: source.instanceId,
|
|
@@ -364,20 +369,23 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
364
369
|
const protocol = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]) ? sourceProtocol : PROTOCOL;
|
|
365
370
|
return rpc(project, { t: "recovery", op, operationId: id, expectedInstanceId: instance, ...(targetFeatures === undefined ? {} : { targetFeatures }) }, protocol).then(() => {});
|
|
366
371
|
};
|
|
372
|
+
const nativeFailure = (planned: PlannedProject, peer: string, reason: RecoveryNativeCause["reason"], message: string,
|
|
373
|
+
details: Omit<RecoveryNativeCause, "kind" | "reason" | "projectId" | "peer"> = {}) =>
|
|
374
|
+
new RecoveryPresentationError(message, { kind: "native_recovery", reason, projectId: planned.project.id, peer, ...details });
|
|
367
375
|
const revalidateTerminal = async (planned: PlannedProject, progress: ProjectProgress, saved: TerminalBinding, exact: boolean): Promise<TerminalBinding> => {
|
|
368
376
|
const options = { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId };
|
|
369
|
-
const found = await inspectTerminals(planned.project.root, { [saved.peer]: saved.sessionId }, options);
|
|
377
|
+
const found = await inspectTerminals(planned.project.root, { [saved.peer]: { sessionId: saved.sessionId, sessionFile: saved.sessionFile, backend: saved.backend, model: saved.model } }, options);
|
|
370
378
|
const current = found.byPeer[saved.peer];
|
|
371
379
|
if (found.manualRequired || !current || (exact && (current.handle !== saved.handle || current.incarnationId !== saved.incarnationId || current.worktreeId !== saved.worktreeId || current.projectRoot !== saved.projectRoot))) {
|
|
372
|
-
throw
|
|
380
|
+
throw nativeFailure(planned, saved.peer, "terminal_unverified", `${saved.peer}: saved terminal identity changed or is not verified; manual-required`, { handle: saved.handle });
|
|
373
381
|
}
|
|
374
382
|
const idle = await waitForIdle(exact ? saved : current, 120_000, options);
|
|
375
|
-
if (!idle.satisfied) throw
|
|
383
|
+
if (!idle.satisfied) throw nativeFailure(planned, saved.peer, "terminal_unverified", `${saved.peer}: terminal is not idle; manual-required`, { handle: saved.handle });
|
|
376
384
|
const observed = await inspectRecovery(planned.project);
|
|
377
|
-
if (observed.instanceId !== progress.instanceId) throw
|
|
385
|
+
if (observed.instanceId !== progress.instanceId) throw nativeFailure(planned, saved.peer, "target_unavailable", `${saved.peer}: daemon instance changed during terminal revalidation`, { handle: saved.handle, targetState: observed.state });
|
|
378
386
|
const peer = observed.peers.find((item) => item.id === saved.peer);
|
|
379
387
|
const session = saved.peer === "codex" ? peer?.threadId : peer?.sessionId;
|
|
380
|
-
if (session !== saved.sessionId) throw
|
|
388
|
+
if (session !== saved.sessionId || saved.peer === "pi" && progress.reattachments?.pi && saved.sessionFile && peer?.sessionFile !== saved.sessionFile) throw nativeFailure(planned, saved.peer, "session_changed", `${saved.peer}: daemon session changed during terminal revalidation; manual-required`, { handle: saved.handle, expectedSession: saved.sessionId, ...(session === undefined ? {} : { observedSession: session }) });
|
|
381
389
|
return current;
|
|
382
390
|
};
|
|
383
391
|
/** Whether Orca's inventory lists the terminal; a failed, not-ok, truncated or non-JSON answer shows nothing. */
|
|
@@ -430,6 +438,31 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
430
438
|
if (attached) return { state: "unknown", idless: true, why: `a ${peer} session is attached to the target without a session id`, step: `wait until it reports one, or end that ${peer} session` };
|
|
431
439
|
return { state: "gone", ...(endedTerminal ? { terminal: endedTerminal } : {}) };
|
|
432
440
|
};
|
|
441
|
+
/** Only independently confirmed terminal loss plus absent sessions and launchers can arm a repair. */
|
|
442
|
+
const detached = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation,
|
|
443
|
+
binding: TerminalBinding, save?: () => void): Promise<void> => {
|
|
444
|
+
const evidence = await peerEvidence(planned, progress, binding.peer);
|
|
445
|
+
if (evidence.state !== "gone") return;
|
|
446
|
+
const terminal = await inspectSavedTerminal(binding, terminalOptions(run));
|
|
447
|
+
const decision = restoreTransition({ state: "done", binding }, {
|
|
448
|
+
target: "running", launcher: "gone", session: "none", allowOther: false,
|
|
449
|
+
terminal: terminal.state === "missing" || terminal.state === "orphaned" ? "gone" : terminal.state === "present" ? "present" : terminal.state === "changed" ? "changed" : "unknown",
|
|
450
|
+
});
|
|
451
|
+
if (decision.action !== "reattach") return;
|
|
452
|
+
const current = await inspectRecovery(planned.project);
|
|
453
|
+
if (current.state !== "running" || current.instanceId !== progress.instanceId || current.version !== op.plan.version || current.recovery?.operationId !== op.id ||
|
|
454
|
+
(await peerEvidence(planned, progress, binding.peer)).state !== "gone") return;
|
|
455
|
+
const existing = progress.reattachments?.[binding.peer];
|
|
456
|
+
const why = `${binding.peer}: detached after restore; saved terminal ${binding.handle} is ${terminal.state}; recorded session ${binding.sessionId}${binding.sessionFile ? ` (${binding.sessionFile})` : ""}`;
|
|
457
|
+
if (existing) throw new ReattachmentRequired(planned.project.id, binding.peer, `${why}; the one reattachment was already attempted; no second terminal will be created`);
|
|
458
|
+
if (!save) throw new Error(`${why}; a durable coordinator save is required before reattachment`);
|
|
459
|
+
(progress.reattachments ??= {})[binding.peer] = { state: "ready", at: now(), cause: "detached after restore", binding };
|
|
460
|
+
(progress.repairedSessions ??= {})[binding.peer] = true;
|
|
461
|
+
progress.terminals[`restored:${binding.peer}`] = decision.next;
|
|
462
|
+
progress.phase = "started";
|
|
463
|
+
save();
|
|
464
|
+
throw new ReattachmentRequired(planned.project.id, binding.peer, `${why}; resume or use the operation's person-only reattach command`);
|
|
465
|
+
};
|
|
433
466
|
/**
|
|
434
467
|
* #225: the terminal of a done restoration of a target instance that died is closed under the close receipt rules
|
|
435
468
|
* (`closedRetired:<peer>` `pending` until Orca no longer lists it; an unreadable or truncated inventory is unknown and
|
|
@@ -453,21 +486,32 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
453
486
|
};
|
|
454
487
|
/** #414: an idle terminal is not an authenticated peer. Keep its receipt while waiting, including on resume. */
|
|
455
488
|
const awaitAttachment = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, waiting: Extract<RestoreReceipt, { state: "waiting" }>, save: () => void, expectedSession: string | null = waiting.binding.sessionId): Promise<string> => {
|
|
489
|
+
const saveTiming = () => {
|
|
490
|
+
try { save(); }
|
|
491
|
+
catch { console.warn("Recovery timing observation could not be saved; recovery continues"); }
|
|
492
|
+
};
|
|
456
493
|
const binding = waiting.binding;
|
|
457
494
|
const matchesTerminal = (terminal: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId">) => terminal.handle === binding.handle && terminal.incarnationId === binding.incarnationId && terminal.worktreeId === binding.worktreeId;
|
|
458
|
-
const
|
|
495
|
+
const startedAt = now(), deadline = startedAt + RECONNECT_WAIT_MS;
|
|
496
|
+
const waitStep = op.step;
|
|
497
|
+
if (waitStep) {
|
|
498
|
+
(op.waits ??= {})[waitStep] = { waitStartedAt: new Date(startedAt).toISOString(), deadlineAt: new Date(deadline).toISOString() };
|
|
499
|
+
saveTiming();
|
|
500
|
+
}
|
|
501
|
+
let waitFailed = false;
|
|
502
|
+
try {
|
|
459
503
|
let live: Inspection;
|
|
460
504
|
for (;;) {
|
|
461
505
|
live = await inspectRecovery(planned.project);
|
|
462
|
-
if (live.state !== "running") throw
|
|
463
|
-
if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw
|
|
506
|
+
if (live.state !== "running") throw nativeFailure(planned, binding.peer, "target_unavailable", `${binding.peer}: target hub reads as ${live.state} while awaiting attachment; saved terminal retained`, { handle: binding.handle, targetState: live.state });
|
|
507
|
+
if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw nativeFailure(planned, binding.peer, "target_unavailable", `${binding.peer}: target identity changed while awaiting attachment`, { handle: binding.handle, targetState: live.state });
|
|
464
508
|
const peer = live.peers.find((p) => p.id === binding.peer && p.state !== "offline");
|
|
465
509
|
const session = binding.peer === "codex" ? peer?.threadId : peer?.sessionId;
|
|
466
510
|
const changed = !!session && expectedSession !== null && session !== expectedSession;
|
|
467
511
|
// #64: only the original zero-turn Claude may change identity without an operator choice. Re-read persistence
|
|
468
512
|
// at attachment, not at the plan or wait start; an already accepted replacement is bound to its own id.
|
|
469
|
-
const zeroTurn = changed && binding.peer === "claude" && (planned.terminals as TerminalBinding[]).some((p) => p.peer === "claude" && p.sessionId === binding.sessionId) && !claudeTranscriptExists(binding);
|
|
470
|
-
if (changed && !zeroTurn) throw
|
|
513
|
+
const zeroTurn = changed && !progress.reattachments?.[binding.peer] && !progress.repairedSessions?.[binding.peer] && binding.peer === "claude" && (planned.terminals as TerminalBinding[]).some((p) => p.peer === "claude" && p.sessionId === binding.sessionId) && !claudeTranscriptExists(binding);
|
|
514
|
+
if (changed && !zeroTurn) throw nativeFailure(planned, binding.peer, "session_changed", `${binding.peer}: a different conversation attached while awaiting ${binding.sessionId}`, { handle: binding.handle, expectedSession: expectedSession!, observedSession: session! });
|
|
471
515
|
if (now() > deadline) break;
|
|
472
516
|
const launcher = await launcherOf(binding.peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId });
|
|
473
517
|
const sameTerminal = launcher?.record && matchesTerminal(launcher.record);
|
|
@@ -497,9 +541,9 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
497
541
|
}
|
|
498
542
|
if (decision.action === "fail" || decision.action === "create") throw new AttachmentLauncherGone(decision);
|
|
499
543
|
if (decision.action === "block") {
|
|
500
|
-
if (launchState === "replaced") throw
|
|
501
|
-
if (launchState === "older-instance") throw
|
|
502
|
-
throw
|
|
544
|
+
if (launchState === "replaced") throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: saved terminal identity changed while awaiting attachment`, { handle: binding.handle, launcherState: launchState });
|
|
545
|
+
if (launchState === "older-instance") throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: an older target's launcher in terminal ${olderHandle ?? "unknown"} may still run; saved terminal retained; no terminal was created`, { ...(olderHandle ? { handle: olderHandle } : {}), launcherState: launchState });
|
|
546
|
+
throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: the chosen restoration launcher's identity cannot be verified (${launchState}); saved terminal retained; no terminal was created`, { handle: binding.handle, launcherState: launchState });
|
|
503
547
|
}
|
|
504
548
|
if (now() >= deadline) break;
|
|
505
549
|
await sleep(Math.min(1000, deadline - now()));
|
|
@@ -520,10 +564,23 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
520
564
|
hub: { state: live.state, ...(live.instanceId ? { instanceId: live.instanceId } : {}), ...(peer ? { peerState: peer.state } : {}), ...(session ? { sessionId: session } : {}) },
|
|
521
565
|
lastHubConnectAttempt: "unknown" };
|
|
522
566
|
save();
|
|
523
|
-
throw
|
|
567
|
+
throw nativeFailure(planned, binding.peer, "attachment_timeout", `${binding.peer}: attachment not verified within ${RECONNECT_WAIT_MS / 1000} s; terminal ${binding.handle} retained; resume after it attaches (native-start diagnostic recorded; last hub connect attempt unknown)`, { handle: binding.handle, timeoutMs: RECONNECT_WAIT_MS });
|
|
568
|
+
} catch (error) { waitFailed = true; throw error; }
|
|
569
|
+
finally {
|
|
570
|
+
const finish = () => {
|
|
571
|
+
if (!waitStep) return;
|
|
572
|
+
op.waits![waitStep] = { ...op.waits![waitStep]!, waitEndedAt: new Date(now()).toISOString() };
|
|
573
|
+
saveTiming();
|
|
574
|
+
};
|
|
575
|
+
// Metadata storage must not hide the attachment failure or change its recovery outcome.
|
|
576
|
+
if (waitFailed) { try { finish(); } catch { /* original attachment failure remains authoritative */ } }
|
|
577
|
+
else finish();
|
|
578
|
+
}
|
|
524
579
|
};
|
|
525
580
|
return {
|
|
581
|
+
...(liveTestPeer === undefined ? {} : { liveTestPeer }),
|
|
526
582
|
now, sleep, inspect: inspectRecovery,
|
|
583
|
+
recordedSessionState: async (binding, kind) => kind === "rollout" ? codexTranscript(binding) : claudeTranscriptState(binding),
|
|
527
584
|
stage: async (op) => {
|
|
528
585
|
if (packageDigest(op.sourceRoot, { ownDependencies: true }) !== op.plan.sourceDigest) throw new FinalRefusal("preserved source changed");
|
|
529
586
|
const target = op.plan.kind === "restart" ? { root: op.sourceRoot, digest: verifyPackage(op.sourceRoot, op.plan.version, { ownDependencies: true }) }
|
|
@@ -549,7 +606,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
549
606
|
const verified = { ...target, targetFeatures };
|
|
550
607
|
// Existing peer/waiver compatibility checks remain properties of the already verified bytes.
|
|
551
608
|
if (op.targetDigest && op.targetDigest === target.digest && op.targetRoot === target.root) return verified;
|
|
552
|
-
if (op.plan.projects.some(p => p.source.peers.some(peer => peer.id
|
|
609
|
+
if (op.plan.projects.some(p => p.source.peers.some(peer => definitionOf(peer.id)?.session.restoreOnStart && peer.state !== "offline"))) {
|
|
553
610
|
const catalog = join(target.root, "src/hub/peer-catalog.ts");
|
|
554
611
|
const peers = existsSync(catalog) ? /export const OPTIONAL_PEERS\s*=\s*\[([^\]]*)\]/.exec(readFileSync(catalog, "utf8"))?.[1] : undefined;
|
|
555
612
|
if (!peers || !/["']qwen["']/.test(peers)) throw new FinalRefusal("target does not support Qwen; end Qwen and make a new plan, or choose a supporting target; runtimes unchanged");
|
|
@@ -568,7 +625,15 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
568
625
|
prepare: (p, id, instance, features) => control(p, "prepare", id, instance, features),
|
|
569
626
|
abort: (p, id, instance) => control(p, "abort", id, instance),
|
|
570
627
|
commit: (p, id, instance, features) => control(p, "commit", id, instance, features),
|
|
571
|
-
release: (p, id, instance) =>
|
|
628
|
+
release: async (p, id, instance) => {
|
|
629
|
+
// Snapshot before the request: a later writer must never be retired by this older release completion.
|
|
630
|
+
const previous = readRecoveryContinuity(p.stateDir, { projectId: p.id, projectRoot: p.root });
|
|
631
|
+
await control(p, "release", id, instance);
|
|
632
|
+
if (previous.state === "ready" && previous.bookmark.operationId !== id) {
|
|
633
|
+
try { retireRecoveryContinuity(p.stateDir, { projectId: p.id, projectRoot: p.root, operationId: previous.bookmark.operationId, digest: previous.digest }, "recovery_released"); }
|
|
634
|
+
catch { console.error("Pi continuity retirement is unconfirmed after successful recovery release; inspect the retained bookmark before another automatic start"); }
|
|
635
|
+
}
|
|
636
|
+
},
|
|
572
637
|
closeTerminals: async (planned, progress, op, save) => {
|
|
573
638
|
for (const binding of planned.terminals as TerminalBinding[]) {
|
|
574
639
|
const key = `closed:${binding.peer}`;
|
|
@@ -618,10 +683,10 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
618
683
|
if (reconnect.length) waiveRecoveryPeers(p.stateDir, op.id, Object.fromEntries(reconnect.map((peer) => [peer, "reconnect-only"])));
|
|
619
684
|
await command(op, ["up"], p);
|
|
620
685
|
},
|
|
621
|
-
restore: async (planned, progress, op, group, save) => {
|
|
686
|
+
restore: async (planned, progress, op, group, save, onlyPeer) => {
|
|
622
687
|
const live = await inspectRecovery(planned.project);
|
|
623
688
|
if (live.instanceId !== progress.instanceId) throw new Error("target daemon changed before peer restore");
|
|
624
|
-
if (group === "native") {
|
|
689
|
+
if (group === "native" && !onlyPeer) {
|
|
625
690
|
for (const peer of planned.source.peers.filter((p) => ["kimi", "qwen", "jcode", "local", "pi"].includes(p.id) && p.state !== "offline" && !(p.id === "pi" && p.args?.mode === "tui"))) {
|
|
626
691
|
const current = live.peers.find((p) => p.id === peer.id);
|
|
627
692
|
if (current && current.state !== "offline") continue;
|
|
@@ -638,7 +703,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
638
703
|
// #269: `ahub pi` without a mode now takes Pi's start mode (tui by default). This Pi ran headless (a terminal
|
|
639
704
|
// one is filtered out above), so a snapshot that recorded no mode still restores it headless.
|
|
640
705
|
if (peer.id === "pi" && typeof peer.args?.mode !== "string") args.push("--mode", "headless");
|
|
641
|
-
if (peer.id
|
|
706
|
+
if (definitionOf(peer.id)?.session.restoreOnStart && peer.sessionId) args.push("--session-id", peer.sessionId);
|
|
642
707
|
if (peer.id === "jcode" && peer.sessionId) args.push("--session-id", peer.sessionId);
|
|
643
708
|
if (peer.id === "jcode" && peer.args?.providerProfile) args.push("--provider-profile", peer.args.providerProfile);
|
|
644
709
|
if (peer.id === "pi") {
|
|
@@ -649,179 +714,221 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
649
714
|
await command(op, args, planned.project);
|
|
650
715
|
}
|
|
651
716
|
}
|
|
717
|
+
let claudeGroup = false;
|
|
652
718
|
for (const plannedBinding of planned.terminals as TerminalBinding[]) {
|
|
653
|
-
if ((plannedBinding.peer === "claude") !== (group === "claude")) continue;
|
|
719
|
+
if ((plannedBinding.peer === "claude") !== (claudeGroup = group === "claude") || onlyPeer && plannedBinding.peer !== onlyPeer) continue;
|
|
654
720
|
const key = `restored:${plannedBinding.peer}`;
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
721
|
+
try {
|
|
722
|
+
// #225: after a restart, the stopped target's terminal for this peer goes first; a session it had accepted
|
|
723
|
+
// instead of the planned one (fresh choice, fresh start, zero-turn Claude) is relaunched by that id, which the
|
|
724
|
+
// operation-fenced waiver already covers.
|
|
725
|
+
await closeRetired(progress, plannedBinding.peer, save);
|
|
726
|
+
const retired = progress.terminals[`retired:${plannedBinding.peer}`] as TerminalBinding | undefined;
|
|
727
|
+
const accepted = retired && typeof retired === "object" && retired.sessionId !== plannedBinding.sessionId ? retired.sessionId : undefined;
|
|
728
|
+
const repair = progress.reattachments?.[plannedBinding.peer];
|
|
729
|
+
const recordedOnly = !!repair || progress.repairedSessions?.[plannedBinding.peer] === true && !progress.fresh?.[plannedBinding.peer]?.repaired;
|
|
730
|
+
const retiredReceipt = recordedOnly && retired !== undefined ? decodeRestoreReceipt({ state: "done", binding: retired }) : undefined;
|
|
731
|
+
const recordedBinding = retiredReceipt?.state === "done" && retiredReceipt.binding.peer === plannedBinding.peer && retiredReceipt.binding.projectRoot === planned.project.root ? retiredReceipt.binding : undefined;
|
|
732
|
+
const original = repair?.binding ?? recordedBinding ?? (accepted ? { ...plannedBinding, sessionId: accepted } : plannedBinding);
|
|
733
|
+
// A new session is accepted only by the operator's choice (#215 dispose --fresh-session) or a planned fresh
|
|
734
|
+
// start of a Codex thread with nothing to lose that still has no rollout; an unreadable store is neither.
|
|
735
|
+
const chosen = !recordedOnly && !!progress.fresh?.[original.peer];
|
|
736
|
+
let read: ReturnType<typeof codexTranscript> | undefined;
|
|
737
|
+
const transcript = () => original.peer !== "codex" || (chosen && !accepted) ? "found" : (read ??= codexTranscript(original));
|
|
738
|
+
// An accepted session starts new again only when it has nothing to resume: a Codex thread that never got a
|
|
739
|
+
// rollout, or a Claude session with no transcript (`--resume` could never succeed, #21).
|
|
740
|
+
const fresh = () => recordedOnly ? false : accepted
|
|
741
|
+
? (original.peer === "codex" ? (chosen || !!planned.freshStart?.includes("codex")) && transcript() === "missing" : original.peer === "claude" && !claudeTranscriptExists(original))
|
|
742
|
+
: chosen || (!!planned.freshStart?.includes(original.peer) && transcript() === "missing");
|
|
743
|
+
// The runner ends this with the choices `status` shows (nextActions); resume comes first and launches a failed
|
|
744
|
+
// peer again, which helps once its cause (a transient Orca create failure, a fixed store) is gone.
|
|
745
|
+
// The waiver's audit label: the operator's choice, a Claude with nothing to resume, or the plan's Codex fresh start.
|
|
746
|
+
const waiver = () => progress.fresh?.[original.peer] ? "fresh-session" : original.peer === "claude" ? "zero-turn" : "fresh-start";
|
|
747
|
+
const notRestored = (why: string) => new Error(fresh()
|
|
748
|
+
? `${original.peer}: ${why}; read that terminal in Orca for the launcher's error and fix its cause`
|
|
749
|
+
: `${original.peer}: ${why}; session ${original.sessionId} was not restored; resume launches it again once the cause is fixed`);
|
|
750
|
+
// Anything live or unknown blocks a create and keeps the receipt: never decide on evidence that could not be read.
|
|
751
|
+
const blocked = (evidence: Exclude<Awaited<ReturnType<typeof peerEvidence>>, { state: "gone" }>) => evidence.state === "unknown"
|
|
752
|
+
? new Error(`${original.peer}: whether a ${original.peer} session or launcher is live cannot be told (${evidence.why}); nothing was recorded or created${evidence.step ? `; ${evidence.step} first` : ""}`)
|
|
753
|
+
: nativeFailure(planned, original.peer, evidence.session ? "session_changed" : "launcher_unverified", evidence.session
|
|
754
|
+
? `${original.peer}: session ${evidence.session} is attached instead of ${original.sessionId}; no terminal was created; end that ${original.peer} session and close its terminal first`
|
|
755
|
+
: `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session ${evidence.attached ? "with an id " : ""}attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`,
|
|
756
|
+
evidence.session
|
|
757
|
+
? { expectedSession: original.sessionId, observedSession: evidence.session }
|
|
758
|
+
: { ...(evidence.handle ? { handle: evidence.handle } : {}), launcherState: evidence.terminal ? "live" : "older-instance" });
|
|
759
|
+
const decide = (receipt: RestoreReceipt, evidence: Awaited<ReturnType<typeof peerEvidence>>) => restoreTransition(receipt, {
|
|
760
|
+
target: "running", launcher: evidence.state === "unknown" ? "unknown" : evidence.state === "gone" ? "gone" : evidence.terminal ? "live" : "older-instance",
|
|
761
|
+
session: evidence.state === "live" && evidence.session ? evidence.session === original.sessionId ? "planned" : "other" : "none",
|
|
762
|
+
allowOther: fresh() || (!recordedOnly && original.peer === "claude" && !claudeTranscriptExists(original)),
|
|
763
|
+
binding: evidence.state === "live" && evidence.terminal ? { ...original, ...evidence.terminal } : original,
|
|
764
|
+
});
|
|
765
|
+
const receipt = decodeRestoreReceipt(progress.terminals[key]);
|
|
766
|
+
function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }> & { origin: "pending" }, expected: string | null): Promise<string>;
|
|
767
|
+
function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined>;
|
|
768
|
+
async function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined> {
|
|
769
|
+
try { return await awaitAttachment(planned, progress, op, waiting, save, expected); }
|
|
770
|
+
catch (error) {
|
|
771
|
+
if (!(error instanceof AttachmentLauncherGone)) throw error;
|
|
772
|
+
progress.terminals[key] = error.transition.next; save();
|
|
773
|
+
// The table decides whether this failed wait blocks or is eligible for a new launch.
|
|
774
|
+
if (error.transition.action === "create") return undefined;
|
|
775
|
+
throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
|
|
776
|
+
}
|
|
702
777
|
}
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
778
|
+
// #215: the pending row of the receipt table, used wherever a pending launch meets an attached session.
|
|
779
|
+
const settle = async (attached: string, recovered?: TerminalBinding) => {
|
|
780
|
+
// #21: a Claude session that never got a first turn has no transcript, so `claude --resume S` can never
|
|
781
|
+
// succeed and accepting its fresh attach loses nothing. Otherwise only the planned id, or a new session
|
|
782
|
+
// the operator or the plan accepted, counts as restored.
|
|
783
|
+
if (attached !== original.sessionId && !fresh() && !(!recordedOnly && original.peer === "claude" && !claudeTranscriptExists(original))) {
|
|
784
|
+
throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${attached} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
|
|
785
|
+
}
|
|
786
|
+
// Validate the proposed transition before committing continuity/waiver effects or a done receipt.
|
|
787
|
+
const verified = await revalidateTerminal(planned, progress, attachedBinding(recovered ?? original, attached), !!recovered);
|
|
788
|
+
// The planned session came back after all: a fresh session chosen for it lost nothing. Compared with the plan,
|
|
789
|
+
// never with a session accepted after a restart (#225), whose choice must stay on record.
|
|
790
|
+
if (attached === plannedBinding.sessionId && progress.fresh?.[original.peer]?.lost === plannedBinding.sessionId) {
|
|
791
|
+
delete progress.fresh[original.peer];
|
|
792
|
+
if (!Object.keys(progress.fresh).length) delete progress.fresh;
|
|
793
|
+
}
|
|
794
|
+
// Any other accepted session needs the target to waive the saved id, or release would wait for it forever;
|
|
795
|
+
// a launch writes the waiver first, but this must not rely on that.
|
|
796
|
+
// The planned session needs no waiver: the target's snapshot already expects it.
|
|
797
|
+
if (attached !== original.sessionId && attached !== plannedBinding.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
|
|
798
|
+
if (repair) repair.state = "attempted"; // a verified spontaneous reattachment also closes this repair allowance
|
|
799
|
+
// The person accepted one loss, not a permanent exemption for later repairs of the replacement.
|
|
800
|
+
if (progress.fresh?.[original.peer]?.repaired) delete progress.fresh[original.peer]!.repaired;
|
|
801
|
+
progress.terminals[key] = { state: "done", binding: verified } satisfies RestoreReceipt; save();
|
|
802
|
+
};
|
|
803
|
+
if (receipt?.state === "done") {
|
|
804
|
+
// A completed restoration is never a waiting launch. A closed terminal blocks exact revalidation;
|
|
805
|
+
// it cannot erase continuity or create a fresh-session choice (PROBE-B).
|
|
806
|
+
await detached(planned, progress, op, receipt.binding, save);
|
|
807
|
+
const observed = await peerEvidence(planned, progress, original.peer);
|
|
808
|
+
const decision = restoreTransition(receipt, { target: "running", launcher: "no-record-yet", session: observed.state === "live" && observed.session === receipt.binding.sessionId ? "planned" : observed.state === "live" && observed.session ? "other" : "none", allowOther: false });
|
|
809
|
+
if (decision.action === "block") {
|
|
810
|
+
const condition = observed.state === "unknown" ? `cannot be verified (${observed.why})`
|
|
811
|
+
: observed.state === "live" && observed.session ? `session ${observed.session} is attached instead of ${receipt.binding.sessionId}`
|
|
812
|
+
: observed.state === "live" ? `is not attached; its launcher in terminal ${observed.handle ?? "unknown"} may still run` : "is not attached";
|
|
813
|
+
throw new Error(`${original.peer}: completed restoration ${condition}; saved done receipt retained; resume its recorded conversation`);
|
|
814
|
+
}
|
|
815
|
+
progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, receipt.binding, true) } satisfies RestoreReceipt; save();
|
|
816
|
+
continue;
|
|
711
817
|
}
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
if (attached === plannedBinding.sessionId && progress.fresh?.[original.peer]) {
|
|
717
|
-
delete progress.fresh[original.peer];
|
|
718
|
-
if (!Object.keys(progress.fresh).length) delete progress.fresh;
|
|
818
|
+
if (receipt?.state === "waiting") {
|
|
819
|
+
const binding = receipt.binding;
|
|
820
|
+
const sessionId = await awaitBound(receipt, fresh() && binding.sessionId === original.sessionId ? null : binding.sessionId);
|
|
821
|
+
if (sessionId !== undefined) { await settle(sessionId, binding); continue; }
|
|
719
822
|
}
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
823
|
+
if (receipt?.state === "pending" || receipt?.state === "failed") {
|
|
824
|
+
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
825
|
+
const decision = decide(receipt, evidence);
|
|
826
|
+
if (decision.action === "settle" && evidence.state === "live" && evidence.session) { await settle(evidence.session); continue; }
|
|
827
|
+
if (decision.action === "poll" && decision.next.state === "waiting") {
|
|
828
|
+
const waiting = decision.next;
|
|
829
|
+
progress.terminals[key] = waiting; save();
|
|
830
|
+
const session = await awaitBound(waiting, fresh() ? null : original.sessionId);
|
|
831
|
+
if (session !== undefined) { await settle(session, waiting.binding); continue; }
|
|
832
|
+
} else if (decision.action === "fail") {
|
|
833
|
+
progress.terminals[key] = decision.next; save();
|
|
834
|
+
throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
|
|
835
|
+
} else if (decision.action === "block") {
|
|
836
|
+
if (evidence.state === "live" && evidence.session) {
|
|
837
|
+
throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${evidence.session} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
|
|
838
|
+
}
|
|
839
|
+
if (evidence.state !== "gone") throw blocked(evidence);
|
|
840
|
+
throw new Error(`${original.peer}: restoration evidence is not actionable`);
|
|
841
|
+
}
|
|
736
842
|
}
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
}
|
|
740
|
-
if (receipt?.state === "waiting") {
|
|
741
|
-
const binding = receipt.binding;
|
|
742
|
-
const sessionId = await awaitBound(receipt, fresh() && binding.sessionId === original.sessionId ? null : binding.sessionId);
|
|
743
|
-
if (sessionId !== undefined) { await settle(sessionId, binding); continue; }
|
|
744
|
-
}
|
|
745
|
-
if (receipt?.state === "pending" || receipt?.state === "failed") {
|
|
843
|
+
// Never create beside a live launch, nor on evidence that could not be read: read now, not at restore start.
|
|
844
|
+
// The planned session already attached is the restoration (the table's row for it), not a reason to block.
|
|
746
845
|
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
progress.terminals[key] =
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
846
|
+
if (evidence.state === "live" && evidence.session === original.sessionId) {
|
|
847
|
+
if (repair) await settle(evidence.session);
|
|
848
|
+
else { progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, original, false) } satisfies RestoreReceipt; save(); }
|
|
849
|
+
continue;
|
|
850
|
+
}
|
|
851
|
+
if (evidence.state !== "gone") throw blocked(evidence);
|
|
852
|
+
if (transcript() === "unknown") throw new Error(`${unresumable(original)}; no terminal was created; make that store readable first`);
|
|
853
|
+
if (transcript() === "missing" && !fresh()) {
|
|
854
|
+
progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
|
|
855
|
+
if (recordedOnly) throw new MissingRecordedSession({ projectId: planned.project.id, peer: original.peer, sessionId: original.sessionId, kind: "rollout", checkedPath: checkedSessionPath(original, "rollout") });
|
|
856
|
+
throw notRestored(unresumable(original).slice("codex: ".length));
|
|
857
|
+
}
|
|
858
|
+
if (fresh()) {
|
|
859
|
+
if (!targetReadsWaivers(op)) throw new Error(`${original.peer}: target ${op.plan.version} predates recovery waivers, so it cannot accept a new session`);
|
|
860
|
+
// The restored daemon must accept the new session instead of the recorded one.
|
|
861
|
+
waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
|
|
862
|
+
}
|
|
863
|
+
if (repair) {
|
|
864
|
+
if (repair.state === "attempted") throw new ReattachmentRequired(planned.project.id, original.peer, `${original.peer}: recorded-session reattachment already attempted; no second terminal will be created`);
|
|
865
|
+
repair.state = "attempted"; save(); // durable before any terminal creation, including an uncertain failure
|
|
866
|
+
}
|
|
867
|
+
const entrypoint = join(op.targetRoot!, "src/cli/main.js");
|
|
868
|
+
const argv = restoredTerminalArgv(entrypoint, planned.project.root, original, fresh());
|
|
869
|
+
const assignments = { ...original.launch.env, AGENTHUB_HOME: hubHome(), AGENTHUB_RECOVERY_OPERATION: op.id };
|
|
870
|
+
const launch = { ...original.launch, packageEntrypoint: entrypoint, argv,
|
|
871
|
+
command: ["env", ...Object.entries(assignments).map(([k, v]) => `${k}=${v}`), ...argv].map(shellQuote).join(" ") };
|
|
872
|
+
const binding = { ...original, launch, launchMetadata: launch };
|
|
873
|
+
progress.terminals[key] = { state: "pending" } satisfies RestoreReceipt; save();
|
|
874
|
+
const restored = await createTerminal(binding, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId }, undefined, fresh());
|
|
875
|
+
const exited = restored.blockers.find((b) => b.code === "launcher-exited");
|
|
876
|
+
if (exited) {
|
|
877
|
+
// #215: the launcher's exit alone is not the peer's state (a target that died or flapped also ends it): only
|
|
878
|
+
// nothing live makes it failed; an attached session is settled as the pending row settles it; anything else
|
|
879
|
+
// stays pending.
|
|
880
|
+
const after = await peerEvidence(planned, progress, original.peer);
|
|
881
|
+
const decision = decide({ state: "pending" }, after);
|
|
882
|
+
if (decision.action === "settle" && after.state === "live" && after.session) { await settle(after.session); continue; }
|
|
883
|
+
if (decision.action === "poll" && decision.next.state === "waiting") {
|
|
884
|
+
progress.terminals[key] = decision.next; save();
|
|
885
|
+
const session = await awaitBound(decision.next, fresh() ? null : original.sessionId);
|
|
886
|
+
if (session !== undefined) { await settle(session, decision.next.binding); continue; }
|
|
760
887
|
}
|
|
761
|
-
if (
|
|
762
|
-
|
|
888
|
+
if (decision.action === "fail") {
|
|
889
|
+
progress.terminals[key] = decision.next; save();
|
|
890
|
+
throw notRestored(exited.message);
|
|
891
|
+
}
|
|
892
|
+
if (after.state !== "gone") throw blocked(after);
|
|
763
893
|
}
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
768
|
-
if (evidence.state === "live" && evidence.session === original.sessionId) {
|
|
769
|
-
progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, original, false) } satisfies RestoreReceipt; save();
|
|
770
|
-
continue;
|
|
771
|
-
}
|
|
772
|
-
if (evidence.state !== "gone") throw blocked(evidence);
|
|
773
|
-
if (transcript() === "unknown") throw new Error(`${unresumable(original)}; no terminal was created; make that store readable first`);
|
|
774
|
-
if (transcript() === "missing" && !fresh()) {
|
|
775
|
-
progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
|
|
776
|
-
throw notRestored(unresumable(original).slice("codex: ".length));
|
|
777
|
-
}
|
|
778
|
-
if (fresh()) {
|
|
779
|
-
if (!targetReadsWaivers(op)) throw new Error(`${original.peer}: target ${op.plan.version} predates recovery waivers, so it cannot accept a new session`);
|
|
780
|
-
// The restored daemon must accept the new session instead of the recorded one.
|
|
781
|
-
waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
|
|
782
|
-
}
|
|
783
|
-
const entrypoint = join(op.targetRoot!, "src/cli/main.js");
|
|
784
|
-
const argv = restoredTerminalArgv(entrypoint, planned.project.root, original, fresh());
|
|
785
|
-
const assignments = { ...original.launch.env, AGENTHUB_HOME: hubHome(), AGENTHUB_RECOVERY_OPERATION: op.id };
|
|
786
|
-
const launch = { ...original.launch, packageEntrypoint: entrypoint, argv,
|
|
787
|
-
command: ["env", ...Object.entries(assignments).map(([k, v]) => `${k}=${v}`), ...argv].map(shellQuote).join(" ") };
|
|
788
|
-
const binding = { ...original, launch, launchMetadata: launch };
|
|
789
|
-
progress.terminals[key] = { state: "pending" } satisfies RestoreReceipt; save();
|
|
790
|
-
const restored = await createTerminal(binding, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId }, undefined, fresh());
|
|
791
|
-
const exited = restored.blockers.find((b) => b.code === "launcher-exited");
|
|
792
|
-
if (exited) {
|
|
793
|
-
// #215: the launcher's exit alone is not the peer's state (a target that died or flapped also ends it): only
|
|
794
|
-
// nothing live makes it failed; an attached session is settled as the pending row settles it; anything else
|
|
795
|
-
// stays pending.
|
|
796
|
-
const after = await peerEvidence(planned, progress, original.peer);
|
|
797
|
-
const decision = decide({ state: "pending" }, after);
|
|
798
|
-
if (decision.action === "settle" && after.state === "live" && after.session) { await settle(after.session); continue; }
|
|
799
|
-
if (decision.action === "poll" && decision.next.state === "waiting") {
|
|
800
|
-
progress.terminals[key] = decision.next; save();
|
|
801
|
-
const session = await awaitBound(decision.next, fresh() ? null : original.sessionId);
|
|
802
|
-
if (session !== undefined) { await settle(session, decision.next.binding); continue; }
|
|
894
|
+
if (restored.manualRequired || !restored.newBinding) {
|
|
895
|
+
const handle = restored.binding?.handle ?? restored.blockers[0]?.handle;
|
|
896
|
+
throw new Error(`${original.peer}: original session restoration needs manual verification${restored.blockers[0] ? ` (${restored.blockers[0].message})` : ""}; wait until it attaches, or end that launch and close its terminal${handle ? ` ${handle}` : ""} first`);
|
|
803
897
|
}
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
898
|
+
// The daemon's id is authoritative: a fresh session is recorded under the id it reports once attached.
|
|
899
|
+
const awaiting = { state: "waiting", origin: "pending", binding: restored.newBinding } satisfies RestoreReceipt;
|
|
900
|
+
progress.terminals[key] = awaiting; save();
|
|
901
|
+
const sessionId = await awaitBound(awaiting, fresh() ? null : restored.newBinding.sessionId);
|
|
902
|
+
await settle(sessionId, awaiting.binding);
|
|
903
|
+
} catch (error) {
|
|
904
|
+
if (error instanceof MissingRecordedSession || isDatabaseBusy(error)) throw error;
|
|
905
|
+
// A failed recorded-only Claude launch may prove that its transcript is gone. No error text is evidence.
|
|
906
|
+
if (claudeGroup && progress.repairedSessions?.[plannedBinding.peer] && !progress.fresh?.[plannedBinding.peer]?.repaired) {
|
|
907
|
+
const chosenReceipt = decodeRestoreReceipt(progress.terminals[key]);
|
|
908
|
+
// A bare pending/failed row can hide an uncertain create. Only a durable exact chosen binding
|
|
909
|
+
// can become the terminal that an explicit fresh choice must close before another create.
|
|
910
|
+
const binding = chosenReceipt?.state === "waiting" ? chosenReceipt.binding : undefined;
|
|
911
|
+
if (binding && claudeTranscriptState(binding) === "missing" && (await peerEvidence(planned, progress, plannedBinding.peer)).state === "gone") {
|
|
912
|
+
// Retain the exact chosen terminal so an explicit later fresh choice closes it before creating again.
|
|
913
|
+
progress.terminals[`retired:${plannedBinding.peer}`] = binding;
|
|
914
|
+
delete progress.terminals[`closedRetired:${plannedBinding.peer}`];
|
|
915
|
+
const marker = progress.reattachments?.[plannedBinding.peer];
|
|
916
|
+
if (marker) marker.binding = binding;
|
|
917
|
+
progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
|
|
918
|
+
throw new MissingRecordedSession({ projectId: planned.project.id, peer: plannedBinding.peer, sessionId: binding.sessionId, kind: "transcript", checkedPath: checkedSessionPath(binding, "transcript") });
|
|
919
|
+
}
|
|
807
920
|
}
|
|
808
|
-
if (
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
throw new Error(`${original.peer}: original session restoration needs manual verification${restored.blockers[0] ? ` (${restored.blockers[0].message})` : ""}; wait until it attaches, or end that launch and close its terminal${handle ? ` ${handle}` : ""} first`);
|
|
921
|
+
if (error instanceof ReattachmentRequired || isDatabaseBusy(error)) throw error;
|
|
922
|
+
if (progress.reattachments?.[plannedBinding.peer]) throw new ReattachmentRequired(planned.project.id, plannedBinding.peer,
|
|
923
|
+
error instanceof Error ? error.message : `${plannedBinding.peer}: recorded-session reattachment is unconfirmed`, error instanceof RecoveryPresentationError ? error : undefined);
|
|
924
|
+
throw error;
|
|
813
925
|
}
|
|
814
|
-
// The daemon's id is authoritative: a fresh session is recorded under the id it reports once attached.
|
|
815
|
-
const awaiting = { state: "waiting", origin: "pending", binding: restored.newBinding } satisfies RestoreReceipt;
|
|
816
|
-
progress.terminals[key] = awaiting; save();
|
|
817
|
-
const sessionId = await awaitBound(awaiting, fresh() ? null : restored.newBinding.sessionId);
|
|
818
|
-
await settle(sessionId, awaiting.binding);
|
|
819
926
|
}
|
|
820
|
-
if (group === "claude") for (const id of planned.reconnectOnly ?? []) {
|
|
927
|
+
if (group === "claude" && !onlyPeer) for (const id of planned.reconnectOnly ?? []) {
|
|
821
928
|
// #206: nothing is launched for an unmanaged session; its plugin reconnects by itself, within a bound.
|
|
822
929
|
const deadline = now() + UNMANAGED_RECONNECT_WAIT_MS;
|
|
823
930
|
while (!(await inspectRecovery(planned.project)).peers.some((p) => p.id === id && p.state !== "offline")) {
|
|
824
|
-
if (now() >= deadline) throw
|
|
931
|
+
if (now() >= deadline) throw nativeFailure(planned, id, "attachment_timeout", `${id}: the unmanaged session did not reconnect within ${UNMANAGED_RECONNECT_WAIT_MS / 1000} s; if that Claude session was closed, start Claude in the project again (its plugin reconnects) first`, { timeoutMs: UNMANAGED_RECONNECT_WAIT_MS });
|
|
825
932
|
await sleep(1000);
|
|
826
933
|
}
|
|
827
934
|
}
|
|
@@ -855,24 +962,61 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
855
962
|
await awaitStopped(project, instance);
|
|
856
963
|
}
|
|
857
964
|
abandonRestartSnapshot(project.stateDir, op.id);
|
|
965
|
+
const planned = op.plan.projects?.find(item => item.project.id === project.id);
|
|
966
|
+
const progress = op.projects?.find(item => item.id === project.id);
|
|
967
|
+
const source = planned?.source.peers.find(peer => peer.id === "pi" && peer.state !== "offline");
|
|
968
|
+
if (source && progress) {
|
|
969
|
+
// abandonRestartSnapshot is deliberately a no-op for another operation or an unreadable snapshot.
|
|
970
|
+
// A continuity bookmark needs positive archival evidence, including when disposal resumes after a crash.
|
|
971
|
+
const archive = join(project.stateDir, `restart.abandoned.${createHash("sha256").update(op.id).digest("hex")}.json`);
|
|
972
|
+
let archived: unknown;
|
|
973
|
+
try { archived = JSON.parse(readFileSync(archive, "utf8")); } catch { return { continuity: "not-archived" }; }
|
|
974
|
+
if (!archived || typeof archived !== "object" || (archived as { operationId?: unknown }).operationId !== op.id) return { continuity: "not-archived" };
|
|
975
|
+
try {
|
|
976
|
+
const binding = planned ? resolveAcceptedBinding(planned, progress, "pi") : undefined;
|
|
977
|
+
const sessionId = binding?.sessionId ?? source.sessionId;
|
|
978
|
+
if (!sessionId) throw new Error("Pi recorded session unavailable; archive continuity bookmark was not written");
|
|
979
|
+
const backend = binding?.backend ?? source.args?.backend;
|
|
980
|
+
writeRecoveryContinuity(project.stateDir, { schema: 1, operationId: op.id, projectId: project.id, projectRoot: project.root, at: now(),
|
|
981
|
+
pi: { sessionId, sessionFile: binding?.sessionFile ?? source.sessionFile ?? source.args?.sessionFile,
|
|
982
|
+
...(backend === "auto" || backend === "dgx" || backend === "mlx" ? { backend } : {}), model: binding?.model ?? source.args?.model } });
|
|
983
|
+
} catch { return { continuity: "unavailable" }; } // optional metadata must not wedge stop-and-archive
|
|
984
|
+
}
|
|
858
985
|
},
|
|
859
986
|
refreshManager: async (op) => { await refreshManager({ home: hubHome(), cli: join(op.targetRoot!, "src/cli/main.ts") }); },
|
|
860
|
-
verify: async (planned, progress, op) => {
|
|
987
|
+
verify: async (planned, progress, op, save) => {
|
|
861
988
|
const live = await inspectRecovery(planned.project);
|
|
862
989
|
if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw new Error("target identity verification failed");
|
|
863
990
|
for (const old of planned.source.peers.filter((p) => p.state !== "offline")) {
|
|
864
991
|
const peer = live.peers.find((p) => p.id === old.id);
|
|
865
|
-
if (!peer || !["idle", "busy", "paused"].includes(peer.state))
|
|
866
|
-
|
|
867
|
-
|
|
992
|
+
if (!peer || !["idle", "busy", "paused"].includes(peer.state)) {
|
|
993
|
+
const receipt = decodeRestoreReceipt(progress.terminals[`restored:${old.id}`]);
|
|
994
|
+
if (receipt?.state === "done") await detached(planned, progress, op, receipt.binding, save);
|
|
995
|
+
throw new Error(`${old.id}: peer reattachment not verified`);
|
|
996
|
+
}
|
|
997
|
+
const accepted = decodeRestoreReceipt(progress.terminals[`restored:${old.id}`]);
|
|
998
|
+
const repair = progress.reattachments?.[old.id as "codex" | "pi" | "claude"];
|
|
999
|
+
const recordedOnly = !!repair || progress.repairedSessions?.[old.id as keyof NonNullable<ProjectProgress["repairedSessions"]>] === true && !progress.fresh?.[old.id]?.repaired;
|
|
1000
|
+
const retiredValue = progress.terminals[`retired:${old.id}`];
|
|
1001
|
+
const retired = retiredValue === undefined ? undefined : decodeRestoreReceipt({ state: "done", binding: retiredValue });
|
|
1002
|
+
// A #225 restart clears the old target's repair fence, never its accepted conversation.
|
|
1003
|
+
const retained = retired?.state === "done" && accepted?.state === "done" && retired.binding.peer === old.id && retired.binding.projectRoot === planned.project.root && retired.binding.sessionId === accepted.binding.sessionId;
|
|
1004
|
+
if ((recordedOnly || retained) && accepted?.state === "done") {
|
|
1005
|
+
const session = old.id === "codex" ? peer.threadId : peer.sessionId;
|
|
1006
|
+
if (session !== accepted.binding.sessionId || repair && accepted.binding.sessionId !== repair.binding.sessionId ||
|
|
1007
|
+
old.id === "pi" && (recordedOnly || old.sessionFile) && accepted.binding.sessionFile && peer.sessionFile !== accepted.binding.sessionFile) throw nativeFailure(planned, old.id, "session_changed", `${old.id}: reattached a different recorded conversation`, { handle: accepted.binding.handle, expectedSession: accepted.binding.sessionId, ...(session === undefined ? {} : { observedSession: session }) });
|
|
1008
|
+
continue;
|
|
1009
|
+
}
|
|
1010
|
+
if (!recordedOnly && (progress.fresh?.[old.id] || planned.freshStart?.includes(old.id))) continue; // #215: a new session was accepted (recorded)
|
|
1011
|
+
if (old.id === "codex" && peer.threadId !== old.threadId) throw nativeFailure(planned, old.id, "session_changed", "Codex resumed a different conversation", { ...(old.threadId ? { expectedSession: old.threadId } : {}), ...(peer.threadId ? { observedSession: peer.threadId } : {}) });
|
|
868
1012
|
if (old.id === "claude" && peer.sessionId !== old.sessionId) {
|
|
869
1013
|
// #64: the restore gate already accepted a fresh session when the original never
|
|
870
1014
|
// persisted a transcript. Re-derive persistence from the binding exactly like that
|
|
871
1015
|
// gate did, so a first turn landing after the plan was made keeps the check strict.
|
|
872
1016
|
const binding = (planned.terminals as TerminalBinding[]).find((t) => t.peer === "claude");
|
|
873
|
-
if (!binding || binding.sessionId !== old.sessionId || claudeTranscriptExists(binding)) throw
|
|
1017
|
+
if (recordedOnly || !binding || binding.sessionId !== old.sessionId || claudeTranscriptExists(binding)) throw nativeFailure(planned, old.id, "session_changed", "Claude resumed a different conversation", { ...(old.sessionId ? { expectedSession: old.sessionId } : {}), ...(peer.sessionId ? { observedSession: peer.sessionId } : {}) });
|
|
874
1018
|
}
|
|
875
|
-
if (old.id === "pi" && (peer.sessionId !== old.sessionId || (old.sessionFile && peer.sessionFile !== old.sessionFile))) throw
|
|
1019
|
+
if (old.id === "pi" && (peer.sessionId !== old.sessionId || (old.sessionFile && peer.sessionFile !== old.sessionFile))) throw nativeFailure(planned, old.id, "session_changed", "Pi resumed a different conversation", { ...(old.sessionId ? { expectedSession: old.sessionId } : {}), ...(peer.sessionId ? { observedSession: peer.sessionId } : {}) });
|
|
876
1020
|
}
|
|
877
1021
|
for (const original of planned.terminals as TerminalBinding[]) {
|
|
878
1022
|
const saved = decodeRestoreReceipt(progress.terminals[`restored:${original.peer}`]);
|
|
@@ -883,6 +1027,13 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
883
1027
|
if (snapshot.ok === false) throw new Error("dashboard snapshot readback failed");
|
|
884
1028
|
const recovery = await rpc(planned.project, { t: "recovery", op: "inspect", expectedInstanceId: progress.instanceId });
|
|
885
1029
|
const integrity = recovery.recovery?.integrity;
|
|
1030
|
+
if (integrity?.expected && integrity.current) {
|
|
1031
|
+
for (const check of ["queues", "manualPaused", "boardDigest", "budgetDigest"] as const) {
|
|
1032
|
+
if (JSON.stringify(integrity.current[check]) !== JSON.stringify(integrity.expected[check])) {
|
|
1033
|
+
throw new RecoveryPreservationError(check, planned.project.id);
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
886
1037
|
if (!integrity?.expected || JSON.stringify(integrity.current) !== JSON.stringify(integrity.expected)) {
|
|
887
1038
|
throw new Error("queue, manual pause, task board or budget preservation was not verified");
|
|
888
1039
|
}
|