@staix/agent-hub 0.12.30 → 0.12.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +29 -9
- package/docs/agent-notes/adapters.md +9 -5
- package/docs/agent-notes/budget.md +1 -1
- package/docs/agent-notes/bus.md +5 -0
- package/docs/agent-notes/daemon.md +14 -8
- package/docs/agent-notes/local-worker.md +3 -1
- package/docs/agent-notes/tasks.md +7 -1
- package/docs/agent-notes/tests.md +6 -2
- package/docs/cooperbench.md +1 -1
- package/docs/gsd-collaboration-analysis.md +1 -1
- package/docs/memory.md +2 -2
- package/docs/operations.md +236 -37
- package/docs/quickstart.md +24 -5
- package/docs/security.md +42 -10
- package/docs/smoke.md +34 -0
- package/docs/specs/2026-09-19-agent-hub-design.md +241 -3
- package/docs/specs/2026-09-20-upgrade-recovery-design.md +275 -10
- package/docs/verification/2026-10-17-0.12.31.md +71 -0
- package/docs/verification/2026-10-17-418-probes.md +21 -0
- package/docs/verification/2026-10-18-0.12.32.md +54 -0
- package/docs/verified.json +20 -15
- package/package.json +2 -2
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +1361 -44
- package/src/adapters/acp.ts +5 -6
- package/src/adapters/claude-channel.ts +29 -11
- package/src/adapters/definitions/builtin-metadata.ts +121 -0
- package/src/adapters/definitions/kimi-native.ts +4 -0
- package/src/adapters/definitions/qwen-native.ts +24 -0
- package/src/adapters/definitions/qwen.ts +71 -0
- package/src/adapters/local-worker.ts +2 -2
- package/src/adapters/pi.ts +11 -2
- package/src/adapters/registry-types.ts +174 -0
- package/src/adapters/registry.ts +36 -0
- package/src/cli/attention.ts +5 -1
- package/src/cli/console-state.ts +130 -17
- package/src/cli/console.ts +54 -8
- package/src/cli/help.ts +3 -1
- package/src/cli/identity.ts +16 -5
- package/src/cli/jcode-mcp.ts +1 -1
- package/src/cli/launch.ts +1 -6
- package/src/cli/main.ts +208 -40
- package/src/cli/output.ts +15 -2
- package/src/cli/preview.ts +8 -5
- package/src/cli/recovery-guidance.ts +137 -0
- package/src/cli/recovery-presentation-options.ts +8 -0
- package/src/cli/recovery-session-file.ts +18 -0
- package/src/cli/terminal-recovery.ts +48 -0
- package/src/cli/upgrade-interactive.ts +129 -74
- package/src/cli/upgrade-labels.ts +30 -0
- package/src/cli/upgrade-presentation-data.ts +41 -0
- package/src/cli/upgrade-presentation-flags.ts +31 -0
- package/src/cli/upgrade-presentation.ts +217 -0
- package/src/cli/upgrade-restore-state.ts +137 -0
- package/src/cli/upgrade-runtime.ts +430 -175
- package/src/cli/upgrade.ts +325 -49
- package/src/hub/board.ts +82 -24
- package/src/hub/bus.ts +89 -29
- package/src/hub/check-runner.ts +514 -0
- package/src/hub/child-process.ts +1 -0
- package/src/hub/cohorts.ts +2 -2
- package/src/hub/config-apply.ts +242 -0
- package/src/hub/config-reload.ts +120 -0
- package/src/hub/config-trust.ts +11 -2
- package/src/hub/control-client.ts +5 -1
- package/src/hub/crash.ts +8 -6
- package/src/hub/daemon.ts +931 -209
- package/src/hub/delivery-hints.ts +30 -0
- package/src/hub/delivery-journal.ts +185 -12
- package/src/hub/delivery-superseded.ts +52 -0
- package/src/hub/envelope.ts +1 -1
- package/src/hub/events.ts +2 -2
- package/src/hub/host-executable.ts +11 -0
- package/src/hub/hub-tools.ts +9 -1
- package/src/hub/manager.ts +1 -1
- package/src/hub/peer-catalog.ts +13 -13
- package/src/hub/peers.ts +12 -2
- package/src/hub/permission-mode.ts +5 -3
- package/src/hub/recovery-continuity.ts +152 -0
- package/src/hub/recovery-store.ts +10 -4
- package/src/hub/routing.ts +35 -16
- package/src/hub/sdlc.ts +5 -1
- package/src/hub/settings.ts +52 -37
- package/src/hub/settled-roles.ts +35 -0
- package/src/hub/spec-read.ts +120 -0
- package/src/hub/supervision.ts +24 -8
- package/src/hub/task-sweep.ts +7 -6
- package/src/hub/tasks.ts +199 -40
- package/src/hub/ui.ts +1 -1
- package/src/memory/brief.ts +6 -5
- package/src/memory/capture.ts +2 -1
- package/src/pi/session.ts +23 -4
- package/src/ui/index.html +35 -16
- package/src/ui/needs.ts +7 -5
- package/src/ui/office-model.ts +4 -1
- package/src/ui/task-board.ts +1 -1
- package/src/ui/task-progress.ts +6 -2
- package/templates/AGENTS.block.md +1 -1
- package/templates/profiles/general.json +1 -1
- package/templates/profiles/research.json +1 -1
- package/templates/profiles/writing.json +1 -1
|
@@ -1,23 +1,34 @@
|
|
|
1
|
+
import { checkedSessionPath, recordedFileState } from "./recovery-session-file.ts";
|
|
2
|
+
import { resolveAcceptedBinding } from "./recovery-guidance.ts";
|
|
3
|
+
import { definitionOf } from "../adapters/registry.ts";
|
|
1
4
|
import { qwenSessionId } from "../hub/crash.ts";
|
|
2
5
|
import { isDatabaseBusy, isDatabaseBusyReply, RecoveryDatabaseBusy } from "../hub/recovery-error.ts";
|
|
3
6
|
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync } from "node:fs";
|
|
4
|
-
import { homedir } from "node:os";
|
|
5
7
|
import { basename, join, resolve } from "node:path";
|
|
6
|
-
import { randomUUID } from "node:crypto";
|
|
8
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
7
9
|
import { ControlClient, PROTOCOL, RECOVERY_SOURCE_PROTOCOLS, readControl } from "../hub/control-client.ts";
|
|
8
10
|
import { awaitStopped, inspectProject } from "../hub/lifecycle.ts";
|
|
9
11
|
import type { Project } from "../hub/registry.ts";
|
|
10
12
|
import { hubHome } from "../hub/project.ts";
|
|
11
13
|
import { packageDigest, preserveRuntimeDependencies, registryRelease, runCommand, stageRelease, verifyPackage, type RunCommand } from "./recovery-package.ts";
|
|
12
|
-
import { inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
|
|
13
|
-
import { FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
|
|
14
|
+
import { inspectSavedTerminal, inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
|
|
15
|
+
import { ReattachmentRequired, MissingRecordedSession, FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
|
|
14
16
|
import { readEvents } from "../hub/events.ts";
|
|
15
17
|
import { refreshManager } from "../hub/manager.ts";
|
|
16
18
|
import { abandonRestartSnapshot, readRestartSnapshot, waiveRecoveryPeers } from "../hub/restart.ts";
|
|
19
|
+
import { readRecoveryContinuity, retireRecoveryContinuity, writeRecoveryContinuity } from "../hub/recovery-continuity.ts";
|
|
17
20
|
import { recoveryLock } from "../hub/recovery-store.ts";
|
|
21
|
+
import { decodeRestoreReceipt, restoreTransition, type RestoreReceipt, type RestoreEvidence, type RestoreTransition } from "./upgrade-restore-state.ts";
|
|
22
|
+
import { RecoveryPreservationError, RecoveryPresentationError, type RecoveryNativeCause } from "./upgrade-presentation-data.ts";
|
|
18
23
|
|
|
19
24
|
/** An unmanaged Claude's plugin retries with a backoff of at most 30 s; three of those bound the reconnect wait (#206). */
|
|
20
|
-
const
|
|
25
|
+
const UNMANAGED_RECONNECT_WAIT_MS = 90_000;
|
|
26
|
+
/** Managed TUI startup can include resume and MCP handshakes; match createTerminal's ten-minute readiness ceiling. */
|
|
27
|
+
const RECONNECT_WAIT_MS = 600_000;
|
|
28
|
+
class AttachmentLauncherGone extends Error {
|
|
29
|
+
constructor(readonly transition: RestoreTransition) { super("chosen attachment launcher exited"); }
|
|
30
|
+
}
|
|
31
|
+
const attachedBinding = (binding: TerminalBinding, sessionId: string): TerminalBinding => ({ ...binding, sessionId });
|
|
21
32
|
|
|
22
33
|
export const PACKAGE_ROOT = resolve(import.meta.dir, "../..");
|
|
23
34
|
const orcaExecutable = () => process.env.ORCA_CLI_COMMAND || (process.env.ORCA_DEV_REPO_ROOT ? "orca-dev" : process.platform === "linux" && !process.env.ORCA_TERMINAL_HANDLE ? "orca-ide" : "orca");
|
|
@@ -27,11 +38,10 @@ const orcaExecutable = () => process.env.ORCA_CLI_COMMAND || (process.env.ORCA_D
|
|
|
27
38
|
* dir, creating the project directory only with the first persisted turn; the slug replaces
|
|
28
39
|
* every non-alphanumeric character of the project root with '-'.
|
|
29
40
|
*/
|
|
30
|
-
function
|
|
31
|
-
|
|
32
|
-
const slug = binding.projectRoot.replace(/[^a-zA-Z0-9]/g, "-");
|
|
33
|
-
return existsSync(join(config, "projects", slug, `${binding.sessionId}.jsonl`));
|
|
41
|
+
function claudeTranscriptState(binding: TerminalBinding): "found" | "missing" | "unknown" {
|
|
42
|
+
return recordedFileState(checkedSessionPath(binding, "transcript"));
|
|
34
43
|
}
|
|
44
|
+
function claudeTranscriptExists(binding: TerminalBinding): boolean { return claudeTranscriptState(binding) === "found"; }
|
|
35
45
|
|
|
36
46
|
/**
|
|
37
47
|
* #215: `codex resume <id>` reads the thread's rollout from the store its launcher uses (the captured CODEX_HOME, else
|
|
@@ -39,7 +49,7 @@ function claudeTranscriptExists(binding: TerminalBinding): boolean {
|
|
|
39
49
|
* ponytail: a `.jsonl` under sessions/ whose name carries the thread id is the evidence; ask Codex itself once it offers
|
|
40
50
|
* a supported resumability query, or when its store layout changes.
|
|
41
51
|
*/
|
|
42
|
-
const codexSessions = (binding: TerminalBinding) =>
|
|
52
|
+
const codexSessions = (binding: TerminalBinding) => checkedSessionPath(binding, "rollout");
|
|
43
53
|
function codexTranscript(binding: TerminalBinding): "found" | "missing" | "unknown" {
|
|
44
54
|
try { return readdirSync(codexSessions(binding), { recursive: true }).some((name) => String(name).endsWith(".jsonl") && basename(String(name)).includes(binding.sessionId)) ? "found" : "missing"; }
|
|
45
55
|
catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "missing" : "unknown"; } // no store has no rollout; an unreadable one says nothing
|
|
@@ -151,7 +161,7 @@ export async function endPlannedPeer(planned: PlannedProject, peer: RecoveryPeer
|
|
|
151
161
|
}
|
|
152
162
|
}
|
|
153
163
|
|
|
154
|
-
export async function inspectRecovery(project: Project): Promise<Inspection> {
|
|
164
|
+
export async function inspectRecovery(project: Project, targetFeatures?: readonly string[]): Promise<Inspection> {
|
|
155
165
|
// #226: the claim is judged by the registry as it is now, never by the plan's frozen copy, which holds the source's
|
|
156
166
|
// pid from planning time (after a reboot another process may hold it). A registry that cannot be read throws: the
|
|
157
167
|
// caller reads that as not inspected, never as stopped.
|
|
@@ -189,9 +199,9 @@ export async function inspectRecovery(project: Project): Promise<Inspection> {
|
|
|
189
199
|
}
|
|
190
200
|
if (!status || !control) return { state: "unavailable", peers: [], blockers: ["authenticated source status unavailable"] };
|
|
191
201
|
let response: any;
|
|
192
|
-
try { response = await rpc(project, { t: "recovery", op: "inspect", expectedInstanceId: status.instanceId }, sourceProtocol ?? PROTOCOL); }
|
|
202
|
+
try { response = await rpc(project, { t: "recovery", op: "inspect", expectedInstanceId: status.instanceId, ...(targetFeatures === undefined ? {} : { targetFeatures }) }, sourceProtocol ?? PROTOCOL); }
|
|
193
203
|
catch (error) {
|
|
194
|
-
if (isDatabaseBusy(error)) throw error;
|
|
204
|
+
if (isDatabaseBusy(error) || targetFeatures !== undefined) throw error;
|
|
195
205
|
// A stop can complete between authenticated status and the second read. Never infer stopped
|
|
196
206
|
// from a connection failure: the coordinator will inspect the ownership manifest again.
|
|
197
207
|
return { state: "unavailable", instanceId: status.instanceId, version: status.version, protocol: status.protocol,
|
|
@@ -239,6 +249,7 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
|
|
|
239
249
|
const reconnectOnly: string[] = [];
|
|
240
250
|
for (const peer of source.peers) {
|
|
241
251
|
if (peer.state === "offline") continue;
|
|
252
|
+
const adapter = definitionOf(peer.id);
|
|
242
253
|
if (peer.id === "kimi" && peer.args?.mode === "tui") {
|
|
243
254
|
blockers.push("kimi: its pull-only terminal has no automatic restore; end it in that terminal, then make a new recovery plan (its session is not resumed by the hub)");
|
|
244
255
|
continue;
|
|
@@ -275,12 +286,13 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
|
|
|
275
286
|
const session = peer.id === "codex" ? peer.threadId : peer.sessionId;
|
|
276
287
|
if (!session) blockers.push(`${peer.id}: original conversation ID is unknown; manual-required; next action: reconnect the original native session and make a new recovery plan`);
|
|
277
288
|
else sessions[peer.id] = session;
|
|
278
|
-
} else if (
|
|
279
|
-
|
|
289
|
+
} else if (adapter?.session.restoreOnStart || peer.id === "jcode") {
|
|
290
|
+
const session = adapter?.session.crashIdentity ? adapter.session.crashIdentity(peer).sessionId : qwenSessionId(peer.sessionId);
|
|
291
|
+
if (!session) blockers.push(`${peer.id}: saved ACP session identity unavailable; end the peer and make a new recovery plan`);
|
|
280
292
|
} else if (peer.id === "pi" && peer.args?.mode === "tui") {
|
|
281
293
|
if (!peer.sessionId) blockers.push("pi: original session ID is unknown; manual-required; next action: reconnect the original Pi session and make a new recovery plan");
|
|
282
294
|
else sessions.pi = { sessionId: peer.sessionId, ...(peer.sessionFile ? { sessionFile: peer.sessionFile } : {}), ...(peer.args.backend ? { backend: peer.args.backend } : {}), ...(peer.args.model ? { model: peer.args.model } : {}) };
|
|
283
|
-
} else if (peer.id !== "kimi" && peer.id !== "
|
|
295
|
+
} else if (peer.id !== "kimi" && peer.id !== "local" && peer.id !== "pi") blockers.push(`${peer.id}: no automatic recovery adapter`);
|
|
284
296
|
}
|
|
285
297
|
const terminals = await inspectTerminals(project.root, sessions, {
|
|
286
298
|
...terminalOptions(run), stateDir: project.stateDir, instanceId: source.instanceId,
|
|
@@ -352,25 +364,28 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
352
364
|
if (result.code !== 0) throw new Error(`target command ${args[0]} failed; inspect recovery state before retrying`);
|
|
353
365
|
return result;
|
|
354
366
|
};
|
|
355
|
-
const control = (project: Project, op: string, id: string, instance: string) => {
|
|
367
|
+
const control = (project: Project, op: string, id: string, instance: string, targetFeatures?: readonly string[]) => {
|
|
356
368
|
const sourceProtocol = readControl(project.stateDir)?.protocol;
|
|
357
369
|
const protocol = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]) ? sourceProtocol : PROTOCOL;
|
|
358
|
-
return rpc(project, { t: "recovery", op, operationId: id, expectedInstanceId: instance }, protocol).then(() => {});
|
|
370
|
+
return rpc(project, { t: "recovery", op, operationId: id, expectedInstanceId: instance, ...(targetFeatures === undefined ? {} : { targetFeatures }) }, protocol).then(() => {});
|
|
359
371
|
};
|
|
372
|
+
const nativeFailure = (planned: PlannedProject, peer: string, reason: RecoveryNativeCause["reason"], message: string,
|
|
373
|
+
details: Omit<RecoveryNativeCause, "kind" | "reason" | "projectId" | "peer"> = {}) =>
|
|
374
|
+
new RecoveryPresentationError(message, { kind: "native_recovery", reason, projectId: planned.project.id, peer, ...details });
|
|
360
375
|
const revalidateTerminal = async (planned: PlannedProject, progress: ProjectProgress, saved: TerminalBinding, exact: boolean): Promise<TerminalBinding> => {
|
|
361
376
|
const options = { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId };
|
|
362
|
-
const found = await inspectTerminals(planned.project.root, { [saved.peer]: saved.sessionId }, options);
|
|
377
|
+
const found = await inspectTerminals(planned.project.root, { [saved.peer]: { sessionId: saved.sessionId, sessionFile: saved.sessionFile, backend: saved.backend, model: saved.model } }, options);
|
|
363
378
|
const current = found.byPeer[saved.peer];
|
|
364
379
|
if (found.manualRequired || !current || (exact && (current.handle !== saved.handle || current.incarnationId !== saved.incarnationId || current.worktreeId !== saved.worktreeId || current.projectRoot !== saved.projectRoot))) {
|
|
365
|
-
throw
|
|
380
|
+
throw nativeFailure(planned, saved.peer, "terminal_unverified", `${saved.peer}: saved terminal identity changed or is not verified; manual-required`, { handle: saved.handle });
|
|
366
381
|
}
|
|
367
382
|
const idle = await waitForIdle(exact ? saved : current, 120_000, options);
|
|
368
|
-
if (!idle.satisfied) throw
|
|
383
|
+
if (!idle.satisfied) throw nativeFailure(planned, saved.peer, "terminal_unverified", `${saved.peer}: terminal is not idle; manual-required`, { handle: saved.handle });
|
|
369
384
|
const observed = await inspectRecovery(planned.project);
|
|
370
|
-
if (observed.instanceId !== progress.instanceId) throw
|
|
385
|
+
if (observed.instanceId !== progress.instanceId) throw nativeFailure(planned, saved.peer, "target_unavailable", `${saved.peer}: daemon instance changed during terminal revalidation`, { handle: saved.handle, targetState: observed.state });
|
|
371
386
|
const peer = observed.peers.find((item) => item.id === saved.peer);
|
|
372
387
|
const session = saved.peer === "codex" ? peer?.threadId : peer?.sessionId;
|
|
373
|
-
if (session !== saved.sessionId) throw
|
|
388
|
+
if (session !== saved.sessionId || saved.peer === "pi" && progress.reattachments?.pi && saved.sessionFile && peer?.sessionFile !== saved.sessionFile) throw nativeFailure(planned, saved.peer, "session_changed", `${saved.peer}: daemon session changed during terminal revalidation; manual-required`, { handle: saved.handle, expectedSession: saved.sessionId, ...(session === undefined ? {} : { observedSession: session }) });
|
|
374
389
|
return current;
|
|
375
390
|
};
|
|
376
391
|
/** Whether Orca's inventory lists the terminal; a failed, not-ok, truncated or non-JSON answer shows nothing. */
|
|
@@ -387,7 +402,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
387
402
|
* expected instance can report "gone"; an unreadable target or launcher is unknown and blocks, never gone.
|
|
388
403
|
*/
|
|
389
404
|
const peerEvidence = async (planned: PlannedProject, progress: ProjectProgress, peer: TerminalBinding["peer"]):
|
|
390
|
-
Promise<{ state: "live"; session?: string; handle?: string; attached?: true } | { state: "gone" } | { state: "unknown"; why: string; step?: string }> => {
|
|
405
|
+
Promise<{ state: "live"; session?: string; handle?: string; attached?: true; terminal?: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId"> } | { state: "gone"; terminal?: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId"> } | { state: "unknown"; why: string; step?: string; idless?: true }> => {
|
|
391
406
|
const target = await inspectRecovery(planned.project);
|
|
392
407
|
if (target.state !== "running" || target.instanceId !== progress.instanceId) {
|
|
393
408
|
// Only a hub that may still answer is waited for; a stopped, missing or other one is settled by the runner's list.
|
|
@@ -397,22 +412,56 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
397
412
|
const attached = target.peers.find((p) => p.id === peer && p.state !== "offline");
|
|
398
413
|
const session = peer === "codex" ? attached?.threadId : attached?.sessionId;
|
|
399
414
|
if (session) return { state: "live", session };
|
|
415
|
+
let endedTerminal: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId"> | undefined;
|
|
400
416
|
// #225: the launch records of every instance this operation started for the project, the current one first: a
|
|
401
417
|
// launcher a dead instance recorded may still run (read before the check below, so a block names its terminal).
|
|
402
418
|
for (const instanceId of [progress.instanceId, ...(progress.restarts ?? []).map((r) => r.instanceId)]) {
|
|
403
419
|
if (!instanceId) continue;
|
|
404
420
|
const launch = await launcherOf(peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId });
|
|
405
|
-
if (launch?.state === "live")
|
|
421
|
+
if (launch?.state === "live") {
|
|
422
|
+
const record = launch.record!;
|
|
423
|
+
return { state: "live", handle: record.handle, ...(attached ? { attached: true as const } : {}),
|
|
424
|
+
...(instanceId === progress.instanceId ? { terminal: { handle: record.handle, incarnationId: record.incarnationId, worktreeId: record.worktreeId } } : {}) };
|
|
425
|
+
}
|
|
406
426
|
if (launch?.state === "unknown") {
|
|
407
427
|
return launch.record ? { state: "unknown", why: `its launcher in terminal ${launch.record.handle} cannot be read`, step: `wait until it attaches, or end it and close terminal ${launch.record.handle}` }
|
|
408
428
|
: launch.invalidRow ? { state: "unknown", why: "a launch record that may be its launcher's cannot be evaluated", step: `inspect ${recordPath(planned.project.stateDir)} and fix that row, or end that launcher and remove its row` }
|
|
409
429
|
: { state: "unknown", why: "the launcher records cannot be read", step: `inspect ${recordPath(planned.project.stateDir)} and move it aside (sessions it recorded then count as unmanaged)` };
|
|
410
430
|
}
|
|
431
|
+
if (instanceId === progress.instanceId && launch?.state === "gone" && launch.record) {
|
|
432
|
+
const record = launch.record;
|
|
433
|
+
endedTerminal = { handle: record.handle, incarnationId: record.incarnationId, worktreeId: record.worktreeId };
|
|
434
|
+
}
|
|
411
435
|
}
|
|
412
436
|
// #225: attached with no provable session id (a Claude of a dead instance whose plugin reconnected, or one that has
|
|
413
437
|
// not reported its id yet) is a session of unknown identity, never "nothing attached".
|
|
414
|
-
if (attached) return { state: "unknown", why: `a ${peer} session is attached to the target without a session id`, step: `wait until it reports one, or end that ${peer} session` };
|
|
415
|
-
return { state: "gone" };
|
|
438
|
+
if (attached) return { state: "unknown", idless: true, why: `a ${peer} session is attached to the target without a session id`, step: `wait until it reports one, or end that ${peer} session` };
|
|
439
|
+
return { state: "gone", ...(endedTerminal ? { terminal: endedTerminal } : {}) };
|
|
440
|
+
};
|
|
441
|
+
/** Only independently confirmed terminal loss plus absent sessions and launchers can arm a repair. */
|
|
442
|
+
const detached = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation,
|
|
443
|
+
binding: TerminalBinding, save?: () => void): Promise<void> => {
|
|
444
|
+
const evidence = await peerEvidence(planned, progress, binding.peer);
|
|
445
|
+
if (evidence.state !== "gone") return;
|
|
446
|
+
const terminal = await inspectSavedTerminal(binding, terminalOptions(run));
|
|
447
|
+
const decision = restoreTransition({ state: "done", binding }, {
|
|
448
|
+
target: "running", launcher: "gone", session: "none", allowOther: false,
|
|
449
|
+
terminal: terminal.state === "missing" || terminal.state === "orphaned" ? "gone" : terminal.state === "present" ? "present" : terminal.state === "changed" ? "changed" : "unknown",
|
|
450
|
+
});
|
|
451
|
+
if (decision.action !== "reattach") return;
|
|
452
|
+
const current = await inspectRecovery(planned.project);
|
|
453
|
+
if (current.state !== "running" || current.instanceId !== progress.instanceId || current.version !== op.plan.version || current.recovery?.operationId !== op.id ||
|
|
454
|
+
(await peerEvidence(planned, progress, binding.peer)).state !== "gone") return;
|
|
455
|
+
const existing = progress.reattachments?.[binding.peer];
|
|
456
|
+
const why = `${binding.peer}: detached after restore; saved terminal ${binding.handle} is ${terminal.state}; recorded session ${binding.sessionId}${binding.sessionFile ? ` (${binding.sessionFile})` : ""}`;
|
|
457
|
+
if (existing) throw new ReattachmentRequired(planned.project.id, binding.peer, `${why}; the one reattachment was already attempted; no second terminal will be created`);
|
|
458
|
+
if (!save) throw new Error(`${why}; a durable coordinator save is required before reattachment`);
|
|
459
|
+
(progress.reattachments ??= {})[binding.peer] = { state: "ready", at: now(), cause: "detached after restore", binding };
|
|
460
|
+
(progress.repairedSessions ??= {})[binding.peer] = true;
|
|
461
|
+
progress.terminals[`restored:${binding.peer}`] = decision.next;
|
|
462
|
+
progress.phase = "started";
|
|
463
|
+
save();
|
|
464
|
+
throw new ReattachmentRequired(planned.project.id, binding.peer, `${why}; resume or use the operation's person-only reattach command`);
|
|
416
465
|
};
|
|
417
466
|
/**
|
|
418
467
|
* #225: the terminal of a done restoration of a target instance that died is closed under the close receipt rules
|
|
@@ -436,24 +485,66 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
436
485
|
progress.terminals[key] = true; save();
|
|
437
486
|
};
|
|
438
487
|
/** #414: an idle terminal is not an authenticated peer. Keep its receipt while waiting, including on resume. */
|
|
439
|
-
const awaitAttachment = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation,
|
|
440
|
-
const
|
|
488
|
+
const awaitAttachment = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, waiting: Extract<RestoreReceipt, { state: "waiting" }>, save: () => void, expectedSession: string | null = waiting.binding.sessionId): Promise<string> => {
|
|
489
|
+
const saveTiming = () => {
|
|
490
|
+
try { save(); }
|
|
491
|
+
catch { console.warn("Recovery timing observation could not be saved; recovery continues"); }
|
|
492
|
+
};
|
|
493
|
+
const binding = waiting.binding;
|
|
494
|
+
const matchesTerminal = (terminal: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId">) => terminal.handle === binding.handle && terminal.incarnationId === binding.incarnationId && terminal.worktreeId === binding.worktreeId;
|
|
495
|
+
const startedAt = now(), deadline = startedAt + RECONNECT_WAIT_MS;
|
|
496
|
+
const waitStep = op.step;
|
|
497
|
+
if (waitStep) {
|
|
498
|
+
(op.waits ??= {})[waitStep] = { waitStartedAt: new Date(startedAt).toISOString(), deadlineAt: new Date(deadline).toISOString() };
|
|
499
|
+
saveTiming();
|
|
500
|
+
}
|
|
501
|
+
let waitFailed = false;
|
|
502
|
+
try {
|
|
441
503
|
let live: Inspection;
|
|
442
504
|
for (;;) {
|
|
443
505
|
live = await inspectRecovery(planned.project);
|
|
444
|
-
if (live.state
|
|
445
|
-
|
|
506
|
+
if (live.state !== "running") throw nativeFailure(planned, binding.peer, "target_unavailable", `${binding.peer}: target hub reads as ${live.state} while awaiting attachment; saved terminal retained`, { handle: binding.handle, targetState: live.state });
|
|
507
|
+
if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw nativeFailure(planned, binding.peer, "target_unavailable", `${binding.peer}: target identity changed while awaiting attachment`, { handle: binding.handle, targetState: live.state });
|
|
508
|
+
const peer = live.peers.find((p) => p.id === binding.peer && p.state !== "offline");
|
|
446
509
|
const session = binding.peer === "codex" ? peer?.threadId : peer?.sessionId;
|
|
447
510
|
const changed = !!session && expectedSession !== null && session !== expectedSession;
|
|
448
511
|
// #64: only the original zero-turn Claude may change identity without an operator choice. Re-read persistence
|
|
449
512
|
// at attachment, not at the plan or wait start; an already accepted replacement is bound to its own id.
|
|
450
|
-
const zeroTurn = changed && binding.peer === "claude" && (planned.terminals as TerminalBinding[]).some((p) => p.peer === "claude" && p.sessionId === binding.sessionId) && !claudeTranscriptExists(binding);
|
|
451
|
-
if (changed && !zeroTurn) throw
|
|
513
|
+
const zeroTurn = changed && !progress.reattachments?.[binding.peer] && !progress.repairedSessions?.[binding.peer] && binding.peer === "claude" && (planned.terminals as TerminalBinding[]).some((p) => p.peer === "claude" && p.sessionId === binding.sessionId) && !claudeTranscriptExists(binding);
|
|
514
|
+
if (changed && !zeroTurn) throw nativeFailure(planned, binding.peer, "session_changed", `${binding.peer}: a different conversation attached while awaiting ${binding.sessionId}`, { handle: binding.handle, expectedSession: expectedSession!, observedSession: session! });
|
|
452
515
|
if (now() > deadline) break;
|
|
453
|
-
|
|
516
|
+
const launcher = await launcherOf(binding.peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId });
|
|
517
|
+
const sameTerminal = launcher?.record && matchesTerminal(launcher.record);
|
|
518
|
+
let launchState: RestoreEvidence["launcher"] = !launcher ? "no-record-yet" : launcher.state === "unknown" || !launcher.record ? "unknown"
|
|
519
|
+
: sameTerminal ? launcher.state : launcher.state === "gone" ? "no-record-yet" : "replaced";
|
|
520
|
+
let olderHandle: string | undefined;
|
|
521
|
+
if ((launchState === "gone" || launchState === "no-record-yet") && !session) {
|
|
522
|
+
// An exact exit is not proof while an older launcher or an id-less session may still run.
|
|
523
|
+
const evidence = await peerEvidence(planned, progress, binding.peer);
|
|
524
|
+
if (evidence.state === "gone" && evidence.terminal && matchesTerminal(evidence.terminal)) launchState = "gone";
|
|
525
|
+
else if (evidence.state === "unknown") {
|
|
526
|
+
if (!(launchState === "no-record-yet" && evidence.idless)) launchState = "unknown";
|
|
527
|
+
} else if (evidence.state === "live" && !evidence.session) {
|
|
528
|
+
if (evidence.terminal) launchState = matchesTerminal(evidence.terminal) ? "live" : "replaced";
|
|
529
|
+
else { launchState = "older-instance"; olderHandle = evidence.handle; }
|
|
530
|
+
} else if (evidence.state === "live" && evidence.session) {
|
|
531
|
+
if (now() >= deadline) break;
|
|
532
|
+
await sleep(Math.min(1000, deadline - now()));
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
const decision = restoreTransition(waiting, { target: "running", launcher: launchState,
|
|
537
|
+
session: session && peer && ["idle", "busy", "paused"].includes(peer.state) ? changed ? "other" : "planned" : "none", allowOther: zeroTurn || expectedSession === null, binding });
|
|
538
|
+
if (decision.action === "settle" && session) {
|
|
454
539
|
if (zeroTurn) waiveRecoveryPeers(planned.project.stateDir, op.id, { claude: "zero-turn" });
|
|
455
540
|
return session;
|
|
456
541
|
}
|
|
542
|
+
if (decision.action === "fail" || decision.action === "create") throw new AttachmentLauncherGone(decision);
|
|
543
|
+
if (decision.action === "block") {
|
|
544
|
+
if (launchState === "replaced") throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: saved terminal identity changed while awaiting attachment`, { handle: binding.handle, launcherState: launchState });
|
|
545
|
+
if (launchState === "older-instance") throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: an older target's launcher in terminal ${olderHandle ?? "unknown"} may still run; saved terminal retained; no terminal was created`, { ...(olderHandle ? { handle: olderHandle } : {}), launcherState: launchState });
|
|
546
|
+
throw nativeFailure(planned, binding.peer, "launcher_unverified", `${binding.peer}: the chosen restoration launcher's identity cannot be verified (${launchState}); saved terminal retained; no terminal was created`, { handle: binding.handle, launcherState: launchState });
|
|
547
|
+
}
|
|
457
548
|
if (now() >= deadline) break;
|
|
458
549
|
await sleep(Math.min(1000, deadline - now()));
|
|
459
550
|
}
|
|
@@ -468,15 +559,28 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
468
559
|
}
|
|
469
560
|
} catch { /* unavailable metadata stays unknown; raw output/errors never enter the receipt */ }
|
|
470
561
|
const peer = live.peers.find((p) => p.id === binding.peer);
|
|
471
|
-
const session = binding.peer === "codex" ? peer?.threadId : peer?.sessionId;
|
|
562
|
+
const session = peer?.state === "offline" ? undefined : binding.peer === "codex" ? peer?.threadId : peer?.sessionId;
|
|
472
563
|
(progress.attachmentTimeouts ??= {})[binding.peer] = { at: now(), terminal,
|
|
473
564
|
hub: { state: live.state, ...(live.instanceId ? { instanceId: live.instanceId } : {}), ...(peer ? { peerState: peer.state } : {}), ...(session ? { sessionId: session } : {}) },
|
|
474
565
|
lastHubConnectAttempt: "unknown" };
|
|
475
566
|
save();
|
|
476
|
-
throw
|
|
567
|
+
throw nativeFailure(planned, binding.peer, "attachment_timeout", `${binding.peer}: attachment not verified within ${RECONNECT_WAIT_MS / 1000} s; terminal ${binding.handle} retained; resume after it attaches (native-start diagnostic recorded; last hub connect attempt unknown)`, { handle: binding.handle, timeoutMs: RECONNECT_WAIT_MS });
|
|
568
|
+
} catch (error) { waitFailed = true; throw error; }
|
|
569
|
+
finally {
|
|
570
|
+
const finish = () => {
|
|
571
|
+
if (!waitStep) return;
|
|
572
|
+
op.waits![waitStep] = { ...op.waits![waitStep]!, waitEndedAt: new Date(now()).toISOString() };
|
|
573
|
+
saveTiming();
|
|
574
|
+
};
|
|
575
|
+
// Metadata storage must not hide the attachment failure or change its recovery outcome.
|
|
576
|
+
if (waitFailed) { try { finish(); } catch { /* original attachment failure remains authoritative */ } }
|
|
577
|
+
else finish();
|
|
578
|
+
}
|
|
477
579
|
};
|
|
478
580
|
return {
|
|
581
|
+
...(liveTestPeer === undefined ? {} : { liveTestPeer }),
|
|
479
582
|
now, sleep, inspect: inspectRecovery,
|
|
583
|
+
recordedSessionState: async (binding, kind) => kind === "rollout" ? codexTranscript(binding) : claudeTranscriptState(binding),
|
|
480
584
|
stage: async (op) => {
|
|
481
585
|
if (packageDigest(op.sourceRoot, { ownDependencies: true }) !== op.plan.sourceDigest) throw new FinalRefusal("preserved source changed");
|
|
482
586
|
const target = op.plan.kind === "restart" ? { root: op.sourceRoot, digest: verifyPackage(op.sourceRoot, op.plan.version, { ownDependencies: true }) }
|
|
@@ -485,16 +589,24 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
485
589
|
if (liveTestPeer && (!existsSync(identityPath) || !readFileSync(identityPath, "utf8").includes("export const LIVE_TEST_SUPPORTED = true;"))) {
|
|
486
590
|
throw new FinalRefusal("target does not support live-test delegation; choose a supporting target before retrying; runtimes unchanged");
|
|
487
591
|
}
|
|
488
|
-
//
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
//
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
if (
|
|
592
|
+
// Reuse capability proof only after verifying the cached root and digest still identify these exact bytes.
|
|
593
|
+
const cached = op.targetDigest === target.digest && op.targetRoot === target.root;
|
|
594
|
+
if (cached && op.targetFeatures !== undefined) return { ...target, targetFeatures: [...op.targetFeatures] };
|
|
595
|
+
// A legacy cached operation already past commit needs no capability proof to finish restoration (#215).
|
|
596
|
+
// A prepared commitSent receipt remains uncertain and must still obtain proof before a possible commit.
|
|
597
|
+
if (cached && op.projects?.every(project => ["stopped", "started", "peers-restored", "verified"].includes(project.phase))) return target;
|
|
598
|
+
// Legacy uncommitted receipts acquire verified feature evidence before source compatibility checks.
|
|
599
|
+
const probe = await run([process.execPath, "-e", `import * as target from ${JSON.stringify(join(target.root, "src/hub/control-client.ts"))}; console.log(JSON.stringify({protocol: target.PROTOCOL, features: target.DAEMON_FEATURES ?? []}))`]);
|
|
600
|
+
let metadata: { protocol?: unknown; features?: unknown } | undefined;
|
|
601
|
+
try { if (probe.code === 0) metadata = JSON.parse(probe.stdout); } catch { /* Unknown metadata refuses before effects. */ }
|
|
602
|
+
if (!metadata || typeof metadata !== "object" || !Number.isInteger(metadata.protocol)) throw new Error(`the staged target's control protocol could not be read (probe exit ${probe.code}); staged package retained, runtimes unchanged; resume again once it can run`);
|
|
603
|
+
if (!Array.isArray(metadata.features) || !metadata.features.every((feature: unknown) => typeof feature === "string" && feature.length > 0)) throw new Error("the staged target features could not be read; staged package retained, runtimes unchanged");
|
|
604
|
+
const targetFeatures: readonly string[] = [...metadata.features];
|
|
605
|
+
if (metadata.protocol !== PROTOCOL) throw new FinalRefusal("target protocol requires a newer coordinator; staged package retained, runtimes unchanged");
|
|
606
|
+
const verified = { ...target, targetFeatures };
|
|
607
|
+
// Existing peer/waiver compatibility checks remain properties of the already verified bytes.
|
|
608
|
+
if (op.targetDigest && op.targetDigest === target.digest && op.targetRoot === target.root) return verified;
|
|
609
|
+
if (op.plan.projects.some(p => p.source.peers.some(peer => definitionOf(peer.id)?.session.restoreOnStart && peer.state !== "offline"))) {
|
|
498
610
|
const catalog = join(target.root, "src/hub/peer-catalog.ts");
|
|
499
611
|
const peers = existsSync(catalog) ? /export const OPTIONAL_PEERS\s*=\s*\[([^\]]*)\]/.exec(readFileSync(catalog, "utf8"))?.[1] : undefined;
|
|
500
612
|
if (!peers || !/["']qwen["']/.test(peers)) throw new FinalRefusal("target does not support Qwen; end Qwen and make a new plan, or choose a supporting target; runtimes unchanged");
|
|
@@ -508,12 +620,20 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
508
620
|
if (op.plan.projects.some((p) => p.reconnectOnly?.length || p.freshStart?.length) && !targetReadsWaivers({ targetRoot: target.root })) {
|
|
509
621
|
throw new FinalRefusal(`target ${op.plan.version} predates recovery waivers, so its hub could never accept a reconnect-only Claude or a fresh Codex start; staged package retained, runtimes unchanged; choose a newer target once this operation is cancelled or ended`);
|
|
510
622
|
}
|
|
511
|
-
return
|
|
623
|
+
return verified;
|
|
512
624
|
},
|
|
513
|
-
prepare: (p, id, instance) => control(p, "prepare", id, instance),
|
|
625
|
+
prepare: (p, id, instance, features) => control(p, "prepare", id, instance, features),
|
|
514
626
|
abort: (p, id, instance) => control(p, "abort", id, instance),
|
|
515
|
-
commit: (p, id, instance) => control(p, "commit", id, instance),
|
|
516
|
-
release: (p, id, instance) =>
|
|
627
|
+
commit: (p, id, instance, features) => control(p, "commit", id, instance, features),
|
|
628
|
+
release: async (p, id, instance) => {
|
|
629
|
+
// Snapshot before the request: a later writer must never be retired by this older release completion.
|
|
630
|
+
const previous = readRecoveryContinuity(p.stateDir, { projectId: p.id, projectRoot: p.root });
|
|
631
|
+
await control(p, "release", id, instance);
|
|
632
|
+
if (previous.state === "ready" && previous.bookmark.operationId !== id) {
|
|
633
|
+
try { retireRecoveryContinuity(p.stateDir, { projectId: p.id, projectRoot: p.root, operationId: previous.bookmark.operationId, digest: previous.digest }, "recovery_released"); }
|
|
634
|
+
catch { console.error("Pi continuity retirement is unconfirmed after successful recovery release; inspect the retained bookmark before another automatic start"); }
|
|
635
|
+
}
|
|
636
|
+
},
|
|
517
637
|
closeTerminals: async (planned, progress, op, save) => {
|
|
518
638
|
for (const binding of planned.terminals as TerminalBinding[]) {
|
|
519
639
|
const key = `closed:${binding.peer}`;
|
|
@@ -563,10 +683,10 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
563
683
|
if (reconnect.length) waiveRecoveryPeers(p.stateDir, op.id, Object.fromEntries(reconnect.map((peer) => [peer, "reconnect-only"])));
|
|
564
684
|
await command(op, ["up"], p);
|
|
565
685
|
},
|
|
566
|
-
restore: async (planned, progress, op, group, save) => {
|
|
686
|
+
restore: async (planned, progress, op, group, save, onlyPeer) => {
|
|
567
687
|
const live = await inspectRecovery(planned.project);
|
|
568
688
|
if (live.instanceId !== progress.instanceId) throw new Error("target daemon changed before peer restore");
|
|
569
|
-
if (group === "native") {
|
|
689
|
+
if (group === "native" && !onlyPeer) {
|
|
570
690
|
for (const peer of planned.source.peers.filter((p) => ["kimi", "qwen", "jcode", "local", "pi"].includes(p.id) && p.state !== "offline" && !(p.id === "pi" && p.args?.mode === "tui"))) {
|
|
571
691
|
const current = live.peers.find((p) => p.id === peer.id);
|
|
572
692
|
if (current && current.state !== "offline") continue;
|
|
@@ -583,7 +703,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
583
703
|
// #269: `ahub pi` without a mode now takes Pi's start mode (tui by default). This Pi ran headless (a terminal
|
|
584
704
|
// one is filtered out above), so a snapshot that recorded no mode still restores it headless.
|
|
585
705
|
if (peer.id === "pi" && typeof peer.args?.mode !== "string") args.push("--mode", "headless");
|
|
586
|
-
if (peer.id
|
|
706
|
+
if (definitionOf(peer.id)?.session.restoreOnStart && peer.sessionId) args.push("--session-id", peer.sessionId);
|
|
587
707
|
if (peer.id === "jcode" && peer.sessionId) args.push("--session-id", peer.sessionId);
|
|
588
708
|
if (peer.id === "jcode" && peer.args?.providerProfile) args.push("--provider-profile", peer.args.providerProfile);
|
|
589
709
|
if (peer.id === "pi") {
|
|
@@ -594,130 +714,221 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
594
714
|
await command(op, args, planned.project);
|
|
595
715
|
}
|
|
596
716
|
}
|
|
717
|
+
let claudeGroup = false;
|
|
597
718
|
for (const plannedBinding of planned.terminals as TerminalBinding[]) {
|
|
598
|
-
if ((plannedBinding.peer === "claude") !== (group === "claude")) continue;
|
|
719
|
+
if ((plannedBinding.peer === "claude") !== (claudeGroup = group === "claude") || onlyPeer && plannedBinding.peer !== onlyPeer) continue;
|
|
599
720
|
const key = `restored:${plannedBinding.peer}`;
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
721
|
+
try {
|
|
722
|
+
// #225: after a restart, the stopped target's terminal for this peer goes first; a session it had accepted
|
|
723
|
+
// instead of the planned one (fresh choice, fresh start, zero-turn Claude) is relaunched by that id, which the
|
|
724
|
+
// operation-fenced waiver already covers.
|
|
725
|
+
await closeRetired(progress, plannedBinding.peer, save);
|
|
726
|
+
const retired = progress.terminals[`retired:${plannedBinding.peer}`] as TerminalBinding | undefined;
|
|
727
|
+
const accepted = retired && typeof retired === "object" && retired.sessionId !== plannedBinding.sessionId ? retired.sessionId : undefined;
|
|
728
|
+
const repair = progress.reattachments?.[plannedBinding.peer];
|
|
729
|
+
const recordedOnly = !!repair || progress.repairedSessions?.[plannedBinding.peer] === true && !progress.fresh?.[plannedBinding.peer]?.repaired;
|
|
730
|
+
const retiredReceipt = recordedOnly && retired !== undefined ? decodeRestoreReceipt({ state: "done", binding: retired }) : undefined;
|
|
731
|
+
const recordedBinding = retiredReceipt?.state === "done" && retiredReceipt.binding.peer === plannedBinding.peer && retiredReceipt.binding.projectRoot === planned.project.root ? retiredReceipt.binding : undefined;
|
|
732
|
+
const original = repair?.binding ?? recordedBinding ?? (accepted ? { ...plannedBinding, sessionId: accepted } : plannedBinding);
|
|
733
|
+
// A new session is accepted only by the operator's choice (#215 dispose --fresh-session) or a planned fresh
|
|
734
|
+
// start of a Codex thread with nothing to lose that still has no rollout; an unreadable store is neither.
|
|
735
|
+
const chosen = !recordedOnly && !!progress.fresh?.[original.peer];
|
|
736
|
+
let read: ReturnType<typeof codexTranscript> | undefined;
|
|
737
|
+
const transcript = () => original.peer !== "codex" || (chosen && !accepted) ? "found" : (read ??= codexTranscript(original));
|
|
738
|
+
// An accepted session starts new again only when it has nothing to resume: a Codex thread that never got a
|
|
739
|
+
// rollout, or a Claude session with no transcript (`--resume` could never succeed, #21).
|
|
740
|
+
const fresh = () => recordedOnly ? false : accepted
|
|
741
|
+
? (original.peer === "codex" ? (chosen || !!planned.freshStart?.includes("codex")) && transcript() === "missing" : original.peer === "claude" && !claudeTranscriptExists(original))
|
|
742
|
+
: chosen || (!!planned.freshStart?.includes(original.peer) && transcript() === "missing");
|
|
743
|
+
// The runner ends this with the choices `status` shows (nextActions); resume comes first and launches a failed
|
|
744
|
+
// peer again, which helps once its cause (a transient Orca create failure, a fixed store) is gone.
|
|
745
|
+
// The waiver's audit label: the operator's choice, a Claude with nothing to resume, or the plan's Codex fresh start.
|
|
746
|
+
const waiver = () => progress.fresh?.[original.peer] ? "fresh-session" : original.peer === "claude" ? "zero-turn" : "fresh-start";
|
|
747
|
+
const notRestored = (why: string) => new Error(fresh()
|
|
748
|
+
? `${original.peer}: ${why}; read that terminal in Orca for the launcher's error and fix its cause`
|
|
749
|
+
: `${original.peer}: ${why}; session ${original.sessionId} was not restored; resume launches it again once the cause is fixed`);
|
|
750
|
+
// Anything live or unknown blocks a create and keeps the receipt: never decide on evidence that could not be read.
|
|
751
|
+
const blocked = (evidence: Exclude<Awaited<ReturnType<typeof peerEvidence>>, { state: "gone" }>) => evidence.state === "unknown"
|
|
752
|
+
? new Error(`${original.peer}: whether a ${original.peer} session or launcher is live cannot be told (${evidence.why}); nothing was recorded or created${evidence.step ? `; ${evidence.step} first` : ""}`)
|
|
753
|
+
: nativeFailure(planned, original.peer, evidence.session ? "session_changed" : "launcher_unverified", evidence.session
|
|
754
|
+
? `${original.peer}: session ${evidence.session} is attached instead of ${original.sessionId}; no terminal was created; end that ${original.peer} session and close its terminal first`
|
|
755
|
+
: `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session ${evidence.attached ? "with an id " : ""}attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`,
|
|
756
|
+
evidence.session
|
|
757
|
+
? { expectedSession: original.sessionId, observedSession: evidence.session }
|
|
758
|
+
: { ...(evidence.handle ? { handle: evidence.handle } : {}), launcherState: evidence.terminal ? "live" : "older-instance" });
|
|
759
|
+
const decide = (receipt: RestoreReceipt, evidence: Awaited<ReturnType<typeof peerEvidence>>) => restoreTransition(receipt, {
|
|
760
|
+
target: "running", launcher: evidence.state === "unknown" ? "unknown" : evidence.state === "gone" ? "gone" : evidence.terminal ? "live" : "older-instance",
|
|
761
|
+
session: evidence.state === "live" && evidence.session ? evidence.session === original.sessionId ? "planned" : "other" : "none",
|
|
762
|
+
allowOther: fresh() || (!recordedOnly && original.peer === "claude" && !claudeTranscriptExists(original)),
|
|
763
|
+
binding: evidence.state === "live" && evidence.terminal ? { ...original, ...evidence.terminal } : original,
|
|
764
|
+
});
|
|
765
|
+
const receipt = decodeRestoreReceipt(progress.terminals[key]);
|
|
766
|
+
function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }> & { origin: "pending" }, expected: string | null): Promise<string>;
|
|
767
|
+
function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined>;
|
|
768
|
+
async function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined> {
|
|
769
|
+
try { return await awaitAttachment(planned, progress, op, waiting, save, expected); }
|
|
770
|
+
catch (error) {
|
|
771
|
+
if (!(error instanceof AttachmentLauncherGone)) throw error;
|
|
772
|
+
progress.terminals[key] = error.transition.next; save();
|
|
773
|
+
// The table decides whether this failed wait blocks or is eligible for a new launch.
|
|
774
|
+
if (error.transition.action === "create") return undefined;
|
|
775
|
+
throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
// #215: the pending row of the receipt table, used wherever a pending launch meets an attached session.
|
|
779
|
+
const settle = async (attached: string, recovered?: TerminalBinding) => {
|
|
780
|
+
// #21: a Claude session that never got a first turn has no transcript, so `claude --resume S` can never
|
|
781
|
+
// succeed and accepting its fresh attach loses nothing. Otherwise only the planned id, or a new session
|
|
782
|
+
// the operator or the plan accepted, counts as restored.
|
|
783
|
+
if (attached !== original.sessionId && !fresh() && !(!recordedOnly && original.peer === "claude" && !claudeTranscriptExists(original))) {
|
|
784
|
+
throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${attached} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
|
|
785
|
+
}
|
|
786
|
+
// Validate the proposed transition before committing continuity/waiver effects or a done receipt.
|
|
787
|
+
const verified = await revalidateTerminal(planned, progress, attachedBinding(recovered ?? original, attached), !!recovered);
|
|
788
|
+
// The planned session came back after all: a fresh session chosen for it lost nothing. Compared with the plan,
|
|
789
|
+
// never with a session accepted after a restart (#225), whose choice must stay on record.
|
|
790
|
+
if (attached === plannedBinding.sessionId && progress.fresh?.[original.peer]?.lost === plannedBinding.sessionId) {
|
|
791
|
+
delete progress.fresh[original.peer];
|
|
792
|
+
if (!Object.keys(progress.fresh).length) delete progress.fresh;
|
|
793
|
+
}
|
|
794
|
+
// Any other accepted session needs the target to waive the saved id, or release would wait for it forever;
|
|
795
|
+
// a launch writes the waiver first, but this must not rely on that.
|
|
796
|
+
// The planned session needs no waiver: the target's snapshot already expects it.
|
|
797
|
+
if (attached !== original.sessionId && attached !== plannedBinding.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
|
|
798
|
+
if (repair) repair.state = "attempted"; // a verified spontaneous reattachment also closes this repair allowance
|
|
799
|
+
// The person accepted one loss, not a permanent exemption for later repairs of the replacement.
|
|
800
|
+
if (progress.fresh?.[original.peer]?.repaired) delete progress.fresh[original.peer]!.repaired;
|
|
801
|
+
progress.terminals[key] = { state: "done", binding: verified } satisfies RestoreReceipt; save();
|
|
802
|
+
};
|
|
803
|
+
if (receipt?.state === "done") {
|
|
804
|
+
// A completed restoration is never a waiting launch. A closed terminal blocks exact revalidation;
|
|
805
|
+
// it cannot erase continuity or create a fresh-session choice (PROBE-B).
|
|
806
|
+
await detached(planned, progress, op, receipt.binding, save);
|
|
807
|
+
const observed = await peerEvidence(planned, progress, original.peer);
|
|
808
|
+
const decision = restoreTransition(receipt, { target: "running", launcher: "no-record-yet", session: observed.state === "live" && observed.session === receipt.binding.sessionId ? "planned" : observed.state === "live" && observed.session ? "other" : "none", allowOther: false });
|
|
809
|
+
if (decision.action === "block") {
|
|
810
|
+
const condition = observed.state === "unknown" ? `cannot be verified (${observed.why})`
|
|
811
|
+
: observed.state === "live" && observed.session ? `session ${observed.session} is attached instead of ${receipt.binding.sessionId}`
|
|
812
|
+
: observed.state === "live" ? `is not attached; its launcher in terminal ${observed.handle ?? "unknown"} may still run` : "is not attached";
|
|
813
|
+
throw new Error(`${original.peer}: completed restoration ${condition}; saved done receipt retained; resume its recorded conversation`);
|
|
814
|
+
}
|
|
815
|
+
progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, receipt.binding, true) } satisfies RestoreReceipt; save();
|
|
816
|
+
continue;
|
|
645
817
|
}
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
if (!Object.keys(progress.fresh).length) delete progress.fresh;
|
|
818
|
+
if (receipt?.state === "waiting") {
|
|
819
|
+
const binding = receipt.binding;
|
|
820
|
+
const sessionId = await awaitBound(receipt, fresh() && binding.sessionId === original.sessionId ? null : binding.sessionId);
|
|
821
|
+
if (sessionId !== undefined) { await settle(sessionId, binding); continue; }
|
|
651
822
|
}
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
823
|
+
if (receipt?.state === "pending" || receipt?.state === "failed") {
|
|
824
|
+
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
825
|
+
const decision = decide(receipt, evidence);
|
|
826
|
+
if (decision.action === "settle" && evidence.state === "live" && evidence.session) { await settle(evidence.session); continue; }
|
|
827
|
+
if (decision.action === "poll" && decision.next.state === "waiting") {
|
|
828
|
+
const waiting = decision.next;
|
|
829
|
+
progress.terminals[key] = waiting; save();
|
|
830
|
+
const session = await awaitBound(waiting, fresh() ? null : original.sessionId);
|
|
831
|
+
if (session !== undefined) { await settle(session, waiting.binding); continue; }
|
|
832
|
+
} else if (decision.action === "fail") {
|
|
833
|
+
progress.terminals[key] = decision.next; save();
|
|
834
|
+
throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
|
|
835
|
+
} else if (decision.action === "block") {
|
|
836
|
+
if (evidence.state === "live" && evidence.session) {
|
|
837
|
+
throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${evidence.session} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
|
|
838
|
+
}
|
|
839
|
+
if (evidence.state !== "gone") throw blocked(evidence);
|
|
840
|
+
throw new Error(`${original.peer}: restoration evidence is not actionable`);
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
// Never create beside a live launch, nor on evidence that could not be read: read now, not at restore start.
|
|
844
|
+
// The planned session already attached is the restoration (the table's row for it), not a reason to block.
|
|
660
845
|
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
661
|
-
if (evidence.state === "live" && evidence.session
|
|
846
|
+
if (evidence.state === "live" && evidence.session === original.sessionId) {
|
|
847
|
+
if (repair) await settle(evidence.session);
|
|
848
|
+
else { progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, original, false) } satisfies RestoreReceipt; save(); }
|
|
849
|
+
continue;
|
|
850
|
+
}
|
|
662
851
|
if (evidence.state !== "gone") throw blocked(evidence);
|
|
663
|
-
if (
|
|
664
|
-
|
|
665
|
-
|
|
852
|
+
if (transcript() === "unknown") throw new Error(`${unresumable(original)}; no terminal was created; make that store readable first`);
|
|
853
|
+
if (transcript() === "missing" && !fresh()) {
|
|
854
|
+
progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
|
|
855
|
+
if (recordedOnly) throw new MissingRecordedSession({ projectId: planned.project.id, peer: original.peer, sessionId: original.sessionId, kind: "rollout", checkedPath: checkedSessionPath(original, "rollout") });
|
|
856
|
+
throw notRestored(unresumable(original).slice("codex: ".length));
|
|
666
857
|
}
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
if (
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
const
|
|
709
|
-
|
|
858
|
+
if (fresh()) {
|
|
859
|
+
if (!targetReadsWaivers(op)) throw new Error(`${original.peer}: target ${op.plan.version} predates recovery waivers, so it cannot accept a new session`);
|
|
860
|
+
// The restored daemon must accept the new session instead of the recorded one.
|
|
861
|
+
waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
|
|
862
|
+
}
|
|
863
|
+
if (repair) {
|
|
864
|
+
if (repair.state === "attempted") throw new ReattachmentRequired(planned.project.id, original.peer, `${original.peer}: recorded-session reattachment already attempted; no second terminal will be created`);
|
|
865
|
+
repair.state = "attempted"; save(); // durable before any terminal creation, including an uncertain failure
|
|
866
|
+
}
|
|
867
|
+
const entrypoint = join(op.targetRoot!, "src/cli/main.js");
|
|
868
|
+
const argv = restoredTerminalArgv(entrypoint, planned.project.root, original, fresh());
|
|
869
|
+
const assignments = { ...original.launch.env, AGENTHUB_HOME: hubHome(), AGENTHUB_RECOVERY_OPERATION: op.id };
|
|
870
|
+
const launch = { ...original.launch, packageEntrypoint: entrypoint, argv,
|
|
871
|
+
command: ["env", ...Object.entries(assignments).map(([k, v]) => `${k}=${v}`), ...argv].map(shellQuote).join(" ") };
|
|
872
|
+
const binding = { ...original, launch, launchMetadata: launch };
|
|
873
|
+
progress.terminals[key] = { state: "pending" } satisfies RestoreReceipt; save();
|
|
874
|
+
const restored = await createTerminal(binding, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId }, undefined, fresh());
|
|
875
|
+
const exited = restored.blockers.find((b) => b.code === "launcher-exited");
|
|
876
|
+
if (exited) {
|
|
877
|
+
// #215: the launcher's exit alone is not the peer's state (a target that died or flapped also ends it): only
|
|
878
|
+
// nothing live makes it failed; an attached session is settled as the pending row settles it; anything else
|
|
879
|
+
// stays pending.
|
|
880
|
+
const after = await peerEvidence(planned, progress, original.peer);
|
|
881
|
+
const decision = decide({ state: "pending" }, after);
|
|
882
|
+
if (decision.action === "settle" && after.state === "live" && after.session) { await settle(after.session); continue; }
|
|
883
|
+
if (decision.action === "poll" && decision.next.state === "waiting") {
|
|
884
|
+
progress.terminals[key] = decision.next; save();
|
|
885
|
+
const session = await awaitBound(decision.next, fresh() ? null : original.sessionId);
|
|
886
|
+
if (session !== undefined) { await settle(session, decision.next.binding); continue; }
|
|
887
|
+
}
|
|
888
|
+
if (decision.action === "fail") {
|
|
889
|
+
progress.terminals[key] = decision.next; save();
|
|
890
|
+
throw notRestored(exited.message);
|
|
891
|
+
}
|
|
892
|
+
if (after.state !== "gone") throw blocked(after);
|
|
893
|
+
}
|
|
894
|
+
if (restored.manualRequired || !restored.newBinding) {
|
|
895
|
+
const handle = restored.binding?.handle ?? restored.blockers[0]?.handle;
|
|
896
|
+
throw new Error(`${original.peer}: original session restoration needs manual verification${restored.blockers[0] ? ` (${restored.blockers[0].message})` : ""}; wait until it attaches, or end that launch and close its terminal${handle ? ` ${handle}` : ""} first`);
|
|
897
|
+
}
|
|
898
|
+
// The daemon's id is authoritative: a fresh session is recorded under the id it reports once attached.
|
|
899
|
+
const awaiting = { state: "waiting", origin: "pending", binding: restored.newBinding } satisfies RestoreReceipt;
|
|
900
|
+
progress.terminals[key] = awaiting; save();
|
|
901
|
+
const sessionId = await awaitBound(awaiting, fresh() ? null : restored.newBinding.sessionId);
|
|
902
|
+
await settle(sessionId, awaiting.binding);
|
|
903
|
+
} catch (error) {
|
|
904
|
+
if (error instanceof MissingRecordedSession || isDatabaseBusy(error)) throw error;
|
|
905
|
+
// A failed recorded-only Claude launch may prove that its transcript is gone. No error text is evidence.
|
|
906
|
+
if (claudeGroup && progress.repairedSessions?.[plannedBinding.peer] && !progress.fresh?.[plannedBinding.peer]?.repaired) {
|
|
907
|
+
const chosenReceipt = decodeRestoreReceipt(progress.terminals[key]);
|
|
908
|
+
// A bare pending/failed row can hide an uncertain create. Only a durable exact chosen binding
|
|
909
|
+
// can become the terminal that an explicit fresh choice must close before another create.
|
|
910
|
+
const binding = chosenReceipt?.state === "waiting" ? chosenReceipt.binding : undefined;
|
|
911
|
+
if (binding && claudeTranscriptState(binding) === "missing" && (await peerEvidence(planned, progress, plannedBinding.peer)).state === "gone") {
|
|
912
|
+
// Retain the exact chosen terminal so an explicit later fresh choice closes it before creating again.
|
|
913
|
+
progress.terminals[`retired:${plannedBinding.peer}`] = binding;
|
|
914
|
+
delete progress.terminals[`closedRetired:${plannedBinding.peer}`];
|
|
915
|
+
const marker = progress.reattachments?.[plannedBinding.peer];
|
|
916
|
+
if (marker) marker.binding = binding;
|
|
917
|
+
progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
|
|
918
|
+
throw new MissingRecordedSession({ projectId: planned.project.id, peer: plannedBinding.peer, sessionId: binding.sessionId, kind: "transcript", checkedPath: checkedSessionPath(binding, "transcript") });
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
if (error instanceof ReattachmentRequired || isDatabaseBusy(error)) throw error;
|
|
922
|
+
if (progress.reattachments?.[plannedBinding.peer]) throw new ReattachmentRequired(planned.project.id, plannedBinding.peer,
|
|
923
|
+
error instanceof Error ? error.message : `${plannedBinding.peer}: recorded-session reattachment is unconfirmed`, error instanceof RecoveryPresentationError ? error : undefined);
|
|
924
|
+
throw error;
|
|
710
925
|
}
|
|
711
|
-
// The daemon's id is authoritative: a fresh session is recorded under the id it reports once attached.
|
|
712
|
-
progress.terminals[key] = restored.newBinding; save();
|
|
713
|
-
const sessionId = await awaitAttachment(planned, progress, op, restored.newBinding, save, fresh() ? null : restored.newBinding.sessionId);
|
|
714
|
-
progress.terminals[key] = { ...restored.newBinding, sessionId }; save();
|
|
715
926
|
}
|
|
716
|
-
if (group === "claude") for (const id of planned.reconnectOnly ?? []) {
|
|
927
|
+
if (group === "claude" && !onlyPeer) for (const id of planned.reconnectOnly ?? []) {
|
|
717
928
|
// #206: nothing is launched for an unmanaged session; its plugin reconnects by itself, within a bound.
|
|
718
|
-
const deadline = now() +
|
|
929
|
+
const deadline = now() + UNMANAGED_RECONNECT_WAIT_MS;
|
|
719
930
|
while (!(await inspectRecovery(planned.project)).peers.some((p) => p.id === id && p.state !== "offline")) {
|
|
720
|
-
if (now() >= deadline) throw
|
|
931
|
+
if (now() >= deadline) throw nativeFailure(planned, id, "attachment_timeout", `${id}: the unmanaged session did not reconnect within ${UNMANAGED_RECONNECT_WAIT_MS / 1000} s; if that Claude session was closed, start Claude in the project again (its plugin reconnects) first`, { timeoutMs: UNMANAGED_RECONNECT_WAIT_MS });
|
|
721
932
|
await sleep(1000);
|
|
722
933
|
}
|
|
723
934
|
}
|
|
@@ -751,34 +962,78 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
751
962
|
await awaitStopped(project, instance);
|
|
752
963
|
}
|
|
753
964
|
abandonRestartSnapshot(project.stateDir, op.id);
|
|
965
|
+
const planned = op.plan.projects?.find(item => item.project.id === project.id);
|
|
966
|
+
const progress = op.projects?.find(item => item.id === project.id);
|
|
967
|
+
const source = planned?.source.peers.find(peer => peer.id === "pi" && peer.state !== "offline");
|
|
968
|
+
if (source && progress) {
|
|
969
|
+
// abandonRestartSnapshot is deliberately a no-op for another operation or an unreadable snapshot.
|
|
970
|
+
// A continuity bookmark needs positive archival evidence, including when disposal resumes after a crash.
|
|
971
|
+
const archive = join(project.stateDir, `restart.abandoned.${createHash("sha256").update(op.id).digest("hex")}.json`);
|
|
972
|
+
let archived: unknown;
|
|
973
|
+
try { archived = JSON.parse(readFileSync(archive, "utf8")); } catch { return { continuity: "not-archived" }; }
|
|
974
|
+
if (!archived || typeof archived !== "object" || (archived as { operationId?: unknown }).operationId !== op.id) return { continuity: "not-archived" };
|
|
975
|
+
try {
|
|
976
|
+
const binding = planned ? resolveAcceptedBinding(planned, progress, "pi") : undefined;
|
|
977
|
+
const sessionId = binding?.sessionId ?? source.sessionId;
|
|
978
|
+
if (!sessionId) throw new Error("Pi recorded session unavailable; archive continuity bookmark was not written");
|
|
979
|
+
const backend = binding?.backend ?? source.args?.backend;
|
|
980
|
+
writeRecoveryContinuity(project.stateDir, { schema: 1, operationId: op.id, projectId: project.id, projectRoot: project.root, at: now(),
|
|
981
|
+
pi: { sessionId, sessionFile: binding?.sessionFile ?? source.sessionFile ?? source.args?.sessionFile,
|
|
982
|
+
...(backend === "auto" || backend === "dgx" || backend === "mlx" ? { backend } : {}), model: binding?.model ?? source.args?.model } });
|
|
983
|
+
} catch { return { continuity: "unavailable" }; } // optional metadata must not wedge stop-and-archive
|
|
984
|
+
}
|
|
754
985
|
},
|
|
755
986
|
refreshManager: async (op) => { await refreshManager({ home: hubHome(), cli: join(op.targetRoot!, "src/cli/main.ts") }); },
|
|
756
|
-
verify: async (planned, progress, op) => {
|
|
987
|
+
verify: async (planned, progress, op, save) => {
|
|
757
988
|
const live = await inspectRecovery(planned.project);
|
|
758
989
|
if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw new Error("target identity verification failed");
|
|
759
990
|
for (const old of planned.source.peers.filter((p) => p.state !== "offline")) {
|
|
760
991
|
const peer = live.peers.find((p) => p.id === old.id);
|
|
761
|
-
if (!peer || !["idle", "busy", "paused"].includes(peer.state))
|
|
762
|
-
|
|
763
|
-
|
|
992
|
+
if (!peer || !["idle", "busy", "paused"].includes(peer.state)) {
|
|
993
|
+
const receipt = decodeRestoreReceipt(progress.terminals[`restored:${old.id}`]);
|
|
994
|
+
if (receipt?.state === "done") await detached(planned, progress, op, receipt.binding, save);
|
|
995
|
+
throw new Error(`${old.id}: peer reattachment not verified`);
|
|
996
|
+
}
|
|
997
|
+
const accepted = decodeRestoreReceipt(progress.terminals[`restored:${old.id}`]);
|
|
998
|
+
const repair = progress.reattachments?.[old.id as "codex" | "pi" | "claude"];
|
|
999
|
+
const recordedOnly = !!repair || progress.repairedSessions?.[old.id as keyof NonNullable<ProjectProgress["repairedSessions"]>] === true && !progress.fresh?.[old.id]?.repaired;
|
|
1000
|
+
const retiredValue = progress.terminals[`retired:${old.id}`];
|
|
1001
|
+
const retired = retiredValue === undefined ? undefined : decodeRestoreReceipt({ state: "done", binding: retiredValue });
|
|
1002
|
+
// A #225 restart clears the old target's repair fence, never its accepted conversation.
|
|
1003
|
+
const retained = retired?.state === "done" && accepted?.state === "done" && retired.binding.peer === old.id && retired.binding.projectRoot === planned.project.root && retired.binding.sessionId === accepted.binding.sessionId;
|
|
1004
|
+
if ((recordedOnly || retained) && accepted?.state === "done") {
|
|
1005
|
+
const session = old.id === "codex" ? peer.threadId : peer.sessionId;
|
|
1006
|
+
if (session !== accepted.binding.sessionId || repair && accepted.binding.sessionId !== repair.binding.sessionId ||
|
|
1007
|
+
old.id === "pi" && (recordedOnly || old.sessionFile) && accepted.binding.sessionFile && peer.sessionFile !== accepted.binding.sessionFile) throw nativeFailure(planned, old.id, "session_changed", `${old.id}: reattached a different recorded conversation`, { handle: accepted.binding.handle, expectedSession: accepted.binding.sessionId, ...(session === undefined ? {} : { observedSession: session }) });
|
|
1008
|
+
continue;
|
|
1009
|
+
}
|
|
1010
|
+
if (!recordedOnly && (progress.fresh?.[old.id] || planned.freshStart?.includes(old.id))) continue; // #215: a new session was accepted (recorded)
|
|
1011
|
+
if (old.id === "codex" && peer.threadId !== old.threadId) throw nativeFailure(planned, old.id, "session_changed", "Codex resumed a different conversation", { ...(old.threadId ? { expectedSession: old.threadId } : {}), ...(peer.threadId ? { observedSession: peer.threadId } : {}) });
|
|
764
1012
|
if (old.id === "claude" && peer.sessionId !== old.sessionId) {
|
|
765
1013
|
// #64: the restore gate already accepted a fresh session when the original never
|
|
766
1014
|
// persisted a transcript. Re-derive persistence from the binding exactly like that
|
|
767
1015
|
// gate did, so a first turn landing after the plan was made keeps the check strict.
|
|
768
1016
|
const binding = (planned.terminals as TerminalBinding[]).find((t) => t.peer === "claude");
|
|
769
|
-
if (!binding || binding.sessionId !== old.sessionId || claudeTranscriptExists(binding)) throw
|
|
1017
|
+
if (recordedOnly || !binding || binding.sessionId !== old.sessionId || claudeTranscriptExists(binding)) throw nativeFailure(planned, old.id, "session_changed", "Claude resumed a different conversation", { ...(old.sessionId ? { expectedSession: old.sessionId } : {}), ...(peer.sessionId ? { observedSession: peer.sessionId } : {}) });
|
|
770
1018
|
}
|
|
771
|
-
if (old.id === "pi" && (peer.sessionId !== old.sessionId || (old.sessionFile && peer.sessionFile !== old.sessionFile))) throw
|
|
1019
|
+
if (old.id === "pi" && (peer.sessionId !== old.sessionId || (old.sessionFile && peer.sessionFile !== old.sessionFile))) throw nativeFailure(planned, old.id, "session_changed", "Pi resumed a different conversation", { ...(old.sessionId ? { expectedSession: old.sessionId } : {}), ...(peer.sessionId ? { observedSession: peer.sessionId } : {}) });
|
|
772
1020
|
}
|
|
773
1021
|
for (const original of planned.terminals as TerminalBinding[]) {
|
|
774
|
-
const saved = progress.terminals[`restored:${original.peer}`];
|
|
775
|
-
if (
|
|
776
|
-
await revalidateTerminal(planned, progress, saved
|
|
1022
|
+
const saved = decodeRestoreReceipt(progress.terminals[`restored:${original.peer}`]);
|
|
1023
|
+
if (saved?.state !== "done") throw new Error(`${original.peer}: restored terminal outcome is not recorded; manual-required`);
|
|
1024
|
+
await revalidateTerminal(planned, progress, saved.binding, true);
|
|
777
1025
|
}
|
|
778
1026
|
const snapshot = await rpc(planned.project, { t: "ui_snapshot", after: 0 });
|
|
779
1027
|
if (snapshot.ok === false) throw new Error("dashboard snapshot readback failed");
|
|
780
1028
|
const recovery = await rpc(planned.project, { t: "recovery", op: "inspect", expectedInstanceId: progress.instanceId });
|
|
781
1029
|
const integrity = recovery.recovery?.integrity;
|
|
1030
|
+
if (integrity?.expected && integrity.current) {
|
|
1031
|
+
for (const check of ["queues", "manualPaused", "boardDigest", "budgetDigest"] as const) {
|
|
1032
|
+
if (JSON.stringify(integrity.current[check]) !== JSON.stringify(integrity.expected[check])) {
|
|
1033
|
+
throw new RecoveryPreservationError(check, planned.project.id);
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
782
1037
|
if (!integrity?.expected || JSON.stringify(integrity.current) !== JSON.stringify(integrity.expected)) {
|
|
783
1038
|
throw new Error("queue, manual pause, task board or budget preservation was not verified");
|
|
784
1039
|
}
|