@staix/agent-hub 0.12.19 → 0.12.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +3 -3
- package/docs/agent-notes/adapters.md +1 -0
- package/docs/agent-notes/bus.md +1 -0
- package/docs/agent-notes/daemon.md +2 -1
- package/docs/agent-notes/models.md +3 -0
- package/docs/agent-notes/tasks.md +3 -2
- package/docs/events.md +4 -1
- package/docs/operations.md +413 -17
- package/docs/quickstart.md +1 -1
- package/docs/security.md +17 -10
- package/docs/smoke.md +58 -0
- package/docs/specs/2026-09-19-agent-hub-design.md +399 -3
- package/docs/specs/2026-09-20-pi-local-models-design.md +2 -0
- package/docs/specs/2026-09-20-upgrade-recovery-design.md +287 -0
- package/docs/specs/2026-10-04-switchyard-source-port-design.md +10 -0
- package/docs/verified.json +13 -13
- package/package.json +1 -1
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +63 -25
- package/src/adapters/claude-channel.ts +61 -20
- package/src/adapters/codex-appserver.ts +3 -0
- package/src/adapters/local-worker.ts +4 -0
- package/src/cli/console-state.ts +325 -102
- package/src/cli/console.ts +44 -23
- package/src/cli/help.ts +120 -0
- package/src/cli/identity.ts +3 -3
- package/src/cli/launch.ts +2 -0
- package/src/cli/main.ts +171 -75
- package/src/cli/preview.ts +1 -0
- package/src/cli/reset.ts +194 -0
- package/src/cli/status-lines.ts +9 -5
- package/src/cli/terminal-recovery.ts +140 -36
- package/src/cli/upgrade-runtime.ts +274 -56
- package/src/cli/upgrade.ts +366 -41
- package/src/hub/ask.ts +1 -1
- package/src/hub/board.ts +15 -7
- package/src/hub/bus.ts +60 -4
- package/src/hub/child-process.ts +1 -0
- package/src/hub/conductor.ts +38 -6
- package/src/hub/control-client.ts +5 -3
- package/src/hub/daemon.ts +56 -9
- package/src/hub/events.ts +7 -1
- package/src/hub/execution-budget.ts +15 -6
- package/src/hub/hub-tools.ts +4 -4
- package/src/hub/inference.ts +136 -1
- package/src/hub/lifecycle.ts +7 -1
- package/src/hub/peers.ts +2 -0
- package/src/hub/recovery-store.ts +51 -3
- package/src/hub/report.ts +43 -1
- package/src/hub/restart.ts +25 -0
- package/src/hub/routing.ts +48 -16
- package/src/hub/snapshots.ts +3 -2
- package/src/hub/tasks.ts +236 -83
- package/src/local/deny.ts +2 -1
- package/src/models/mlx.ts +11 -8
- package/src/models/ollama.ts +2 -2
- package/src/models/relay.ts +190 -42
- package/src/models/route/relay-selector.ts +43 -31
- package/src/models/route/runtime.ts +17 -7
- package/src/models/route/signals.ts +12 -1
- package/src/models/route/stage.ts +57 -3
- package/src/omniroute/client.ts +1 -1
- package/src/ui/index.html +1 -0
- package/templates/AGENTS.block.md +1 -1
- package/templates/routing.toml +8 -0
|
@@ -1,15 +1,20 @@
|
|
|
1
|
-
import { cpSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync } from "node:fs";
|
|
1
|
+
import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync } from "node:fs";
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
|
-
import { join, resolve } from "node:path";
|
|
3
|
+
import { basename, join, resolve } from "node:path";
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
5
|
import { ControlClient, PROTOCOL, RECOVERY_SOURCE_PROTOCOLS, readControl } from "../hub/control-client.ts";
|
|
6
|
-
import { inspectProject } from "../hub/lifecycle.ts";
|
|
6
|
+
import { awaitStopped, inspectProject } from "../hub/lifecycle.ts";
|
|
7
7
|
import type { Project } from "../hub/registry.ts";
|
|
8
8
|
import { hubHome } from "../hub/project.ts";
|
|
9
9
|
import { packageDigest, registryRelease, runCommand, stageRelease, verifyPackage, type RunCommand } from "./recovery-package.ts";
|
|
10
|
-
import { inspectTerminals, closeTerminal, createTerminal, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
|
|
11
|
-
import { planFingerprint, registeredProjects, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type UpgradePlan } from "./upgrade.ts";
|
|
10
|
+
import { inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
|
|
11
|
+
import { FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type UpgradePlan } from "./upgrade.ts";
|
|
12
|
+
import { readEvents } from "../hub/events.ts";
|
|
12
13
|
import { refreshManager } from "../hub/manager.ts";
|
|
14
|
+
import { abandonRestartSnapshot, readRestartSnapshot, waiveRecoveryPeers } from "../hub/restart.ts";
|
|
15
|
+
|
|
16
|
+
/** An unmanaged Claude's plugin retries with a backoff of at most 30 s; three of those bound the reconnect wait (#206). */
|
|
17
|
+
const RECONNECT_WAIT_MS = 90_000;
|
|
13
18
|
|
|
14
19
|
export const PACKAGE_ROOT = resolve(import.meta.dir, "../..");
|
|
15
20
|
const orcaExecutable = () => process.env.ORCA_CLI_COMMAND || (process.env.ORCA_DEV_REPO_ROOT ? "orca-dev" : process.platform === "linux" && !process.env.ORCA_TERMINAL_HANDLE ? "orca-ide" : "orca");
|
|
@@ -25,6 +30,42 @@ function claudeTranscriptExists(binding: TerminalBinding): boolean {
|
|
|
25
30
|
return existsSync(join(config, "projects", slug, `${binding.sessionId}.jsonl`));
|
|
26
31
|
}
|
|
27
32
|
|
|
33
|
+
/**
|
|
34
|
+
* #215: `codex resume <id>` reads the thread's rollout from the store its launcher uses (the captured CODEX_HOME, else
|
|
35
|
+
* the default), so an app-server thread id alone proves nothing.
|
|
36
|
+
* ponytail: a `.jsonl` under sessions/ whose name carries the thread id is the evidence; ask Codex itself once it offers
|
|
37
|
+
* a supported resumability query, or when its store layout changes.
|
|
38
|
+
*/
|
|
39
|
+
const codexSessions = (binding: TerminalBinding) => join(binding.launch.env.CODEX_HOME ?? join(homedir(), ".codex"), "sessions");
|
|
40
|
+
function codexTranscript(binding: TerminalBinding): "found" | "missing" | "unknown" {
|
|
41
|
+
try { return readdirSync(codexSessions(binding), { recursive: true }).some((name) => String(name).endsWith(".jsonl") && basename(String(name)).includes(binding.sessionId)) ? "found" : "missing"; }
|
|
42
|
+
catch (error) { return (error as NodeJS.ErrnoException).code === "ENOENT" ? "missing" : "unknown"; } // no store has no rollout; an unreadable one says nothing
|
|
43
|
+
}
|
|
44
|
+
const unresumable = (binding: TerminalBinding) => codexTranscript(binding) === "unknown"
|
|
45
|
+
? `codex: the session store ${codexSessions(binding)} cannot be read, so whether thread ${binding.sessionId} can resume is unknown`
|
|
46
|
+
: `codex: thread ${binding.sessionId} has no resumable transcript under ${codexSessions(binding)}`;
|
|
47
|
+
|
|
48
|
+
const closeUnsettled = (binding: TerminalBinding, seen: "listed" | "unreadable") => seen === "listed"
|
|
49
|
+
? `${binding.peer}: terminal close outcome needs manual reconciliation: Orca still lists terminal ${binding.handle}; close that terminal (the login shell it runs in) by hand in Orca first`
|
|
50
|
+
: `${binding.peer}: Orca's terminal list could not be read, so whether terminal ${binding.handle} is closed is unknown; check it in Orca (close it by hand, with the login shell it runs in, if it is still open) first`;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* #215: Codex writes a thread's rollout with its first message, so a thread this hub saw Codex start (`native_thread`
|
|
54
|
+
* with `fresh`) on which no turn ran has nothing to lose. A codex turn_start belongs to the thread most recently
|
|
55
|
+
* adopted before it; detaching forgets nothing. A thread whose start the log does not show (resumed, older hub, pruned
|
|
56
|
+
* log) is unsure, and unsure counts as turned.
|
|
57
|
+
*/
|
|
58
|
+
function codexZeroTurn(stateDir: string, thread: string): boolean {
|
|
59
|
+
let current: string | undefined, started = false, turned = false;
|
|
60
|
+
for (const e of readEvents(join(stateDir, "events.jsonl"))) {
|
|
61
|
+
if (e.type === "native_thread" && e.peer === "codex") {
|
|
62
|
+
current = e.thread;
|
|
63
|
+
if (e.thread === thread && e.fresh) started = true;
|
|
64
|
+
} else if (e.type === "turn_start" && e.peer === "codex" && current === thread) turned = true;
|
|
65
|
+
}
|
|
66
|
+
return started && !turned;
|
|
67
|
+
}
|
|
68
|
+
|
|
28
69
|
function terminalOptions(run: RunCommand = runCommand): TerminalRecoveryOptions {
|
|
29
70
|
return { orcaExecutable: orcaExecutable(), runner: async (args) => {
|
|
30
71
|
const result = await run([orcaExecutable(), ...args], { timeoutMs: 610_000 });
|
|
@@ -32,9 +73,10 @@ function terminalOptions(run: RunCommand = runCommand): TerminalRecoveryOptions
|
|
|
32
73
|
} };
|
|
33
74
|
}
|
|
34
75
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
if (binding.peer === "
|
|
76
|
+
/** `fresh` (#215: the operator's choice, or a planned fresh start) starts Codex or Claude without resuming its recorded session. */
|
|
77
|
+
export function restoredTerminalArgv(entrypoint: string, projectRoot: string, binding: TerminalBinding, fresh = false): string[] {
|
|
78
|
+
if (binding.peer === "codex") return [process.execPath, entrypoint, "--project", projectRoot, "codex", ...(fresh ? [] : ["resume", binding.sessionId])];
|
|
79
|
+
if (binding.peer === "claude") return [process.execPath, entrypoint, "--project", projectRoot, "claude", ...(fresh ? [] : ["--resume", binding.sessionId])];
|
|
38
80
|
return [process.execPath, entrypoint, "--project", projectRoot, "pi", "--mode", "tui",
|
|
39
81
|
...(binding.backend ? ["--backend", binding.backend] : []), ...(binding.model ? ["--model", binding.model] : []),
|
|
40
82
|
...(binding.sessionFile ? ["--session-file", binding.sessionFile] : ["--session-id", binding.sessionId])];
|
|
@@ -54,9 +96,13 @@ export async function inspectRecovery(project: Project): Promise<Inspection> {
|
|
|
54
96
|
const control = readControl(project.stateDir);
|
|
55
97
|
const sourceProtocol = control?.protocol;
|
|
56
98
|
const legacySupported = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]);
|
|
57
|
-
|
|
99
|
+
// A stopped runtime reports whose unreleased restart snapshot it holds: only that one can start it again (#215).
|
|
100
|
+
const snapshot = base.state === "stopped" ? readRestartSnapshot(project.stateDir, { projectRoot: project.root, projectId: project.id })?.operationId : undefined;
|
|
101
|
+
// A dead pid or a missing project directory is final whatever protocol its manifest names: probing it would read
|
|
102
|
+
// "unavailable" forever and wedge abort, resume and stop-and-archive (#215).
|
|
103
|
+
if (base.state === "stopped" || base.state === "missing" || (base.state !== "running" && !legacySupported)) return {
|
|
58
104
|
state: base.state, peers: [], blockers: base.state === "stopped" ? [] : [base.state === "incompatible" ? "manual-bootstrap-required: source lacks the recovery contract; use its matching CLI" : base.error ?? base.state],
|
|
59
|
-
...(control?.instanceId ? { instanceId: control.instanceId } : {}), ...(control?.protocol ? { protocol: control.protocol } : {}),
|
|
105
|
+
...(control?.instanceId ? { instanceId: control.instanceId } : {}), ...(control?.protocol ? { protocol: control.protocol } : {}), ...(snapshot ? { snapshot } : {}),
|
|
60
106
|
};
|
|
61
107
|
let status = base.status;
|
|
62
108
|
if (!status && legacySupported) {
|
|
@@ -114,11 +160,39 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
|
|
|
114
160
|
catch { source = { state: "unavailable", peers: [], blockers: ["source recovery metadata could not be authenticated"] }; }
|
|
115
161
|
if (source.state === "stopped" || source.state === "missing") continue;
|
|
116
162
|
const blockers = [...source.blockers];
|
|
117
|
-
if (!RECOVERY_SOURCE_PROTOCOLS.includes(source.protocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]) || source.state !== "running") blockers.push("manual-bootstrap-required: an authenticated protocol-9, protocol-10, protocol-11, protocol-12, protocol-13, protocol-14 or protocol-
|
|
163
|
+
if (!RECOVERY_SOURCE_PROTOCOLS.includes(source.protocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]) || source.state !== "running") blockers.push("manual-bootstrap-required: an authenticated protocol-9, protocol-10, protocol-11, protocol-12, protocol-13, protocol-14, protocol-15 or protocol-16 source is required");
|
|
118
164
|
if (source.recovery?.operationId && source.recovery.phase !== "released") blockers.push(`existing recovery operation ${source.recovery.operationId} must be resolved first`);
|
|
119
165
|
const sessions: { codex?: string; claude?: string; pi?: SessionRef } = {};
|
|
166
|
+
const reconnectOnly: string[] = [];
|
|
120
167
|
for (const peer of source.peers) {
|
|
121
168
|
if (peer.state === "offline") continue;
|
|
169
|
+
if (peer.id === "claude") {
|
|
170
|
+
// #206: managed means a live `ahub claude` launcher recorded for this daemon and a session record written by
|
|
171
|
+
// that launch. Anything else cannot be bound to the attached session, so its record never becomes a target.
|
|
172
|
+
// ponytail: the channel's hello carries no launch id, so a plain `claude` that took the peer id from a live
|
|
173
|
+
// managed session reads as managed; send the launch id at hello (a protocol bump) if that case shows up.
|
|
174
|
+
const recorded = await launcherOf("claude", project.root, { ...terminalOptions(run), stateDir: project.stateDir, instanceId: source.instanceId });
|
|
175
|
+
if (recorded?.state === "unknown") {
|
|
176
|
+
// An unreadable launcher is never gone (#215): whether the session is managed cannot be told.
|
|
177
|
+
delete peer.sessionId;
|
|
178
|
+
blockers.push(`claude: ${recorded.record ? `the launcher recorded in terminal ${recorded.record.handle}` : "the launcher record (terminal-recovery.json)"} cannot be read, so whether the attached session is managed is unknown; manual-required; next action: make it readable (or end that session and close its terminal), then make a new plan`);
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
const launcher = recorded?.state === "live" ? recorded.record! : undefined;
|
|
182
|
+
if (!launcher) {
|
|
183
|
+
delete peer.sessionId;
|
|
184
|
+
if (source.protocol === PROTOCOL) reconnectOnly.push("claude");
|
|
185
|
+
else blockers.push(`claude: unmanaged session (no live ahub claude launcher recorded) runs a protocol-${source.protocol} plugin and cannot reconnect to a protocol-${PROTOCOL} hub; next action: end that Claude session, or relaunch it with ahub claude in an Orca terminal, then make a new plan`);
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
let record: { instanceId?: unknown; launchId?: unknown; sessionId?: unknown } | undefined;
|
|
189
|
+
try { record = JSON.parse(readFileSync(join(project.stateDir, "claude-session.json"), "utf8")); } catch { /* no record: the id check below reports it */ }
|
|
190
|
+
if (peer.sessionId && (record?.instanceId !== source.instanceId || record?.launchId !== launcher.launchId || record?.sessionId !== peer.sessionId)) {
|
|
191
|
+
delete peer.sessionId;
|
|
192
|
+
blockers.push(`claude: the recorded session was not written by the live launcher in terminal ${launcher.handle}; manual-required; next action: close any other Claude session in this project, send one message in that terminal, then make a new plan`);
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
122
196
|
if (peer.id === "codex" || peer.id === "claude") {
|
|
123
197
|
const session = peer.id === "codex" ? peer.threadId : peer.sessionId;
|
|
124
198
|
if (!session) blockers.push(`${peer.id}: original conversation ID is unknown; manual-required; next action: reconnect the original native session and make a new recovery plan`);
|
|
@@ -132,7 +206,16 @@ export async function makeUpgradePlan(kind: "restart" | "upgrade", version: stri
|
|
|
132
206
|
...terminalOptions(run), stateDir: project.stateDir, instanceId: source.instanceId,
|
|
133
207
|
});
|
|
134
208
|
blockers.push(...terminals.blockers.map((b) => `${b.message}${b.terminalReference ? ` (terminal ${b.terminalReference})` : ""}${b.nextAction ? `; next action: ${b.nextAction}` : ""}`));
|
|
135
|
-
|
|
209
|
+
const freshStart: string[] = [];
|
|
210
|
+
for (const binding of terminals.bindings) if (binding.peer === "codex") {
|
|
211
|
+
const transcript = codexTranscript(binding);
|
|
212
|
+
if (transcript === "found") continue;
|
|
213
|
+
if (transcript === "missing" && codexZeroTurn(project.stateDir, binding.sessionId)) { freshStart.push("codex"); continue; }
|
|
214
|
+
blockers.push(transcript === "unknown"
|
|
215
|
+
? `${unresumable(binding)}; manual-required; next action: make that store readable, then make a new plan`
|
|
216
|
+
: `${unresumable(binding)}, and the hub cannot show that no turn ran on it, so its conversation may not come back after the restart; manual-required; next action: end that Codex session and close its Orca terminal ${binding.handle} (ahub codex starts a new one later), then make a new plan`);
|
|
217
|
+
}
|
|
218
|
+
body.projects.push({ project, source, terminals: terminals.bindings, blockers, ...(reconnectOnly.length ? { reconnectOnly } : {}), ...(freshStart.length ? { freshStart } : {}) });
|
|
136
219
|
}
|
|
137
220
|
if (!body.projects.length) body.blockers.push("no running registered projects in scope");
|
|
138
221
|
return { ...body, fingerprint: planFingerprint(body) };
|
|
@@ -208,14 +291,66 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
|
|
|
208
291
|
if (session !== saved.sessionId) throw new Error(`${saved.peer}: daemon session changed during terminal revalidation; manual-required`);
|
|
209
292
|
return current;
|
|
210
293
|
};
|
|
294
|
+
/** Whether Orca's inventory lists the terminal; a failed, not-ok, truncated or non-JSON answer shows nothing. */
|
|
295
|
+
const listed = async (binding: TerminalBinding): Promise<"listed" | "absent" | "unreadable"> => {
|
|
296
|
+
const result = await run([orcaExecutable(), "terminal", "list", "--json"]);
|
|
297
|
+
let inventory: any;
|
|
298
|
+
try { inventory = JSON.parse(result.stdout); } catch { return "unreadable"; }
|
|
299
|
+
const rows = inventory?.result?.terminals;
|
|
300
|
+
if (result.code !== 0 || inventory?.ok !== true || !Array.isArray(rows) || inventory.result?.truncated) return "unreadable";
|
|
301
|
+
return rows.some((t: any) => t.handle === binding.handle || t.incarnationId === binding.incarnationId) ? "listed" : "absent";
|
|
302
|
+
};
|
|
303
|
+
/**
|
|
304
|
+
* #215: what runs for a peer on the target, the one place restore reads it. Only a running target answering as the
|
|
305
|
+
* expected instance can report "gone"; an unreadable target or launcher is unknown and blocks, never gone.
|
|
306
|
+
*/
|
|
307
|
+
const peerEvidence = async (planned: PlannedProject, progress: ProjectProgress, peer: TerminalBinding["peer"]):
|
|
308
|
+
Promise<{ state: "live"; session?: string; handle?: string } | { state: "gone" } | { state: "unknown"; why: string; step?: string }> => {
|
|
309
|
+
const target = await inspectRecovery(planned.project);
|
|
310
|
+
if (target.state !== "running" || target.instanceId !== progress.instanceId) {
|
|
311
|
+
// Only a hub that may still answer is waited for; a stopped, missing or other one is settled by the runner's list.
|
|
312
|
+
const settled = ["running", "stopped", "missing"].includes(target.state);
|
|
313
|
+
return { state: "unknown", why: `the target hub reads as ${target.state === "running" ? `instance ${target.instanceId}, not ${progress.instanceId}` : target.state}`, ...(settled ? {} : { step: "wait until the target answers" }) };
|
|
314
|
+
}
|
|
315
|
+
const attached = target.peers.find((p) => p.id === peer && p.state !== "offline");
|
|
316
|
+
const session = peer === "codex" ? attached?.threadId : attached?.sessionId;
|
|
317
|
+
if (session) return { state: "live", session };
|
|
318
|
+
const launch = await launcherOf(peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId });
|
|
319
|
+
if (launch?.state === "live") return { state: "live", handle: launch.record!.handle };
|
|
320
|
+
if (launch?.state === "unknown") {
|
|
321
|
+
return launch.record ? { state: "unknown", why: `its launcher in terminal ${launch.record.handle} cannot be read`, step: `wait until it attaches, or end it and close terminal ${launch.record.handle}` }
|
|
322
|
+
: { state: "unknown", why: "the launcher records cannot be read", step: `inspect ${recordPath(planned.project.stateDir)} and move it aside (sessions it recorded then count as unmanaged)` };
|
|
323
|
+
}
|
|
324
|
+
return { state: "gone" };
|
|
325
|
+
};
|
|
326
|
+
const attachedId = async (planned: PlannedProject, peer: string, op: RecoveryOperation): Promise<string> => {
|
|
327
|
+
for (const deadline = now() + RECONNECT_WAIT_MS; ;) {
|
|
328
|
+
const current = (await inspectRecovery(planned.project)).peers.find((p) => p.id === peer && p.state !== "offline");
|
|
329
|
+
const id = peer === "codex" ? current?.threadId : current?.sessionId;
|
|
330
|
+
if (id) return id;
|
|
331
|
+
if (now() >= deadline) throw new Error(`${peer}: the new session reported no id within ${RECONNECT_WAIT_MS / 1000} s; wait until it attaches first`);
|
|
332
|
+
await sleep(1000);
|
|
333
|
+
}
|
|
334
|
+
};
|
|
211
335
|
return {
|
|
212
336
|
now, sleep, inspect: inspectRecovery,
|
|
213
337
|
stage: async (op) => {
|
|
214
|
-
if (packageDigest(op.sourceRoot) !== op.plan.sourceDigest) throw new
|
|
338
|
+
if (packageDigest(op.sourceRoot) !== op.plan.sourceDigest) throw new FinalRefusal("preserved source changed");
|
|
215
339
|
const target = op.plan.kind === "restart" ? { root: op.sourceRoot, digest: verifyPackage(op.sourceRoot, op.plan.version) }
|
|
216
340
|
: await stageRelease(op.plan.version, op.plan.integrity!, run);
|
|
341
|
+
// #215: the protocol and waiver checks are properties of these bytes. Once they passed (the runner recorded the
|
|
342
|
+
// digest), a resume does not check again, so a transient probe failure cannot refuse an operation for good;
|
|
343
|
+
// changed bytes are the runner's final "staged target changed".
|
|
344
|
+
if (op.targetDigest && op.targetDigest === target.digest && op.targetRoot === target.root) return target;
|
|
217
345
|
const protocol = await run([process.execPath, "-e", `import { PROTOCOL } from ${JSON.stringify(join(target.root, "src/hub/control-client.ts"))}; console.log(PROTOCOL)`]);
|
|
218
|
-
|
|
346
|
+
const read = protocol.code === 0 ? Number.parseInt(protocol.stdout.trim(), 10) : NaN;
|
|
347
|
+
// A probe that did not answer (killed, a null exit code maps to 1) says nothing about the target: retry.
|
|
348
|
+
if (!Number.isInteger(read)) throw new Error(`the staged target's control protocol could not be read (probe exit ${protocol.code}); staged package retained, runtimes unchanged; resume again once it can run`);
|
|
349
|
+
if (read !== PROTOCOL) throw new FinalRefusal("target protocol requires a newer coordinator; staged package retained, runtimes unchanged");
|
|
350
|
+
// Reconnect-only (#206) and fresh sessions (#215) need a target hub that reads recovery-waivers.json.
|
|
351
|
+
if (op.plan.projects.some((p) => p.reconnectOnly?.length || p.freshStart?.length) && !targetReadsWaivers({ targetRoot: target.root })) {
|
|
352
|
+
throw new FinalRefusal(`target ${op.plan.version} predates recovery waivers, so its hub could never accept a reconnect-only Claude or a fresh Codex start; staged package retained, runtimes unchanged; choose a newer target once this operation is cancelled or ended`);
|
|
353
|
+
}
|
|
219
354
|
return target;
|
|
220
355
|
},
|
|
221
356
|
prepare: (p, id, instance) => control(p, "prepare", id, instance),
|
|
@@ -227,35 +362,33 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
|
|
|
227
362
|
const key = `closed:${binding.peer}`;
|
|
228
363
|
if (progress.terminals[key] === true) continue;
|
|
229
364
|
if (progress.terminals[key] === "pending") {
|
|
230
|
-
const
|
|
231
|
-
|
|
232
|
-
const rows = inventory.result?.terminals;
|
|
233
|
-
if (result.code !== 0 || inventory.ok !== true || !Array.isArray(rows) || inventory.result?.truncated ||
|
|
234
|
-
rows.some((t: any) => t.handle === binding.handle || t.incarnationId === binding.incarnationId)) {
|
|
235
|
-
throw new Error(`${binding.peer}: terminal close outcome needs manual reconciliation`);
|
|
236
|
-
}
|
|
365
|
+
const seen = await listed(binding);
|
|
366
|
+
if (seen !== "absent") throw new Error(closeUnsettled(binding, seen)); // the receipt stays pending
|
|
237
367
|
progress.terminals[key] = true; save(); continue;
|
|
238
368
|
}
|
|
239
369
|
// A peer whose TUI already exited left nothing to close: the source reports it
|
|
240
370
|
// detached, and inventory silence is the same proof the pending-reconcile path
|
|
241
371
|
// accepts. No mutation is issued for a terminal that no longer exists (#21).
|
|
242
|
-
const
|
|
372
|
+
const sourceNow = await inspectRecovery(planned.project);
|
|
373
|
+
const attached = sourceNow.peers.find((item) => item.id === binding.peer);
|
|
243
374
|
if (attached?.state === "offline") {
|
|
244
|
-
const
|
|
245
|
-
|
|
246
|
-
const rows = inventory.result?.terminals;
|
|
247
|
-
if (result.code !== 0 || inventory.ok !== true || !Array.isArray(rows) || inventory.result?.truncated ||
|
|
248
|
-
rows.some((t: any) => t.handle === binding.handle || t.incarnationId === binding.incarnationId)) {
|
|
249
|
-
throw new Error(`${binding.peer}: terminal close outcome needs manual reconciliation`);
|
|
250
|
-
}
|
|
375
|
+
const seen = await listed(binding);
|
|
376
|
+
if (seen !== "absent") throw new Error(closeUnsettled(binding, seen));
|
|
251
377
|
progress.terminals[key] = true; save(); continue;
|
|
252
378
|
}
|
|
379
|
+
// #215: check resume viability again right before the destructive effect (a planned fresh start has none to lose).
|
|
380
|
+
const transcript = binding.peer === "codex" ? codexTranscript(binding) : "found";
|
|
381
|
+
if (transcript === "unknown") throw new Error(`${unresumable(binding)}; no terminal was closed; make that store readable first`);
|
|
382
|
+
if (transcript === "missing" && !planned.freshStart?.includes("codex")) {
|
|
383
|
+
throw new Error(`${unresumable(binding)}; no terminal was closed. To continue without its conversation, end that Codex session and close its Orca terminal ${binding.handle} first, then resume`);
|
|
384
|
+
}
|
|
253
385
|
const idle = await waitForIdle(binding, 600_000, terminalOptions(run));
|
|
254
|
-
if (!idle.satisfied) throw new Error(`${binding.peer}: terminal is not verified idle; source retained`);
|
|
386
|
+
if (!idle.satisfied) throw new Error(`${binding.peer}: terminal is not verified idle; source retained; finish or cancel its turn first`);
|
|
255
387
|
const source = await inspectRecovery(planned.project);
|
|
256
388
|
if (source.instanceId !== planned.source.instanceId || source.recovery?.operationId !== op.id || !source.recovery.ready) {
|
|
257
|
-
throw new Error(
|
|
258
|
-
}
|
|
389
|
+
throw new Error(`source preparation expired or changed while waiting for the terminal; no terminal was closed`);
|
|
390
|
+
}
|
|
391
|
+
// Refresh the same prepared lease immediately before the terminal effect. Preserve
|
|
259
392
|
// the original peer roster, including any terminal already closed in this operation.
|
|
260
393
|
await control(planned.project, "prepare", op.id, planned.source.instanceId!);
|
|
261
394
|
progress.terminals[key] = "pending"; save();
|
|
@@ -269,6 +402,8 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
|
|
|
269
402
|
if (checkpoint.projectId !== p.id || checkpoint.projectRoot !== p.root || checkpoint.operationId !== op.id) {
|
|
270
403
|
throw new Error("committed restart snapshot is missing or belongs to another operation");
|
|
271
404
|
}
|
|
405
|
+
const reconnect = op.plan.projects.find((planned) => planned.project.id === p.id)?.reconnectOnly ?? [];
|
|
406
|
+
if (reconnect.length) waiveRecoveryPeers(p.stateDir, op.id, Object.fromEntries(reconnect.map((peer) => [peer, "reconnect-only"])));
|
|
272
407
|
await command(op, ["up"], p);
|
|
273
408
|
},
|
|
274
409
|
restore: async (planned, progress, op, group, save) => {
|
|
@@ -291,39 +426,109 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
|
|
|
291
426
|
for (const original of planned.terminals as TerminalBinding[]) {
|
|
292
427
|
if ((original.peer === "claude") !== (group === "claude")) continue;
|
|
293
428
|
const key = `restored:${original.peer}`;
|
|
294
|
-
|
|
295
|
-
|
|
429
|
+
// A new session is accepted only by the operator's choice (#215 dispose --fresh-session) or a planned fresh
|
|
430
|
+
// start of a Codex thread with nothing to lose that still has no rollout; an unreadable store is neither.
|
|
431
|
+
const chosen = !!progress.fresh?.[original.peer];
|
|
432
|
+
let read: ReturnType<typeof codexTranscript> | undefined;
|
|
433
|
+
const transcript = () => original.peer !== "codex" || chosen ? "found" : (read ??= codexTranscript(original));
|
|
434
|
+
const fresh = () => chosen || (!!planned.freshStart?.includes(original.peer) && transcript() === "missing");
|
|
435
|
+
// The runner ends this with the choices `status` shows (nextActions); resume comes first and launches a failed
|
|
436
|
+
// peer again, which helps once its cause (a transient Orca create failure, a fixed store) is gone.
|
|
437
|
+
const notRestored = (why: string) => new Error(fresh()
|
|
438
|
+
? `${original.peer}: ${why}; read that terminal in Orca for the launcher's error and fix its cause`
|
|
439
|
+
: `${original.peer}: ${why}; session ${original.sessionId} was not restored; resume launches it again once the cause is fixed`);
|
|
440
|
+
// Anything live or unknown blocks a create and keeps the receipt: never decide on evidence that could not be read.
|
|
441
|
+
const blocked = (evidence: Exclude<Awaited<ReturnType<typeof peerEvidence>>, { state: "gone" }>) => new Error(evidence.state === "unknown"
|
|
442
|
+
? `${original.peer}: whether a ${original.peer} session or launcher is live cannot be told (${evidence.why}); nothing was recorded or created${evidence.step ? `; ${evidence.step} first` : ""}`
|
|
443
|
+
: evidence.session
|
|
444
|
+
? `${original.peer}: session ${evidence.session} is attached instead of ${original.sessionId}; no terminal was created; end that ${original.peer} session and close its terminal first`
|
|
445
|
+
: `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`);
|
|
446
|
+
const receipt = progress.terminals[key];
|
|
447
|
+
if (receipt && receipt !== "pending" && receipt !== "failed") {
|
|
448
|
+
progress.terminals[key] = await revalidateTerminal(planned, progress, receipt as TerminalBinding, true); save();
|
|
296
449
|
continue;
|
|
297
450
|
}
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
if (attached !== original.sessionId) {
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
if (
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
451
|
+
// #215: the pending row of the receipt table, used wherever a pending launch meets an attached session.
|
|
452
|
+
const settle = async (attached: string) => {
|
|
453
|
+
// #21: a Claude session that never got a first turn has no transcript, so `claude --resume S` can never
|
|
454
|
+
// succeed and accepting its fresh attach loses nothing. Otherwise only the planned id, or a new session
|
|
455
|
+
// the operator or the plan accepted, counts as restored.
|
|
456
|
+
if (attached !== original.sessionId && !fresh() && !(original.peer === "claude" && !claudeTranscriptExists(original))) {
|
|
457
|
+
throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${attached} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
|
|
458
|
+
}
|
|
459
|
+
// The original came back after all: a fresh session chosen for it lost nothing.
|
|
460
|
+
if (attached === original.sessionId && progress.fresh?.[original.peer]) {
|
|
461
|
+
delete progress.fresh[original.peer];
|
|
462
|
+
if (!Object.keys(progress.fresh).length) delete progress.fresh;
|
|
463
|
+
}
|
|
464
|
+
// Any other accepted session needs the target to waive the saved id, or release would wait for it forever;
|
|
465
|
+
// a launch writes the waiver first, but this must not rely on that.
|
|
466
|
+
if (attached !== original.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: progress.fresh?.[original.peer] ? "fresh-session" : fresh() ? "fresh-start" : "zero-turn" });
|
|
467
|
+
progress.terminals[key] = await revalidateTerminal(planned, progress, { ...original, sessionId: attached }, false); save();
|
|
468
|
+
};
|
|
469
|
+
// #215: a pending or failed receipt is settled by what is live now (the receipt table in the recovery spec).
|
|
470
|
+
if (receipt === "pending" || receipt === "failed") {
|
|
471
|
+
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
472
|
+
if (evidence.state === "live" && evidence.session) { await settle(evidence.session); continue; }
|
|
473
|
+
if (evidence.state !== "gone") throw blocked(evidence);
|
|
474
|
+
if (receipt === "pending") {
|
|
475
|
+
progress.terminals[key] = "failed"; save();
|
|
476
|
+
throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
|
|
313
477
|
}
|
|
314
|
-
|
|
315
|
-
|
|
478
|
+
// failed, nothing live: launched again below; the receipt stays `failed` until `pending` replaces it, so a
|
|
479
|
+
// refusal on the way keeps the failed-restoration choices
|
|
480
|
+
}
|
|
481
|
+
// Never create beside a live launch, nor on evidence that could not be read: read now, not at restore start.
|
|
482
|
+
// The planned session already attached is the restoration (the table's row for it), not a reason to block.
|
|
483
|
+
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
484
|
+
if (evidence.state === "live" && evidence.session === original.sessionId) {
|
|
485
|
+
progress.terminals[key] = await revalidateTerminal(planned, progress, original, false); save();
|
|
486
|
+
continue;
|
|
487
|
+
}
|
|
488
|
+
if (evidence.state !== "gone") throw blocked(evidence);
|
|
489
|
+
if (transcript() === "unknown") throw new Error(`${unresumable(original)}; no terminal was created; make that store readable first`);
|
|
490
|
+
if (transcript() === "missing" && !fresh()) {
|
|
491
|
+
progress.terminals[key] = "failed"; save();
|
|
492
|
+
throw notRestored(unresumable(original).slice("codex: ".length));
|
|
493
|
+
}
|
|
494
|
+
if (fresh()) {
|
|
495
|
+
if (!targetReadsWaivers(op)) throw new Error(`${original.peer}: target ${op.plan.version} predates recovery waivers, so it cannot accept a new session`);
|
|
496
|
+
// The restored daemon must accept the new session instead of the recorded one.
|
|
497
|
+
waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: progress.fresh?.[original.peer] ? "fresh-session" : "fresh-start" });
|
|
316
498
|
}
|
|
317
499
|
const entrypoint = join(op.targetRoot!, "src/cli/main.js");
|
|
318
|
-
const argv = restoredTerminalArgv(entrypoint, planned.project.root, original);
|
|
500
|
+
const argv = restoredTerminalArgv(entrypoint, planned.project.root, original, fresh());
|
|
319
501
|
const assignments = { ...original.launch.env, AGENTHUB_HOME: hubHome(), AGENTHUB_RECOVERY_OPERATION: op.id };
|
|
320
502
|
const launch = { ...original.launch, packageEntrypoint: entrypoint, argv,
|
|
321
503
|
command: ["env", ...Object.entries(assignments).map(([k, v]) => `${k}=${v}`), ...argv].map(shellQuote).join(" ") };
|
|
322
504
|
const binding = { ...original, launch, launchMetadata: launch };
|
|
323
505
|
progress.terminals[key] = "pending"; save();
|
|
324
|
-
const restored = await createTerminal(binding, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId });
|
|
325
|
-
|
|
326
|
-
|
|
506
|
+
const restored = await createTerminal(binding, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId }, undefined, fresh());
|
|
507
|
+
const exited = restored.blockers.find((b) => b.code === "launcher-exited");
|
|
508
|
+
if (exited) {
|
|
509
|
+
// #215: the launcher's exit alone is not the peer's state (a target that died or flapped also ends it): only
|
|
510
|
+
// nothing live makes it failed; an attached session is settled as the pending row settles it; anything else
|
|
511
|
+
// stays pending.
|
|
512
|
+
const after = await peerEvidence(planned, progress, original.peer);
|
|
513
|
+
if (after.state === "live" && after.session) { await settle(after.session); continue; }
|
|
514
|
+
if (after.state !== "gone") throw blocked(after);
|
|
515
|
+
progress.terminals[key] = "failed"; save();
|
|
516
|
+
throw notRestored(exited.message);
|
|
517
|
+
}
|
|
518
|
+
if (restored.manualRequired || !restored.newBinding) {
|
|
519
|
+
const handle = restored.binding?.handle ?? restored.blockers[0]?.handle;
|
|
520
|
+
throw new Error(`${original.peer}: original session restoration needs manual verification${restored.blockers[0] ? ` (${restored.blockers[0].message})` : ""}; wait until it attaches, or end that launch and close its terminal${handle ? ` ${handle}` : ""} first`);
|
|
521
|
+
}
|
|
522
|
+
// The daemon's id is authoritative: a fresh session is recorded under the id it reports once attached.
|
|
523
|
+
progress.terminals[key] = fresh() ? { ...restored.newBinding, sessionId: await attachedId(planned, original.peer, op) } : restored.newBinding; save();
|
|
524
|
+
}
|
|
525
|
+
if (group === "claude") for (const id of planned.reconnectOnly ?? []) {
|
|
526
|
+
// #206: nothing is launched for an unmanaged session; its plugin reconnects by itself, within a bound.
|
|
527
|
+
const deadline = now() + RECONNECT_WAIT_MS;
|
|
528
|
+
while (!(await inspectRecovery(planned.project)).peers.some((p) => p.id === id && p.state !== "offline")) {
|
|
529
|
+
if (now() >= deadline) throw new Error(`${id}: the unmanaged session did not reconnect within ${RECONNECT_WAIT_MS / 1000} s; if that Claude session was closed, start Claude in the project again (its plugin reconnects) first`);
|
|
530
|
+
await sleep(1000);
|
|
531
|
+
}
|
|
327
532
|
}
|
|
328
533
|
},
|
|
329
534
|
installPlugin: async (op) => {
|
|
@@ -344,6 +549,18 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
|
|
|
344
549
|
const readback = await run(["ahub", "--version"], { env: env(op) });
|
|
345
550
|
if (readback.code !== 0 || readback.stdout.trim() !== op.plan.version) throw new Error("global CLI version was not verified");
|
|
346
551
|
},
|
|
552
|
+
stopAndArchive: async (project, op, instance) => {
|
|
553
|
+
if (instance) {
|
|
554
|
+
// #215: a target an older coordinator started speaks its own protocol, which the lifecycle stop's inspection
|
|
555
|
+
// refuses. Ask it to stop at that protocol, fenced by its instance (the daemon refuses another), then wait as
|
|
556
|
+
// the lifecycle stop does until its manifest and registry claim are gone.
|
|
557
|
+
const sourceProtocol = readControl(project.stateDir)?.protocol;
|
|
558
|
+
const protocol = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]) ? sourceProtocol : PROTOCOL;
|
|
559
|
+
await rpc(project, { t: "kill", instanceId: instance }, protocol);
|
|
560
|
+
await awaitStopped(project, instance);
|
|
561
|
+
}
|
|
562
|
+
abandonRestartSnapshot(project.stateDir, op.id);
|
|
563
|
+
},
|
|
347
564
|
refreshManager: async (op) => { await refreshManager({ home: hubHome(), cli: join(op.targetRoot!, "src/cli/main.ts") }); },
|
|
348
565
|
verify: async (planned, progress, op) => {
|
|
349
566
|
const live = await inspectRecovery(planned.project);
|
|
@@ -351,6 +568,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
|
|
|
351
568
|
for (const old of planned.source.peers.filter((p) => p.state !== "offline")) {
|
|
352
569
|
const peer = live.peers.find((p) => p.id === old.id);
|
|
353
570
|
if (!peer || !["idle", "busy", "paused"].includes(peer.state)) throw new Error(`${old.id}: peer reattachment not verified`);
|
|
571
|
+
if (progress.fresh?.[old.id] || planned.freshStart?.includes(old.id)) continue; // #215: a new session was accepted (recorded)
|
|
354
572
|
if (old.id === "codex" && peer.threadId !== old.threadId) throw new Error("Codex resumed a different conversation");
|
|
355
573
|
if (old.id === "claude" && peer.sessionId !== old.sessionId) {
|
|
356
574
|
// #64: the restore gate already accepted a fresh session when the original never
|
|
@@ -363,7 +581,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand): RecoveryDriver
|
|
|
363
581
|
}
|
|
364
582
|
for (const original of planned.terminals as TerminalBinding[]) {
|
|
365
583
|
const saved = progress.terminals[`restored:${original.peer}`];
|
|
366
|
-
if (!saved || saved === "
|
|
584
|
+
if (!saved || typeof saved === "string") throw new Error(`${original.peer}: restored terminal outcome is not recorded; manual-required`);
|
|
367
585
|
await revalidateTerminal(planned, progress, saved as TerminalBinding, true);
|
|
368
586
|
}
|
|
369
587
|
const snapshot = await rpc(planned.project, { t: "ui_snapshot", after: 0 });
|