@staix/agent-hub 0.12.29 → 0.12.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/README.md +92 -218
- package/docs/agent-notes/README.md +12 -0
- package/docs/agent-notes/adapters.md +3 -2
- package/docs/agent-notes/budget.md +1 -1
- package/docs/agent-notes/bus.md +7 -1
- package/docs/agent-notes/daemon.md +14 -1
- package/docs/agent-notes/local-worker.md +4 -2
- package/docs/agent-notes/tasks.md +14 -0
- package/docs/agent-notes/tests.md +6 -3
- package/docs/memory.md +70 -152
- package/docs/operations.md +317 -25
- package/docs/plans/2026-10-16-amem-failure-registry.md +101 -0
- package/docs/plans/2026-10-16-amem-p0-baseline.md +110 -0
- package/docs/plans/2026-10-16-amem-p2-read-tools.md +28 -0
- package/docs/plans/amem-cases.json +482 -0
- package/docs/quickstart.md +24 -2
- package/docs/security.md +42 -2
- package/docs/smoke.md +11 -0
- package/docs/specs/2026-09-19-agent-hub-design.md +142 -7
- package/docs/specs/2026-09-20-upgrade-recovery-design.md +84 -10
- package/docs/verification/2026-10-17-0.12.30.md +70 -0
- package/docs/verification/2026-10-17-0.12.31.md +71 -0
- package/docs/verification/2026-10-17-418-probes.md +21 -0
- package/docs/verified.json +13 -13
- package/package.json +5 -2
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +129 -62
- package/src/adapters/claude-channel.ts +29 -11
- package/src/adapters/local-worker.ts +2 -2
- package/src/adapters/pi.ts +5 -0
- package/src/cli/attention.ts +5 -1
- package/src/cli/console-state.ts +145 -29
- package/src/cli/console.ts +40 -10
- package/src/cli/help.ts +3 -2
- package/src/cli/identity.ts +4 -3
- package/src/cli/jcode-mcp.ts +1 -1
- package/src/cli/main.js +2 -2
- package/src/cli/main.ts +43 -14
- package/src/cli/memory.ts +87 -187
- package/src/cli/output.ts +19 -2
- package/src/cli/recovery-package.ts +59 -10
- package/src/cli/upgrade-interactive.ts +2 -1
- package/src/cli/upgrade-restore-state.ts +133 -0
- package/src/cli/upgrade-runtime.ts +204 -63
- package/src/cli/upgrade.ts +57 -24
- package/src/hub/board.ts +83 -24
- package/src/hub/bus.ts +108 -38
- package/src/hub/check-runner.ts +514 -0
- package/src/hub/child-process.ts +1 -0
- package/src/hub/control-client.ts +5 -1
- package/src/hub/daemon.ts +529 -89
- package/src/hub/delivery-hints.ts +30 -0
- package/src/hub/delivery-journal.ts +185 -12
- package/src/hub/delivery-superseded.ts +52 -0
- package/src/hub/envelope.ts +31 -1
- package/src/hub/git.ts +2 -38
- package/src/hub/host-executable.ts +11 -0
- package/src/hub/hub-tools.ts +14 -2
- package/src/hub/manager.ts +1 -1
- package/src/hub/memory-candidates.ts +103 -0
- package/src/hub/memory-fail.ts +25 -0
- package/src/hub/memory-observations.ts +47 -0
- package/src/hub/memory-plan-guard.ts +33 -0
- package/src/hub/memory-policy.ts +42 -0
- package/src/hub/memory-profile-check.ts +170 -0
- package/src/hub/memory-profile.ts +192 -0
- package/src/hub/memory-staleness.ts +55 -0
- package/src/hub/path-policy.ts +8 -0
- package/src/hub/peer-catalog.ts +3 -0
- package/src/hub/project.ts +4 -37
- package/src/hub/read-tools.ts +127 -0
- package/src/hub/routing.ts +35 -16
- package/src/hub/settings.ts +2 -1
- package/src/hub/spec-read.ts +120 -0
- package/src/hub/supervision.ts +21 -6
- package/src/hub/task-sweep.ts +7 -6
- package/src/hub/tasks.ts +370 -71
- package/src/local/deny.ts +4 -17
- package/src/memory/brief.ts +8 -5
- package/src/memory/capture.ts +3 -1
- package/src/memory/recall.ts +50 -110
- package/src/pi/process-signature.ts +2 -42
- package/src/ui/index.html +12 -9
- package/src/ui/needs.ts +7 -5
- package/src/ui/office-model.ts +4 -1
- package/src/ui/task-board.ts +1 -1
- package/src/ui/task-progress.ts +6 -2
- package/templates/AGENTS.block.md +1 -1
- package/templates/profiles/general.json +1 -1
- package/templates/profiles/research.json +1 -1
- package/templates/profiles/writing.json +1 -1
- package/src/memory/import.ts +0 -90
- package/src/memory/store.ts +0 -805
- package/src/memory/sync.ts +0 -145
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import type { TerminalBinding } from "./terminal-recovery.ts";
|
|
2
|
+
|
|
3
|
+
export type RestoreReceipt =
|
|
4
|
+
| { state: "pending" }
|
|
5
|
+
| { state: "waiting"; origin: "pending" | "failed"; binding: TerminalBinding }
|
|
6
|
+
| { state: "done"; binding: TerminalBinding }
|
|
7
|
+
| { state: "failed" };
|
|
8
|
+
export interface RestoreEvidence {
|
|
9
|
+
target: "running" | "stopped" | "changed";
|
|
10
|
+
launcher: "live" | "gone" | "unknown" | "no-record-yet" | "replaced" | "older-instance";
|
|
11
|
+
session: "planned" | "other" | "none";
|
|
12
|
+
allowOther: boolean;
|
|
13
|
+
/** The original or verified selected terminal when a bare receipt first acquires a binding. */
|
|
14
|
+
binding?: TerminalBinding;
|
|
15
|
+
}
|
|
16
|
+
export type RestoreAction = "poll" | "settle" | "revalidate" | "block" | "fail" | "create";
|
|
17
|
+
export interface RestoreTransition { next: RestoreReceipt; action: RestoreAction }
|
|
18
|
+
export type RestoreReceiptStatus = "unstarted" | "pending" | "waiting:pending" | "waiting:failed" | "done" | "failed";
|
|
19
|
+
|
|
20
|
+
function malformed(): never { throw new Error("invalid restore receipt"); }
|
|
21
|
+
const object = (value: unknown): value is Record<string, unknown> => !!value && typeof value === "object" && !Array.isArray(value);
|
|
22
|
+
const text = (value: unknown): value is string => typeof value === "string" && value.length > 0;
|
|
23
|
+
function terminalBinding(value: unknown): TerminalBinding {
|
|
24
|
+
if (!object(value) || typeof value.peer !== "string" || !["codex", "claude", "pi"].includes(value.peer)) malformed();
|
|
25
|
+
for (const key of ["handle", "incarnationId", "worktreeId", "projectRoot", "sessionId"]) if (!text(value[key])) malformed();
|
|
26
|
+
for (const key of ["sessionFile", "backend", "model"]) if (value[key] !== undefined && typeof value[key] !== "string") malformed();
|
|
27
|
+
const launch = value.launch;
|
|
28
|
+
const validLaunch = (v: unknown) => object(v) && text(v.packageEntrypoint) && text(v.command)
|
|
29
|
+
&& Array.isArray(v.argv) && v.argv.every(arg => typeof arg === "string")
|
|
30
|
+
&& object(v.env) && Object.values(v.env).every(entry => typeof entry === "string");
|
|
31
|
+
if (!validLaunch(launch) || value.launchMetadata !== undefined && !validLaunch(value.launchMetadata)) malformed();
|
|
32
|
+
return { ...value, launchMetadata: value.launchMetadata ?? launch } as unknown as TerminalBinding;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Decode legacy representations only here. Absence is unstarted, never pending or failed. */
|
|
36
|
+
export function decodeRestoreReceipt(value: unknown): RestoreReceipt | undefined {
|
|
37
|
+
if (value === undefined) return undefined;
|
|
38
|
+
if (value === "pending" || value === "failed") return { state: value };
|
|
39
|
+
if (!object(value)) malformed();
|
|
40
|
+
if (Object.hasOwn(value, "state")) {
|
|
41
|
+
const allowed = value.state === "waiting" ? ["state", "origin", "binding"] : value.state === "done" ? ["state", "binding"] : ["state"];
|
|
42
|
+
if (Object.keys(value).some(key => !allowed.includes(key))) malformed();
|
|
43
|
+
switch (value.state) {
|
|
44
|
+
case "pending": case "failed": return { state: value.state };
|
|
45
|
+
case "waiting": {
|
|
46
|
+
if (value.origin !== "pending" && value.origin !== "failed") malformed();
|
|
47
|
+
return { state: "waiting", origin: value.origin, binding: terminalBinding(value.binding) };
|
|
48
|
+
}
|
|
49
|
+
case "done": return { state: "done", binding: terminalBinding(value.binding) };
|
|
50
|
+
default: return malformed();
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (Object.hasOwn(value, "attachmentReceipt")) malformed();
|
|
54
|
+
return { state: "done", binding: terminalBinding(value) };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function restoreReceiptStatus(value: unknown): RestoreReceiptStatus {
|
|
58
|
+
const receipt = decodeRestoreReceipt(value);
|
|
59
|
+
return !receipt ? "unstarted" : receipt.state === "waiting" ? `waiting:${receipt.origin}` : receipt.state;
|
|
60
|
+
}
|
|
61
|
+
export function canChooseFresh(value: unknown): boolean {
|
|
62
|
+
const receipt = decodeRestoreReceipt(value);
|
|
63
|
+
return receipt?.state === "failed" || receipt?.state === "waiting" && receipt.origin === "failed";
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Pure decision only: settlement/revalidation must succeed before the runtime persists its proposed next receipt. */
|
|
67
|
+
export function restoreTransition(receipt: RestoreReceipt, evidence: RestoreEvidence): RestoreTransition {
|
|
68
|
+
const keep = (action: RestoreAction): RestoreTransition => ({ next: receipt, action });
|
|
69
|
+
if (evidence.target !== "running") return keep("block");
|
|
70
|
+
// A chosen waiting terminal cannot be replaced by another identity, even if a session has attached meanwhile.
|
|
71
|
+
if (receipt.state === "waiting" && ["unknown", "replaced", "older-instance"].includes(evidence.launcher)) return keep("block");
|
|
72
|
+
if (evidence.session === "other" && (!evidence.allowOther || receipt.state === "done")) return keep("block");
|
|
73
|
+
if (evidence.session === "planned" || evidence.session === "other" && evidence.allowOther) {
|
|
74
|
+
if (receipt.state === "done") return keep("revalidate");
|
|
75
|
+
const binding = receipt.state === "waiting" ? receipt.binding : evidence.binding;
|
|
76
|
+
return binding ? { next: { state: "done", binding }, action: "settle" } : keep("block");
|
|
77
|
+
}
|
|
78
|
+
if (receipt.state === "done") return keep("block");
|
|
79
|
+
if (evidence.launcher === "unknown" || evidence.launcher === "replaced" || evidence.launcher === "older-instance") return keep("block");
|
|
80
|
+
if (receipt.state === "waiting" && (evidence.launcher === "live" || evidence.launcher === "no-record-yet")) return keep("poll");
|
|
81
|
+
const origin = receipt.state === "waiting" ? receipt.origin : receipt.state;
|
|
82
|
+
if (evidence.launcher === "live") return evidence.binding
|
|
83
|
+
? { next: { state: "waiting", origin, binding: evidence.binding }, action: "poll" }
|
|
84
|
+
: keep("block");
|
|
85
|
+
// Bare pending/failed have no chosen saved terminal: never infer that an absent record is an outstanding bound wait.
|
|
86
|
+
return { next: { state: "failed" }, action: origin === "pending" ? "fail" : "create" };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Checkable compact documentation, enumerated through the same pure decision function. */
|
|
90
|
+
export function renderRestoreTransitionTable(): string {
|
|
91
|
+
const launch = { packageEntrypoint: "table", command: "table", argv: [], env: {} };
|
|
92
|
+
const binding: TerminalBinding = { peer: "codex", handle: "selected", incarnationId: "selected", worktreeId: "selected", projectRoot: "table", sessionId: "planned", launch, launchMetadata: launch };
|
|
93
|
+
const receipts: RestoreReceipt[] = [{ state: "pending" }, { state: "waiting", origin: "pending", binding }, { state: "waiting", origin: "failed", binding }, { state: "done", binding }, { state: "failed" }];
|
|
94
|
+
const launchers: RestoreEvidence["launcher"][] = ["live", "gone", "unknown", "no-record-yet", "replaced", "older-instance"];
|
|
95
|
+
const result = (receipt: RestoreReceipt, evidence: RestoreEvidence, unchanged = false) => {
|
|
96
|
+
const next = restoreTransition(receipt, evidence);
|
|
97
|
+
const status = restoreReceiptStatus(next.next);
|
|
98
|
+
return `${next.action} -> ${unchanged && status === restoreReceiptStatus(receipt) ? "unchanged" : status}${canChooseFresh(next.next) && !(unchanged && status === restoreReceiptStatus(receipt)) ? "; fresh" : ""}`;
|
|
99
|
+
};
|
|
100
|
+
const table = (title: string, sessions: readonly { session: RestoreEvidence["session"]; allowOther: boolean }[]) => [
|
|
101
|
+
title,
|
|
102
|
+
"",
|
|
103
|
+
`| Receipt | ${launchers.join(" | ")} |`,
|
|
104
|
+
`|---|${launchers.map(() => "---").join("|")}|`,
|
|
105
|
+
...receipts.map(receipt => `| ${restoreReceiptStatus(receipt)} | ${launchers.map(launcher => {
|
|
106
|
+
const variants = sessions.map(session => ({ session: session.session, result: result(receipt, { target: "running", launcher, ...session, binding }) }));
|
|
107
|
+
const outcomes = [...new Set(variants.map(value => value.result))];
|
|
108
|
+
return outcomes.length === 1 ? outcomes[0]! : [...new Set(variants.map(value => `${value.session}: ${value.result}`))].join("; ");
|
|
109
|
+
}).join(" | ")} |`),
|
|
110
|
+
].join("\n");
|
|
111
|
+
const priority = (targets: readonly RestoreEvidence["target"][], sessions: readonly { session: RestoreEvidence["session"]; allowOther: boolean }[]) => {
|
|
112
|
+
const outcomes = new Set<string>();
|
|
113
|
+
for (const receipt of receipts) for (const target of targets) for (const launcher of launchers) for (const session of sessions) {
|
|
114
|
+
outcomes.add(result(receipt, { target, launcher, ...session, binding }, true));
|
|
115
|
+
}
|
|
116
|
+
return [...outcomes].join(" / ");
|
|
117
|
+
};
|
|
118
|
+
const allSessions = (["planned", "other", "none"] as const).flatMap(session => [false, true].map(allowOther => ({ session, allowOther })));
|
|
119
|
+
return [
|
|
120
|
+
"Cells show action -> next receipt status. Fresh-session eligibility is marked fresh; peer/operation-specific restrictions still apply. Exact terminal revalidation and settlement must succeed before persisting a proposed done receipt.",
|
|
121
|
+
"",
|
|
122
|
+
"| Priority condition | Decision |",
|
|
123
|
+
"|---|---|",
|
|
124
|
+
`| Target stopped or changed | ${priority(["stopped", "changed"], allSessions)} |`,
|
|
125
|
+
`| Running target, other session not allowed | ${priority(["running"], [{ session: "other", allowOther: false }])} |`,
|
|
126
|
+
"| Absent receipt | unstarted; initial launch is handled by the runtime outside this receipt table |",
|
|
127
|
+
"",
|
|
128
|
+
table("Running target, no attached session (either allowOther value):", [{ session: "none", allowOther: false }, { session: "none", allowOther: true }]),
|
|
129
|
+
"",
|
|
130
|
+
table("Running target, planned session or explicitly allowed other session (different decisions are labeled):", [{ session: "planned", allowOther: false }, { session: "planned", allowOther: true }, { session: "other", allowOther: true }]),
|
|
131
|
+
].join("\n");
|
|
132
|
+
}
|
|
133
|
+
export const RESTORE_TRANSITION_TABLE = renderRestoreTransitionTable();
|
|
@@ -8,16 +8,23 @@ import { ControlClient, PROTOCOL, RECOVERY_SOURCE_PROTOCOLS, readControl } from
|
|
|
8
8
|
import { awaitStopped, inspectProject } from "../hub/lifecycle.ts";
|
|
9
9
|
import type { Project } from "../hub/registry.ts";
|
|
10
10
|
import { hubHome } from "../hub/project.ts";
|
|
11
|
-
import { packageDigest, registryRelease, runCommand, stageRelease, verifyPackage, type RunCommand } from "./recovery-package.ts";
|
|
11
|
+
import { packageDigest, preserveRuntimeDependencies, registryRelease, runCommand, stageRelease, verifyPackage, type RunCommand } from "./recovery-package.ts";
|
|
12
12
|
import { inspectTerminals, closeTerminal, createTerminal, launcherOf, recordPath, waitForIdle, shellQuote, type SessionRef, type TerminalBinding, type TerminalRecoveryOptions } from "./terminal-recovery.ts";
|
|
13
13
|
import { FinalRefusal, planFingerprint, registeredProjects, targetReadsWaivers, type Inspection, type PlannedProject, type ProjectProgress, type RecoveryDriver, type RecoveryOperation, type RecoveryPeer, type UpgradePlan } from "./upgrade.ts";
|
|
14
14
|
import { readEvents } from "../hub/events.ts";
|
|
15
15
|
import { refreshManager } from "../hub/manager.ts";
|
|
16
16
|
import { abandonRestartSnapshot, readRestartSnapshot, waiveRecoveryPeers } from "../hub/restart.ts";
|
|
17
17
|
import { recoveryLock } from "../hub/recovery-store.ts";
|
|
18
|
+
import { decodeRestoreReceipt, restoreTransition, type RestoreReceipt, type RestoreEvidence, type RestoreTransition } from "./upgrade-restore-state.ts";
|
|
18
19
|
|
|
19
20
|
/** An unmanaged Claude's plugin retries with a backoff of at most 30 s; three of those bound the reconnect wait (#206). */
|
|
20
|
-
const
|
|
21
|
+
const UNMANAGED_RECONNECT_WAIT_MS = 90_000;
|
|
22
|
+
/** Managed TUI startup can include resume and MCP handshakes; match createTerminal's ten-minute readiness ceiling. */
|
|
23
|
+
const RECONNECT_WAIT_MS = 600_000;
|
|
24
|
+
class AttachmentLauncherGone extends Error {
|
|
25
|
+
constructor(readonly transition: RestoreTransition) { super("chosen attachment launcher exited"); }
|
|
26
|
+
}
|
|
27
|
+
const attachedBinding = (binding: TerminalBinding, sessionId: string): TerminalBinding => ({ ...binding, sessionId });
|
|
21
28
|
|
|
22
29
|
export const PACKAGE_ROOT = resolve(import.meta.dir, "../..");
|
|
23
30
|
const orcaExecutable = () => process.env.ORCA_CLI_COMMAND || (process.env.ORCA_DEV_REPO_ROOT ? "orca-dev" : process.platform === "linux" && !process.env.ORCA_TERMINAL_HANDLE ? "orca-ide" : "orca");
|
|
@@ -151,7 +158,7 @@ export async function endPlannedPeer(planned: PlannedProject, peer: RecoveryPeer
|
|
|
151
158
|
}
|
|
152
159
|
}
|
|
153
160
|
|
|
154
|
-
export async function inspectRecovery(project: Project): Promise<Inspection> {
|
|
161
|
+
export async function inspectRecovery(project: Project, targetFeatures?: readonly string[]): Promise<Inspection> {
|
|
155
162
|
// #226: the claim is judged by the registry as it is now, never by the plan's frozen copy, which holds the source's
|
|
156
163
|
// pid from planning time (after a reboot another process may hold it). A registry that cannot be read throws: the
|
|
157
164
|
// caller reads that as not inspected, never as stopped.
|
|
@@ -189,9 +196,9 @@ export async function inspectRecovery(project: Project): Promise<Inspection> {
|
|
|
189
196
|
}
|
|
190
197
|
if (!status || !control) return { state: "unavailable", peers: [], blockers: ["authenticated source status unavailable"] };
|
|
191
198
|
let response: any;
|
|
192
|
-
try { response = await rpc(project, { t: "recovery", op: "inspect", expectedInstanceId: status.instanceId }, sourceProtocol ?? PROTOCOL); }
|
|
199
|
+
try { response = await rpc(project, { t: "recovery", op: "inspect", expectedInstanceId: status.instanceId, ...(targetFeatures === undefined ? {} : { targetFeatures }) }, sourceProtocol ?? PROTOCOL); }
|
|
193
200
|
catch (error) {
|
|
194
|
-
if (isDatabaseBusy(error)) throw error;
|
|
201
|
+
if (isDatabaseBusy(error) || targetFeatures !== undefined) throw error;
|
|
195
202
|
// A stop can complete between authenticated status and the second read. Never infer stopped
|
|
196
203
|
// from a connection failure: the coordinator will inspect the ownership manifest again.
|
|
197
204
|
return { state: "unavailable", instanceId: status.instanceId, version: status.version, protocol: status.protocol,
|
|
@@ -307,7 +314,7 @@ export function preserveSource(plan: UpgradePlan, home = hubHome()): string {
|
|
|
307
314
|
const base = join(home, "releases");
|
|
308
315
|
const dest = join(base, `source-${plan.sourceDigest}`);
|
|
309
316
|
if (existsSync(dest)) {
|
|
310
|
-
if (packageDigest(dest) !== plan.sourceDigest) throw new Error("preserved source package changed");
|
|
317
|
+
if (packageDigest(dest, { ownDependencies: true }) !== plan.sourceDigest) throw new Error("preserved source package changed");
|
|
311
318
|
return dest;
|
|
312
319
|
}
|
|
313
320
|
mkdirSync(base, { recursive: true, mode: 0o700 });
|
|
@@ -317,15 +324,16 @@ export function preserveSource(plan: UpgradePlan, home = hubHome()): string {
|
|
|
317
324
|
for (const name of ["src", "plugins", "templates", ".claude-plugin", "package.json", "README.md", "LICENSE", "CHANGELOG.md"]) {
|
|
318
325
|
cpSync(join(plan.sourceRoot, name), join(temporary, name), { recursive: true, dereference: false });
|
|
319
326
|
}
|
|
320
|
-
|
|
327
|
+
preserveRuntimeDependencies(plan.sourceRoot, temporary);
|
|
328
|
+
if (packageDigest(temporary, { ownDependencies: true }) !== plan.sourceDigest) throw new Error("source changed while being preserved");
|
|
321
329
|
renameSync(temporary, dest);
|
|
322
330
|
} finally { rmSync(temporary, { recursive: true, force: true }); }
|
|
323
331
|
return dest;
|
|
324
332
|
}
|
|
325
333
|
|
|
326
|
-
export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?: string): RecoveryDriver {
|
|
327
|
-
const now = () => Date.now();
|
|
328
|
-
const sleep = (ms: number) => Bun.sleep(ms);
|
|
334
|
+
export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?: string, clock?: Pick<RecoveryDriver, "now" | "sleep">): RecoveryDriver {
|
|
335
|
+
const now = clock?.now ?? (() => Date.now());
|
|
336
|
+
const sleep = clock?.sleep ?? ((ms: number) => Bun.sleep(ms));
|
|
329
337
|
const env = (op: RecoveryOperation, project?: Project): NodeJS.ProcessEnv => {
|
|
330
338
|
const value = { ...process.env };
|
|
331
339
|
// Do not carry one project's gateway, auth overrides or unattended mode into another.
|
|
@@ -351,10 +359,10 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
351
359
|
if (result.code !== 0) throw new Error(`target command ${args[0]} failed; inspect recovery state before retrying`);
|
|
352
360
|
return result;
|
|
353
361
|
};
|
|
354
|
-
const control = (project: Project, op: string, id: string, instance: string) => {
|
|
362
|
+
const control = (project: Project, op: string, id: string, instance: string, targetFeatures?: readonly string[]) => {
|
|
355
363
|
const sourceProtocol = readControl(project.stateDir)?.protocol;
|
|
356
364
|
const protocol = sourceProtocol !== undefined && RECOVERY_SOURCE_PROTOCOLS.includes(sourceProtocol as (typeof RECOVERY_SOURCE_PROTOCOLS)[number]) ? sourceProtocol : PROTOCOL;
|
|
357
|
-
return rpc(project, { t: "recovery", op, operationId: id, expectedInstanceId: instance }, protocol).then(() => {});
|
|
365
|
+
return rpc(project, { t: "recovery", op, operationId: id, expectedInstanceId: instance, ...(targetFeatures === undefined ? {} : { targetFeatures }) }, protocol).then(() => {});
|
|
358
366
|
};
|
|
359
367
|
const revalidateTerminal = async (planned: PlannedProject, progress: ProjectProgress, saved: TerminalBinding, exact: boolean): Promise<TerminalBinding> => {
|
|
360
368
|
const options = { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId };
|
|
@@ -386,7 +394,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
386
394
|
* expected instance can report "gone"; an unreadable target or launcher is unknown and blocks, never gone.
|
|
387
395
|
*/
|
|
388
396
|
const peerEvidence = async (planned: PlannedProject, progress: ProjectProgress, peer: TerminalBinding["peer"]):
|
|
389
|
-
Promise<{ state: "live"; session?: string; handle?: string; attached?: true } | { state: "gone" } | { state: "unknown"; why: string; step?: string }> => {
|
|
397
|
+
Promise<{ state: "live"; session?: string; handle?: string; attached?: true; terminal?: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId"> } | { state: "gone"; terminal?: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId"> } | { state: "unknown"; why: string; step?: string; idless?: true }> => {
|
|
390
398
|
const target = await inspectRecovery(planned.project);
|
|
391
399
|
if (target.state !== "running" || target.instanceId !== progress.instanceId) {
|
|
392
400
|
// Only a hub that may still answer is waited for; a stopped, missing or other one is settled by the runner's list.
|
|
@@ -396,22 +404,31 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
396
404
|
const attached = target.peers.find((p) => p.id === peer && p.state !== "offline");
|
|
397
405
|
const session = peer === "codex" ? attached?.threadId : attached?.sessionId;
|
|
398
406
|
if (session) return { state: "live", session };
|
|
407
|
+
let endedTerminal: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId"> | undefined;
|
|
399
408
|
// #225: the launch records of every instance this operation started for the project, the current one first: a
|
|
400
409
|
// launcher a dead instance recorded may still run (read before the check below, so a block names its terminal).
|
|
401
410
|
for (const instanceId of [progress.instanceId, ...(progress.restarts ?? []).map((r) => r.instanceId)]) {
|
|
402
411
|
if (!instanceId) continue;
|
|
403
412
|
const launch = await launcherOf(peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId });
|
|
404
|
-
if (launch?.state === "live")
|
|
413
|
+
if (launch?.state === "live") {
|
|
414
|
+
const record = launch.record!;
|
|
415
|
+
return { state: "live", handle: record.handle, ...(attached ? { attached: true as const } : {}),
|
|
416
|
+
...(instanceId === progress.instanceId ? { terminal: { handle: record.handle, incarnationId: record.incarnationId, worktreeId: record.worktreeId } } : {}) };
|
|
417
|
+
}
|
|
405
418
|
if (launch?.state === "unknown") {
|
|
406
419
|
return launch.record ? { state: "unknown", why: `its launcher in terminal ${launch.record.handle} cannot be read`, step: `wait until it attaches, or end it and close terminal ${launch.record.handle}` }
|
|
407
420
|
: launch.invalidRow ? { state: "unknown", why: "a launch record that may be its launcher's cannot be evaluated", step: `inspect ${recordPath(planned.project.stateDir)} and fix that row, or end that launcher and remove its row` }
|
|
408
421
|
: { state: "unknown", why: "the launcher records cannot be read", step: `inspect ${recordPath(planned.project.stateDir)} and move it aside (sessions it recorded then count as unmanaged)` };
|
|
409
422
|
}
|
|
423
|
+
if (instanceId === progress.instanceId && launch?.state === "gone" && launch.record) {
|
|
424
|
+
const record = launch.record;
|
|
425
|
+
endedTerminal = { handle: record.handle, incarnationId: record.incarnationId, worktreeId: record.worktreeId };
|
|
426
|
+
}
|
|
410
427
|
}
|
|
411
428
|
// #225: attached with no provable session id (a Claude of a dead instance whose plugin reconnected, or one that has
|
|
412
429
|
// not reported its id yet) is a session of unknown identity, never "nothing attached".
|
|
413
|
-
if (attached) return { state: "unknown", why: `a ${peer} session is attached to the target without a session id`, step: `wait until it reports one, or end that ${peer} session` };
|
|
414
|
-
return { state: "gone" };
|
|
430
|
+
if (attached) return { state: "unknown", idless: true, why: `a ${peer} session is attached to the target without a session id`, step: `wait until it reports one, or end that ${peer} session` };
|
|
431
|
+
return { state: "gone", ...(endedTerminal ? { terminal: endedTerminal } : {}) };
|
|
415
432
|
};
|
|
416
433
|
/**
|
|
417
434
|
* #225: the terminal of a done restoration of a target instance that died is closed under the close receipt rules
|
|
@@ -434,34 +451,104 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
434
451
|
if (result.manualRequired) throw new Error(`${peer}: closing terminal ${retired.handle} of the stopped target's session was not verified; manual-required`);
|
|
435
452
|
progress.terminals[key] = true; save();
|
|
436
453
|
};
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
454
|
+
/** #414: an idle terminal is not an authenticated peer. Keep its receipt while waiting, including on resume. */
|
|
455
|
+
const awaitAttachment = async (planned: PlannedProject, progress: ProjectProgress, op: RecoveryOperation, waiting: Extract<RestoreReceipt, { state: "waiting" }>, save: () => void, expectedSession: string | null = waiting.binding.sessionId): Promise<string> => {
|
|
456
|
+
const binding = waiting.binding;
|
|
457
|
+
const matchesTerminal = (terminal: Pick<TerminalBinding, "handle" | "incarnationId" | "worktreeId">) => terminal.handle === binding.handle && terminal.incarnationId === binding.incarnationId && terminal.worktreeId === binding.worktreeId;
|
|
458
|
+
const deadline = now() + RECONNECT_WAIT_MS;
|
|
459
|
+
let live: Inspection;
|
|
460
|
+
for (;;) {
|
|
461
|
+
live = await inspectRecovery(planned.project);
|
|
462
|
+
if (live.state !== "running") throw new Error(`${binding.peer}: target hub reads as ${live.state} while awaiting attachment; saved terminal retained`);
|
|
463
|
+
if (live.instanceId !== progress.instanceId || live.version !== op.plan.version || live.recovery?.operationId !== op.id) throw new Error(`${binding.peer}: target identity changed while awaiting attachment`);
|
|
464
|
+
const peer = live.peers.find((p) => p.id === binding.peer && p.state !== "offline");
|
|
465
|
+
const session = binding.peer === "codex" ? peer?.threadId : peer?.sessionId;
|
|
466
|
+
const changed = !!session && expectedSession !== null && session !== expectedSession;
|
|
467
|
+
// #64: only the original zero-turn Claude may change identity without an operator choice. Re-read persistence
|
|
468
|
+
// at attachment, not at the plan or wait start; an already accepted replacement is bound to its own id.
|
|
469
|
+
const zeroTurn = changed && binding.peer === "claude" && (planned.terminals as TerminalBinding[]).some((p) => p.peer === "claude" && p.sessionId === binding.sessionId) && !claudeTranscriptExists(binding);
|
|
470
|
+
if (changed && !zeroTurn) throw new Error(`${binding.peer}: a different conversation attached while awaiting ${binding.sessionId}`);
|
|
471
|
+
if (now() > deadline) break;
|
|
472
|
+
const launcher = await launcherOf(binding.peer, planned.project.root, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId });
|
|
473
|
+
const sameTerminal = launcher?.record && matchesTerminal(launcher.record);
|
|
474
|
+
let launchState: RestoreEvidence["launcher"] = !launcher ? "no-record-yet" : launcher.state === "unknown" || !launcher.record ? "unknown"
|
|
475
|
+
: sameTerminal ? launcher.state : launcher.state === "gone" ? "no-record-yet" : "replaced";
|
|
476
|
+
let olderHandle: string | undefined;
|
|
477
|
+
if ((launchState === "gone" || launchState === "no-record-yet") && !session) {
|
|
478
|
+
// An exact exit is not proof while an older launcher or an id-less session may still run.
|
|
479
|
+
const evidence = await peerEvidence(planned, progress, binding.peer);
|
|
480
|
+
if (evidence.state === "gone" && evidence.terminal && matchesTerminal(evidence.terminal)) launchState = "gone";
|
|
481
|
+
else if (evidence.state === "unknown") {
|
|
482
|
+
if (!(launchState === "no-record-yet" && evidence.idless)) launchState = "unknown";
|
|
483
|
+
} else if (evidence.state === "live" && !evidence.session) {
|
|
484
|
+
if (evidence.terminal) launchState = matchesTerminal(evidence.terminal) ? "live" : "replaced";
|
|
485
|
+
else { launchState = "older-instance"; olderHandle = evidence.handle; }
|
|
486
|
+
} else if (evidence.state === "live" && evidence.session) {
|
|
487
|
+
if (now() >= deadline) break;
|
|
488
|
+
await sleep(Math.min(1000, deadline - now()));
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
const decision = restoreTransition(waiting, { target: "running", launcher: launchState,
|
|
493
|
+
session: session && peer && ["idle", "busy", "paused"].includes(peer.state) ? changed ? "other" : "planned" : "none", allowOther: zeroTurn || expectedSession === null, binding });
|
|
494
|
+
if (decision.action === "settle" && session) {
|
|
495
|
+
if (zeroTurn) waiveRecoveryPeers(planned.project.stateDir, op.id, { claude: "zero-turn" });
|
|
496
|
+
return session;
|
|
497
|
+
}
|
|
498
|
+
if (decision.action === "fail" || decision.action === "create") throw new AttachmentLauncherGone(decision);
|
|
499
|
+
if (decision.action === "block") {
|
|
500
|
+
if (launchState === "replaced") throw new Error(`${binding.peer}: saved terminal identity changed while awaiting attachment`);
|
|
501
|
+
if (launchState === "older-instance") throw new Error(`${binding.peer}: an older target's launcher in terminal ${olderHandle ?? "unknown"} may still run; saved terminal retained; no terminal was created`);
|
|
502
|
+
throw new Error(`${binding.peer}: the chosen restoration launcher's identity cannot be verified (${launchState}); saved terminal retained; no terminal was created`);
|
|
503
|
+
}
|
|
504
|
+
if (now() >= deadline) break;
|
|
505
|
+
await sleep(Math.min(1000, deadline - now()));
|
|
444
506
|
}
|
|
507
|
+
const terminal: NonNullable<ProjectProgress["attachmentTimeouts"]>[string]["terminal"] = { handle: binding.handle, incarnationId: binding.incarnationId, state: "unknown" };
|
|
508
|
+
try {
|
|
509
|
+
const result = await run([orcaExecutable(), "terminal", "show", "--terminal", binding.handle, "--json"], { timeoutMs: 3000 });
|
|
510
|
+
const value = JSON.parse(result.stdout), shown = value.result?.terminal;
|
|
511
|
+
if (result.code === 0 && value.ok === true && shown?.handle === binding.handle && shown.incarnationId === binding.incarnationId) {
|
|
512
|
+
if (["idle", "busy", "exited"].includes(shown.state)) terminal.state = shown.state;
|
|
513
|
+
if (typeof shown.connected === "boolean") terminal.connected = shown.connected;
|
|
514
|
+
if (typeof shown.orphaned === "boolean") terminal.orphaned = shown.orphaned;
|
|
515
|
+
}
|
|
516
|
+
} catch { /* unavailable metadata stays unknown; raw output/errors never enter the receipt */ }
|
|
517
|
+
const peer = live.peers.find((p) => p.id === binding.peer);
|
|
518
|
+
const session = peer?.state === "offline" ? undefined : binding.peer === "codex" ? peer?.threadId : peer?.sessionId;
|
|
519
|
+
(progress.attachmentTimeouts ??= {})[binding.peer] = { at: now(), terminal,
|
|
520
|
+
hub: { state: live.state, ...(live.instanceId ? { instanceId: live.instanceId } : {}), ...(peer ? { peerState: peer.state } : {}), ...(session ? { sessionId: session } : {}) },
|
|
521
|
+
lastHubConnectAttempt: "unknown" };
|
|
522
|
+
save();
|
|
523
|
+
throw new Error(`${binding.peer}: attachment not verified within ${RECONNECT_WAIT_MS / 1000} s; terminal ${binding.handle} retained; resume after it attaches (native-start diagnostic recorded; last hub connect attempt unknown)`);
|
|
445
524
|
};
|
|
446
525
|
return {
|
|
447
526
|
now, sleep, inspect: inspectRecovery,
|
|
448
527
|
stage: async (op) => {
|
|
449
|
-
if (packageDigest(op.sourceRoot) !== op.plan.sourceDigest) throw new FinalRefusal("preserved source changed");
|
|
450
|
-
const target = op.plan.kind === "restart" ? { root: op.sourceRoot, digest: verifyPackage(op.sourceRoot, op.plan.version) }
|
|
528
|
+
if (packageDigest(op.sourceRoot, { ownDependencies: true }) !== op.plan.sourceDigest) throw new FinalRefusal("preserved source changed");
|
|
529
|
+
const target = op.plan.kind === "restart" ? { root: op.sourceRoot, digest: verifyPackage(op.sourceRoot, op.plan.version, { ownDependencies: true }) }
|
|
451
530
|
: await stageRelease(op.plan.version, op.plan.integrity!, run);
|
|
452
531
|
const identityPath = join(target.root, "src/cli/identity.ts");
|
|
453
532
|
if (liveTestPeer && (!existsSync(identityPath) || !readFileSync(identityPath, "utf8").includes("export const LIVE_TEST_SUPPORTED = true;"))) {
|
|
454
533
|
throw new FinalRefusal("target does not support live-test delegation; choose a supporting target before retrying; runtimes unchanged");
|
|
455
534
|
}
|
|
456
|
-
//
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
//
|
|
463
|
-
|
|
464
|
-
|
|
535
|
+
// Reuse capability proof only after verifying the cached root and digest still identify these exact bytes.
|
|
536
|
+
const cached = op.targetDigest === target.digest && op.targetRoot === target.root;
|
|
537
|
+
if (cached && op.targetFeatures !== undefined) return { ...target, targetFeatures: [...op.targetFeatures] };
|
|
538
|
+
// A legacy cached operation already past commit needs no capability proof to finish restoration (#215).
|
|
539
|
+
// A prepared commitSent receipt remains uncertain and must still obtain proof before a possible commit.
|
|
540
|
+
if (cached && op.projects?.every(project => ["stopped", "started", "peers-restored", "verified"].includes(project.phase))) return target;
|
|
541
|
+
// Legacy uncommitted receipts acquire verified feature evidence before source compatibility checks.
|
|
542
|
+
const probe = await run([process.execPath, "-e", `import * as target from ${JSON.stringify(join(target.root, "src/hub/control-client.ts"))}; console.log(JSON.stringify({protocol: target.PROTOCOL, features: target.DAEMON_FEATURES ?? []}))`]);
|
|
543
|
+
let metadata: { protocol?: unknown; features?: unknown } | undefined;
|
|
544
|
+
try { if (probe.code === 0) metadata = JSON.parse(probe.stdout); } catch { /* Unknown metadata refuses before effects. */ }
|
|
545
|
+
if (!metadata || typeof metadata !== "object" || !Number.isInteger(metadata.protocol)) throw new Error(`the staged target's control protocol could not be read (probe exit ${probe.code}); staged package retained, runtimes unchanged; resume again once it can run`);
|
|
546
|
+
if (!Array.isArray(metadata.features) || !metadata.features.every((feature: unknown) => typeof feature === "string" && feature.length > 0)) throw new Error("the staged target features could not be read; staged package retained, runtimes unchanged");
|
|
547
|
+
const targetFeatures: readonly string[] = [...metadata.features];
|
|
548
|
+
if (metadata.protocol !== PROTOCOL) throw new FinalRefusal("target protocol requires a newer coordinator; staged package retained, runtimes unchanged");
|
|
549
|
+
const verified = { ...target, targetFeatures };
|
|
550
|
+
// Existing peer/waiver compatibility checks remain properties of the already verified bytes.
|
|
551
|
+
if (op.targetDigest && op.targetDigest === target.digest && op.targetRoot === target.root) return verified;
|
|
465
552
|
if (op.plan.projects.some(p => p.source.peers.some(peer => peer.id === "qwen" && peer.state !== "offline"))) {
|
|
466
553
|
const catalog = join(target.root, "src/hub/peer-catalog.ts");
|
|
467
554
|
const peers = existsSync(catalog) ? /export const OPTIONAL_PEERS\s*=\s*\[([^\]]*)\]/.exec(readFileSync(catalog, "utf8"))?.[1] : undefined;
|
|
@@ -476,11 +563,11 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
476
563
|
if (op.plan.projects.some((p) => p.reconnectOnly?.length || p.freshStart?.length) && !targetReadsWaivers({ targetRoot: target.root })) {
|
|
477
564
|
throw new FinalRefusal(`target ${op.plan.version} predates recovery waivers, so its hub could never accept a reconnect-only Claude or a fresh Codex start; staged package retained, runtimes unchanged; choose a newer target once this operation is cancelled or ended`);
|
|
478
565
|
}
|
|
479
|
-
return
|
|
566
|
+
return verified;
|
|
480
567
|
},
|
|
481
|
-
prepare: (p, id, instance) => control(p, "prepare", id, instance),
|
|
568
|
+
prepare: (p, id, instance, features) => control(p, "prepare", id, instance, features),
|
|
482
569
|
abort: (p, id, instance) => control(p, "abort", id, instance),
|
|
483
|
-
commit: (p, id, instance) => control(p, "commit", id, instance),
|
|
570
|
+
commit: (p, id, instance, features) => control(p, "commit", id, instance, features),
|
|
484
571
|
release: (p, id, instance) => control(p, "release", id, instance),
|
|
485
572
|
closeTerminals: async (planned, progress, op, save) => {
|
|
486
573
|
for (const binding of planned.terminals as TerminalBinding[]) {
|
|
@@ -595,19 +682,35 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
595
682
|
: evidence.session
|
|
596
683
|
? `${original.peer}: session ${evidence.session} is attached instead of ${original.sessionId}; no terminal was created; end that ${original.peer} session and close its terminal first`
|
|
597
684
|
: `${original.peer}: its launcher in terminal ${evidence.handle} is running but no ${original.peer} session ${evidence.attached ? "with an id " : ""}attached yet; no terminal was created; wait until it attaches, or end it and close terminal ${evidence.handle} first`);
|
|
598
|
-
const
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
685
|
+
const decide = (receipt: RestoreReceipt, evidence: Awaited<ReturnType<typeof peerEvidence>>) => restoreTransition(receipt, {
|
|
686
|
+
target: "running", launcher: evidence.state === "unknown" ? "unknown" : evidence.state === "gone" ? "gone" : evidence.terminal ? "live" : "older-instance",
|
|
687
|
+
session: evidence.state === "live" && evidence.session ? evidence.session === original.sessionId ? "planned" : "other" : "none",
|
|
688
|
+
allowOther: fresh() || (original.peer === "claude" && !claudeTranscriptExists(original)),
|
|
689
|
+
binding: evidence.state === "live" && evidence.terminal ? { ...original, ...evidence.terminal } : original,
|
|
690
|
+
});
|
|
691
|
+
const receipt = decodeRestoreReceipt(progress.terminals[key]);
|
|
692
|
+
function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }> & { origin: "pending" }, expected: string | null): Promise<string>;
|
|
693
|
+
function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined>;
|
|
694
|
+
async function awaitBound(waiting: Extract<RestoreReceipt, { state: "waiting" }>, expected: string | null): Promise<string | undefined> {
|
|
695
|
+
try { return await awaitAttachment(planned, progress, op, waiting, save, expected); }
|
|
696
|
+
catch (error) {
|
|
697
|
+
if (!(error instanceof AttachmentLauncherGone)) throw error;
|
|
698
|
+
progress.terminals[key] = error.transition.next; save();
|
|
699
|
+
// The table decides whether this failed wait blocks or is eligible for a new launch.
|
|
700
|
+
if (error.transition.action === "create") return undefined;
|
|
701
|
+
throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
|
|
702
|
+
}
|
|
602
703
|
}
|
|
603
704
|
// #215: the pending row of the receipt table, used wherever a pending launch meets an attached session.
|
|
604
|
-
const settle = async (attached: string) => {
|
|
705
|
+
const settle = async (attached: string, recovered?: TerminalBinding) => {
|
|
605
706
|
// #21: a Claude session that never got a first turn has no transcript, so `claude --resume S` can never
|
|
606
707
|
// succeed and accepting its fresh attach loses nothing. Otherwise only the planned id, or a new session
|
|
607
708
|
// the operator or the plan accepted, counts as restored.
|
|
608
709
|
if (attached !== original.sessionId && !fresh() && !(original.peer === "claude" && !claudeTranscriptExists(original))) {
|
|
609
710
|
throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${attached} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
|
|
610
711
|
}
|
|
712
|
+
// Validate the proposed transition before committing continuity/waiver effects or a done receipt.
|
|
713
|
+
const verified = await revalidateTerminal(planned, progress, attachedBinding(recovered ?? original, attached), !!recovered);
|
|
611
714
|
// The planned session came back after all: a fresh session chosen for it lost nothing. Compared with the plan,
|
|
612
715
|
// never with a session accepted after a restart (#225), whose choice must stay on record.
|
|
613
716
|
if (attached === plannedBinding.sessionId && progress.fresh?.[original.peer]) {
|
|
@@ -618,31 +721,58 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
618
721
|
// a launch writes the waiver first, but this must not rely on that.
|
|
619
722
|
// The planned session needs no waiver: the target's snapshot already expects it.
|
|
620
723
|
if (attached !== original.sessionId && attached !== plannedBinding.sessionId) waiveRecoveryPeers(planned.project.stateDir, op.id, { [original.peer]: waiver() });
|
|
621
|
-
progress.terminals[key] =
|
|
724
|
+
progress.terminals[key] = { state: "done", binding: verified } satisfies RestoreReceipt; save();
|
|
622
725
|
};
|
|
623
|
-
|
|
624
|
-
|
|
726
|
+
if (receipt?.state === "done") {
|
|
727
|
+
// A completed restoration is never a waiting launch. A closed terminal blocks exact revalidation;
|
|
728
|
+
// it cannot erase continuity or create a fresh-session choice (PROBE-B).
|
|
729
|
+
const observed = await peerEvidence(planned, progress, original.peer);
|
|
730
|
+
const decision = restoreTransition(receipt, { target: "running", launcher: "no-record-yet", session: observed.state === "live" && observed.session === receipt.binding.sessionId ? "planned" : observed.state === "live" && observed.session ? "other" : "none", allowOther: false });
|
|
731
|
+
if (decision.action === "block") {
|
|
732
|
+
const condition = observed.state === "unknown" ? `cannot be verified (${observed.why})`
|
|
733
|
+
: observed.state === "live" && observed.session ? `session ${observed.session} is attached instead of ${receipt.binding.sessionId}`
|
|
734
|
+
: observed.state === "live" ? `is not attached; its launcher in terminal ${observed.handle ?? "unknown"} may still run` : "is not attached";
|
|
735
|
+
throw new Error(`${original.peer}: completed restoration ${condition}; saved done receipt retained; resume its recorded conversation`);
|
|
736
|
+
}
|
|
737
|
+
progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, receipt.binding, true) } satisfies RestoreReceipt; save();
|
|
738
|
+
continue;
|
|
739
|
+
}
|
|
740
|
+
if (receipt?.state === "waiting") {
|
|
741
|
+
const binding = receipt.binding;
|
|
742
|
+
const sessionId = await awaitBound(receipt, fresh() && binding.sessionId === original.sessionId ? null : binding.sessionId);
|
|
743
|
+
if (sessionId !== undefined) { await settle(sessionId, binding); continue; }
|
|
744
|
+
}
|
|
745
|
+
if (receipt?.state === "pending" || receipt?.state === "failed") {
|
|
625
746
|
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
626
|
-
|
|
627
|
-
if (evidence.state
|
|
628
|
-
if (
|
|
629
|
-
|
|
747
|
+
const decision = decide(receipt, evidence);
|
|
748
|
+
if (decision.action === "settle" && evidence.state === "live" && evidence.session) { await settle(evidence.session); continue; }
|
|
749
|
+
if (decision.action === "poll" && decision.next.state === "waiting") {
|
|
750
|
+
const waiting = decision.next;
|
|
751
|
+
progress.terminals[key] = waiting; save();
|
|
752
|
+
const session = await awaitBound(waiting, fresh() ? null : original.sessionId);
|
|
753
|
+
if (session !== undefined) { await settle(session, waiting.binding); continue; }
|
|
754
|
+
} else if (decision.action === "fail") {
|
|
755
|
+
progress.terminals[key] = decision.next; save();
|
|
630
756
|
throw notRestored(`its launcher no longer runs and no ${original.peer} session attached`);
|
|
757
|
+
} else if (decision.action === "block") {
|
|
758
|
+
if (evidence.state === "live" && evidence.session) {
|
|
759
|
+
throw new Error(`${original.peer}: terminal creation outcome is uncertain: session ${evidence.session} is attached instead of ${original.sessionId}; end that ${original.peer} session and close its terminal first`);
|
|
760
|
+
}
|
|
761
|
+
if (evidence.state !== "gone") throw blocked(evidence);
|
|
762
|
+
throw new Error(`${original.peer}: restoration evidence is not actionable`);
|
|
631
763
|
}
|
|
632
|
-
// failed, nothing live: launched again below; the receipt stays `failed` until `pending` replaces it, so a
|
|
633
|
-
// refusal on the way keeps the failed-restoration choices
|
|
634
764
|
}
|
|
635
765
|
// Never create beside a live launch, nor on evidence that could not be read: read now, not at restore start.
|
|
636
766
|
// The planned session already attached is the restoration (the table's row for it), not a reason to block.
|
|
637
767
|
const evidence = await peerEvidence(planned, progress, original.peer);
|
|
638
768
|
if (evidence.state === "live" && evidence.session === original.sessionId) {
|
|
639
|
-
progress.terminals[key] = await revalidateTerminal(planned, progress, original, false); save();
|
|
769
|
+
progress.terminals[key] = { state: "done", binding: await revalidateTerminal(planned, progress, original, false) } satisfies RestoreReceipt; save();
|
|
640
770
|
continue;
|
|
641
771
|
}
|
|
642
772
|
if (evidence.state !== "gone") throw blocked(evidence);
|
|
643
773
|
if (transcript() === "unknown") throw new Error(`${unresumable(original)}; no terminal was created; make that store readable first`);
|
|
644
774
|
if (transcript() === "missing" && !fresh()) {
|
|
645
|
-
progress.terminals[key] = "failed"; save();
|
|
775
|
+
progress.terminals[key] = { state: "failed" } satisfies RestoreReceipt; save();
|
|
646
776
|
throw notRestored(unresumable(original).slice("codex: ".length));
|
|
647
777
|
}
|
|
648
778
|
if (fresh()) {
|
|
@@ -656,7 +786,7 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
656
786
|
const launch = { ...original.launch, packageEntrypoint: entrypoint, argv,
|
|
657
787
|
command: ["env", ...Object.entries(assignments).map(([k, v]) => `${k}=${v}`), ...argv].map(shellQuote).join(" ") };
|
|
658
788
|
const binding = { ...original, launch, launchMetadata: launch };
|
|
659
|
-
progress.terminals[key] = "pending"; save();
|
|
789
|
+
progress.terminals[key] = { state: "pending" } satisfies RestoreReceipt; save();
|
|
660
790
|
const restored = await createTerminal(binding, { ...terminalOptions(run), stateDir: planned.project.stateDir, instanceId: progress.instanceId }, undefined, fresh());
|
|
661
791
|
const exited = restored.blockers.find((b) => b.code === "launcher-exited");
|
|
662
792
|
if (exited) {
|
|
@@ -664,23 +794,34 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
664
794
|
// nothing live makes it failed; an attached session is settled as the pending row settles it; anything else
|
|
665
795
|
// stays pending.
|
|
666
796
|
const after = await peerEvidence(planned, progress, original.peer);
|
|
667
|
-
|
|
797
|
+
const decision = decide({ state: "pending" }, after);
|
|
798
|
+
if (decision.action === "settle" && after.state === "live" && after.session) { await settle(after.session); continue; }
|
|
799
|
+
if (decision.action === "poll" && decision.next.state === "waiting") {
|
|
800
|
+
progress.terminals[key] = decision.next; save();
|
|
801
|
+
const session = await awaitBound(decision.next, fresh() ? null : original.sessionId);
|
|
802
|
+
if (session !== undefined) { await settle(session, decision.next.binding); continue; }
|
|
803
|
+
}
|
|
804
|
+
if (decision.action === "fail") {
|
|
805
|
+
progress.terminals[key] = decision.next; save();
|
|
806
|
+
throw notRestored(exited.message);
|
|
807
|
+
}
|
|
668
808
|
if (after.state !== "gone") throw blocked(after);
|
|
669
|
-
progress.terminals[key] = "failed"; save();
|
|
670
|
-
throw notRestored(exited.message);
|
|
671
809
|
}
|
|
672
810
|
if (restored.manualRequired || !restored.newBinding) {
|
|
673
811
|
const handle = restored.binding?.handle ?? restored.blockers[0]?.handle;
|
|
674
812
|
throw new Error(`${original.peer}: original session restoration needs manual verification${restored.blockers[0] ? ` (${restored.blockers[0].message})` : ""}; wait until it attaches, or end that launch and close its terminal${handle ? ` ${handle}` : ""} first`);
|
|
675
813
|
}
|
|
676
814
|
// The daemon's id is authoritative: a fresh session is recorded under the id it reports once attached.
|
|
677
|
-
|
|
815
|
+
const awaiting = { state: "waiting", origin: "pending", binding: restored.newBinding } satisfies RestoreReceipt;
|
|
816
|
+
progress.terminals[key] = awaiting; save();
|
|
817
|
+
const sessionId = await awaitBound(awaiting, fresh() ? null : restored.newBinding.sessionId);
|
|
818
|
+
await settle(sessionId, awaiting.binding);
|
|
678
819
|
}
|
|
679
820
|
if (group === "claude") for (const id of planned.reconnectOnly ?? []) {
|
|
680
821
|
// #206: nothing is launched for an unmanaged session; its plugin reconnects by itself, within a bound.
|
|
681
|
-
const deadline = now() +
|
|
822
|
+
const deadline = now() + UNMANAGED_RECONNECT_WAIT_MS;
|
|
682
823
|
while (!(await inspectRecovery(planned.project)).peers.some((p) => p.id === id && p.state !== "offline")) {
|
|
683
|
-
if (now() >= deadline) throw new Error(`${id}: the unmanaged session did not reconnect within ${
|
|
824
|
+
if (now() >= deadline) throw new Error(`${id}: the unmanaged session did not reconnect within ${UNMANAGED_RECONNECT_WAIT_MS / 1000} s; if that Claude session was closed, start Claude in the project again (its plugin reconnects) first`);
|
|
684
825
|
await sleep(1000);
|
|
685
826
|
}
|
|
686
827
|
}
|
|
@@ -734,9 +875,9 @@ export function makeRecoveryDriver(run: RunCommand = runCommand, liveTestPeer?:
|
|
|
734
875
|
if (old.id === "pi" && (peer.sessionId !== old.sessionId || (old.sessionFile && peer.sessionFile !== old.sessionFile))) throw new Error("Pi resumed a different conversation");
|
|
735
876
|
}
|
|
736
877
|
for (const original of planned.terminals as TerminalBinding[]) {
|
|
737
|
-
const saved = progress.terminals[`restored:${original.peer}`];
|
|
738
|
-
if (
|
|
739
|
-
await revalidateTerminal(planned, progress, saved
|
|
878
|
+
const saved = decodeRestoreReceipt(progress.terminals[`restored:${original.peer}`]);
|
|
879
|
+
if (saved?.state !== "done") throw new Error(`${original.peer}: restored terminal outcome is not recorded; manual-required`);
|
|
880
|
+
await revalidateTerminal(planned, progress, saved.binding, true);
|
|
740
881
|
}
|
|
741
882
|
const snapshot = await rpc(planned.project, { t: "ui_snapshot", after: 0 });
|
|
742
883
|
if (snapshot.ok === false) throw new Error("dashboard snapshot readback failed");
|