@neta-art/cohub-cli 8.0.0 → 8.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/runtime.js +53 -53
- package/dist/commands/sandboxd-binary.d.ts +1 -1
- package/dist/commands/sandboxd-binary.js +7 -1
- package/dist/runtime/archive-store.d.ts +3 -1
- package/dist/runtime/archive-store.js +2 -0
- package/dist/runtime/connection.js +34 -22
- package/dist/runtime/harness.d.ts +4 -0
- package/dist/runtime/harness.js +29 -16
- package/dist/runtime/instance.js +5 -5
- package/dist/runtime/json-rpc.d.ts +2 -0
- package/dist/runtime/json-rpc.js +2 -0
- package/dist/runtime/launch.js +22 -22
- package/dist/runtime/native-codex-hook.js +3 -3
- package/dist/runtime/native-install.js +9 -9
- package/dist/runtime/native-ipc.js +4 -4
- package/dist/runtime/native-pi-extension.js +1 -1
- package/dist/runtime/native-sync-store.js +19 -19
- package/dist/runtime/native-sync.js +5 -5
- package/dist/runtime/native-transcript.js +13 -13
- package/dist/runtime/presentation.js +31 -29
- package/dist/runtime/process-group.d.ts +2 -0
- package/dist/runtime/process-group.js +124 -31
- package/dist/runtime/session-store.d.ts +15 -2
- package/dist/runtime/session-store.js +79 -5
- package/dist/runtime/space-binding.js +3 -3
- package/dist/runtime/supervisor.js +4 -4
- package/package.json +1 -1
|
@@ -3,26 +3,28 @@ export const diagnosticLevels = ["debug", "info", "warn", "error"];
|
|
|
3
3
|
export const atLeastLevel = (level, minimum) => diagnosticLevels.indexOf(level) >= diagnosticLevels.indexOf(minimum);
|
|
4
4
|
export const runtimeWebUrl = (spaceId) => `https://${resolveCohubEnvironment() === "prod" ? "" : "dev."}cohub.live/spaces/${spaceId}`;
|
|
5
5
|
const messages = {
|
|
6
|
-
"runtime.ready": "Harness connected
|
|
7
|
-
"runtime.available": "Runtime ready
|
|
8
|
-
"runtime.websocket.closed": "Connection lost; reconnecting
|
|
9
|
-
"runtime.heartbeat_timeout": "Connection timed out; reconnecting
|
|
10
|
-
"runtime.auth_token_failed": "Cannot obtain credentials; retrying
|
|
11
|
-
"runtime.auth_required": "Sign in with cohub auth login
|
|
12
|
-
"runtime.stopped": "Runtime stopped
|
|
13
|
-
"runtime.failed": "Runtime needs attention
|
|
14
|
-
"runtime.turn_failed": "Turn failed; local files retained
|
|
15
|
-
"runtime.
|
|
16
|
-
"runtime.
|
|
17
|
-
"
|
|
18
|
-
"
|
|
19
|
-
"archive.
|
|
20
|
-
"
|
|
21
|
-
"archive.
|
|
22
|
-
"
|
|
23
|
-
"
|
|
24
|
-
"sandboxd.
|
|
25
|
-
"sandboxd.
|
|
6
|
+
"runtime.ready": "Harness connected",
|
|
7
|
+
"runtime.available": "Runtime ready",
|
|
8
|
+
"runtime.websocket.closed": "Connection lost; reconnecting",
|
|
9
|
+
"runtime.heartbeat_timeout": "Connection timed out; reconnecting",
|
|
10
|
+
"runtime.auth_token_failed": "Cannot obtain credentials; retrying",
|
|
11
|
+
"runtime.auth_required": "Sign in with cohub auth login",
|
|
12
|
+
"runtime.stopped": "Runtime stopped",
|
|
13
|
+
"runtime.failed": "Runtime needs attention",
|
|
14
|
+
"runtime.turn_failed": "Turn failed; local files retained",
|
|
15
|
+
"runtime.turn_cleanup_pending": "Turn result saved; tool cleanup still unconfirmed",
|
|
16
|
+
"runtime.connection_failed": "Connection attempt failed; retrying",
|
|
17
|
+
"runtime.execution_transport_detached": "Execution continues locally; result replays after reconnect",
|
|
18
|
+
"runtime.execution_transport_invalidated": "Execution interrupted; outcome needs reconciliation",
|
|
19
|
+
"archive.upload_pending": "Archive upload pending; local data retained",
|
|
20
|
+
"native.sync_pending": "Native sync pending; local records retained",
|
|
21
|
+
"archive.capture_pending": "Archive capture pending",
|
|
22
|
+
"archive.capture_unavailable": "Archive unavailable; original receipt retained",
|
|
23
|
+
"archive.restore_failed": "Native restore unavailable; using saved history",
|
|
24
|
+
"sandboxd.process_exit": "File bridge stopped; restarting",
|
|
25
|
+
"sandboxd.download": "Preparing file bridge",
|
|
26
|
+
"sandboxd.connected": "File bridge connected",
|
|
27
|
+
"sandboxd.disconnected": "File bridge disconnected; reconnecting",
|
|
26
28
|
};
|
|
27
29
|
export function formatDiagnostic(event, verbose = false) {
|
|
28
30
|
const text = messages[event.event] ?? (typeof event.data?.message === "string" ? event.data.message : event.event);
|
|
@@ -46,7 +48,7 @@ export function createDiagnosticConsole(verbose = false, write = (line) => proce
|
|
|
46
48
|
if (last.size >= 256)
|
|
47
49
|
last.delete(last.keys().next().value ?? "");
|
|
48
50
|
last.set(key, { at: now, suppressed: 0 });
|
|
49
|
-
const repeated = previous?.suppressed ? ` (+${previous.suppressed} repeated
|
|
51
|
+
const repeated = previous?.suppressed ? ` (+${previous.suppressed} repeated)` : "";
|
|
50
52
|
write(`${formatDiagnostic(event, verbose).trimEnd()}${repeated}\n`);
|
|
51
53
|
};
|
|
52
54
|
}
|
|
@@ -56,21 +58,21 @@ export function printRuntimeSummary(summary, json = false, reused = false) {
|
|
|
56
58
|
process.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
|
|
57
59
|
return;
|
|
58
60
|
}
|
|
59
|
-
const label = summary.state === "ready" ? "Runtime ready
|
|
60
|
-
process.stdout.write(`\n${label}${reused ? " · reused
|
|
61
|
+
const label = summary.state === "ready" ? "Runtime ready" : `Runtime ${summary.state}`;
|
|
62
|
+
process.stdout.write(`\n${label}${reused ? " · reused" : ""}\n\n`);
|
|
61
63
|
const rows = [
|
|
62
|
-
["Space
|
|
63
|
-
["URL
|
|
64
|
-
["Directory
|
|
64
|
+
["Space", summary.spaceId],
|
|
65
|
+
["URL", value.url],
|
|
66
|
+
["Directory", summary.root],
|
|
65
67
|
["Harness", summary.harnesses.join(" · ")],
|
|
66
|
-
["Mode
|
|
68
|
+
["Mode", summary.background ? "Background" : "Foreground"],
|
|
67
69
|
["PID", String(summary.pid)],
|
|
68
|
-
["Logs
|
|
70
|
+
["Logs", summary.diagnosticsPath],
|
|
69
71
|
];
|
|
70
72
|
for (const [name, text] of rows)
|
|
71
73
|
process.stdout.write(` ${name} ${text}\n`);
|
|
72
74
|
process.stdout.write(`\n cohub runtime logs --space ${summary.spaceId} --follow\n cohub runtime down --space ${summary.spaceId}\n`);
|
|
73
75
|
if (!summary.background && !reused)
|
|
74
|
-
process.stdout.write(" Ctrl+C to stop
|
|
76
|
+
process.stdout.write(" Ctrl+C to stop\n");
|
|
75
77
|
process.stdout.write("\n");
|
|
76
78
|
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
export declare class ProcessCleanupUncertainError extends Error {
|
|
2
2
|
}
|
|
3
|
+
/** True only when the process group is gone or holds nothing that can execute. */
|
|
4
|
+
export declare function confirmQuiescentProcessGroup(pid: number): Promise<boolean>;
|
|
3
5
|
export declare function stopProcessGroup(pid: number): Promise<void>;
|
|
@@ -3,43 +3,143 @@ import { setTimeout as delay } from "node:timers/promises";
|
|
|
3
3
|
import { spawn } from "node:child_process";
|
|
4
4
|
export class ProcessCleanupUncertainError extends Error {
|
|
5
5
|
}
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
if (error.code === "ESRCH")
|
|
12
|
-
return false;
|
|
13
|
-
throw error;
|
|
14
|
-
}
|
|
15
|
-
if (process.platform !== "linux")
|
|
16
|
-
return true;
|
|
17
|
-
// Linux can retain reparented zombies: they cannot execute, but kill(0) still sees them.
|
|
6
|
+
/** A member that can still execute code. Zombies cannot, so they never block cleanup. */
|
|
7
|
+
const executable = (member) => member.state !== "Z" && member.state !== "X";
|
|
8
|
+
/** Linux: /proc/<pid>/stat exposes pgrp and state for every member of the group. */
|
|
9
|
+
async function probeLinux(pid) {
|
|
10
|
+
const members = [];
|
|
18
11
|
for (const entry of await readdir("/proc")) {
|
|
19
12
|
if (!/^\d+$/.test(entry))
|
|
20
13
|
continue;
|
|
21
14
|
try {
|
|
22
15
|
const stat = await readFile(`/proc/${entry}/stat`, "utf8");
|
|
23
16
|
// After removing `pid (comm)`: state, ppid, pgrp, session, ...
|
|
24
|
-
const [state,
|
|
25
|
-
if (
|
|
26
|
-
|
|
17
|
+
const [state, , processGroupId] = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
|
|
18
|
+
if (processGroupId !== undefined && Number(processGroupId) === pid)
|
|
19
|
+
members.push({ pid: Number(entry), state: state ?? "U" });
|
|
27
20
|
}
|
|
28
21
|
catch (error) {
|
|
29
|
-
|
|
22
|
+
// The member exited between readdir and read; it cannot execute anymore.
|
|
23
|
+
if (error.code !== "ENOENT")
|
|
30
24
|
throw error;
|
|
31
25
|
}
|
|
32
26
|
}
|
|
33
|
-
return
|
|
27
|
+
return members.some(executable) ? { state: "alive", members } : { state: "quiescent", members };
|
|
28
|
+
}
|
|
29
|
+
/** macOS/BSD: one `ps` snapshot; a zombie's STAT starts with Z. */
|
|
30
|
+
async function probePosixSnapshot(pid) {
|
|
31
|
+
const output = await new Promise((resolve, reject) => {
|
|
32
|
+
const task = spawn("ps", ["-A", "-o", "pid=,pgid=,stat="], { stdio: ["ignore", "pipe", "ignore"] });
|
|
33
|
+
let text = "";
|
|
34
|
+
task.stdout.on("data", (chunk) => { text += chunk.toString(); });
|
|
35
|
+
task.once("error", reject);
|
|
36
|
+
task.once("close", (code) => code === 0 ? resolve(text) : reject(new Error(`ps exited ${code}`)));
|
|
37
|
+
});
|
|
38
|
+
const members = [];
|
|
39
|
+
for (const line of output.split("\n")) {
|
|
40
|
+
const [processId, processGroupId, stat] = line.trim().split(/\s+/);
|
|
41
|
+
if (processGroupId !== undefined && Number(processGroupId) === pid)
|
|
42
|
+
members.push({ pid: Number(processId), state: stat?.[0] ?? "U" });
|
|
43
|
+
}
|
|
44
|
+
return members.some(executable) ? { state: "alive", members } : { state: "quiescent", members };
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* `kill(-pgid)` outcomes that carry no liveness information on their own.
|
|
48
|
+
* ESRCH: the group is gone. EPERM: xnu's killpg1() reports it when no member can be
|
|
49
|
+
* signalled anymore (exited or zombie), so on macOS it is the normal end state; our
|
|
50
|
+
* harnesses run as the same user, so a real permission denial cannot occur here.
|
|
51
|
+
*/
|
|
52
|
+
const signalCode = (error) => error.code;
|
|
53
|
+
const groupUnsignallable = (error) => signalCode(error) === "ESRCH" || signalCode(error) === "EPERM";
|
|
54
|
+
/**
|
|
55
|
+
* Windows has no process groups: cleanup kills the tree with taskkill, so the leader's
|
|
56
|
+
* absence is the practical quiescence signal. A reused PID can only keep a result
|
|
57
|
+
* uncertain, never wrongly confirm it.
|
|
58
|
+
*/
|
|
59
|
+
async function probeWindows(pid) {
|
|
60
|
+
try {
|
|
61
|
+
process.kill(pid, 0);
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
if (signalCode(error) === "ESRCH")
|
|
65
|
+
return { state: "quiescent", members: [] };
|
|
66
|
+
return { state: "unknown", cause: error };
|
|
67
|
+
}
|
|
68
|
+
return { state: "alive", members: [{ pid, state: "R" }] };
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* A group is alive only while a member can still execute. `kill(0)` is only a fast
|
|
72
|
+
* path; the platform snapshot is the source of truth, and only a failing snapshot is
|
|
73
|
+
* uncertain.
|
|
74
|
+
*/
|
|
75
|
+
async function probeProcessGroup(pid) {
|
|
76
|
+
if (process.platform === "win32")
|
|
77
|
+
return probeWindows(pid);
|
|
78
|
+
try {
|
|
79
|
+
process.kill(-pid, 0);
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
if (signalCode(error) === "ESRCH")
|
|
83
|
+
return { state: "quiescent", members: [] };
|
|
84
|
+
if (signalCode(error) !== "EPERM")
|
|
85
|
+
return { state: "unknown", cause: error };
|
|
86
|
+
}
|
|
87
|
+
if (process.platform === "linux") {
|
|
88
|
+
try {
|
|
89
|
+
return await probeLinux(pid);
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
return { state: "unknown", cause: error };
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
try {
|
|
96
|
+
return await probePosixSnapshot(pid);
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
return { state: "unknown", cause: error };
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const describeMembers = (members) => members.map((member) => `${member.pid}:${member.state}`).join(" ") || "none";
|
|
103
|
+
/** True only when the process group is gone or holds nothing that can execute. */
|
|
104
|
+
export async function confirmQuiescentProcessGroup(pid) {
|
|
105
|
+
const probe = await probeProcessGroup(pid);
|
|
106
|
+
return probe.state === "quiescent";
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Poll until the probe confirms quiescence. The interval backs off from 25ms to 200ms:
|
|
110
|
+
* normal exits confirm on the first probe, while a resistant group stops spawning
|
|
111
|
+
* snapshot subprocesses at full rate.
|
|
112
|
+
*/
|
|
113
|
+
async function awaitQuiescence(pid, probe, escalate) {
|
|
114
|
+
const started = Date.now();
|
|
115
|
+
let escalated = false;
|
|
116
|
+
let pollMs = 25;
|
|
117
|
+
for (;;) {
|
|
118
|
+
const state = await probe(pid);
|
|
119
|
+
if (state.state === "quiescent")
|
|
120
|
+
return;
|
|
121
|
+
if (state.state === "unknown" || Date.now() - started >= 5000) {
|
|
122
|
+
throw new Error(state.state === "unknown" ? `probe failed: ${state.cause instanceof Error ? state.cause.message : String(state.cause)}` : `still running: ${describeMembers(state.members)}`);
|
|
123
|
+
}
|
|
124
|
+
if (escalate && !escalated && Date.now() - started >= 2000) {
|
|
125
|
+
escalate();
|
|
126
|
+
escalated = true;
|
|
127
|
+
}
|
|
128
|
+
await delay(pollMs);
|
|
129
|
+
pollMs = Math.min(pollMs * 2, 200);
|
|
130
|
+
}
|
|
34
131
|
}
|
|
35
132
|
export async function stopProcessGroup(pid) {
|
|
36
133
|
try {
|
|
37
134
|
if (process.platform === "win32") {
|
|
38
|
-
|
|
135
|
+
// taskkill's exit code is not authoritative: a PID that already exited is an
|
|
136
|
+
// "error". The request is sent, then liveness decides.
|
|
137
|
+
await new Promise((resolve) => {
|
|
39
138
|
const task = spawn("taskkill", ["/PID", String(pid), "/T", "/F"], { stdio: "ignore", timeout: 5000 });
|
|
40
|
-
task.once("error",
|
|
41
|
-
task.once("exit", (
|
|
139
|
+
task.once("error", () => resolve());
|
|
140
|
+
task.once("exit", () => resolve());
|
|
42
141
|
});
|
|
142
|
+
await awaitQuiescence(pid, probeWindows);
|
|
43
143
|
return;
|
|
44
144
|
}
|
|
45
145
|
const signal = (value) => {
|
|
@@ -47,22 +147,15 @@ export async function stopProcessGroup(pid) {
|
|
|
47
147
|
process.kill(-pid, value);
|
|
48
148
|
}
|
|
49
149
|
catch (error) {
|
|
50
|
-
if (error
|
|
150
|
+
if (!groupUnsignallable(error))
|
|
51
151
|
throw error;
|
|
52
152
|
}
|
|
53
153
|
};
|
|
54
154
|
signal("SIGTERM");
|
|
55
|
-
|
|
56
|
-
while (await groupAlive(pid)) {
|
|
57
|
-
const elapsed = Date.now() - started;
|
|
58
|
-
if (elapsed >= 5000)
|
|
59
|
-
throw new Error("Process group is still running");
|
|
60
|
-
if (elapsed >= 2000)
|
|
61
|
-
signal("SIGKILL");
|
|
62
|
-
await delay(25);
|
|
63
|
-
}
|
|
155
|
+
await awaitQuiescence(pid, probeProcessGroup, () => signal("SIGKILL"));
|
|
64
156
|
}
|
|
65
157
|
catch (cause) {
|
|
66
|
-
|
|
158
|
+
// Any cleanup failure leaves the outcome unresolved; callers treat it as uncertain.
|
|
159
|
+
throw new ProcessCleanupUncertainError(`Tool process cleanup could not be confirmed; execution remains unresolved (${cause instanceof Error ? cause.message : String(cause)})`, { cause });
|
|
67
160
|
}
|
|
68
161
|
}
|
|
@@ -7,7 +7,8 @@ export declare class ContextRequiredError extends Error {
|
|
|
7
7
|
}
|
|
8
8
|
export type RuntimeSessionStoreOptions = {
|
|
9
9
|
stateRoot?: string;
|
|
10
|
-
|
|
10
|
+
/** Explicit, so a serve path can never silently lose archive uploads. */
|
|
11
|
+
archiveTransport: ArchiveTransport | null;
|
|
11
12
|
projectionSource: SessionTurnProjectionClient;
|
|
12
13
|
};
|
|
13
14
|
export type NativeSession = {
|
|
@@ -61,11 +62,23 @@ export declare class RuntimeSessionStore {
|
|
|
61
62
|
}>;
|
|
62
63
|
started(state: NativeSession, turnId: string): Promise<void>;
|
|
63
64
|
private resultPath;
|
|
64
|
-
recordResult(state: NativeSession, requestId: string, events: RuntimeExecutionEvent[]
|
|
65
|
+
recordResult(state: NativeSession, requestId: string | null, events: RuntimeExecutionEvent[], options?: {
|
|
66
|
+
uncertainCleanup?: {
|
|
67
|
+
processGroupId: number;
|
|
68
|
+
error: string;
|
|
69
|
+
};
|
|
70
|
+
}): Promise<void>;
|
|
65
71
|
recoverResult(input: Pick<RuntimeTurnInput, "sessionId" | "harness" | "turnId">, requestId?: string): Promise<{
|
|
66
72
|
state: NativeSession;
|
|
67
73
|
events: RuntimeExecutionEvent[];
|
|
68
74
|
} | null>;
|
|
75
|
+
/**
|
|
76
|
+
* Receipt-less recovery: the native file is the authority. When a Turn completed but no
|
|
77
|
+
* receipt survived (a crash between native completion and recordResult), rebuild the
|
|
78
|
+
* result from transcript bytes appended after the Turn started, without ever rewriting
|
|
79
|
+
* native data. Anything ambiguous stays unrecovered.
|
|
80
|
+
*/
|
|
81
|
+
private reconstructResult;
|
|
69
82
|
archive(state: NativeSession, turnId: string, diagnosticContext?: RuntimeDiagnosticContext): Promise<HarnessArchive | null>;
|
|
70
83
|
acknowledge(state: NativeSession, turnId: string, revision: string): Promise<void>;
|
|
71
84
|
}
|
|
@@ -7,6 +7,8 @@ import { RuntimeArchiveStore, checksumNativeFile, atomicRuntimeJson as atomicJso
|
|
|
7
7
|
import { importNativeArchive, readCodexArchiveTotals } from "./native-archive.js";
|
|
8
8
|
import { serializeDiagnosticError } from "./diagnostics.js";
|
|
9
9
|
import { ProjectionStore, rebindProjectionNativeSession } from "./projection-store.js";
|
|
10
|
+
import { confirmQuiescentProcessGroup } from "./process-group.js";
|
|
11
|
+
import { readNativeTranscript } from "./native-transcript.js";
|
|
10
12
|
export class ContextRequiredError extends Error {
|
|
11
13
|
}
|
|
12
14
|
/** Reverse lookup for native clients; the existing Runtime state remains authoritative. */
|
|
@@ -35,7 +37,7 @@ export async function findRuntimeNativeSession(root, harness, nativeSessionId, p
|
|
|
35
37
|
return exact[0] ?? null;
|
|
36
38
|
}
|
|
37
39
|
if (matches.length > 1)
|
|
38
|
-
throw new Error("Native Session has ambiguous Runtime bindings
|
|
40
|
+
throw new Error("Native Session has ambiguous Runtime bindings");
|
|
39
41
|
return matches[0] ?? null;
|
|
40
42
|
}
|
|
41
43
|
const checksum = (data) => createHash("sha256").update(data).digest("hex");
|
|
@@ -280,6 +282,9 @@ export class RuntimeSessionStore {
|
|
|
280
282
|
return this.archiveFlush;
|
|
281
283
|
}
|
|
282
284
|
async flushArchiveOutbox(signal) {
|
|
285
|
+
if (!this.archives.hasTransport && await this.archives.pendingCount() > 0) {
|
|
286
|
+
this.diagnostics?.log("debug", "archive.transport_absent", { note: "captures stay local" }, { component: "archive" });
|
|
287
|
+
}
|
|
283
288
|
const captures = join(this.archives.root, "captures");
|
|
284
289
|
const names = await readdir(captures).catch((error) => { if (missing(error))
|
|
285
290
|
return []; throw error; });
|
|
@@ -450,7 +455,7 @@ export class RuntimeSessionStore {
|
|
|
450
455
|
}
|
|
451
456
|
async started(state, turnId) { state.pendingTurnId = turnId; state.resultChecksum = undefined; await atomicJson(this.statePath(state), state); }
|
|
452
457
|
resultPath(state) { return join(this.root, "results", `${state.sessionId}.${state.harness}.json`); }
|
|
453
|
-
async recordResult(state, requestId, events) {
|
|
458
|
+
async recordResult(state, requestId, events, options) {
|
|
454
459
|
if (!state.pendingTurnId)
|
|
455
460
|
throw new Error("Cannot record a result for an idle native session");
|
|
456
461
|
if (state.harness === "pi")
|
|
@@ -458,7 +463,7 @@ export class RuntimeSessionStore {
|
|
|
458
463
|
state.resultChecksum = await checksumNativeFile(state.path);
|
|
459
464
|
if (state.archivePendingTurnId)
|
|
460
465
|
await atomicJson(join(this.archives.root, "captures", `${state.archivePendingTurnId}.json`), state);
|
|
461
|
-
await atomicJson(this.resultPath(state), { requestId, state, events });
|
|
466
|
+
await atomicJson(this.resultPath(state), { requestId, state, events, ...(options?.uncertainCleanup ? { uncertainCleanup: options.uncertainCleanup } : {}) });
|
|
462
467
|
await atomicJson(this.statePath(state), state);
|
|
463
468
|
}
|
|
464
469
|
async recoverResult(input, requestId) {
|
|
@@ -467,16 +472,85 @@ export class RuntimeSessionStore {
|
|
|
467
472
|
if (saved.state.sessionId !== input.sessionId || saved.state.harness !== input.harness)
|
|
468
473
|
throw new Error("Runtime result identity mismatch");
|
|
469
474
|
if (saved.state.pendingTurnId !== input.turnId)
|
|
470
|
-
return
|
|
471
|
-
if (requestId && saved.requestId !== requestId)
|
|
475
|
+
return this.reconstructResult(input);
|
|
476
|
+
if (requestId && saved.requestId != null && saved.requestId !== requestId)
|
|
472
477
|
throw new Error("Runtime result execution identity mismatch");
|
|
478
|
+
// A quiescent process group upgrades the receipt to confirmed; a live one stays uncertain.
|
|
479
|
+
if (saved.uncertainCleanup && !saved.uncertainCleanup.confirmedAt) {
|
|
480
|
+
if (!(await confirmQuiescentProcessGroup(saved.uncertainCleanup.processGroupId)))
|
|
481
|
+
return null;
|
|
482
|
+
saved.uncertainCleanup.confirmedAt = new Date().toISOString();
|
|
483
|
+
await atomicJson(this.resultPath(input), saved);
|
|
484
|
+
}
|
|
473
485
|
return { state: saved.state, events: saved.events.map((event) => runtimeEventSchema.parse(event)) };
|
|
474
486
|
}
|
|
487
|
+
catch (error) {
|
|
488
|
+
if (missing(error))
|
|
489
|
+
return this.reconstructResult(input);
|
|
490
|
+
throw error;
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
/**
|
|
494
|
+
* Receipt-less recovery: the native file is the authority. When a Turn completed but no
|
|
495
|
+
* receipt survived (a crash between native completion and recordResult), rebuild the
|
|
496
|
+
* result from transcript bytes appended after the Turn started, without ever rewriting
|
|
497
|
+
* native data. Anything ambiguous stays unrecovered.
|
|
498
|
+
*/
|
|
499
|
+
async reconstructResult(input) {
|
|
500
|
+
const statePath = this.statePath(input);
|
|
501
|
+
let state;
|
|
502
|
+
try {
|
|
503
|
+
state = JSON.parse(await readFile(statePath, "utf8"));
|
|
504
|
+
}
|
|
475
505
|
catch (error) {
|
|
476
506
|
if (missing(error))
|
|
477
507
|
return null;
|
|
478
508
|
throw error;
|
|
479
509
|
}
|
|
510
|
+
if (state.sessionId !== input.sessionId || state.harness !== input.harness || state.pendingTurnId !== input.turnId)
|
|
511
|
+
return null;
|
|
512
|
+
let transcript;
|
|
513
|
+
try {
|
|
514
|
+
transcript = await readNativeTranscript(state.path, state.harness, { settled: state.harness === "pi" });
|
|
515
|
+
}
|
|
516
|
+
catch {
|
|
517
|
+
return null;
|
|
518
|
+
}
|
|
519
|
+
// The Turn must start at the byte boundary the file had when the Turn began.
|
|
520
|
+
const boundary = state.checksum
|
|
521
|
+
? [...transcript.prefixes].find(([, sha]) => sha === state.checksum)?.[0]
|
|
522
|
+
: 0;
|
|
523
|
+
if (boundary === undefined)
|
|
524
|
+
return null;
|
|
525
|
+
const turns = transcript.turns.filter((turn) => turn.startBytes >= boundary);
|
|
526
|
+
const completed = turns.length === 1 ? turns[0] : null;
|
|
527
|
+
const result = completed?.result ?? null;
|
|
528
|
+
// An interrupted Turn may still be executing in a surviving native process.
|
|
529
|
+
if (!completed || !result || result.status === "interrupted")
|
|
530
|
+
return null;
|
|
531
|
+
const messages = result.messages.filter((message) => message.content.length);
|
|
532
|
+
if (!messages.length)
|
|
533
|
+
return null;
|
|
534
|
+
const last = messages.at(-1);
|
|
535
|
+
const asRuntimeMessage = (message, ordinal) => ({
|
|
536
|
+
ordinal,
|
|
537
|
+
content: message.content,
|
|
538
|
+
...(message.provider != null ? { provider: message.provider } : {}),
|
|
539
|
+
...(message.model != null ? { model: message.model } : {}),
|
|
540
|
+
...(message.usage != null ? { usage: message.usage } : {}),
|
|
541
|
+
...(message.stopReason != null ? { stopReason: message.stopReason } : {}),
|
|
542
|
+
...(message.errorMessage != null ? { errorMessage: message.errorMessage } : {}),
|
|
543
|
+
});
|
|
544
|
+
const events = messages.slice(0, -1).map((message, index) => ({ type: "message.commit", message: asRuntimeMessage(message, index + 1) }));
|
|
545
|
+
const archive = await this.archive(state, input.turnId, { component: "recovery", sessionId: input.sessionId, turnId: input.turnId, harness: input.harness }).catch(() => null);
|
|
546
|
+
const final = { type: "turn.end", message: asRuntimeMessage(last, messages.length), archive, resume: "native" };
|
|
547
|
+
await this.recordResult(state, null, [...events, final]);
|
|
548
|
+
this.diagnostics?.log("warn", "runtime.result_reconstructed", {
|
|
549
|
+
startBytes: completed.startBytes,
|
|
550
|
+
messageCount: messages.length,
|
|
551
|
+
status: result.status,
|
|
552
|
+
}, { component: "recovery", sessionId: input.sessionId, turnId: input.turnId, harness: input.harness });
|
|
553
|
+
return { state, events: [...events, final] };
|
|
480
554
|
}
|
|
481
555
|
async archive(state, turnId, diagnosticContext = {}) {
|
|
482
556
|
if (!state.path)
|
|
@@ -292,7 +292,7 @@ export async function resolveRuntimeSpace(input) {
|
|
|
292
292
|
}
|
|
293
293
|
const existing = findRuntimeSpaceBinding(file, { root, key: bindingKey });
|
|
294
294
|
if (input.newSpace && (existing?.spaceId ?? null) !== (input.expectedSpaceId ?? null)) {
|
|
295
|
-
throw new RuntimeSpaceBindingsError("Directory binding changed; run up again
|
|
295
|
+
throw new RuntimeSpaceBindingsError("Directory binding changed; run up again");
|
|
296
296
|
}
|
|
297
297
|
if (existing && !input.newSpace) {
|
|
298
298
|
await input.validateSpace?.(existing.spaceId);
|
|
@@ -304,7 +304,7 @@ export async function resolveRuntimeSpace(input) {
|
|
|
304
304
|
try {
|
|
305
305
|
receipt = JSON.parse(await readFile(receiptPath, "utf8"));
|
|
306
306
|
if (!receipt || !nonEmptyString(receipt.operationId) || receipt.spaceId !== undefined && !nonEmptyString(receipt.spaceId)) {
|
|
307
|
-
throw new RuntimeSpaceBindingsError(`Invalid creation receipt; original retained
|
|
307
|
+
throw new RuntimeSpaceBindingsError(`Invalid creation receipt; original retained: ${receiptPath}`);
|
|
308
308
|
}
|
|
309
309
|
}
|
|
310
310
|
catch (error) {
|
|
@@ -312,7 +312,7 @@ export async function resolveRuntimeSpace(input) {
|
|
|
312
312
|
throw error;
|
|
313
313
|
}
|
|
314
314
|
if (receipt && !receipt.spaceId)
|
|
315
|
-
throw new RuntimeSpaceBindingsError(`A previous Space creation has an unknown outcome. Check your Spaces and use --space <id
|
|
315
|
+
throw new RuntimeSpaceBindingsError(`A previous Space creation has an unknown outcome. Check your Spaces and use --space <id>. ${receiptPath}`);
|
|
316
316
|
if (!receipt) {
|
|
317
317
|
receipt = { operationId: randomUUID() };
|
|
318
318
|
await writeRuntimeJson(receiptPath, receipt);
|
|
@@ -29,7 +29,7 @@ export async function runRuntime(config, onState, externalSignal, onDiagnostic)
|
|
|
29
29
|
process.once("SIGTERM", stop);
|
|
30
30
|
const client = createClient();
|
|
31
31
|
const space = client.space(config.spaceId);
|
|
32
|
-
const store = new RuntimeSessionStore(config.spaceId, { projectionSource: space });
|
|
32
|
+
const store = new RuntimeSessionStore(config.spaceId, { projectionSource: space, archiveTransport: space });
|
|
33
33
|
const consoleSink = config.background ? undefined : createDiagnosticConsole(config.verbose);
|
|
34
34
|
const diagnostics = new RuntimeDiagnostics({
|
|
35
35
|
root: store.root, spaceId: config.spaceId, runtimeId: randomUUID(),
|
|
@@ -67,12 +67,12 @@ export async function runRuntime(config, onState, externalSignal, onDiagnostic)
|
|
|
67
67
|
const token = (forceRefresh = false) => {
|
|
68
68
|
tokenInFlight ??= (async () => {
|
|
69
69
|
if (currentIdentityKey() !== config.identity)
|
|
70
|
-
throw new AuthRequiredError("Runtime account changed; sign in to the original account
|
|
70
|
+
throw new AuthRequiredError("Runtime account changed; sign in to the original account");
|
|
71
71
|
const value = await resolveAccessToken({ forceRefresh });
|
|
72
72
|
if (!value)
|
|
73
73
|
throw new AuthRequiredError();
|
|
74
74
|
if (currentIdentityKey() !== config.identity)
|
|
75
|
-
throw new AuthRequiredError("Runtime account changed
|
|
75
|
+
throw new AuthRequiredError("Runtime account changed");
|
|
76
76
|
return value;
|
|
77
77
|
})().finally(() => { tokenInFlight = null; });
|
|
78
78
|
return tokenInFlight;
|
|
@@ -82,7 +82,7 @@ export async function runRuntime(config, onState, externalSignal, onDiagnostic)
|
|
|
82
82
|
if (!force)
|
|
83
83
|
for await (const batch of store.pendingExecutionBatches()) {
|
|
84
84
|
if (batch.length)
|
|
85
|
-
throw new Error("Unconfirmed executions remain. Use down --yes to stop; results and files are retained
|
|
85
|
+
throw new Error("Unconfirmed executions remain. Use down --yes to stop; results and files are retained");
|
|
86
86
|
}
|
|
87
87
|
update({ state: "stopping" });
|
|
88
88
|
setTimeout(stop, 30);
|