@neta-art/cohub-cli 7.1.2 → 8.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -2
- package/dist/auth.js +38 -5
- package/dist/client.js +5 -2
- package/dist/commands/runtime.d.ts +1 -2
- package/dist/commands/runtime.js +169 -293
- package/dist/commands/sandboxd-binary.d.ts +1 -1
- package/dist/commands/sandboxd-binary.js +13 -5
- package/dist/runtime/archive-store.d.ts +5 -1
- package/dist/runtime/archive-store.js +22 -6
- package/dist/runtime/connection.d.ts +4 -2
- package/dist/runtime/connection.js +113 -44
- package/dist/runtime/diagnostics.d.ts +3 -0
- package/dist/runtime/diagnostics.js +3 -0
- package/dist/runtime/harness.d.ts +7 -0
- package/dist/runtime/harness.js +63 -17
- package/dist/runtime/instance.d.ts +5 -0
- package/dist/runtime/instance.js +159 -0
- package/dist/runtime/json-rpc.d.ts +2 -0
- package/dist/runtime/json-rpc.js +2 -0
- package/dist/runtime/launch.d.ts +20 -0
- package/dist/runtime/launch.js +176 -0
- package/dist/runtime/native-codex-hook.d.ts +1 -0
- package/dist/runtime/native-codex-hook.js +28 -0
- package/dist/runtime/native-install.d.ts +21 -0
- package/dist/runtime/native-install.js +130 -0
- package/dist/runtime/native-ipc.d.ts +26 -0
- package/dist/runtime/native-ipc.js +101 -0
- package/dist/runtime/native-pi-extension.d.ts +20 -0
- package/dist/runtime/native-pi-extension.js +47 -0
- package/dist/runtime/native-sync-store.d.ts +97 -0
- package/dist/runtime/native-sync-store.js +365 -0
- package/dist/runtime/native-sync.d.ts +25 -0
- package/dist/runtime/native-sync.js +128 -0
- package/dist/runtime/native-transcript.d.ts +27 -0
- package/dist/runtime/native-transcript.js +281 -0
- package/dist/runtime/presentation.d.ts +21 -0
- package/dist/runtime/presentation.js +78 -0
- package/dist/runtime/process-group.d.ts +2 -0
- package/dist/runtime/process-group.js +124 -31
- package/dist/runtime/session-store.d.ts +17 -2
- package/dist/runtime/session-store.js +118 -9
- package/dist/runtime/space-binding.d.ts +3 -0
- package/dist/runtime/space-binding.js +43 -6
- package/dist/runtime/supervisor.d.ts +16 -0
- package/dist/runtime/supervisor.js +277 -0
- package/dist/runtime/worker.d.ts +1 -0
- package/dist/runtime/worker.js +20 -0
- package/package.json +3 -2
|
@@ -3,43 +3,143 @@ import { setTimeout as delay } from "node:timers/promises";
|
|
|
3
3
|
import { spawn } from "node:child_process";
|
|
4
4
|
export class ProcessCleanupUncertainError extends Error {
|
|
5
5
|
}
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
if (error.code === "ESRCH")
|
|
12
|
-
return false;
|
|
13
|
-
throw error;
|
|
14
|
-
}
|
|
15
|
-
if (process.platform !== "linux")
|
|
16
|
-
return true;
|
|
17
|
-
// Linux can retain reparented zombies: they cannot execute, but kill(0) still sees them.
|
|
6
|
+
/** A member that can still execute code. Zombies cannot, so they never block cleanup. */
|
|
7
|
+
const executable = (member) => member.state !== "Z" && member.state !== "X";
|
|
8
|
+
/** Linux: /proc/<pid>/stat exposes pgrp and state for every member of the group. */
|
|
9
|
+
async function probeLinux(pid) {
|
|
10
|
+
const members = [];
|
|
18
11
|
for (const entry of await readdir("/proc")) {
|
|
19
12
|
if (!/^\d+$/.test(entry))
|
|
20
13
|
continue;
|
|
21
14
|
try {
|
|
22
15
|
const stat = await readFile(`/proc/${entry}/stat`, "utf8");
|
|
23
16
|
// After removing `pid (comm)`: state, ppid, pgrp, session, ...
|
|
24
|
-
const [state,
|
|
25
|
-
if (
|
|
26
|
-
|
|
17
|
+
const [state, , processGroupId] = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
|
|
18
|
+
if (processGroupId !== undefined && Number(processGroupId) === pid)
|
|
19
|
+
members.push({ pid: Number(entry), state: state ?? "U" });
|
|
27
20
|
}
|
|
28
21
|
catch (error) {
|
|
29
|
-
|
|
22
|
+
// The member exited between readdir and read; it cannot execute anymore.
|
|
23
|
+
if (error.code !== "ENOENT")
|
|
30
24
|
throw error;
|
|
31
25
|
}
|
|
32
26
|
}
|
|
33
|
-
return
|
|
27
|
+
return members.some(executable) ? { state: "alive", members } : { state: "quiescent", members };
|
|
28
|
+
}
|
|
29
|
+
/** macOS/BSD: one `ps` snapshot; a zombie's STAT starts with Z. */
|
|
30
|
+
async function probePosixSnapshot(pid) {
|
|
31
|
+
const output = await new Promise((resolve, reject) => {
|
|
32
|
+
const task = spawn("ps", ["-A", "-o", "pid=,pgid=,stat="], { stdio: ["ignore", "pipe", "ignore"] });
|
|
33
|
+
let text = "";
|
|
34
|
+
task.stdout.on("data", (chunk) => { text += chunk.toString(); });
|
|
35
|
+
task.once("error", reject);
|
|
36
|
+
task.once("close", (code) => code === 0 ? resolve(text) : reject(new Error(`ps exited ${code}`)));
|
|
37
|
+
});
|
|
38
|
+
const members = [];
|
|
39
|
+
for (const line of output.split("\n")) {
|
|
40
|
+
const [processId, processGroupId, stat] = line.trim().split(/\s+/);
|
|
41
|
+
if (processGroupId !== undefined && Number(processGroupId) === pid)
|
|
42
|
+
members.push({ pid: Number(processId), state: stat?.[0] ?? "U" });
|
|
43
|
+
}
|
|
44
|
+
return members.some(executable) ? { state: "alive", members } : { state: "quiescent", members };
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* `kill(-pgid)` outcomes that carry no liveness information on their own.
|
|
48
|
+
* ESRCH: the group is gone. EPERM: xnu's killpg1() reports it when no member can be
|
|
49
|
+
* signalled anymore (exited or zombie), so on macOS it is the normal end state; our
|
|
50
|
+
* harnesses run as the same user, so a real permission denial cannot occur here.
|
|
51
|
+
*/
|
|
52
|
+
const signalCode = (error) => error.code;
|
|
53
|
+
const groupUnsignallable = (error) => signalCode(error) === "ESRCH" || signalCode(error) === "EPERM";
|
|
54
|
+
/**
|
|
55
|
+
* Windows has no process groups: cleanup kills the tree with taskkill, so the leader's
|
|
56
|
+
* absence is the practical quiescence signal. A reused PID can only keep a result
|
|
57
|
+
* uncertain, never wrongly confirm it.
|
|
58
|
+
*/
|
|
59
|
+
async function probeWindows(pid) {
|
|
60
|
+
try {
|
|
61
|
+
process.kill(pid, 0);
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
if (signalCode(error) === "ESRCH")
|
|
65
|
+
return { state: "quiescent", members: [] };
|
|
66
|
+
return { state: "unknown", cause: error };
|
|
67
|
+
}
|
|
68
|
+
return { state: "alive", members: [{ pid, state: "R" }] };
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* A group is alive only while a member can still execute. `kill(0)` is only a fast
|
|
72
|
+
* path; the platform snapshot is the source of truth, and only a failing snapshot is
|
|
73
|
+
* uncertain.
|
|
74
|
+
*/
|
|
75
|
+
async function probeProcessGroup(pid) {
|
|
76
|
+
if (process.platform === "win32")
|
|
77
|
+
return probeWindows(pid);
|
|
78
|
+
try {
|
|
79
|
+
process.kill(-pid, 0);
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
if (signalCode(error) === "ESRCH")
|
|
83
|
+
return { state: "quiescent", members: [] };
|
|
84
|
+
if (signalCode(error) !== "EPERM")
|
|
85
|
+
return { state: "unknown", cause: error };
|
|
86
|
+
}
|
|
87
|
+
if (process.platform === "linux") {
|
|
88
|
+
try {
|
|
89
|
+
return await probeLinux(pid);
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
return { state: "unknown", cause: error };
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
try {
|
|
96
|
+
return await probePosixSnapshot(pid);
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
return { state: "unknown", cause: error };
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const describeMembers = (members) => members.map((member) => `${member.pid}:${member.state}`).join(" ") || "none";
|
|
103
|
+
/** True only when the process group is gone or holds nothing that can execute. */
|
|
104
|
+
export async function confirmQuiescentProcessGroup(pid) {
|
|
105
|
+
const probe = await probeProcessGroup(pid);
|
|
106
|
+
return probe.state === "quiescent";
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Poll until the probe confirms quiescence. The interval backs off from 25ms to 200ms:
|
|
110
|
+
* normal exits confirm on the first probe, while a resistant group stops spawning
|
|
111
|
+
* snapshot subprocesses at full rate.
|
|
112
|
+
*/
|
|
113
|
+
async function awaitQuiescence(pid, probe, escalate) {
|
|
114
|
+
const started = Date.now();
|
|
115
|
+
let escalated = false;
|
|
116
|
+
let pollMs = 25;
|
|
117
|
+
for (;;) {
|
|
118
|
+
const state = await probe(pid);
|
|
119
|
+
if (state.state === "quiescent")
|
|
120
|
+
return;
|
|
121
|
+
if (state.state === "unknown" || Date.now() - started >= 5000) {
|
|
122
|
+
throw new Error(state.state === "unknown" ? `probe failed: ${state.cause instanceof Error ? state.cause.message : String(state.cause)}` : `still running: ${describeMembers(state.members)}`);
|
|
123
|
+
}
|
|
124
|
+
if (escalate && !escalated && Date.now() - started >= 2000) {
|
|
125
|
+
escalate();
|
|
126
|
+
escalated = true;
|
|
127
|
+
}
|
|
128
|
+
await delay(pollMs);
|
|
129
|
+
pollMs = Math.min(pollMs * 2, 200);
|
|
130
|
+
}
|
|
34
131
|
}
|
|
35
132
|
export async function stopProcessGroup(pid) {
|
|
36
133
|
try {
|
|
37
134
|
if (process.platform === "win32") {
|
|
38
|
-
|
|
135
|
+
// taskkill's exit code is not authoritative: a PID that already exited is an
|
|
136
|
+
// "error". The request is sent, then liveness decides.
|
|
137
|
+
await new Promise((resolve) => {
|
|
39
138
|
const task = spawn("taskkill", ["/PID", String(pid), "/T", "/F"], { stdio: "ignore", timeout: 5000 });
|
|
40
|
-
task.once("error",
|
|
41
|
-
task.once("exit", (
|
|
139
|
+
task.once("error", () => resolve());
|
|
140
|
+
task.once("exit", () => resolve());
|
|
42
141
|
});
|
|
142
|
+
await awaitQuiescence(pid, probeWindows);
|
|
43
143
|
return;
|
|
44
144
|
}
|
|
45
145
|
const signal = (value) => {
|
|
@@ -47,22 +147,15 @@ export async function stopProcessGroup(pid) {
|
|
|
47
147
|
process.kill(-pid, value);
|
|
48
148
|
}
|
|
49
149
|
catch (error) {
|
|
50
|
-
if (error
|
|
150
|
+
if (!groupUnsignallable(error))
|
|
51
151
|
throw error;
|
|
52
152
|
}
|
|
53
153
|
};
|
|
54
154
|
signal("SIGTERM");
|
|
55
|
-
|
|
56
|
-
while (await groupAlive(pid)) {
|
|
57
|
-
const elapsed = Date.now() - started;
|
|
58
|
-
if (elapsed >= 5000)
|
|
59
|
-
throw new Error("Process group is still running");
|
|
60
|
-
if (elapsed >= 2000)
|
|
61
|
-
signal("SIGKILL");
|
|
62
|
-
await delay(25);
|
|
63
|
-
}
|
|
155
|
+
await awaitQuiescence(pid, probeProcessGroup, () => signal("SIGKILL"));
|
|
64
156
|
}
|
|
65
157
|
catch (cause) {
|
|
66
|
-
|
|
158
|
+
// Any cleanup failure leaves the outcome unresolved; callers treat it as uncertain.
|
|
159
|
+
throw new ProcessCleanupUncertainError(`Tool process cleanup could not be confirmed; execution remains unresolved (${cause instanceof Error ? cause.message : String(cause)})`, { cause });
|
|
67
160
|
}
|
|
68
161
|
}
|
|
@@ -7,7 +7,8 @@ export declare class ContextRequiredError extends Error {
|
|
|
7
7
|
}
|
|
8
8
|
export type RuntimeSessionStoreOptions = {
|
|
9
9
|
stateRoot?: string;
|
|
10
|
-
|
|
10
|
+
/** Explicit, so a serve path can never silently lose archive uploads. */
|
|
11
|
+
archiveTransport: ArchiveTransport | null;
|
|
11
12
|
projectionSource: SessionTurnProjectionClient;
|
|
12
13
|
};
|
|
13
14
|
export type NativeSession = {
|
|
@@ -30,6 +31,8 @@ export type NativeSession = {
|
|
|
30
31
|
sourceFingerprint?: string | null;
|
|
31
32
|
nativeLeafId?: string | null;
|
|
32
33
|
};
|
|
34
|
+
/** Reverse lookup for native clients; the existing Runtime state remains authoritative. */
|
|
35
|
+
export declare function findRuntimeNativeSession(root: string, harness: "pi" | "codex", nativeSessionId: string, path?: string): Promise<NativeSession | null>;
|
|
33
36
|
/** Local references are hints validated against actual native files, never cloud existence claims. */
|
|
34
37
|
export declare class RuntimeSessionStore {
|
|
35
38
|
readonly root: string;
|
|
@@ -59,11 +62,23 @@ export declare class RuntimeSessionStore {
|
|
|
59
62
|
}>;
|
|
60
63
|
started(state: NativeSession, turnId: string): Promise<void>;
|
|
61
64
|
private resultPath;
|
|
62
|
-
recordResult(state: NativeSession, requestId: string, events: RuntimeExecutionEvent[]
|
|
65
|
+
recordResult(state: NativeSession, requestId: string | null, events: RuntimeExecutionEvent[], options?: {
|
|
66
|
+
uncertainCleanup?: {
|
|
67
|
+
processGroupId: number;
|
|
68
|
+
error: string;
|
|
69
|
+
};
|
|
70
|
+
}): Promise<void>;
|
|
63
71
|
recoverResult(input: Pick<RuntimeTurnInput, "sessionId" | "harness" | "turnId">, requestId?: string): Promise<{
|
|
64
72
|
state: NativeSession;
|
|
65
73
|
events: RuntimeExecutionEvent[];
|
|
66
74
|
} | null>;
|
|
75
|
+
/**
|
|
76
|
+
* Receipt-less recovery: the native file is the authority. When a Turn completed but no
|
|
77
|
+
* receipt survived (a crash between native completion and recordResult), rebuild the
|
|
78
|
+
* result from transcript bytes appended after the Turn started, without ever rewriting
|
|
79
|
+
* native data. Anything ambiguous stays unrecovered.
|
|
80
|
+
*/
|
|
81
|
+
private reconstructResult;
|
|
67
82
|
archive(state: NativeSession, turnId: string, diagnosticContext?: RuntimeDiagnosticContext): Promise<HarnessArchive | null>;
|
|
68
83
|
acknowledge(state: NativeSession, turnId: string, revision: string): Promise<void>;
|
|
69
84
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
-
import { mkdir, open, readFile, readdir, rename, rm } from "node:fs/promises";
|
|
2
|
+
import { mkdir, open, readFile, readdir, realpath, rename, rm } from "node:fs/promises";
|
|
3
3
|
import { homedir } from "node:os";
|
|
4
4
|
import { dirname, join } from "node:path";
|
|
5
5
|
import { RUNTIME_RECOVERY_BATCH_SIZE, runtimeEventSchema, serializeProjectionRecords } from "@neta-art/cohub";
|
|
@@ -7,8 +7,39 @@ import { RuntimeArchiveStore, checksumNativeFile, atomicRuntimeJson as atomicJso
|
|
|
7
7
|
import { importNativeArchive, readCodexArchiveTotals } from "./native-archive.js";
|
|
8
8
|
import { serializeDiagnosticError } from "./diagnostics.js";
|
|
9
9
|
import { ProjectionStore, rebindProjectionNativeSession } from "./projection-store.js";
|
|
10
|
+
import { confirmQuiescentProcessGroup } from "./process-group.js";
|
|
11
|
+
import { readNativeTranscript } from "./native-transcript.js";
|
|
10
12
|
export class ContextRequiredError extends Error {
|
|
11
13
|
}
|
|
14
|
+
/** Reverse lookup for native clients; the existing Runtime state remains authoritative. */
|
|
15
|
+
export async function findRuntimeNativeSession(root, harness, nativeSessionId, path) {
|
|
16
|
+
const directory = join(root, harness);
|
|
17
|
+
const names = await readdir(directory).catch((error) => { if (error.code === "ENOENT")
|
|
18
|
+
return []; throw error; });
|
|
19
|
+
const matches = [];
|
|
20
|
+
for (const name of names) {
|
|
21
|
+
if (!name.endsWith(".json"))
|
|
22
|
+
continue;
|
|
23
|
+
const state = JSON.parse(await readFile(join(directory, name), "utf8"));
|
|
24
|
+
if (state.version === 1 && state.harness === harness && state.nativeSessionId === nativeSessionId)
|
|
25
|
+
matches.push(state);
|
|
26
|
+
}
|
|
27
|
+
if (path && matches.length) {
|
|
28
|
+
const canonical = await realpath(path);
|
|
29
|
+
const exact = [];
|
|
30
|
+
for (const state of matches) {
|
|
31
|
+
const candidate = await realpath(state.path).catch((error) => { if (error.code === "ENOENT")
|
|
32
|
+
return null; throw error; });
|
|
33
|
+
if (candidate === canonical)
|
|
34
|
+
exact.push(state);
|
|
35
|
+
}
|
|
36
|
+
if (exact.length === 1)
|
|
37
|
+
return exact[0] ?? null;
|
|
38
|
+
}
|
|
39
|
+
if (matches.length > 1)
|
|
40
|
+
throw new Error("Native Session has ambiguous Runtime bindings");
|
|
41
|
+
return matches[0] ?? null;
|
|
42
|
+
}
|
|
12
43
|
const checksum = (data) => createHash("sha256").update(data).digest("hex");
|
|
13
44
|
const missing = (error) => error?.code === "ENOENT";
|
|
14
45
|
class CaptureUnavailableError extends Error {
|
|
@@ -240,7 +271,6 @@ export class RuntimeSessionStore {
|
|
|
240
271
|
}
|
|
241
272
|
catch (error) {
|
|
242
273
|
this.diagnostics?.log("error", "runtime.session_state_unreadable", { path: join(directory, name), error: serializeDiagnosticError(error) });
|
|
243
|
-
console.error(`Runtime session state unreadable: ${join(directory, name)}`, error);
|
|
244
274
|
}
|
|
245
275
|
}
|
|
246
276
|
}
|
|
@@ -252,6 +282,9 @@ export class RuntimeSessionStore {
|
|
|
252
282
|
return this.archiveFlush;
|
|
253
283
|
}
|
|
254
284
|
async flushArchiveOutbox(signal) {
|
|
285
|
+
if (!this.archives.hasTransport && await this.archives.pendingCount() > 0) {
|
|
286
|
+
this.diagnostics?.log("debug", "archive.transport_absent", { note: "captures stay local" }, { component: "archive" });
|
|
287
|
+
}
|
|
255
288
|
const captures = join(this.archives.root, "captures");
|
|
256
289
|
const names = await readdir(captures).catch((error) => { if (missing(error))
|
|
257
290
|
return []; throw error; });
|
|
@@ -295,11 +328,9 @@ export class RuntimeSessionStore {
|
|
|
295
328
|
});
|
|
296
329
|
await rm(join(captures, name), { force: true });
|
|
297
330
|
this.diagnostics?.log("error", "archive.capture_unavailable", { reason: error.message, receipt: true }, { component: "archive" });
|
|
298
|
-
console.error("Archive capture unavailable; receipt retained:", error.message);
|
|
299
331
|
}
|
|
300
332
|
else {
|
|
301
333
|
this.diagnostics?.log("warn", "archive.capture_pending", { error: serializeDiagnosticError(error) }, { component: "archive" });
|
|
302
|
-
console.error("Archive capture pending:", error);
|
|
303
334
|
}
|
|
304
335
|
}
|
|
305
336
|
}
|
|
@@ -368,6 +399,16 @@ export class RuntimeSessionStore {
|
|
|
368
399
|
}
|
|
369
400
|
}
|
|
370
401
|
}
|
|
402
|
+
if (previous) {
|
|
403
|
+
const previousPath = previous.path;
|
|
404
|
+
const canonical = await realpath(previousPath).catch((error) => { if (missing(error))
|
|
405
|
+
return previousPath; throw error; });
|
|
406
|
+
const externallyOwned = await readFile(join(this.root, "native-owners", `${checksum(canonical)}.json`), "utf8").then(() => true).catch((error) => { if (missing(error))
|
|
407
|
+
return false; throw error; });
|
|
408
|
+
// Interactive clients own their native files, including symlink aliases. Use an independent projection.
|
|
409
|
+
if (externallyOwned)
|
|
410
|
+
previous = null;
|
|
411
|
+
}
|
|
371
412
|
if (previous) {
|
|
372
413
|
try {
|
|
373
414
|
const nativeChecksum = await checksumNativeFile(previous.path);
|
|
@@ -408,14 +449,13 @@ export class RuntimeSessionStore {
|
|
|
408
449
|
catch (error) {
|
|
409
450
|
signal?.throwIfAborted();
|
|
410
451
|
this.diagnostics?.log("warn", "archive.restore_failed", { error: serializeDiagnosticError(error) }, { component: "archive" });
|
|
411
|
-
console.error("Native archive unavailable; rebuilding from durable history:", error);
|
|
412
452
|
}
|
|
413
453
|
}
|
|
414
454
|
return await this.syncNativeProjection(input, cwd, state, signal);
|
|
415
455
|
}
|
|
416
456
|
async started(state, turnId) { state.pendingTurnId = turnId; state.resultChecksum = undefined; await atomicJson(this.statePath(state), state); }
|
|
417
457
|
resultPath(state) { return join(this.root, "results", `${state.sessionId}.${state.harness}.json`); }
|
|
418
|
-
async recordResult(state, requestId, events) {
|
|
458
|
+
async recordResult(state, requestId, events, options) {
|
|
419
459
|
if (!state.pendingTurnId)
|
|
420
460
|
throw new Error("Cannot record a result for an idle native session");
|
|
421
461
|
if (state.harness === "pi")
|
|
@@ -423,7 +463,7 @@ export class RuntimeSessionStore {
|
|
|
423
463
|
state.resultChecksum = await checksumNativeFile(state.path);
|
|
424
464
|
if (state.archivePendingTurnId)
|
|
425
465
|
await atomicJson(join(this.archives.root, "captures", `${state.archivePendingTurnId}.json`), state);
|
|
426
|
-
await atomicJson(this.resultPath(state), { requestId, state, events });
|
|
466
|
+
await atomicJson(this.resultPath(state), { requestId, state, events, ...(options?.uncertainCleanup ? { uncertainCleanup: options.uncertainCleanup } : {}) });
|
|
427
467
|
await atomicJson(this.statePath(state), state);
|
|
428
468
|
}
|
|
429
469
|
async recoverResult(input, requestId) {
|
|
@@ -432,16 +472,85 @@ export class RuntimeSessionStore {
|
|
|
432
472
|
if (saved.state.sessionId !== input.sessionId || saved.state.harness !== input.harness)
|
|
433
473
|
throw new Error("Runtime result identity mismatch");
|
|
434
474
|
if (saved.state.pendingTurnId !== input.turnId)
|
|
435
|
-
return
|
|
436
|
-
if (requestId && saved.requestId !== requestId)
|
|
475
|
+
return this.reconstructResult(input);
|
|
476
|
+
if (requestId && saved.requestId != null && saved.requestId !== requestId)
|
|
437
477
|
throw new Error("Runtime result execution identity mismatch");
|
|
478
|
+
// A quiescent process group upgrades the receipt to confirmed; a live one stays uncertain.
|
|
479
|
+
if (saved.uncertainCleanup && !saved.uncertainCleanup.confirmedAt) {
|
|
480
|
+
if (!(await confirmQuiescentProcessGroup(saved.uncertainCleanup.processGroupId)))
|
|
481
|
+
return null;
|
|
482
|
+
saved.uncertainCleanup.confirmedAt = new Date().toISOString();
|
|
483
|
+
await atomicJson(this.resultPath(input), saved);
|
|
484
|
+
}
|
|
438
485
|
return { state: saved.state, events: saved.events.map((event) => runtimeEventSchema.parse(event)) };
|
|
439
486
|
}
|
|
487
|
+
catch (error) {
|
|
488
|
+
if (missing(error))
|
|
489
|
+
return this.reconstructResult(input);
|
|
490
|
+
throw error;
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
/**
|
|
494
|
+
* Receipt-less recovery: the native file is the authority. When a Turn completed but no
|
|
495
|
+
* receipt survived (a crash between native completion and recordResult), rebuild the
|
|
496
|
+
* result from transcript bytes appended after the Turn started, without ever rewriting
|
|
497
|
+
* native data. Anything ambiguous stays unrecovered.
|
|
498
|
+
*/
|
|
499
|
+
async reconstructResult(input) {
|
|
500
|
+
const statePath = this.statePath(input);
|
|
501
|
+
let state;
|
|
502
|
+
try {
|
|
503
|
+
state = JSON.parse(await readFile(statePath, "utf8"));
|
|
504
|
+
}
|
|
440
505
|
catch (error) {
|
|
441
506
|
if (missing(error))
|
|
442
507
|
return null;
|
|
443
508
|
throw error;
|
|
444
509
|
}
|
|
510
|
+
if (state.sessionId !== input.sessionId || state.harness !== input.harness || state.pendingTurnId !== input.turnId)
|
|
511
|
+
return null;
|
|
512
|
+
let transcript;
|
|
513
|
+
try {
|
|
514
|
+
transcript = await readNativeTranscript(state.path, state.harness, { settled: state.harness === "pi" });
|
|
515
|
+
}
|
|
516
|
+
catch {
|
|
517
|
+
return null;
|
|
518
|
+
}
|
|
519
|
+
// The Turn must start at the byte boundary the file had when the Turn began.
|
|
520
|
+
const boundary = state.checksum
|
|
521
|
+
? [...transcript.prefixes].find(([, sha]) => sha === state.checksum)?.[0]
|
|
522
|
+
: 0;
|
|
523
|
+
if (boundary === undefined)
|
|
524
|
+
return null;
|
|
525
|
+
const turns = transcript.turns.filter((turn) => turn.startBytes >= boundary);
|
|
526
|
+
const completed = turns.length === 1 ? turns[0] : null;
|
|
527
|
+
const result = completed?.result ?? null;
|
|
528
|
+
// An interrupted Turn may still be executing in a surviving native process.
|
|
529
|
+
if (!completed || !result || result.status === "interrupted")
|
|
530
|
+
return null;
|
|
531
|
+
const messages = result.messages.filter((message) => message.content.length);
|
|
532
|
+
if (!messages.length)
|
|
533
|
+
return null;
|
|
534
|
+
const last = messages.at(-1);
|
|
535
|
+
const asRuntimeMessage = (message, ordinal) => ({
|
|
536
|
+
ordinal,
|
|
537
|
+
content: message.content,
|
|
538
|
+
...(message.provider != null ? { provider: message.provider } : {}),
|
|
539
|
+
...(message.model != null ? { model: message.model } : {}),
|
|
540
|
+
...(message.usage != null ? { usage: message.usage } : {}),
|
|
541
|
+
...(message.stopReason != null ? { stopReason: message.stopReason } : {}),
|
|
542
|
+
...(message.errorMessage != null ? { errorMessage: message.errorMessage } : {}),
|
|
543
|
+
});
|
|
544
|
+
const events = messages.slice(0, -1).map((message, index) => ({ type: "message.commit", message: asRuntimeMessage(message, index + 1) }));
|
|
545
|
+
const archive = await this.archive(state, input.turnId, { component: "recovery", sessionId: input.sessionId, turnId: input.turnId, harness: input.harness }).catch(() => null);
|
|
546
|
+
const final = { type: "turn.end", message: asRuntimeMessage(last, messages.length), archive, resume: "native" };
|
|
547
|
+
await this.recordResult(state, null, [...events, final]);
|
|
548
|
+
this.diagnostics?.log("warn", "runtime.result_reconstructed", {
|
|
549
|
+
startBytes: completed.startBytes,
|
|
550
|
+
messageCount: messages.length,
|
|
551
|
+
status: result.status,
|
|
552
|
+
}, { component: "recovery", sessionId: input.sessionId, turnId: input.turnId, harness: input.harness });
|
|
553
|
+
return { state, events: [...events, final] };
|
|
445
554
|
}
|
|
446
555
|
async archive(state, turnId, diagnosticContext = {}) {
|
|
447
556
|
if (!state.path)
|
|
@@ -39,6 +39,9 @@ export declare function resolveRuntimeSpace(input: {
|
|
|
39
39
|
root: string;
|
|
40
40
|
identityKey: string | null | undefined;
|
|
41
41
|
explicitSpaceId?: string | null;
|
|
42
|
+
/** Explicit new selection; compare the binding observed before prompting. */
|
|
43
|
+
newSpace?: boolean;
|
|
44
|
+
expectedSpaceId?: string | null;
|
|
42
45
|
createSpace: () => Promise<string>;
|
|
43
46
|
validateSpace?: (spaceId: string) => Promise<void>;
|
|
44
47
|
path?: string;
|
|
@@ -99,7 +99,7 @@ function upsertRuntimeSpaceBinding(file, binding) {
|
|
|
99
99
|
bindings[index] = nextBinding;
|
|
100
100
|
return { file: { version: 1, bindings }, changed: true };
|
|
101
101
|
}
|
|
102
|
-
async function
|
|
102
|
+
async function writeRuntimeJson(path, file) {
|
|
103
103
|
await mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
|
104
104
|
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
105
105
|
try {
|
|
@@ -248,7 +248,7 @@ async function persistRuntimeSpaceBinding(path, binding) {
|
|
|
248
248
|
const file = await readRuntimeSpaceBindings(path);
|
|
249
249
|
const result = upsertRuntimeSpaceBinding(file, binding);
|
|
250
250
|
if (result.changed)
|
|
251
|
-
await
|
|
251
|
+
await writeRuntimeJson(path, result.file);
|
|
252
252
|
}, { path, lockPath: `${path}${GLOBAL_LOCK_SUFFIX}` });
|
|
253
253
|
}
|
|
254
254
|
export async function getRuntimeSpaceBinding(root, key, path = runtimeSpaceBindingsPath()) {
|
|
@@ -281,25 +281,62 @@ export async function resolveRuntimeSpace(input) {
|
|
|
281
281
|
const lockPath = bindingLockPath(path, root, bindingKey);
|
|
282
282
|
return withRuntimeSpaceBindingsLock(async () => {
|
|
283
283
|
const file = await readRuntimeSpaceBindings(path);
|
|
284
|
+
const receiptPath = `${lockPath}.creation.json`;
|
|
284
285
|
if (explicitSpaceId) {
|
|
285
286
|
await input.validateSpace?.(explicitSpaceId);
|
|
286
287
|
await persistRuntimeSpaceBinding(path, { root, key: bindingKey, spaceId: explicitSpaceId });
|
|
288
|
+
// Preserve an ambiguous creation receipt for diagnosis, rather than deleting it.
|
|
289
|
+
if (await statIfPresent(receiptPath))
|
|
290
|
+
await rename(receiptPath, `${receiptPath}.${randomUUID()}.resolved`);
|
|
287
291
|
return { spaceId: explicitSpaceId, source: "explicit" };
|
|
288
292
|
}
|
|
289
293
|
const existing = findRuntimeSpaceBinding(file, { root, key: bindingKey });
|
|
290
|
-
if (existing) {
|
|
294
|
+
if (input.newSpace && (existing?.spaceId ?? null) !== (input.expectedSpaceId ?? null)) {
|
|
295
|
+
throw new RuntimeSpaceBindingsError("Directory binding changed; run up again");
|
|
296
|
+
}
|
|
297
|
+
if (existing && !input.newSpace) {
|
|
291
298
|
await input.validateSpace?.(existing.spaceId);
|
|
292
299
|
return { spaceId: existing.spaceId, source: "binding" };
|
|
293
300
|
}
|
|
294
|
-
//
|
|
295
|
-
//
|
|
296
|
-
|
|
301
|
+
// Fail closed across the remote-create/local-commit crash window. A saved ID
|
|
302
|
+
// resumes binding; an ambiguous request must be resolved with --space, never replayed.
|
|
303
|
+
let receipt = null;
|
|
304
|
+
try {
|
|
305
|
+
receipt = JSON.parse(await readFile(receiptPath, "utf8"));
|
|
306
|
+
if (!receipt || !nonEmptyString(receipt.operationId) || receipt.spaceId !== undefined && !nonEmptyString(receipt.spaceId)) {
|
|
307
|
+
throw new RuntimeSpaceBindingsError(`Invalid creation receipt; original retained: ${receiptPath}`);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
catch (error) {
|
|
311
|
+
if (!missing(error))
|
|
312
|
+
throw error;
|
|
313
|
+
}
|
|
314
|
+
if (receipt && !receipt.spaceId)
|
|
315
|
+
throw new RuntimeSpaceBindingsError(`A previous Space creation has an unknown outcome. Check your Spaces and use --space <id>. ${receiptPath}`);
|
|
316
|
+
if (!receipt) {
|
|
317
|
+
receipt = { operationId: randomUUID() };
|
|
318
|
+
await writeRuntimeJson(receiptPath, receipt);
|
|
319
|
+
}
|
|
320
|
+
let createdSpaceId = receipt.spaceId;
|
|
321
|
+
if (!createdSpaceId) {
|
|
322
|
+
try {
|
|
323
|
+
createdSpaceId = await input.createSpace();
|
|
324
|
+
}
|
|
325
|
+
catch (error) {
|
|
326
|
+
const status = error?.status;
|
|
327
|
+
if (status && status >= 400 && status < 500 && status !== 408)
|
|
328
|
+
await rm(receiptPath);
|
|
329
|
+
throw error;
|
|
330
|
+
}
|
|
331
|
+
await writeRuntimeJson(receiptPath, { ...receipt, spaceId: createdSpaceId });
|
|
332
|
+
}
|
|
297
333
|
if (!nonEmptyString(createdSpaceId)) {
|
|
298
334
|
throw new RuntimeSpaceBindingsError("Local Runtime Space creation returned no Space ID");
|
|
299
335
|
}
|
|
300
336
|
const spaceId = createdSpaceId.trim();
|
|
301
337
|
await input.validateSpace?.(spaceId);
|
|
302
338
|
await persistRuntimeSpaceBinding(path, { root, key: bindingKey, spaceId });
|
|
339
|
+
await rm(receiptPath);
|
|
303
340
|
return { spaceId, source: "created" };
|
|
304
341
|
}, { path, lockPath });
|
|
305
342
|
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type RuntimeCapabilities } from "@neta-art/cohub";
|
|
2
|
+
import { type HarnessOptions } from "./harness.js";
|
|
3
|
+
import { type RuntimeDiagnostic, type RuntimeDiagnosticLevel } from "./diagnostics.js";
|
|
4
|
+
import { type RuntimeSummary } from "./presentation.js";
|
|
5
|
+
export type RuntimeLaunch = {
|
|
6
|
+
spaceId: string;
|
|
7
|
+
root: string;
|
|
8
|
+
identity: string;
|
|
9
|
+
harnesses: ("pi" | "codex")[];
|
|
10
|
+
executables: HarnessOptions;
|
|
11
|
+
capabilities?: RuntimeCapabilities;
|
|
12
|
+
background: boolean;
|
|
13
|
+
verbose?: boolean;
|
|
14
|
+
};
|
|
15
|
+
export declare function sandboxOutputLevel(value: unknown, stream: "stdout" | "stderr"): RuntimeDiagnosticLevel;
|
|
16
|
+
export declare function runRuntime(config: RuntimeLaunch, onState: (status: RuntimeSummary) => void, externalSignal?: AbortSignal, onDiagnostic?: (event: RuntimeDiagnostic) => void): Promise<void>;
|