@neta-art/cohub-cli 7.1.2 → 8.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -2
- package/dist/auth.js +38 -5
- package/dist/client.js +5 -2
- package/dist/commands/runtime.d.ts +1 -2
- package/dist/commands/runtime.js +169 -293
- package/dist/commands/sandboxd-binary.d.ts +1 -1
- package/dist/commands/sandboxd-binary.js +13 -5
- package/dist/runtime/archive-store.d.ts +5 -1
- package/dist/runtime/archive-store.js +22 -6
- package/dist/runtime/connection.d.ts +4 -2
- package/dist/runtime/connection.js +113 -44
- package/dist/runtime/diagnostics.d.ts +3 -0
- package/dist/runtime/diagnostics.js +3 -0
- package/dist/runtime/harness.d.ts +7 -0
- package/dist/runtime/harness.js +63 -17
- package/dist/runtime/instance.d.ts +5 -0
- package/dist/runtime/instance.js +159 -0
- package/dist/runtime/json-rpc.d.ts +2 -0
- package/dist/runtime/json-rpc.js +2 -0
- package/dist/runtime/launch.d.ts +20 -0
- package/dist/runtime/launch.js +176 -0
- package/dist/runtime/native-codex-hook.d.ts +1 -0
- package/dist/runtime/native-codex-hook.js +28 -0
- package/dist/runtime/native-install.d.ts +21 -0
- package/dist/runtime/native-install.js +130 -0
- package/dist/runtime/native-ipc.d.ts +26 -0
- package/dist/runtime/native-ipc.js +101 -0
- package/dist/runtime/native-pi-extension.d.ts +20 -0
- package/dist/runtime/native-pi-extension.js +47 -0
- package/dist/runtime/native-sync-store.d.ts +97 -0
- package/dist/runtime/native-sync-store.js +365 -0
- package/dist/runtime/native-sync.d.ts +25 -0
- package/dist/runtime/native-sync.js +128 -0
- package/dist/runtime/native-transcript.d.ts +27 -0
- package/dist/runtime/native-transcript.js +281 -0
- package/dist/runtime/presentation.d.ts +21 -0
- package/dist/runtime/presentation.js +78 -0
- package/dist/runtime/process-group.d.ts +2 -0
- package/dist/runtime/process-group.js +124 -31
- package/dist/runtime/session-store.d.ts +17 -2
- package/dist/runtime/session-store.js +118 -9
- package/dist/runtime/space-binding.d.ts +3 -0
- package/dist/runtime/space-binding.js +43 -6
- package/dist/runtime/supervisor.d.ts +16 -0
- package/dist/runtime/supervisor.js +277 -0
- package/dist/runtime/worker.d.ts +1 -0
- package/dist/runtime/worker.js +20 -0
- package/package.json +3 -2
|
@@ -49,6 +49,8 @@ export class RuntimeArchiveStore {
|
|
|
49
49
|
this.root = root;
|
|
50
50
|
this.transport = transport;
|
|
51
51
|
}
|
|
52
|
+
/** Distinguishes "no transport configured" from transient upload failures. */
|
|
53
|
+
get hasTransport() { return this.transport != null; }
|
|
52
54
|
setErrorReporter(reporter) {
|
|
53
55
|
this.errorReporter = reporter;
|
|
54
56
|
}
|
|
@@ -115,6 +117,8 @@ export class RuntimeArchiveStore {
|
|
|
115
117
|
if (saved) {
|
|
116
118
|
if (saved.sessionId !== state.sessionId || saved.harness !== state.harness)
|
|
117
119
|
throw new Error("Archive identity mismatch");
|
|
120
|
+
if (state.expectedChecksum && state.expectedChecksum !== saved.sha256)
|
|
121
|
+
throw new Error("Native Turn bytes changed; original archive retained");
|
|
118
122
|
const committed = await stat(join(this.root, "ready", `${turnId}.json`)).catch((error) => { if (missing(error))
|
|
119
123
|
return null; throw error; });
|
|
120
124
|
if (!committed)
|
|
@@ -130,12 +134,15 @@ export class RuntimeArchiveStore {
|
|
|
130
134
|
const before = await file.stat();
|
|
131
135
|
if (!before.isFile() || !before.size)
|
|
132
136
|
throw new Error("Native archive is empty");
|
|
137
|
+
const sizeBytes = state.sizeBytes ?? before.size;
|
|
138
|
+
if (!Number.isSafeInteger(sizeBytes) || sizeBytes < 1 || sizeBytes > before.size)
|
|
139
|
+
throw new Error("Native archive boundary is unavailable");
|
|
133
140
|
const buffer = Buffer.alloc(RUNTIME_ARCHIVE_SEGMENT_BYTES);
|
|
134
141
|
let offset = 0;
|
|
135
142
|
let digest = createHash("sha256");
|
|
136
143
|
let parent = null;
|
|
137
144
|
// Hash the old prefix, not just its size: equal-size and growing rewrites are valid.
|
|
138
|
-
if (previous && previous.nativeSessionId === state.nativeSessionId && previous.sizeBytes <=
|
|
145
|
+
if (previous && previous.nativeSessionId === state.nativeSessionId && previous.sizeBytes <= sizeBytes) {
|
|
139
146
|
while (offset < previous.sizeBytes) {
|
|
140
147
|
const { bytesRead } = await file.read(buffer, 0, Math.min(buffer.length, previous.sizeBytes - offset), offset);
|
|
141
148
|
if (!bytesRead)
|
|
@@ -152,8 +159,8 @@ export class RuntimeArchiveStore {
|
|
|
152
159
|
}
|
|
153
160
|
const segments = [];
|
|
154
161
|
await mkdir(join(this.root, "objects"), { recursive: true, mode: 0o700 });
|
|
155
|
-
while (offset <
|
|
156
|
-
const { bytesRead } = await file.read(buffer, 0, Math.min(buffer.length,
|
|
162
|
+
while (offset < sizeBytes) {
|
|
163
|
+
const { bytesRead } = await file.read(buffer, 0, Math.min(buffer.length, sizeBytes - offset), offset);
|
|
157
164
|
if (!bytesRead)
|
|
158
165
|
throw new Error("Native file changed during capture");
|
|
159
166
|
const bytes = buffer.subarray(0, bytesRead);
|
|
@@ -164,8 +171,16 @@ export class RuntimeArchiveStore {
|
|
|
164
171
|
offset += bytesRead;
|
|
165
172
|
}
|
|
166
173
|
const after = await stat(state.path);
|
|
167
|
-
if (before.ino !== after.ino || before.size !== after.size || before.mtimeMs !== after.mtimeMs)
|
|
174
|
+
if (before.ino !== after.ino || after.size < sizeBytes || state.sizeBytes === undefined && (before.size !== after.size || before.mtimeMs !== after.mtimeMs))
|
|
168
175
|
throw new Error("Native file changed during capture");
|
|
176
|
+
if (state.sizeBytes !== undefined) {
|
|
177
|
+
// Native clients may append while an earlier Turn is captured. Validate the exact prefix twice.
|
|
178
|
+
const verified = createHash("sha256");
|
|
179
|
+
for await (const bytes of createReadStream(state.path, { end: sizeBytes - 1 }))
|
|
180
|
+
verified.update(bytes);
|
|
181
|
+
if (verified.digest("hex") !== digest.copy().digest("hex"))
|
|
182
|
+
throw new Error("Native prefix changed during capture");
|
|
183
|
+
}
|
|
169
184
|
if (process.platform !== "win32") {
|
|
170
185
|
const directory = await open(join(this.root, "objects"), "r");
|
|
171
186
|
try {
|
|
@@ -177,8 +192,10 @@ export class RuntimeArchiveStore {
|
|
|
177
192
|
}
|
|
178
193
|
index = harnessArchiveIndexSchema.parse({ ...identity, version: 1, nativeSessionId: state.nativeSessionId,
|
|
179
194
|
nativeFormat: state.harness === "pi" ? "pi.jsonl" : "codex.rollout", parentTurnId: parent?.turnId ?? null,
|
|
180
|
-
sizeBytes
|
|
195
|
+
sizeBytes, sha256: digest.digest("hex"), segments });
|
|
181
196
|
validateArchiveBoundary(index, parent);
|
|
197
|
+
if (state.expectedChecksum && index.sha256 !== state.expectedChecksum)
|
|
198
|
+
throw new Error("Native Turn bytes changed during capture; original retained");
|
|
182
199
|
}
|
|
183
200
|
finally {
|
|
184
201
|
await file.close();
|
|
@@ -254,7 +271,6 @@ export class RuntimeArchiveStore {
|
|
|
254
271
|
catch (error) {
|
|
255
272
|
if (!signal.aborted) {
|
|
256
273
|
this.errorReporter?.(error, index);
|
|
257
|
-
console.error("Archive pending; native segments retained:", error);
|
|
258
274
|
}
|
|
259
275
|
}
|
|
260
276
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type RuntimeCapabilities } from "@neta-art/cohub";
|
|
1
|
+
import { type RuntimeCapabilities, type NativeRuntimeEvent } from "@neta-art/cohub";
|
|
2
2
|
import { type HarnessOptions } from "./harness.js";
|
|
3
3
|
import { type RuntimeSessionStore } from "./session-store.js";
|
|
4
4
|
import { type RuntimeDiagnostics } from "./diagnostics.js";
|
|
@@ -8,12 +8,14 @@ export type RuntimeConnectionOptions = {
|
|
|
8
8
|
url: string;
|
|
9
9
|
capabilities: RuntimeCapabilities;
|
|
10
10
|
harnesses: HarnessOptions;
|
|
11
|
-
token: () => Promise<string>;
|
|
11
|
+
token: (forceRefresh?: boolean) => Promise<string>;
|
|
12
12
|
signal: AbortSignal;
|
|
13
13
|
store: RuntimeSessionStore;
|
|
14
14
|
onReady: () => void;
|
|
15
|
+
onDisconnected?: () => void;
|
|
15
16
|
runtimeId?: string;
|
|
16
17
|
diagnostics?: RuntimeDiagnostics;
|
|
17
18
|
leaseConflictTimeoutMs?: number;
|
|
19
|
+
onNativeChannel?: (send: (event: NativeRuntimeEvent) => Promise<unknown>) => void;
|
|
18
20
|
};
|
|
19
21
|
export declare function serveRuntime(options: RuntimeConnectionOptions): Promise<void>;
|
|
@@ -16,7 +16,7 @@ export async function serveRuntime(options) {
|
|
|
16
16
|
const flush = () => options.store.flushArchives(uploadSignal).catch((error) => {
|
|
17
17
|
if (!uploadSignal.aborted) {
|
|
18
18
|
log("warn", "archive.flush_failed", { error: serializeDiagnosticError(error) });
|
|
19
|
-
|
|
19
|
+
// The diagnostic sink handles terminal presentation and throttling.
|
|
20
20
|
}
|
|
21
21
|
});
|
|
22
22
|
const timer = setInterval(() => {
|
|
@@ -31,22 +31,36 @@ export async function serveRuntime(options) {
|
|
|
31
31
|
try {
|
|
32
32
|
while (!options.signal.aborted) {
|
|
33
33
|
attempt += 1;
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
attempt
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
34
|
+
let readyAt = 0;
|
|
35
|
+
let outcome;
|
|
36
|
+
try {
|
|
37
|
+
outcome = await connect({
|
|
38
|
+
...options,
|
|
39
|
+
runtimeId,
|
|
40
|
+
attempt,
|
|
41
|
+
onReady: () => {
|
|
42
|
+
readyAt = Date.now();
|
|
43
|
+
conflictSince = null;
|
|
44
|
+
options.onReady();
|
|
45
|
+
void flush();
|
|
46
|
+
},
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
if (options.signal.aborted)
|
|
51
|
+
return;
|
|
52
|
+
log("warn", "runtime.connection_failed", { error: serializeDiagnosticError(error) });
|
|
53
|
+
outcome = "retry";
|
|
54
|
+
}
|
|
55
|
+
options.onDisconnected?.();
|
|
46
56
|
if (options.signal.aborted)
|
|
47
57
|
return;
|
|
48
58
|
if (outcome === "fatal")
|
|
49
|
-
throw new Error("Runtime connection rejected");
|
|
59
|
+
throw new Error("Runtime connection rejected; check permissions or upgrade the CLI");
|
|
60
|
+
if (readyAt && Date.now() - readyAt >= 60_000) {
|
|
61
|
+
backoff = 500;
|
|
62
|
+
attempt = 0;
|
|
63
|
+
}
|
|
50
64
|
if (outcome === "conflict") {
|
|
51
65
|
conflictSince ??= Date.now();
|
|
52
66
|
if (Date.now() - conflictSince >= (options.leaseConflictTimeoutMs ?? 90_000)) {
|
|
@@ -58,8 +72,8 @@ export async function serveRuntime(options) {
|
|
|
58
72
|
delayMs: backoff,
|
|
59
73
|
outcome,
|
|
60
74
|
});
|
|
61
|
-
await delay(backoff, undefined, { signal: options.signal }).catch(() => undefined);
|
|
62
|
-
backoff = Math.min(
|
|
75
|
+
await delay(backoff / 2 + Math.random() * backoff / 2, undefined, { signal: options.signal }).catch(() => undefined);
|
|
76
|
+
backoff = Math.min(30_000, backoff * 2);
|
|
63
77
|
}
|
|
64
78
|
}
|
|
65
79
|
finally {
|
|
@@ -91,7 +105,7 @@ async function connect(options) {
|
|
|
91
105
|
currentToken = await options.token();
|
|
92
106
|
}
|
|
93
107
|
catch (error) {
|
|
94
|
-
log("
|
|
108
|
+
log("warn", error instanceof Error && error.name === "AuthRequiredError" ? "runtime.auth_required" : "runtime.auth_token_failed", { error: serializeDiagnosticError(error) });
|
|
95
109
|
throw error;
|
|
96
110
|
}
|
|
97
111
|
const runtimeUrl = new URL(options.url);
|
|
@@ -105,7 +119,10 @@ async function connect(options) {
|
|
|
105
119
|
let connectionId = null;
|
|
106
120
|
let fatal = false;
|
|
107
121
|
let conflict = false;
|
|
122
|
+
let unauthorized = false;
|
|
108
123
|
let readyTimer;
|
|
124
|
+
const nativePending = new Map();
|
|
125
|
+
let nativeChannelReady = false;
|
|
109
126
|
const send = (frame) => {
|
|
110
127
|
if (socket.readyState !== WebSocket.OPEN || socket.bufferedAmount > RUNTIME_MAX_FRAME_BYTES) {
|
|
111
128
|
log("warn", "runtime.frame_send_unavailable", {
|
|
@@ -122,12 +139,30 @@ async function connect(options) {
|
|
|
122
139
|
}
|
|
123
140
|
socket.send(data);
|
|
124
141
|
};
|
|
125
|
-
const
|
|
142
|
+
const closeTransport = () => socket.close();
|
|
143
|
+
const shutdown = () => {
|
|
126
144
|
for (const execution of active.values())
|
|
127
145
|
execution.controller.abort();
|
|
128
|
-
|
|
146
|
+
closeTransport();
|
|
129
147
|
};
|
|
130
|
-
|
|
148
|
+
const sendNative = (event) => new Promise((resolve, reject) => {
|
|
149
|
+
if (!nativeChannelReady || !connectionId) {
|
|
150
|
+
reject(new Error("Runtime native channel is unavailable"));
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
const requestId = randomUUID();
|
|
154
|
+
const timer = setTimeout(() => { nativePending.delete(requestId); reject(new Error("Native Runtime event timed out")); }, 30_000);
|
|
155
|
+
nativePending.set(requestId, { resolve, reject, timer });
|
|
156
|
+
try {
|
|
157
|
+
send({ type: "runtime.native", requestId, event });
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
clearTimeout(timer);
|
|
161
|
+
nativePending.delete(requestId);
|
|
162
|
+
reject(error instanceof Error ? error : new Error(String(error)));
|
|
163
|
+
}
|
|
164
|
+
});
|
|
165
|
+
options.signal.addEventListener("abort", shutdown, { once: true });
|
|
131
166
|
const heartbeat = setInterval(() => {
|
|
132
167
|
const heartbeatAgeMs = Date.now() - lastHeartbeat;
|
|
133
168
|
if (heartbeatAgeMs > 30_000) {
|
|
@@ -135,12 +170,7 @@ async function connect(options) {
|
|
|
135
170
|
ageMs: heartbeatAgeMs,
|
|
136
171
|
activeExecutions: active.size,
|
|
137
172
|
}, { connectionId });
|
|
138
|
-
|
|
139
|
-
log("error", "runtime.execution_heartbeat_timeout", {
|
|
140
|
-
ageMs: heartbeatAgeMs,
|
|
141
|
-
}, executionContext(connectionId, execution));
|
|
142
|
-
}
|
|
143
|
-
stop();
|
|
173
|
+
closeTransport();
|
|
144
174
|
}
|
|
145
175
|
else if (connectionId) {
|
|
146
176
|
try {
|
|
@@ -150,7 +180,7 @@ async function connect(options) {
|
|
|
150
180
|
log("warn", "runtime.heartbeat_send_failed", {
|
|
151
181
|
error: serializeDiagnosticError(error),
|
|
152
182
|
}, { connectionId });
|
|
153
|
-
|
|
183
|
+
closeTransport();
|
|
154
184
|
}
|
|
155
185
|
void options.token().then((next) => {
|
|
156
186
|
if (next !== currentToken) {
|
|
@@ -163,21 +193,22 @@ async function connect(options) {
|
|
|
163
193
|
log("warn", "runtime.auth_refresh_send_failed", {
|
|
164
194
|
error: serializeDiagnosticError(error),
|
|
165
195
|
}, { connectionId });
|
|
166
|
-
|
|
196
|
+
closeTransport();
|
|
167
197
|
}
|
|
168
198
|
}
|
|
169
199
|
}).catch((error) => {
|
|
170
200
|
log("warn", "runtime.auth_refresh_failed", { error: serializeDiagnosticError(error) }, { connectionId });
|
|
171
|
-
|
|
201
|
+
closeTransport();
|
|
172
202
|
});
|
|
173
203
|
}
|
|
174
204
|
}, 10_000);
|
|
175
205
|
const closed = new Promise((resolve) => {
|
|
176
206
|
socket.addEventListener("close", (event) => {
|
|
177
|
-
|
|
207
|
+
unauthorized = event.code === 4401;
|
|
208
|
+
fatal = [4400, 4403].includes(event.code);
|
|
178
209
|
conflict = event.code === 4409;
|
|
179
210
|
disconnected.abort();
|
|
180
|
-
log(fatal || conflict ? "error" : "warn", "runtime.websocket.closed", {
|
|
211
|
+
log(options.signal.aborted ? "debug" : fatal || conflict ? "error" : "warn", "runtime.websocket.closed", {
|
|
181
212
|
code: event.code,
|
|
182
213
|
reason: event.reason,
|
|
183
214
|
durationMs: Date.now() - connectedAt,
|
|
@@ -185,14 +216,21 @@ async function connect(options) {
|
|
|
185
216
|
fatal,
|
|
186
217
|
conflict,
|
|
187
218
|
}, { connectionId });
|
|
188
|
-
|
|
189
|
-
|
|
219
|
+
nativeChannelReady = false;
|
|
220
|
+
for (const pending of nativePending.values()) {
|
|
221
|
+
clearTimeout(pending.timer);
|
|
222
|
+
pending.reject(new Error("Runtime native channel disconnected"));
|
|
223
|
+
}
|
|
224
|
+
nativePending.clear();
|
|
225
|
+
options.onDisconnected?.();
|
|
226
|
+
const executionMustStop = options.signal.aborted || fatal || conflict;
|
|
190
227
|
for (const execution of active.values()) {
|
|
191
|
-
log("error", "runtime.
|
|
228
|
+
log(executionMustStop ? "error" : "warn", executionMustStop ? "runtime.execution_transport_invalidated" : "runtime.execution_transport_detached", {
|
|
192
229
|
code: event.code,
|
|
193
230
|
reason: event.reason,
|
|
194
231
|
}, executionContext(connectionId, execution));
|
|
195
|
-
|
|
232
|
+
if (executionMustStop)
|
|
233
|
+
execution.controller.abort();
|
|
196
234
|
}
|
|
197
235
|
resolve();
|
|
198
236
|
}, { once: true });
|
|
@@ -221,7 +259,7 @@ async function connect(options) {
|
|
|
221
259
|
}
|
|
222
260
|
catch (error) {
|
|
223
261
|
log("error", "runtime.hello_failed", { error: serializeDiagnosticError(error) });
|
|
224
|
-
|
|
262
|
+
closeTransport();
|
|
225
263
|
}
|
|
226
264
|
});
|
|
227
265
|
socket.addEventListener("message", (event) => {
|
|
@@ -234,6 +272,8 @@ async function connect(options) {
|
|
|
234
272
|
log("error", "runtime.protocol.invalid_json", { error: serializeDiagnosticError(error) }, { connectionId });
|
|
235
273
|
throw error;
|
|
236
274
|
}
|
|
275
|
+
// Any inbound frame proves the transport is live; heartbeat frames are one case.
|
|
276
|
+
lastHeartbeat = Date.now();
|
|
237
277
|
if (raw.type === "runtime.ready") {
|
|
238
278
|
const frame = runtimeReadySchema.parse(raw);
|
|
239
279
|
if (connectionId === frame.connectionId)
|
|
@@ -246,6 +286,8 @@ async function connect(options) {
|
|
|
246
286
|
connectionId,
|
|
247
287
|
durationMs: Date.now() - connectedAt,
|
|
248
288
|
}, { connectionId });
|
|
289
|
+
nativeChannelReady = true;
|
|
290
|
+
options.onNativeChannel?.(sendNative);
|
|
249
291
|
options.onReady();
|
|
250
292
|
let recoveryBatch = 0;
|
|
251
293
|
for await (const executions of options.store.pendingExecutionBatches()) {
|
|
@@ -261,7 +303,19 @@ async function connect(options) {
|
|
|
261
303
|
if (!connectionId)
|
|
262
304
|
throw new Error("Runtime handshake is incomplete");
|
|
263
305
|
if (raw.type === "runtime.heartbeat") {
|
|
264
|
-
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
if (raw.type === "runtime.native.result") {
|
|
309
|
+
const result = raw;
|
|
310
|
+
const pending = result.requestId ? nativePending.get(result.requestId) : null;
|
|
311
|
+
if (!pending)
|
|
312
|
+
return;
|
|
313
|
+
clearTimeout(pending.timer);
|
|
314
|
+
nativePending.delete(result.requestId ?? "");
|
|
315
|
+
if (result.error)
|
|
316
|
+
pending.reject(new Error(result.error));
|
|
317
|
+
else
|
|
318
|
+
pending.resolve(result.result);
|
|
265
319
|
return;
|
|
266
320
|
}
|
|
267
321
|
const frame = runtimeCommandSchema.parse(raw);
|
|
@@ -294,7 +348,7 @@ async function connect(options) {
|
|
|
294
348
|
revision: frame.revision,
|
|
295
349
|
error: serializeDiagnosticError(error),
|
|
296
350
|
}, context);
|
|
297
|
-
|
|
351
|
+
// Keep the receipt; the next reconciliation can retry acknowledgement.
|
|
298
352
|
active.delete(frame.requestId);
|
|
299
353
|
send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.error", message: "Local acknowledgement failed; result retained" } });
|
|
300
354
|
return;
|
|
@@ -353,7 +407,7 @@ async function connect(options) {
|
|
|
353
407
|
catch (error) {
|
|
354
408
|
if (!recovery.controller.signal.aborted) {
|
|
355
409
|
log("error", "runtime.recovery_failed", { error: serializeDiagnosticError(error) }, context);
|
|
356
|
-
|
|
410
|
+
// Original files and receipts remain available for reconciliation.
|
|
357
411
|
try {
|
|
358
412
|
send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.error", uncertain: true, message: "Result unavailable; files retained" } });
|
|
359
413
|
}
|
|
@@ -453,7 +507,17 @@ async function connect(options) {
|
|
|
453
507
|
const emit = (value) => {
|
|
454
508
|
if (value.type === "message.commit")
|
|
455
509
|
durableEvents.push(value);
|
|
456
|
-
|
|
510
|
+
// Streaming is best-effort. A detached transport must not cancel local model or tool work;
|
|
511
|
+
// durable commits and the final result are replayed after reconnect.
|
|
512
|
+
if (disconnected.signal.aborted || socket.readyState !== WebSocket.OPEN || socket.bufferedAmount > RUNTIME_MAX_FRAME_BYTES)
|
|
513
|
+
return;
|
|
514
|
+
try {
|
|
515
|
+
send({ type: "runtime.event", requestId: frame.requestId, event: value });
|
|
516
|
+
}
|
|
517
|
+
catch (error) {
|
|
518
|
+
if (!disconnected.signal.aborted && socket.readyState === WebSocket.OPEN)
|
|
519
|
+
throw error;
|
|
520
|
+
}
|
|
457
521
|
};
|
|
458
522
|
execution.promise = (async () => {
|
|
459
523
|
try {
|
|
@@ -486,7 +550,10 @@ async function connect(options) {
|
|
|
486
550
|
frame.input.context = await requestContext(controller.signal);
|
|
487
551
|
}
|
|
488
552
|
}
|
|
489
|
-
await options.store.recordResult(execution.result.state, frame.requestId, [...durableEvents, execution.result.event]);
|
|
553
|
+
await options.store.recordResult(execution.result.state, frame.requestId, [...durableEvents, execution.result.event], execution.result.uncertainCleanup ? { uncertainCleanup: execution.result.uncertainCleanup } : undefined);
|
|
554
|
+
if (execution.result.uncertainCleanup) {
|
|
555
|
+
log("warn", "runtime.turn_cleanup_pending", { cleanup: execution.result.uncertainCleanup.error }, context);
|
|
556
|
+
}
|
|
490
557
|
emit(execution.result.event);
|
|
491
558
|
}
|
|
492
559
|
catch (error) {
|
|
@@ -505,7 +572,7 @@ async function connect(options) {
|
|
|
505
572
|
})();
|
|
506
573
|
})().catch((error) => {
|
|
507
574
|
log("error", "runtime.protocol_error", { error: serializeDiagnosticError(error) }, { connectionId });
|
|
508
|
-
|
|
575
|
+
closeTransport();
|
|
509
576
|
});
|
|
510
577
|
});
|
|
511
578
|
readyTimer = setTimeout(() => {
|
|
@@ -513,7 +580,7 @@ async function connect(options) {
|
|
|
513
580
|
socket.close(4408, "Runtime handshake timed out");
|
|
514
581
|
}, 15_000);
|
|
515
582
|
if (options.signal.aborted)
|
|
516
|
-
|
|
583
|
+
shutdown();
|
|
517
584
|
try {
|
|
518
585
|
await closed;
|
|
519
586
|
await Promise.allSettled([...active.values()].map((entry) => entry.promise));
|
|
@@ -521,7 +588,9 @@ async function connect(options) {
|
|
|
521
588
|
finally {
|
|
522
589
|
clearTimeout(readyTimer);
|
|
523
590
|
clearInterval(heartbeat);
|
|
524
|
-
options.signal.removeEventListener("abort",
|
|
591
|
+
options.signal.removeEventListener("abort", shutdown);
|
|
525
592
|
}
|
|
593
|
+
if (unauthorized && !options.signal.aborted)
|
|
594
|
+
await options.token(true);
|
|
526
595
|
return fatal ? "fatal" : conflict ? "conflict" : "retry";
|
|
527
596
|
}
|
|
@@ -50,6 +50,8 @@ export type RuntimeDiagnosticsOptions = {
|
|
|
50
50
|
logFlushIntervalMs?: number;
|
|
51
51
|
maxLogFileBytes?: number;
|
|
52
52
|
maxTotalLogBytes?: number;
|
|
53
|
+
/** Receives only the redacted event, before asynchronous disk I/O. */
|
|
54
|
+
onEvent?: (event: RuntimeDiagnostic) => void;
|
|
53
55
|
};
|
|
54
56
|
export type ReadRuntimeDiagnosticsOptions = {
|
|
55
57
|
limit?: number;
|
|
@@ -64,6 +66,7 @@ export declare class RuntimeDiagnostics {
|
|
|
64
66
|
readonly logPath: string;
|
|
65
67
|
private readonly spaceId;
|
|
66
68
|
private readonly component;
|
|
69
|
+
private readonly onEvent?;
|
|
67
70
|
private readonly logFlushIntervalMs;
|
|
68
71
|
private readonly maxLogFileBytes;
|
|
69
72
|
private readonly maxTotalLogBytes;
|
|
@@ -134,6 +134,7 @@ export class RuntimeDiagnostics {
|
|
|
134
134
|
logPath;
|
|
135
135
|
spaceId;
|
|
136
136
|
component;
|
|
137
|
+
onEvent;
|
|
137
138
|
logFlushIntervalMs;
|
|
138
139
|
maxLogFileBytes;
|
|
139
140
|
maxTotalLogBytes;
|
|
@@ -153,6 +154,7 @@ export class RuntimeDiagnostics {
|
|
|
153
154
|
this.runtimeId = options.runtimeId ?? randomUUID();
|
|
154
155
|
this.spaceId = options.spaceId;
|
|
155
156
|
this.component = options.component ?? "runtime";
|
|
157
|
+
this.onEvent = options.onEvent;
|
|
156
158
|
this.logFlushIntervalMs = options.logFlushIntervalMs ?? DEFAULT_LOG_FLUSH_INTERVAL_MS;
|
|
157
159
|
this.maxLogFileBytes = options.maxLogFileBytes ?? DEFAULT_MAX_LOG_FILE_BYTES;
|
|
158
160
|
this.maxTotalLogBytes = options.maxTotalLogBytes ?? DEFAULT_MAX_TOTAL_LOG_BYTES;
|
|
@@ -193,6 +195,7 @@ export class RuntimeDiagnostics {
|
|
|
193
195
|
...(diagnosticError ? { error: diagnosticError } : {}),
|
|
194
196
|
});
|
|
195
197
|
this.enqueueWrite(value);
|
|
198
|
+
this.onEvent?.(value);
|
|
196
199
|
}
|
|
197
200
|
async close() {
|
|
198
201
|
this.closed = true;
|
|
@@ -6,11 +6,18 @@ export type HarnessOptions = {
|
|
|
6
6
|
pi?: string;
|
|
7
7
|
codex?: string;
|
|
8
8
|
};
|
|
9
|
+
export declare function harnessExecutableCandidates(binary: string, cwd: string, path: string, platform?: NodeJS.Platform, pathExt?: string | undefined): string[];
|
|
10
|
+
/** Discover installed executables only; authentication/capability errors stay explicit. */
|
|
11
|
+
export declare function installedHarnesses(cwd: string, options: HarnessOptions, path?: string): Promise<("pi" | "codex")[]>;
|
|
9
12
|
export type HarnessResult = {
|
|
10
13
|
state: NativeSession;
|
|
11
14
|
event: Extract<RuntimeExecutionEvent, {
|
|
12
15
|
type: "turn.end";
|
|
13
16
|
}>;
|
|
17
|
+
uncertainCleanup?: {
|
|
18
|
+
processGroupId: number;
|
|
19
|
+
error: string;
|
|
20
|
+
};
|
|
14
21
|
};
|
|
15
22
|
export declare function piContent(value: unknown): ContentBlock[];
|
|
16
23
|
export declare function promptText(content: ContentBlock[]): string;
|
package/dist/runtime/harness.js
CHANGED
|
@@ -1,8 +1,41 @@
|
|
|
1
|
+
import { access, stat } from "node:fs/promises";
|
|
2
|
+
import { constants } from "node:fs";
|
|
3
|
+
import { delimiter, join, resolve, win32 } from "node:path";
|
|
1
4
|
import { JsonRpcProcess, record } from "./json-rpc.js";
|
|
5
|
+
import { ProcessCleanupUncertainError } from "./process-group.js";
|
|
2
6
|
import { codexModelCatalog } from "./model-catalog.js";
|
|
3
7
|
import { codexTokenTotals, codexUsage, subtractCodexTokens } from "./codex-usage.js";
|
|
4
8
|
import { downloadPublicImage } from "../safe-remote-image.js";
|
|
5
9
|
import { serializeDiagnosticError } from "./diagnostics.js";
|
|
10
|
+
export function harnessExecutableCandidates(binary, cwd, path, platform = process.platform, pathExt = process.env.PATHEXT) {
|
|
11
|
+
const windows = platform === "win32";
|
|
12
|
+
const paths = windows ? win32 : { delimiter, join, resolve };
|
|
13
|
+
const extensions = windows && !win32.extname(binary)
|
|
14
|
+
? ["", ...(pathExt || ".COM;.EXE;.BAT;.CMD").split(";").filter((ext) => /^\.[a-z0-9]+$/i.test(ext))]
|
|
15
|
+
: [""];
|
|
16
|
+
const roots = binary.includes("/") || binary.includes("\\")
|
|
17
|
+
? [paths.resolve(cwd, binary)]
|
|
18
|
+
: path.split(paths.delimiter).map((directory) => directory.replace(/^"(.*)"$/, "$1")).filter(Boolean).map((directory) => paths.resolve(cwd, directory, binary));
|
|
19
|
+
return roots.flatMap((root) => extensions.map((ext) => `${root}${ext}`));
|
|
20
|
+
}
|
|
21
|
+
/** Discover installed executables only; authentication/capability errors stay explicit. */
|
|
22
|
+
export async function installedHarnesses(cwd, options, path = process.env.PATH ?? "") {
|
|
23
|
+
const names = options.pi || options.codex ? ["pi", "codex"].filter((name) => options[name]) : ["pi", "codex"];
|
|
24
|
+
const found = await Promise.all(names.map(async (name) => {
|
|
25
|
+
const binary = options[name] || name;
|
|
26
|
+
const candidates = harnessExecutableCandidates(binary, cwd, path);
|
|
27
|
+
for (const candidate of candidates) {
|
|
28
|
+
try {
|
|
29
|
+
await access(candidate, process.platform === "win32" ? constants.F_OK : constants.X_OK);
|
|
30
|
+
if ((await stat(candidate)).isFile())
|
|
31
|
+
return name;
|
|
32
|
+
}
|
|
33
|
+
catch { /* Try the next PATH entry. */ }
|
|
34
|
+
}
|
|
35
|
+
return null;
|
|
36
|
+
}));
|
|
37
|
+
return found.filter((name) => name !== null);
|
|
38
|
+
}
|
|
6
39
|
const runtimeEnvironment = (input) => ({ COHUB_SPACE_ID: input.spaceId, COHUB_SESSION_ID: input.sessionId, COHUB_TURN_ID: input.turnId });
|
|
7
40
|
const array = (value) => Array.isArray(value) ? value : [];
|
|
8
41
|
const text = (value) => typeof value === "string" ? value : "";
|
|
@@ -49,9 +82,20 @@ function createAbortEscalation(rpc, signal, interrupt) {
|
|
|
49
82
|
}
|
|
50
83
|
/** Native files stay authoritative; archival failure only degrades cross-host resume. */
|
|
51
84
|
async function finishHarnessTurn(store, state, message, resume, turnId, diagnosticContext) {
|
|
52
|
-
|
|
85
|
+
// The store records a redacted diagnostic; native files remain authoritative.
|
|
86
|
+
const archive = await store.archive(state, turnId, diagnosticContext).catch(() => null);
|
|
53
87
|
return { state, event: { type: "turn.end", message, resume, archive } };
|
|
54
88
|
}
|
|
89
|
+
/** Close the RPC process. An unconfirmable cleanup degrades the result to uncertain, never masks it. */
|
|
90
|
+
async function closeHarness(rpc, executionError) {
|
|
91
|
+
return rpc.close().then(() => undefined, (cleanupError) => {
|
|
92
|
+
const processGroupId = rpc.processGroupId;
|
|
93
|
+
if (!(cleanupError instanceof ProcessCleanupUncertainError) || processGroupId == null)
|
|
94
|
+
throw cleanupError;
|
|
95
|
+
const detail = executionError ? `; execution error: ${executionError}` : "";
|
|
96
|
+
return { processGroupId, error: `${cleanupError.message}${detail}` };
|
|
97
|
+
});
|
|
98
|
+
}
|
|
55
99
|
export async function discoverHarnesses(harnesses, options, cwd) {
|
|
56
100
|
const models = [];
|
|
57
101
|
await Promise.all(harnesses.map(async (harness) => {
|
|
@@ -100,6 +144,7 @@ export async function executePi(input, options, cwd, store, emit, signal, diagno
|
|
|
100
144
|
let currentContent = [];
|
|
101
145
|
let last = { ordinal: 0, content: [] };
|
|
102
146
|
const abortEscalation = createAbortEscalation(rpc, signal, () => { void rpc.request("abort").catch(() => undefined); });
|
|
147
|
+
let uncertainCleanup;
|
|
103
148
|
const stopFailureLogging = diagnostics
|
|
104
149
|
? rpc.onFailure((error) => diagnostics.log("error", "harness.rpc_process_failed", { error: serializeDiagnosticError(error) }, { ...diagnosticContext, component: "harness" }))
|
|
105
150
|
: () => undefined;
|
|
@@ -202,17 +247,17 @@ export async function executePi(input, options, cwd, store, emit, signal, diagno
|
|
|
202
247
|
abortEscalation.clear();
|
|
203
248
|
stopFailureLogging();
|
|
204
249
|
stopTimeoutLogging();
|
|
205
|
-
await rpc.
|
|
250
|
+
uncertainCleanup = await closeHarness(rpc, last.errorMessage);
|
|
206
251
|
}
|
|
207
252
|
if (signal.aborted)
|
|
208
253
|
last = { ...last, stopReason: "aborted" };
|
|
209
|
-
return finishHarnessTurn(store, state, last, resume, input.turnId, {
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
254
|
+
return { ...await finishHarnessTurn(store, state, last, resume, input.turnId, {
|
|
255
|
+
component: "harness",
|
|
256
|
+
sessionId: input.sessionId,
|
|
257
|
+
turnId: input.turnId,
|
|
258
|
+
harness: "pi",
|
|
259
|
+
traceContext: input.traceContext,
|
|
260
|
+
}), ...(uncertainCleanup ? { uncertainCleanup } : {}) };
|
|
216
261
|
}
|
|
217
262
|
export function codexItemContent(item) {
|
|
218
263
|
if (item.type === "agentMessage" || item.type === "plan")
|
|
@@ -262,6 +307,7 @@ export async function executeCodex(input, options, cwd, store, emit, signal, dia
|
|
|
262
307
|
if (nativeTurnId)
|
|
263
308
|
void rpc.request("turn/interrupt", { threadId: state.nativeSessionId, turnId: nativeTurnId }).catch(() => undefined);
|
|
264
309
|
});
|
|
310
|
+
let uncertainCleanup;
|
|
265
311
|
try {
|
|
266
312
|
await initializeCodex(rpc);
|
|
267
313
|
const threadOptions = {
|
|
@@ -418,17 +464,17 @@ export async function executeCodex(input, options, cwd, store, emit, signal, dia
|
|
|
418
464
|
abortEscalation.clear();
|
|
419
465
|
stopFailureLogging();
|
|
420
466
|
stopTimeoutLogging();
|
|
421
|
-
await rpc.
|
|
467
|
+
uncertainCleanup = await closeHarness(rpc, final.errorMessage);
|
|
422
468
|
}
|
|
423
469
|
if (signal.aborted)
|
|
424
470
|
final = { ...final, stopReason: "aborted" };
|
|
425
471
|
if (usage)
|
|
426
472
|
final = { ...final, usage };
|
|
427
|
-
return finishHarnessTurn(store, state, final, resume, input.turnId, {
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
473
|
+
return { ...await finishHarnessTurn(store, state, final, resume, input.turnId, {
|
|
474
|
+
component: "harness",
|
|
475
|
+
sessionId: input.sessionId,
|
|
476
|
+
turnId: input.turnId,
|
|
477
|
+
harness: "codex",
|
|
478
|
+
traceContext: input.traceContext,
|
|
479
|
+
}), ...(uncertainCleanup ? { uncertainCleanup } : {}) };
|
|
434
480
|
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { RuntimeSummary } from "./presentation.js";
|
|
2
|
+
export declare function runtimeInstanceDirectory(identity: string, spaceId: string): string;
|
|
3
|
+
export declare function requestRuntimeInstance(directory: string, action?: "status" | "stop", force?: boolean): Promise<RuntimeSummary | null>;
|
|
4
|
+
/** Private local IPC is both the single-instance guard and the control surface. */
|
|
5
|
+
export declare function ownRuntimeInstance(directory: string, status: () => RuntimeSummary, stop: (force: boolean) => Promise<void>): Promise<() => Promise<void>>;
|