@neta-art/cohub-cli 7.0.1 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/dist/commands/desktop.d.ts +9 -0
- package/dist/commands/desktop.js +13 -5
- package/dist/commands/run.d.ts +11 -0
- package/dist/commands/run.js +4 -4
- package/dist/commands/runtime.js +254 -43
- package/dist/commands/sandboxd-binary.d.ts +2 -0
- package/dist/commands/sandboxd-binary.js +37 -16
- package/dist/index.js +4 -2
- package/dist/runtime/archive-store.d.ts +2 -0
- package/dist/runtime/archive-store.js +7 -1
- package/dist/runtime/connection.d.ts +3 -0
- package/dist/runtime/connection.js +242 -36
- package/dist/runtime/diagnostics.d.ts +104 -0
- package/dist/runtime/diagnostics.js +382 -0
- package/dist/runtime/harness.d.ts +3 -2
- package/dist/runtime/harness.js +39 -8
- package/dist/runtime/json-rpc.d.ts +2 -0
- package/dist/runtime/json-rpc.js +12 -1
- package/dist/runtime/projection-store.d.ts +34 -0
- package/dist/runtime/projection-store.js +103 -0
- package/dist/runtime/session-store.d.ts +26 -4
- package/dist/runtime/session-store.js +238 -60
- package/dist/runtime/space-binding.d.ts +45 -0
- package/dist/runtime/space-binding.js +305 -0
- package/dist/runtime/turn-projection.d.ts +43 -0
- package/dist/runtime/turn-projection.js +127 -0
- package/dist/space.d.ts +11 -3
- package/dist/space.js +18 -6
- package/package.json +3 -2
|
@@ -44,10 +44,14 @@ export class RuntimeArchiveStore {
|
|
|
44
44
|
transport;
|
|
45
45
|
flushing = null;
|
|
46
46
|
capturing = new Map();
|
|
47
|
+
errorReporter = null;
|
|
47
48
|
constructor(root, transport) {
|
|
48
49
|
this.root = root;
|
|
49
50
|
this.transport = transport;
|
|
50
51
|
}
|
|
52
|
+
setErrorReporter(reporter) {
|
|
53
|
+
this.errorReporter = reporter;
|
|
54
|
+
}
|
|
51
55
|
async pendingCount() {
|
|
52
56
|
const pending = new Set();
|
|
53
57
|
for (const directory of ["pending", "captures"]) {
|
|
@@ -248,8 +252,10 @@ export class RuntimeArchiveStore {
|
|
|
248
252
|
queue.push(...children.get(index.turnId) ?? []);
|
|
249
253
|
}
|
|
250
254
|
catch (error) {
|
|
251
|
-
if (!signal.aborted)
|
|
255
|
+
if (!signal.aborted) {
|
|
256
|
+
this.errorReporter?.(error, index);
|
|
252
257
|
console.error("Archive pending; native segments retained:", error);
|
|
258
|
+
}
|
|
253
259
|
}
|
|
254
260
|
}
|
|
255
261
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type RuntimeCapabilities } from "@neta-art/cohub";
|
|
2
2
|
import { type HarnessOptions } from "./harness.js";
|
|
3
3
|
import { type RuntimeSessionStore } from "./session-store.js";
|
|
4
|
+
import { type RuntimeDiagnostics } from "./diagnostics.js";
|
|
4
5
|
export type RuntimeConnectionOptions = {
|
|
5
6
|
spaceId: string;
|
|
6
7
|
cwd: string;
|
|
@@ -11,6 +12,8 @@ export type RuntimeConnectionOptions = {
|
|
|
11
12
|
signal: AbortSignal;
|
|
12
13
|
store: RuntimeSessionStore;
|
|
13
14
|
onReady: () => void;
|
|
15
|
+
runtimeId?: string;
|
|
16
|
+
diagnostics?: RuntimeDiagnostics;
|
|
14
17
|
leaseConflictTimeoutMs?: number;
|
|
15
18
|
};
|
|
16
19
|
export declare function serveRuntime(options: RuntimeConnectionOptions): Promise<void>;
|
|
@@ -1,31 +1,63 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
1
2
|
import { setTimeout as delay } from "node:timers/promises";
|
|
2
|
-
import { RUNTIME_MAX_FRAME_BYTES, RUNTIME_PROTOCOL_VERSION, runtimeCommandSchema, runtimeReadySchema } from "@neta-art/cohub";
|
|
3
|
+
import { RUNTIME_MAX_FRAME_BYTES, RUNTIME_PROTOCOL_VERSION, runtimeCommandSchema, runtimeReadySchema, } from "@neta-art/cohub";
|
|
3
4
|
import { executeCodex, executePi } from "./harness.js";
|
|
4
5
|
import { ProcessCleanupUncertainError } from "./process-group.js";
|
|
5
6
|
import { ContextRequiredError } from "./session-store.js";
|
|
7
|
+
import { serializeDiagnosticError, } from "./diagnostics.js";
|
|
6
8
|
export async function serveRuntime(options) {
|
|
9
|
+
const runtimeId = options.runtimeId ?? options.diagnostics?.runtimeId ?? randomUUID();
|
|
7
10
|
let backoff = 500;
|
|
11
|
+
let attempt = 0;
|
|
8
12
|
let conflictSince = null;
|
|
9
13
|
const uploads = new AbortController();
|
|
10
14
|
const uploadSignal = AbortSignal.any([options.signal, uploads.signal]);
|
|
15
|
+
const log = (level, event, data, context) => options.diagnostics?.log(level, event, data, context);
|
|
11
16
|
const flush = () => options.store.flushArchives(uploadSignal).catch((error) => {
|
|
12
|
-
if (!uploadSignal.aborted)
|
|
17
|
+
if (!uploadSignal.aborted) {
|
|
18
|
+
log("warn", "archive.flush_failed", { error: serializeDiagnosticError(error) });
|
|
13
19
|
console.error("Archive pending:", error);
|
|
20
|
+
}
|
|
14
21
|
});
|
|
15
|
-
const timer = setInterval(() => {
|
|
22
|
+
const timer = setInterval(() => {
|
|
23
|
+
void flush();
|
|
24
|
+
}, 10_000);
|
|
16
25
|
void flush();
|
|
26
|
+
log("info", "runtime.started", {
|
|
27
|
+
runtimeId,
|
|
28
|
+
workspace: options.cwd,
|
|
29
|
+
harnesses: options.capabilities.harnesses,
|
|
30
|
+
});
|
|
17
31
|
try {
|
|
18
32
|
while (!options.signal.aborted) {
|
|
19
|
-
|
|
33
|
+
attempt += 1;
|
|
34
|
+
const outcome = await connect({
|
|
35
|
+
...options,
|
|
36
|
+
runtimeId,
|
|
37
|
+
attempt,
|
|
38
|
+
onReady: () => {
|
|
39
|
+
backoff = 500;
|
|
40
|
+
attempt = 0;
|
|
41
|
+
conflictSince = null;
|
|
42
|
+
options.onReady();
|
|
43
|
+
void flush();
|
|
44
|
+
},
|
|
45
|
+
});
|
|
20
46
|
if (options.signal.aborted)
|
|
21
47
|
return;
|
|
22
48
|
if (outcome === "fatal")
|
|
23
49
|
throw new Error("Runtime connection rejected");
|
|
24
50
|
if (outcome === "conflict") {
|
|
25
51
|
conflictSince ??= Date.now();
|
|
26
|
-
if (Date.now() - conflictSince >= (options.leaseConflictTimeoutMs ?? 90_000))
|
|
52
|
+
if (Date.now() - conflictSince >= (options.leaseConflictTimeoutMs ?? 90_000)) {
|
|
27
53
|
throw new Error("Space is already connected to another Runtime");
|
|
54
|
+
}
|
|
28
55
|
}
|
|
56
|
+
log("debug", "runtime.reconnect_scheduled", {
|
|
57
|
+
attempt: attempt || 1,
|
|
58
|
+
delayMs: backoff,
|
|
59
|
+
outcome,
|
|
60
|
+
});
|
|
29
61
|
await delay(backoff, undefined, { signal: options.signal }).catch(() => undefined);
|
|
30
62
|
backoff = Math.min(10_000, backoff * 2);
|
|
31
63
|
}
|
|
@@ -34,11 +66,37 @@ export async function serveRuntime(options) {
|
|
|
34
66
|
clearInterval(timer);
|
|
35
67
|
uploads.abort();
|
|
36
68
|
await flush();
|
|
69
|
+
log("info", "runtime.stopped", { runtimeId });
|
|
37
70
|
}
|
|
38
71
|
}
|
|
72
|
+
function executionContext(connectionId, input) {
|
|
73
|
+
return {
|
|
74
|
+
connectionId,
|
|
75
|
+
sessionId: input.sessionId,
|
|
76
|
+
turnId: input.turnId,
|
|
77
|
+
harness: input.harness,
|
|
78
|
+
requestId: input.traceContext?.requestId ?? input.requestId ?? null,
|
|
79
|
+
traceContext: input.traceContext,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
39
82
|
async function connect(options) {
|
|
40
|
-
|
|
41
|
-
const
|
|
83
|
+
const log = (level, event, data, context) => options.diagnostics?.log(level, event, data, context);
|
|
84
|
+
const connectedAt = Date.now();
|
|
85
|
+
log("debug", "runtime.websocket.connecting", {
|
|
86
|
+
attempt: options.attempt,
|
|
87
|
+
url: options.url,
|
|
88
|
+
});
|
|
89
|
+
let currentToken;
|
|
90
|
+
try {
|
|
91
|
+
currentToken = await options.token();
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
log("error", "runtime.auth_token_failed", { error: serializeDiagnosticError(error) });
|
|
95
|
+
throw error;
|
|
96
|
+
}
|
|
97
|
+
const runtimeUrl = new URL(options.url);
|
|
98
|
+
runtimeUrl.searchParams.set("runtimeId", options.runtimeId);
|
|
99
|
+
const socket = new WebSocket(runtimeUrl.toString());
|
|
42
100
|
const active = new Map();
|
|
43
101
|
const seen = new Set();
|
|
44
102
|
const disconnected = new AbortController();
|
|
@@ -49,30 +107,69 @@ async function connect(options) {
|
|
|
49
107
|
let conflict = false;
|
|
50
108
|
let readyTimer;
|
|
51
109
|
const send = (frame) => {
|
|
52
|
-
if (socket.readyState !== WebSocket.OPEN || socket.bufferedAmount > RUNTIME_MAX_FRAME_BYTES)
|
|
110
|
+
if (socket.readyState !== WebSocket.OPEN || socket.bufferedAmount > RUNTIME_MAX_FRAME_BYTES) {
|
|
111
|
+
log("warn", "runtime.frame_send_unavailable", {
|
|
112
|
+
readyState: socket.readyState,
|
|
113
|
+
bufferedBytes: socket.bufferedAmount,
|
|
114
|
+
}, { connectionId });
|
|
53
115
|
throw new Error("Runtime connection unavailable");
|
|
116
|
+
}
|
|
54
117
|
const data = JSON.stringify(frame);
|
|
55
|
-
|
|
118
|
+
const bytes = Buffer.byteLength(data);
|
|
119
|
+
if (bytes > RUNTIME_MAX_FRAME_BYTES) {
|
|
120
|
+
log("error", "runtime.frame_too_large", { bytes, maxBytes: RUNTIME_MAX_FRAME_BYTES }, { connectionId });
|
|
56
121
|
throw new Error("Runtime frame exceeds transfer limit");
|
|
122
|
+
}
|
|
57
123
|
socket.send(data);
|
|
58
124
|
};
|
|
59
|
-
const stop = () => {
|
|
60
|
-
|
|
125
|
+
const stop = () => {
|
|
126
|
+
for (const execution of active.values())
|
|
127
|
+
execution.controller.abort();
|
|
128
|
+
socket.close();
|
|
129
|
+
};
|
|
61
130
|
options.signal.addEventListener("abort", stop, { once: true });
|
|
62
131
|
const heartbeat = setInterval(() => {
|
|
63
|
-
|
|
132
|
+
const heartbeatAgeMs = Date.now() - lastHeartbeat;
|
|
133
|
+
if (heartbeatAgeMs > 30_000) {
|
|
134
|
+
log("error", "runtime.heartbeat_timeout", {
|
|
135
|
+
ageMs: heartbeatAgeMs,
|
|
136
|
+
activeExecutions: active.size,
|
|
137
|
+
}, { connectionId });
|
|
138
|
+
for (const execution of active.values()) {
|
|
139
|
+
log("error", "runtime.execution_heartbeat_timeout", {
|
|
140
|
+
ageMs: heartbeatAgeMs,
|
|
141
|
+
}, executionContext(connectionId, execution));
|
|
142
|
+
}
|
|
64
143
|
stop();
|
|
144
|
+
}
|
|
65
145
|
else if (connectionId) {
|
|
66
146
|
try {
|
|
67
147
|
send({ type: "runtime.heartbeat" });
|
|
68
148
|
}
|
|
69
|
-
catch {
|
|
149
|
+
catch (error) {
|
|
150
|
+
log("warn", "runtime.heartbeat_send_failed", {
|
|
151
|
+
error: serializeDiagnosticError(error),
|
|
152
|
+
}, { connectionId });
|
|
70
153
|
stop();
|
|
71
154
|
}
|
|
72
|
-
void options.token().then((next) => {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
155
|
+
void options.token().then((next) => {
|
|
156
|
+
if (next !== currentToken) {
|
|
157
|
+
try {
|
|
158
|
+
send({ type: "runtime.auth", token: next });
|
|
159
|
+
currentToken = next;
|
|
160
|
+
log("debug", "runtime.auth_refreshed", undefined, { connectionId });
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
log("warn", "runtime.auth_refresh_send_failed", {
|
|
164
|
+
error: serializeDiagnosticError(error),
|
|
165
|
+
}, { connectionId });
|
|
166
|
+
stop();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}).catch((error) => {
|
|
170
|
+
log("warn", "runtime.auth_refresh_failed", { error: serializeDiagnosticError(error) }, { connectionId });
|
|
171
|
+
stop();
|
|
172
|
+
});
|
|
76
173
|
}
|
|
77
174
|
}, 10_000);
|
|
78
175
|
const closed = new Promise((resolve) => {
|
|
@@ -80,25 +177,63 @@ async function connect(options) {
|
|
|
80
177
|
fatal = [4400, 4401, 4403].includes(event.code);
|
|
81
178
|
conflict = event.code === 4409;
|
|
82
179
|
disconnected.abort();
|
|
180
|
+
log(fatal || conflict ? "error" : "warn", "runtime.websocket.closed", {
|
|
181
|
+
code: event.code,
|
|
182
|
+
reason: event.reason,
|
|
183
|
+
durationMs: Date.now() - connectedAt,
|
|
184
|
+
activeExecutions: active.size,
|
|
185
|
+
fatal,
|
|
186
|
+
conflict,
|
|
187
|
+
}, { connectionId });
|
|
83
188
|
if (!options.signal.aborted)
|
|
84
189
|
console.error(`Runtime disconnected (${event.code}): ${event.reason}`);
|
|
85
|
-
for (const execution of active.values())
|
|
190
|
+
for (const execution of active.values()) {
|
|
191
|
+
log("error", "runtime.execution_transport_lost", {
|
|
192
|
+
code: event.code,
|
|
193
|
+
reason: event.reason,
|
|
194
|
+
}, executionContext(connectionId, execution));
|
|
86
195
|
execution.controller.abort();
|
|
196
|
+
}
|
|
87
197
|
resolve();
|
|
88
198
|
}, { once: true });
|
|
89
199
|
});
|
|
90
|
-
socket.addEventListener("error", () =>
|
|
200
|
+
socket.addEventListener("error", (event) => {
|
|
201
|
+
const detail = event.error;
|
|
202
|
+
log("warn", "runtime.websocket.error", {
|
|
203
|
+
type: event.type,
|
|
204
|
+
error: serializeDiagnosticError(detail ?? new Error("WebSocket error")),
|
|
205
|
+
}, { connectionId });
|
|
206
|
+
socket.close();
|
|
207
|
+
});
|
|
91
208
|
socket.addEventListener("open", () => {
|
|
92
209
|
try {
|
|
93
|
-
|
|
210
|
+
log("info", "runtime.websocket.open", {
|
|
211
|
+
attempt: options.attempt,
|
|
212
|
+
durationMs: Date.now() - connectedAt,
|
|
213
|
+
});
|
|
214
|
+
send({
|
|
215
|
+
type: "runtime.hello",
|
|
216
|
+
version: RUNTIME_PROTOCOL_VERSION,
|
|
217
|
+
spaceId: options.spaceId,
|
|
218
|
+
token: currentToken,
|
|
219
|
+
capabilities: options.capabilities,
|
|
220
|
+
});
|
|
94
221
|
}
|
|
95
|
-
catch {
|
|
222
|
+
catch (error) {
|
|
223
|
+
log("error", "runtime.hello_failed", { error: serializeDiagnosticError(error) });
|
|
96
224
|
stop();
|
|
97
225
|
}
|
|
98
226
|
});
|
|
99
227
|
socket.addEventListener("message", (event) => {
|
|
100
228
|
void (async () => {
|
|
101
|
-
|
|
229
|
+
let raw;
|
|
230
|
+
try {
|
|
231
|
+
raw = JSON.parse(String(event.data));
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
log("error", "runtime.protocol.invalid_json", { error: serializeDiagnosticError(error) }, { connectionId });
|
|
235
|
+
throw error;
|
|
236
|
+
}
|
|
102
237
|
if (raw.type === "runtime.ready") {
|
|
103
238
|
const frame = runtimeReadySchema.parse(raw);
|
|
104
239
|
if (connectionId === frame.connectionId)
|
|
@@ -107,8 +242,18 @@ async function connect(options) {
|
|
|
107
242
|
throw new Error("Runtime connection identity changed");
|
|
108
243
|
connectionId = frame.connectionId;
|
|
109
244
|
clearTimeout(readyTimer);
|
|
245
|
+
log("info", "runtime.ready", {
|
|
246
|
+
connectionId,
|
|
247
|
+
durationMs: Date.now() - connectedAt,
|
|
248
|
+
}, { connectionId });
|
|
110
249
|
options.onReady();
|
|
250
|
+
let recoveryBatch = 0;
|
|
111
251
|
for await (const executions of options.store.pendingExecutionBatches()) {
|
|
252
|
+
recoveryBatch += 1;
|
|
253
|
+
log("info", "runtime.recovery_batch_sent", {
|
|
254
|
+
batch: recoveryBatch,
|
|
255
|
+
count: executions.length,
|
|
256
|
+
}, { connectionId });
|
|
112
257
|
send({ type: "runtime.recovery", executions });
|
|
113
258
|
}
|
|
114
259
|
return;
|
|
@@ -121,11 +266,15 @@ async function connect(options) {
|
|
|
121
266
|
}
|
|
122
267
|
const frame = runtimeCommandSchema.parse(raw);
|
|
123
268
|
if (frame.type === "session.context") {
|
|
269
|
+
log("debug", "runtime.context_received", {
|
|
270
|
+
messageCount: frame.context.messages.length,
|
|
271
|
+
}, { connectionId });
|
|
124
272
|
contexts.get(frame.requestId)?.(frame.context);
|
|
125
273
|
contexts.delete(frame.requestId);
|
|
126
274
|
return;
|
|
127
275
|
}
|
|
128
276
|
if (frame.type === "turn.abort") {
|
|
277
|
+
log("info", "runtime.turn_abort_received", undefined, { connectionId, requestId: frame.requestId });
|
|
129
278
|
active.get(frame.requestId)?.controller.abort();
|
|
130
279
|
return;
|
|
131
280
|
}
|
|
@@ -133,6 +282,7 @@ async function connect(options) {
|
|
|
133
282
|
const execution = active.get(frame.requestId);
|
|
134
283
|
if (!execution)
|
|
135
284
|
return;
|
|
285
|
+
const context = executionContext(connectionId, execution);
|
|
136
286
|
try {
|
|
137
287
|
await execution.promise;
|
|
138
288
|
if (!execution.result)
|
|
@@ -140,11 +290,16 @@ async function connect(options) {
|
|
|
140
290
|
await options.store.acknowledge(execution.result.state, frame.turnId, frame.revision);
|
|
141
291
|
}
|
|
142
292
|
catch (error) {
|
|
293
|
+
log("error", "runtime.turn_ack_failed", {
|
|
294
|
+
revision: frame.revision,
|
|
295
|
+
error: serializeDiagnosticError(error),
|
|
296
|
+
}, context);
|
|
143
297
|
console.error("Runtime acknowledgement failed; result retained:", error);
|
|
144
298
|
active.delete(frame.requestId);
|
|
145
299
|
send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.error", message: "Local acknowledgement failed; result retained" } });
|
|
146
300
|
return;
|
|
147
301
|
}
|
|
302
|
+
log("info", "runtime.turn_acknowledged", { revision: frame.revision }, context);
|
|
148
303
|
active.delete(frame.requestId);
|
|
149
304
|
send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.acknowledged" } });
|
|
150
305
|
return;
|
|
@@ -153,6 +308,12 @@ async function connect(options) {
|
|
|
153
308
|
const identity = frame.execution;
|
|
154
309
|
if (identity.spaceId !== options.spaceId)
|
|
155
310
|
throw new Error("Invalid Runtime target");
|
|
311
|
+
const context = executionContext(connectionId, {
|
|
312
|
+
...identity,
|
|
313
|
+
requestId: frame.traceContext?.requestId,
|
|
314
|
+
traceContext: frame.traceContext,
|
|
315
|
+
});
|
|
316
|
+
log("info", "runtime.recovery_requested", undefined, context);
|
|
156
317
|
const previous = active.get(frame.requestId);
|
|
157
318
|
if (previous) {
|
|
158
319
|
if (previous.sessionId !== identity.sessionId || previous.turnId !== identity.turnId || previous.harness !== identity.harness)
|
|
@@ -165,7 +326,13 @@ async function connect(options) {
|
|
|
165
326
|
return;
|
|
166
327
|
}
|
|
167
328
|
const running = [...active].find(([, entry]) => entry.turnId === identity.turnId && entry.sessionId === identity.sessionId && entry.harness === identity.harness);
|
|
168
|
-
const recovery = {
|
|
329
|
+
const recovery = {
|
|
330
|
+
...identity,
|
|
331
|
+
requestId: frame.traceContext?.requestId,
|
|
332
|
+
traceContext: frame.traceContext,
|
|
333
|
+
controller: new AbortController(),
|
|
334
|
+
promise: Promise.resolve(),
|
|
335
|
+
};
|
|
169
336
|
active.set(frame.requestId, recovery);
|
|
170
337
|
recovery.promise = (async () => {
|
|
171
338
|
try {
|
|
@@ -185,11 +352,14 @@ async function connect(options) {
|
|
|
185
352
|
}
|
|
186
353
|
catch (error) {
|
|
187
354
|
if (!recovery.controller.signal.aborted) {
|
|
355
|
+
log("error", "runtime.recovery_failed", { error: serializeDiagnosticError(error) }, context);
|
|
188
356
|
console.error("Runtime recovery failed; original files retained:", error);
|
|
189
357
|
try {
|
|
190
358
|
send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.error", uncertain: true, message: "Result unavailable; files retained" } });
|
|
191
359
|
}
|
|
192
|
-
catch {
|
|
360
|
+
catch {
|
|
361
|
+
// The next connection can read the same result.
|
|
362
|
+
}
|
|
193
363
|
}
|
|
194
364
|
active.delete(frame.requestId);
|
|
195
365
|
}
|
|
@@ -198,6 +368,11 @@ async function connect(options) {
|
|
|
198
368
|
}
|
|
199
369
|
if (frame.input.spaceId !== options.spaceId || !options.capabilities.harnesses.includes(frame.input.harness))
|
|
200
370
|
throw new Error("Invalid Runtime target");
|
|
371
|
+
const context = executionContext(connectionId, frame.input);
|
|
372
|
+
log("info", "runtime.turn_received", {
|
|
373
|
+
requestId: frame.requestId,
|
|
374
|
+
resumeOnly: frame.resumeOnly === true,
|
|
375
|
+
}, context);
|
|
201
376
|
const previous = active.get(frame.requestId);
|
|
202
377
|
if (previous) {
|
|
203
378
|
if (previous.sessionId !== frame.input.sessionId || previous.turnId !== frame.input.turnId || previous.harness !== frame.input.harness)
|
|
@@ -209,17 +384,27 @@ async function connect(options) {
|
|
|
209
384
|
send({ type: "runtime.event", requestId: frame.requestId, event });
|
|
210
385
|
return;
|
|
211
386
|
}
|
|
212
|
-
const requestContext = async (executionSignal
|
|
387
|
+
const requestContext = async (executionSignal) => {
|
|
213
388
|
const signal = AbortSignal.any([executionSignal, disconnected.signal]);
|
|
214
389
|
signal.throwIfAborted();
|
|
215
390
|
const pendingTurnIds = await options.store.pendingTurnIds(frame.input.sessionId);
|
|
391
|
+
log("debug", "runtime.context_requested", { pendingTurnCount: pendingTurnIds.length }, context);
|
|
216
392
|
return new Promise((resolve, reject) => {
|
|
217
|
-
const abort = () => {
|
|
393
|
+
const abort = () => {
|
|
394
|
+
clearTimeout(timeout);
|
|
395
|
+
contexts.delete(frame.requestId);
|
|
396
|
+
signal.removeEventListener("abort", abort);
|
|
397
|
+
reject(new Error("Runtime context request aborted"));
|
|
398
|
+
};
|
|
218
399
|
const timeout = setTimeout(abort, 60_000);
|
|
219
400
|
signal.addEventListener("abort", abort, { once: true });
|
|
220
|
-
contexts.set(frame.requestId, (
|
|
401
|
+
contexts.set(frame.requestId, (nextContext) => {
|
|
402
|
+
clearTimeout(timeout);
|
|
403
|
+
signal.removeEventListener("abort", abort);
|
|
404
|
+
resolve(nextContext);
|
|
405
|
+
});
|
|
221
406
|
try {
|
|
222
|
-
send({ type: "runtime.event", requestId: frame.requestId, event: { type: "context.required", pendingTurnIds
|
|
407
|
+
send({ type: "runtime.event", requestId: frame.requestId, event: { type: "context.required", pendingTurnIds } });
|
|
223
408
|
}
|
|
224
409
|
catch {
|
|
225
410
|
abort();
|
|
@@ -243,6 +428,7 @@ async function connect(options) {
|
|
|
243
428
|
active.delete(requestId);
|
|
244
429
|
}
|
|
245
430
|
if ([...active.values()].some((entry) => entry.sessionId === frame.input.sessionId) || active.size >= 8) {
|
|
431
|
+
log("warn", "runtime.turn_rejected_busy", { activeExecutions: active.size }, context);
|
|
246
432
|
send({ type: "runtime.event", requestId: frame.requestId, event: { type: "turn.error", message: "Local Runtime is busy" } });
|
|
247
433
|
return;
|
|
248
434
|
}
|
|
@@ -253,7 +439,15 @@ async function connect(options) {
|
|
|
253
439
|
seen.delete(oldest);
|
|
254
440
|
}
|
|
255
441
|
const controller = new AbortController();
|
|
256
|
-
const execution = {
|
|
442
|
+
const execution = {
|
|
443
|
+
controller,
|
|
444
|
+
sessionId: frame.input.sessionId,
|
|
445
|
+
turnId: frame.input.turnId,
|
|
446
|
+
harness: frame.input.harness,
|
|
447
|
+
requestId: frame.input.requestId,
|
|
448
|
+
traceContext: frame.input.traceContext,
|
|
449
|
+
promise: Promise.resolve(),
|
|
450
|
+
};
|
|
257
451
|
active.set(frame.requestId, execution);
|
|
258
452
|
const durableEvents = [];
|
|
259
453
|
const emit = (value) => {
|
|
@@ -278,34 +472,46 @@ async function connect(options) {
|
|
|
278
472
|
active.delete(frame.requestId);
|
|
279
473
|
return;
|
|
280
474
|
}
|
|
281
|
-
const run = () => (frame.input.harness === "pi" ? executePi : executeCodex)(frame.input, options.harnesses, options.cwd, options.store, emit, controller.signal);
|
|
475
|
+
const run = () => (frame.input.harness === "pi" ? executePi : executeCodex)(frame.input, options.harnesses, options.cwd, options.store, emit, controller.signal, options.diagnostics, context);
|
|
282
476
|
// Preparation can request context, then fall back once from native archive to DB.
|
|
283
477
|
// These retries precede started(), so they never replay model or tool work.
|
|
284
|
-
for (let
|
|
478
|
+
for (let retry = 0;; retry += 1) {
|
|
285
479
|
try {
|
|
286
480
|
execution.result = await run();
|
|
287
481
|
break;
|
|
288
482
|
}
|
|
289
483
|
catch (error) {
|
|
290
|
-
if (!(error instanceof ContextRequiredError) ||
|
|
484
|
+
if (!(error instanceof ContextRequiredError) || retry >= 2)
|
|
291
485
|
throw error;
|
|
292
|
-
frame.input.context = await requestContext(controller.signal
|
|
486
|
+
frame.input.context = await requestContext(controller.signal);
|
|
293
487
|
}
|
|
294
488
|
}
|
|
295
489
|
await options.store.recordResult(execution.result.state, frame.requestId, [...durableEvents, execution.result.event]);
|
|
296
490
|
emit(execution.result.event);
|
|
297
491
|
}
|
|
298
492
|
catch (error) {
|
|
493
|
+
log("error", "runtime.turn_failed", {
|
|
494
|
+
uncertain: Boolean(execution.result) || frame.resumeOnly === true || error instanceof ProcessCleanupUncertainError,
|
|
495
|
+
error: serializeDiagnosticError(error),
|
|
496
|
+
}, context);
|
|
299
497
|
try {
|
|
300
498
|
emit({ type: "turn.error", message: error instanceof Error ? error.message : String(error), uncertain: !!execution.result || frame.resumeOnly === true || error instanceof ProcessCleanupUncertainError });
|
|
301
499
|
}
|
|
302
|
-
catch {
|
|
500
|
+
catch {
|
|
501
|
+
// Native files remain for recovery.
|
|
502
|
+
}
|
|
303
503
|
active.delete(frame.requestId);
|
|
304
504
|
}
|
|
305
505
|
})();
|
|
306
|
-
})().catch((error) => {
|
|
506
|
+
})().catch((error) => {
|
|
507
|
+
log("error", "runtime.protocol_error", { error: serializeDiagnosticError(error) }, { connectionId });
|
|
508
|
+
stop();
|
|
509
|
+
});
|
|
307
510
|
});
|
|
308
|
-
readyTimer = setTimeout(() =>
|
|
511
|
+
readyTimer = setTimeout(() => {
|
|
512
|
+
log("error", "runtime.handshake_timeout", { timeoutMs: 15_000 }, { connectionId });
|
|
513
|
+
socket.close(4408, "Runtime handshake timed out");
|
|
514
|
+
}, 15_000);
|
|
309
515
|
if (options.signal.aborted)
|
|
310
516
|
stop();
|
|
311
517
|
try {
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import type { RuntimeTraceContext } from "@neta-art/cohub";
|
|
2
|
+
export type RuntimeDiagnosticLevel = "debug" | "info" | "warn" | "error";
|
|
3
|
+
export type RuntimeDiagnosticError = {
|
|
4
|
+
name?: string;
|
|
5
|
+
message: string;
|
|
6
|
+
code?: string;
|
|
7
|
+
status?: number;
|
|
8
|
+
syscall?: string;
|
|
9
|
+
hostname?: string;
|
|
10
|
+
causeCode?: string;
|
|
11
|
+
causeMessage?: string;
|
|
12
|
+
causeSyscall?: string;
|
|
13
|
+
causeHostname?: string;
|
|
14
|
+
traceContext?: RuntimeTraceContext;
|
|
15
|
+
stack?: string;
|
|
16
|
+
};
|
|
17
|
+
export type RuntimeDiagnostic = {
|
|
18
|
+
schemaVersion: 1;
|
|
19
|
+
sequence: number;
|
|
20
|
+
timestamp: string;
|
|
21
|
+
elapsedMs: number;
|
|
22
|
+
level: RuntimeDiagnosticLevel;
|
|
23
|
+
component: string;
|
|
24
|
+
event: string;
|
|
25
|
+
runtimeId: string;
|
|
26
|
+
connectionId?: string | null;
|
|
27
|
+
spaceId: string;
|
|
28
|
+
sessionId?: string | null;
|
|
29
|
+
turnId?: string | null;
|
|
30
|
+
harness?: "pi" | "codex" | null;
|
|
31
|
+
traceContext?: RuntimeTraceContext;
|
|
32
|
+
data?: Record<string, unknown>;
|
|
33
|
+
error?: RuntimeDiagnosticError;
|
|
34
|
+
};
|
|
35
|
+
export type RuntimeDiagnosticContext = {
|
|
36
|
+
component?: string;
|
|
37
|
+
connectionId?: string | null;
|
|
38
|
+
requestId?: string | null;
|
|
39
|
+
spaceId?: string;
|
|
40
|
+
sessionId?: string | null;
|
|
41
|
+
turnId?: string | null;
|
|
42
|
+
harness?: "pi" | "codex" | null;
|
|
43
|
+
traceContext?: RuntimeTraceContext;
|
|
44
|
+
};
|
|
45
|
+
export type RuntimeDiagnosticsOptions = {
|
|
46
|
+
root: string;
|
|
47
|
+
runtimeId?: string;
|
|
48
|
+
spaceId: string;
|
|
49
|
+
component?: string;
|
|
50
|
+
logFlushIntervalMs?: number;
|
|
51
|
+
maxLogFileBytes?: number;
|
|
52
|
+
maxTotalLogBytes?: number;
|
|
53
|
+
};
|
|
54
|
+
export type ReadRuntimeDiagnosticsOptions = {
|
|
55
|
+
limit?: number;
|
|
56
|
+
};
|
|
57
|
+
export declare const runtimeDiagnosticsDirectory: (root: string) => string;
|
|
58
|
+
export declare const runtimeDiagnosticsPath: (root: string, runtimeId: string) => string;
|
|
59
|
+
export declare function redactDiagnosticString(value: string): string;
|
|
60
|
+
export declare function serializeDiagnosticError(error: unknown): RuntimeDiagnosticError;
|
|
61
|
+
export declare class RuntimeDiagnostics {
|
|
62
|
+
readonly runtimeId: string;
|
|
63
|
+
readonly directory: string;
|
|
64
|
+
readonly logPath: string;
|
|
65
|
+
private readonly spaceId;
|
|
66
|
+
private readonly component;
|
|
67
|
+
private readonly logFlushIntervalMs;
|
|
68
|
+
private readonly maxLogFileBytes;
|
|
69
|
+
private readonly maxTotalLogBytes;
|
|
70
|
+
private readonly startedAtMs;
|
|
71
|
+
private readonly ready;
|
|
72
|
+
private logFile;
|
|
73
|
+
private logFileSize;
|
|
74
|
+
private logFileStartedAt;
|
|
75
|
+
private writeChain;
|
|
76
|
+
private writeBuffer;
|
|
77
|
+
private writeBufferBytes;
|
|
78
|
+
private droppedWriteEvents;
|
|
79
|
+
private writeTimer;
|
|
80
|
+
private sequence;
|
|
81
|
+
private closed;
|
|
82
|
+
constructor(options: RuntimeDiagnosticsOptions);
|
|
83
|
+
log(level: RuntimeDiagnosticLevel, event: string, data?: Record<string, unknown>, context?: RuntimeDiagnosticContext): void;
|
|
84
|
+
close(): Promise<void>;
|
|
85
|
+
private enqueueWrite;
|
|
86
|
+
private scheduleLogFlush;
|
|
87
|
+
private flushLogBuffer;
|
|
88
|
+
private writeEntries;
|
|
89
|
+
private ensureLogFile;
|
|
90
|
+
private rotateLog;
|
|
91
|
+
private enforceLogCapacity;
|
|
92
|
+
}
|
|
93
|
+
export declare class RuntimeDiagnosticReader {
|
|
94
|
+
readonly root: string;
|
|
95
|
+
private readonly files;
|
|
96
|
+
private readonly lastSequenceByRuntime;
|
|
97
|
+
private initialized;
|
|
98
|
+
constructor(root: string);
|
|
99
|
+
read(options?: {
|
|
100
|
+
limit?: number;
|
|
101
|
+
}): Promise<RuntimeDiagnostic[]>;
|
|
102
|
+
}
|
|
103
|
+
export declare function readRuntimeDiagnosticEvents(root: string, options?: ReadRuntimeDiagnosticsOptions): Promise<RuntimeDiagnostic[]>;
|
|
104
|
+
export declare function isValidRuntimeTraceparent(value: string | null | undefined): value is string;
|