@botlearn-course/daemon 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/agent-service-session.d.ts +67 -0
- package/dist/agent-service-session.js +796 -0
- package/dist/agent-service-ws-protocol.d.ts +28 -0
- package/dist/agent-service-ws-protocol.js +128 -0
- package/dist/cli.js +112 -2
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/run-dispatcher.d.ts +15 -0
- package/dist/run-dispatcher.js +42 -6
- package/dist/runtime-env.d.ts +5 -0
- package/dist/runtime-env.js +23 -0
- package/dist/runtime-profile.js +25 -13
- package/dist/runtimes/acp-stream.js +2 -0
- package/dist/runtimes/codex.d.ts +1 -1
- package/dist/runtimes/codex.js +2 -2
- package/dist/runtimes/deepseek-tui.js +10 -9
- package/dist/runtimes/engine.d.ts +1 -0
- package/dist/runtimes/engine.js +3 -1
- package/dist/runtimes/hermes-agent.d.ts +1 -1
- package/dist/runtimes/hermes-agent.js +3 -2
- package/dist/runtimes/ndjson-stream.d.ts +1 -1
- package/dist/runtimes/ndjson-stream.js +4 -2
- package/dist/runtimes/openclaw-acp.js +3 -1
- package/dist/sandbox-supervisor.d.ts +4 -0
- package/dist/sandbox-supervisor.js +187 -0
- package/dist/types.d.ts +9 -1
- package/dist/websocket-client.d.ts +43 -0
- package/dist/websocket-client.js +320 -0
- package/dist/workspace.d.ts +9 -0
- package/dist/workspace.js +43 -2
- package/package.json +3 -2
|
@@ -0,0 +1,796 @@
|
|
|
1
|
+
import { chmodSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync, } from "node:fs";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { ensureDaemonHome } from "./auth-store.js";
|
|
5
|
+
import { AGENT_SERVICE_WS_SUBPROTOCOL, createSandboxFrame, parseSandboxFrame, } from "./agent-service-ws-protocol.js";
|
|
6
|
+
import { log as defaultLog } from "./log.js";
|
|
7
|
+
import { assertNoInjectedCredentials, redactSecretString } from "./redaction.js";
|
|
8
|
+
import { RunDispatcher, } from "./run-dispatcher.js";
|
|
9
|
+
import { ensureRuntimeSessionWorkspace } from "./workspace.js";
|
|
10
|
+
import { WebSocketClient, } from "./websocket-client.js";
|
|
11
|
+
const RECONNECT_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000];
|
|
12
|
+
const MAX_SPOOL_FRAMES = 1024;
|
|
13
|
+
const MAX_SPOOL_BYTES = 8 * 1024 * 1024;
|
|
14
|
+
class SessionClosedError extends Error {
|
|
15
|
+
code;
|
|
16
|
+
reason;
|
|
17
|
+
constructor(code, reason) {
|
|
18
|
+
super(`sandbox WebSocket closed (${code}): ${reason}`);
|
|
19
|
+
this.code = code;
|
|
20
|
+
this.reason = reason;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
class SocketInbox {
|
|
24
|
+
queued = [];
|
|
25
|
+
waiting = [];
|
|
26
|
+
constructor(socket) {
|
|
27
|
+
socket.on("message", (data, isBinary) => {
|
|
28
|
+
this.push(isBinary ? new Error("binary WebSocket frames are not supported") : data.toString());
|
|
29
|
+
});
|
|
30
|
+
socket.on("error", (error) => this.push(error));
|
|
31
|
+
socket.on("close", (code, reason) => {
|
|
32
|
+
this.push(new SessionClosedError(code, reason.toString() || "closed"));
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
next() {
|
|
36
|
+
const value = this.queued.shift();
|
|
37
|
+
if (value !== undefined) {
|
|
38
|
+
return value instanceof Error ? Promise.reject(value) : Promise.resolve(value);
|
|
39
|
+
}
|
|
40
|
+
return new Promise((resolve, reject) => this.waiting.push({ resolve, reject }));
|
|
41
|
+
}
|
|
42
|
+
push(value) {
|
|
43
|
+
const waiter = this.waiting.shift();
|
|
44
|
+
if (!waiter) {
|
|
45
|
+
this.queued.push(value);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
if (value instanceof Error)
|
|
49
|
+
waiter.reject(value);
|
|
50
|
+
else
|
|
51
|
+
waiter.resolve(value);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
function waitForOpen(socket) {
|
|
55
|
+
return new Promise((resolve, reject) => {
|
|
56
|
+
const onOpen = () => {
|
|
57
|
+
cleanup();
|
|
58
|
+
resolve();
|
|
59
|
+
};
|
|
60
|
+
const onError = (error) => {
|
|
61
|
+
cleanup();
|
|
62
|
+
reject(error);
|
|
63
|
+
};
|
|
64
|
+
const onUnexpected = (_request, response) => {
|
|
65
|
+
cleanup();
|
|
66
|
+
reject(new Error(`sandbox WebSocket upgrade failed: ${response.statusCode ?? "unknown"}`));
|
|
67
|
+
};
|
|
68
|
+
const cleanup = () => {
|
|
69
|
+
socket.off("open", onOpen);
|
|
70
|
+
socket.off("error", onError);
|
|
71
|
+
socket.off("unexpected-response", onUnexpected);
|
|
72
|
+
};
|
|
73
|
+
socket.on("open", onOpen);
|
|
74
|
+
socket.on("error", onError);
|
|
75
|
+
socket.on("unexpected-response", onUnexpected);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
function sessionStatePath(sessionId) {
|
|
79
|
+
const root = path.join(ensureDaemonHome(), "agent-service-sessions", sessionId);
|
|
80
|
+
mkdirSync(root, { recursive: true, mode: 0o700 });
|
|
81
|
+
try {
|
|
82
|
+
chmodSync(root, 0o700);
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
// Windows best effort.
|
|
86
|
+
}
|
|
87
|
+
return path.join(root, "state.json");
|
|
88
|
+
}
|
|
89
|
+
function loadState(sessionId, token) {
|
|
90
|
+
const file = sessionStatePath(sessionId);
|
|
91
|
+
if (existsSync(file)) {
|
|
92
|
+
try {
|
|
93
|
+
const value = JSON.parse(readFileSync(file, "utf8"));
|
|
94
|
+
return {
|
|
95
|
+
sessionGeneration: Number(value.sessionGeneration ?? 0),
|
|
96
|
+
nextOutboundSeq: Number(value.nextOutboundSeq ?? 0),
|
|
97
|
+
reconnectToken: typeof value.reconnectToken === "string" ? value.reconnectToken : token,
|
|
98
|
+
completedCommands: Array.isArray(value.completedCommands)
|
|
99
|
+
? value.completedCommands.filter((item) => typeof item === "string")
|
|
100
|
+
: [],
|
|
101
|
+
acceptedCommands: value.acceptedCommands &&
|
|
102
|
+
typeof value.acceptedCommands === "object" &&
|
|
103
|
+
!Array.isArray(value.acceptedCommands)
|
|
104
|
+
? value.acceptedCommands
|
|
105
|
+
: {},
|
|
106
|
+
runtime: {
|
|
107
|
+
runtimeId: typeof value.runtime?.runtimeId === "string" ? value.runtime.runtimeId : null,
|
|
108
|
+
nativeSessionId: typeof value.runtime?.nativeSessionId === "string"
|
|
109
|
+
? value.runtime.nativeSessionId
|
|
110
|
+
: null,
|
|
111
|
+
contextRevision: Number(value.runtime?.contextRevision ?? 0),
|
|
112
|
+
},
|
|
113
|
+
spool: Array.isArray(value.spool) ? value.spool : [],
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
// Corrupt state is reconstructed from the server desired state.
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return {
|
|
121
|
+
sessionGeneration: 0,
|
|
122
|
+
nextOutboundSeq: 0,
|
|
123
|
+
reconnectToken: token,
|
|
124
|
+
completedCommands: [],
|
|
125
|
+
acceptedCommands: {},
|
|
126
|
+
runtime: {
|
|
127
|
+
runtimeId: null,
|
|
128
|
+
nativeSessionId: null,
|
|
129
|
+
contextRevision: 0,
|
|
130
|
+
},
|
|
131
|
+
spool: [],
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
function saveState(sessionId, state) {
|
|
135
|
+
const file = sessionStatePath(sessionId);
|
|
136
|
+
const tmp = `${file}.tmp-${process.pid}`;
|
|
137
|
+
const fd = openSync(tmp, "w", 0o600);
|
|
138
|
+
try {
|
|
139
|
+
writeFileSync(fd, JSON.stringify(state), { encoding: "utf8" });
|
|
140
|
+
fsyncSync(fd);
|
|
141
|
+
}
|
|
142
|
+
finally {
|
|
143
|
+
closeSync(fd);
|
|
144
|
+
}
|
|
145
|
+
renameSync(tmp, file);
|
|
146
|
+
try {
|
|
147
|
+
const directoryFd = openSync(path.dirname(file), "r");
|
|
148
|
+
try {
|
|
149
|
+
fsyncSync(directoryFd);
|
|
150
|
+
}
|
|
151
|
+
finally {
|
|
152
|
+
closeSync(directoryFd);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
// Directory fsync is unavailable on Windows; the file itself is still flushed.
|
|
157
|
+
}
|
|
158
|
+
try {
|
|
159
|
+
chmodSync(file, 0o600);
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
// Windows best effort.
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
/** Long-running daemon client for one persistent managed sandbox session. */
|
|
166
|
+
export class AgentServiceSessionClient {
|
|
167
|
+
options;
|
|
168
|
+
log;
|
|
169
|
+
random;
|
|
170
|
+
sleep;
|
|
171
|
+
state;
|
|
172
|
+
dispatcher;
|
|
173
|
+
attempts = new Map();
|
|
174
|
+
runtimeProfiles = new Map();
|
|
175
|
+
pendingAcks = new Map();
|
|
176
|
+
pendingFilePrepares = new Map();
|
|
177
|
+
pendingFileCommits = new Map();
|
|
178
|
+
fileGrants = new Map();
|
|
179
|
+
inflightCommands = new Set();
|
|
180
|
+
socket = null;
|
|
181
|
+
sessionGeneration = 0;
|
|
182
|
+
connectionEpoch = 0;
|
|
183
|
+
inboundSeq = 0;
|
|
184
|
+
heartbeatMs = 15_000;
|
|
185
|
+
staleMs = 45_000;
|
|
186
|
+
stopped = false;
|
|
187
|
+
permanentFailure = false;
|
|
188
|
+
constructor(options) {
|
|
189
|
+
this.options = options;
|
|
190
|
+
this.log = options.log ?? defaultLog;
|
|
191
|
+
this.random = options.random ?? Math.random;
|
|
192
|
+
this.sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
193
|
+
this.state = loadState(options.sessionId, options.sessionToken);
|
|
194
|
+
this.dispatcher = new RunDispatcher(this, options.runtimes, {
|
|
195
|
+
persistentSession: this,
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
prepareTurn(payload) {
|
|
199
|
+
const activation = this.runtimeActivation(payload);
|
|
200
|
+
if (activation.contextRevision !== this.state.runtime.contextRevision) {
|
|
201
|
+
throw new Error(`runtime context revision mismatch: accepted ${this.state.runtime.contextRevision}, ` +
|
|
202
|
+
`received ${activation.contextRevision}`);
|
|
203
|
+
}
|
|
204
|
+
const prepared = ensureRuntimeSessionWorkspace(this.options.sessionId, this.sessionGeneration, payload.agent_run_id);
|
|
205
|
+
return {
|
|
206
|
+
workspaceDir: prepared.workspaceDir,
|
|
207
|
+
transcriptFile: prepared.transcriptFile,
|
|
208
|
+
nativeSessionId: this.state.runtime.nativeSessionId,
|
|
209
|
+
contextRevision: activation.contextRevision,
|
|
210
|
+
...(this.options.runtimeEnv
|
|
211
|
+
? { runtimeEnv: { ...process.env, ...this.options.runtimeEnv } }
|
|
212
|
+
: {}),
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
persistNativeSession(sessionId) {
|
|
216
|
+
this.state.runtime.nativeSessionId = sessionId.trim() || null;
|
|
217
|
+
this.persist();
|
|
218
|
+
}
|
|
219
|
+
async run() {
|
|
220
|
+
let failures = 0;
|
|
221
|
+
let authFailures = 0;
|
|
222
|
+
while (!this.stopped && !this.permanentFailure) {
|
|
223
|
+
try {
|
|
224
|
+
await this.connectOnce();
|
|
225
|
+
failures = 0;
|
|
226
|
+
}
|
|
227
|
+
catch (error) {
|
|
228
|
+
const close = error instanceof SessionClosedError ? error : null;
|
|
229
|
+
if (close?.code === 4403 || close?.code === 4410) {
|
|
230
|
+
this.permanentFailure = true;
|
|
231
|
+
break;
|
|
232
|
+
}
|
|
233
|
+
if (close?.code === 4401) {
|
|
234
|
+
authFailures += 1;
|
|
235
|
+
if (authFailures >= 3) {
|
|
236
|
+
this.permanentFailure = true;
|
|
237
|
+
break;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
241
|
+
authFailures = 0;
|
|
242
|
+
}
|
|
243
|
+
if (this.stopped)
|
|
244
|
+
break;
|
|
245
|
+
const base = RECONNECT_DELAYS_MS[Math.min(failures, RECONNECT_DELAYS_MS.length - 1)];
|
|
246
|
+
failures += 1;
|
|
247
|
+
const delay = Math.round(base * (1 + this.random() * 0.25));
|
|
248
|
+
this.log.warn("Agent Service sandbox WebSocket disconnected; reconnecting", {
|
|
249
|
+
runtimeSessionId: this.options.sessionId,
|
|
250
|
+
delayMs: delay,
|
|
251
|
+
error: error instanceof Error ? redactSecretString(error.message) : String(error),
|
|
252
|
+
});
|
|
253
|
+
await this.sleep(delay);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
if (this.permanentFailure) {
|
|
257
|
+
throw new Error("Agent Service sandbox session stopped after a permanent protocol failure");
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
stop() {
|
|
261
|
+
this.stopped = true;
|
|
262
|
+
this.dispatcher.cancelAll();
|
|
263
|
+
this.socket?.close(1000, "daemon_stopping");
|
|
264
|
+
}
|
|
265
|
+
async postEvent(agentRunId, event) {
|
|
266
|
+
const attempt = this.attempts.get(agentRunId);
|
|
267
|
+
if (!attempt)
|
|
268
|
+
throw new Error("Agent Service session event has no active turn attempt");
|
|
269
|
+
if (!this.sessionGeneration || !this.connectionEpoch) {
|
|
270
|
+
throw new Error("Agent Service session is not authenticated");
|
|
271
|
+
}
|
|
272
|
+
const frame = createSandboxFrame({
|
|
273
|
+
type: "turn.event",
|
|
274
|
+
sessionId: this.options.sessionId,
|
|
275
|
+
sessionGeneration: this.sessionGeneration,
|
|
276
|
+
connectionEpoch: this.connectionEpoch,
|
|
277
|
+
seq: this.nextOutboundSeq(),
|
|
278
|
+
agentRunId,
|
|
279
|
+
workerAttempt: attempt,
|
|
280
|
+
payload: event,
|
|
281
|
+
});
|
|
282
|
+
this.state.spool.push(frame);
|
|
283
|
+
try {
|
|
284
|
+
this.enforceSpoolLimit();
|
|
285
|
+
}
|
|
286
|
+
catch (error) {
|
|
287
|
+
this.state.spool.pop();
|
|
288
|
+
this.persist();
|
|
289
|
+
throw error;
|
|
290
|
+
}
|
|
291
|
+
this.persist();
|
|
292
|
+
const ack = new Promise((resolve, reject) => {
|
|
293
|
+
this.pendingAcks.set(frame.frame_id, { resolve, reject });
|
|
294
|
+
});
|
|
295
|
+
await this.sendFrame(frame);
|
|
296
|
+
await ack;
|
|
297
|
+
}
|
|
298
|
+
async postFile(agentRunId, file) {
|
|
299
|
+
const attempt = this.attempts.get(agentRunId);
|
|
300
|
+
if (!attempt)
|
|
301
|
+
throw new Error("Agent Service session file has no active turn attempt");
|
|
302
|
+
if (!file.sha256 || !Number.isInteger(file.size_bytes) || (file.size_bytes ?? -1) < 0) {
|
|
303
|
+
throw new Error("Agent Service session file requires size_bytes and sha256");
|
|
304
|
+
}
|
|
305
|
+
const frame = createSandboxFrame({
|
|
306
|
+
type: "turn.file.prepare",
|
|
307
|
+
sessionId: this.options.sessionId,
|
|
308
|
+
sessionGeneration: this.sessionGeneration,
|
|
309
|
+
connectionEpoch: this.connectionEpoch,
|
|
310
|
+
seq: this.nextOutboundSeq(),
|
|
311
|
+
agentRunId,
|
|
312
|
+
workerAttempt: attempt,
|
|
313
|
+
payload: file,
|
|
314
|
+
});
|
|
315
|
+
const prepared = new Promise((resolve, reject) => {
|
|
316
|
+
this.pendingFilePrepares.set(frame.frame_id, { resolve, reject });
|
|
317
|
+
});
|
|
318
|
+
await this.sendFrame(frame);
|
|
319
|
+
const grant = await prepared;
|
|
320
|
+
this.fileGrants.set(`${agentRunId}:${grant.record.id}`, grant);
|
|
321
|
+
return grant.record;
|
|
322
|
+
}
|
|
323
|
+
async uploadFileContent(agentRunId, fileId, absPath, mimeType) {
|
|
324
|
+
const attempt = this.attempts.get(agentRunId);
|
|
325
|
+
if (!attempt)
|
|
326
|
+
throw new Error("Agent Service session file has no active turn attempt");
|
|
327
|
+
const key = `${agentRunId}:${fileId}`;
|
|
328
|
+
const grant = this.fileGrants.get(key);
|
|
329
|
+
if (!grant)
|
|
330
|
+
throw new Error("Agent Service session file has no upload grant");
|
|
331
|
+
const data = await readFile(absPath);
|
|
332
|
+
assertNoInjectedCredentials(data, [this.state.reconnectToken, grant.uploadGrant]);
|
|
333
|
+
const response = await fetch(grant.uploadUrl, {
|
|
334
|
+
method: "PUT",
|
|
335
|
+
headers: {
|
|
336
|
+
authorization: `Bearer ${grant.uploadGrant}`,
|
|
337
|
+
"content-type": mimeType ?? grant.record.mime_type ?? "application/octet-stream",
|
|
338
|
+
},
|
|
339
|
+
body: data,
|
|
340
|
+
});
|
|
341
|
+
if (!response.ok) {
|
|
342
|
+
const text = await response.text().catch(() => "");
|
|
343
|
+
throw new Error(`Agent Service file upload failed (${response.status}): ${redactSecretString(text.slice(0, 500))}`);
|
|
344
|
+
}
|
|
345
|
+
const frame = createSandboxFrame({
|
|
346
|
+
type: "turn.file.committed",
|
|
347
|
+
sessionId: this.options.sessionId,
|
|
348
|
+
sessionGeneration: this.sessionGeneration,
|
|
349
|
+
connectionEpoch: this.connectionEpoch,
|
|
350
|
+
seq: this.nextOutboundSeq(),
|
|
351
|
+
agentRunId,
|
|
352
|
+
workerAttempt: attempt,
|
|
353
|
+
payload: { file_id: fileId, sha256: grant.record.sha256 },
|
|
354
|
+
});
|
|
355
|
+
const committed = new Promise((resolve, reject) => {
|
|
356
|
+
this.pendingFileCommits.set(frame.frame_id, { resolve, reject });
|
|
357
|
+
});
|
|
358
|
+
await this.sendFrame(frame);
|
|
359
|
+
try {
|
|
360
|
+
return await committed;
|
|
361
|
+
}
|
|
362
|
+
finally {
|
|
363
|
+
this.fileGrants.delete(key);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
async getRunRuntimeProfile(agentRunId) {
|
|
367
|
+
const profile = this.runtimeProfiles.get(agentRunId);
|
|
368
|
+
if (!profile)
|
|
369
|
+
throw new Error("Agent Service session runtime profile is unavailable");
|
|
370
|
+
return profile;
|
|
371
|
+
}
|
|
372
|
+
async connectOnce() {
|
|
373
|
+
const socket = new WebSocketClient(this.options.wsUrl, AGENT_SERVICE_WS_SUBPROTOCOL, {
|
|
374
|
+
headers: { Authorization: `Bearer ${this.state.reconnectToken}` },
|
|
375
|
+
maxPayload: 262_144,
|
|
376
|
+
});
|
|
377
|
+
// Attach message/close listeners before awaiting open; the server may send
|
|
378
|
+
// session.hello immediately after the upgrade completes.
|
|
379
|
+
const inbox = new SocketInbox(socket);
|
|
380
|
+
await waitForOpen(socket);
|
|
381
|
+
if (socket.protocol !== AGENT_SERVICE_WS_SUBPROTOCOL) {
|
|
382
|
+
socket.close(4410, "protocol_incompatible");
|
|
383
|
+
throw new SessionClosedError(4410, "protocol_incompatible");
|
|
384
|
+
}
|
|
385
|
+
this.socket = socket;
|
|
386
|
+
let heartbeat = null;
|
|
387
|
+
let staleCheck = null;
|
|
388
|
+
let lastServerFrameAt = Date.now();
|
|
389
|
+
try {
|
|
390
|
+
while (!this.stopped) {
|
|
391
|
+
const raw = await inbox.next();
|
|
392
|
+
lastServerFrameAt = Date.now();
|
|
393
|
+
const frame = parseSandboxFrame(raw);
|
|
394
|
+
if (frame.type === "session.hello") {
|
|
395
|
+
await this.handleHello(frame);
|
|
396
|
+
if (heartbeat === null) {
|
|
397
|
+
heartbeat = setInterval(() => {
|
|
398
|
+
void this.sendHeartbeat().catch((error) => {
|
|
399
|
+
this.log.warn("Agent Service session heartbeat failed", {
|
|
400
|
+
runtimeSessionId: this.options.sessionId,
|
|
401
|
+
error: error instanceof Error ? redactSecretString(error.message) : String(error),
|
|
402
|
+
});
|
|
403
|
+
});
|
|
404
|
+
}, this.heartbeatMs);
|
|
405
|
+
if (typeof heartbeat.unref === "function")
|
|
406
|
+
heartbeat.unref();
|
|
407
|
+
staleCheck = setInterval(() => {
|
|
408
|
+
if (Date.now() - lastServerFrameAt >= this.staleMs &&
|
|
409
|
+
socket.readyState === WebSocketClient.OPEN) {
|
|
410
|
+
this.log.warn("Agent Service session server frame timeout; reconnecting", {
|
|
411
|
+
runtimeSessionId: this.options.sessionId,
|
|
412
|
+
staleMs: this.staleMs,
|
|
413
|
+
});
|
|
414
|
+
socket.close(4000, "server_stale");
|
|
415
|
+
}
|
|
416
|
+
}, Math.max(1_000, Math.min(this.heartbeatMs, Math.floor(this.staleMs / 3))));
|
|
417
|
+
if (typeof staleCheck.unref === "function")
|
|
418
|
+
staleCheck.unref();
|
|
419
|
+
}
|
|
420
|
+
continue;
|
|
421
|
+
}
|
|
422
|
+
this.assertServerFrame(frame);
|
|
423
|
+
this.inboundSeq = frame.seq;
|
|
424
|
+
await this.handleServerFrame(frame);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
finally {
|
|
428
|
+
if (heartbeat !== null)
|
|
429
|
+
clearInterval(heartbeat);
|
|
430
|
+
if (staleCheck !== null)
|
|
431
|
+
clearInterval(staleCheck);
|
|
432
|
+
this.rejectPendingFiles(new Error("Agent Service file transport disconnected"));
|
|
433
|
+
if (this.socket === socket)
|
|
434
|
+
this.socket = null;
|
|
435
|
+
if (socket.readyState === WebSocketClient.OPEN)
|
|
436
|
+
socket.close(1000, "reconnecting");
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
async handleHello(frame) {
|
|
440
|
+
if (frame.session_id !== this.options.sessionId) {
|
|
441
|
+
throw new SessionClosedError(4403, "session_mismatch");
|
|
442
|
+
}
|
|
443
|
+
if (this.sessionGeneration && frame.session_generation < this.sessionGeneration) {
|
|
444
|
+
throw new SessionClosedError(4409, "stale_generation");
|
|
445
|
+
}
|
|
446
|
+
if (frame.session_generation !== this.state.sessionGeneration) {
|
|
447
|
+
this.rejectPending(new Error("Agent Service session generation changed"));
|
|
448
|
+
this.state.spool = [];
|
|
449
|
+
this.state.completedCommands = [];
|
|
450
|
+
this.state.acceptedCommands = {};
|
|
451
|
+
this.state.runtime = {
|
|
452
|
+
runtimeId: null,
|
|
453
|
+
nativeSessionId: null,
|
|
454
|
+
contextRevision: 0,
|
|
455
|
+
};
|
|
456
|
+
this.state.nextOutboundSeq = 0;
|
|
457
|
+
this.state.sessionGeneration = frame.session_generation;
|
|
458
|
+
}
|
|
459
|
+
this.sessionGeneration = frame.session_generation;
|
|
460
|
+
this.connectionEpoch = frame.connection_epoch;
|
|
461
|
+
this.inboundSeq = frame.seq;
|
|
462
|
+
const heartbeatSeconds = Number(frame.payload.heartbeat_seconds ?? 15);
|
|
463
|
+
this.heartbeatMs = Math.max(1_000, Math.min(60_000, heartbeatSeconds * 1000));
|
|
464
|
+
const staleSeconds = Number(frame.payload.stale_seconds ?? 45);
|
|
465
|
+
// The server owns this deadline. Keep only a defensive protocol floor/ceiling rather
|
|
466
|
+
// than stretching it relative to the heartbeat and silently ignoring its contract.
|
|
467
|
+
this.staleMs = Math.min(300_000, Math.max(1_000, staleSeconds * 1000));
|
|
468
|
+
this.persist();
|
|
469
|
+
await this.sendControlFrame("session.ready", {
|
|
470
|
+
daemon_version: this.options.daemonVersion,
|
|
471
|
+
protocol_versions: ["botlearn-agent-sandbox-ws/0.1"],
|
|
472
|
+
local_event_cursor: this.state.spool.at(-1)?.seq ?? 0,
|
|
473
|
+
spool_frames: this.state.spool.length,
|
|
474
|
+
spool_bytes: this.spoolBytes(),
|
|
475
|
+
active_run_id: this.dispatcher.activeCount > 0 ? "active" : null,
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
assertServerFrame(frame) {
|
|
479
|
+
if (frame.session_id !== this.options.sessionId ||
|
|
480
|
+
frame.session_generation !== this.sessionGeneration ||
|
|
481
|
+
frame.connection_epoch !== this.connectionEpoch) {
|
|
482
|
+
throw new SessionClosedError(4409, "connection_fenced");
|
|
483
|
+
}
|
|
484
|
+
if (frame.seq !== this.inboundSeq + 1)
|
|
485
|
+
throw new Error("server frame sequence gap");
|
|
486
|
+
}
|
|
487
|
+
async handleServerFrame(frame) {
|
|
488
|
+
switch (frame.type) {
|
|
489
|
+
case "session.sync":
|
|
490
|
+
await this.replaySpool();
|
|
491
|
+
await this.applyDesiredState(frame);
|
|
492
|
+
return;
|
|
493
|
+
case "event.ack":
|
|
494
|
+
this.ackEventOrFile(frame);
|
|
495
|
+
return;
|
|
496
|
+
case "turn.file.upload_grant":
|
|
497
|
+
this.acceptFileGrant(frame);
|
|
498
|
+
return;
|
|
499
|
+
case "ping":
|
|
500
|
+
await this.sendControlFrame("pong", { ping_frame_id: frame.frame_id });
|
|
501
|
+
return;
|
|
502
|
+
case "auth.rotate": {
|
|
503
|
+
const token = frame.payload.reconnect_token;
|
|
504
|
+
if (typeof token !== "string" || !token)
|
|
505
|
+
throw new Error("invalid reconnect token");
|
|
506
|
+
this.state.reconnectToken = token;
|
|
507
|
+
this.persist();
|
|
508
|
+
await this.sendCommandAck(frame);
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
case "session.shutdown":
|
|
512
|
+
this.stop();
|
|
513
|
+
return;
|
|
514
|
+
case "turn.cancel":
|
|
515
|
+
if (frame.agent_run_id)
|
|
516
|
+
this.dispatcher.cancel(frame.agent_run_id);
|
|
517
|
+
await this.sendCommandAck(frame);
|
|
518
|
+
return;
|
|
519
|
+
case "session.drain":
|
|
520
|
+
await this.sendCommandAck(frame);
|
|
521
|
+
if (await this.dispatcher.drain(10_000)) {
|
|
522
|
+
await this.sendControlFrame("session.drained", {});
|
|
523
|
+
}
|
|
524
|
+
return;
|
|
525
|
+
case "protocol.error":
|
|
526
|
+
throw new Error(`server protocol error: ${String(frame.payload.code ?? "unknown")}`);
|
|
527
|
+
default:
|
|
528
|
+
throw new Error(`unexpected server frame: ${frame.type}`);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
async applyDesiredState(frame) {
|
|
532
|
+
const state = frame.payload.state;
|
|
533
|
+
const runId = typeof frame.payload.active_agent_run_id === "string"
|
|
534
|
+
? frame.payload.active_agent_run_id
|
|
535
|
+
: null;
|
|
536
|
+
const attempt = Number(frame.payload.worker_attempt ?? 0);
|
|
537
|
+
if (state === "idle")
|
|
538
|
+
return;
|
|
539
|
+
if (state === "shutdown") {
|
|
540
|
+
this.stop();
|
|
541
|
+
return;
|
|
542
|
+
}
|
|
543
|
+
if (state === "drain") {
|
|
544
|
+
if (await this.dispatcher.drain(10_000)) {
|
|
545
|
+
await this.sendControlFrame("session.drained", {});
|
|
546
|
+
}
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
if (!runId || !Number.isInteger(attempt) || attempt < 1) {
|
|
550
|
+
throw new Error("desired turn is missing fencing fields");
|
|
551
|
+
}
|
|
552
|
+
if (state === "cancel") {
|
|
553
|
+
this.dispatcher.cancel(runId);
|
|
554
|
+
await this.sendCommandAck(frame);
|
|
555
|
+
return;
|
|
556
|
+
}
|
|
557
|
+
if (state !== "run" || !frame.payload.run || typeof frame.payload.run !== "object") {
|
|
558
|
+
throw new Error("invalid desired session state");
|
|
559
|
+
}
|
|
560
|
+
const commandId = `run:${runId}:${attempt}`;
|
|
561
|
+
this.attempts.set(runId, attempt);
|
|
562
|
+
if (this.state.completedCommands.includes(commandId) ||
|
|
563
|
+
this.inflightCommands.has(commandId)) {
|
|
564
|
+
await this.sendCommandAck(frame, commandId);
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
const payload = frame.payload.run;
|
|
568
|
+
const activation = this.runtimeActivation(payload);
|
|
569
|
+
const accepted = this.state.acceptedCommands[commandId];
|
|
570
|
+
if (accepted && !this.inflightCommands.has(commandId)) {
|
|
571
|
+
await this.sendCommandAck(frame, commandId);
|
|
572
|
+
void this.reconcileInterruptedCommand(commandId, accepted);
|
|
573
|
+
return;
|
|
574
|
+
}
|
|
575
|
+
if (!accepted) {
|
|
576
|
+
if (this.state.runtime.runtimeId !== null &&
|
|
577
|
+
this.state.runtime.runtimeId !== activation.runtimeId) {
|
|
578
|
+
throw new Error("runtime changed inside one persistent session generation");
|
|
579
|
+
}
|
|
580
|
+
if (activation.contextRevision !== this.state.runtime.contextRevision + 1) {
|
|
581
|
+
throw new Error(`runtime context revision gap: expected ${this.state.runtime.contextRevision + 1}, ` +
|
|
582
|
+
`received ${activation.contextRevision}`);
|
|
583
|
+
}
|
|
584
|
+
this.state.runtime.runtimeId = activation.runtimeId;
|
|
585
|
+
this.state.runtime.contextRevision = activation.contextRevision;
|
|
586
|
+
this.state.acceptedCommands[commandId] = {
|
|
587
|
+
agentRunId: runId,
|
|
588
|
+
workerAttempt: attempt,
|
|
589
|
+
contextRevision: activation.contextRevision,
|
|
590
|
+
acceptedAt: new Date().toISOString(),
|
|
591
|
+
};
|
|
592
|
+
this.persist();
|
|
593
|
+
}
|
|
594
|
+
const embeddedProfile = payload.context.runtimeProfile;
|
|
595
|
+
if (embeddedProfile && typeof embeddedProfile === "object") {
|
|
596
|
+
this.runtimeProfiles.set(runId, embeddedProfile);
|
|
597
|
+
}
|
|
598
|
+
this.inflightCommands.add(commandId);
|
|
599
|
+
await this.sendCommandAck(frame, commandId);
|
|
600
|
+
void this.dispatcher.dispatch(payload)
|
|
601
|
+
.then(() => {
|
|
602
|
+
this.state.completedCommands.push(commandId);
|
|
603
|
+
this.state.completedCommands = this.state.completedCommands.slice(-256);
|
|
604
|
+
delete this.state.acceptedCommands[commandId];
|
|
605
|
+
this.persist();
|
|
606
|
+
})
|
|
607
|
+
.finally(() => {
|
|
608
|
+
this.inflightCommands.delete(commandId);
|
|
609
|
+
this.runtimeProfiles.delete(runId);
|
|
610
|
+
});
|
|
611
|
+
}
|
|
612
|
+
async reconcileInterruptedCommand(commandId, accepted) {
|
|
613
|
+
try {
|
|
614
|
+
this.attempts.set(accepted.agentRunId, accepted.workerAttempt);
|
|
615
|
+
await this.postEvent(accepted.agentRunId, {
|
|
616
|
+
type: "run.failed",
|
|
617
|
+
event_id: `recovery-${commandId.replaceAll(":", "-")}`,
|
|
618
|
+
seq: 1,
|
|
619
|
+
error: "daemon restarted after accepting the turn; execution outcome is unknown",
|
|
620
|
+
payload: {
|
|
621
|
+
error_type: "execution_outcome_unknown",
|
|
622
|
+
retryable: true,
|
|
623
|
+
session_disposition: "close_generation",
|
|
624
|
+
context_revision: accepted.contextRevision,
|
|
625
|
+
},
|
|
626
|
+
});
|
|
627
|
+
this.state.completedCommands.push(commandId);
|
|
628
|
+
this.state.completedCommands = this.state.completedCommands.slice(-256);
|
|
629
|
+
delete this.state.acceptedCommands[commandId];
|
|
630
|
+
this.persist();
|
|
631
|
+
}
|
|
632
|
+
catch (error) {
|
|
633
|
+
this.log.error("failed to reconcile an interrupted accepted turn", {
|
|
634
|
+
runtimeSessionId: this.options.sessionId,
|
|
635
|
+
agentRunId: accepted.agentRunId,
|
|
636
|
+
workerAttempt: accepted.workerAttempt,
|
|
637
|
+
error: error instanceof Error ? redactSecretString(error.message) : String(error),
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
runtimeActivation(payload) {
|
|
642
|
+
const raw = payload.context.runtimeSession;
|
|
643
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
644
|
+
throw new Error("managed turn is missing runtime session context");
|
|
645
|
+
}
|
|
646
|
+
const value = raw;
|
|
647
|
+
const runtimeSessionId = value.runtimeSessionId;
|
|
648
|
+
const generation = Number(value.sessionGeneration ?? 0);
|
|
649
|
+
const contextRevision = Number(value.contextRevision ?? 0);
|
|
650
|
+
const runtimeId = typeof payload.runtime.id === "string" ? payload.runtime.id.trim() : "";
|
|
651
|
+
if (value.schemaVersion !== "agent-runtime-session-context/0.1" ||
|
|
652
|
+
runtimeSessionId !== this.options.sessionId ||
|
|
653
|
+
generation !== this.sessionGeneration ||
|
|
654
|
+
!Number.isInteger(contextRevision) ||
|
|
655
|
+
contextRevision < 1 ||
|
|
656
|
+
!runtimeId) {
|
|
657
|
+
throw new Error("managed turn runtime session context is invalid");
|
|
658
|
+
}
|
|
659
|
+
return { runtimeId, contextRevision };
|
|
660
|
+
}
|
|
661
|
+
ackEventOrFile(frame) {
|
|
662
|
+
const frameId = frame.payload.frame_id;
|
|
663
|
+
if (typeof frameId !== "string")
|
|
664
|
+
throw new Error("event.ack has no frame_id");
|
|
665
|
+
const pendingFile = this.pendingFileCommits.get(frameId);
|
|
666
|
+
if (pendingFile) {
|
|
667
|
+
this.pendingFileCommits.delete(frameId);
|
|
668
|
+
const file = frame.payload.file;
|
|
669
|
+
if (!file || typeof file !== "object" || Array.isArray(file)) {
|
|
670
|
+
pendingFile.reject(new Error("file commit ACK has no file record"));
|
|
671
|
+
}
|
|
672
|
+
else {
|
|
673
|
+
pendingFile.resolve(file);
|
|
674
|
+
}
|
|
675
|
+
return;
|
|
676
|
+
}
|
|
677
|
+
const index = this.state.spool.findIndex((item) => item.frame_id === frameId);
|
|
678
|
+
if (index >= 0)
|
|
679
|
+
this.state.spool.splice(index, 1);
|
|
680
|
+
this.persist();
|
|
681
|
+
const pending = this.pendingAcks.get(frameId);
|
|
682
|
+
if (pending) {
|
|
683
|
+
this.pendingAcks.delete(frameId);
|
|
684
|
+
pending.resolve();
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
acceptFileGrant(frame) {
|
|
688
|
+
const requestFrameId = frame.payload.request_frame_id;
|
|
689
|
+
if (typeof requestFrameId !== "string")
|
|
690
|
+
throw new Error("file grant has no request_frame_id");
|
|
691
|
+
const pending = this.pendingFilePrepares.get(requestFrameId);
|
|
692
|
+
if (!pending)
|
|
693
|
+
return;
|
|
694
|
+
this.pendingFilePrepares.delete(requestFrameId);
|
|
695
|
+
const file = frame.payload.file;
|
|
696
|
+
const uploadUrl = frame.payload.upload_url;
|
|
697
|
+
const uploadGrant = frame.payload.upload_grant;
|
|
698
|
+
if (!file ||
|
|
699
|
+
typeof file !== "object" ||
|
|
700
|
+
Array.isArray(file) ||
|
|
701
|
+
typeof uploadUrl !== "string" ||
|
|
702
|
+
!uploadUrl ||
|
|
703
|
+
typeof uploadGrant !== "string" ||
|
|
704
|
+
!uploadGrant) {
|
|
705
|
+
pending.reject(new Error("invalid Agent Service file upload grant"));
|
|
706
|
+
return;
|
|
707
|
+
}
|
|
708
|
+
pending.resolve({
|
|
709
|
+
record: file,
|
|
710
|
+
uploadUrl,
|
|
711
|
+
uploadGrant,
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
async replaySpool() {
|
|
715
|
+
for (const original of this.state.spool) {
|
|
716
|
+
const replay = {
|
|
717
|
+
...original,
|
|
718
|
+
session_generation: this.sessionGeneration,
|
|
719
|
+
connection_epoch: this.connectionEpoch,
|
|
720
|
+
seq: this.nextOutboundSeq(),
|
|
721
|
+
sent_at: new Date().toISOString(),
|
|
722
|
+
};
|
|
723
|
+
Object.assign(original, replay);
|
|
724
|
+
this.persist();
|
|
725
|
+
await this.sendFrame(original);
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
async sendHeartbeat() {
|
|
729
|
+
if (!this.socket ||
|
|
730
|
+
this.socket.readyState !== WebSocketClient.OPEN ||
|
|
731
|
+
!this.sessionGeneration)
|
|
732
|
+
return;
|
|
733
|
+
await this.sendControlFrame("session.heartbeat", {
|
|
734
|
+
active_turns: this.dispatcher.activeCount,
|
|
735
|
+
spool_frames: this.state.spool.length,
|
|
736
|
+
spool_bytes: this.spoolBytes(),
|
|
737
|
+
});
|
|
738
|
+
}
|
|
739
|
+
async sendCommandAck(frame, commandId) {
|
|
740
|
+
await this.sendControlFrame("command.ack", {
|
|
741
|
+
command_id: commandId ?? frame.frame_id,
|
|
742
|
+
command_seq: frame.seq,
|
|
743
|
+
accepted: true,
|
|
744
|
+
});
|
|
745
|
+
}
|
|
746
|
+
async sendControlFrame(type, payload) {
|
|
747
|
+
await this.sendFrame(createSandboxFrame({
|
|
748
|
+
type,
|
|
749
|
+
sessionId: this.options.sessionId,
|
|
750
|
+
sessionGeneration: this.sessionGeneration,
|
|
751
|
+
connectionEpoch: this.connectionEpoch,
|
|
752
|
+
seq: this.nextOutboundSeq(),
|
|
753
|
+
payload,
|
|
754
|
+
}));
|
|
755
|
+
}
|
|
756
|
+
async sendFrame(frame) {
|
|
757
|
+
const socket = this.socket;
|
|
758
|
+
if (!socket || socket.readyState !== WebSocketClient.OPEN)
|
|
759
|
+
return;
|
|
760
|
+
await new Promise((resolve, reject) => {
|
|
761
|
+
socket.send(JSON.stringify(frame), (error) => (error ? reject(error) : resolve()));
|
|
762
|
+
});
|
|
763
|
+
}
|
|
764
|
+
nextOutboundSeq() {
|
|
765
|
+
this.state.nextOutboundSeq += 1;
|
|
766
|
+
this.persist();
|
|
767
|
+
return this.state.nextOutboundSeq;
|
|
768
|
+
}
|
|
769
|
+
spoolBytes() {
|
|
770
|
+
return Buffer.byteLength(JSON.stringify(this.state.spool), "utf8");
|
|
771
|
+
}
|
|
772
|
+
enforceSpoolLimit() {
|
|
773
|
+
if (this.state.spool.length > MAX_SPOOL_FRAMES || this.spoolBytes() > MAX_SPOOL_BYTES) {
|
|
774
|
+
this.dispatcher.cancelAll();
|
|
775
|
+
throw new Error("Agent Service session event spool exceeded its bounded limit");
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
rejectPending(error) {
|
|
779
|
+
for (const pending of this.pendingAcks.values())
|
|
780
|
+
pending.reject(error);
|
|
781
|
+
this.pendingAcks.clear();
|
|
782
|
+
this.rejectPendingFiles(error);
|
|
783
|
+
}
|
|
784
|
+
rejectPendingFiles(error) {
|
|
785
|
+
for (const pending of this.pendingFilePrepares.values())
|
|
786
|
+
pending.reject(error);
|
|
787
|
+
this.pendingFilePrepares.clear();
|
|
788
|
+
for (const pending of this.pendingFileCommits.values())
|
|
789
|
+
pending.reject(error);
|
|
790
|
+
this.pendingFileCommits.clear();
|
|
791
|
+
this.fileGrants.clear();
|
|
792
|
+
}
|
|
793
|
+
persist() {
|
|
794
|
+
saveState(this.options.sessionId, this.state);
|
|
795
|
+
}
|
|
796
|
+
}
|