crewx-agent-cli 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -4
- package/dist/adapters.d.ts +12 -0
- package/dist/adapters.d.ts.map +1 -1
- package/dist/adapters.js +112 -40
- package/dist/adapters.js.map +1 -1
- package/dist/api.d.ts +40 -17
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +187 -11
- package/dist/api.js.map +1 -1
- package/dist/bridge-watchdog.d.ts +10 -0
- package/dist/bridge-watchdog.d.ts.map +1 -0
- package/dist/bridge-watchdog.js +45 -0
- package/dist/bridge-watchdog.js.map +1 -0
- package/dist/config.d.ts +9 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +53 -2
- package/dist/config.js.map +1 -1
- package/dist/constants.d.ts +6 -1
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +11 -1
- package/dist/constants.js.map +1 -1
- package/dist/daemon.d.ts +32 -1
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +479 -100
- package/dist/daemon.js.map +1 -1
- package/dist/errors.d.ts +4 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +3 -1
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -6
- package/dist/index.js.map +1 -1
- package/dist/join.d.ts +1 -0
- package/dist/join.d.ts.map +1 -1
- package/dist/join.js +13 -0
- package/dist/join.js.map +1 -1
- package/dist/outbox.d.ts +22 -0
- package/dist/outbox.d.ts.map +1 -0
- package/dist/outbox.js +189 -0
- package/dist/outbox.js.map +1 -0
- package/dist/process-lock.d.ts +31 -0
- package/dist/process-lock.d.ts.map +1 -0
- package/dist/process-lock.js +272 -0
- package/dist/process-lock.js.map +1 -0
- package/dist/prompt.d.ts.map +1 -1
- package/dist/prompt.js +7 -0
- package/dist/prompt.js.map +1 -1
- package/dist/runtime-proxy.d.ts +18 -0
- package/dist/runtime-proxy.d.ts.map +1 -0
- package/dist/runtime-proxy.js +221 -0
- package/dist/runtime-proxy.js.map +1 -0
- package/package.json +6 -2
package/dist/daemon.js
CHANGED
|
@@ -3,13 +3,17 @@ import { realpathSync } from "node:fs";
|
|
|
3
3
|
import { hostname, platform, arch } from "node:os";
|
|
4
4
|
import { isAbsolute, relative, resolve, sep } from "node:path";
|
|
5
5
|
import { runAdapter, verifyOpenClawRuntime, } from "./adapters.js";
|
|
6
|
-
import { ApiError } from "./api.js";
|
|
7
|
-
import { CLI_VERSION, MAX_EVENT_CONTENT_LENGTH } from "./constants.js";
|
|
8
|
-
import { errorMessage, redactSecrets } from "./errors.js";
|
|
6
|
+
import { ApiError, serializeOutboundEventBatch, } from "./api.js";
|
|
7
|
+
import { CLI_VERSION, MAX_BUFFERED_EVENT_BYTES, MAX_BUFFERED_EVENTS, MAX_EVENT_BATCH_BYTES, MAX_EVENT_CONTENT_LENGTH, } from "./constants.js";
|
|
8
|
+
import { CliError, errorMessage, redactSecrets } from "./errors.js";
|
|
9
9
|
import { assemblePrompt, assignmentFromEvent } from "./prompt.js";
|
|
10
|
+
import { startRuntimeCredentialProxy, } from "./runtime-proxy.js";
|
|
11
|
+
import { DurableEventOutbox, agentEventOutboxPath, eventOutboxLockPath, } from "./outbox.js";
|
|
12
|
+
import { AgentProcessLock, } from "./process-lock.js";
|
|
10
13
|
const ASSIGNMENT_CONTROL_INTERVAL_MS = 5_000;
|
|
11
14
|
const EVENT_RETRY_BASE_DELAY_MS = 2_000;
|
|
12
15
|
const EVENT_RETRY_MAX_DELAY_MS = 30_000;
|
|
16
|
+
const MAX_IDLE_POLL_DELAY_MS = 8_000;
|
|
13
17
|
const MAX_RUNTIME_DIAGNOSTICS = 20;
|
|
14
18
|
function timestamp() {
|
|
15
19
|
return new Date().toISOString();
|
|
@@ -22,6 +26,19 @@ function event(type, payload) {
|
|
|
22
26
|
occurred_at: timestamp(),
|
|
23
27
|
};
|
|
24
28
|
}
|
|
29
|
+
function assignmentEvent(assignment, type, payload) {
|
|
30
|
+
return {
|
|
31
|
+
id: crypto.randomUUID(),
|
|
32
|
+
type,
|
|
33
|
+
payload: {
|
|
34
|
+
...payload,
|
|
35
|
+
...(assignment.claim_token
|
|
36
|
+
? { claim_token: assignment.claim_token }
|
|
37
|
+
: {}),
|
|
38
|
+
},
|
|
39
|
+
occurred_at: timestamp(),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
25
42
|
function truncate(value) {
|
|
26
43
|
return value.length > MAX_EVENT_CONTENT_LENGTH
|
|
27
44
|
? `${value.slice(0, MAX_EVENT_CONTENT_LENGTH)}… [truncated]`
|
|
@@ -43,6 +60,30 @@ function record(value) {
|
|
|
43
60
|
function nonEmptyString(value) {
|
|
44
61
|
return typeof value === "string" && value.trim() ? value.trim() : undefined;
|
|
45
62
|
}
|
|
63
|
+
function runtimeFailure(adapter, detail) {
|
|
64
|
+
if (adapter !== "openclaw") {
|
|
65
|
+
return { code: "runtime_exit_nonzero", message: detail };
|
|
66
|
+
}
|
|
67
|
+
if (/\b429\b|rate[ -]?limit|quota|too many requests/i.test(detail)) {
|
|
68
|
+
return {
|
|
69
|
+
code: "openclaw_provider_rate_limited",
|
|
70
|
+
message: `${detail} Check \`openclaw models status --agent <agent-id> --json\` for provider cooldowns, then retry after the provider limit resets.`,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
if (/unauthori[sz]ed|authentication|credential|oauth|invalid token|expired/i.test(detail)) {
|
|
74
|
+
return {
|
|
75
|
+
code: "openclaw_auth_failed",
|
|
76
|
+
message: `${detail} Run \`openclaw models auth --agent <agent-id> list --json\` and repair the selected provider credentials.`,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
if (/model.+(?:not found|unknown|unavailable|not allowed)/i.test(detail)) {
|
|
80
|
+
return {
|
|
81
|
+
code: "openclaw_model_unavailable",
|
|
82
|
+
message: `${detail} Run \`openclaw models status --agent <agent-id> --json\` and select an available model.`,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
return { code: "openclaw_runtime_failed", message: detail };
|
|
86
|
+
}
|
|
46
87
|
function identifier(value) {
|
|
47
88
|
if (typeof value === "number" && Number.isSafeInteger(value))
|
|
48
89
|
return String(value);
|
|
@@ -130,7 +171,7 @@ export function assignmentWorkingDirectory(assignment, context, fallback, locked
|
|
|
130
171
|
if (relativePath === ".." ||
|
|
131
172
|
relativePath.startsWith(`..${sep}`) ||
|
|
132
173
|
isAbsolute(relativePath)) {
|
|
133
|
-
throw new Error(
|
|
174
|
+
throw new Error("CrewX Bridge rejected the assignment directory because it is outside the approved folder.");
|
|
134
175
|
}
|
|
135
176
|
return canonicalCandidate;
|
|
136
177
|
}
|
|
@@ -138,18 +179,65 @@ export class EventBuffer {
|
|
|
138
179
|
api;
|
|
139
180
|
sessionId;
|
|
140
181
|
queue = [];
|
|
182
|
+
inFlight = new Set();
|
|
141
183
|
flushing;
|
|
142
184
|
retryNotBefore = 0;
|
|
143
185
|
consecutiveFailures = 0;
|
|
144
|
-
|
|
186
|
+
maxEvents;
|
|
187
|
+
maxBytes;
|
|
188
|
+
random;
|
|
189
|
+
durableSnapshot;
|
|
190
|
+
constructor(api, sessionId, options = {}) {
|
|
145
191
|
this.api = api;
|
|
146
192
|
this.sessionId = sessionId;
|
|
193
|
+
this.maxEvents = options.maxEvents ?? MAX_BUFFERED_EVENTS;
|
|
194
|
+
this.maxBytes = options.maxBytes ?? MAX_BUFFERED_EVENT_BYTES;
|
|
195
|
+
this.random = options.random ?? Math.random;
|
|
196
|
+
this.outbox = options.outbox;
|
|
197
|
+
if (this.outbox) {
|
|
198
|
+
this.queue.push(...this.outbox.load());
|
|
199
|
+
this.enforceBounds();
|
|
200
|
+
}
|
|
201
|
+
this.durableSnapshot = this.serializeDurableEvents();
|
|
147
202
|
}
|
|
203
|
+
outbox;
|
|
148
204
|
push(item) {
|
|
149
|
-
this.
|
|
205
|
+
this.pushMany([item]);
|
|
206
|
+
}
|
|
207
|
+
pushMany(items) {
|
|
208
|
+
if (items.length === 0)
|
|
209
|
+
return;
|
|
210
|
+
const previous = [...this.queue];
|
|
211
|
+
try {
|
|
212
|
+
for (const item of items) {
|
|
213
|
+
if (!this.coalesce(item))
|
|
214
|
+
this.queue.push(item);
|
|
215
|
+
}
|
|
216
|
+
this.enforceBounds();
|
|
217
|
+
this.persistDurableEvents();
|
|
218
|
+
}
|
|
219
|
+
catch (error) {
|
|
220
|
+
this.queue.splice(0, this.queue.length, ...previous);
|
|
221
|
+
throw error;
|
|
222
|
+
}
|
|
150
223
|
if (this.queue.length >= 25)
|
|
151
224
|
void this.flush().catch(() => undefined);
|
|
152
225
|
}
|
|
226
|
+
pendingCount() {
|
|
227
|
+
return this.queue.length;
|
|
228
|
+
}
|
|
229
|
+
pendingBytes() {
|
|
230
|
+
return this.queue.reduce((total, item) => total + this.eventBytes(item), 0);
|
|
231
|
+
}
|
|
232
|
+
hasPendingTerminalOutcome(assignmentId) {
|
|
233
|
+
const isTerminalOutcome = (item) => item.type === "lifecycle" &&
|
|
234
|
+
item.payload.assignment_id === assignmentId &&
|
|
235
|
+
(item.payload.stage === "completed" ||
|
|
236
|
+
item.payload.stage === "failed" ||
|
|
237
|
+
item.payload.stage === "cancelled");
|
|
238
|
+
return (this.queue.some(isTerminalOutcome) ||
|
|
239
|
+
[...this.inFlight].some(isTerminalOutcome));
|
|
240
|
+
}
|
|
153
241
|
async flush(signal) {
|
|
154
242
|
if (this.flushing) {
|
|
155
243
|
await this.flushing;
|
|
@@ -171,21 +259,143 @@ export class EventBuffer {
|
|
|
171
259
|
await sleep(retryDelay, signal);
|
|
172
260
|
if (signal?.aborted)
|
|
173
261
|
return;
|
|
174
|
-
const batch =
|
|
262
|
+
const batch = [];
|
|
263
|
+
for (const item of this.queue) {
|
|
264
|
+
if (batch.length >= 100) {
|
|
265
|
+
break;
|
|
266
|
+
}
|
|
267
|
+
const candidate = [...batch, item];
|
|
268
|
+
const candidateBytes = Buffer.byteLength(serializeOutboundEventBatch(this.sessionId, candidate), "utf8");
|
|
269
|
+
if (candidateBytes > MAX_EVENT_BATCH_BYTES) {
|
|
270
|
+
if (batch.length === 0) {
|
|
271
|
+
throw new CliError(`CrewX cannot send event ${item.id ?? "(without an ID)"} because its exact serialized batch exceeds ${String(MAX_EVENT_BATCH_BYTES)} bytes.`, 1, { code: "event_batch_too_large" });
|
|
272
|
+
}
|
|
273
|
+
break;
|
|
274
|
+
}
|
|
275
|
+
batch.push(item);
|
|
276
|
+
}
|
|
277
|
+
for (const item of batch)
|
|
278
|
+
this.inFlight.add(item);
|
|
175
279
|
try {
|
|
176
280
|
await this.api.sendEvents(this.sessionId, batch, signal);
|
|
177
|
-
this.queue.splice(0, batch.length);
|
|
281
|
+
const delivered = this.queue.splice(0, batch.length);
|
|
282
|
+
try {
|
|
283
|
+
this.persistDurableEvents();
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
this.queue.unshift(...delivered);
|
|
287
|
+
throw error;
|
|
288
|
+
}
|
|
178
289
|
this.consecutiveFailures = 0;
|
|
179
290
|
this.retryNotBefore = 0;
|
|
180
291
|
}
|
|
181
292
|
catch (error) {
|
|
182
293
|
this.consecutiveFailures += 1;
|
|
183
294
|
const exponentialDelay = Math.min(EVENT_RETRY_MAX_DELAY_MS, EVENT_RETRY_BASE_DELAY_MS * 2 ** (this.consecutiveFailures - 1));
|
|
295
|
+
const jitterDelay = Math.floor(this.random() * (exponentialDelay + 1));
|
|
184
296
|
const serverDelay = error instanceof ApiError ? error.retryAfterMs : undefined;
|
|
185
297
|
this.retryNotBefore =
|
|
186
|
-
Date.now() + Math.max(
|
|
298
|
+
Date.now() + Math.max(jitterDelay, serverDelay ?? 0);
|
|
187
299
|
throw error;
|
|
188
300
|
}
|
|
301
|
+
finally {
|
|
302
|
+
for (const item of batch)
|
|
303
|
+
this.inFlight.delete(item);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
coalesce(item) {
|
|
308
|
+
if (item.id) {
|
|
309
|
+
const duplicateIndex = this.findNewestIndex((candidate) => candidate.id === item.id);
|
|
310
|
+
if (duplicateIndex >= 0) {
|
|
311
|
+
this.queue[duplicateIndex] = item;
|
|
312
|
+
return true;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
if (item.type === "heartbeat") {
|
|
316
|
+
const existingIndex = this.findNewestIndex((candidate) => candidate.type === "heartbeat");
|
|
317
|
+
if (existingIndex >= 0) {
|
|
318
|
+
this.queue[existingIndex] = item;
|
|
319
|
+
return true;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
if (item.type === "output") {
|
|
323
|
+
const existingIndex = this.findNewestIndex((candidate) => candidate.type === "output" &&
|
|
324
|
+
candidate.payload.assignment_id === item.payload.assignment_id &&
|
|
325
|
+
candidate.payload.stream === item.payload.stream);
|
|
326
|
+
const existing = this.queue[existingIndex];
|
|
327
|
+
if (existingIndex >= 0 && existing?.type === "output") {
|
|
328
|
+
const combined = `${existing.payload.content}\n${item.payload.content}`;
|
|
329
|
+
const content = combined.length <= MAX_EVENT_CONTENT_LENGTH
|
|
330
|
+
? combined
|
|
331
|
+
: `[earlier output omitted during backpressure]\n${combined.slice(-(MAX_EVENT_CONTENT_LENGTH - 47))}`;
|
|
332
|
+
this.queue[existingIndex] = {
|
|
333
|
+
...item,
|
|
334
|
+
payload: {
|
|
335
|
+
...item.payload,
|
|
336
|
+
content,
|
|
337
|
+
},
|
|
338
|
+
};
|
|
339
|
+
return true;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
return false;
|
|
343
|
+
}
|
|
344
|
+
findNewestIndex(predicate) {
|
|
345
|
+
for (let index = this.queue.length - 1; index >= 0; index -= 1) {
|
|
346
|
+
const candidate = this.queue[index];
|
|
347
|
+
if (candidate &&
|
|
348
|
+
!this.inFlight.has(candidate) &&
|
|
349
|
+
predicate(candidate)) {
|
|
350
|
+
return index;
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
return -1;
|
|
354
|
+
}
|
|
355
|
+
eventPriority(item) {
|
|
356
|
+
if (item.type === "heartbeat" || item.type === "output")
|
|
357
|
+
return 0;
|
|
358
|
+
if (item.type === "error" ||
|
|
359
|
+
(item.type === "message" && item.payload.role === "assistant") ||
|
|
360
|
+
(item.type === "lifecycle" &&
|
|
361
|
+
["completed", "failed", "cancelled", "stopped"].includes(item.payload.stage))) {
|
|
362
|
+
return 2;
|
|
363
|
+
}
|
|
364
|
+
return 1;
|
|
365
|
+
}
|
|
366
|
+
eventBytes(item) {
|
|
367
|
+
return Buffer.byteLength(JSON.stringify(item));
|
|
368
|
+
}
|
|
369
|
+
persistDurableEvents() {
|
|
370
|
+
if (!this.outbox)
|
|
371
|
+
return;
|
|
372
|
+
const durable = this.queue.filter((item) => this.eventPriority(item) === 2);
|
|
373
|
+
const snapshot = JSON.stringify(durable);
|
|
374
|
+
if (snapshot === this.durableSnapshot)
|
|
375
|
+
return;
|
|
376
|
+
this.outbox.replace(durable);
|
|
377
|
+
this.durableSnapshot = snapshot;
|
|
378
|
+
}
|
|
379
|
+
serializeDurableEvents() {
|
|
380
|
+
return JSON.stringify(this.queue.filter((item) => this.eventPriority(item) === 2));
|
|
381
|
+
}
|
|
382
|
+
enforceBounds() {
|
|
383
|
+
let bytes = this.pendingBytes();
|
|
384
|
+
while (this.queue.length > this.maxEvents ||
|
|
385
|
+
bytes > this.maxBytes) {
|
|
386
|
+
let removeIndex = -1;
|
|
387
|
+
for (const priority of [0, 1]) {
|
|
388
|
+
removeIndex = this.queue.findIndex((item) => !this.inFlight.has(item) &&
|
|
389
|
+
this.eventPriority(item) === priority);
|
|
390
|
+
if (removeIndex >= 0)
|
|
391
|
+
break;
|
|
392
|
+
}
|
|
393
|
+
if (removeIndex < 0) {
|
|
394
|
+
throw new CliError("CrewX cannot buffer another final agent outcome while event delivery is unavailable. Existing final outcomes were retained; restore connectivity before accepting more work.", 1, { code: "event_buffer_capacity_exceeded" });
|
|
395
|
+
}
|
|
396
|
+
const [removed] = this.queue.splice(removeIndex, 1);
|
|
397
|
+
if (removed)
|
|
398
|
+
bytes -= this.eventBytes(removed);
|
|
189
399
|
}
|
|
190
400
|
}
|
|
191
401
|
}
|
|
@@ -202,6 +412,18 @@ function sleep(milliseconds, signal) {
|
|
|
202
412
|
signal?.addEventListener("abort", finish, { once: true });
|
|
203
413
|
});
|
|
204
414
|
}
|
|
415
|
+
export function adaptiveIdlePollDelay(baseDelayMs, consecutiveEmptyPolls, options = {}) {
|
|
416
|
+
const base = Math.max(250, Math.floor(baseDelayMs));
|
|
417
|
+
const maximum = Math.max(base, MAX_IDLE_POLL_DELAY_MS);
|
|
418
|
+
const exponent = Math.max(0, Math.min(8, consecutiveEmptyPolls - 1));
|
|
419
|
+
const cap = Math.min(maximum, base * 2 ** exponent);
|
|
420
|
+
const random = options.random ?? Math.random;
|
|
421
|
+
const jittered = cap <= base
|
|
422
|
+
? base
|
|
423
|
+
: base +
|
|
424
|
+
Math.round(Math.max(0, Math.min(1, random())) * (cap - base));
|
|
425
|
+
return Math.max(jittered, options.retryAfterMs ?? 0);
|
|
426
|
+
}
|
|
205
427
|
function linkedAbortController(parent) {
|
|
206
428
|
const controller = new AbortController();
|
|
207
429
|
const abort = () => controller.abort(parent?.reason);
|
|
@@ -222,17 +444,24 @@ async function monitorAssignmentControl(options) {
|
|
|
222
444
|
while (!options.isFinished() && !options.controller.signal.aborted) {
|
|
223
445
|
let nextCheckDelay = ASSIGNMENT_CONTROL_INTERVAL_MS;
|
|
224
446
|
try {
|
|
225
|
-
const control = await options.api.control(options.sessionId, options.assignmentId, options.controller.signal);
|
|
447
|
+
const control = await options.api.control(options.sessionId, options.assignmentId, options.controller.signal, options.assignmentEventCursor);
|
|
226
448
|
if (control.cancelled) {
|
|
227
449
|
options.log(`CrewX cancelled ${options.assignmentId}; stopping the local agent…`);
|
|
228
450
|
options.controller.abort(new Error("Assignment cancelled by CrewX"));
|
|
229
|
-
return;
|
|
451
|
+
return undefined;
|
|
230
452
|
}
|
|
231
453
|
lastLoggedError = undefined;
|
|
232
454
|
}
|
|
233
455
|
catch (error) {
|
|
234
|
-
if (options.controller.signal.aborted || options.isFinished())
|
|
235
|
-
return;
|
|
456
|
+
if (options.controller.signal.aborted || options.isFinished()) {
|
|
457
|
+
return undefined;
|
|
458
|
+
}
|
|
459
|
+
if (error instanceof ApiError &&
|
|
460
|
+
(error.status === 401 || error.status === 403)) {
|
|
461
|
+
options.log("CrewX revoked this agent connection; stopping the local agent and daemon…");
|
|
462
|
+
options.controller.abort(error);
|
|
463
|
+
return error;
|
|
464
|
+
}
|
|
236
465
|
const message = errorMessage(error);
|
|
237
466
|
const now = Date.now();
|
|
238
467
|
if (message !== lastLoggedError || now - lastErrorLoggedAt >= 30_000) {
|
|
@@ -246,38 +475,68 @@ async function monitorAssignmentControl(options) {
|
|
|
246
475
|
}
|
|
247
476
|
await sleep(nextCheckDelay, options.controller.signal);
|
|
248
477
|
}
|
|
478
|
+
return undefined;
|
|
249
479
|
}
|
|
250
480
|
export async function executeAssignment(options) {
|
|
251
481
|
const { assignment, events } = options;
|
|
252
|
-
const secrets =
|
|
253
|
-
|
|
254
|
-
|
|
482
|
+
const secrets = [
|
|
483
|
+
...(options.environment?.CREWX_TOKEN
|
|
484
|
+
? [options.environment.CREWX_TOKEN]
|
|
485
|
+
: []),
|
|
486
|
+
...(assignment.claim_token ? [assignment.claim_token] : []),
|
|
487
|
+
];
|
|
255
488
|
const nativeJsonRuntime = !options.codingCommand &&
|
|
256
489
|
(options.adapter === "codex" ||
|
|
257
490
|
options.adapter === "claude" ||
|
|
258
491
|
options.adapter === "pi");
|
|
259
492
|
const runtimeDiagnostics = new Set();
|
|
260
|
-
|
|
493
|
+
const pendingRuntimeMessages = [];
|
|
494
|
+
let terminalizationStarted = false;
|
|
495
|
+
events.push(assignmentEvent(assignment, "lifecycle", {
|
|
261
496
|
stage: "claimed",
|
|
262
497
|
assignment_id: assignment.id,
|
|
263
498
|
adapter: options.adapter,
|
|
264
499
|
}));
|
|
265
|
-
events.push(
|
|
500
|
+
events.push(assignmentEvent(assignment, "message", {
|
|
266
501
|
assignment_id: assignment.id,
|
|
267
502
|
role: "user",
|
|
268
503
|
content: assignment.prompt,
|
|
269
504
|
}));
|
|
270
|
-
events.push(
|
|
505
|
+
events.push(assignmentEvent(assignment, "lifecycle", {
|
|
271
506
|
stage: "started",
|
|
272
507
|
assignment_id: assignment.id,
|
|
273
508
|
adapter: options.adapter,
|
|
274
509
|
}));
|
|
275
510
|
await events.flush(options.signal);
|
|
511
|
+
let runtimeProxy;
|
|
276
512
|
try {
|
|
277
513
|
const agent = record(options.context.agent);
|
|
278
514
|
const permissionPreset = agent?.permission_preset;
|
|
279
515
|
const configuredModel = nonEmptyString(agent?.model);
|
|
280
516
|
const runtimeAgentId = nonEmptyString(agent?.runtime_agent_id);
|
|
517
|
+
const upstreamToken = options.environment?.CREWX_TOKEN?.trim();
|
|
518
|
+
const cliPath = options.environment?.CREWX_CLI_PATH?.trim();
|
|
519
|
+
if (upstreamToken && cliPath) {
|
|
520
|
+
runtimeProxy = await startRuntimeCredentialProxy({
|
|
521
|
+
upstreamUrl: options.serverUrl,
|
|
522
|
+
upstreamToken,
|
|
523
|
+
cliPath,
|
|
524
|
+
assignmentId: assignment.id,
|
|
525
|
+
...(assignment.claim_token
|
|
526
|
+
? { claimToken: assignment.claim_token }
|
|
527
|
+
: {}),
|
|
528
|
+
...(options.sessionId ? { sessionId: options.sessionId } : {}),
|
|
529
|
+
permissionPreset: permissionPreset === "read_only" ||
|
|
530
|
+
permissionPreset === "standard" ||
|
|
531
|
+
permissionPreset === "full_access"
|
|
532
|
+
? permissionPreset
|
|
533
|
+
: "read_only",
|
|
534
|
+
chatOnly: agent?.chat_only === true,
|
|
535
|
+
});
|
|
536
|
+
const proxyToken = runtimeProxy.environment.CREWX_TOKEN;
|
|
537
|
+
if (proxyToken)
|
|
538
|
+
secrets.push(proxyToken);
|
|
539
|
+
}
|
|
281
540
|
const result = await (options.runAgent ?? runAdapter)({
|
|
282
541
|
adapter: options.adapter,
|
|
283
542
|
prompt: assemblePrompt(assignment, options.context),
|
|
@@ -297,6 +556,9 @@ export async function executeAssignment(options) {
|
|
|
297
556
|
},
|
|
298
557
|
...(options.codingCommand ? { codingCommand: options.codingCommand } : {}),
|
|
299
558
|
...(options.environment ? { environment: options.environment } : {}),
|
|
559
|
+
...(runtimeProxy
|
|
560
|
+
? { controlEnvironment: runtimeProxy.environment }
|
|
561
|
+
: {}),
|
|
300
562
|
...(options.signal ? { signal: options.signal } : {}),
|
|
301
563
|
onStdout(line, parsed) {
|
|
302
564
|
if (nativeJsonRuntime) {
|
|
@@ -309,7 +571,7 @@ export async function executeAssignment(options) {
|
|
|
309
571
|
return;
|
|
310
572
|
}
|
|
311
573
|
runtimeDiagnostics.add(content);
|
|
312
|
-
events.push(
|
|
574
|
+
events.push(assignmentEvent(assignment, "output", {
|
|
313
575
|
assignment_id: assignment.id,
|
|
314
576
|
stream: "stdout",
|
|
315
577
|
content,
|
|
@@ -318,7 +580,7 @@ export async function executeAssignment(options) {
|
|
|
318
580
|
}
|
|
319
581
|
const content = truncate(redactSecrets(line, secrets));
|
|
320
582
|
const raw = sanitizedRaw(parsed.raw, secrets);
|
|
321
|
-
events.push(
|
|
583
|
+
events.push(assignmentEvent(assignment, "output", {
|
|
322
584
|
assignment_id: assignment.id,
|
|
323
585
|
stream: "stdout",
|
|
324
586
|
content,
|
|
@@ -330,14 +592,20 @@ export async function executeAssignment(options) {
|
|
|
330
592
|
}));
|
|
331
593
|
},
|
|
332
594
|
onMessage(message, role) {
|
|
333
|
-
|
|
595
|
+
const runtimeMessage = assignmentEvent(assignment, "message", {
|
|
334
596
|
assignment_id: assignment.id,
|
|
335
597
|
role,
|
|
336
598
|
content: truncate(redactSecrets(message, secrets)),
|
|
337
|
-
})
|
|
599
|
+
});
|
|
600
|
+
if (role === "assistant" || role === "tool") {
|
|
601
|
+
pendingRuntimeMessages.push(runtimeMessage);
|
|
602
|
+
}
|
|
603
|
+
else {
|
|
604
|
+
events.push(runtimeMessage);
|
|
605
|
+
}
|
|
338
606
|
},
|
|
339
607
|
onStderr(line) {
|
|
340
|
-
events.push(
|
|
608
|
+
events.push(assignmentEvent(assignment, "output", {
|
|
341
609
|
assignment_id: assignment.id,
|
|
342
610
|
stream: "stderr",
|
|
343
611
|
content: truncate(redactSecrets(line, secrets)),
|
|
@@ -345,55 +613,106 @@ export async function executeAssignment(options) {
|
|
|
345
613
|
},
|
|
346
614
|
});
|
|
347
615
|
if (options.signal?.aborted) {
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
616
|
+
terminalizationStarted = true;
|
|
617
|
+
events.pushMany([
|
|
618
|
+
...pendingRuntimeMessages,
|
|
619
|
+
assignmentEvent(assignment, "lifecycle", {
|
|
620
|
+
stage: "cancelled",
|
|
621
|
+
assignment_id: assignment.id,
|
|
622
|
+
adapter: options.adapter,
|
|
623
|
+
}),
|
|
624
|
+
]);
|
|
353
625
|
}
|
|
354
626
|
else if (result.exitCode === 0) {
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
627
|
+
terminalizationStarted = true;
|
|
628
|
+
events.pushMany([
|
|
629
|
+
...pendingRuntimeMessages,
|
|
630
|
+
assignmentEvent(assignment, "lifecycle", {
|
|
631
|
+
stage: "completed",
|
|
632
|
+
assignment_id: assignment.id,
|
|
633
|
+
adapter: options.adapter,
|
|
634
|
+
exit_code: result.exitCode,
|
|
635
|
+
}),
|
|
636
|
+
]);
|
|
361
637
|
}
|
|
362
638
|
else {
|
|
363
639
|
const detail = result.stderr.at(-1) ??
|
|
364
640
|
`${options.adapter} exited with code ${String(result.exitCode)}`;
|
|
365
|
-
const
|
|
366
|
-
|
|
641
|
+
const failure = runtimeFailure(options.adapter, detail);
|
|
642
|
+
const safeDetail = redactSecrets(failure.message, secrets);
|
|
643
|
+
terminalizationStarted = true;
|
|
644
|
+
events.pushMany([
|
|
645
|
+
...pendingRuntimeMessages,
|
|
646
|
+
assignmentEvent(assignment, "error", {
|
|
647
|
+
assignment_id: assignment.id,
|
|
648
|
+
message: truncate(safeDetail),
|
|
649
|
+
code: failure.code,
|
|
650
|
+
retryable: false,
|
|
651
|
+
}),
|
|
652
|
+
assignmentEvent(assignment, "lifecycle", {
|
|
653
|
+
stage: "failed",
|
|
654
|
+
assignment_id: assignment.id,
|
|
655
|
+
adapter: options.adapter,
|
|
656
|
+
exit_code: result.exitCode,
|
|
657
|
+
detail: truncate(safeDetail),
|
|
658
|
+
}),
|
|
659
|
+
]);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
catch (error) {
|
|
663
|
+
if (terminalizationStarted)
|
|
664
|
+
throw error;
|
|
665
|
+
const message = truncate(redactSecrets(errorMessage(error), secrets));
|
|
666
|
+
terminalizationStarted = true;
|
|
667
|
+
events.pushMany([
|
|
668
|
+
...pendingRuntimeMessages,
|
|
669
|
+
assignmentEvent(assignment, "error", {
|
|
367
670
|
assignment_id: assignment.id,
|
|
368
|
-
message
|
|
671
|
+
message,
|
|
672
|
+
...(error instanceof CliError && error.code
|
|
673
|
+
? { code: error.code }
|
|
674
|
+
: {}),
|
|
369
675
|
retryable: false,
|
|
370
|
-
})
|
|
371
|
-
|
|
372
|
-
stage: "failed",
|
|
676
|
+
}),
|
|
677
|
+
assignmentEvent(assignment, "lifecycle", {
|
|
678
|
+
stage: options.signal?.aborted ? "cancelled" : "failed",
|
|
373
679
|
assignment_id: assignment.id,
|
|
374
680
|
adapter: options.adapter,
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
}
|
|
681
|
+
detail: message,
|
|
682
|
+
}),
|
|
683
|
+
]);
|
|
379
684
|
}
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
events.push(event("error", {
|
|
383
|
-
assignment_id: assignment.id,
|
|
384
|
-
message,
|
|
385
|
-
retryable: false,
|
|
386
|
-
}));
|
|
387
|
-
events.push(event("lifecycle", {
|
|
388
|
-
stage: options.signal?.aborted ? "cancelled" : "failed",
|
|
389
|
-
assignment_id: assignment.id,
|
|
390
|
-
adapter: options.adapter,
|
|
391
|
-
detail: message,
|
|
392
|
-
}));
|
|
685
|
+
finally {
|
|
686
|
+
await runtimeProxy?.close();
|
|
393
687
|
}
|
|
394
688
|
await events.flush(options.signal?.aborted ? undefined : options.signal);
|
|
395
689
|
}
|
|
396
690
|
export async function runDaemon(options) {
|
|
691
|
+
const agentToken = options.environment?.CREWX_TOKEN?.trim();
|
|
692
|
+
const automaticOutboxPath = agentToken
|
|
693
|
+
? agentEventOutboxPath(options.api.baseUrl, agentToken, options.adapter, options.environment)
|
|
694
|
+
: undefined;
|
|
695
|
+
const durablePath = options.outbox?.path ?? automaticOutboxPath;
|
|
696
|
+
const processLock = options.processLock ??
|
|
697
|
+
(durablePath
|
|
698
|
+
? new AgentProcessLock(eventOutboxLockPath(durablePath))
|
|
699
|
+
: undefined);
|
|
700
|
+
// The lock precedes both session registration and outbox loading. This is
|
|
701
|
+
// intentionally wider than the network loop: two same-profile daemons must
|
|
702
|
+
// never hold independent cached snapshots of the same durable outcomes.
|
|
703
|
+
await processLock?.acquire();
|
|
704
|
+
try {
|
|
705
|
+
const durableOutbox = options.outbox ??
|
|
706
|
+
(automaticOutboxPath
|
|
707
|
+
? new DurableEventOutbox(automaticOutboxPath)
|
|
708
|
+
: undefined);
|
|
709
|
+
await runOwnedDaemon(options, durableOutbox);
|
|
710
|
+
}
|
|
711
|
+
finally {
|
|
712
|
+
await processLock?.release();
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
async function runOwnedDaemon(options, durableOutbox) {
|
|
397
716
|
const log = options.log ?? (() => undefined);
|
|
398
717
|
const startedAt = Date.now();
|
|
399
718
|
const session = await options.api.createSession({
|
|
@@ -409,14 +728,20 @@ export async function runDaemon(options) {
|
|
|
409
728
|
cwd: options.cwd,
|
|
410
729
|
},
|
|
411
730
|
}, options.signal);
|
|
412
|
-
const events = new EventBuffer(options.api, session.session_id);
|
|
413
731
|
let cursor = session.cursor;
|
|
414
732
|
let activeAssignmentId;
|
|
733
|
+
let activeAssignmentClaimToken;
|
|
415
734
|
let stopping = false;
|
|
416
735
|
const seenAssignmentEvents = new Set();
|
|
417
736
|
let workspaceContext = await options.api.context(session.session_id, options.signal);
|
|
737
|
+
const agent = record(workspaceContext.agent);
|
|
738
|
+
const events = new EventBuffer(options.api, session.session_id, {
|
|
739
|
+
...(durableOutbox ? { outbox: durableOutbox } : {}),
|
|
740
|
+
});
|
|
741
|
+
// Reuse original event UUIDs against the new session so the server can
|
|
742
|
+
// idempotently acknowledge outcomes persisted before a crash or restart.
|
|
743
|
+
await events.flush(options.signal);
|
|
418
744
|
if (options.adapter === "openclaw") {
|
|
419
|
-
const agent = record(workspaceContext.agent);
|
|
420
745
|
const runtimeAgentId = nonEmptyString(agent?.runtime_agent_id);
|
|
421
746
|
if (!runtimeAgentId) {
|
|
422
747
|
throw new Error("CrewX did not provide an OpenClaw runtime agent ID for this profile.");
|
|
@@ -439,6 +764,9 @@ export async function runDaemon(options) {
|
|
|
439
764
|
events.push(event("heartbeat", {
|
|
440
765
|
status: activeAssignmentId ? "running" : stopping ? "stopping" : "idle",
|
|
441
766
|
...(activeAssignmentId ? { assignment_id: activeAssignmentId } : {}),
|
|
767
|
+
...(activeAssignmentClaimToken
|
|
768
|
+
? { claim_token: activeAssignmentClaimToken }
|
|
769
|
+
: {}),
|
|
442
770
|
uptime_seconds: Math.floor((Date.now() - startedAt) / 1_000),
|
|
443
771
|
}));
|
|
444
772
|
void events
|
|
@@ -448,6 +776,8 @@ export async function runDaemon(options) {
|
|
|
448
776
|
heartbeatTimer.unref();
|
|
449
777
|
try {
|
|
450
778
|
let polls = 0;
|
|
779
|
+
let consecutivePollFailures = 0;
|
|
780
|
+
let consecutiveEmptyPolls = 0;
|
|
451
781
|
while (!options.signal?.aborted) {
|
|
452
782
|
try {
|
|
453
783
|
// Never acknowledge a claimed page while its lifecycle/output batch is
|
|
@@ -455,7 +785,7 @@ export async function runDaemon(options) {
|
|
|
455
785
|
await events.flush(options.signal);
|
|
456
786
|
const requestedCursor = cursor;
|
|
457
787
|
const response = await options.api.poll(session.session_id, cursor, options.signal);
|
|
458
|
-
|
|
788
|
+
consecutivePollFailures = 0;
|
|
459
789
|
polls += 1;
|
|
460
790
|
for (const inbound of response.events) {
|
|
461
791
|
const assignment = assignmentFromEvent(inbound);
|
|
@@ -469,49 +799,84 @@ export async function runDaemon(options) {
|
|
|
469
799
|
continue;
|
|
470
800
|
seenAssignmentEvents.add(executionKey);
|
|
471
801
|
activeAssignmentId = assignment.id;
|
|
802
|
+
activeAssignmentClaimToken = assignment.claim_token;
|
|
472
803
|
log(`Running ${assignment.title ?? assignment.id}…`);
|
|
473
804
|
try {
|
|
474
|
-
|
|
805
|
+
try {
|
|
806
|
+
workspaceContext = await options.api.context(session.session_id, options.signal);
|
|
807
|
+
}
|
|
808
|
+
catch (error) {
|
|
809
|
+
log(`Context refresh failed; using the previous snapshot: ${errorMessage(error)}`);
|
|
810
|
+
}
|
|
811
|
+
const linked = linkedAbortController(options.signal);
|
|
812
|
+
let assignmentFinished = false;
|
|
813
|
+
const assignmentEventCursor = typeof assignment.metadata?.assignment_event_cursor === "number"
|
|
814
|
+
? assignment.metadata.assignment_event_cursor
|
|
815
|
+
: undefined;
|
|
816
|
+
const canMonitor = typeof options.api.control === "function" &&
|
|
817
|
+
/^\d+$/.test(assignment.id);
|
|
818
|
+
const monitor = canMonitor
|
|
819
|
+
? monitorAssignmentControl({
|
|
820
|
+
api: options.api,
|
|
821
|
+
sessionId: session.session_id,
|
|
822
|
+
assignmentId: assignment.id,
|
|
823
|
+
assignmentEventCursor,
|
|
824
|
+
controller: linked.controller,
|
|
825
|
+
isFinished: () => assignmentFinished,
|
|
826
|
+
log,
|
|
827
|
+
})
|
|
828
|
+
: Promise.resolve(undefined);
|
|
829
|
+
let fatalControlError;
|
|
830
|
+
try {
|
|
831
|
+
await (options.executeAssignment ?? executeAssignment)({
|
|
832
|
+
assignment,
|
|
833
|
+
adapter: options.adapter,
|
|
834
|
+
cwd: assignmentWorkingDirectory(assignment, workspaceContext, options.cwd, options.lockedRoot ?? process.env.CREWX_BRIDGE_ROOT),
|
|
835
|
+
context: workspaceContext,
|
|
836
|
+
events,
|
|
837
|
+
serverUrl: options.api.baseUrl,
|
|
838
|
+
sessionId: session.session_id,
|
|
839
|
+
...(options.codingCommand
|
|
840
|
+
? { codingCommand: options.codingCommand }
|
|
841
|
+
: {}),
|
|
842
|
+
...(options.environment
|
|
843
|
+
? { environment: options.environment }
|
|
844
|
+
: {}),
|
|
845
|
+
signal: linked.controller.signal,
|
|
846
|
+
});
|
|
847
|
+
}
|
|
848
|
+
finally {
|
|
849
|
+
assignmentFinished = true;
|
|
850
|
+
linked.controller.abort();
|
|
851
|
+
linked.detach();
|
|
852
|
+
fatalControlError = await monitor;
|
|
853
|
+
}
|
|
854
|
+
if (fatalControlError)
|
|
855
|
+
throw fatalControlError;
|
|
856
|
+
await events.flush(options.signal);
|
|
475
857
|
}
|
|
476
858
|
catch (error) {
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
assignmentId: assignment.id,
|
|
488
|
-
controller: linked.controller,
|
|
489
|
-
isFinished: () => assignmentFinished,
|
|
490
|
-
log,
|
|
491
|
-
})
|
|
492
|
-
: Promise.resolve();
|
|
493
|
-
try {
|
|
494
|
-
await (options.executeAssignment ?? executeAssignment)({
|
|
495
|
-
assignment,
|
|
496
|
-
adapter: options.adapter,
|
|
497
|
-
cwd: assignmentWorkingDirectory(assignment, workspaceContext, options.cwd, options.lockedRoot ?? process.env.CREWX_BRIDGE_ROOT),
|
|
498
|
-
context: workspaceContext,
|
|
499
|
-
events,
|
|
500
|
-
serverUrl: options.api.baseUrl,
|
|
501
|
-
...(options.codingCommand ? { codingCommand: options.codingCommand } : {}),
|
|
502
|
-
...(options.environment ? { environment: options.environment } : {}),
|
|
503
|
-
signal: linked.controller.signal,
|
|
504
|
-
});
|
|
859
|
+
// A failed pre-run lifecycle flush means the runtime never
|
|
860
|
+
// started. Leave the page unacknowledged and allow that event to
|
|
861
|
+
// execute after delivery recovers. If a terminal outcome is
|
|
862
|
+
// queued, retain the seen fence: the next loop first delivers
|
|
863
|
+
// that durable outcome, then acknowledges the page without
|
|
864
|
+
// executing the runtime twice.
|
|
865
|
+
if (!events.hasPendingTerminalOutcome(assignment.id)) {
|
|
866
|
+
seenAssignmentEvents.delete(executionKey);
|
|
867
|
+
}
|
|
868
|
+
throw error;
|
|
505
869
|
}
|
|
506
870
|
finally {
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
linked.detach();
|
|
510
|
-
await monitor;
|
|
871
|
+
activeAssignmentId = undefined;
|
|
872
|
+
activeAssignmentClaimToken = undefined;
|
|
511
873
|
}
|
|
512
|
-
await events.flush(options.signal);
|
|
513
|
-
activeAssignmentId = undefined;
|
|
514
874
|
}
|
|
875
|
+
// Advancing the local cursor is the acknowledgement boundary. Do it
|
|
876
|
+
// only after every assignment on the page has a delivered terminal
|
|
877
|
+
// outcome (or was previously terminalized and fenced).
|
|
878
|
+
cursor = response.cursor;
|
|
879
|
+
seenAssignmentEvents.clear();
|
|
515
880
|
if (options.once && polls >= 1) {
|
|
516
881
|
if (String(cursor) !== String(requestedCursor)) {
|
|
517
882
|
// A final poll acknowledges the page that was just processed. Any
|
|
@@ -520,10 +885,21 @@ export async function runDaemon(options) {
|
|
|
520
885
|
}
|
|
521
886
|
break;
|
|
522
887
|
}
|
|
523
|
-
if (response.has_more)
|
|
888
|
+
if (response.has_more) {
|
|
889
|
+
consecutiveEmptyPolls = 0;
|
|
524
890
|
continue;
|
|
891
|
+
}
|
|
892
|
+
consecutiveEmptyPolls =
|
|
893
|
+
response.events.length === 0 ? consecutiveEmptyPolls + 1 : 0;
|
|
525
894
|
const serverDelay = response.retry_after_ms;
|
|
526
|
-
|
|
895
|
+
const pollDelay = consecutiveEmptyPolls > 0
|
|
896
|
+
? adaptiveIdlePollDelay(options.pollIntervalMs, consecutiveEmptyPolls, {
|
|
897
|
+
...(serverDelay !== undefined
|
|
898
|
+
? { retryAfterMs: serverDelay }
|
|
899
|
+
: {}),
|
|
900
|
+
})
|
|
901
|
+
: Math.max(options.pollIntervalMs, serverDelay ?? 0);
|
|
902
|
+
await sleep(pollDelay, options.signal);
|
|
527
903
|
}
|
|
528
904
|
catch (error) {
|
|
529
905
|
if (options.signal?.aborted)
|
|
@@ -532,9 +908,12 @@ export async function runDaemon(options) {
|
|
|
532
908
|
throw error;
|
|
533
909
|
const message = redactSecrets(errorMessage(error), [options.api.token]);
|
|
534
910
|
log(`Polling failed: ${message}. Retrying…`);
|
|
535
|
-
|
|
911
|
+
consecutivePollFailures += 1;
|
|
536
912
|
const serverDelay = error instanceof ApiError ? error.retryAfterMs : undefined;
|
|
537
|
-
|
|
913
|
+
const retryCap = Math.min(EVENT_RETRY_MAX_DELAY_MS, Math.max(options.pollIntervalMs, EVENT_RETRY_BASE_DELAY_MS) *
|
|
914
|
+
2 ** Math.min(8, consecutivePollFailures - 1));
|
|
915
|
+
const jitterDelay = Math.floor(Math.random() * (retryCap + 1));
|
|
916
|
+
await sleep(Math.max(250, jitterDelay, serverDelay ?? 0), options.signal);
|
|
538
917
|
}
|
|
539
918
|
}
|
|
540
919
|
}
|