crewx-agent-cli 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -5
- package/dist/adapters.d.ts +12 -0
- package/dist/adapters.d.ts.map +1 -1
- package/dist/adapters.js +139 -42
- package/dist/adapters.js.map +1 -1
- package/dist/api.d.ts +40 -17
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +187 -11
- package/dist/api.js.map +1 -1
- package/dist/bridge-watchdog.d.ts +10 -0
- package/dist/bridge-watchdog.d.ts.map +1 -0
- package/dist/bridge-watchdog.js +45 -0
- package/dist/bridge-watchdog.js.map +1 -0
- package/dist/config.d.ts +9 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +53 -2
- package/dist/config.js.map +1 -1
- package/dist/constants.d.ts +6 -1
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +11 -1
- package/dist/constants.js.map +1 -1
- package/dist/daemon.d.ts +34 -2
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +495 -100
- package/dist/daemon.js.map +1 -1
- package/dist/errors.d.ts +4 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +3 -1
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -6
- package/dist/index.js.map +1 -1
- package/dist/join.d.ts +1 -0
- package/dist/join.d.ts.map +1 -1
- package/dist/join.js +13 -0
- package/dist/join.js.map +1 -1
- package/dist/outbox.d.ts +22 -0
- package/dist/outbox.d.ts.map +1 -0
- package/dist/outbox.js +189 -0
- package/dist/outbox.js.map +1 -0
- package/dist/process-lock.d.ts +31 -0
- package/dist/process-lock.d.ts.map +1 -0
- package/dist/process-lock.js +272 -0
- package/dist/process-lock.js.map +1 -0
- package/dist/prompt.d.ts.map +1 -1
- package/dist/prompt.js +7 -0
- package/dist/prompt.js.map +1 -1
- package/dist/runtime-proxy.d.ts +18 -0
- package/dist/runtime-proxy.d.ts.map +1 -0
- package/dist/runtime-proxy.js +221 -0
- package/dist/runtime-proxy.js.map +1 -0
- package/package.json +6 -2
package/dist/daemon.js
CHANGED
|
@@ -2,14 +2,18 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import { realpathSync } from "node:fs";
|
|
3
3
|
import { hostname, platform, arch } from "node:os";
|
|
4
4
|
import { isAbsolute, relative, resolve, sep } from "node:path";
|
|
5
|
-
import { runAdapter } from "./adapters.js";
|
|
6
|
-
import { ApiError } from "./api.js";
|
|
7
|
-
import { CLI_VERSION, MAX_EVENT_CONTENT_LENGTH } from "./constants.js";
|
|
8
|
-
import { errorMessage, redactSecrets } from "./errors.js";
|
|
5
|
+
import { runAdapter, verifyOpenClawRuntime, } from "./adapters.js";
|
|
6
|
+
import { ApiError, serializeOutboundEventBatch, } from "./api.js";
|
|
7
|
+
import { CLI_VERSION, MAX_BUFFERED_EVENT_BYTES, MAX_BUFFERED_EVENTS, MAX_EVENT_BATCH_BYTES, MAX_EVENT_CONTENT_LENGTH, } from "./constants.js";
|
|
8
|
+
import { CliError, errorMessage, redactSecrets } from "./errors.js";
|
|
9
9
|
import { assemblePrompt, assignmentFromEvent } from "./prompt.js";
|
|
10
|
+
import { startRuntimeCredentialProxy, } from "./runtime-proxy.js";
|
|
11
|
+
import { DurableEventOutbox, agentEventOutboxPath, eventOutboxLockPath, } from "./outbox.js";
|
|
12
|
+
import { AgentProcessLock, } from "./process-lock.js";
|
|
10
13
|
const ASSIGNMENT_CONTROL_INTERVAL_MS = 5_000;
|
|
11
14
|
const EVENT_RETRY_BASE_DELAY_MS = 2_000;
|
|
12
15
|
const EVENT_RETRY_MAX_DELAY_MS = 30_000;
|
|
16
|
+
const MAX_IDLE_POLL_DELAY_MS = 8_000;
|
|
13
17
|
const MAX_RUNTIME_DIAGNOSTICS = 20;
|
|
14
18
|
function timestamp() {
|
|
15
19
|
return new Date().toISOString();
|
|
@@ -22,6 +26,19 @@ function event(type, payload) {
|
|
|
22
26
|
occurred_at: timestamp(),
|
|
23
27
|
};
|
|
24
28
|
}
|
|
29
|
+
function assignmentEvent(assignment, type, payload) {
|
|
30
|
+
return {
|
|
31
|
+
id: crypto.randomUUID(),
|
|
32
|
+
type,
|
|
33
|
+
payload: {
|
|
34
|
+
...payload,
|
|
35
|
+
...(assignment.claim_token
|
|
36
|
+
? { claim_token: assignment.claim_token }
|
|
37
|
+
: {}),
|
|
38
|
+
},
|
|
39
|
+
occurred_at: timestamp(),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
25
42
|
function truncate(value) {
|
|
26
43
|
return value.length > MAX_EVENT_CONTENT_LENGTH
|
|
27
44
|
? `${value.slice(0, MAX_EVENT_CONTENT_LENGTH)}… [truncated]`
|
|
@@ -43,6 +60,30 @@ function record(value) {
|
|
|
43
60
|
function nonEmptyString(value) {
|
|
44
61
|
return typeof value === "string" && value.trim() ? value.trim() : undefined;
|
|
45
62
|
}
|
|
63
|
+
function runtimeFailure(adapter, detail) {
|
|
64
|
+
if (adapter !== "openclaw") {
|
|
65
|
+
return { code: "runtime_exit_nonzero", message: detail };
|
|
66
|
+
}
|
|
67
|
+
if (/\b429\b|rate[ -]?limit|quota|too many requests/i.test(detail)) {
|
|
68
|
+
return {
|
|
69
|
+
code: "openclaw_provider_rate_limited",
|
|
70
|
+
message: `${detail} Check \`openclaw models status --agent <agent-id> --json\` for provider cooldowns, then retry after the provider limit resets.`,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
if (/unauthori[sz]ed|authentication|credential|oauth|invalid token|expired/i.test(detail)) {
|
|
74
|
+
return {
|
|
75
|
+
code: "openclaw_auth_failed",
|
|
76
|
+
message: `${detail} Run \`openclaw models auth --agent <agent-id> list --json\` and repair the selected provider credentials.`,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
if (/model.+(?:not found|unknown|unavailable|not allowed)/i.test(detail)) {
|
|
80
|
+
return {
|
|
81
|
+
code: "openclaw_model_unavailable",
|
|
82
|
+
message: `${detail} Run \`openclaw models status --agent <agent-id> --json\` and select an available model.`,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
return { code: "openclaw_runtime_failed", message: detail };
|
|
86
|
+
}
|
|
46
87
|
function identifier(value) {
|
|
47
88
|
if (typeof value === "number" && Number.isSafeInteger(value))
|
|
48
89
|
return String(value);
|
|
@@ -130,7 +171,7 @@ export function assignmentWorkingDirectory(assignment, context, fallback, locked
|
|
|
130
171
|
if (relativePath === ".." ||
|
|
131
172
|
relativePath.startsWith(`..${sep}`) ||
|
|
132
173
|
isAbsolute(relativePath)) {
|
|
133
|
-
throw new Error(
|
|
174
|
+
throw new Error("CrewX Bridge rejected the assignment directory because it is outside the approved folder.");
|
|
134
175
|
}
|
|
135
176
|
return canonicalCandidate;
|
|
136
177
|
}
|
|
@@ -138,18 +179,65 @@ export class EventBuffer {
|
|
|
138
179
|
api;
|
|
139
180
|
sessionId;
|
|
140
181
|
queue = [];
|
|
182
|
+
inFlight = new Set();
|
|
141
183
|
flushing;
|
|
142
184
|
retryNotBefore = 0;
|
|
143
185
|
consecutiveFailures = 0;
|
|
144
|
-
|
|
186
|
+
maxEvents;
|
|
187
|
+
maxBytes;
|
|
188
|
+
random;
|
|
189
|
+
durableSnapshot;
|
|
190
|
+
constructor(api, sessionId, options = {}) {
|
|
145
191
|
this.api = api;
|
|
146
192
|
this.sessionId = sessionId;
|
|
193
|
+
this.maxEvents = options.maxEvents ?? MAX_BUFFERED_EVENTS;
|
|
194
|
+
this.maxBytes = options.maxBytes ?? MAX_BUFFERED_EVENT_BYTES;
|
|
195
|
+
this.random = options.random ?? Math.random;
|
|
196
|
+
this.outbox = options.outbox;
|
|
197
|
+
if (this.outbox) {
|
|
198
|
+
this.queue.push(...this.outbox.load());
|
|
199
|
+
this.enforceBounds();
|
|
200
|
+
}
|
|
201
|
+
this.durableSnapshot = this.serializeDurableEvents();
|
|
147
202
|
}
|
|
203
|
+
outbox;
|
|
148
204
|
push(item) {
|
|
149
|
-
this.
|
|
205
|
+
this.pushMany([item]);
|
|
206
|
+
}
|
|
207
|
+
pushMany(items) {
|
|
208
|
+
if (items.length === 0)
|
|
209
|
+
return;
|
|
210
|
+
const previous = [...this.queue];
|
|
211
|
+
try {
|
|
212
|
+
for (const item of items) {
|
|
213
|
+
if (!this.coalesce(item))
|
|
214
|
+
this.queue.push(item);
|
|
215
|
+
}
|
|
216
|
+
this.enforceBounds();
|
|
217
|
+
this.persistDurableEvents();
|
|
218
|
+
}
|
|
219
|
+
catch (error) {
|
|
220
|
+
this.queue.splice(0, this.queue.length, ...previous);
|
|
221
|
+
throw error;
|
|
222
|
+
}
|
|
150
223
|
if (this.queue.length >= 25)
|
|
151
224
|
void this.flush().catch(() => undefined);
|
|
152
225
|
}
|
|
226
|
+
pendingCount() {
|
|
227
|
+
return this.queue.length;
|
|
228
|
+
}
|
|
229
|
+
pendingBytes() {
|
|
230
|
+
return this.queue.reduce((total, item) => total + this.eventBytes(item), 0);
|
|
231
|
+
}
|
|
232
|
+
hasPendingTerminalOutcome(assignmentId) {
|
|
233
|
+
const isTerminalOutcome = (item) => item.type === "lifecycle" &&
|
|
234
|
+
item.payload.assignment_id === assignmentId &&
|
|
235
|
+
(item.payload.stage === "completed" ||
|
|
236
|
+
item.payload.stage === "failed" ||
|
|
237
|
+
item.payload.stage === "cancelled");
|
|
238
|
+
return (this.queue.some(isTerminalOutcome) ||
|
|
239
|
+
[...this.inFlight].some(isTerminalOutcome));
|
|
240
|
+
}
|
|
153
241
|
async flush(signal) {
|
|
154
242
|
if (this.flushing) {
|
|
155
243
|
await this.flushing;
|
|
@@ -171,21 +259,143 @@ export class EventBuffer {
|
|
|
171
259
|
await sleep(retryDelay, signal);
|
|
172
260
|
if (signal?.aborted)
|
|
173
261
|
return;
|
|
174
|
-
const batch =
|
|
262
|
+
const batch = [];
|
|
263
|
+
for (const item of this.queue) {
|
|
264
|
+
if (batch.length >= 100) {
|
|
265
|
+
break;
|
|
266
|
+
}
|
|
267
|
+
const candidate = [...batch, item];
|
|
268
|
+
const candidateBytes = Buffer.byteLength(serializeOutboundEventBatch(this.sessionId, candidate), "utf8");
|
|
269
|
+
if (candidateBytes > MAX_EVENT_BATCH_BYTES) {
|
|
270
|
+
if (batch.length === 0) {
|
|
271
|
+
throw new CliError(`CrewX cannot send event ${item.id ?? "(without an ID)"} because its exact serialized batch exceeds ${String(MAX_EVENT_BATCH_BYTES)} bytes.`, 1, { code: "event_batch_too_large" });
|
|
272
|
+
}
|
|
273
|
+
break;
|
|
274
|
+
}
|
|
275
|
+
batch.push(item);
|
|
276
|
+
}
|
|
277
|
+
for (const item of batch)
|
|
278
|
+
this.inFlight.add(item);
|
|
175
279
|
try {
|
|
176
280
|
await this.api.sendEvents(this.sessionId, batch, signal);
|
|
177
|
-
this.queue.splice(0, batch.length);
|
|
281
|
+
const delivered = this.queue.splice(0, batch.length);
|
|
282
|
+
try {
|
|
283
|
+
this.persistDurableEvents();
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
this.queue.unshift(...delivered);
|
|
287
|
+
throw error;
|
|
288
|
+
}
|
|
178
289
|
this.consecutiveFailures = 0;
|
|
179
290
|
this.retryNotBefore = 0;
|
|
180
291
|
}
|
|
181
292
|
catch (error) {
|
|
182
293
|
this.consecutiveFailures += 1;
|
|
183
294
|
const exponentialDelay = Math.min(EVENT_RETRY_MAX_DELAY_MS, EVENT_RETRY_BASE_DELAY_MS * 2 ** (this.consecutiveFailures - 1));
|
|
295
|
+
const jitterDelay = Math.floor(this.random() * (exponentialDelay + 1));
|
|
184
296
|
const serverDelay = error instanceof ApiError ? error.retryAfterMs : undefined;
|
|
185
297
|
this.retryNotBefore =
|
|
186
|
-
Date.now() + Math.max(
|
|
298
|
+
Date.now() + Math.max(jitterDelay, serverDelay ?? 0);
|
|
187
299
|
throw error;
|
|
188
300
|
}
|
|
301
|
+
finally {
|
|
302
|
+
for (const item of batch)
|
|
303
|
+
this.inFlight.delete(item);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
coalesce(item) {
|
|
308
|
+
if (item.id) {
|
|
309
|
+
const duplicateIndex = this.findNewestIndex((candidate) => candidate.id === item.id);
|
|
310
|
+
if (duplicateIndex >= 0) {
|
|
311
|
+
this.queue[duplicateIndex] = item;
|
|
312
|
+
return true;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
if (item.type === "heartbeat") {
|
|
316
|
+
const existingIndex = this.findNewestIndex((candidate) => candidate.type === "heartbeat");
|
|
317
|
+
if (existingIndex >= 0) {
|
|
318
|
+
this.queue[existingIndex] = item;
|
|
319
|
+
return true;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
if (item.type === "output") {
|
|
323
|
+
const existingIndex = this.findNewestIndex((candidate) => candidate.type === "output" &&
|
|
324
|
+
candidate.payload.assignment_id === item.payload.assignment_id &&
|
|
325
|
+
candidate.payload.stream === item.payload.stream);
|
|
326
|
+
const existing = this.queue[existingIndex];
|
|
327
|
+
if (existingIndex >= 0 && existing?.type === "output") {
|
|
328
|
+
const combined = `${existing.payload.content}\n${item.payload.content}`;
|
|
329
|
+
const content = combined.length <= MAX_EVENT_CONTENT_LENGTH
|
|
330
|
+
? combined
|
|
331
|
+
: `[earlier output omitted during backpressure]\n${combined.slice(-(MAX_EVENT_CONTENT_LENGTH - 47))}`;
|
|
332
|
+
this.queue[existingIndex] = {
|
|
333
|
+
...item,
|
|
334
|
+
payload: {
|
|
335
|
+
...item.payload,
|
|
336
|
+
content,
|
|
337
|
+
},
|
|
338
|
+
};
|
|
339
|
+
return true;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
return false;
|
|
343
|
+
}
|
|
344
|
+
findNewestIndex(predicate) {
|
|
345
|
+
for (let index = this.queue.length - 1; index >= 0; index -= 1) {
|
|
346
|
+
const candidate = this.queue[index];
|
|
347
|
+
if (candidate &&
|
|
348
|
+
!this.inFlight.has(candidate) &&
|
|
349
|
+
predicate(candidate)) {
|
|
350
|
+
return index;
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
return -1;
|
|
354
|
+
}
|
|
355
|
+
eventPriority(item) {
|
|
356
|
+
if (item.type === "heartbeat" || item.type === "output")
|
|
357
|
+
return 0;
|
|
358
|
+
if (item.type === "error" ||
|
|
359
|
+
(item.type === "message" && item.payload.role === "assistant") ||
|
|
360
|
+
(item.type === "lifecycle" &&
|
|
361
|
+
["completed", "failed", "cancelled", "stopped"].includes(item.payload.stage))) {
|
|
362
|
+
return 2;
|
|
363
|
+
}
|
|
364
|
+
return 1;
|
|
365
|
+
}
|
|
366
|
+
eventBytes(item) {
|
|
367
|
+
return Buffer.byteLength(JSON.stringify(item));
|
|
368
|
+
}
|
|
369
|
+
persistDurableEvents() {
|
|
370
|
+
if (!this.outbox)
|
|
371
|
+
return;
|
|
372
|
+
const durable = this.queue.filter((item) => this.eventPriority(item) === 2);
|
|
373
|
+
const snapshot = JSON.stringify(durable);
|
|
374
|
+
if (snapshot === this.durableSnapshot)
|
|
375
|
+
return;
|
|
376
|
+
this.outbox.replace(durable);
|
|
377
|
+
this.durableSnapshot = snapshot;
|
|
378
|
+
}
|
|
379
|
+
serializeDurableEvents() {
|
|
380
|
+
return JSON.stringify(this.queue.filter((item) => this.eventPriority(item) === 2));
|
|
381
|
+
}
|
|
382
|
+
enforceBounds() {
|
|
383
|
+
let bytes = this.pendingBytes();
|
|
384
|
+
while (this.queue.length > this.maxEvents ||
|
|
385
|
+
bytes > this.maxBytes) {
|
|
386
|
+
let removeIndex = -1;
|
|
387
|
+
for (const priority of [0, 1]) {
|
|
388
|
+
removeIndex = this.queue.findIndex((item) => !this.inFlight.has(item) &&
|
|
389
|
+
this.eventPriority(item) === priority);
|
|
390
|
+
if (removeIndex >= 0)
|
|
391
|
+
break;
|
|
392
|
+
}
|
|
393
|
+
if (removeIndex < 0) {
|
|
394
|
+
throw new CliError("CrewX cannot buffer another final agent outcome while event delivery is unavailable. Existing final outcomes were retained; restore connectivity before accepting more work.", 1, { code: "event_buffer_capacity_exceeded" });
|
|
395
|
+
}
|
|
396
|
+
const [removed] = this.queue.splice(removeIndex, 1);
|
|
397
|
+
if (removed)
|
|
398
|
+
bytes -= this.eventBytes(removed);
|
|
189
399
|
}
|
|
190
400
|
}
|
|
191
401
|
}
|
|
@@ -202,6 +412,18 @@ function sleep(milliseconds, signal) {
|
|
|
202
412
|
signal?.addEventListener("abort", finish, { once: true });
|
|
203
413
|
});
|
|
204
414
|
}
|
|
415
|
+
export function adaptiveIdlePollDelay(baseDelayMs, consecutiveEmptyPolls, options = {}) {
|
|
416
|
+
const base = Math.max(250, Math.floor(baseDelayMs));
|
|
417
|
+
const maximum = Math.max(base, MAX_IDLE_POLL_DELAY_MS);
|
|
418
|
+
const exponent = Math.max(0, Math.min(8, consecutiveEmptyPolls - 1));
|
|
419
|
+
const cap = Math.min(maximum, base * 2 ** exponent);
|
|
420
|
+
const random = options.random ?? Math.random;
|
|
421
|
+
const jittered = cap <= base
|
|
422
|
+
? base
|
|
423
|
+
: base +
|
|
424
|
+
Math.round(Math.max(0, Math.min(1, random())) * (cap - base));
|
|
425
|
+
return Math.max(jittered, options.retryAfterMs ?? 0);
|
|
426
|
+
}
|
|
205
427
|
function linkedAbortController(parent) {
|
|
206
428
|
const controller = new AbortController();
|
|
207
429
|
const abort = () => controller.abort(parent?.reason);
|
|
@@ -222,17 +444,24 @@ async function monitorAssignmentControl(options) {
|
|
|
222
444
|
while (!options.isFinished() && !options.controller.signal.aborted) {
|
|
223
445
|
let nextCheckDelay = ASSIGNMENT_CONTROL_INTERVAL_MS;
|
|
224
446
|
try {
|
|
225
|
-
const control = await options.api.control(options.sessionId, options.assignmentId, options.controller.signal);
|
|
447
|
+
const control = await options.api.control(options.sessionId, options.assignmentId, options.controller.signal, options.assignmentEventCursor);
|
|
226
448
|
if (control.cancelled) {
|
|
227
449
|
options.log(`CrewX cancelled ${options.assignmentId}; stopping the local agent…`);
|
|
228
450
|
options.controller.abort(new Error("Assignment cancelled by CrewX"));
|
|
229
|
-
return;
|
|
451
|
+
return undefined;
|
|
230
452
|
}
|
|
231
453
|
lastLoggedError = undefined;
|
|
232
454
|
}
|
|
233
455
|
catch (error) {
|
|
234
|
-
if (options.controller.signal.aborted || options.isFinished())
|
|
235
|
-
return;
|
|
456
|
+
if (options.controller.signal.aborted || options.isFinished()) {
|
|
457
|
+
return undefined;
|
|
458
|
+
}
|
|
459
|
+
if (error instanceof ApiError &&
|
|
460
|
+
(error.status === 401 || error.status === 403)) {
|
|
461
|
+
options.log("CrewX revoked this agent connection; stopping the local agent and daemon…");
|
|
462
|
+
options.controller.abort(error);
|
|
463
|
+
return error;
|
|
464
|
+
}
|
|
236
465
|
const message = errorMessage(error);
|
|
237
466
|
const now = Date.now();
|
|
238
467
|
if (message !== lastLoggedError || now - lastErrorLoggedAt >= 30_000) {
|
|
@@ -246,38 +475,68 @@ async function monitorAssignmentControl(options) {
|
|
|
246
475
|
}
|
|
247
476
|
await sleep(nextCheckDelay, options.controller.signal);
|
|
248
477
|
}
|
|
478
|
+
return undefined;
|
|
249
479
|
}
|
|
250
480
|
export async function executeAssignment(options) {
|
|
251
481
|
const { assignment, events } = options;
|
|
252
|
-
const secrets =
|
|
253
|
-
|
|
254
|
-
|
|
482
|
+
const secrets = [
|
|
483
|
+
...(options.environment?.CREWX_TOKEN
|
|
484
|
+
? [options.environment.CREWX_TOKEN]
|
|
485
|
+
: []),
|
|
486
|
+
...(assignment.claim_token ? [assignment.claim_token] : []),
|
|
487
|
+
];
|
|
255
488
|
const nativeJsonRuntime = !options.codingCommand &&
|
|
256
489
|
(options.adapter === "codex" ||
|
|
257
490
|
options.adapter === "claude" ||
|
|
258
491
|
options.adapter === "pi");
|
|
259
492
|
const runtimeDiagnostics = new Set();
|
|
260
|
-
|
|
493
|
+
const pendingRuntimeMessages = [];
|
|
494
|
+
let terminalizationStarted = false;
|
|
495
|
+
events.push(assignmentEvent(assignment, "lifecycle", {
|
|
261
496
|
stage: "claimed",
|
|
262
497
|
assignment_id: assignment.id,
|
|
263
498
|
adapter: options.adapter,
|
|
264
499
|
}));
|
|
265
|
-
events.push(
|
|
500
|
+
events.push(assignmentEvent(assignment, "message", {
|
|
266
501
|
assignment_id: assignment.id,
|
|
267
502
|
role: "user",
|
|
268
503
|
content: assignment.prompt,
|
|
269
504
|
}));
|
|
270
|
-
events.push(
|
|
505
|
+
events.push(assignmentEvent(assignment, "lifecycle", {
|
|
271
506
|
stage: "started",
|
|
272
507
|
assignment_id: assignment.id,
|
|
273
508
|
adapter: options.adapter,
|
|
274
509
|
}));
|
|
275
510
|
await events.flush(options.signal);
|
|
511
|
+
let runtimeProxy;
|
|
276
512
|
try {
|
|
277
513
|
const agent = record(options.context.agent);
|
|
278
514
|
const permissionPreset = agent?.permission_preset;
|
|
279
515
|
const configuredModel = nonEmptyString(agent?.model);
|
|
280
516
|
const runtimeAgentId = nonEmptyString(agent?.runtime_agent_id);
|
|
517
|
+
const upstreamToken = options.environment?.CREWX_TOKEN?.trim();
|
|
518
|
+
const cliPath = options.environment?.CREWX_CLI_PATH?.trim();
|
|
519
|
+
if (upstreamToken && cliPath) {
|
|
520
|
+
runtimeProxy = await startRuntimeCredentialProxy({
|
|
521
|
+
upstreamUrl: options.serverUrl,
|
|
522
|
+
upstreamToken,
|
|
523
|
+
cliPath,
|
|
524
|
+
assignmentId: assignment.id,
|
|
525
|
+
...(assignment.claim_token
|
|
526
|
+
? { claimToken: assignment.claim_token }
|
|
527
|
+
: {}),
|
|
528
|
+
...(options.sessionId ? { sessionId: options.sessionId } : {}),
|
|
529
|
+
permissionPreset: permissionPreset === "read_only" ||
|
|
530
|
+
permissionPreset === "standard" ||
|
|
531
|
+
permissionPreset === "full_access"
|
|
532
|
+
? permissionPreset
|
|
533
|
+
: "read_only",
|
|
534
|
+
chatOnly: agent?.chat_only === true,
|
|
535
|
+
});
|
|
536
|
+
const proxyToken = runtimeProxy.environment.CREWX_TOKEN;
|
|
537
|
+
if (proxyToken)
|
|
538
|
+
secrets.push(proxyToken);
|
|
539
|
+
}
|
|
281
540
|
const result = await (options.runAgent ?? runAdapter)({
|
|
282
541
|
adapter: options.adapter,
|
|
283
542
|
prompt: assemblePrompt(assignment, options.context),
|
|
@@ -297,6 +556,9 @@ export async function executeAssignment(options) {
|
|
|
297
556
|
},
|
|
298
557
|
...(options.codingCommand ? { codingCommand: options.codingCommand } : {}),
|
|
299
558
|
...(options.environment ? { environment: options.environment } : {}),
|
|
559
|
+
...(runtimeProxy
|
|
560
|
+
? { controlEnvironment: runtimeProxy.environment }
|
|
561
|
+
: {}),
|
|
300
562
|
...(options.signal ? { signal: options.signal } : {}),
|
|
301
563
|
onStdout(line, parsed) {
|
|
302
564
|
if (nativeJsonRuntime) {
|
|
@@ -309,7 +571,7 @@ export async function executeAssignment(options) {
|
|
|
309
571
|
return;
|
|
310
572
|
}
|
|
311
573
|
runtimeDiagnostics.add(content);
|
|
312
|
-
events.push(
|
|
574
|
+
events.push(assignmentEvent(assignment, "output", {
|
|
313
575
|
assignment_id: assignment.id,
|
|
314
576
|
stream: "stdout",
|
|
315
577
|
content,
|
|
@@ -318,7 +580,7 @@ export async function executeAssignment(options) {
|
|
|
318
580
|
}
|
|
319
581
|
const content = truncate(redactSecrets(line, secrets));
|
|
320
582
|
const raw = sanitizedRaw(parsed.raw, secrets);
|
|
321
|
-
events.push(
|
|
583
|
+
events.push(assignmentEvent(assignment, "output", {
|
|
322
584
|
assignment_id: assignment.id,
|
|
323
585
|
stream: "stdout",
|
|
324
586
|
content,
|
|
@@ -330,14 +592,20 @@ export async function executeAssignment(options) {
|
|
|
330
592
|
}));
|
|
331
593
|
},
|
|
332
594
|
onMessage(message, role) {
|
|
333
|
-
|
|
595
|
+
const runtimeMessage = assignmentEvent(assignment, "message", {
|
|
334
596
|
assignment_id: assignment.id,
|
|
335
597
|
role,
|
|
336
598
|
content: truncate(redactSecrets(message, secrets)),
|
|
337
|
-
})
|
|
599
|
+
});
|
|
600
|
+
if (role === "assistant" || role === "tool") {
|
|
601
|
+
pendingRuntimeMessages.push(runtimeMessage);
|
|
602
|
+
}
|
|
603
|
+
else {
|
|
604
|
+
events.push(runtimeMessage);
|
|
605
|
+
}
|
|
338
606
|
},
|
|
339
607
|
onStderr(line) {
|
|
340
|
-
events.push(
|
|
608
|
+
events.push(assignmentEvent(assignment, "output", {
|
|
341
609
|
assignment_id: assignment.id,
|
|
342
610
|
stream: "stderr",
|
|
343
611
|
content: truncate(redactSecrets(line, secrets)),
|
|
@@ -345,55 +613,106 @@ export async function executeAssignment(options) {
|
|
|
345
613
|
},
|
|
346
614
|
});
|
|
347
615
|
if (options.signal?.aborted) {
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
616
|
+
terminalizationStarted = true;
|
|
617
|
+
events.pushMany([
|
|
618
|
+
...pendingRuntimeMessages,
|
|
619
|
+
assignmentEvent(assignment, "lifecycle", {
|
|
620
|
+
stage: "cancelled",
|
|
621
|
+
assignment_id: assignment.id,
|
|
622
|
+
adapter: options.adapter,
|
|
623
|
+
}),
|
|
624
|
+
]);
|
|
353
625
|
}
|
|
354
626
|
else if (result.exitCode === 0) {
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
627
|
+
terminalizationStarted = true;
|
|
628
|
+
events.pushMany([
|
|
629
|
+
...pendingRuntimeMessages,
|
|
630
|
+
assignmentEvent(assignment, "lifecycle", {
|
|
631
|
+
stage: "completed",
|
|
632
|
+
assignment_id: assignment.id,
|
|
633
|
+
adapter: options.adapter,
|
|
634
|
+
exit_code: result.exitCode,
|
|
635
|
+
}),
|
|
636
|
+
]);
|
|
361
637
|
}
|
|
362
638
|
else {
|
|
363
639
|
const detail = result.stderr.at(-1) ??
|
|
364
640
|
`${options.adapter} exited with code ${String(result.exitCode)}`;
|
|
365
|
-
const
|
|
366
|
-
|
|
641
|
+
const failure = runtimeFailure(options.adapter, detail);
|
|
642
|
+
const safeDetail = redactSecrets(failure.message, secrets);
|
|
643
|
+
terminalizationStarted = true;
|
|
644
|
+
events.pushMany([
|
|
645
|
+
...pendingRuntimeMessages,
|
|
646
|
+
assignmentEvent(assignment, "error", {
|
|
647
|
+
assignment_id: assignment.id,
|
|
648
|
+
message: truncate(safeDetail),
|
|
649
|
+
code: failure.code,
|
|
650
|
+
retryable: false,
|
|
651
|
+
}),
|
|
652
|
+
assignmentEvent(assignment, "lifecycle", {
|
|
653
|
+
stage: "failed",
|
|
654
|
+
assignment_id: assignment.id,
|
|
655
|
+
adapter: options.adapter,
|
|
656
|
+
exit_code: result.exitCode,
|
|
657
|
+
detail: truncate(safeDetail),
|
|
658
|
+
}),
|
|
659
|
+
]);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
catch (error) {
|
|
663
|
+
if (terminalizationStarted)
|
|
664
|
+
throw error;
|
|
665
|
+
const message = truncate(redactSecrets(errorMessage(error), secrets));
|
|
666
|
+
terminalizationStarted = true;
|
|
667
|
+
events.pushMany([
|
|
668
|
+
...pendingRuntimeMessages,
|
|
669
|
+
assignmentEvent(assignment, "error", {
|
|
367
670
|
assignment_id: assignment.id,
|
|
368
|
-
message
|
|
671
|
+
message,
|
|
672
|
+
...(error instanceof CliError && error.code
|
|
673
|
+
? { code: error.code }
|
|
674
|
+
: {}),
|
|
369
675
|
retryable: false,
|
|
370
|
-
})
|
|
371
|
-
|
|
372
|
-
stage: "failed",
|
|
676
|
+
}),
|
|
677
|
+
assignmentEvent(assignment, "lifecycle", {
|
|
678
|
+
stage: options.signal?.aborted ? "cancelled" : "failed",
|
|
373
679
|
assignment_id: assignment.id,
|
|
374
680
|
adapter: options.adapter,
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
}
|
|
681
|
+
detail: message,
|
|
682
|
+
}),
|
|
683
|
+
]);
|
|
379
684
|
}
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
events.push(event("error", {
|
|
383
|
-
assignment_id: assignment.id,
|
|
384
|
-
message,
|
|
385
|
-
retryable: false,
|
|
386
|
-
}));
|
|
387
|
-
events.push(event("lifecycle", {
|
|
388
|
-
stage: options.signal?.aborted ? "cancelled" : "failed",
|
|
389
|
-
assignment_id: assignment.id,
|
|
390
|
-
adapter: options.adapter,
|
|
391
|
-
detail: message,
|
|
392
|
-
}));
|
|
685
|
+
finally {
|
|
686
|
+
await runtimeProxy?.close();
|
|
393
687
|
}
|
|
394
688
|
await events.flush(options.signal?.aborted ? undefined : options.signal);
|
|
395
689
|
}
|
|
396
690
|
export async function runDaemon(options) {
|
|
691
|
+
const agentToken = options.environment?.CREWX_TOKEN?.trim();
|
|
692
|
+
const automaticOutboxPath = agentToken
|
|
693
|
+
? agentEventOutboxPath(options.api.baseUrl, agentToken, options.adapter, options.environment)
|
|
694
|
+
: undefined;
|
|
695
|
+
const durablePath = options.outbox?.path ?? automaticOutboxPath;
|
|
696
|
+
const processLock = options.processLock ??
|
|
697
|
+
(durablePath
|
|
698
|
+
? new AgentProcessLock(eventOutboxLockPath(durablePath))
|
|
699
|
+
: undefined);
|
|
700
|
+
// The lock precedes both session registration and outbox loading. This is
|
|
701
|
+
// intentionally wider than the network loop: two same-profile daemons must
|
|
702
|
+
// never hold independent cached snapshots of the same durable outcomes.
|
|
703
|
+
await processLock?.acquire();
|
|
704
|
+
try {
|
|
705
|
+
const durableOutbox = options.outbox ??
|
|
706
|
+
(automaticOutboxPath
|
|
707
|
+
? new DurableEventOutbox(automaticOutboxPath)
|
|
708
|
+
: undefined);
|
|
709
|
+
await runOwnedDaemon(options, durableOutbox);
|
|
710
|
+
}
|
|
711
|
+
finally {
|
|
712
|
+
await processLock?.release();
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
async function runOwnedDaemon(options, durableOutbox) {
|
|
397
716
|
const log = options.log ?? (() => undefined);
|
|
398
717
|
const startedAt = Date.now();
|
|
399
718
|
const session = await options.api.createSession({
|
|
@@ -409,12 +728,34 @@ export async function runDaemon(options) {
|
|
|
409
728
|
cwd: options.cwd,
|
|
410
729
|
},
|
|
411
730
|
}, options.signal);
|
|
412
|
-
const events = new EventBuffer(options.api, session.session_id);
|
|
413
731
|
let cursor = session.cursor;
|
|
414
732
|
let activeAssignmentId;
|
|
733
|
+
let activeAssignmentClaimToken;
|
|
415
734
|
let stopping = false;
|
|
416
735
|
const seenAssignmentEvents = new Set();
|
|
417
736
|
let workspaceContext = await options.api.context(session.session_id, options.signal);
|
|
737
|
+
const agent = record(workspaceContext.agent);
|
|
738
|
+
const events = new EventBuffer(options.api, session.session_id, {
|
|
739
|
+
...(durableOutbox ? { outbox: durableOutbox } : {}),
|
|
740
|
+
});
|
|
741
|
+
// Reuse original event UUIDs against the new session so the server can
|
|
742
|
+
// idempotently acknowledge outcomes persisted before a crash or restart.
|
|
743
|
+
await events.flush(options.signal);
|
|
744
|
+
if (options.adapter === "openclaw") {
|
|
745
|
+
const runtimeAgentId = nonEmptyString(agent?.runtime_agent_id);
|
|
746
|
+
if (!runtimeAgentId) {
|
|
747
|
+
throw new Error("CrewX did not provide an OpenClaw runtime agent ID for this profile.");
|
|
748
|
+
}
|
|
749
|
+
await (options.verifyOpenClaw ?? verifyOpenClawRuntime)({
|
|
750
|
+
cwd: options.cwd,
|
|
751
|
+
profile: {
|
|
752
|
+
runtimeAgentId,
|
|
753
|
+
sessionKey: "crewx-bootstrap",
|
|
754
|
+
},
|
|
755
|
+
...(options.environment ? { environment: options.environment } : {}),
|
|
756
|
+
});
|
|
757
|
+
log(`OpenClaw agent ${runtimeAgentId} is ready in ${options.cwd}.`);
|
|
758
|
+
}
|
|
418
759
|
events.push(event("lifecycle", { stage: "connected", adapter: options.adapter }));
|
|
419
760
|
await events.flush(options.signal);
|
|
420
761
|
log(`Connected session ${session.session_id} with ${options.adapter}.`);
|
|
@@ -423,6 +764,9 @@ export async function runDaemon(options) {
|
|
|
423
764
|
events.push(event("heartbeat", {
|
|
424
765
|
status: activeAssignmentId ? "running" : stopping ? "stopping" : "idle",
|
|
425
766
|
...(activeAssignmentId ? { assignment_id: activeAssignmentId } : {}),
|
|
767
|
+
...(activeAssignmentClaimToken
|
|
768
|
+
? { claim_token: activeAssignmentClaimToken }
|
|
769
|
+
: {}),
|
|
426
770
|
uptime_seconds: Math.floor((Date.now() - startedAt) / 1_000),
|
|
427
771
|
}));
|
|
428
772
|
void events
|
|
@@ -432,6 +776,8 @@ export async function runDaemon(options) {
|
|
|
432
776
|
heartbeatTimer.unref();
|
|
433
777
|
try {
|
|
434
778
|
let polls = 0;
|
|
779
|
+
let consecutivePollFailures = 0;
|
|
780
|
+
let consecutiveEmptyPolls = 0;
|
|
435
781
|
while (!options.signal?.aborted) {
|
|
436
782
|
try {
|
|
437
783
|
// Never acknowledge a claimed page while its lifecycle/output batch is
|
|
@@ -439,7 +785,7 @@ export async function runDaemon(options) {
|
|
|
439
785
|
await events.flush(options.signal);
|
|
440
786
|
const requestedCursor = cursor;
|
|
441
787
|
const response = await options.api.poll(session.session_id, cursor, options.signal);
|
|
442
|
-
|
|
788
|
+
consecutivePollFailures = 0;
|
|
443
789
|
polls += 1;
|
|
444
790
|
for (const inbound of response.events) {
|
|
445
791
|
const assignment = assignmentFromEvent(inbound);
|
|
@@ -453,49 +799,84 @@ export async function runDaemon(options) {
|
|
|
453
799
|
continue;
|
|
454
800
|
seenAssignmentEvents.add(executionKey);
|
|
455
801
|
activeAssignmentId = assignment.id;
|
|
802
|
+
activeAssignmentClaimToken = assignment.claim_token;
|
|
456
803
|
log(`Running ${assignment.title ?? assignment.id}…`);
|
|
457
804
|
try {
|
|
458
|
-
|
|
805
|
+
try {
|
|
806
|
+
workspaceContext = await options.api.context(session.session_id, options.signal);
|
|
807
|
+
}
|
|
808
|
+
catch (error) {
|
|
809
|
+
log(`Context refresh failed; using the previous snapshot: ${errorMessage(error)}`);
|
|
810
|
+
}
|
|
811
|
+
const linked = linkedAbortController(options.signal);
|
|
812
|
+
let assignmentFinished = false;
|
|
813
|
+
const assignmentEventCursor = typeof assignment.metadata?.assignment_event_cursor === "number"
|
|
814
|
+
? assignment.metadata.assignment_event_cursor
|
|
815
|
+
: undefined;
|
|
816
|
+
const canMonitor = typeof options.api.control === "function" &&
|
|
817
|
+
/^\d+$/.test(assignment.id);
|
|
818
|
+
const monitor = canMonitor
|
|
819
|
+
? monitorAssignmentControl({
|
|
820
|
+
api: options.api,
|
|
821
|
+
sessionId: session.session_id,
|
|
822
|
+
assignmentId: assignment.id,
|
|
823
|
+
assignmentEventCursor,
|
|
824
|
+
controller: linked.controller,
|
|
825
|
+
isFinished: () => assignmentFinished,
|
|
826
|
+
log,
|
|
827
|
+
})
|
|
828
|
+
: Promise.resolve(undefined);
|
|
829
|
+
let fatalControlError;
|
|
830
|
+
try {
|
|
831
|
+
await (options.executeAssignment ?? executeAssignment)({
|
|
832
|
+
assignment,
|
|
833
|
+
adapter: options.adapter,
|
|
834
|
+
cwd: assignmentWorkingDirectory(assignment, workspaceContext, options.cwd, options.lockedRoot ?? process.env.CREWX_BRIDGE_ROOT),
|
|
835
|
+
context: workspaceContext,
|
|
836
|
+
events,
|
|
837
|
+
serverUrl: options.api.baseUrl,
|
|
838
|
+
sessionId: session.session_id,
|
|
839
|
+
...(options.codingCommand
|
|
840
|
+
? { codingCommand: options.codingCommand }
|
|
841
|
+
: {}),
|
|
842
|
+
...(options.environment
|
|
843
|
+
? { environment: options.environment }
|
|
844
|
+
: {}),
|
|
845
|
+
signal: linked.controller.signal,
|
|
846
|
+
});
|
|
847
|
+
}
|
|
848
|
+
finally {
|
|
849
|
+
assignmentFinished = true;
|
|
850
|
+
linked.controller.abort();
|
|
851
|
+
linked.detach();
|
|
852
|
+
fatalControlError = await monitor;
|
|
853
|
+
}
|
|
854
|
+
if (fatalControlError)
|
|
855
|
+
throw fatalControlError;
|
|
856
|
+
await events.flush(options.signal);
|
|
459
857
|
}
|
|
460
858
|
catch (error) {
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
assignmentId: assignment.id,
|
|
472
|
-
controller: linked.controller,
|
|
473
|
-
isFinished: () => assignmentFinished,
|
|
474
|
-
log,
|
|
475
|
-
})
|
|
476
|
-
: Promise.resolve();
|
|
477
|
-
try {
|
|
478
|
-
await (options.executeAssignment ?? executeAssignment)({
|
|
479
|
-
assignment,
|
|
480
|
-
adapter: options.adapter,
|
|
481
|
-
cwd: assignmentWorkingDirectory(assignment, workspaceContext, options.cwd, options.lockedRoot ?? process.env.CREWX_BRIDGE_ROOT),
|
|
482
|
-
context: workspaceContext,
|
|
483
|
-
events,
|
|
484
|
-
serverUrl: options.api.baseUrl,
|
|
485
|
-
...(options.codingCommand ? { codingCommand: options.codingCommand } : {}),
|
|
486
|
-
...(options.environment ? { environment: options.environment } : {}),
|
|
487
|
-
signal: linked.controller.signal,
|
|
488
|
-
});
|
|
859
|
+
// A failed pre-run lifecycle flush means the runtime never
|
|
860
|
+
// started. Leave the page unacknowledged and allow that event to
|
|
861
|
+
// execute after delivery recovers. If a terminal outcome is
|
|
862
|
+
// queued, retain the seen fence: the next loop first delivers
|
|
863
|
+
// that durable outcome, then acknowledges the page without
|
|
864
|
+
// executing the runtime twice.
|
|
865
|
+
if (!events.hasPendingTerminalOutcome(assignment.id)) {
|
|
866
|
+
seenAssignmentEvents.delete(executionKey);
|
|
867
|
+
}
|
|
868
|
+
throw error;
|
|
489
869
|
}
|
|
490
870
|
finally {
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
linked.detach();
|
|
494
|
-
await monitor;
|
|
871
|
+
activeAssignmentId = undefined;
|
|
872
|
+
activeAssignmentClaimToken = undefined;
|
|
495
873
|
}
|
|
496
|
-
await events.flush(options.signal);
|
|
497
|
-
activeAssignmentId = undefined;
|
|
498
874
|
}
|
|
875
|
+
// Advancing the local cursor is the acknowledgement boundary. Do it
|
|
876
|
+
// only after every assignment on the page has a delivered terminal
|
|
877
|
+
// outcome (or was previously terminalized and fenced).
|
|
878
|
+
cursor = response.cursor;
|
|
879
|
+
seenAssignmentEvents.clear();
|
|
499
880
|
if (options.once && polls >= 1) {
|
|
500
881
|
if (String(cursor) !== String(requestedCursor)) {
|
|
501
882
|
// A final poll acknowledges the page that was just processed. Any
|
|
@@ -504,10 +885,21 @@ export async function runDaemon(options) {
|
|
|
504
885
|
}
|
|
505
886
|
break;
|
|
506
887
|
}
|
|
507
|
-
if (response.has_more)
|
|
888
|
+
if (response.has_more) {
|
|
889
|
+
consecutiveEmptyPolls = 0;
|
|
508
890
|
continue;
|
|
891
|
+
}
|
|
892
|
+
consecutiveEmptyPolls =
|
|
893
|
+
response.events.length === 0 ? consecutiveEmptyPolls + 1 : 0;
|
|
509
894
|
const serverDelay = response.retry_after_ms;
|
|
510
|
-
|
|
895
|
+
const pollDelay = consecutiveEmptyPolls > 0
|
|
896
|
+
? adaptiveIdlePollDelay(options.pollIntervalMs, consecutiveEmptyPolls, {
|
|
897
|
+
...(serverDelay !== undefined
|
|
898
|
+
? { retryAfterMs: serverDelay }
|
|
899
|
+
: {}),
|
|
900
|
+
})
|
|
901
|
+
: Math.max(options.pollIntervalMs, serverDelay ?? 0);
|
|
902
|
+
await sleep(pollDelay, options.signal);
|
|
511
903
|
}
|
|
512
904
|
catch (error) {
|
|
513
905
|
if (options.signal?.aborted)
|
|
@@ -516,9 +908,12 @@ export async function runDaemon(options) {
|
|
|
516
908
|
throw error;
|
|
517
909
|
const message = redactSecrets(errorMessage(error), [options.api.token]);
|
|
518
910
|
log(`Polling failed: ${message}. Retrying…`);
|
|
519
|
-
|
|
911
|
+
consecutivePollFailures += 1;
|
|
520
912
|
const serverDelay = error instanceof ApiError ? error.retryAfterMs : undefined;
|
|
521
|
-
|
|
913
|
+
const retryCap = Math.min(EVENT_RETRY_MAX_DELAY_MS, Math.max(options.pollIntervalMs, EVENT_RETRY_BASE_DELAY_MS) *
|
|
914
|
+
2 ** Math.min(8, consecutivePollFailures - 1));
|
|
915
|
+
const jitterDelay = Math.floor(Math.random() * (retryCap + 1));
|
|
916
|
+
await sleep(Math.max(250, jitterDelay, serverDelay ?? 0), options.signal);
|
|
522
917
|
}
|
|
523
918
|
}
|
|
524
919
|
}
|