crewx-agent-cli 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +80 -5
  2. package/dist/adapters.d.ts +12 -0
  3. package/dist/adapters.d.ts.map +1 -1
  4. package/dist/adapters.js +139 -42
  5. package/dist/adapters.js.map +1 -1
  6. package/dist/api.d.ts +40 -17
  7. package/dist/api.d.ts.map +1 -1
  8. package/dist/api.js +187 -11
  9. package/dist/api.js.map +1 -1
  10. package/dist/bridge-watchdog.d.ts +10 -0
  11. package/dist/bridge-watchdog.d.ts.map +1 -0
  12. package/dist/bridge-watchdog.js +45 -0
  13. package/dist/bridge-watchdog.js.map +1 -0
  14. package/dist/config.d.ts +9 -1
  15. package/dist/config.d.ts.map +1 -1
  16. package/dist/config.js +53 -2
  17. package/dist/config.js.map +1 -1
  18. package/dist/constants.d.ts +6 -1
  19. package/dist/constants.d.ts.map +1 -1
  20. package/dist/constants.js +11 -1
  21. package/dist/constants.js.map +1 -1
  22. package/dist/daemon.d.ts +34 -2
  23. package/dist/daemon.d.ts.map +1 -1
  24. package/dist/daemon.js +495 -100
  25. package/dist/daemon.js.map +1 -1
  26. package/dist/errors.d.ts +4 -1
  27. package/dist/errors.d.ts.map +1 -1
  28. package/dist/errors.js +3 -1
  29. package/dist/errors.js.map +1 -1
  30. package/dist/index.d.ts +3 -2
  31. package/dist/index.d.ts.map +1 -1
  32. package/dist/index.js +17 -6
  33. package/dist/index.js.map +1 -1
  34. package/dist/join.d.ts +1 -0
  35. package/dist/join.d.ts.map +1 -1
  36. package/dist/join.js +13 -0
  37. package/dist/join.js.map +1 -1
  38. package/dist/outbox.d.ts +22 -0
  39. package/dist/outbox.d.ts.map +1 -0
  40. package/dist/outbox.js +189 -0
  41. package/dist/outbox.js.map +1 -0
  42. package/dist/process-lock.d.ts +31 -0
  43. package/dist/process-lock.d.ts.map +1 -0
  44. package/dist/process-lock.js +272 -0
  45. package/dist/process-lock.js.map +1 -0
  46. package/dist/prompt.d.ts.map +1 -1
  47. package/dist/prompt.js +7 -0
  48. package/dist/prompt.js.map +1 -1
  49. package/dist/runtime-proxy.d.ts +18 -0
  50. package/dist/runtime-proxy.d.ts.map +1 -0
  51. package/dist/runtime-proxy.js +221 -0
  52. package/dist/runtime-proxy.js.map +1 -0
  53. package/package.json +6 -2
package/dist/daemon.js CHANGED
@@ -2,14 +2,18 @@ import { createHash } from "node:crypto";
2
2
  import { realpathSync } from "node:fs";
3
3
  import { hostname, platform, arch } from "node:os";
4
4
  import { isAbsolute, relative, resolve, sep } from "node:path";
5
- import { runAdapter } from "./adapters.js";
6
- import { ApiError } from "./api.js";
7
- import { CLI_VERSION, MAX_EVENT_CONTENT_LENGTH } from "./constants.js";
8
- import { errorMessage, redactSecrets } from "./errors.js";
5
+ import { runAdapter, verifyOpenClawRuntime, } from "./adapters.js";
6
+ import { ApiError, serializeOutboundEventBatch, } from "./api.js";
7
+ import { CLI_VERSION, MAX_BUFFERED_EVENT_BYTES, MAX_BUFFERED_EVENTS, MAX_EVENT_BATCH_BYTES, MAX_EVENT_CONTENT_LENGTH, } from "./constants.js";
8
+ import { CliError, errorMessage, redactSecrets } from "./errors.js";
9
9
  import { assemblePrompt, assignmentFromEvent } from "./prompt.js";
10
+ import { startRuntimeCredentialProxy, } from "./runtime-proxy.js";
11
+ import { DurableEventOutbox, agentEventOutboxPath, eventOutboxLockPath, } from "./outbox.js";
12
+ import { AgentProcessLock, } from "./process-lock.js";
10
13
  const ASSIGNMENT_CONTROL_INTERVAL_MS = 5_000;
11
14
  const EVENT_RETRY_BASE_DELAY_MS = 2_000;
12
15
  const EVENT_RETRY_MAX_DELAY_MS = 30_000;
16
+ const MAX_IDLE_POLL_DELAY_MS = 8_000;
13
17
  const MAX_RUNTIME_DIAGNOSTICS = 20;
14
18
  function timestamp() {
15
19
  return new Date().toISOString();
@@ -22,6 +26,19 @@ function event(type, payload) {
22
26
  occurred_at: timestamp(),
23
27
  };
24
28
  }
29
+ function assignmentEvent(assignment, type, payload) {
30
+ return {
31
+ id: crypto.randomUUID(),
32
+ type,
33
+ payload: {
34
+ ...payload,
35
+ ...(assignment.claim_token
36
+ ? { claim_token: assignment.claim_token }
37
+ : {}),
38
+ },
39
+ occurred_at: timestamp(),
40
+ };
41
+ }
25
42
  function truncate(value) {
26
43
  return value.length > MAX_EVENT_CONTENT_LENGTH
27
44
  ? `${value.slice(0, MAX_EVENT_CONTENT_LENGTH)}… [truncated]`
@@ -43,6 +60,30 @@ function record(value) {
43
60
  function nonEmptyString(value) {
44
61
  return typeof value === "string" && value.trim() ? value.trim() : undefined;
45
62
  }
63
+ function runtimeFailure(adapter, detail) {
64
+ if (adapter !== "openclaw") {
65
+ return { code: "runtime_exit_nonzero", message: detail };
66
+ }
67
+ if (/\b429\b|rate[ -]?limit|quota|too many requests/i.test(detail)) {
68
+ return {
69
+ code: "openclaw_provider_rate_limited",
70
+ message: `${detail} Check \`openclaw models status --agent <agent-id> --json\` for provider cooldowns, then retry after the provider limit resets.`,
71
+ };
72
+ }
73
+ if (/unauthori[sz]ed|authentication|credential|oauth|invalid token|expired/i.test(detail)) {
74
+ return {
75
+ code: "openclaw_auth_failed",
76
+ message: `${detail} Run \`openclaw models auth --agent <agent-id> list --json\` and repair the selected provider credentials.`,
77
+ };
78
+ }
79
+ if (/model.+(?:not found|unknown|unavailable|not allowed)/i.test(detail)) {
80
+ return {
81
+ code: "openclaw_model_unavailable",
82
+ message: `${detail} Run \`openclaw models status --agent <agent-id> --json\` and select an available model.`,
83
+ };
84
+ }
85
+ return { code: "openclaw_runtime_failed", message: detail };
86
+ }
46
87
  function identifier(value) {
47
88
  if (typeof value === "number" && Number.isSafeInteger(value))
48
89
  return String(value);
@@ -130,7 +171,7 @@ export function assignmentWorkingDirectory(assignment, context, fallback, locked
130
171
  if (relativePath === ".." ||
131
172
  relativePath.startsWith(`..${sep}`) ||
132
173
  isAbsolute(relativePath)) {
133
- throw new Error(`CrewX Bridge rejected assignment directory "${requested}" because it is outside the approved folder.`);
174
+ throw new Error("CrewX Bridge rejected the assignment directory because it is outside the approved folder.");
134
175
  }
135
176
  return canonicalCandidate;
136
177
  }
@@ -138,18 +179,65 @@ export class EventBuffer {
138
179
  api;
139
180
  sessionId;
140
181
  queue = [];
182
+ inFlight = new Set();
141
183
  flushing;
142
184
  retryNotBefore = 0;
143
185
  consecutiveFailures = 0;
144
- constructor(api, sessionId) {
186
+ maxEvents;
187
+ maxBytes;
188
+ random;
189
+ durableSnapshot;
190
+ constructor(api, sessionId, options = {}) {
145
191
  this.api = api;
146
192
  this.sessionId = sessionId;
193
+ this.maxEvents = options.maxEvents ?? MAX_BUFFERED_EVENTS;
194
+ this.maxBytes = options.maxBytes ?? MAX_BUFFERED_EVENT_BYTES;
195
+ this.random = options.random ?? Math.random;
196
+ this.outbox = options.outbox;
197
+ if (this.outbox) {
198
+ this.queue.push(...this.outbox.load());
199
+ this.enforceBounds();
200
+ }
201
+ this.durableSnapshot = this.serializeDurableEvents();
147
202
  }
203
+ outbox;
148
204
  push(item) {
149
- this.queue.push(item);
205
+ this.pushMany([item]);
206
+ }
207
+ pushMany(items) {
208
+ if (items.length === 0)
209
+ return;
210
+ const previous = [...this.queue];
211
+ try {
212
+ for (const item of items) {
213
+ if (!this.coalesce(item))
214
+ this.queue.push(item);
215
+ }
216
+ this.enforceBounds();
217
+ this.persistDurableEvents();
218
+ }
219
+ catch (error) {
220
+ this.queue.splice(0, this.queue.length, ...previous);
221
+ throw error;
222
+ }
150
223
  if (this.queue.length >= 25)
151
224
  void this.flush().catch(() => undefined);
152
225
  }
226
+ pendingCount() {
227
+ return this.queue.length;
228
+ }
229
+ pendingBytes() {
230
+ return this.queue.reduce((total, item) => total + this.eventBytes(item), 0);
231
+ }
232
+ hasPendingTerminalOutcome(assignmentId) {
233
+ const isTerminalOutcome = (item) => item.type === "lifecycle" &&
234
+ item.payload.assignment_id === assignmentId &&
235
+ (item.payload.stage === "completed" ||
236
+ item.payload.stage === "failed" ||
237
+ item.payload.stage === "cancelled");
238
+ return (this.queue.some(isTerminalOutcome) ||
239
+ [...this.inFlight].some(isTerminalOutcome));
240
+ }
153
241
  async flush(signal) {
154
242
  if (this.flushing) {
155
243
  await this.flushing;
@@ -171,21 +259,143 @@ export class EventBuffer {
171
259
  await sleep(retryDelay, signal);
172
260
  if (signal?.aborted)
173
261
  return;
174
- const batch = this.queue.slice(0, 100);
262
+ const batch = [];
263
+ for (const item of this.queue) {
264
+ if (batch.length >= 100) {
265
+ break;
266
+ }
267
+ const candidate = [...batch, item];
268
+ const candidateBytes = Buffer.byteLength(serializeOutboundEventBatch(this.sessionId, candidate), "utf8");
269
+ if (candidateBytes > MAX_EVENT_BATCH_BYTES) {
270
+ if (batch.length === 0) {
271
+ throw new CliError(`CrewX cannot send event ${item.id ?? "(without an ID)"} because its exact serialized batch exceeds ${String(MAX_EVENT_BATCH_BYTES)} bytes.`, 1, { code: "event_batch_too_large" });
272
+ }
273
+ break;
274
+ }
275
+ batch.push(item);
276
+ }
277
+ for (const item of batch)
278
+ this.inFlight.add(item);
175
279
  try {
176
280
  await this.api.sendEvents(this.sessionId, batch, signal);
177
- this.queue.splice(0, batch.length);
281
+ const delivered = this.queue.splice(0, batch.length);
282
+ try {
283
+ this.persistDurableEvents();
284
+ }
285
+ catch (error) {
286
+ this.queue.unshift(...delivered);
287
+ throw error;
288
+ }
178
289
  this.consecutiveFailures = 0;
179
290
  this.retryNotBefore = 0;
180
291
  }
181
292
  catch (error) {
182
293
  this.consecutiveFailures += 1;
183
294
  const exponentialDelay = Math.min(EVENT_RETRY_MAX_DELAY_MS, EVENT_RETRY_BASE_DELAY_MS * 2 ** (this.consecutiveFailures - 1));
295
+ const jitterDelay = Math.floor(this.random() * (exponentialDelay + 1));
184
296
  const serverDelay = error instanceof ApiError ? error.retryAfterMs : undefined;
185
297
  this.retryNotBefore =
186
- Date.now() + Math.max(exponentialDelay, serverDelay ?? 0);
298
+ Date.now() + Math.max(jitterDelay, serverDelay ?? 0);
187
299
  throw error;
188
300
  }
301
+ finally {
302
+ for (const item of batch)
303
+ this.inFlight.delete(item);
304
+ }
305
+ }
306
+ }
307
+ coalesce(item) {
308
+ if (item.id) {
309
+ const duplicateIndex = this.findNewestIndex((candidate) => candidate.id === item.id);
310
+ if (duplicateIndex >= 0) {
311
+ this.queue[duplicateIndex] = item;
312
+ return true;
313
+ }
314
+ }
315
+ if (item.type === "heartbeat") {
316
+ const existingIndex = this.findNewestIndex((candidate) => candidate.type === "heartbeat");
317
+ if (existingIndex >= 0) {
318
+ this.queue[existingIndex] = item;
319
+ return true;
320
+ }
321
+ }
322
+ if (item.type === "output") {
323
+ const existingIndex = this.findNewestIndex((candidate) => candidate.type === "output" &&
324
+ candidate.payload.assignment_id === item.payload.assignment_id &&
325
+ candidate.payload.stream === item.payload.stream);
326
+ const existing = this.queue[existingIndex];
327
+ if (existingIndex >= 0 && existing?.type === "output") {
328
+ const combined = `${existing.payload.content}\n${item.payload.content}`;
329
+ const content = combined.length <= MAX_EVENT_CONTENT_LENGTH
330
+ ? combined
331
+ : `[earlier output omitted during backpressure]\n${combined.slice(-(MAX_EVENT_CONTENT_LENGTH - 47))}`;
332
+ this.queue[existingIndex] = {
333
+ ...item,
334
+ payload: {
335
+ ...item.payload,
336
+ content,
337
+ },
338
+ };
339
+ return true;
340
+ }
341
+ }
342
+ return false;
343
+ }
344
+ findNewestIndex(predicate) {
345
+ for (let index = this.queue.length - 1; index >= 0; index -= 1) {
346
+ const candidate = this.queue[index];
347
+ if (candidate &&
348
+ !this.inFlight.has(candidate) &&
349
+ predicate(candidate)) {
350
+ return index;
351
+ }
352
+ }
353
+ return -1;
354
+ }
355
+ eventPriority(item) {
356
+ if (item.type === "heartbeat" || item.type === "output")
357
+ return 0;
358
+ if (item.type === "error" ||
359
+ (item.type === "message" && item.payload.role === "assistant") ||
360
+ (item.type === "lifecycle" &&
361
+ ["completed", "failed", "cancelled", "stopped"].includes(item.payload.stage))) {
362
+ return 2;
363
+ }
364
+ return 1;
365
+ }
366
+ eventBytes(item) {
367
+ return Buffer.byteLength(JSON.stringify(item));
368
+ }
369
+ persistDurableEvents() {
370
+ if (!this.outbox)
371
+ return;
372
+ const durable = this.queue.filter((item) => this.eventPriority(item) === 2);
373
+ const snapshot = JSON.stringify(durable);
374
+ if (snapshot === this.durableSnapshot)
375
+ return;
376
+ this.outbox.replace(durable);
377
+ this.durableSnapshot = snapshot;
378
+ }
379
+ serializeDurableEvents() {
380
+ return JSON.stringify(this.queue.filter((item) => this.eventPriority(item) === 2));
381
+ }
382
+ enforceBounds() {
383
+ let bytes = this.pendingBytes();
384
+ while (this.queue.length > this.maxEvents ||
385
+ bytes > this.maxBytes) {
386
+ let removeIndex = -1;
387
+ for (const priority of [0, 1]) {
388
+ removeIndex = this.queue.findIndex((item) => !this.inFlight.has(item) &&
389
+ this.eventPriority(item) === priority);
390
+ if (removeIndex >= 0)
391
+ break;
392
+ }
393
+ if (removeIndex < 0) {
394
+ throw new CliError("CrewX cannot buffer another final agent outcome while event delivery is unavailable. Existing final outcomes were retained; restore connectivity before accepting more work.", 1, { code: "event_buffer_capacity_exceeded" });
395
+ }
396
+ const [removed] = this.queue.splice(removeIndex, 1);
397
+ if (removed)
398
+ bytes -= this.eventBytes(removed);
189
399
  }
190
400
  }
191
401
  }
@@ -202,6 +412,18 @@ function sleep(milliseconds, signal) {
202
412
  signal?.addEventListener("abort", finish, { once: true });
203
413
  });
204
414
  }
415
+ export function adaptiveIdlePollDelay(baseDelayMs, consecutiveEmptyPolls, options = {}) {
416
+ const base = Math.max(250, Math.floor(baseDelayMs));
417
+ const maximum = Math.max(base, MAX_IDLE_POLL_DELAY_MS);
418
+ const exponent = Math.max(0, Math.min(8, consecutiveEmptyPolls - 1));
419
+ const cap = Math.min(maximum, base * 2 ** exponent);
420
+ const random = options.random ?? Math.random;
421
+ const jittered = cap <= base
422
+ ? base
423
+ : base +
424
+ Math.round(Math.max(0, Math.min(1, random())) * (cap - base));
425
+ return Math.max(jittered, options.retryAfterMs ?? 0);
426
+ }
205
427
  function linkedAbortController(parent) {
206
428
  const controller = new AbortController();
207
429
  const abort = () => controller.abort(parent?.reason);
@@ -222,17 +444,24 @@ async function monitorAssignmentControl(options) {
222
444
  while (!options.isFinished() && !options.controller.signal.aborted) {
223
445
  let nextCheckDelay = ASSIGNMENT_CONTROL_INTERVAL_MS;
224
446
  try {
225
- const control = await options.api.control(options.sessionId, options.assignmentId, options.controller.signal);
447
+ const control = await options.api.control(options.sessionId, options.assignmentId, options.controller.signal, options.assignmentEventCursor);
226
448
  if (control.cancelled) {
227
449
  options.log(`CrewX cancelled ${options.assignmentId}; stopping the local agent…`);
228
450
  options.controller.abort(new Error("Assignment cancelled by CrewX"));
229
- return;
451
+ return undefined;
230
452
  }
231
453
  lastLoggedError = undefined;
232
454
  }
233
455
  catch (error) {
234
- if (options.controller.signal.aborted || options.isFinished())
235
- return;
456
+ if (options.controller.signal.aborted || options.isFinished()) {
457
+ return undefined;
458
+ }
459
+ if (error instanceof ApiError &&
460
+ (error.status === 401 || error.status === 403)) {
461
+ options.log("CrewX revoked this agent connection; stopping the local agent and daemon…");
462
+ options.controller.abort(error);
463
+ return error;
464
+ }
236
465
  const message = errorMessage(error);
237
466
  const now = Date.now();
238
467
  if (message !== lastLoggedError || now - lastErrorLoggedAt >= 30_000) {
@@ -246,38 +475,68 @@ async function monitorAssignmentControl(options) {
246
475
  }
247
476
  await sleep(nextCheckDelay, options.controller.signal);
248
477
  }
478
+ return undefined;
249
479
  }
250
480
  export async function executeAssignment(options) {
251
481
  const { assignment, events } = options;
252
- const secrets = options.environment?.CREWX_TOKEN
253
- ? [options.environment.CREWX_TOKEN]
254
- : [];
482
+ const secrets = [
483
+ ...(options.environment?.CREWX_TOKEN
484
+ ? [options.environment.CREWX_TOKEN]
485
+ : []),
486
+ ...(assignment.claim_token ? [assignment.claim_token] : []),
487
+ ];
255
488
  const nativeJsonRuntime = !options.codingCommand &&
256
489
  (options.adapter === "codex" ||
257
490
  options.adapter === "claude" ||
258
491
  options.adapter === "pi");
259
492
  const runtimeDiagnostics = new Set();
260
- events.push(event("lifecycle", {
493
+ const pendingRuntimeMessages = [];
494
+ let terminalizationStarted = false;
495
+ events.push(assignmentEvent(assignment, "lifecycle", {
261
496
  stage: "claimed",
262
497
  assignment_id: assignment.id,
263
498
  adapter: options.adapter,
264
499
  }));
265
- events.push(event("message", {
500
+ events.push(assignmentEvent(assignment, "message", {
266
501
  assignment_id: assignment.id,
267
502
  role: "user",
268
503
  content: assignment.prompt,
269
504
  }));
270
- events.push(event("lifecycle", {
505
+ events.push(assignmentEvent(assignment, "lifecycle", {
271
506
  stage: "started",
272
507
  assignment_id: assignment.id,
273
508
  adapter: options.adapter,
274
509
  }));
275
510
  await events.flush(options.signal);
511
+ let runtimeProxy;
276
512
  try {
277
513
  const agent = record(options.context.agent);
278
514
  const permissionPreset = agent?.permission_preset;
279
515
  const configuredModel = nonEmptyString(agent?.model);
280
516
  const runtimeAgentId = nonEmptyString(agent?.runtime_agent_id);
517
+ const upstreamToken = options.environment?.CREWX_TOKEN?.trim();
518
+ const cliPath = options.environment?.CREWX_CLI_PATH?.trim();
519
+ if (upstreamToken && cliPath) {
520
+ runtimeProxy = await startRuntimeCredentialProxy({
521
+ upstreamUrl: options.serverUrl,
522
+ upstreamToken,
523
+ cliPath,
524
+ assignmentId: assignment.id,
525
+ ...(assignment.claim_token
526
+ ? { claimToken: assignment.claim_token }
527
+ : {}),
528
+ ...(options.sessionId ? { sessionId: options.sessionId } : {}),
529
+ permissionPreset: permissionPreset === "read_only" ||
530
+ permissionPreset === "standard" ||
531
+ permissionPreset === "full_access"
532
+ ? permissionPreset
533
+ : "read_only",
534
+ chatOnly: agent?.chat_only === true,
535
+ });
536
+ const proxyToken = runtimeProxy.environment.CREWX_TOKEN;
537
+ if (proxyToken)
538
+ secrets.push(proxyToken);
539
+ }
281
540
  const result = await (options.runAgent ?? runAdapter)({
282
541
  adapter: options.adapter,
283
542
  prompt: assemblePrompt(assignment, options.context),
@@ -297,6 +556,9 @@ export async function executeAssignment(options) {
297
556
  },
298
557
  ...(options.codingCommand ? { codingCommand: options.codingCommand } : {}),
299
558
  ...(options.environment ? { environment: options.environment } : {}),
559
+ ...(runtimeProxy
560
+ ? { controlEnvironment: runtimeProxy.environment }
561
+ : {}),
300
562
  ...(options.signal ? { signal: options.signal } : {}),
301
563
  onStdout(line, parsed) {
302
564
  if (nativeJsonRuntime) {
@@ -309,7 +571,7 @@ export async function executeAssignment(options) {
309
571
  return;
310
572
  }
311
573
  runtimeDiagnostics.add(content);
312
- events.push(event("output", {
574
+ events.push(assignmentEvent(assignment, "output", {
313
575
  assignment_id: assignment.id,
314
576
  stream: "stdout",
315
577
  content,
@@ -318,7 +580,7 @@ export async function executeAssignment(options) {
318
580
  }
319
581
  const content = truncate(redactSecrets(line, secrets));
320
582
  const raw = sanitizedRaw(parsed.raw, secrets);
321
- events.push(event("output", {
583
+ events.push(assignmentEvent(assignment, "output", {
322
584
  assignment_id: assignment.id,
323
585
  stream: "stdout",
324
586
  content,
@@ -330,14 +592,20 @@ export async function executeAssignment(options) {
330
592
  }));
331
593
  },
332
594
  onMessage(message, role) {
333
- events.push(event("message", {
595
+ const runtimeMessage = assignmentEvent(assignment, "message", {
334
596
  assignment_id: assignment.id,
335
597
  role,
336
598
  content: truncate(redactSecrets(message, secrets)),
337
- }));
599
+ });
600
+ if (role === "assistant" || role === "tool") {
601
+ pendingRuntimeMessages.push(runtimeMessage);
602
+ }
603
+ else {
604
+ events.push(runtimeMessage);
605
+ }
338
606
  },
339
607
  onStderr(line) {
340
- events.push(event("output", {
608
+ events.push(assignmentEvent(assignment, "output", {
341
609
  assignment_id: assignment.id,
342
610
  stream: "stderr",
343
611
  content: truncate(redactSecrets(line, secrets)),
@@ -345,55 +613,106 @@ export async function executeAssignment(options) {
345
613
  },
346
614
  });
347
615
  if (options.signal?.aborted) {
348
- events.push(event("lifecycle", {
349
- stage: "cancelled",
350
- assignment_id: assignment.id,
351
- adapter: options.adapter,
352
- }));
616
+ terminalizationStarted = true;
617
+ events.pushMany([
618
+ ...pendingRuntimeMessages,
619
+ assignmentEvent(assignment, "lifecycle", {
620
+ stage: "cancelled",
621
+ assignment_id: assignment.id,
622
+ adapter: options.adapter,
623
+ }),
624
+ ]);
353
625
  }
354
626
  else if (result.exitCode === 0) {
355
- events.push(event("lifecycle", {
356
- stage: "completed",
357
- assignment_id: assignment.id,
358
- adapter: options.adapter,
359
- exit_code: result.exitCode,
360
- }));
627
+ terminalizationStarted = true;
628
+ events.pushMany([
629
+ ...pendingRuntimeMessages,
630
+ assignmentEvent(assignment, "lifecycle", {
631
+ stage: "completed",
632
+ assignment_id: assignment.id,
633
+ adapter: options.adapter,
634
+ exit_code: result.exitCode,
635
+ }),
636
+ ]);
361
637
  }
362
638
  else {
363
639
  const detail = result.stderr.at(-1) ??
364
640
  `${options.adapter} exited with code ${String(result.exitCode)}`;
365
- const safeDetail = redactSecrets(detail, secrets);
366
- events.push(event("error", {
641
+ const failure = runtimeFailure(options.adapter, detail);
642
+ const safeDetail = redactSecrets(failure.message, secrets);
643
+ terminalizationStarted = true;
644
+ events.pushMany([
645
+ ...pendingRuntimeMessages,
646
+ assignmentEvent(assignment, "error", {
647
+ assignment_id: assignment.id,
648
+ message: truncate(safeDetail),
649
+ code: failure.code,
650
+ retryable: false,
651
+ }),
652
+ assignmentEvent(assignment, "lifecycle", {
653
+ stage: "failed",
654
+ assignment_id: assignment.id,
655
+ adapter: options.adapter,
656
+ exit_code: result.exitCode,
657
+ detail: truncate(safeDetail),
658
+ }),
659
+ ]);
660
+ }
661
+ }
662
+ catch (error) {
663
+ if (terminalizationStarted)
664
+ throw error;
665
+ const message = truncate(redactSecrets(errorMessage(error), secrets));
666
+ terminalizationStarted = true;
667
+ events.pushMany([
668
+ ...pendingRuntimeMessages,
669
+ assignmentEvent(assignment, "error", {
367
670
  assignment_id: assignment.id,
368
- message: truncate(safeDetail),
671
+ message,
672
+ ...(error instanceof CliError && error.code
673
+ ? { code: error.code }
674
+ : {}),
369
675
  retryable: false,
370
- }));
371
- events.push(event("lifecycle", {
372
- stage: "failed",
676
+ }),
677
+ assignmentEvent(assignment, "lifecycle", {
678
+ stage: options.signal?.aborted ? "cancelled" : "failed",
373
679
  assignment_id: assignment.id,
374
680
  adapter: options.adapter,
375
- exit_code: result.exitCode,
376
- detail: truncate(safeDetail),
377
- }));
378
- }
681
+ detail: message,
682
+ }),
683
+ ]);
379
684
  }
380
- catch (error) {
381
- const message = truncate(redactSecrets(errorMessage(error), secrets));
382
- events.push(event("error", {
383
- assignment_id: assignment.id,
384
- message,
385
- retryable: false,
386
- }));
387
- events.push(event("lifecycle", {
388
- stage: options.signal?.aborted ? "cancelled" : "failed",
389
- assignment_id: assignment.id,
390
- adapter: options.adapter,
391
- detail: message,
392
- }));
685
+ finally {
686
+ await runtimeProxy?.close();
393
687
  }
394
688
  await events.flush(options.signal?.aborted ? undefined : options.signal);
395
689
  }
396
690
  export async function runDaemon(options) {
691
+ const agentToken = options.environment?.CREWX_TOKEN?.trim();
692
+ const automaticOutboxPath = agentToken
693
+ ? agentEventOutboxPath(options.api.baseUrl, agentToken, options.adapter, options.environment)
694
+ : undefined;
695
+ const durablePath = options.outbox?.path ?? automaticOutboxPath;
696
+ const processLock = options.processLock ??
697
+ (durablePath
698
+ ? new AgentProcessLock(eventOutboxLockPath(durablePath))
699
+ : undefined);
700
+ // The lock precedes both session registration and outbox loading. This is
701
+ // intentionally wider than the network loop: two same-profile daemons must
702
+ // never hold independent cached snapshots of the same durable outcomes.
703
+ await processLock?.acquire();
704
+ try {
705
+ const durableOutbox = options.outbox ??
706
+ (automaticOutboxPath
707
+ ? new DurableEventOutbox(automaticOutboxPath)
708
+ : undefined);
709
+ await runOwnedDaemon(options, durableOutbox);
710
+ }
711
+ finally {
712
+ await processLock?.release();
713
+ }
714
+ }
715
+ async function runOwnedDaemon(options, durableOutbox) {
397
716
  const log = options.log ?? (() => undefined);
398
717
  const startedAt = Date.now();
399
718
  const session = await options.api.createSession({
@@ -409,12 +728,34 @@ export async function runDaemon(options) {
409
728
  cwd: options.cwd,
410
729
  },
411
730
  }, options.signal);
412
- const events = new EventBuffer(options.api, session.session_id);
413
731
  let cursor = session.cursor;
414
732
  let activeAssignmentId;
733
+ let activeAssignmentClaimToken;
415
734
  let stopping = false;
416
735
  const seenAssignmentEvents = new Set();
417
736
  let workspaceContext = await options.api.context(session.session_id, options.signal);
737
+ const agent = record(workspaceContext.agent);
738
+ const events = new EventBuffer(options.api, session.session_id, {
739
+ ...(durableOutbox ? { outbox: durableOutbox } : {}),
740
+ });
741
+ // Reuse original event UUIDs against the new session so the server can
742
+ // idempotently acknowledge outcomes persisted before a crash or restart.
743
+ await events.flush(options.signal);
744
+ if (options.adapter === "openclaw") {
745
+ const runtimeAgentId = nonEmptyString(agent?.runtime_agent_id);
746
+ if (!runtimeAgentId) {
747
+ throw new Error("CrewX did not provide an OpenClaw runtime agent ID for this profile.");
748
+ }
749
+ await (options.verifyOpenClaw ?? verifyOpenClawRuntime)({
750
+ cwd: options.cwd,
751
+ profile: {
752
+ runtimeAgentId,
753
+ sessionKey: "crewx-bootstrap",
754
+ },
755
+ ...(options.environment ? { environment: options.environment } : {}),
756
+ });
757
+ log(`OpenClaw agent ${runtimeAgentId} is ready in ${options.cwd}.`);
758
+ }
418
759
  events.push(event("lifecycle", { stage: "connected", adapter: options.adapter }));
419
760
  await events.flush(options.signal);
420
761
  log(`Connected session ${session.session_id} with ${options.adapter}.`);
@@ -423,6 +764,9 @@ export async function runDaemon(options) {
423
764
  events.push(event("heartbeat", {
424
765
  status: activeAssignmentId ? "running" : stopping ? "stopping" : "idle",
425
766
  ...(activeAssignmentId ? { assignment_id: activeAssignmentId } : {}),
767
+ ...(activeAssignmentClaimToken
768
+ ? { claim_token: activeAssignmentClaimToken }
769
+ : {}),
426
770
  uptime_seconds: Math.floor((Date.now() - startedAt) / 1_000),
427
771
  }));
428
772
  void events
@@ -432,6 +776,8 @@ export async function runDaemon(options) {
432
776
  heartbeatTimer.unref();
433
777
  try {
434
778
  let polls = 0;
779
+ let consecutivePollFailures = 0;
780
+ let consecutiveEmptyPolls = 0;
435
781
  while (!options.signal?.aborted) {
436
782
  try {
437
783
  // Never acknowledge a claimed page while its lifecycle/output batch is
@@ -439,7 +785,7 @@ export async function runDaemon(options) {
439
785
  await events.flush(options.signal);
440
786
  const requestedCursor = cursor;
441
787
  const response = await options.api.poll(session.session_id, cursor, options.signal);
442
- cursor = response.cursor;
788
+ consecutivePollFailures = 0;
443
789
  polls += 1;
444
790
  for (const inbound of response.events) {
445
791
  const assignment = assignmentFromEvent(inbound);
@@ -453,49 +799,84 @@ export async function runDaemon(options) {
453
799
  continue;
454
800
  seenAssignmentEvents.add(executionKey);
455
801
  activeAssignmentId = assignment.id;
802
+ activeAssignmentClaimToken = assignment.claim_token;
456
803
  log(`Running ${assignment.title ?? assignment.id}…`);
457
804
  try {
458
- workspaceContext = await options.api.context(session.session_id, options.signal);
805
+ try {
806
+ workspaceContext = await options.api.context(session.session_id, options.signal);
807
+ }
808
+ catch (error) {
809
+ log(`Context refresh failed; using the previous snapshot: ${errorMessage(error)}`);
810
+ }
811
+ const linked = linkedAbortController(options.signal);
812
+ let assignmentFinished = false;
813
+ const assignmentEventCursor = typeof assignment.metadata?.assignment_event_cursor === "number"
814
+ ? assignment.metadata.assignment_event_cursor
815
+ : undefined;
816
+ const canMonitor = typeof options.api.control === "function" &&
817
+ /^\d+$/.test(assignment.id);
818
+ const monitor = canMonitor
819
+ ? monitorAssignmentControl({
820
+ api: options.api,
821
+ sessionId: session.session_id,
822
+ assignmentId: assignment.id,
823
+ assignmentEventCursor,
824
+ controller: linked.controller,
825
+ isFinished: () => assignmentFinished,
826
+ log,
827
+ })
828
+ : Promise.resolve(undefined);
829
+ let fatalControlError;
830
+ try {
831
+ await (options.executeAssignment ?? executeAssignment)({
832
+ assignment,
833
+ adapter: options.adapter,
834
+ cwd: assignmentWorkingDirectory(assignment, workspaceContext, options.cwd, options.lockedRoot ?? process.env.CREWX_BRIDGE_ROOT),
835
+ context: workspaceContext,
836
+ events,
837
+ serverUrl: options.api.baseUrl,
838
+ sessionId: session.session_id,
839
+ ...(options.codingCommand
840
+ ? { codingCommand: options.codingCommand }
841
+ : {}),
842
+ ...(options.environment
843
+ ? { environment: options.environment }
844
+ : {}),
845
+ signal: linked.controller.signal,
846
+ });
847
+ }
848
+ finally {
849
+ assignmentFinished = true;
850
+ linked.controller.abort();
851
+ linked.detach();
852
+ fatalControlError = await monitor;
853
+ }
854
+ if (fatalControlError)
855
+ throw fatalControlError;
856
+ await events.flush(options.signal);
459
857
  }
460
858
  catch (error) {
461
- log(`Context refresh failed; using the previous snapshot: ${errorMessage(error)}`);
462
- }
463
- const linked = linkedAbortController(options.signal);
464
- let assignmentFinished = false;
465
- const canMonitor = typeof options.api.control === "function" &&
466
- /^\d+$/.test(assignment.id);
467
- const monitor = canMonitor
468
- ? monitorAssignmentControl({
469
- api: options.api,
470
- sessionId: session.session_id,
471
- assignmentId: assignment.id,
472
- controller: linked.controller,
473
- isFinished: () => assignmentFinished,
474
- log,
475
- })
476
- : Promise.resolve();
477
- try {
478
- await (options.executeAssignment ?? executeAssignment)({
479
- assignment,
480
- adapter: options.adapter,
481
- cwd: assignmentWorkingDirectory(assignment, workspaceContext, options.cwd, options.lockedRoot ?? process.env.CREWX_BRIDGE_ROOT),
482
- context: workspaceContext,
483
- events,
484
- serverUrl: options.api.baseUrl,
485
- ...(options.codingCommand ? { codingCommand: options.codingCommand } : {}),
486
- ...(options.environment ? { environment: options.environment } : {}),
487
- signal: linked.controller.signal,
488
- });
859
+ // A failed pre-run lifecycle flush means the runtime never
860
+ // started. Leave the page unacknowledged and allow that event to
861
+ // execute after delivery recovers. If a terminal outcome is
862
+ // queued, retain the seen fence: the next loop first delivers
863
+ // that durable outcome, then acknowledges the page without
864
+ // executing the runtime twice.
865
+ if (!events.hasPendingTerminalOutcome(assignment.id)) {
866
+ seenAssignmentEvents.delete(executionKey);
867
+ }
868
+ throw error;
489
869
  }
490
870
  finally {
491
- assignmentFinished = true;
492
- linked.controller.abort();
493
- linked.detach();
494
- await monitor;
871
+ activeAssignmentId = undefined;
872
+ activeAssignmentClaimToken = undefined;
495
873
  }
496
- await events.flush(options.signal);
497
- activeAssignmentId = undefined;
498
874
  }
875
+ // Advancing the local cursor is the acknowledgement boundary. Do it
876
+ // only after every assignment on the page has a delivered terminal
877
+ // outcome (or was previously terminalized and fenced).
878
+ cursor = response.cursor;
879
+ seenAssignmentEvents.clear();
499
880
  if (options.once && polls >= 1) {
500
881
  if (String(cursor) !== String(requestedCursor)) {
501
882
  // A final poll acknowledges the page that was just processed. Any
@@ -504,10 +885,21 @@ export async function runDaemon(options) {
504
885
  }
505
886
  break;
506
887
  }
507
- if (response.has_more)
888
+ if (response.has_more) {
889
+ consecutiveEmptyPolls = 0;
508
890
  continue;
891
+ }
892
+ consecutiveEmptyPolls =
893
+ response.events.length === 0 ? consecutiveEmptyPolls + 1 : 0;
509
894
  const serverDelay = response.retry_after_ms;
510
- await sleep(serverDelay ?? options.pollIntervalMs, options.signal);
895
+ const pollDelay = consecutiveEmptyPolls > 0
896
+ ? adaptiveIdlePollDelay(options.pollIntervalMs, consecutiveEmptyPolls, {
897
+ ...(serverDelay !== undefined
898
+ ? { retryAfterMs: serverDelay }
899
+ : {}),
900
+ })
901
+ : Math.max(options.pollIntervalMs, serverDelay ?? 0);
902
+ await sleep(pollDelay, options.signal);
511
903
  }
512
904
  catch (error) {
513
905
  if (options.signal?.aborted)
@@ -516,9 +908,12 @@ export async function runDaemon(options) {
516
908
  throw error;
517
909
  const message = redactSecrets(errorMessage(error), [options.api.token]);
518
910
  log(`Polling failed: ${message}. Retrying…`);
519
- events.push(event("error", { message, retryable: true }));
911
+ consecutivePollFailures += 1;
520
912
  const serverDelay = error instanceof ApiError ? error.retryAfterMs : undefined;
521
- await sleep(Math.max(options.pollIntervalMs, 2_000, serverDelay ?? 0), options.signal);
913
+ const retryCap = Math.min(EVENT_RETRY_MAX_DELAY_MS, Math.max(options.pollIntervalMs, EVENT_RETRY_BASE_DELAY_MS) *
914
+ 2 ** Math.min(8, consecutivePollFailures - 1));
915
+ const jitterDelay = Math.floor(Math.random() * (retryCap + 1));
916
+ await sleep(Math.max(250, jitterDelay, serverDelay ?? 0), options.signal);
522
917
  }
523
918
  }
524
919
  }