@botbuddy/cli 1.59.1 → 1.59.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/run.mjs +295 -32
- package/src/wait-core.mjs +7 -2
- package/src/wait.mjs +16 -2
package/package.json
CHANGED
package/src/run.mjs
CHANGED
|
@@ -215,6 +215,66 @@ export function commandHash(command) { return createHash("sha256").update(JSON.s
|
|
|
215
215
|
export function environmentHash(cwd, environment) { return createHash("sha256").update(`${cwd}\0${environment}`).digest("hex"); }
|
|
216
216
|
export function receiptPath(runId) { return join(homedir(), ".botbuddy", "command-runs", `${runId}.receipt.json`); }
|
|
217
217
|
export function contextPath(runId) { return join(homedir(), ".botbuddy", "command-runs", `${runId}.context.json`); }
|
|
218
|
+
function cancellationPath(contextFile) { return `${contextFile}.cancelled`; }
|
|
219
|
+
|
|
220
|
+
async function markCancellation(contextFile, runId) {
|
|
221
|
+
try {
|
|
222
|
+
await writeFile(cancellationPath(contextFile), JSON.stringify({ run_id: runId, reason: "cancelled_before_workload_start", requested_at: new Date().toISOString() }), { mode: 0o600, flag: "wx" });
|
|
223
|
+
return true;
|
|
224
|
+
} catch (error) {
|
|
225
|
+
if (error?.code === "EEXIST") return false;
|
|
226
|
+
throw error;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
async function hasCancellationMarker(contextFile) {
|
|
231
|
+
try { await readFile(cancellationPath(contextFile), "utf8"); return true; } catch (error) {
|
|
232
|
+
if (error?.code === "ENOENT") return false;
|
|
233
|
+
throw error;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
async function preWorkloadCancellationReceipt(context) {
|
|
238
|
+
const now = new Date().toISOString();
|
|
239
|
+
const receipt = {
|
|
240
|
+
schema_version: RUN_SCHEMA_VERSION,
|
|
241
|
+
run_id: context.run_id,
|
|
242
|
+
status: "cancelled",
|
|
243
|
+
started_at: now,
|
|
244
|
+
ended_at: now,
|
|
245
|
+
exit_code: null,
|
|
246
|
+
signal: "SIGTERM",
|
|
247
|
+
recovery_command: null,
|
|
248
|
+
bounded_log: "",
|
|
249
|
+
output_characters: 0,
|
|
250
|
+
reason: "cancelled_before_workload_start",
|
|
251
|
+
};
|
|
252
|
+
await writeJson(context.receipt_path, receipt);
|
|
253
|
+
return receipt;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async function retryCancellationHandoff(context, call, { sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) } = {}) {
|
|
257
|
+
let attempts = 0;
|
|
258
|
+
let delay = 1_000;
|
|
259
|
+
for (;;) {
|
|
260
|
+
let result;
|
|
261
|
+
try {
|
|
262
|
+
result = await call("cancel_command_run", {
|
|
263
|
+
run_id: context.run_id,
|
|
264
|
+
run_credential: context.run_credential,
|
|
265
|
+
reason: "cancelled_before_workload_start",
|
|
266
|
+
});
|
|
267
|
+
} catch (error) { result = { ok: false, error: error instanceof Error ? error.message : String(error) }; }
|
|
268
|
+
if (result?.ok && !result.isError && result.data?.status === "cancelled") return result;
|
|
269
|
+
attempts += 1;
|
|
270
|
+
await writeJson(context.context_path, {
|
|
271
|
+
...context,
|
|
272
|
+
cancellation_handoff: { attempts, last_error: result?.error ?? result?.data?.error ?? "cancellation_handoff_rejected", next_retry_at: new Date(Date.now() + delay).toISOString() },
|
|
273
|
+
});
|
|
274
|
+
await sleep(delay);
|
|
275
|
+
delay = Math.min(delay * 2, 60_000);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
218
278
|
|
|
219
279
|
async function writeJson(path, value) {
|
|
220
280
|
await mkdir(dirname(path), { recursive: true });
|
|
@@ -222,7 +282,7 @@ async function writeJson(path, value) {
|
|
|
222
282
|
// file, or a poller) never observes a torn/empty file mid-write. rename(2)
|
|
223
283
|
// is atomic within a directory, so readers see either the old or the new
|
|
224
284
|
// complete JSON, never a partial one.
|
|
225
|
-
const tmp = `${path}.${process.pid}.tmp`;
|
|
285
|
+
const tmp = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
226
286
|
await writeFile(tmp, JSON.stringify(value), { mode: 0o600 });
|
|
227
287
|
await chmod(tmp, 0o600);
|
|
228
288
|
await rename(tmp, path);
|
|
@@ -340,7 +400,12 @@ export async function launchRun(argv, {
|
|
|
340
400
|
}
|
|
341
401
|
const context = contextPath(runId);
|
|
342
402
|
const receipt = receiptPath(runId);
|
|
343
|
-
|
|
403
|
+
// The detached worker has no ambient credential after its launcher exits.
|
|
404
|
+
// Seal the already-resolved owning session in the private (0600) context so
|
|
405
|
+
// every callback is attributable to the same agent. It deliberately stays
|
|
406
|
+
// outside child_env, receipts, and the workload's process environment.
|
|
407
|
+
const knownSession = opts.sessionId ?? managedCredential?.identity?.sessionId ?? managedCredential?.sessionId;
|
|
408
|
+
await writeJson(context, { schema_version: RUN_SCHEMA_VERSION, context_path: context, run_id: runId, run_credential: registration.data.run_credential, receipt_upload: registration.data.receipt_upload, command, cwd, timeout_seconds: opts.timeout, receipt_path: receipt, launched_at: new Date().toISOString(), ...(opts.sessionToken ? { callback_session_token: opts.sessionToken } : {}), ...(knownSession ? { callback_session_id: knownSession } : {}), ...(childEnv && Object.keys(childEnv).length ? { child_env: childEnv } : {}), ...(testRun ? { test_run: testRun } : {}) });
|
|
344
409
|
const [file, args] = workerInvocation(context);
|
|
345
410
|
try {
|
|
346
411
|
const worker = spawnImpl(file, args, { detached: true, stdio: "ignore", windowsHide: true });
|
|
@@ -351,25 +416,85 @@ export async function launchRun(argv, {
|
|
|
351
416
|
await effectiveCall("update_command_run", { run_id: runId, run_credential: registration.data.run_credential, status: "owner_lost", invalidated_reason: `launcher_failed:${error instanceof Error ? error.message : String(error)}` });
|
|
352
417
|
return { exitCode: EXIT.INTERNAL, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "owner_lost", run_id: runId, exit_code: EXIT.INTERNAL } };
|
|
353
418
|
}
|
|
354
|
-
const knownSession = opts.sessionId ?? managedCredential?.identity?.sessionId ?? managedCredential?.sessionId;
|
|
355
419
|
return { exitCode: EXIT.OK, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "launched", run_id: runId,
|
|
356
420
|
...(knownSession ? { session_id: knownSession } : { session_binding: "server_derived" }),
|
|
357
421
|
durable_owner: "detached_worker", receipt_path: receipt, reentry: "update_command_run publishes the terminal callback for this run_id", exit_code: EXIT.OK } };
|
|
358
422
|
}
|
|
359
423
|
|
|
360
|
-
export async function runWorker(contextFile, {
|
|
424
|
+
export async function runWorker(contextFile, {
|
|
425
|
+
spawnImpl = spawn,
|
|
426
|
+
call = callToolJson,
|
|
427
|
+
signalEmitter = process,
|
|
428
|
+
terminate = terminateProcessGroup,
|
|
429
|
+
} = {}) {
|
|
361
430
|
const context = JSON.parse(await readFile(contextFile, "utf8"));
|
|
431
|
+
const callbackCall = callWithContextIdentity(context, call);
|
|
432
|
+
const ownerHeartbeat = startOwnerHeartbeat(context, callbackCall);
|
|
362
433
|
const startedAt = new Date().toISOString();
|
|
363
434
|
let timedOut = false;
|
|
364
435
|
let child;
|
|
436
|
+
let cancellationStateWrite = Promise.resolve();
|
|
437
|
+
let contextWrite = Promise.resolve();
|
|
438
|
+
const persistWorkerContext = (state) => {
|
|
439
|
+
const next = contextWrite.then(() => writeJson(context.context_path, {
|
|
440
|
+
...context,
|
|
441
|
+
worker_pid: process.pid,
|
|
442
|
+
...state,
|
|
443
|
+
}));
|
|
444
|
+
contextWrite = next;
|
|
445
|
+
return next;
|
|
446
|
+
};
|
|
447
|
+
// `cancelRun` creates a durable marker before it acknowledges an unstarted
|
|
448
|
+
// cancellation. The worker owns its own process group, so SIGTERM is also
|
|
449
|
+
// the fence for the tiny interval after the final marker check: if spawn won
|
|
450
|
+
// that interval, the handler terminates the just-created workload instead.
|
|
451
|
+
const cancellationHandler = () => {
|
|
452
|
+
void hasCancellationMarker(context.context_path).then(async (marked) => {
|
|
453
|
+
if (!marked) {
|
|
454
|
+
// Preserve the old default SIGTERM behaviour for a signal that was not
|
|
455
|
+
// issued by `cancelRun` for this durable context.
|
|
456
|
+
signalEmitter.removeListener("SIGTERM", cancellationHandler);
|
|
457
|
+
process.kill(process.pid, "SIGTERM");
|
|
458
|
+
return;
|
|
459
|
+
}
|
|
460
|
+
if (child) {
|
|
461
|
+
terminate(child, "SIGTERM");
|
|
462
|
+
cancellationStateWrite = persistWorkerContext({ workload_pid: child.pid, worker_state: "cancelling_workload" });
|
|
463
|
+
} else {
|
|
464
|
+
cancellationStateWrite = persistWorkerContext({ worker_state: "cancelled_before_workload_start" });
|
|
465
|
+
}
|
|
466
|
+
await cancellationStateWrite;
|
|
467
|
+
}).catch(() => { /* the durable marker remains the source of truth for recovery */ });
|
|
468
|
+
};
|
|
469
|
+
signalEmitter.once("SIGTERM", cancellationHandler);
|
|
365
470
|
try {
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
471
|
+
if (await hasCancellationMarker(context.context_path)) {
|
|
472
|
+
if (!context.cancellation_handoff?.accepted) await retryCancellationHandoff(context, callbackCall);
|
|
473
|
+
if (context.test_run) {
|
|
474
|
+
const endedAt = new Date().toISOString();
|
|
475
|
+
await completeTestRun(context, { status: "cancelled", exitCode: null, signal: "SIGTERM", startedAt: context.launched_at ?? endedAt, endedAt, call: callbackCall });
|
|
476
|
+
}
|
|
477
|
+
return await preWorkloadCancellationReceipt(context);
|
|
478
|
+
}
|
|
479
|
+
// Publish worker ownership before the first workload spawn. Cancellation
|
|
480
|
+
// can now signal this private process group instead of terminalizing the
|
|
481
|
+
// server run while a local worker still has authority to start work.
|
|
482
|
+
await persistWorkerContext({ worker_state: "starting" });
|
|
483
|
+
if (await hasCancellationMarker(context.context_path)) {
|
|
484
|
+
if (!context.cancellation_handoff?.accepted) await retryCancellationHandoff(context, callbackCall);
|
|
485
|
+
if (context.test_run) {
|
|
486
|
+
const endedAt = new Date().toISOString();
|
|
487
|
+
await completeTestRun(context, { status: "cancelled", exitCode: null, signal: "SIGTERM", startedAt: context.launched_at ?? endedAt, endedAt, call: callbackCall });
|
|
488
|
+
}
|
|
489
|
+
return await preWorkloadCancellationReceipt(context);
|
|
490
|
+
}
|
|
491
|
+
try {
|
|
492
|
+
child = spawnImpl(context.command[0], context.command.slice(1), { cwd: context.cwd, detached: true, stdio: ["ignore", "pipe", "pipe"], windowsHide: true, ...(context.child_env ? { env: { ...process.env, ...context.child_env } } : {}) });
|
|
493
|
+
} catch (error) {
|
|
494
|
+
const endedAt = new Date().toISOString();
|
|
495
|
+
const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${error instanceof Error ? error.message : String(error)}` }, callbackCall);
|
|
496
|
+
return receipt;
|
|
497
|
+
}
|
|
373
498
|
// A failed spawn can report through the child `error` event on the next turn.
|
|
374
499
|
// Subscribe before any filesystem I/O so a busy parallel test run (or a real
|
|
375
500
|
// fast failure) cannot lose the sole terminal event.
|
|
@@ -381,32 +506,57 @@ export async function runWorker(contextFile, { spawnImpl = spawn, call = callToo
|
|
|
381
506
|
const capture = boundedCapture();
|
|
382
507
|
child.stdout?.on("data", (chunk) => capture.append(chunk));
|
|
383
508
|
child.stderr?.on("data", (chunk) => capture.append(chunk));
|
|
384
|
-
|
|
509
|
+
await persistWorkerContext({ workload_pid: child.pid, worker_state: "running" });
|
|
385
510
|
// BOT-1510: for a test lane, tail the reporter's NDJSON and flush per-case
|
|
386
511
|
// verdicts while the child runs. The test process never touches the network.
|
|
387
|
-
const lane = context.test_run ? startLaneFlusher(context,
|
|
512
|
+
const lane = context.test_run ? startLaneFlusher(context, callbackCall) : null;
|
|
388
513
|
let killTimer;
|
|
389
514
|
const timer = setTimeout(() => {
|
|
390
515
|
timedOut = true;
|
|
391
|
-
|
|
392
|
-
killTimer = setTimeout(() =>
|
|
516
|
+
terminate(child, "SIGTERM");
|
|
517
|
+
killTimer = setTimeout(() => terminate(child, "SIGKILL"), context.timeout_grace_ms ?? TIMEOUT_GRACE_MS);
|
|
393
518
|
}, context.timeout_seconds * 1000);
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
519
|
+
const result = await completion;
|
|
520
|
+
clearTimeout(timer);
|
|
521
|
+
clearTimeout(killTimer);
|
|
522
|
+
await cancellationStateWrite;
|
|
523
|
+
if (lane) await lane.stop();
|
|
524
|
+
const endedAt = new Date().toISOString();
|
|
525
|
+
if (result.error) {
|
|
526
|
+
const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${result.error.message}`, output: capture.output, outputCharacters: capture.characters }, callbackCall);
|
|
527
|
+
return receipt;
|
|
528
|
+
}
|
|
529
|
+
const status = terminalStatus({ ...result, timedOut });
|
|
530
|
+
const receipt = await finalize(context, { status, startedAt, endedAt, exitCode: result.exitCode, signal: result.signal, output: capture.output, outputCharacters: capture.characters }, callbackCall);
|
|
402
531
|
return receipt;
|
|
532
|
+
} finally {
|
|
533
|
+
if (ownerHeartbeat) await ownerHeartbeat.stop();
|
|
534
|
+
signalEmitter.removeListener("SIGTERM", cancellationHandler);
|
|
403
535
|
}
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
//
|
|
408
|
-
|
|
409
|
-
return
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
function startOwnerHeartbeat(context, call, { intervalMs = 5 * 60_000, setIntervalFn = setInterval } = {}) {
|
|
539
|
+
// A detached worker must never acquire liveness through its ambient default
|
|
540
|
+
// identity. It can only refresh the bearer sealed for its owner.
|
|
541
|
+
if (typeof context.callback_session_token !== "string" || !context.callback_session_token) return null;
|
|
542
|
+
// A token-only launch intentionally omits its public session id. The server
|
|
543
|
+
// derives the exact active session from the sealed bearer, so it must still
|
|
544
|
+
// heartbeat rather than waiting for the eight-hour token window to lapse.
|
|
545
|
+
// Supplying an id when one was sealed retains the same explicit scope fence.
|
|
546
|
+
const heartbeatArgs = typeof context.callback_session_id === "string" && context.callback_session_id
|
|
547
|
+
? { session_id: context.callback_session_id } : {};
|
|
548
|
+
let stopped = false;
|
|
549
|
+
let chain = Promise.resolve();
|
|
550
|
+
const tick = () => {
|
|
551
|
+
chain = chain.then(async () => {
|
|
552
|
+
if (stopped) return;
|
|
553
|
+
try { await call("heartbeat", heartbeatArgs); } catch { /* terminal delivery retains its own retry path */ }
|
|
554
|
+
});
|
|
555
|
+
};
|
|
556
|
+
tick();
|
|
557
|
+
const timer = setIntervalFn(tick, intervalMs);
|
|
558
|
+
timer.unref?.();
|
|
559
|
+
return { async stop() { stopped = true; clearInterval(timer); await chain; } };
|
|
410
560
|
}
|
|
411
561
|
|
|
412
562
|
// Periodic lane flusher: reduce the NDJSON events file to the current case state
|
|
@@ -496,6 +646,30 @@ function terminateProcessGroup(child, signal) {
|
|
|
496
646
|
try { child.kill?.(signal); } catch { /* already exited */ }
|
|
497
647
|
}
|
|
498
648
|
|
|
649
|
+
function workerProcessGroupAlive(pid) {
|
|
650
|
+
if (!Number.isInteger(pid) || pid <= 0 || process.platform === "win32") return false;
|
|
651
|
+
try { process.kill(-pid, 0); return true; } catch (error) { return error?.code !== "ESRCH"; }
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
async function waitForWorkerCancellation(contextFile, { sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), attempts = 25 } = {}) {
|
|
655
|
+
let latest = null;
|
|
656
|
+
for (let attempt = 0; attempt < attempts; attempt++) {
|
|
657
|
+
try { latest = JSON.parse(await readFile(contextFile, "utf8")); } catch { /* the context is atomically replaced */ }
|
|
658
|
+
if (Number.isInteger(latest?.workload_pid) || latest?.worker_state === "cancelled_before_workload_start") return latest;
|
|
659
|
+
await sleep(20);
|
|
660
|
+
}
|
|
661
|
+
return latest;
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
async function sealedTerminalStatus(context) {
|
|
665
|
+
if (typeof context.terminal_update?.status === "string") return context.terminal_update.status;
|
|
666
|
+
if (!context.receipt_path) return null;
|
|
667
|
+
try {
|
|
668
|
+
const receipt = JSON.parse(await readFile(context.receipt_path, "utf8"));
|
|
669
|
+
return receipt?.run_id === context.run_id && typeof receipt.status === "string" ? receipt.status : null;
|
|
670
|
+
} catch { return null; }
|
|
671
|
+
}
|
|
672
|
+
|
|
499
673
|
function forwardTerminationSignals(child, signalEmitter) {
|
|
500
674
|
let signal = null;
|
|
501
675
|
let escalation = null;
|
|
@@ -525,15 +699,87 @@ function waitForChild(child) {
|
|
|
525
699
|
});
|
|
526
700
|
}
|
|
527
701
|
|
|
528
|
-
export async function cancelRun(runId, {
|
|
702
|
+
export async function cancelRun(runId, {
|
|
703
|
+
call = callToolJson,
|
|
704
|
+
waitForCancellation = waitForWorkerCancellation,
|
|
705
|
+
killWorker = (pid) => process.kill(-pid, "SIGTERM"),
|
|
706
|
+
workerAlive = workerProcessGroupAlive,
|
|
707
|
+
} = {}) {
|
|
529
708
|
const context = JSON.parse(await readFile(contextPath(runId), "utf8"));
|
|
709
|
+
const callbackCall = callWithContextIdentity(context, call);
|
|
710
|
+
const sealedStatus = await sealedTerminalStatus(context);
|
|
711
|
+
if (sealedStatus) {
|
|
712
|
+
return { exitCode: EXIT.OK, receipt: {
|
|
713
|
+
schema_version: RUN_SCHEMA_VERSION,
|
|
714
|
+
outcome: "terminal_delivery_pending",
|
|
715
|
+
run_id: runId,
|
|
716
|
+
terminal_status: sealedStatus,
|
|
717
|
+
exit_code: EXIT.OK,
|
|
718
|
+
} };
|
|
719
|
+
}
|
|
720
|
+
const requestCancellation = async () => {
|
|
721
|
+
// No worker remains to publish the linked test-run terminal event. Complete
|
|
722
|
+
// it before revoking the one-run credential so an armed `test-run:` wait
|
|
723
|
+
// cannot be stranded active after the command cancellation is accepted.
|
|
724
|
+
if (context.test_run) {
|
|
725
|
+
const endedAt = new Date().toISOString();
|
|
726
|
+
await completeTestRun(context, {
|
|
727
|
+
status: "cancelled",
|
|
728
|
+
exitCode: null,
|
|
729
|
+
signal: "SIGTERM",
|
|
730
|
+
startedAt: context.launched_at ?? endedAt,
|
|
731
|
+
endedAt,
|
|
732
|
+
call: callbackCall,
|
|
733
|
+
});
|
|
734
|
+
}
|
|
735
|
+
let cancellation;
|
|
736
|
+
try {
|
|
737
|
+
cancellation = await callbackCall("cancel_command_run", {
|
|
738
|
+
run_id: context.run_id,
|
|
739
|
+
run_credential: context.run_credential,
|
|
740
|
+
reason: "cancelled_before_workload_start",
|
|
741
|
+
});
|
|
742
|
+
} catch (error) {
|
|
743
|
+
cancellation = { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
744
|
+
}
|
|
745
|
+
if (cancellation?.ok && !cancellation.isError && cancellation.data?.status === "cancelled") {
|
|
746
|
+
await writeJson(context.context_path ?? contextPath(runId), { ...context, cancellation_handoff: { accepted: true, accepted_at: new Date().toISOString() } });
|
|
747
|
+
return { exitCode: EXIT.OK, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "cancellation_requested", run_id: runId, exit_code: EXIT.OK } };
|
|
748
|
+
}
|
|
749
|
+
return { exitCode: EXIT.BACKEND, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "rejected", run_id: runId, error: cancellation?.error ?? cancellation?.data?.error ?? "cancellation_handoff_failed", exit_code: EXIT.BACKEND } };
|
|
750
|
+
};
|
|
530
751
|
if (!Number.isInteger(context.workload_pid) || context.workload_pid <= 0) {
|
|
531
|
-
|
|
752
|
+
// The marker is durable and checked twice by the worker. It must exist
|
|
753
|
+
// before the terminal API transition so a launcher-returned worker cannot
|
|
754
|
+
// begin executing after cancellation has been acknowledged.
|
|
755
|
+
const contextFile = context.context_path ?? contextPath(runId);
|
|
756
|
+
try {
|
|
757
|
+
await markCancellation(contextFile, context.run_id);
|
|
758
|
+
} catch (error) {
|
|
759
|
+
return { exitCode: EXIT.INTERNAL, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "rejected", run_id: runId, error: `cancellation_fence_failed:${error instanceof Error ? error.message : String(error)}`, exit_code: EXIT.INTERNAL } };
|
|
760
|
+
}
|
|
761
|
+
if (Number.isInteger(context.worker_pid) && context.worker_pid > 0) {
|
|
762
|
+
// A live worker may have claimed the interval after launcher handoff.
|
|
763
|
+
// Fence it first; its SIGTERM handler either records a no-workload
|
|
764
|
+
// acknowledgement (safe for cancel_command_run) or terminates the
|
|
765
|
+
// workload and keeps ownership of the final update_command_run callback.
|
|
766
|
+
try { killWorker(context.worker_pid); } catch (error) {
|
|
767
|
+
if (error?.code !== "ESRCH") return { exitCode: EXIT.INTERNAL, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "rejected", run_id: runId, error: `cancellation_worker_fence_failed:${error instanceof Error ? error.message : String(error)}`, exit_code: EXIT.INTERNAL } };
|
|
768
|
+
}
|
|
769
|
+
const fenced = await waitForCancellation(contextFile);
|
|
770
|
+
if (Number.isInteger(fenced?.workload_pid) && fenced.workload_pid > 0) {
|
|
771
|
+
terminateProcessGroup({ pid: fenced.workload_pid }, "SIGTERM");
|
|
772
|
+
return { exitCode: EXIT.OK, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "cancellation_requested", run_id: runId, exit_code: EXIT.OK } };
|
|
773
|
+
}
|
|
774
|
+
if (fenced?.worker_state === "cancelled_before_workload_start" || !workerAlive(context.worker_pid)) return requestCancellation();
|
|
775
|
+
return { exitCode: EXIT.BACKEND, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "rejected", run_id: runId, error: "cancellation_worker_fence_unconfirmed", exit_code: EXIT.BACKEND } };
|
|
776
|
+
}
|
|
777
|
+
return requestCancellation();
|
|
532
778
|
}
|
|
533
779
|
try {
|
|
534
780
|
process.kill(-context.workload_pid, "SIGTERM");
|
|
535
781
|
} catch (error) {
|
|
536
|
-
await
|
|
782
|
+
await callbackCall("update_command_run", { run_id: context.run_id, run_credential: context.run_credential, status: "owner_lost", invalidated_reason: `cancel_owner_unreachable:${error instanceof Error ? error.message : String(error)}` });
|
|
537
783
|
return { exitCode: EXIT.INTERNAL, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "owner_lost", run_id: runId, exit_code: EXIT.INTERNAL } };
|
|
538
784
|
}
|
|
539
785
|
return { exitCode: EXIT.OK, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "cancellation_requested", run_id: runId, exit_code: EXIT.OK } };
|
|
@@ -548,11 +794,28 @@ async function finalize(context, state, call) {
|
|
|
548
794
|
const sha256 = createHash("sha256").update(await readFile(context.receipt_path)).digest("hex");
|
|
549
795
|
const apiStatus = ({ success: "succeeded", failure: "failed", cancelled: "cancelled", timed_out: "timed_out", owner_lost: "owner_lost" })[state.status];
|
|
550
796
|
const update = { run_id: context.run_id, run_credential: context.run_credential, status: apiStatus, exit_code: state.exitCode ?? undefined, invalidated_reason: state.reason ?? (state.status === "timed_out" ? "timeout" : state.status === "owner_lost" ? "owner_lost" : undefined), receipt_path: context.receipt_upload, receipt_content_type: "application/json", receipt_byte_size: bytes.size, receipt_sha256: sha256, output_characters: outputCharacters, truncated: outputCharacters > MAX_CAPTURE_BYTES };
|
|
797
|
+
// A locally durable receipt is enough to publish the linked test run. Do it
|
|
798
|
+
// before the command callback's intentionally unbounded retry: otherwise an
|
|
799
|
+
// auth outage (for example agent_identity_required) strands a 0-case run in
|
|
800
|
+
// active state forever even though its command has already failed locally.
|
|
801
|
+
if (context.test_run) await completeTestRun(context, { status: state.status, exitCode: state.exitCode, signal: state.signal, startedAt: state.startedAt, endedAt: state.endedAt, call });
|
|
551
802
|
await retryTerminalUpdate(context, update, call);
|
|
552
803
|
try { await unlink(context.context_path); } catch { /* receipt is authoritative; context cleanup is best effort */ }
|
|
553
804
|
return receipt;
|
|
554
805
|
}
|
|
555
806
|
|
|
807
|
+
function callWithContextIdentity(context, call) {
|
|
808
|
+
const token = typeof context.callback_session_token === "string" && SESSION_TOKEN_RE.test(context.callback_session_token)
|
|
809
|
+
? context.callback_session_token
|
|
810
|
+
: null;
|
|
811
|
+
// Old contexts cannot safely fall back to the machine-level owner key. Ask
|
|
812
|
+
// the API layer to require a session instead; the retained receipt/context
|
|
813
|
+
// makes recovery possible without attributing a callback to the wrong agent.
|
|
814
|
+
return (name, args) => call(name, args, token
|
|
815
|
+
? { auth: { "x-agent-api-key": token } }
|
|
816
|
+
: { requireSessionToken: true });
|
|
817
|
+
}
|
|
818
|
+
|
|
556
819
|
export async function retryTerminalUpdate(context, update, call, { sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) } = {}) {
|
|
557
820
|
let attempts = 0;
|
|
558
821
|
let delay = 1_000;
|
package/src/wait-core.mjs
CHANGED
|
@@ -676,9 +676,12 @@ export function parseSseFrames(buffer) {
|
|
|
676
676
|
}
|
|
677
677
|
|
|
678
678
|
// ---------------------------------------------------------------------------
|
|
679
|
-
// Cursor —
|
|
679
|
+
// Cursor — replay uses JavaScript Numbers at the relay boundary, so only exact
|
|
680
|
+
// non-negative safe integers can be replayed without changing the requested floor.
|
|
680
681
|
// ---------------------------------------------------------------------------
|
|
681
682
|
|
|
683
|
+
const MAX_SAFE_SIGNAL_SEQUENCE = BigInt(Number.MAX_SAFE_INTEGER);
|
|
684
|
+
|
|
682
685
|
/**
|
|
683
686
|
* @param {string|null|undefined} raw
|
|
684
687
|
* @returns {{kind: "absent"} | {kind: "ok", value: string} | {kind: "invalid", raw: string}}
|
|
@@ -688,7 +691,9 @@ export function normalizeSince(raw) {
|
|
|
688
691
|
const t = String(raw).trim();
|
|
689
692
|
if (t === "") return { kind: "absent" };
|
|
690
693
|
if (!/^\d+$/.test(t)) return { kind: "invalid", raw };
|
|
691
|
-
|
|
694
|
+
const value = BigInt(t);
|
|
695
|
+
if (value > MAX_SAFE_SIGNAL_SEQUENCE) return { kind: "invalid", raw };
|
|
696
|
+
return { kind: "ok", value: String(value) };
|
|
692
697
|
}
|
|
693
698
|
|
|
694
699
|
// ---------------------------------------------------------------------------
|
package/src/wait.mjs
CHANGED
|
@@ -153,7 +153,7 @@ CONDITIONS (TYPE:key=val,key=val — repeat for several; --any wakes on the fir
|
|
|
153
153
|
OPTIONS
|
|
154
154
|
--any wake on the first matching condition (default; only mode in v1)
|
|
155
155
|
--timeout <seconds> max wait (default 7200; hard cap 28800). Timeout is exit 2, not an error.
|
|
156
|
-
--since <seq> resume from a prior receipt's next_cursor (replay what you missed)
|
|
156
|
+
--since <seq> resume from a prior receipt's next_cursor (0..9007199254740991; replay what you missed)
|
|
157
157
|
--receipt-max-bytes <n> cap the receipt (512..65536; default 10240; payloads truncate to pointers)
|
|
158
158
|
--heartbeat keep this agent session alive while waiting (so it is not reaped)
|
|
159
159
|
--url <base> relay base URL (default $BOTBUDDY_RELAY_URL or https://api.bot-buddy.ai/functions/v1)
|
|
@@ -216,6 +216,7 @@ function parseArgv(argv) {
|
|
|
216
216
|
mode: "any",
|
|
217
217
|
timeout: 7200,
|
|
218
218
|
since: null,
|
|
219
|
+
sinceProvided: false,
|
|
219
220
|
receiptMaxBytes: 10240,
|
|
220
221
|
heartbeat: false,
|
|
221
222
|
url: process.env.BOTBUDDY_RELAY_URL || "https://api.bot-buddy.ai/functions/v1",
|
|
@@ -248,7 +249,7 @@ function parseArgv(argv) {
|
|
|
248
249
|
else if (a === "--heartbeat") opts.heartbeat = true;
|
|
249
250
|
else if (a === "--timeout") opts.timeout = Number(optionValue());
|
|
250
251
|
else if (a === "--deadline-at") opts.deadlineAt = optionValue(); // internal recovery argv; never shown as public API
|
|
251
|
-
else if (a === "--since") opts.since = optionValue();
|
|
252
|
+
else if (a === "--since") { opts.sinceProvided = true; opts.since = optionValue(); }
|
|
252
253
|
else if (a === "--receipt-max-bytes") opts.receiptMaxBytes = Number(optionValue());
|
|
253
254
|
else if (a === "--url") opts.url = optionValue();
|
|
254
255
|
// BOT-1574: --token and --profile are RETIRED. A wait authenticates from
|
|
@@ -1572,6 +1573,19 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
|
|
|
1572
1573
|
process.exit(EXIT.INVALID);
|
|
1573
1574
|
}
|
|
1574
1575
|
|
|
1576
|
+
// BOT-2453: an explicit replay cursor is a request to replay, never permission
|
|
1577
|
+
// to silently fall back to live-only. Validate it before creating a recoverable
|
|
1578
|
+
// checkpoint, resolving credentials, registering a wait, or opening SSE.
|
|
1579
|
+
if (opts.sinceProvided) {
|
|
1580
|
+
const since = normalizeSince(opts.since);
|
|
1581
|
+
if (since.kind !== "ok") {
|
|
1582
|
+
process.stderr.write("bb-wait: --since must be a non-negative signal sequence <= 9007199254740991\n");
|
|
1583
|
+
emitReceipt({ schema_version: 1, outcome: "error", error: "invalid_since", cursor: opts.since });
|
|
1584
|
+
process.exit(EXIT.INVALID);
|
|
1585
|
+
}
|
|
1586
|
+
opts.since = since.value;
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1575
1589
|
const { conditions, errors } = parseConditions(opts.conditions);
|
|
1576
1590
|
if (errors.length > 0) {
|
|
1577
1591
|
for (const e of errors) process.stderr.write(`bb-wait: invalid condition '${e.spec}': ${e.message}\n`);
|