@botbuddy/cli 1.59.8 → 1.59.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/botbuddy.mjs CHANGED
@@ -1,7 +1,16 @@
1
1
  #!/usr/bin/env node
2
2
  import { run } from "../src/commands.mjs";
3
+ import { ensureRunDelivery } from "../src/run-delivery.mjs";
3
4
 
4
- run(process.argv.slice(2)).then((code) => {
5
+ const argv = process.argv.slice(2);
6
+
7
+ // BOT-2600: any `bb` invocation starts the single per-machine delivery process
8
+ // when finished `bb run` commands still have an unsent terminal update. A stat
9
+ // and two small reads when nothing changed; it never fails the command.
10
+ const deliveryProcessArgs = argv[0] === "run" && (argv[1] === "--worker" || argv[1] === "--deliver");
11
+ const delivery = deliveryProcessArgs ? Promise.resolve() : ensureRunDelivery().catch(() => {});
12
+
13
+ delivery.then(() => run(argv)).then((code) => {
5
14
  if (typeof code === "number") process.exitCode = code;
6
15
  }).catch((err) => {
7
16
  console.error(`\x1b[31m✗\x1b[0m ${err?.message || err}`);
@@ -1,7 +1,7 @@
1
1
  {
2
- "release_version": "1.59.8",
3
- "source_commit": "d72599a3ac43155a4d66e30b2d73e5f2a46cf956",
4
- "source_sha256": "5ccca5bc66c82f02e856a25c0cedf9056f69b3e72b95a668dc62a3dff4a0f72f",
2
+ "release_version": "1.59.10",
3
+ "source_commit": "0a69b9e8cf14b345f8cc9e23e8ed68a8f240147c",
4
+ "source_sha256": "126242299e994c1e88bf377dbc0046672b5cdfe3c1454657f06abb64275b0301",
5
5
  "target": "darwin-arm64",
6
- "sha256": "24810c0fd70f40e77d860b0c247f1a7112a6a2a09b8754c83c7ca50b023e38c1"
6
+ "sha256": "8c41e97c983e235c710b8624849998f0e231cdab27d7147499e5ba6e4e7e5abb"
7
7
  }
@@ -1,7 +1,7 @@
1
1
  {
2
- "release_version": "1.59.8",
3
- "source_commit": "d72599a3ac43155a4d66e30b2d73e5f2a46cf956",
4
- "source_sha256": "5ccca5bc66c82f02e856a25c0cedf9056f69b3e72b95a668dc62a3dff4a0f72f",
2
+ "release_version": "1.59.10",
3
+ "source_commit": "0a69b9e8cf14b345f8cc9e23e8ed68a8f240147c",
4
+ "source_sha256": "126242299e994c1e88bf377dbc0046672b5cdfe3c1454657f06abb64275b0301",
5
5
  "target": "linux-x64",
6
- "sha256": "61b9f79104394a9fe11e43b25cf3aa3c84dc1538f44c4439b7ed0555b8e8b858"
6
+ "sha256": "92a173432e8a40e2ed5d91cd2a93a5fd0cfcd23c63b56ecec42d53127e272b7a"
7
7
  }
@@ -1,21 +1,21 @@
1
1
  {
2
2
  "schema_version": 1,
3
- "release_version": "1.59.8",
4
- "source_commit": "d72599a3ac43155a4d66e30b2d73e5f2a46cf956",
5
- "source_sha256": "5ccca5bc66c82f02e856a25c0cedf9056f69b3e72b95a668dc62a3dff4a0f72f",
3
+ "release_version": "1.59.10",
4
+ "source_commit": "0a69b9e8cf14b345f8cc9e23e8ed68a8f240147c",
5
+ "source_sha256": "126242299e994c1e88bf377dbc0046672b5cdfe3c1454657f06abb64275b0301",
6
6
  "targets": {
7
7
  "darwin-arm64": {
8
8
  "goos": "darwin",
9
9
  "goarch": "arm64",
10
10
  "binary": "daemon/bin/darwin-arm64/botbuddy-daemon",
11
- "sha256": "24810c0fd70f40e77d860b0c247f1a7112a6a2a09b8754c83c7ca50b023e38c1",
11
+ "sha256": "8c41e97c983e235c710b8624849998f0e231cdab27d7147499e5ba6e4e7e5abb",
12
12
  "mode": "0755"
13
13
  },
14
14
  "linux-x64": {
15
15
  "goos": "linux",
16
16
  "goarch": "amd64",
17
17
  "binary": "daemon/bin/linux-x64/botbuddy-daemon",
18
- "sha256": "61b9f79104394a9fe11e43b25cf3aa3c84dc1538f44c4439b7ed0555b8e8b858",
18
+ "sha256": "92a173432e8a40e2ed5d91cd2a93a5fd0cfcd23c63b56ecec42d53127e272b7a",
19
19
  "mode": "0755"
20
20
  }
21
21
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@botbuddy/cli",
3
- "version": "1.59.8",
3
+ "version": "1.59.10",
4
4
  "description": "BotBuddy \u2014 Swarm coordination CLI for multi-agent workflows",
5
5
  "type": "module",
6
6
  "bin": {
package/src/api.mjs CHANGED
@@ -385,9 +385,9 @@ export const SESSION_TOKEN_DENIED_TOOLS = new Set([
385
385
  // call to a specific tenant by appending ?tenant=<slug> to the endpoint (the
386
386
  // same mechanism as the documented BOTBUDDY_SERVER_URL override), letting the
387
387
  // server resolve/confirm exactly that tenant.
388
- function toolEndpoint(tenant) {
389
- if (!tenant) return SERVER_URL;
390
- const url = new URL(SERVER_URL);
388
+ function toolEndpoint(tenant, base = SERVER_URL) {
389
+ if (!tenant) return base;
390
+ const url = new URL(base);
391
391
  url.searchParams.set("tenant", tenant);
392
392
  return url.toString();
393
393
  }
@@ -457,6 +457,9 @@ export async function registerContainerHost(body, {
457
457
  export async function callToolJson(toolName, args = {}, {
458
458
  fetchImpl = fetch, signal, auth: pinnedAuth = null, tenant = null, resolvePin = resolveDefaultTenantPin,
459
459
  requireSessionToken = false, resolveAuth = resolveCallAuth,
460
+ // BOT-2600: a delivery for a sealed run goes to the endpoint the run was
461
+ // registered with, not this process's ambient one.
462
+ serverUrl = SERVER_URL,
460
463
  } = {}) {
461
464
  let auth = pinnedAuth;
462
465
  let sessionTokenSelected = false;
@@ -481,7 +484,7 @@ export async function callToolJson(toolName, args = {}, {
481
484
  const body = { jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: toolName, arguments: args } };
482
485
  let res;
483
486
  try {
484
- res = await fetchImpl(toolEndpoint(pin), {
487
+ res = await fetchImpl(toolEndpoint(pin, serverUrl), {
485
488
  method: "POST",
486
489
  headers: { "Content-Type": "application/json", ...auth },
487
490
  body: JSON.stringify(body),
package/src/config.mjs CHANGED
@@ -9,8 +9,8 @@ const CONFIG_FILE = join(CONFIG_DIR, "config.json");
9
9
  // BOT-876: overridable so the CLI can be pointed at a local Supabase stack
10
10
  // (`BOTBUDDY_SERVER_URL=http://127.0.0.1:56321/functions/v1/mcp-server`).
11
11
  // Production is the default; nothing changes for normal use.
12
- export const SERVER_URL = process.env.BOTBUDDY_SERVER_URL
13
- || "https://api.bot-buddy.ai/functions/v1/mcp-server";
12
+ export const DEFAULT_SERVER_URL = "https://api.bot-buddy.ai/functions/v1/mcp-server";
13
+ export const SERVER_URL = process.env.BOTBUDDY_SERVER_URL || DEFAULT_SERVER_URL;
14
14
 
15
15
  let config = {};
16
16
 
@@ -0,0 +1,13 @@
1
+ {
2
+ "source": "production MCP (api.bot-buddy.ai) via cli/src/api.mjs callToolJson, CLI worktree of BOT-2600",
3
+ "captured_at": "2026-10-20T19:15:53.070Z",
4
+ "note": "update_command_run for a random run id. Both requests are refused before any write; no credential value is recorded.",
5
+ "invalid_run_credential": {
6
+ "request": "x-agent-api-key: the caller's live session token; run_credential: 64 random hex",
7
+ "result": { "ok": true, "data": { "error": "run_credential_invalid" }, "isError": true }
8
+ },
9
+ "unknown_session_token": {
10
+ "request": "x-agent-api-key: bb_sess_ + 64 zeros",
11
+ "result": { "ok": false, "auth": true, "status": 401, "error": "unauthorized", "error_code": "session_token_revoked" }
12
+ }
13
+ }
@@ -0,0 +1,496 @@
1
+ // BOT-2600 — bounded terminal delivery for durable `bb run` commands.
2
+ //
3
+ // A detached `bb run` worker owns its workload and writes exactly one receipt.
4
+ // Once that receipt is durable, publishing it (update_command_run) is only a
5
+ // delivery problem, and it must not keep a resident Node process per run alive:
6
+ // on 2026-10-20 one Mac had 142 finished workers (6 GB) retrying rejections
7
+ // that could never succeed. So:
8
+ //
9
+ // - the worker retries transient failures within a bounded monotonic budget
10
+ // and parks permanent rejections immediately, recording
11
+ // `terminal_update.parked` in its private context, then exits;
12
+ // - a single per-machine delivery process (`bb run --deliver`, guarded by a
13
+ // PID lock under ~/.botbuddy/) retries what is still retryable and exits
14
+ // when nothing retryable remains. Any `bb` invocation starts it on demand;
15
+ // - workers left by older CLI versions (which retry forever) are adopted:
16
+ // their delivery is taken over and the worker is stopped only when it is
17
+ // provably finished (exact command line, same user, terminal receipt, no
18
+ // child processes, already in its terminal retry loop).
19
+ //
20
+ // The receipt and context files stay authoritative; nothing is deleted except a
21
+ // context whose delivery was accepted (exactly what the worker always did).
22
+ // The identity model is unchanged (BOT-2052): a delivery is only ever sent with
23
+ // the session token sealed for the run's owner, never an ambient credential.
24
+
25
+ import { createHash, randomUUID } from "node:crypto";
26
+ import { chmod, link, mkdir, readdir, readFile, rename, stat, unlink, writeFile } from "node:fs/promises";
27
+ import { execFile, spawn } from "node:child_process";
28
+ import { dirname, join } from "node:path";
29
+ import { homedir } from "node:os";
30
+ import { fileURLToPath } from "node:url";
31
+ import { performance } from "node:perf_hooks";
32
+ import { AGENT_KEY_RE } from "./agent-key.mjs";
33
+ import { DEFAULT_SERVER_URL } from "./config.mjs";
34
+
35
+ /** The worker's own retry budget for transient failures (AC-1: ≤ 10 minutes). */
36
+ export const TERMINAL_RETRY_BUDGET_MS = 10 * 60_000;
37
+ /** One update_command_run attempt never waits longer than this. */
38
+ export const DELIVERY_CALL_TIMEOUT_MS = 30_000;
39
+ /** Transient attempts the delivery process makes before parking for good. */
40
+ export const DELIVERY_MAX_ATTEMPTS = 50;
41
+ /** The delivery process exits after this long even with retryable work left. */
42
+ export const DELIVERY_LIFETIME_MS = 30 * 60_000;
43
+ /** A non-forced `bb` invocation re-scans at most this often. */
44
+ export const DELIVERY_RESCAN_COOLDOWN_MS = 60_000;
45
+ const DELIVERY_FIRST_BACKOFF_MS = 30_000;
46
+ const DELIVERY_MAX_BACKOFF_MS = 5 * 60_000;
47
+ const STOP_GRACE_MS = 3_000;
48
+
49
+ export const TERMINAL_RECEIPT_STATUSES = new Set(["success", "failure", "cancelled", "timed_out", "owner_lost"]);
50
+ const API_STATUS = { success: "succeeded", failure: "failed", cancelled: "cancelled", timed_out: "timed_out", owner_lost: "owner_lost" };
51
+ const MAX_CAPTURE_BYTES = 8_192;
52
+ // The update_command_run arguments a persisted terminal_update may carry.
53
+ const UPDATE_FIELDS = ["run_id", "run_credential", "status", "exit_code", "invalidated_reason", "receipt_path", "receipt_content_type", "receipt_byte_size", "receipt_sha256", "output_characters", "truncated"];
54
+
55
+ // Never retried: decided locally without a request reaching the server (no
56
+ // usable session token), or a definitive server state (a different terminal
57
+ // status is already recorded; the payload itself is refused).
58
+ const PERMANENT_ERRORS = new Set([
59
+ "session_token_required", "session_token_malformed", "not_authenticated", "token_expired",
60
+ "command_terminal_immutable", "artifact_required",
61
+ ]);
62
+ // Trust boundary for server rejections. The server can answer with an auth or
63
+ // credential rejection when its own lookup failed transiently (it discards the
64
+ // read error: BOT-2611), so none of these is proof that a retry cannot
65
+ // succeed. The worker parks every one at once and exits (it never spends its
66
+ // retry budget on them), and the delivery process gives each a few spaced
67
+ // attempts before parking it for good, so a recovered server recovers the
68
+ // terminal result while a dead credential costs at most a handful of calls.
69
+ const AMBIGUOUS_ERRORS = new Set([
70
+ "agent_identity_required", "authentication_required", "unauthorized", "run_credential_invalid",
71
+ "session_token_revoked", "credential_unknown",
72
+ ]);
73
+ /** Delivery-process attempts for an ambiguous rejection before parking for good. */
74
+ export const DELIVERY_AMBIGUOUS_MAX_ATTEMPTS = 5;
75
+ const LOCAL_ENVIRONMENT_RE =/uv_os_get_passwd|\bENOENT\b|\bEACCES\b|\bEPERM\b/;
76
+
77
+ export function commandRunsDir(home = homedir()) { return join(home, ".botbuddy", "command-runs"); }
78
+ export function deliveryLockPath(home = homedir()) { return join(home, ".botbuddy", "run-delivery.lock"); }
79
+ export function deliveryStatePath(home = homedir()) { return join(home, ".botbuddy", "run-delivery.json"); }
80
+
81
+ function errorText(result) {
82
+ const value = result?.data?.error ?? result?.error ?? result?.code ?? null;
83
+ return typeof value === "string" && value ? value : "terminal_update_rejected";
84
+ }
85
+
86
+ export function terminalUpdateAccepted(result) {
87
+ return Boolean(result?.ok && !result.isError && result.data?.error == null && result.data?.accepted !== false);
88
+ }
89
+
90
+ /**
91
+ * Classify a failed update_command_run attempt.
92
+ * `parkNow` marks a retryable failure the worker must not spend its own
93
+ * budget on: it parks at once and only the delivery process retries it.
94
+ * @returns {{ reason: string, retryable: boolean, kind: string, parkNow?: boolean }}
95
+ */
96
+ export function classifyTerminalUpdateFailure(result, thrown = null) {
97
+ if (thrown) {
98
+ const message = thrown instanceof Error ? thrown.message : String(thrown);
99
+ if (thrown?.code === "ERR_SYSTEM_ERROR" || LOCAL_ENVIRONMENT_RE.test(message)) {
100
+ return { reason: message, retryable: false, kind: "local_environment" };
101
+ }
102
+ return { reason: message, retryable: true, kind: "transient" };
103
+ }
104
+ const reason = errorText(result);
105
+ const codes = [result?.data?.error, result?.error, result?.code, result?.error_code].filter((v) => typeof v === "string");
106
+ if (codes.some((code) => PERMANENT_ERRORS.has(code))) return { reason, retryable: false, kind: "permanent_rejection" };
107
+ if (result?.auth === true || codes.some((code) => AMBIGUOUS_ERRORS.has(code))) return { reason, retryable: true, kind: "ambiguous_rejection", parkNow: true };
108
+ if (codes.some((code) => code.startsWith("usage:"))) return { reason, retryable: false, kind: "local_environment" };
109
+ // Transport failures, timeouts, 5xx/429 and anything unrecognised stay
110
+ // retryable; the budget (worker) or attempt cap (delivery process) bounds them.
111
+ return { reason, retryable: true, kind: "transient" };
112
+ }
113
+
114
+ /** The sealed owner's identity, never an ambient or machine-level credential. */
115
+ export function callWithContextIdentity(context, call) {
116
+ const token = sealedToken(context);
117
+ // Old contexts cannot safely fall back to the machine-level owner key. Ask
118
+ // the API layer to require a session instead; the retained receipt/context
119
+ // makes recovery possible without attributing a callback to the wrong agent.
120
+ return (name, args, extra = {}) => call(name, args, token
121
+ ? { ...extra, auth: { "x-agent-api-key": token } }
122
+ : { ...extra, requireSessionToken: true });
123
+ }
124
+
125
+ function sealedToken(context) {
126
+ return typeof context?.callback_session_token === "string" && AGENT_KEY_RE.test(context.callback_session_token)
127
+ ? context.callback_session_token : null;
128
+ }
129
+
130
+ export async function writeJsonAtomic(path, value) {
131
+ await mkdir(dirname(path), { recursive: true });
132
+ // Write-then-rename so a concurrent reader never observes a torn file.
133
+ const tmp = `${path}.${process.pid}.${randomUUID()}.tmp`;
134
+ await writeFile(tmp, JSON.stringify(value), { mode: 0o600 });
135
+ await chmod(tmp, 0o600);
136
+ await rename(tmp, path);
137
+ }
138
+
139
+ async function readJson(path) {
140
+ try { return JSON.parse(await readFile(path, "utf8")); } catch (error) {
141
+ if (error?.code === "ENOENT") return null;
142
+ if (error instanceof SyntaxError) return undefined;
143
+ throw error;
144
+ }
145
+ }
146
+
147
+ function withTimeout(extra = {}) {
148
+ return { ...extra, signal: AbortSignal.timeout(DELIVERY_CALL_TIMEOUT_MS) };
149
+ }
150
+
151
+ /**
152
+ * The endpoint a run was registered with. A sealed token is only ever sent
153
+ * back there, whatever endpoint the delivery process itself was started with.
154
+ * Contexts written before BOT-2600 did not record it; the CLI's default
155
+ * (production) endpoint is the only one a context without an override could
156
+ * have been registered with by default, so it is used for those.
157
+ */
158
+ export function contextServerUrl(context) {
159
+ return typeof context?.server_url === "string" && /^https?:\/\//.test(context.server_url) ? context.server_url : DEFAULT_SERVER_URL;
160
+ }
161
+
162
+ /** One update_command_run attempt with the sealed identity and a bounded wait. */
163
+ export async function attemptTerminalUpdate(context, update, call) {
164
+ const identified = callWithContextIdentity(context, call);
165
+ try {
166
+ const result = await identified("update_command_run", update, withTimeout({ serverUrl: contextServerUrl(context) }));
167
+ if (terminalUpdateAccepted(result)) return { accepted: true, result };
168
+ return { accepted: false, result, failure: classifyTerminalUpdateFailure(result) };
169
+ } catch (error) {
170
+ return { accepted: false, result: null, failure: classifyTerminalUpdateFailure(null, error) };
171
+ }
172
+ }
173
+
174
+ export function parkedRecord(failure, by) {
175
+ return { reason: failure.reason, retryable: failure.retryable, kind: failure.kind, by, parked_at: new Date().toISOString() };
176
+ }
177
+
178
+ /** Rebuild the terminal update from a durable receipt (a worker that crashed first). */
179
+ export async function terminalUpdateFromReceipt(context, receipt) {
180
+ const bytes = await readFile(context.receipt_path);
181
+ const outputCharacters = Number.isInteger(receipt.output_characters) ? receipt.output_characters : 0;
182
+ const reason = receipt.reason ?? (receipt.status === "timed_out" ? "timeout" : receipt.status === "owner_lost" ? "owner_lost" : undefined);
183
+ return {
184
+ run_id: context.run_id, run_credential: context.run_credential, status: API_STATUS[receipt.status],
185
+ exit_code: receipt.exit_code ?? undefined, invalidated_reason: reason ?? undefined,
186
+ receipt_path: context.receipt_upload, receipt_content_type: "application/json",
187
+ receipt_byte_size: bytes.length, receipt_sha256: createHash("sha256").update(bytes).digest("hex"),
188
+ output_characters: outputCharacters, truncated: outputCharacters > MAX_CAPTURE_BYTES,
189
+ };
190
+ }
191
+
192
+ function updateFromPersisted(terminalUpdate) {
193
+ const update = {};
194
+ for (const key of UPDATE_FIELDS) if (terminalUpdate[key] !== undefined) update[key] = terminalUpdate[key];
195
+ return update;
196
+ }
197
+
198
+ // ── processes ────────────────────────────────────────────────────────────────
199
+
200
+ export function parseProcessList(text) {
201
+ const rows = [];
202
+ for (const line of String(text).split("\n")) {
203
+ const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s(.*)$/.exec(line);
204
+ if (match) rows.push({ pid: Number(match[1]), ppid: Number(match[2]), uid: Number(match[3]), args: match[4].trim() });
205
+ }
206
+ return rows;
207
+ }
208
+
209
+ export function listProcessesWithPs() {
210
+ return new Promise((resolve, reject) => {
211
+ execFile("ps", ["-ww", "-A", "-o", "pid=,ppid=,uid=,args="], { maxBuffer: 32 * 1024 * 1024 }, (error, stdout) => {
212
+ if (error) reject(error); else resolve(parseProcessList(stdout));
213
+ });
214
+ });
215
+ }
216
+
217
+ export function pidAlive(pid) {
218
+ if (!Number.isInteger(pid) || pid <= 0) return false;
219
+ try { process.kill(pid, 0); return true; } catch (error) { return error?.code === "EPERM"; }
220
+ }
221
+
222
+ // Every process whose argument string names this context as its worker.
223
+ function workersFor(processes, contextFile) {
224
+ const suffix = ` run --worker ${contextFile}`;
225
+ return processes.filter((proc) => proc.args.endsWith(suffix) || proc.args.includes(`${suffix} `));
226
+ }
227
+
228
+ // AC-4: exactly `<node> <…>botbuddy.mjs run --worker <this context>`, owned by
229
+ // this user, and with no child process. The receipt/terminal checks are the
230
+ // caller's, made on the same context read.
231
+ export function isStoppableLegacyWorker(proc, { contextFile, uid, processes }) {
232
+ const suffix = ` run --worker ${contextFile}`;
233
+ if (!proc.args.endsWith(suffix)) return false;
234
+ if (!/^\S+\s+\S*botbuddy\.mjs$/.test(proc.args.slice(0, -suffix.length))) return false;
235
+ if (proc.uid !== uid) return false;
236
+ return !processes.some((other) => other.ppid === proc.pid);
237
+ }
238
+
239
+ async function stopLegacyWorker(pid, { contextFile, uid, listProcesses, killProcess, isAlive, sleep }) {
240
+ // Re-verify on a fresh listing right before signalling (PID reuse, a new child).
241
+ const fresh = await listProcesses();
242
+ const proc = fresh.find((candidate) => candidate.pid === pid);
243
+ if (!proc || !isStoppableLegacyWorker(proc, { contextFile, uid, processes: fresh })) return !proc;
244
+ killProcess(pid, "SIGTERM");
245
+ for (let waited = 0; waited < STOP_GRACE_MS; waited += 100) {
246
+ if (!isAlive(pid)) return true;
247
+ await sleep(100);
248
+ }
249
+ return !isAlive(pid);
250
+ }
251
+
252
+ // ── one drain pass ────────────────────────────────────────────────────────────
253
+
254
+ /**
255
+ * Deliver every pending terminal update once.
256
+ * @returns {Promise<{ delivered: number, parked: number, pending: number, stopped: number, owned: number, wrote: boolean }>}
257
+ */
258
+ export async function drainOnce({
259
+ home = homedir(), call, listProcesses = listProcessesWithPs, killProcess = (pid, signal) => process.kill(pid, signal),
260
+ isAlive = pidAlive, uid = process.getuid?.() ?? -1, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
261
+ listContexts = async () => (await readdir(commandRunsDir(home)).catch((error) => { if (error?.code === "ENOENT") return []; throw error; }))
262
+ .filter((name) => name.endsWith(".context.json")).sort(),
263
+ maxAttempts = DELIVERY_MAX_ATTEMPTS,
264
+ } = {}) {
265
+ const totals = { delivered: 0, parked: 0, pending: 0, stopped: 0, owned: 0, wrote: false };
266
+ const names = await listContexts();
267
+ if (!names.length) return totals;
268
+ let processes = null;
269
+ for (const name of names) {
270
+ const contextFile = join(commandRunsDir(home), name);
271
+ const context = await readJson(contextFile);
272
+ // Deleted by a successful delivery mid-scan, or unreadable: not ours.
273
+ if (!context || typeof context.run_id !== "string" || typeof context.receipt_path !== "string") continue;
274
+ // A pre-workload cancellation is owned by the cancel handoff path. A
275
+ // marker written by a cancel that raced workload start leaves the worker
276
+ // owning a normal terminal update, which stays eligible here.
277
+ if (!context.terminal_update && await readFile(`${contextFile}.cancelled`).then(() => true, () => false)) continue;
278
+ const receipt = await readJson(context.receipt_path);
279
+ // Still executing (no terminal receipt yet): never touched.
280
+ if (!receipt || receipt.run_id !== context.run_id || !TERMINAL_RECEIPT_STATUSES.has(receipt.status)) continue;
281
+ const persisted = context.terminal_update ?? null;
282
+ if (persisted?.parked && persisted.parked.retryable === false) continue;
283
+ processes ??= await listProcesses();
284
+ const workers = workersFor(processes, contextFile).filter((proc) => isAlive(proc.pid));
285
+ if (workers.length && !persisted?.parked) {
286
+ // A bounded (current) worker owns its delivery until it parks it. A
287
+ // legacy worker is adopted only once it is provably in its terminal
288
+ // retry loop and stopping it cannot touch running work.
289
+ const legacyLoop = persisted && persisted.retry_budget_ms === undefined;
290
+ const stoppable = legacyLoop && workers.every((proc) => isStoppableLegacyWorker(proc, { contextFile, uid, processes }));
291
+ if (!stoppable) { totals.owned += 1; continue; }
292
+ let allStopped = true;
293
+ for (const proc of workers) {
294
+ if (await stopLegacyWorker(proc.pid, { contextFile, uid, listProcesses, killProcess, isAlive, sleep })) totals.stopped += 1;
295
+ else allStopped = false;
296
+ }
297
+ processes = null;
298
+ if (!allStopped) { totals.owned += 1; continue; }
299
+ }
300
+ // Re-read: a stopped legacy worker may have rewritten the context last.
301
+ const current = await readJson(contextFile);
302
+ if (!current) continue;
303
+ const update = current.terminal_update ? updateFromPersisted(current.terminal_update) : await terminalUpdateFromReceipt(current, receipt);
304
+ const previous = current.terminal_update ?? {};
305
+ const attempts = (Number.isInteger(previous.attempts) ? previous.attempts : 0) + 1;
306
+ const deliveryAttempts = (Number.isInteger(previous.delivery_attempts) ? previous.delivery_attempts : 0) + 1;
307
+ if (!sealedToken(current)) {
308
+ // No sealed owner: a delivery from this process could only use an
309
+ // ambient identity, which BOT-2052 forbids. Park it visibly instead.
310
+ const failure = { reason: "callback_session_token_missing", retryable: false, kind: "callback_session_token_missing" };
311
+ await writeJsonAtomic(contextFile, { ...current, terminal_update: { ...previous, ...update, parked: parkedRecord(failure, "delivery") } });
312
+ totals.parked += 1; totals.wrote = true;
313
+ continue;
314
+ }
315
+ const outcome = await attemptTerminalUpdate(current, update, call);
316
+ if (outcome.accepted) {
317
+ try { await unlink(contextFile); } catch { /* already removed */ }
318
+ totals.delivered += 1; totals.wrote = true;
319
+ continue;
320
+ }
321
+ let failure = outcome.failure;
322
+ const cap = failure.kind === "ambiguous_rejection" ? Math.min(maxAttempts, DELIVERY_AMBIGUOUS_MAX_ATTEMPTS) : maxAttempts;
323
+ if (failure.retryable && deliveryAttempts >= cap) failure = { ...failure, retryable: false, kind: "delivery_attempts_exhausted" };
324
+ await writeJsonAtomic(contextFile, { ...current, terminal_update: {
325
+ ...previous, ...update, attempts, delivery_attempts: deliveryAttempts, last_error: failure.reason,
326
+ parked: failure.retryable ? { ...(previous.parked ?? parkedRecord(failure, "delivery")), reason: failure.reason, retryable: true } : parkedRecord(failure, "delivery"),
327
+ } });
328
+ totals.wrote = true;
329
+ if (failure.retryable) totals.pending += 1; else totals.parked += 1;
330
+ }
331
+ return totals;
332
+ }
333
+
334
+ // ── the per-machine lock ──────────────────────────────────────────────────────
335
+
336
+ /**
337
+ * Take the per-machine delivery lock. A lock whose holder PID is gone is stale
338
+ * and is taken over.
339
+ */
340
+ export async function acquireDeliveryLock({ home = homedir(), pid = process.pid, isAlive = pidAlive } = {}) {
341
+ const lockFile = deliveryLockPath(home);
342
+ await mkdir(dirname(lockFile), { recursive: true });
343
+ const candidate = `${lockFile}.${pid}.${randomUUID()}.tmp`;
344
+ await writeFile(candidate, JSON.stringify({ pid, started_at: new Date().toISOString() }), { mode: 0o600 });
345
+ try {
346
+ for (let round = 0; round < 3; round++) {
347
+ try {
348
+ await link(candidate, lockFile); // atomic create-if-absent
349
+ return { acquired: true, holderPid: pid, release: async () => {
350
+ const holder = await readJson(lockFile);
351
+ if (holder?.pid === pid) await unlink(lockFile).catch(() => {});
352
+ } };
353
+ } catch (error) {
354
+ if (error?.code !== "EEXIST") throw error;
355
+ }
356
+ const holder = await readJson(lockFile);
357
+ if (holder === null) continue; // released between link and read
358
+ const holderPid = holder?.pid;
359
+ if (Number.isInteger(holderPid) && isAlive(holderPid)) return { acquired: false, holderPid, release: async () => {} };
360
+ // Stale. Only the holder of the reclaim mutex may remove the lock file,
361
+ // so it can never move aside a lock another contender just acquired:
362
+ // while the lock file exists, nobody else can replace it (link fails),
363
+ // and no other reclaimer can run, so what we re-read is what we remove.
364
+ const reclaim = await acquireReclaimMutex(lockFile, candidate, isAlive);
365
+ if (!reclaim) return { acquired: false, holderPid: null, release: async () => {} };
366
+ try {
367
+ const current = await readJson(lockFile);
368
+ if (current && !(Number.isInteger(current.pid) && !isAlive(current.pid))) {
369
+ return { acquired: false, holderPid: current?.pid ?? null, release: async () => {} };
370
+ }
371
+ if (current) await unlink(lockFile).catch((error) => { if (error?.code !== "ENOENT") throw error; });
372
+ } finally {
373
+ await reclaim.release();
374
+ }
375
+ }
376
+ const holder = await readJson(lockFile);
377
+ return { acquired: false, holderPid: holder?.pid ?? null, release: async () => {} };
378
+ } finally {
379
+ await unlink(candidate).catch(() => {});
380
+ }
381
+ }
382
+
383
+ // The reclaim mutex serialises stale-lock removal. Its critical section is a
384
+ // read and an unlink, so a reclaim file left by a crashed contender is taken
385
+ // over only once its holder is dead AND it is older than RECLAIM_STALE_MS.
386
+ // Trust boundary: two contenders racing to clear a crashed reclaimer's mutex
387
+ // in the same instant is not defended further (it needs a crash inside a
388
+ // sub-millisecond section plus concurrent contention).
389
+ const RECLAIM_STALE_MS = 10_000;
390
+ async function acquireReclaimMutex(lockFile, candidate, isAlive) {
391
+ const mutex = `${lockFile}.reclaim`;
392
+ for (let attempt = 0; attempt < 2; attempt++) {
393
+ try {
394
+ await link(candidate, mutex);
395
+ return { release: () => unlink(mutex).catch(() => {}) };
396
+ } catch (error) {
397
+ if (error?.code !== "EEXIST") throw error;
398
+ }
399
+ const holder = await readJson(mutex);
400
+ if (holder === null) continue;
401
+ let ageMs = 0;
402
+ try { ageMs = Date.now() - (await stat(mutex)).mtimeMs; } catch { continue; }
403
+ const dead = !(Number.isInteger(holder?.pid) && isAlive(holder.pid));
404
+ if (!dead || ageMs < RECLAIM_STALE_MS) return null; // another contender is reclaiming
405
+ await unlink(mutex).catch(() => {});
406
+ }
407
+ return null;
408
+ }
409
+
410
+ async function dirMtimeMs(home) {
411
+ try { return (await stat(commandRunsDir(home))).mtimeMs; } catch (error) {
412
+ if (error?.code === "ENOENT") return null;
413
+ throw error;
414
+ }
415
+ }
416
+
417
+ /**
418
+ * The delivery process: hold the lock, drain until nothing retryable remains
419
+ * (or the lifetime cap), record the settled directory state, release, exit.
420
+ */
421
+ export async function runDelivery({
422
+ home = homedir(), call, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), now = () => performance.now(),
423
+ lifetimeMs = DELIVERY_LIFETIME_MS, ...drainOptions
424
+ } = {}) {
425
+ const lock = await acquireDeliveryLock({ home });
426
+ if (!lock.acquired) return { outcome: "held", holder_pid: lock.holderPid, delivered: 0, parked: 0, pending: 0, stopped: 0 };
427
+ const totals = { delivered: 0, parked: 0, pending: 0, stopped: 0, owned: 0 };
428
+ const started = now();
429
+ let backoff = DELIVERY_FIRST_BACKOFF_MS;
430
+ let settlePasses = 0;
431
+ try {
432
+ for (;;) {
433
+ const mtime = await dirMtimeMs(home);
434
+ const pass = await drainOnce({ home, call, sleep, ...drainOptions });
435
+ for (const key of ["delivered", "parked", "stopped"]) totals[key] += pass[key];
436
+ totals.pending = pass.pending;
437
+ totals.owned = pass.owned;
438
+ if (pass.pending === 0) {
439
+ // Our own writes move the directory mtime; one more pass confirms the
440
+ // directory is settled so `bb` invocations stop re-starting us.
441
+ if (pass.wrote && settlePasses++ < 3) continue;
442
+ // A delivery still owned by a live worker is not settled: if that
443
+ // worker dies before it writes again, the directory never changes, so
444
+ // leave the state open for a re-scan after the cooldown.
445
+ await writeJsonAtomic(deliveryStatePath(home), { dir_mtime_ms: pass.owned > 0 ? null : mtime, settled_at: new Date().toISOString(), owned: pass.owned });
446
+ return { outcome: "drained", ...totals };
447
+ }
448
+ settlePasses = 0;
449
+ if (now() - started + backoff > lifetimeMs) {
450
+ // Retryable work remains: leave the state unsettled so the next `bb`
451
+ // invocation starts a fresh delivery process.
452
+ await writeJsonAtomic(deliveryStatePath(home), { dir_mtime_ms: null, settled_at: null, pending: pass.pending });
453
+ return { outcome: "lifetime_exhausted", ...totals };
454
+ }
455
+ await sleep(backoff);
456
+ backoff = Math.min(backoff * 2, DELIVERY_MAX_BACKOFF_MS);
457
+ }
458
+ } finally {
459
+ await lock.release();
460
+ }
461
+ }
462
+
463
+ /**
464
+ * Called by every `bb` invocation (and by a worker that parked a retryable
465
+ * delivery): start the delivery process when the run directory changed since
466
+ * it last settled and no live process holds the lock. Cheap: a stat and two
467
+ * small reads. Never throws into the caller's command.
468
+ */
469
+ export async function ensureRunDelivery({
470
+ home = homedir(), env = process.env, spawnImpl = spawn, isAlive = pidAlive, force = false, cooldownMs = DELIVERY_RESCAN_COOLDOWN_MS,
471
+ } = {}) {
472
+ try {
473
+ if (env.BOTBUDDY_RUN_DELIVERY === "off") return { started: false, reason: "disabled" };
474
+ const mtime = await dirMtimeMs(home);
475
+ if (mtime === null) return { started: false, reason: "no_runs" };
476
+ const state = await readJson(deliveryStatePath(home));
477
+ if (!force && state && state.dir_mtime_ms === mtime) return { started: false, reason: "up_to_date" };
478
+ // A worker still inside its own retry budget rewrites its context on every
479
+ // attempt. Re-scanning on each of those changes is wasted work: the worker
480
+ // starts us itself (force) when it parks, so otherwise scan at most once
481
+ // per cooldown. Both times are this machine's filesystem clock.
482
+ if (!force && state) {
483
+ const stateAge = Date.now() - (await stat(deliveryStatePath(home))).mtimeMs;
484
+ if (stateAge >= 0 && stateAge < cooldownMs) return { started: false, reason: "cooldown" };
485
+ }
486
+ const holder = await readJson(deliveryLockPath(home));
487
+ if (Number.isInteger(holder?.pid) && isAlive(holder.pid)) return { started: false, reason: "held" };
488
+ const bin = join(dirname(fileURLToPath(import.meta.url)), "..", "bin", "botbuddy.mjs");
489
+ const child = spawnImpl(process.execPath, [bin, "run", "--deliver"], { detached: true, stdio: "ignore", cwd: home, windowsHide: true });
490
+ child.on?.("error", () => { /* best effort: the next invocation retries */ });
491
+ child.unref?.();
492
+ return { started: true, reason: "pending" };
493
+ } catch (error) {
494
+ return { started: false, reason: `error:${error instanceof Error ? error.message : String(error)}` };
495
+ }
496
+ }
package/src/run.mjs CHANGED
@@ -13,7 +13,9 @@ import { dirname, join } from "node:path";
13
13
  import { homedir } from "node:os";
14
14
  import { spawn } from "node:child_process";
15
15
  import { fileURLToPath } from "node:url";
16
+ import { performance } from "node:perf_hooks";
16
17
  import { callToolJson } from "./api.mjs";
18
+ import { SERVER_URL } from "./config.mjs";
17
19
  import { parseLaneEvents, laneCaseCounts, flushLaneCases, buildLaneSummary, laneSummaryFilename } from "./test-lane-events.mjs";
18
20
  import { AGENT_KEY_RE, readAgentSessionTokenEnv, legacyAgentAliasHint } from "./agent-key.mjs";
19
21
  import { hasAgentSessionRuntime, isRejectedCachedMcpSession, resolveAgentSessionCredential, selfHealAgentSession } from "./agent-session.mjs";
@@ -23,6 +25,10 @@ import { createTelemetryOutbox } from "./telemetry-outbox.mjs";
23
25
  import { deliverExecutionEvent } from "./telemetry-delivery.mjs";
24
26
  import { attestTelemetryCredential, loadTelemetryIdentity } from "./telemetry-config.mjs";
25
27
  import { VERSION } from "./version.mjs";
28
+ import {
29
+ TERMINAL_RETRY_BUDGET_MS, DELIVERY_CALL_TIMEOUT_MS, callWithContextIdentity, classifyTerminalUpdateFailure,
30
+ ensureRunDelivery, parkedRecord, runDelivery, terminalUpdateAccepted,
31
+ } from "./run-delivery.mjs";
26
32
 
27
33
  export const RUN_SCHEMA_VERSION = 1;
28
34
  export const EXIT = Object.freeze({ OK: 0, AUTH: 3, INVALID: 4, BACKEND: 5, INTERNAL: 7 });
@@ -405,7 +411,7 @@ export async function launchRun(argv, {
405
411
  // every callback is attributable to the same agent. It deliberately stays
406
412
  // outside child_env, receipts, and the workload's process environment.
407
413
  const knownSession = opts.sessionId ?? managedCredential?.identity?.sessionId ?? managedCredential?.sessionId;
408
- await writeJson(context, { schema_version: RUN_SCHEMA_VERSION, context_path: context, run_id: runId, run_credential: registration.data.run_credential, receipt_upload: registration.data.receipt_upload, command, cwd, timeout_seconds: opts.timeout, receipt_path: receipt, launched_at: new Date().toISOString(), ...(opts.sessionToken ? { callback_session_token: opts.sessionToken } : {}), ...(knownSession ? { callback_session_id: knownSession } : {}), ...(childEnv && Object.keys(childEnv).length ? { child_env: childEnv } : {}), ...(testRun ? { test_run: testRun } : {}) });
414
+ await writeJson(context, { schema_version: RUN_SCHEMA_VERSION, context_path: context, run_id: runId, run_credential: registration.data.run_credential, receipt_upload: registration.data.receipt_upload, command, cwd, timeout_seconds: opts.timeout, receipt_path: receipt, launched_at: new Date().toISOString(), server_url: SERVER_URL,...(opts.sessionToken ? { callback_session_token: opts.sessionToken } : {}), ...(knownSession ? { callback_session_id: knownSession } : {}), ...(childEnv && Object.keys(childEnv).length ? { child_env: childEnv } : {}), ...(testRun ? { test_run: testRun } : {}) });
409
415
  const [file, args] = workerInvocation(context);
410
416
  try {
411
417
  const worker = spawnImpl(file, args, { detached: true, stdio: "ignore", windowsHide: true });
@@ -426,7 +432,10 @@ export async function runWorker(contextFile, {
426
432
  call = callToolJson,
427
433
  signalEmitter = process,
428
434
  terminate = terminateProcessGroup,
435
+ // BOT-2600: the bounded terminal delivery (see retryTerminalUpdate).
436
+ sleep, now, terminalBudgetMs, startDelivery,
429
437
  } = {}) {
438
+ const retry = { sleep, now, budgetMs: terminalBudgetMs, startDelivery };
430
439
  const context = JSON.parse(await readFile(contextFile, "utf8"));
431
440
  const callbackCall = callWithContextIdentity(context, call);
432
441
  const ownerHeartbeat = startOwnerHeartbeat(context, callbackCall);
@@ -492,7 +501,7 @@ export async function runWorker(contextFile, {
492
501
  child = spawnImpl(context.command[0], context.command.slice(1), { cwd: context.cwd, detached: true, stdio: ["ignore", "pipe", "pipe"], windowsHide: true, ...(context.child_env ? { env: { ...process.env, ...context.child_env } } : {}) });
493
502
  } catch (error) {
494
503
  const endedAt = new Date().toISOString();
495
- const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${error instanceof Error ? error.message : String(error)}` }, callbackCall);
504
+ const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${error instanceof Error ? error.message : String(error)}` }, callbackCall, retry);
496
505
  return receipt;
497
506
  }
498
507
  // A failed spawn can report through the child `error` event on the next turn.
@@ -523,11 +532,11 @@ export async function runWorker(contextFile, {
523
532
  if (lane) await lane.stop();
524
533
  const endedAt = new Date().toISOString();
525
534
  if (result.error) {
526
- const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${result.error.message}`, output: capture.output, outputCharacters: capture.characters }, callbackCall);
535
+ const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${result.error.message}`, output: capture.output, outputCharacters: capture.characters }, callbackCall, retry);
527
536
  return receipt;
528
537
  }
529
538
  const status = terminalStatus({ ...result, timedOut });
530
- const receipt = await finalize(context, { status, startedAt, endedAt, exitCode: result.exitCode, signal: result.signal, output: capture.output, outputCharacters: capture.characters }, callbackCall);
539
+ const receipt = await finalize(context, { status, startedAt, endedAt, exitCode: result.exitCode, signal: result.signal, output: capture.output, outputCharacters: capture.characters }, callbackCall, retry);
531
540
  return receipt;
532
541
  } finally {
533
542
  if (ownerHeartbeat) await ownerHeartbeat.stop();
@@ -556,7 +565,14 @@ function startOwnerHeartbeat(context, call, { intervalMs = 5 * 60_000, setInterv
556
565
  tick();
557
566
  const timer = setIntervalFn(tick, intervalMs);
558
567
  timer.unref?.();
559
- return { async stop() { stopped = true; clearInterval(timer); await chain; } };
568
+ // BOT-2600: an in-flight heartbeat must never hold a finished worker open.
569
+ return { async stop() {
570
+ stopped = true;
571
+ clearInterval(timer);
572
+ let bound;
573
+ await Promise.race([chain, new Promise((resolve) => { bound = setTimeout(resolve, 2_000); bound.unref?.(); })]);
574
+ clearTimeout(bound);
575
+ } };
560
576
  }
561
577
 
562
578
  // Periodic lane flusher: reduce the NDJSON events file to the current case state
@@ -785,7 +801,7 @@ export async function cancelRun(runId, {
785
801
  return { exitCode: EXIT.OK, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "cancellation_requested", run_id: runId, exit_code: EXIT.OK } };
786
802
  }
787
803
 
788
- async function finalize(context, state, call) {
804
+ async function finalize(context, state, call, retry = {}) {
789
805
  const boundedLog = String(state.output ?? "").slice(-8192);
790
806
  const outputCharacters = state.outputCharacters ?? String(state.output ?? "").length;
791
807
  const receipt = { schema_version: RUN_SCHEMA_VERSION, run_id: context.run_id, status: state.status, started_at: state.startedAt, ended_at: state.endedAt, exit_code: state.exitCode, signal: state.signal, recovery_command: state.status === "failure" ? `botbuddy run --session-id <new-session-id> --environment local -- ${context.command.map((x) => JSON.stringify(x)).join(" ")}` : null, bounded_log: boundedLog, output_characters: outputCharacters, reason: state.reason ?? null };
@@ -795,43 +811,77 @@ async function finalize(context, state, call) {
795
811
  const apiStatus = ({ success: "succeeded", failure: "failed", cancelled: "cancelled", timed_out: "timed_out", owner_lost: "owner_lost" })[state.status];
796
812
  const update = { run_id: context.run_id, run_credential: context.run_credential, status: apiStatus, exit_code: state.exitCode ?? undefined, invalidated_reason: state.reason ?? (state.status === "timed_out" ? "timeout" : state.status === "owner_lost" ? "owner_lost" : undefined), receipt_path: context.receipt_upload, receipt_content_type: "application/json", receipt_byte_size: bytes.size, receipt_sha256: sha256, output_characters: outputCharacters, truncated: outputCharacters > MAX_CAPTURE_BYTES };
797
813
  // A locally durable receipt is enough to publish the linked test run. Do it
798
- // before the command callback's intentionally unbounded retry: otherwise an
799
- // auth outage (for example agent_identity_required) strands a 0-case run in
800
- // active state forever even though its command has already failed locally.
814
+ // before the command callback's retry: otherwise an auth outage (for example
815
+ // agent_identity_required) strands a 0-case run in active state even though
816
+ // its command has already failed locally.
801
817
  if (context.test_run) await completeTestRun(context, { status: state.status, exitCode: state.exitCode, signal: state.signal, startedAt: state.startedAt, endedAt: state.endedAt, call });
802
- await retryTerminalUpdate(context, update, call);
803
- try { await unlink(context.context_path); } catch { /* receipt is authoritative; context cleanup is best effort */ }
818
+ const delivery = await retryTerminalUpdate(context, update, call, retry);
819
+ // A parked delivery keeps its context: it is the durable record the
820
+ // per-machine delivery process (and an operator) reads.
821
+ if (delivery.delivered) {
822
+ try { await unlink(context.context_path); } catch { /* receipt is authoritative; context cleanup is best effort */ }
823
+ }
804
824
  return receipt;
805
825
  }
806
826
 
807
- function callWithContextIdentity(context, call) {
808
- const token = typeof context.callback_session_token === "string" && SESSION_TOKEN_RE.test(context.callback_session_token)
809
- ? context.callback_session_token
810
- : null;
811
- // Old contexts cannot safely fall back to the machine-level owner key. Ask
812
- // the API layer to require a session instead; the retained receipt/context
813
- // makes recovery possible without attributing a callback to the wrong agent.
814
- return (name, args) => call(name, args, token
815
- ? { auth: { "x-agent-api-key": token } }
816
- : { requireSessionToken: true });
817
- }
818
-
819
- export async function retryTerminalUpdate(context, update, call, { sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) } = {}) {
827
+ // BOT-2600: the terminal callback is bounded. The worker that owns the receipt
828
+ // retries transient failures (transport, timeout, 5xx) within a monotonic
829
+ // budget of at most ten minutes and parks a permanent rejection (no owning
830
+ // identity, an invalid run credential, a local environment error) at once.
831
+ // Either way it records `terminal_update.parked` in the private context and
832
+ // returns so the worker process exits; a retryable parked delivery is handed to
833
+ // the single per-machine delivery process (`bb run --deliver`).
834
+ export async function retryTerminalUpdate(context, update, call, {
835
+ sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
836
+ now = () => performance.now(),
837
+ budgetMs = TERMINAL_RETRY_BUDGET_MS,
838
+ startDelivery = () => ensureRunDelivery({ force: true }),
839
+ } = {}) {
840
+ const started = now();
820
841
  let attempts = 0;
821
842
  let delay = 1_000;
822
843
  for (;;) {
823
- let result;
824
- try { result = await call("update_command_run", update); } catch (error) { result = { ok: false, error: error instanceof Error ? error.message : String(error) }; }
825
- if (result?.ok && !result.isError && result.data?.error == null && result.data?.accepted !== false) return result;
844
+ let result = null;
845
+ let failure;
846
+ try {
847
+ result = await call("update_command_run", update, {
848
+ signal: AbortSignal.timeout(DELIVERY_CALL_TIMEOUT_MS),
849
+ // The endpoint this run was registered with (BOT-2600; sealed at launch).
850
+ ...(typeof context.server_url === "string" && context.server_url ? { serverUrl: context.server_url } : {}),
851
+ });
852
+ if (terminalUpdateAccepted(result)) return { delivered: true, attempts: attempts + 1, result };
853
+ failure = classifyTerminalUpdateFailure(result);
854
+ } catch (error) {
855
+ failure = classifyTerminalUpdateFailure(null, error);
856
+ }
826
857
  attempts += 1;
827
- await writeJson(context.context_path, { ...context, terminal_update: { ...update, attempts, last_error: result?.error ?? result?.data?.error ?? "terminal_update_rejected", next_retry_at: new Date(Date.now() + delay).toISOString() } });
858
+ const exhausted = failure.retryable && !failure.parkNow && now() - started + delay > budgetMs;
859
+ if (!failure.retryable || failure.parkNow || exhausted) {
860
+ const parked = parkedRecord(exhausted ? { ...failure, kind: "retry_budget_exhausted" } : failure, "worker");
861
+ await writeJson(context.context_path, { ...context, terminal_update: { ...update, attempts, last_error: failure.reason, retry_budget_ms: budgetMs, parked } });
862
+ if (parked.retryable) { try { await startDelivery(); } catch { /* the next bb invocation starts it */ } }
863
+ return { delivered: false, attempts, parked, result };
864
+ }
865
+ await writeJson(context.context_path, { ...context, terminal_update: { ...update, attempts, last_error: failure.reason, retry_budget_ms: budgetMs, next_retry_at: new Date(Date.now() + delay).toISOString() } });
828
866
  await sleep(delay);
829
867
  delay = Math.min(delay * 2, 60_000);
830
868
  }
831
869
  }
832
870
 
833
871
  export async function cmdRun(args) {
834
- if (args[0] === "--worker") { await runWorker(args[1]); return; }
872
+ if (args[0] === "--worker") {
873
+ await runWorker(args[1]);
874
+ // BOT-2600: the receipt is durable and its delivery is accepted or parked.
875
+ // Nothing may keep a finished worker resident; an unref'd guard exits even
876
+ // if a stray handle (a keep-alive socket) would otherwise hold the loop.
877
+ setTimeout(() => process.exit(process.exitCode ?? 0), 1_000).unref();
878
+ return;
879
+ }
880
+ if (args[0] === "--deliver") {
881
+ const result = await runDelivery({ call: callToolJson });
882
+ if (args.includes("--json")) oneLine({ schema_version: RUN_SCHEMA_VERSION, ...result }, false);
883
+ return;
884
+ }
835
885
  if (args[0] === "cancel") {
836
886
  const result = await cancelRun(args[1] ?? "");
837
887
  oneLine(result.receipt, false);
package/src/wait-core.mjs CHANGED
@@ -287,9 +287,10 @@ const VALIDATORS = {
287
287
  // BOT-1507 — scope=next adds a fourth, LIVE-ONLY selector: it names no run or PR
288
288
  // (they must already exist at arm time) and wakes on the first terminal CI run for
289
289
  // the repo delivered AFTER the wait was armed — the post-merge / cron train case.
290
- // Optional branch=/workflow= filter that first run; branch is valid only with
291
- // scope=next. A workflow also pins a level-triggered sha= recovery to one
292
- // canonical workflow identity.
290
+ // Optional branch=/workflow=/workflow_id=/workflow_path= filters narrow that
291
+ // first run; branch and stable workflow identities are valid only with
292
+ // scope=next. The legacy display-name filter also pins a level-triggered
293
+ // sha= recovery to one historical workflow.
293
294
  // BOT-1685 — scope=next also accepts an optional sha= FILTER (distinct from the
294
295
  // level-triggered sha= selector): a wait armed BEFORE re-dispatching a run for a
295
296
  // known commit then wakes on the NEXT terminal run for THAT commit, instead of
@@ -309,9 +310,13 @@ const VALIDATORS = {
309
310
  }
310
311
  const hasBranchFilter = p.branch !== undefined;
311
312
  const hasWorkflowFilter = p.workflow !== undefined;
313
+ const hasWorkflowIdFilter = p.workflow_id !== undefined;
314
+ const hasWorkflowPathFilter = p.workflow_path !== undefined;
312
315
  if (p.scope !== undefined) {
313
316
  if (p.scope === "latest") {
314
- if (hasBranchFilter || hasWorkflowFilter) throw new Error("ci branch=/workflow= are only valid with scope=next");
317
+ if (hasBranchFilter || hasWorkflowFilter || hasWorkflowIdFilter || hasWorkflowPathFilter) {
318
+ throw new Error("ci branch=/workflow=/workflow_id=/workflow_path= are only valid with scope=next");
319
+ }
315
320
  if (p.pr === undefined || !/^\d+$/.test(String(p.pr).trim())) {
316
321
  throw new Error("ci scope=latest needs pr=<number>");
317
322
  }
@@ -329,12 +334,23 @@ const VALIDATORS = {
329
334
  if (v === "") throw new Error(`ci ${key}= must be non-empty`);
330
335
  return v;
331
336
  };
332
- return { repo, scope: "next", branch: filter("branch"), workflow: filter("workflow"), sha: filter("sha"), pr: null };
337
+ const workflowId = filter("workflow_id");
338
+ const workflowPath = filter("workflow_path");
339
+ return {
340
+ repo,
341
+ scope: "next",
342
+ branch: filter("branch"),
343
+ workflow: filter("workflow"),
344
+ ...(workflowId == null ? {} : { workflowId }),
345
+ ...(workflowPath == null ? {} : { workflowPath }),
346
+ sha: filter("sha"),
347
+ pr: null,
348
+ };
333
349
  }
334
350
  throw new Error("ci scope must be 'latest' or 'next' (or use run_id=/sha=)");
335
351
  }
336
- if (hasBranchFilter || (hasWorkflowFilter && p.sha === undefined)) {
337
- throw new Error("ci branch= is only valid with scope=next; workflow= also accepts an exact sha selector");
352
+ if (hasBranchFilter || hasWorkflowIdFilter || hasWorkflowPathFilter || (hasWorkflowFilter && p.sha === undefined)) {
353
+ throw new Error("ci branch=/workflow_id=/workflow_path= are only valid with scope=next; legacy workflow= also accepts an exact sha selector");
338
354
  }
339
355
  if (p.run_id !== undefined) {
340
356
  if (!String(p.run_id).trim()) throw new Error("ci run_id must be non-empty");
@@ -343,7 +359,13 @@ const VALIDATORS = {
343
359
  if (!String(p.sha).trim()) throw new Error("ci sha must be non-empty");
344
360
  const workflow = p.workflow === undefined ? null : String(p.workflow).trim();
345
361
  if (p.workflow !== undefined && workflow === "") throw new Error("ci workflow= must be non-empty");
346
- return { repo, scope: "sha", sha: String(p.sha).trim(), workflow, pr: p.pr != null ? String(p.pr).trim() : null };
362
+ return {
363
+ repo,
364
+ scope: "sha",
365
+ sha: String(p.sha).trim(),
366
+ workflow,
367
+ pr: p.pr != null ? String(p.pr).trim() : null,
368
+ };
347
369
  },
348
370
  "staging-green"(p) {
349
371
  const repo = p.repo != null ? String(p.repo).trim() : null;
@@ -1013,7 +1035,7 @@ export function formatUnknownWorkflowGuidance(detail, candidates) {
1013
1035
  const named = Array.isArray(candidates) ? candidates.filter((c) => typeof c === "string" && c.trim() !== "") : [];
1014
1036
  if (named.length === 0) return lines;
1015
1037
  lines.push("closest workflow names this repo has actually reported:");
1016
- for (const name of named) lines.push(` workflow=${name}`);
1038
+ for (const name of named) lines.push(` ${name.includes("=") ? name : `workflow=${name}`}`);
1017
1039
  return lines;
1018
1040
  }
1019
1041
 
@@ -1152,16 +1174,20 @@ function conditionMatchesSignal(condition, signal, waitSessionId) {
1152
1174
  if (params.scope === "run_id") return String(signal.payload?.run_id) === params.runId;
1153
1175
  if (params.scope === "sha") {
1154
1176
  if (signal.payload?.head_sha !== params.sha) return false;
1155
- if (params.workflow == null) return true;
1156
- const workflow = signal.payload?.workflow_name;
1157
- return typeof workflow === "string" && workflow.trim().toLowerCase() === params.workflow.trim().toLowerCase();
1177
+ if (params.workflow != null) {
1178
+ const workflow = signal.payload?.workflow_name;
1179
+ if (typeof workflow !== "string" || workflow.trim().toLowerCase() !== params.workflow.trim().toLowerCase()) return false;
1180
+ }
1181
+ if (params.workflowId != null && String(signal.payload?.workflow_id ?? "").trim() !== params.workflowId) return false;
1182
+ if (params.workflowPath != null && String(signal.payload?.workflow_path ?? "").trim() !== params.workflowPath) return false;
1183
+ return true;
1158
1184
  }
1159
1185
  if (params.scope === "next") {
1160
1186
  // BOT-1507 — live-only: the first TERMINAL run for the repo. Never keys on
1161
1187
  // subject_key (it is run:<id> for post-merge runs and owner/repo#n for PR
1162
1188
  // runs — both are legitimate wakes); a null/empty conclusion is an
1163
1189
  // in-progress signal and never terminal. Optional branch/workflow filters:
1164
- // branch is exact, workflow is the GitHub display name (case/space-insensitive).
1190
+ // branch/path/id are exact; the display name is case/space-insensitive.
1165
1191
  const conclusion = signal.payload?.conclusion;
1166
1192
  if (typeof conclusion !== "string" || conclusion.trim() === "") return false;
1167
1193
  // BOT-1685: an optional sha filter pins scope=next to a commit, so a
@@ -1174,6 +1200,8 @@ function conditionMatchesSignal(condition, signal, waitSessionId) {
1174
1200
  return false;
1175
1201
  }
1176
1202
  }
1203
+ if (params.workflowId != null && String(signal.payload?.workflow_id ?? "").trim() !== params.workflowId) return false;
1204
+ if (params.workflowPath != null && String(signal.payload?.workflow_path ?? "").trim() !== params.workflowPath) return false;
1177
1205
  return true;
1178
1206
  }
1179
1207
  // scope=latest: the PR's run (by subject owner/repo#pr or payload pr_number).