@botbuddy/cli 1.59.8 → 1.59.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/botbuddy.mjs +10 -1
- package/daemon/bin/darwin-arm64/botbuddy-daemon +0 -0
- package/daemon/bin/linux-x64/botbuddy-daemon +0 -0
- package/daemon-release/darwin-arm64.json +4 -4
- package/daemon-release/linux-x64.json +4 -4
- package/daemon-release/manifest.json +5 -5
- package/package.json +1 -1
- package/src/api.mjs +7 -4
- package/src/config.mjs +2 -2
- package/src/fixtures/bot-2600-update-command-run-rejections-2026-10-20.json +13 -0
- package/src/run-delivery.mjs +496 -0
- package/src/run.mjs +79 -29
- package/src/wait-core.mjs +41 -13
package/bin/botbuddy.mjs
CHANGED
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { run } from "../src/commands.mjs";
|
|
3
|
+
import { ensureRunDelivery } from "../src/run-delivery.mjs";
|
|
3
4
|
|
|
4
|
-
|
|
5
|
+
const argv = process.argv.slice(2);
|
|
6
|
+
|
|
7
|
+
// BOT-2600: any `bb` invocation starts the single per-machine delivery process
|
|
8
|
+
// when finished `bb run` commands still have an unsent terminal update. A stat
|
|
9
|
+
// and two small reads when nothing changed; it never fails the command.
|
|
10
|
+
const deliveryProcessArgs = argv[0] === "run" && (argv[1] === "--worker" || argv[1] === "--deliver");
|
|
11
|
+
const delivery = deliveryProcessArgs ? Promise.resolve() : ensureRunDelivery().catch(() => {});
|
|
12
|
+
|
|
13
|
+
delivery.then(() => run(argv)).then((code) => {
|
|
5
14
|
if (typeof code === "number") process.exitCode = code;
|
|
6
15
|
}).catch((err) => {
|
|
7
16
|
console.error(`\x1b[31m✗\x1b[0m ${err?.message || err}`);
|
|
Binary file
|
|
Binary file
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"release_version": "1.59.
|
|
3
|
-
"source_commit": "
|
|
4
|
-
"source_sha256": "
|
|
2
|
+
"release_version": "1.59.10",
|
|
3
|
+
"source_commit": "0a69b9e8cf14b345f8cc9e23e8ed68a8f240147c",
|
|
4
|
+
"source_sha256": "126242299e994c1e88bf377dbc0046672b5cdfe3c1454657f06abb64275b0301",
|
|
5
5
|
"target": "darwin-arm64",
|
|
6
|
-
"sha256": "
|
|
6
|
+
"sha256": "8c41e97c983e235c710b8624849998f0e231cdab27d7147499e5ba6e4e7e5abb"
|
|
7
7
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"release_version": "1.59.
|
|
3
|
-
"source_commit": "
|
|
4
|
-
"source_sha256": "
|
|
2
|
+
"release_version": "1.59.10",
|
|
3
|
+
"source_commit": "0a69b9e8cf14b345f8cc9e23e8ed68a8f240147c",
|
|
4
|
+
"source_sha256": "126242299e994c1e88bf377dbc0046672b5cdfe3c1454657f06abb64275b0301",
|
|
5
5
|
"target": "linux-x64",
|
|
6
|
-
"sha256": "
|
|
6
|
+
"sha256": "92a173432e8a40e2ed5d91cd2a93a5fd0cfcd23c63b56ecec42d53127e272b7a"
|
|
7
7
|
}
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schema_version": 1,
|
|
3
|
-
"release_version": "1.59.
|
|
4
|
-
"source_commit": "
|
|
5
|
-
"source_sha256": "
|
|
3
|
+
"release_version": "1.59.10",
|
|
4
|
+
"source_commit": "0a69b9e8cf14b345f8cc9e23e8ed68a8f240147c",
|
|
5
|
+
"source_sha256": "126242299e994c1e88bf377dbc0046672b5cdfe3c1454657f06abb64275b0301",
|
|
6
6
|
"targets": {
|
|
7
7
|
"darwin-arm64": {
|
|
8
8
|
"goos": "darwin",
|
|
9
9
|
"goarch": "arm64",
|
|
10
10
|
"binary": "daemon/bin/darwin-arm64/botbuddy-daemon",
|
|
11
|
-
"sha256": "
|
|
11
|
+
"sha256": "8c41e97c983e235c710b8624849998f0e231cdab27d7147499e5ba6e4e7e5abb",
|
|
12
12
|
"mode": "0755"
|
|
13
13
|
},
|
|
14
14
|
"linux-x64": {
|
|
15
15
|
"goos": "linux",
|
|
16
16
|
"goarch": "amd64",
|
|
17
17
|
"binary": "daemon/bin/linux-x64/botbuddy-daemon",
|
|
18
|
-
"sha256": "
|
|
18
|
+
"sha256": "92a173432e8a40e2ed5d91cd2a93a5fd0cfcd23c63b56ecec42d53127e272b7a",
|
|
19
19
|
"mode": "0755"
|
|
20
20
|
}
|
|
21
21
|
}
|
package/package.json
CHANGED
package/src/api.mjs
CHANGED
|
@@ -385,9 +385,9 @@ export const SESSION_TOKEN_DENIED_TOOLS = new Set([
|
|
|
385
385
|
// call to a specific tenant by appending ?tenant=<slug> to the endpoint (the
|
|
386
386
|
// same mechanism as the documented BOTBUDDY_SERVER_URL override), letting the
|
|
387
387
|
// server resolve/confirm exactly that tenant.
|
|
388
|
-
function toolEndpoint(tenant) {
|
|
389
|
-
if (!tenant) return
|
|
390
|
-
const url = new URL(
|
|
388
|
+
function toolEndpoint(tenant, base = SERVER_URL) {
|
|
389
|
+
if (!tenant) return base;
|
|
390
|
+
const url = new URL(base);
|
|
391
391
|
url.searchParams.set("tenant", tenant);
|
|
392
392
|
return url.toString();
|
|
393
393
|
}
|
|
@@ -457,6 +457,9 @@ export async function registerContainerHost(body, {
|
|
|
457
457
|
export async function callToolJson(toolName, args = {}, {
|
|
458
458
|
fetchImpl = fetch, signal, auth: pinnedAuth = null, tenant = null, resolvePin = resolveDefaultTenantPin,
|
|
459
459
|
requireSessionToken = false, resolveAuth = resolveCallAuth,
|
|
460
|
+
// BOT-2600: a delivery for a sealed run goes to the endpoint the run was
|
|
461
|
+
// registered with, not this process's ambient one.
|
|
462
|
+
serverUrl = SERVER_URL,
|
|
460
463
|
} = {}) {
|
|
461
464
|
let auth = pinnedAuth;
|
|
462
465
|
let sessionTokenSelected = false;
|
|
@@ -481,7 +484,7 @@ export async function callToolJson(toolName, args = {}, {
|
|
|
481
484
|
const body = { jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: toolName, arguments: args } };
|
|
482
485
|
let res;
|
|
483
486
|
try {
|
|
484
|
-
res = await fetchImpl(toolEndpoint(pin), {
|
|
487
|
+
res = await fetchImpl(toolEndpoint(pin, serverUrl), {
|
|
485
488
|
method: "POST",
|
|
486
489
|
headers: { "Content-Type": "application/json", ...auth },
|
|
487
490
|
body: JSON.stringify(body),
|
package/src/config.mjs
CHANGED
|
@@ -9,8 +9,8 @@ const CONFIG_FILE = join(CONFIG_DIR, "config.json");
|
|
|
9
9
|
// BOT-876: overridable so the CLI can be pointed at a local Supabase stack
|
|
10
10
|
// (`BOTBUDDY_SERVER_URL=http://127.0.0.1:56321/functions/v1/mcp-server`).
|
|
11
11
|
// Production is the default; nothing changes for normal use.
|
|
12
|
-
export const
|
|
13
|
-
|
|
12
|
+
export const DEFAULT_SERVER_URL = "https://api.bot-buddy.ai/functions/v1/mcp-server";
|
|
13
|
+
export const SERVER_URL = process.env.BOTBUDDY_SERVER_URL || DEFAULT_SERVER_URL;
|
|
14
14
|
|
|
15
15
|
let config = {};
|
|
16
16
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"source": "production MCP (api.bot-buddy.ai) via cli/src/api.mjs callToolJson, CLI worktree of BOT-2600",
|
|
3
|
+
"captured_at": "2026-10-20T19:15:53.070Z",
|
|
4
|
+
"note": "update_command_run for a random run id. Both requests are refused before any write; no credential value is recorded.",
|
|
5
|
+
"invalid_run_credential": {
|
|
6
|
+
"request": "x-agent-api-key: the caller's live session token; run_credential: 64 random hex",
|
|
7
|
+
"result": { "ok": true, "data": { "error": "run_credential_invalid" }, "isError": true }
|
|
8
|
+
},
|
|
9
|
+
"unknown_session_token": {
|
|
10
|
+
"request": "x-agent-api-key: bb_sess_ + 64 zeros",
|
|
11
|
+
"result": { "ok": false, "auth": true, "status": 401, "error": "unauthorized", "error_code": "session_token_revoked" }
|
|
12
|
+
}
|
|
13
|
+
}
|
|
@@ -0,0 +1,496 @@
|
|
|
1
|
+
// BOT-2600 — bounded terminal delivery for durable `bb run` commands.
|
|
2
|
+
//
|
|
3
|
+
// A detached `bb run` worker owns its workload and writes exactly one receipt.
|
|
4
|
+
// Once that receipt is durable, publishing it (update_command_run) is only a
|
|
5
|
+
// delivery problem, and it must not keep a resident Node process per run alive:
|
|
6
|
+
// on 2026-10-20 one Mac had 142 finished workers (6 GB) retrying rejections
|
|
7
|
+
// that could never succeed. So:
|
|
8
|
+
//
|
|
9
|
+
// - the worker retries transient failures within a bounded monotonic budget
|
|
10
|
+
// and parks permanent rejections immediately, recording
|
|
11
|
+
// `terminal_update.parked` in its private context, then exits;
|
|
12
|
+
// - a single per-machine delivery process (`bb run --deliver`, guarded by a
|
|
13
|
+
// PID lock under ~/.botbuddy/) retries what is still retryable and exits
|
|
14
|
+
// when nothing retryable remains. Any `bb` invocation starts it on demand;
|
|
15
|
+
// - workers left by older CLI versions (which retry forever) are adopted:
|
|
16
|
+
// their delivery is taken over and the worker is stopped only when it is
|
|
17
|
+
// provably finished (exact command line, same user, terminal receipt, no
|
|
18
|
+
// child processes, already in its terminal retry loop).
|
|
19
|
+
//
|
|
20
|
+
// The receipt and context files stay authoritative; nothing is deleted except a
|
|
21
|
+
// context whose delivery was accepted (exactly what the worker always did).
|
|
22
|
+
// The identity model is unchanged (BOT-2052): a delivery is only ever sent with
|
|
23
|
+
// the session token sealed for the run's owner, never an ambient credential.
|
|
24
|
+
|
|
25
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
26
|
+
import { chmod, link, mkdir, readdir, readFile, rename, stat, unlink, writeFile } from "node:fs/promises";
|
|
27
|
+
import { execFile, spawn } from "node:child_process";
|
|
28
|
+
import { dirname, join } from "node:path";
|
|
29
|
+
import { homedir } from "node:os";
|
|
30
|
+
import { fileURLToPath } from "node:url";
|
|
31
|
+
import { performance } from "node:perf_hooks";
|
|
32
|
+
import { AGENT_KEY_RE } from "./agent-key.mjs";
|
|
33
|
+
import { DEFAULT_SERVER_URL } from "./config.mjs";
|
|
34
|
+
|
|
35
|
+
/** The worker's own retry budget for transient failures (AC-1: ≤ 10 minutes). */
|
|
36
|
+
export const TERMINAL_RETRY_BUDGET_MS = 10 * 60_000;
|
|
37
|
+
/** One update_command_run attempt never waits longer than this. */
|
|
38
|
+
export const DELIVERY_CALL_TIMEOUT_MS = 30_000;
|
|
39
|
+
/** Transient attempts the delivery process makes before parking for good. */
|
|
40
|
+
export const DELIVERY_MAX_ATTEMPTS = 50;
|
|
41
|
+
/** The delivery process exits after this long even with retryable work left. */
|
|
42
|
+
export const DELIVERY_LIFETIME_MS = 30 * 60_000;
|
|
43
|
+
/** A non-forced `bb` invocation re-scans at most this often. */
|
|
44
|
+
export const DELIVERY_RESCAN_COOLDOWN_MS = 60_000;
|
|
45
|
+
const DELIVERY_FIRST_BACKOFF_MS = 30_000;
|
|
46
|
+
const DELIVERY_MAX_BACKOFF_MS = 5 * 60_000;
|
|
47
|
+
const STOP_GRACE_MS = 3_000;
|
|
48
|
+
|
|
49
|
+
export const TERMINAL_RECEIPT_STATUSES = new Set(["success", "failure", "cancelled", "timed_out", "owner_lost"]);
|
|
50
|
+
const API_STATUS = { success: "succeeded", failure: "failed", cancelled: "cancelled", timed_out: "timed_out", owner_lost: "owner_lost" };
|
|
51
|
+
const MAX_CAPTURE_BYTES = 8_192;
|
|
52
|
+
// The update_command_run arguments a persisted terminal_update may carry.
|
|
53
|
+
const UPDATE_FIELDS = ["run_id", "run_credential", "status", "exit_code", "invalidated_reason", "receipt_path", "receipt_content_type", "receipt_byte_size", "receipt_sha256", "output_characters", "truncated"];
|
|
54
|
+
|
|
55
|
+
// Never retried: decided locally without a request reaching the server (no
|
|
56
|
+
// usable session token), or a definitive server state (a different terminal
|
|
57
|
+
// status is already recorded; the payload itself is refused).
|
|
58
|
+
const PERMANENT_ERRORS = new Set([
|
|
59
|
+
"session_token_required", "session_token_malformed", "not_authenticated", "token_expired",
|
|
60
|
+
"command_terminal_immutable", "artifact_required",
|
|
61
|
+
]);
|
|
62
|
+
// Trust boundary for server rejections. The server can answer with an auth or
|
|
63
|
+
// credential rejection when its own lookup failed transiently (it discards the
|
|
64
|
+
// read error: BOT-2611), so none of these is proof that a retry cannot
|
|
65
|
+
// succeed. The worker parks every one at once and exits (it never spends its
|
|
66
|
+
// retry budget on them), and the delivery process gives each a few spaced
|
|
67
|
+
// attempts before parking it for good, so a recovered server recovers the
|
|
68
|
+
// terminal result while a dead credential costs at most a handful of calls.
|
|
69
|
+
const AMBIGUOUS_ERRORS = new Set([
|
|
70
|
+
"agent_identity_required", "authentication_required", "unauthorized", "run_credential_invalid",
|
|
71
|
+
"session_token_revoked", "credential_unknown",
|
|
72
|
+
]);
|
|
73
|
+
/** Delivery-process attempts for an ambiguous rejection before parking for good. */
|
|
74
|
+
export const DELIVERY_AMBIGUOUS_MAX_ATTEMPTS = 5;
|
|
75
|
+
const LOCAL_ENVIRONMENT_RE =/uv_os_get_passwd|\bENOENT\b|\bEACCES\b|\bEPERM\b/;
|
|
76
|
+
|
|
77
|
+
export function commandRunsDir(home = homedir()) { return join(home, ".botbuddy", "command-runs"); }
|
|
78
|
+
export function deliveryLockPath(home = homedir()) { return join(home, ".botbuddy", "run-delivery.lock"); }
|
|
79
|
+
export function deliveryStatePath(home = homedir()) { return join(home, ".botbuddy", "run-delivery.json"); }
|
|
80
|
+
|
|
81
|
+
function errorText(result) {
|
|
82
|
+
const value = result?.data?.error ?? result?.error ?? result?.code ?? null;
|
|
83
|
+
return typeof value === "string" && value ? value : "terminal_update_rejected";
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function terminalUpdateAccepted(result) {
|
|
87
|
+
return Boolean(result?.ok && !result.isError && result.data?.error == null && result.data?.accepted !== false);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Classify a failed update_command_run attempt.
|
|
92
|
+
* `parkNow` marks a retryable failure the worker must not spend its own
|
|
93
|
+
* budget on: it parks at once and only the delivery process retries it.
|
|
94
|
+
* @returns {{ reason: string, retryable: boolean, kind: string, parkNow?: boolean }}
|
|
95
|
+
*/
|
|
96
|
+
export function classifyTerminalUpdateFailure(result, thrown = null) {
|
|
97
|
+
if (thrown) {
|
|
98
|
+
const message = thrown instanceof Error ? thrown.message : String(thrown);
|
|
99
|
+
if (thrown?.code === "ERR_SYSTEM_ERROR" || LOCAL_ENVIRONMENT_RE.test(message)) {
|
|
100
|
+
return { reason: message, retryable: false, kind: "local_environment" };
|
|
101
|
+
}
|
|
102
|
+
return { reason: message, retryable: true, kind: "transient" };
|
|
103
|
+
}
|
|
104
|
+
const reason = errorText(result);
|
|
105
|
+
const codes = [result?.data?.error, result?.error, result?.code, result?.error_code].filter((v) => typeof v === "string");
|
|
106
|
+
if (codes.some((code) => PERMANENT_ERRORS.has(code))) return { reason, retryable: false, kind: "permanent_rejection" };
|
|
107
|
+
if (result?.auth === true || codes.some((code) => AMBIGUOUS_ERRORS.has(code))) return { reason, retryable: true, kind: "ambiguous_rejection", parkNow: true };
|
|
108
|
+
if (codes.some((code) => code.startsWith("usage:"))) return { reason, retryable: false, kind: "local_environment" };
|
|
109
|
+
// Transport failures, timeouts, 5xx/429 and anything unrecognised stay
|
|
110
|
+
// retryable; the budget (worker) or attempt cap (delivery process) bounds them.
|
|
111
|
+
return { reason, retryable: true, kind: "transient" };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** The sealed owner's identity, never an ambient or machine-level credential. */
|
|
115
|
+
export function callWithContextIdentity(context, call) {
|
|
116
|
+
const token = sealedToken(context);
|
|
117
|
+
// Old contexts cannot safely fall back to the machine-level owner key. Ask
|
|
118
|
+
// the API layer to require a session instead; the retained receipt/context
|
|
119
|
+
// makes recovery possible without attributing a callback to the wrong agent.
|
|
120
|
+
return (name, args, extra = {}) => call(name, args, token
|
|
121
|
+
? { ...extra, auth: { "x-agent-api-key": token } }
|
|
122
|
+
: { ...extra, requireSessionToken: true });
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function sealedToken(context) {
|
|
126
|
+
return typeof context?.callback_session_token === "string" && AGENT_KEY_RE.test(context.callback_session_token)
|
|
127
|
+
? context.callback_session_token : null;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export async function writeJsonAtomic(path, value) {
|
|
131
|
+
await mkdir(dirname(path), { recursive: true });
|
|
132
|
+
// Write-then-rename so a concurrent reader never observes a torn file.
|
|
133
|
+
const tmp = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
134
|
+
await writeFile(tmp, JSON.stringify(value), { mode: 0o600 });
|
|
135
|
+
await chmod(tmp, 0o600);
|
|
136
|
+
await rename(tmp, path);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function readJson(path) {
|
|
140
|
+
try { return JSON.parse(await readFile(path, "utf8")); } catch (error) {
|
|
141
|
+
if (error?.code === "ENOENT") return null;
|
|
142
|
+
if (error instanceof SyntaxError) return undefined;
|
|
143
|
+
throw error;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function withTimeout(extra = {}) {
|
|
148
|
+
return { ...extra, signal: AbortSignal.timeout(DELIVERY_CALL_TIMEOUT_MS) };
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* The endpoint a run was registered with. A sealed token is only ever sent
|
|
153
|
+
* back there, whatever endpoint the delivery process itself was started with.
|
|
154
|
+
* Contexts written before BOT-2600 did not record it; the CLI's default
|
|
155
|
+
* (production) endpoint is the only one a context without an override could
|
|
156
|
+
* have been registered with by default, so it is used for those.
|
|
157
|
+
*/
|
|
158
|
+
export function contextServerUrl(context) {
|
|
159
|
+
return typeof context?.server_url === "string" && /^https?:\/\//.test(context.server_url) ? context.server_url : DEFAULT_SERVER_URL;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** One update_command_run attempt with the sealed identity and a bounded wait. */
|
|
163
|
+
export async function attemptTerminalUpdate(context, update, call) {
|
|
164
|
+
const identified = callWithContextIdentity(context, call);
|
|
165
|
+
try {
|
|
166
|
+
const result = await identified("update_command_run", update, withTimeout({ serverUrl: contextServerUrl(context) }));
|
|
167
|
+
if (terminalUpdateAccepted(result)) return { accepted: true, result };
|
|
168
|
+
return { accepted: false, result, failure: classifyTerminalUpdateFailure(result) };
|
|
169
|
+
} catch (error) {
|
|
170
|
+
return { accepted: false, result: null, failure: classifyTerminalUpdateFailure(null, error) };
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export function parkedRecord(failure, by) {
|
|
175
|
+
return { reason: failure.reason, retryable: failure.retryable, kind: failure.kind, by, parked_at: new Date().toISOString() };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Rebuild the terminal update from a durable receipt (a worker that crashed first). */
|
|
179
|
+
export async function terminalUpdateFromReceipt(context, receipt) {
|
|
180
|
+
const bytes = await readFile(context.receipt_path);
|
|
181
|
+
const outputCharacters = Number.isInteger(receipt.output_characters) ? receipt.output_characters : 0;
|
|
182
|
+
const reason = receipt.reason ?? (receipt.status === "timed_out" ? "timeout" : receipt.status === "owner_lost" ? "owner_lost" : undefined);
|
|
183
|
+
return {
|
|
184
|
+
run_id: context.run_id, run_credential: context.run_credential, status: API_STATUS[receipt.status],
|
|
185
|
+
exit_code: receipt.exit_code ?? undefined, invalidated_reason: reason ?? undefined,
|
|
186
|
+
receipt_path: context.receipt_upload, receipt_content_type: "application/json",
|
|
187
|
+
receipt_byte_size: bytes.length, receipt_sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
188
|
+
output_characters: outputCharacters, truncated: outputCharacters > MAX_CAPTURE_BYTES,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function updateFromPersisted(terminalUpdate) {
|
|
193
|
+
const update = {};
|
|
194
|
+
for (const key of UPDATE_FIELDS) if (terminalUpdate[key] !== undefined) update[key] = terminalUpdate[key];
|
|
195
|
+
return update;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// ── processes ────────────────────────────────────────────────────────────────
|
|
199
|
+
|
|
200
|
+
export function parseProcessList(text) {
|
|
201
|
+
const rows = [];
|
|
202
|
+
for (const line of String(text).split("\n")) {
|
|
203
|
+
const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s(.*)$/.exec(line);
|
|
204
|
+
if (match) rows.push({ pid: Number(match[1]), ppid: Number(match[2]), uid: Number(match[3]), args: match[4].trim() });
|
|
205
|
+
}
|
|
206
|
+
return rows;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export function listProcessesWithPs() {
|
|
210
|
+
return new Promise((resolve, reject) => {
|
|
211
|
+
execFile("ps", ["-ww", "-A", "-o", "pid=,ppid=,uid=,args="], { maxBuffer: 32 * 1024 * 1024 }, (error, stdout) => {
|
|
212
|
+
if (error) reject(error); else resolve(parseProcessList(stdout));
|
|
213
|
+
});
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
export function pidAlive(pid) {
|
|
218
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
219
|
+
try { process.kill(pid, 0); return true; } catch (error) { return error?.code === "EPERM"; }
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// Every process whose argument string names this context as its worker.
|
|
223
|
+
function workersFor(processes, contextFile) {
|
|
224
|
+
const suffix = ` run --worker ${contextFile}`;
|
|
225
|
+
return processes.filter((proc) => proc.args.endsWith(suffix) || proc.args.includes(`${suffix} `));
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// AC-4: exactly `<node> <…>botbuddy.mjs run --worker <this context>`, owned by
|
|
229
|
+
// this user, and with no child process. The receipt/terminal checks are the
|
|
230
|
+
// caller's, made on the same context read.
|
|
231
|
+
export function isStoppableLegacyWorker(proc, { contextFile, uid, processes }) {
|
|
232
|
+
const suffix = ` run --worker ${contextFile}`;
|
|
233
|
+
if (!proc.args.endsWith(suffix)) return false;
|
|
234
|
+
if (!/^\S+\s+\S*botbuddy\.mjs$/.test(proc.args.slice(0, -suffix.length))) return false;
|
|
235
|
+
if (proc.uid !== uid) return false;
|
|
236
|
+
return !processes.some((other) => other.ppid === proc.pid);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
async function stopLegacyWorker(pid, { contextFile, uid, listProcesses, killProcess, isAlive, sleep }) {
|
|
240
|
+
// Re-verify on a fresh listing right before signalling (PID reuse, a new child).
|
|
241
|
+
const fresh = await listProcesses();
|
|
242
|
+
const proc = fresh.find((candidate) => candidate.pid === pid);
|
|
243
|
+
if (!proc || !isStoppableLegacyWorker(proc, { contextFile, uid, processes: fresh })) return !proc;
|
|
244
|
+
killProcess(pid, "SIGTERM");
|
|
245
|
+
for (let waited = 0; waited < STOP_GRACE_MS; waited += 100) {
|
|
246
|
+
if (!isAlive(pid)) return true;
|
|
247
|
+
await sleep(100);
|
|
248
|
+
}
|
|
249
|
+
return !isAlive(pid);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// ── one drain pass ────────────────────────────────────────────────────────────
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Deliver every pending terminal update once.
|
|
256
|
+
* @returns {Promise<{ delivered: number, parked: number, pending: number, stopped: number, owned: number, wrote: boolean }>}
|
|
257
|
+
*/
|
|
258
|
+
export async function drainOnce({
|
|
259
|
+
home = homedir(), call, listProcesses = listProcessesWithPs, killProcess = (pid, signal) => process.kill(pid, signal),
|
|
260
|
+
isAlive = pidAlive, uid = process.getuid?.() ?? -1, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
261
|
+
listContexts = async () => (await readdir(commandRunsDir(home)).catch((error) => { if (error?.code === "ENOENT") return []; throw error; }))
|
|
262
|
+
.filter((name) => name.endsWith(".context.json")).sort(),
|
|
263
|
+
maxAttempts = DELIVERY_MAX_ATTEMPTS,
|
|
264
|
+
} = {}) {
|
|
265
|
+
const totals = { delivered: 0, parked: 0, pending: 0, stopped: 0, owned: 0, wrote: false };
|
|
266
|
+
const names = await listContexts();
|
|
267
|
+
if (!names.length) return totals;
|
|
268
|
+
let processes = null;
|
|
269
|
+
for (const name of names) {
|
|
270
|
+
const contextFile = join(commandRunsDir(home), name);
|
|
271
|
+
const context = await readJson(contextFile);
|
|
272
|
+
// Deleted by a successful delivery mid-scan, or unreadable: not ours.
|
|
273
|
+
if (!context || typeof context.run_id !== "string" || typeof context.receipt_path !== "string") continue;
|
|
274
|
+
// A pre-workload cancellation is owned by the cancel handoff path. A
|
|
275
|
+
// marker written by a cancel that raced workload start leaves the worker
|
|
276
|
+
// owning a normal terminal update, which stays eligible here.
|
|
277
|
+
if (!context.terminal_update && await readFile(`${contextFile}.cancelled`).then(() => true, () => false)) continue;
|
|
278
|
+
const receipt = await readJson(context.receipt_path);
|
|
279
|
+
// Still executing (no terminal receipt yet): never touched.
|
|
280
|
+
if (!receipt || receipt.run_id !== context.run_id || !TERMINAL_RECEIPT_STATUSES.has(receipt.status)) continue;
|
|
281
|
+
const persisted = context.terminal_update ?? null;
|
|
282
|
+
if (persisted?.parked && persisted.parked.retryable === false) continue;
|
|
283
|
+
processes ??= await listProcesses();
|
|
284
|
+
const workers = workersFor(processes, contextFile).filter((proc) => isAlive(proc.pid));
|
|
285
|
+
if (workers.length && !persisted?.parked) {
|
|
286
|
+
// A bounded (current) worker owns its delivery until it parks it. A
|
|
287
|
+
// legacy worker is adopted only once it is provably in its terminal
|
|
288
|
+
// retry loop and stopping it cannot touch running work.
|
|
289
|
+
const legacyLoop = persisted && persisted.retry_budget_ms === undefined;
|
|
290
|
+
const stoppable = legacyLoop && workers.every((proc) => isStoppableLegacyWorker(proc, { contextFile, uid, processes }));
|
|
291
|
+
if (!stoppable) { totals.owned += 1; continue; }
|
|
292
|
+
let allStopped = true;
|
|
293
|
+
for (const proc of workers) {
|
|
294
|
+
if (await stopLegacyWorker(proc.pid, { contextFile, uid, listProcesses, killProcess, isAlive, sleep })) totals.stopped += 1;
|
|
295
|
+
else allStopped = false;
|
|
296
|
+
}
|
|
297
|
+
processes = null;
|
|
298
|
+
if (!allStopped) { totals.owned += 1; continue; }
|
|
299
|
+
}
|
|
300
|
+
// Re-read: a stopped legacy worker may have rewritten the context last.
|
|
301
|
+
const current = await readJson(contextFile);
|
|
302
|
+
if (!current) continue;
|
|
303
|
+
const update = current.terminal_update ? updateFromPersisted(current.terminal_update) : await terminalUpdateFromReceipt(current, receipt);
|
|
304
|
+
const previous = current.terminal_update ?? {};
|
|
305
|
+
const attempts = (Number.isInteger(previous.attempts) ? previous.attempts : 0) + 1;
|
|
306
|
+
const deliveryAttempts = (Number.isInteger(previous.delivery_attempts) ? previous.delivery_attempts : 0) + 1;
|
|
307
|
+
if (!sealedToken(current)) {
|
|
308
|
+
// No sealed owner: a delivery from this process could only use an
|
|
309
|
+
// ambient identity, which BOT-2052 forbids. Park it visibly instead.
|
|
310
|
+
const failure = { reason: "callback_session_token_missing", retryable: false, kind: "callback_session_token_missing" };
|
|
311
|
+
await writeJsonAtomic(contextFile, { ...current, terminal_update: { ...previous, ...update, parked: parkedRecord(failure, "delivery") } });
|
|
312
|
+
totals.parked += 1; totals.wrote = true;
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
const outcome = await attemptTerminalUpdate(current, update, call);
|
|
316
|
+
if (outcome.accepted) {
|
|
317
|
+
try { await unlink(contextFile); } catch { /* already removed */ }
|
|
318
|
+
totals.delivered += 1; totals.wrote = true;
|
|
319
|
+
continue;
|
|
320
|
+
}
|
|
321
|
+
let failure = outcome.failure;
|
|
322
|
+
const cap = failure.kind === "ambiguous_rejection" ? Math.min(maxAttempts, DELIVERY_AMBIGUOUS_MAX_ATTEMPTS) : maxAttempts;
|
|
323
|
+
if (failure.retryable && deliveryAttempts >= cap) failure = { ...failure, retryable: false, kind: "delivery_attempts_exhausted" };
|
|
324
|
+
await writeJsonAtomic(contextFile, { ...current, terminal_update: {
|
|
325
|
+
...previous, ...update, attempts, delivery_attempts: deliveryAttempts, last_error: failure.reason,
|
|
326
|
+
parked: failure.retryable ? { ...(previous.parked ?? parkedRecord(failure, "delivery")), reason: failure.reason, retryable: true } : parkedRecord(failure, "delivery"),
|
|
327
|
+
} });
|
|
328
|
+
totals.wrote = true;
|
|
329
|
+
if (failure.retryable) totals.pending += 1; else totals.parked += 1;
|
|
330
|
+
}
|
|
331
|
+
return totals;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// ── the per-machine lock ──────────────────────────────────────────────────────
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* Take the per-machine delivery lock. A lock whose holder PID is gone is stale
|
|
338
|
+
* and is taken over.
|
|
339
|
+
*/
|
|
340
|
+
export async function acquireDeliveryLock({ home = homedir(), pid = process.pid, isAlive = pidAlive } = {}) {
|
|
341
|
+
const lockFile = deliveryLockPath(home);
|
|
342
|
+
await mkdir(dirname(lockFile), { recursive: true });
|
|
343
|
+
const candidate = `${lockFile}.${pid}.${randomUUID()}.tmp`;
|
|
344
|
+
await writeFile(candidate, JSON.stringify({ pid, started_at: new Date().toISOString() }), { mode: 0o600 });
|
|
345
|
+
try {
|
|
346
|
+
for (let round = 0; round < 3; round++) {
|
|
347
|
+
try {
|
|
348
|
+
await link(candidate, lockFile); // atomic create-if-absent
|
|
349
|
+
return { acquired: true, holderPid: pid, release: async () => {
|
|
350
|
+
const holder = await readJson(lockFile);
|
|
351
|
+
if (holder?.pid === pid) await unlink(lockFile).catch(() => {});
|
|
352
|
+
} };
|
|
353
|
+
} catch (error) {
|
|
354
|
+
if (error?.code !== "EEXIST") throw error;
|
|
355
|
+
}
|
|
356
|
+
const holder = await readJson(lockFile);
|
|
357
|
+
if (holder === null) continue; // released between link and read
|
|
358
|
+
const holderPid = holder?.pid;
|
|
359
|
+
if (Number.isInteger(holderPid) && isAlive(holderPid)) return { acquired: false, holderPid, release: async () => {} };
|
|
360
|
+
// Stale. Only the holder of the reclaim mutex may remove the lock file,
|
|
361
|
+
// so it can never move aside a lock another contender just acquired:
|
|
362
|
+
// while the lock file exists, nobody else can replace it (link fails),
|
|
363
|
+
// and no other reclaimer can run, so what we re-read is what we remove.
|
|
364
|
+
const reclaim = await acquireReclaimMutex(lockFile, candidate, isAlive);
|
|
365
|
+
if (!reclaim) return { acquired: false, holderPid: null, release: async () => {} };
|
|
366
|
+
try {
|
|
367
|
+
const current = await readJson(lockFile);
|
|
368
|
+
if (current && !(Number.isInteger(current.pid) && !isAlive(current.pid))) {
|
|
369
|
+
return { acquired: false, holderPid: current?.pid ?? null, release: async () => {} };
|
|
370
|
+
}
|
|
371
|
+
if (current) await unlink(lockFile).catch((error) => { if (error?.code !== "ENOENT") throw error; });
|
|
372
|
+
} finally {
|
|
373
|
+
await reclaim.release();
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
const holder = await readJson(lockFile);
|
|
377
|
+
return { acquired: false, holderPid: holder?.pid ?? null, release: async () => {} };
|
|
378
|
+
} finally {
|
|
379
|
+
await unlink(candidate).catch(() => {});
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// The reclaim mutex serialises stale-lock removal. Its critical section is a
|
|
384
|
+
// read and an unlink, so a reclaim file left by a crashed contender is taken
|
|
385
|
+
// over only once its holder is dead AND it is older than RECLAIM_STALE_MS.
|
|
386
|
+
// Trust boundary: two contenders racing to clear a crashed reclaimer's mutex
|
|
387
|
+
// in the same instant is not defended further (it needs a crash inside a
|
|
388
|
+
// sub-millisecond section plus concurrent contention).
|
|
389
|
+
const RECLAIM_STALE_MS = 10_000;
|
|
390
|
+
async function acquireReclaimMutex(lockFile, candidate, isAlive) {
|
|
391
|
+
const mutex = `${lockFile}.reclaim`;
|
|
392
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
393
|
+
try {
|
|
394
|
+
await link(candidate, mutex);
|
|
395
|
+
return { release: () => unlink(mutex).catch(() => {}) };
|
|
396
|
+
} catch (error) {
|
|
397
|
+
if (error?.code !== "EEXIST") throw error;
|
|
398
|
+
}
|
|
399
|
+
const holder = await readJson(mutex);
|
|
400
|
+
if (holder === null) continue;
|
|
401
|
+
let ageMs = 0;
|
|
402
|
+
try { ageMs = Date.now() - (await stat(mutex)).mtimeMs; } catch { continue; }
|
|
403
|
+
const dead = !(Number.isInteger(holder?.pid) && isAlive(holder.pid));
|
|
404
|
+
if (!dead || ageMs < RECLAIM_STALE_MS) return null; // another contender is reclaiming
|
|
405
|
+
await unlink(mutex).catch(() => {});
|
|
406
|
+
}
|
|
407
|
+
return null;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
async function dirMtimeMs(home) {
|
|
411
|
+
try { return (await stat(commandRunsDir(home))).mtimeMs; } catch (error) {
|
|
412
|
+
if (error?.code === "ENOENT") return null;
|
|
413
|
+
throw error;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* The delivery process: hold the lock, drain until nothing retryable remains
|
|
419
|
+
* (or the lifetime cap), record the settled directory state, release, exit.
|
|
420
|
+
*/
|
|
421
|
+
export async function runDelivery({
|
|
422
|
+
home = homedir(), call, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), now = () => performance.now(),
|
|
423
|
+
lifetimeMs = DELIVERY_LIFETIME_MS, ...drainOptions
|
|
424
|
+
} = {}) {
|
|
425
|
+
const lock = await acquireDeliveryLock({ home });
|
|
426
|
+
if (!lock.acquired) return { outcome: "held", holder_pid: lock.holderPid, delivered: 0, parked: 0, pending: 0, stopped: 0 };
|
|
427
|
+
const totals = { delivered: 0, parked: 0, pending: 0, stopped: 0, owned: 0 };
|
|
428
|
+
const started = now();
|
|
429
|
+
let backoff = DELIVERY_FIRST_BACKOFF_MS;
|
|
430
|
+
let settlePasses = 0;
|
|
431
|
+
try {
|
|
432
|
+
for (;;) {
|
|
433
|
+
const mtime = await dirMtimeMs(home);
|
|
434
|
+
const pass = await drainOnce({ home, call, sleep, ...drainOptions });
|
|
435
|
+
for (const key of ["delivered", "parked", "stopped"]) totals[key] += pass[key];
|
|
436
|
+
totals.pending = pass.pending;
|
|
437
|
+
totals.owned = pass.owned;
|
|
438
|
+
if (pass.pending === 0) {
|
|
439
|
+
// Our own writes move the directory mtime; one more pass confirms the
|
|
440
|
+
// directory is settled so `bb` invocations stop re-starting us.
|
|
441
|
+
if (pass.wrote && settlePasses++ < 3) continue;
|
|
442
|
+
// A delivery still owned by a live worker is not settled: if that
|
|
443
|
+
// worker dies before it writes again, the directory never changes, so
|
|
444
|
+
// leave the state open for a re-scan after the cooldown.
|
|
445
|
+
await writeJsonAtomic(deliveryStatePath(home), { dir_mtime_ms: pass.owned > 0 ? null : mtime, settled_at: new Date().toISOString(), owned: pass.owned });
|
|
446
|
+
return { outcome: "drained", ...totals };
|
|
447
|
+
}
|
|
448
|
+
settlePasses = 0;
|
|
449
|
+
if (now() - started + backoff > lifetimeMs) {
|
|
450
|
+
// Retryable work remains: leave the state unsettled so the next `bb`
|
|
451
|
+
// invocation starts a fresh delivery process.
|
|
452
|
+
await writeJsonAtomic(deliveryStatePath(home), { dir_mtime_ms: null, settled_at: null, pending: pass.pending });
|
|
453
|
+
return { outcome: "lifetime_exhausted", ...totals };
|
|
454
|
+
}
|
|
455
|
+
await sleep(backoff);
|
|
456
|
+
backoff = Math.min(backoff * 2, DELIVERY_MAX_BACKOFF_MS);
|
|
457
|
+
}
|
|
458
|
+
} finally {
|
|
459
|
+
await lock.release();
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* Called by every `bb` invocation (and by a worker that parked a retryable
|
|
465
|
+
* delivery): start the delivery process when the run directory changed since
|
|
466
|
+
* it last settled and no live process holds the lock. Cheap: a stat and two
|
|
467
|
+
* small reads. Never throws into the caller's command.
|
|
468
|
+
*/
|
|
469
|
+
export async function ensureRunDelivery({
|
|
470
|
+
home = homedir(), env = process.env, spawnImpl = spawn, isAlive = pidAlive, force = false, cooldownMs = DELIVERY_RESCAN_COOLDOWN_MS,
|
|
471
|
+
} = {}) {
|
|
472
|
+
try {
|
|
473
|
+
if (env.BOTBUDDY_RUN_DELIVERY === "off") return { started: false, reason: "disabled" };
|
|
474
|
+
const mtime = await dirMtimeMs(home);
|
|
475
|
+
if (mtime === null) return { started: false, reason: "no_runs" };
|
|
476
|
+
const state = await readJson(deliveryStatePath(home));
|
|
477
|
+
if (!force && state && state.dir_mtime_ms === mtime) return { started: false, reason: "up_to_date" };
|
|
478
|
+
// A worker still inside its own retry budget rewrites its context on every
|
|
479
|
+
// attempt. Re-scanning on each of those changes is wasted work: the worker
|
|
480
|
+
// starts us itself (force) when it parks, so otherwise scan at most once
|
|
481
|
+
// per cooldown. Both times are this machine's filesystem clock.
|
|
482
|
+
if (!force && state) {
|
|
483
|
+
const stateAge = Date.now() - (await stat(deliveryStatePath(home))).mtimeMs;
|
|
484
|
+
if (stateAge >= 0 && stateAge < cooldownMs) return { started: false, reason: "cooldown" };
|
|
485
|
+
}
|
|
486
|
+
const holder = await readJson(deliveryLockPath(home));
|
|
487
|
+
if (Number.isInteger(holder?.pid) && isAlive(holder.pid)) return { started: false, reason: "held" };
|
|
488
|
+
const bin = join(dirname(fileURLToPath(import.meta.url)), "..", "bin", "botbuddy.mjs");
|
|
489
|
+
const child = spawnImpl(process.execPath, [bin, "run", "--deliver"], { detached: true, stdio: "ignore", cwd: home, windowsHide: true });
|
|
490
|
+
child.on?.("error", () => { /* best effort: the next invocation retries */ });
|
|
491
|
+
child.unref?.();
|
|
492
|
+
return { started: true, reason: "pending" };
|
|
493
|
+
} catch (error) {
|
|
494
|
+
return { started: false, reason: `error:${error instanceof Error ? error.message : String(error)}` };
|
|
495
|
+
}
|
|
496
|
+
}
|
package/src/run.mjs
CHANGED
|
@@ -13,7 +13,9 @@ import { dirname, join } from "node:path";
|
|
|
13
13
|
import { homedir } from "node:os";
|
|
14
14
|
import { spawn } from "node:child_process";
|
|
15
15
|
import { fileURLToPath } from "node:url";
|
|
16
|
+
import { performance } from "node:perf_hooks";
|
|
16
17
|
import { callToolJson } from "./api.mjs";
|
|
18
|
+
import { SERVER_URL } from "./config.mjs";
|
|
17
19
|
import { parseLaneEvents, laneCaseCounts, flushLaneCases, buildLaneSummary, laneSummaryFilename } from "./test-lane-events.mjs";
|
|
18
20
|
import { AGENT_KEY_RE, readAgentSessionTokenEnv, legacyAgentAliasHint } from "./agent-key.mjs";
|
|
19
21
|
import { hasAgentSessionRuntime, isRejectedCachedMcpSession, resolveAgentSessionCredential, selfHealAgentSession } from "./agent-session.mjs";
|
|
@@ -23,6 +25,10 @@ import { createTelemetryOutbox } from "./telemetry-outbox.mjs";
|
|
|
23
25
|
import { deliverExecutionEvent } from "./telemetry-delivery.mjs";
|
|
24
26
|
import { attestTelemetryCredential, loadTelemetryIdentity } from "./telemetry-config.mjs";
|
|
25
27
|
import { VERSION } from "./version.mjs";
|
|
28
|
+
import {
|
|
29
|
+
TERMINAL_RETRY_BUDGET_MS, DELIVERY_CALL_TIMEOUT_MS, callWithContextIdentity, classifyTerminalUpdateFailure,
|
|
30
|
+
ensureRunDelivery, parkedRecord, runDelivery, terminalUpdateAccepted,
|
|
31
|
+
} from "./run-delivery.mjs";
|
|
26
32
|
|
|
27
33
|
export const RUN_SCHEMA_VERSION = 1;
|
|
28
34
|
export const EXIT = Object.freeze({ OK: 0, AUTH: 3, INVALID: 4, BACKEND: 5, INTERNAL: 7 });
|
|
@@ -405,7 +411,7 @@ export async function launchRun(argv, {
|
|
|
405
411
|
// every callback is attributable to the same agent. It deliberately stays
|
|
406
412
|
// outside child_env, receipts, and the workload's process environment.
|
|
407
413
|
const knownSession = opts.sessionId ?? managedCredential?.identity?.sessionId ?? managedCredential?.sessionId;
|
|
408
|
-
await writeJson(context, { schema_version: RUN_SCHEMA_VERSION, context_path: context, run_id: runId, run_credential: registration.data.run_credential, receipt_upload: registration.data.receipt_upload, command, cwd, timeout_seconds: opts.timeout, receipt_path: receipt, launched_at: new Date().toISOString(),
|
|
414
|
+
await writeJson(context, { schema_version: RUN_SCHEMA_VERSION, context_path: context, run_id: runId, run_credential: registration.data.run_credential, receipt_upload: registration.data.receipt_upload, command, cwd, timeout_seconds: opts.timeout, receipt_path: receipt, launched_at: new Date().toISOString(), server_url: SERVER_URL,...(opts.sessionToken ? { callback_session_token: opts.sessionToken } : {}), ...(knownSession ? { callback_session_id: knownSession } : {}), ...(childEnv && Object.keys(childEnv).length ? { child_env: childEnv } : {}), ...(testRun ? { test_run: testRun } : {}) });
|
|
409
415
|
const [file, args] = workerInvocation(context);
|
|
410
416
|
try {
|
|
411
417
|
const worker = spawnImpl(file, args, { detached: true, stdio: "ignore", windowsHide: true });
|
|
@@ -426,7 +432,10 @@ export async function runWorker(contextFile, {
|
|
|
426
432
|
call = callToolJson,
|
|
427
433
|
signalEmitter = process,
|
|
428
434
|
terminate = terminateProcessGroup,
|
|
435
|
+
// BOT-2600: the bounded terminal delivery (see retryTerminalUpdate).
|
|
436
|
+
sleep, now, terminalBudgetMs, startDelivery,
|
|
429
437
|
} = {}) {
|
|
438
|
+
const retry = { sleep, now, budgetMs: terminalBudgetMs, startDelivery };
|
|
430
439
|
const context = JSON.parse(await readFile(contextFile, "utf8"));
|
|
431
440
|
const callbackCall = callWithContextIdentity(context, call);
|
|
432
441
|
const ownerHeartbeat = startOwnerHeartbeat(context, callbackCall);
|
|
@@ -492,7 +501,7 @@ export async function runWorker(contextFile, {
|
|
|
492
501
|
child = spawnImpl(context.command[0], context.command.slice(1), { cwd: context.cwd, detached: true, stdio: ["ignore", "pipe", "pipe"], windowsHide: true, ...(context.child_env ? { env: { ...process.env, ...context.child_env } } : {}) });
|
|
493
502
|
} catch (error) {
|
|
494
503
|
const endedAt = new Date().toISOString();
|
|
495
|
-
const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${error instanceof Error ? error.message : String(error)}` }, callbackCall);
|
|
504
|
+
const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${error instanceof Error ? error.message : String(error)}` }, callbackCall, retry);
|
|
496
505
|
return receipt;
|
|
497
506
|
}
|
|
498
507
|
// A failed spawn can report through the child `error` event on the next turn.
|
|
@@ -523,11 +532,11 @@ export async function runWorker(contextFile, {
|
|
|
523
532
|
if (lane) await lane.stop();
|
|
524
533
|
const endedAt = new Date().toISOString();
|
|
525
534
|
if (result.error) {
|
|
526
|
-
const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${result.error.message}`, output: capture.output, outputCharacters: capture.characters }, callbackCall);
|
|
535
|
+
const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${result.error.message}`, output: capture.output, outputCharacters: capture.characters }, callbackCall, retry);
|
|
527
536
|
return receipt;
|
|
528
537
|
}
|
|
529
538
|
const status = terminalStatus({ ...result, timedOut });
|
|
530
|
-
const receipt = await finalize(context, { status, startedAt, endedAt, exitCode: result.exitCode, signal: result.signal, output: capture.output, outputCharacters: capture.characters }, callbackCall);
|
|
539
|
+
const receipt = await finalize(context, { status, startedAt, endedAt, exitCode: result.exitCode, signal: result.signal, output: capture.output, outputCharacters: capture.characters }, callbackCall, retry);
|
|
531
540
|
return receipt;
|
|
532
541
|
} finally {
|
|
533
542
|
if (ownerHeartbeat) await ownerHeartbeat.stop();
|
|
@@ -556,7 +565,14 @@ function startOwnerHeartbeat(context, call, { intervalMs = 5 * 60_000, setInterv
|
|
|
556
565
|
tick();
|
|
557
566
|
const timer = setIntervalFn(tick, intervalMs);
|
|
558
567
|
timer.unref?.();
|
|
559
|
-
|
|
568
|
+
// BOT-2600: an in-flight heartbeat must never hold a finished worker open.
|
|
569
|
+
return { async stop() {
|
|
570
|
+
stopped = true;
|
|
571
|
+
clearInterval(timer);
|
|
572
|
+
let bound;
|
|
573
|
+
await Promise.race([chain, new Promise((resolve) => { bound = setTimeout(resolve, 2_000); bound.unref?.(); })]);
|
|
574
|
+
clearTimeout(bound);
|
|
575
|
+
} };
|
|
560
576
|
}
|
|
561
577
|
|
|
562
578
|
// Periodic lane flusher: reduce the NDJSON events file to the current case state
|
|
@@ -785,7 +801,7 @@ export async function cancelRun(runId, {
|
|
|
785
801
|
return { exitCode: EXIT.OK, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "cancellation_requested", run_id: runId, exit_code: EXIT.OK } };
|
|
786
802
|
}
|
|
787
803
|
|
|
788
|
-
async function finalize(context, state, call) {
|
|
804
|
+
async function finalize(context, state, call, retry = {}) {
|
|
789
805
|
const boundedLog = String(state.output ?? "").slice(-8192);
|
|
790
806
|
const outputCharacters = state.outputCharacters ?? String(state.output ?? "").length;
|
|
791
807
|
const receipt = { schema_version: RUN_SCHEMA_VERSION, run_id: context.run_id, status: state.status, started_at: state.startedAt, ended_at: state.endedAt, exit_code: state.exitCode, signal: state.signal, recovery_command: state.status === "failure" ? `botbuddy run --session-id <new-session-id> --environment local -- ${context.command.map((x) => JSON.stringify(x)).join(" ")}` : null, bounded_log: boundedLog, output_characters: outputCharacters, reason: state.reason ?? null };
|
|
@@ -795,43 +811,77 @@ async function finalize(context, state, call) {
|
|
|
795
811
|
const apiStatus = ({ success: "succeeded", failure: "failed", cancelled: "cancelled", timed_out: "timed_out", owner_lost: "owner_lost" })[state.status];
|
|
796
812
|
const update = { run_id: context.run_id, run_credential: context.run_credential, status: apiStatus, exit_code: state.exitCode ?? undefined, invalidated_reason: state.reason ?? (state.status === "timed_out" ? "timeout" : state.status === "owner_lost" ? "owner_lost" : undefined), receipt_path: context.receipt_upload, receipt_content_type: "application/json", receipt_byte_size: bytes.size, receipt_sha256: sha256, output_characters: outputCharacters, truncated: outputCharacters > MAX_CAPTURE_BYTES };
|
|
797
813
|
// A locally durable receipt is enough to publish the linked test run. Do it
|
|
798
|
-
// before the command callback's
|
|
799
|
-
//
|
|
800
|
-
//
|
|
814
|
+
// before the command callback's retry: otherwise an auth outage (for example
|
|
815
|
+
// agent_identity_required) strands a 0-case run in active state even though
|
|
816
|
+
// its command has already failed locally.
|
|
801
817
|
if (context.test_run) await completeTestRun(context, { status: state.status, exitCode: state.exitCode, signal: state.signal, startedAt: state.startedAt, endedAt: state.endedAt, call });
|
|
802
|
-
await retryTerminalUpdate(context, update, call);
|
|
803
|
-
|
|
818
|
+
const delivery = await retryTerminalUpdate(context, update, call, retry);
|
|
819
|
+
// A parked delivery keeps its context: it is the durable record the
|
|
820
|
+
// per-machine delivery process (and an operator) reads.
|
|
821
|
+
if (delivery.delivered) {
|
|
822
|
+
try { await unlink(context.context_path); } catch { /* receipt is authoritative; context cleanup is best effort */ }
|
|
823
|
+
}
|
|
804
824
|
return receipt;
|
|
805
825
|
}
|
|
806
826
|
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
827
|
+
// BOT-2600: the terminal callback is bounded. The worker that owns the receipt
|
|
828
|
+
// retries transient failures (transport, timeout, 5xx) within a monotonic
|
|
829
|
+
// budget of at most ten minutes and parks a permanent rejection (no owning
|
|
830
|
+
// identity, an invalid run credential, a local environment error) at once.
|
|
831
|
+
// Either way it records `terminal_update.parked` in the private context and
|
|
832
|
+
// returns so the worker process exits; a retryable parked delivery is handed to
|
|
833
|
+
// the single per-machine delivery process (`bb run --deliver`).
|
|
834
|
+
export async function retryTerminalUpdate(context, update, call, {
|
|
835
|
+
sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
836
|
+
now = () => performance.now(),
|
|
837
|
+
budgetMs = TERMINAL_RETRY_BUDGET_MS,
|
|
838
|
+
startDelivery = () => ensureRunDelivery({ force: true }),
|
|
839
|
+
} = {}) {
|
|
840
|
+
const started = now();
|
|
820
841
|
let attempts = 0;
|
|
821
842
|
let delay = 1_000;
|
|
822
843
|
for (;;) {
|
|
823
|
-
let result;
|
|
824
|
-
|
|
825
|
-
|
|
844
|
+
let result = null;
|
|
845
|
+
let failure;
|
|
846
|
+
try {
|
|
847
|
+
result = await call("update_command_run", update, {
|
|
848
|
+
signal: AbortSignal.timeout(DELIVERY_CALL_TIMEOUT_MS),
|
|
849
|
+
// The endpoint this run was registered with (BOT-2600; sealed at launch).
|
|
850
|
+
...(typeof context.server_url === "string" && context.server_url ? { serverUrl: context.server_url } : {}),
|
|
851
|
+
});
|
|
852
|
+
if (terminalUpdateAccepted(result)) return { delivered: true, attempts: attempts + 1, result };
|
|
853
|
+
failure = classifyTerminalUpdateFailure(result);
|
|
854
|
+
} catch (error) {
|
|
855
|
+
failure = classifyTerminalUpdateFailure(null, error);
|
|
856
|
+
}
|
|
826
857
|
attempts += 1;
|
|
827
|
-
|
|
858
|
+
const exhausted = failure.retryable && !failure.parkNow && now() - started + delay > budgetMs;
|
|
859
|
+
if (!failure.retryable || failure.parkNow || exhausted) {
|
|
860
|
+
const parked = parkedRecord(exhausted ? { ...failure, kind: "retry_budget_exhausted" } : failure, "worker");
|
|
861
|
+
await writeJson(context.context_path, { ...context, terminal_update: { ...update, attempts, last_error: failure.reason, retry_budget_ms: budgetMs, parked } });
|
|
862
|
+
if (parked.retryable) { try { await startDelivery(); } catch { /* the next bb invocation starts it */ } }
|
|
863
|
+
return { delivered: false, attempts, parked, result };
|
|
864
|
+
}
|
|
865
|
+
await writeJson(context.context_path, { ...context, terminal_update: { ...update, attempts, last_error: failure.reason, retry_budget_ms: budgetMs, next_retry_at: new Date(Date.now() + delay).toISOString() } });
|
|
828
866
|
await sleep(delay);
|
|
829
867
|
delay = Math.min(delay * 2, 60_000);
|
|
830
868
|
}
|
|
831
869
|
}
|
|
832
870
|
|
|
833
871
|
export async function cmdRun(args) {
|
|
834
|
-
if (args[0] === "--worker") {
|
|
872
|
+
if (args[0] === "--worker") {
|
|
873
|
+
await runWorker(args[1]);
|
|
874
|
+
// BOT-2600: the receipt is durable and its delivery is accepted or parked.
|
|
875
|
+
// Nothing may keep a finished worker resident; an unref'd guard exits even
|
|
876
|
+
// if a stray handle (a keep-alive socket) would otherwise hold the loop.
|
|
877
|
+
setTimeout(() => process.exit(process.exitCode ?? 0), 1_000).unref();
|
|
878
|
+
return;
|
|
879
|
+
}
|
|
880
|
+
if (args[0] === "--deliver") {
|
|
881
|
+
const result = await runDelivery({ call: callToolJson });
|
|
882
|
+
if (args.includes("--json")) oneLine({ schema_version: RUN_SCHEMA_VERSION, ...result }, false);
|
|
883
|
+
return;
|
|
884
|
+
}
|
|
835
885
|
if (args[0] === "cancel") {
|
|
836
886
|
const result = await cancelRun(args[1] ?? "");
|
|
837
887
|
oneLine(result.receipt, false);
|
package/src/wait-core.mjs
CHANGED
|
@@ -287,9 +287,10 @@ const VALIDATORS = {
|
|
|
287
287
|
// BOT-1507 — scope=next adds a fourth, LIVE-ONLY selector: it names no run or PR
|
|
288
288
|
// (they must already exist at arm time) and wakes on the first terminal CI run for
|
|
289
289
|
// the repo delivered AFTER the wait was armed — the post-merge / cron train case.
|
|
290
|
-
// Optional branch=/workflow=
|
|
291
|
-
//
|
|
292
|
-
//
|
|
290
|
+
// Optional branch=/workflow=/workflow_id=/workflow_path= filters narrow that
|
|
291
|
+
// first run; branch and stable workflow identities are valid only with
|
|
292
|
+
// scope=next. The legacy display-name filter also pins a level-triggered
|
|
293
|
+
// sha= recovery to one historical workflow.
|
|
293
294
|
// BOT-1685 — scope=next also accepts an optional sha= FILTER (distinct from the
|
|
294
295
|
// level-triggered sha= selector): a wait armed BEFORE re-dispatching a run for a
|
|
295
296
|
// known commit then wakes on the NEXT terminal run for THAT commit, instead of
|
|
@@ -309,9 +310,13 @@ const VALIDATORS = {
|
|
|
309
310
|
}
|
|
310
311
|
const hasBranchFilter = p.branch !== undefined;
|
|
311
312
|
const hasWorkflowFilter = p.workflow !== undefined;
|
|
313
|
+
const hasWorkflowIdFilter = p.workflow_id !== undefined;
|
|
314
|
+
const hasWorkflowPathFilter = p.workflow_path !== undefined;
|
|
312
315
|
if (p.scope !== undefined) {
|
|
313
316
|
if (p.scope === "latest") {
|
|
314
|
-
if (hasBranchFilter || hasWorkflowFilter
|
|
317
|
+
if (hasBranchFilter || hasWorkflowFilter || hasWorkflowIdFilter || hasWorkflowPathFilter) {
|
|
318
|
+
throw new Error("ci branch=/workflow=/workflow_id=/workflow_path= are only valid with scope=next");
|
|
319
|
+
}
|
|
315
320
|
if (p.pr === undefined || !/^\d+$/.test(String(p.pr).trim())) {
|
|
316
321
|
throw new Error("ci scope=latest needs pr=<number>");
|
|
317
322
|
}
|
|
@@ -329,12 +334,23 @@ const VALIDATORS = {
|
|
|
329
334
|
if (v === "") throw new Error(`ci ${key}= must be non-empty`);
|
|
330
335
|
return v;
|
|
331
336
|
};
|
|
332
|
-
|
|
337
|
+
const workflowId = filter("workflow_id");
|
|
338
|
+
const workflowPath = filter("workflow_path");
|
|
339
|
+
return {
|
|
340
|
+
repo,
|
|
341
|
+
scope: "next",
|
|
342
|
+
branch: filter("branch"),
|
|
343
|
+
workflow: filter("workflow"),
|
|
344
|
+
...(workflowId == null ? {} : { workflowId }),
|
|
345
|
+
...(workflowPath == null ? {} : { workflowPath }),
|
|
346
|
+
sha: filter("sha"),
|
|
347
|
+
pr: null,
|
|
348
|
+
};
|
|
333
349
|
}
|
|
334
350
|
throw new Error("ci scope must be 'latest' or 'next' (or use run_id=/sha=)");
|
|
335
351
|
}
|
|
336
|
-
if (hasBranchFilter || (hasWorkflowFilter && p.sha === undefined)) {
|
|
337
|
-
throw new Error("ci branch=
|
|
352
|
+
if (hasBranchFilter || hasWorkflowIdFilter || hasWorkflowPathFilter || (hasWorkflowFilter && p.sha === undefined)) {
|
|
353
|
+
throw new Error("ci branch=/workflow_id=/workflow_path= are only valid with scope=next; legacy workflow= also accepts an exact sha selector");
|
|
338
354
|
}
|
|
339
355
|
if (p.run_id !== undefined) {
|
|
340
356
|
if (!String(p.run_id).trim()) throw new Error("ci run_id must be non-empty");
|
|
@@ -343,7 +359,13 @@ const VALIDATORS = {
|
|
|
343
359
|
if (!String(p.sha).trim()) throw new Error("ci sha must be non-empty");
|
|
344
360
|
const workflow = p.workflow === undefined ? null : String(p.workflow).trim();
|
|
345
361
|
if (p.workflow !== undefined && workflow === "") throw new Error("ci workflow= must be non-empty");
|
|
346
|
-
return {
|
|
362
|
+
return {
|
|
363
|
+
repo,
|
|
364
|
+
scope: "sha",
|
|
365
|
+
sha: String(p.sha).trim(),
|
|
366
|
+
workflow,
|
|
367
|
+
pr: p.pr != null ? String(p.pr).trim() : null,
|
|
368
|
+
};
|
|
347
369
|
},
|
|
348
370
|
"staging-green"(p) {
|
|
349
371
|
const repo = p.repo != null ? String(p.repo).trim() : null;
|
|
@@ -1013,7 +1035,7 @@ export function formatUnknownWorkflowGuidance(detail, candidates) {
|
|
|
1013
1035
|
const named = Array.isArray(candidates) ? candidates.filter((c) => typeof c === "string" && c.trim() !== "") : [];
|
|
1014
1036
|
if (named.length === 0) return lines;
|
|
1015
1037
|
lines.push("closest workflow names this repo has actually reported:");
|
|
1016
|
-
for (const name of named) lines.push(` workflow=${name}`);
|
|
1038
|
+
for (const name of named) lines.push(` ${name.includes("=") ? name : `workflow=${name}`}`);
|
|
1017
1039
|
return lines;
|
|
1018
1040
|
}
|
|
1019
1041
|
|
|
@@ -1152,16 +1174,20 @@ function conditionMatchesSignal(condition, signal, waitSessionId) {
|
|
|
1152
1174
|
if (params.scope === "run_id") return String(signal.payload?.run_id) === params.runId;
|
|
1153
1175
|
if (params.scope === "sha") {
|
|
1154
1176
|
if (signal.payload?.head_sha !== params.sha) return false;
|
|
1155
|
-
if (params.workflow
|
|
1156
|
-
|
|
1157
|
-
|
|
1177
|
+
if (params.workflow != null) {
|
|
1178
|
+
const workflow = signal.payload?.workflow_name;
|
|
1179
|
+
if (typeof workflow !== "string" || workflow.trim().toLowerCase() !== params.workflow.trim().toLowerCase()) return false;
|
|
1180
|
+
}
|
|
1181
|
+
if (params.workflowId != null && String(signal.payload?.workflow_id ?? "").trim() !== params.workflowId) return false;
|
|
1182
|
+
if (params.workflowPath != null && String(signal.payload?.workflow_path ?? "").trim() !== params.workflowPath) return false;
|
|
1183
|
+
return true;
|
|
1158
1184
|
}
|
|
1159
1185
|
if (params.scope === "next") {
|
|
1160
1186
|
// BOT-1507 — live-only: the first TERMINAL run for the repo. Never keys on
|
|
1161
1187
|
// subject_key (it is run:<id> for post-merge runs and owner/repo#n for PR
|
|
1162
1188
|
// runs — both are legitimate wakes); a null/empty conclusion is an
|
|
1163
1189
|
// in-progress signal and never terminal. Optional branch/workflow filters:
|
|
1164
|
-
// branch
|
|
1190
|
+
// branch/path/id are exact; the display name is case/space-insensitive.
|
|
1165
1191
|
const conclusion = signal.payload?.conclusion;
|
|
1166
1192
|
if (typeof conclusion !== "string" || conclusion.trim() === "") return false;
|
|
1167
1193
|
// BOT-1685: an optional sha filter pins scope=next to a commit, so a
|
|
@@ -1174,6 +1200,8 @@ function conditionMatchesSignal(condition, signal, waitSessionId) {
|
|
|
1174
1200
|
return false;
|
|
1175
1201
|
}
|
|
1176
1202
|
}
|
|
1203
|
+
if (params.workflowId != null && String(signal.payload?.workflow_id ?? "").trim() !== params.workflowId) return false;
|
|
1204
|
+
if (params.workflowPath != null && String(signal.payload?.workflow_path ?? "").trim() !== params.workflowPath) return false;
|
|
1177
1205
|
return true;
|
|
1178
1206
|
}
|
|
1179
1207
|
// scope=latest: the PR's run (by subject owner/repo#pr or payload pr_number).
|