@botbuddy/cli 1.47.1 → 1.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/commands.mjs +20 -2
- package/src/helper-host.mjs +223 -0
- package/src/wait-core.mjs +18 -0
- package/src/wait.mjs +63 -19
package/package.json
CHANGED
package/src/commands.mjs
CHANGED
|
@@ -30,6 +30,7 @@ import { attestTelemetryCredential, loadTelemetryIdentity, loadTelemetryLocation
|
|
|
30
30
|
import { deliverExecutionEvent } from "./telemetry-delivery.mjs";
|
|
31
31
|
import { doctorAgentAuth, doctorHelp, parseDoctorArgs, readClientKeyStrict, repairFailureRecovery } from "./agent-doctor.mjs";
|
|
32
32
|
import { formatPublishIp, isLoopbackIp, probeEnginePublishIp, publishIpFix } from "./docker-publish-ip.mjs";
|
|
33
|
+
import { cmdHost, fixHelperWorkers, formatHelperWorkersLines, inspectHelperWorkers } from "./helper-host.mjs";
|
|
33
34
|
import { clearAgentState, selfHealAgentSession } from "./agent-session.mjs";
|
|
34
35
|
import { runUsage, usageStatus } from "./usage/sync.mjs";
|
|
35
36
|
import { installUsageAgent, usageAgentStatus } from "./usage/launchd.mjs";
|
|
@@ -83,6 +84,8 @@ export async function run(argv, {
|
|
|
83
84
|
case "logout": return cmdLogout();
|
|
84
85
|
case "status": return cmdStatus();
|
|
85
86
|
case "doctor": return cmdDoctor(args);
|
|
87
|
+
// BOT-2367: this Mac's Helper host key, whatever the status file's age.
|
|
88
|
+
case "host": return cmdHost(args);
|
|
86
89
|
case "update": return cmdUpdate();
|
|
87
90
|
// Agent-only commands (used by MCP agents, not humans)
|
|
88
91
|
case "heartbeat": return cmdHeartbeat(args);
|
|
@@ -188,7 +191,9 @@ ${bold("AUTH")}
|
|
|
188
191
|
logout Remove saved credentials
|
|
189
192
|
status Show current auth status (local metadata + server check)
|
|
190
193
|
doctor [--fix] [--print-exports] [--json]
|
|
191
|
-
Diagnose or repair the cached agent session
|
|
194
|
+
Diagnose or repair the cached agent session; on macOS also
|
|
195
|
+
reports duplicate Helper workers (--fix terminates them)
|
|
196
|
+
host key [--json] This Mac's Helper host key, whatever the status file's age
|
|
192
197
|
|
|
193
198
|
${bold("MCP CONFIG KEY")}
|
|
194
199
|
mcp setup [--env <NAME>] [--tenant <slug>] [--label <l>] [--expiry-days <n>]
|
|
@@ -257,6 +262,8 @@ export async function cmdDoctor(args, {
|
|
|
257
262
|
log = (line) => console.log(line), errorLog = (line) => console.error(line),
|
|
258
263
|
doctor = doctorAgentAuth, usageReport = doctorUsageSync, bootstrapReport = doctorBootstrapState,
|
|
259
264
|
publishReport = () => probeEnginePublishIp(),
|
|
265
|
+
helperWorkersReport = async () => inspectHelperWorkers(),
|
|
266
|
+
helperWorkersFix = (report) => fixHelperWorkers(report),
|
|
260
267
|
} = {}) {
|
|
261
268
|
const parsed = parseDoctorArgs(args);
|
|
262
269
|
if (parsed.help) { log(doctorHelp()); return 0; }
|
|
@@ -271,8 +278,18 @@ export async function cmdDoctor(args, {
|
|
|
271
278
|
const usage = await usageReport().catch(() => null);
|
|
272
279
|
// BOT-1945 (AC-3): informational, like usage sync — never changes the exit code.
|
|
273
280
|
const bootstrap = await bootstrapReport().catch(() => null);
|
|
281
|
+
// BOT-2367: duplicate Helper workers / stale Helper locks (macOS). Reported on
|
|
282
|
+
// every run; `--fix` SIGTERMs a duplicate launchd is not running, then
|
|
283
|
+
// re-checks. Informational: never changes doctor's exit code.
|
|
284
|
+
let helperWorkers = await helperWorkersReport().catch(() => null);
|
|
285
|
+
let helperFix = null;
|
|
286
|
+
if (parsed.fix && helperWorkers?.status === "duplicate") {
|
|
287
|
+
helperFix = await helperWorkersFix(helperWorkers).catch(() => null);
|
|
288
|
+
const recheck = await helperWorkersReport().catch(() => null);
|
|
289
|
+
helperWorkers = recheck ? { ...recheck, fix: helperFix } : { ...helperWorkers, fix: helperFix };
|
|
290
|
+
}
|
|
274
291
|
if (parsed.json) {
|
|
275
|
-
log(JSON.stringify({ ...result.report, ...(usage ? { usage_sync: usage } : {}), ...(bootstrap ? { bootstrap } : {}) }));
|
|
292
|
+
log(JSON.stringify({ ...result.report, ...(usage ? { usage_sync: usage } : {}), ...(bootstrap ? { bootstrap } : {}), ...(helperWorkers ? { helper_workers: helperWorkers } : {}) }));
|
|
276
293
|
} else {
|
|
277
294
|
log(`binding: ${result.report.binding.status}${result.report.binding.tenant ? ` (${result.report.binding.tenant})` : ""}`);
|
|
278
295
|
log(`client key: ${result.report.client_key.status}`);
|
|
@@ -288,6 +305,7 @@ export async function cmdDoctor(args, {
|
|
|
288
305
|
if (result.report.recovery) log(`recovery: ${result.report.recovery}`);
|
|
289
306
|
if (bootstrap) log(formatBootstrapLine(bootstrap));
|
|
290
307
|
if (usage) log(formatUsageSyncLine(usage));
|
|
308
|
+
for (const line of formatHelperWorkersLines(helperWorkers, helperFix)) log(line);
|
|
291
309
|
}
|
|
292
310
|
if (result.exports) log(result.exports);
|
|
293
311
|
return result.exitCode;
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
// BOT-2367 — this Mac's BotBuddy Helper host key, and the `bb doctor` Helper
|
|
2
|
+
// duplicate-worker / stale-lock check.
|
|
3
|
+
//
|
|
4
|
+
// The Helper writes `managed-container-host-status.json` with this machine's
|
|
5
|
+
// `host_key` and `checked_at` (Apple-epoch seconds; there is no `updated_at`).
|
|
6
|
+
// The host key is derived from the machine's P-256 identity and is static, so
|
|
7
|
+
// it never depends on the file's age: `bb host key` returns it whatever the
|
|
8
|
+
// age and reports freshness separately. On 2026-10-14 a 7-minute Helper job
|
|
9
|
+
// tick aged the file past Supply Guard's 300 s limit on a healthy host and SG
|
|
10
|
+
// refused to name the host (exit 4); Helper 0.2.37 refreshes the file on every
|
|
11
|
+
// beacon, and this command gives wrappers a resolver that cannot fail that way.
|
|
12
|
+
//
|
|
13
|
+
// Read-only except `fixHelperWorkers`, which only SIGTERMs a
|
|
14
|
+
// `botbuddyhelper-worker` process launchd is not running, and only when the
|
|
15
|
+
// launchd PID was read twice and agreed (fail-closed otherwise).
|
|
16
|
+
import { spawnSync } from "node:child_process";
|
|
17
|
+
import { readFileSync } from "node:fs";
|
|
18
|
+
import { homedir, platform } from "node:os";
|
|
19
|
+
import { basename, join } from "node:path";
|
|
20
|
+
|
|
21
|
+
export const HELPER_WORKER_LABEL = "com.botbuddy.helper.worker";
|
|
22
|
+
export const HELPER_WORKER_BINARY = "botbuddyhelper-worker";
|
|
23
|
+
/** Supply Guard's freshness limit; Helper 0.2.37 refreshes every ~20 s. */
|
|
24
|
+
export const HELPER_STATUS_FRESH_SECONDS = 300;
|
|
25
|
+
const APPLE_REFERENCE_EPOCH_SECONDS = 978_307_200;
|
|
26
|
+
const HOST_KEY_PATTERN = /^[A-Za-z0-9._:-]+$/;
|
|
27
|
+
const HELPER_LOCK_FILES = [".worker.lock", ".beacon.lock", ".metrics.lock"];
|
|
28
|
+
const START_REMEDY = "start the BotBuddy Helper (open \"BotBuddy Helper.app\") so it writes this file";
|
|
29
|
+
|
|
30
|
+
export function helperSupportDir(home = homedir()) {
|
|
31
|
+
return join(home, "Library", "Application Support", "BotBuddyHelper");
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function helperStatusPath(home = homedir()) {
|
|
35
|
+
return join(helperSupportDir(home), "managed-container-host-status.json");
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* `{ ok: true, host_key, checked_at, age_seconds, fresh, path }`, or
|
|
40
|
+
* `{ ok: false, code, path, message }` when the file is missing or names no
|
|
41
|
+
* well-formed host key. Freshness never decides whether the key is returned.
|
|
42
|
+
*/
|
|
43
|
+
export function readHelperHostKey({ home = homedir(), nowMs = Date.now(), readFile = (p) => readFileSync(p, "utf8") } = {}) {
|
|
44
|
+
const path = helperStatusPath(home);
|
|
45
|
+
let contents;
|
|
46
|
+
try {
|
|
47
|
+
contents = readFile(path);
|
|
48
|
+
} catch (error) {
|
|
49
|
+
return { ok: false, code: "HELPER_STATUS_MISSING", path, message: `could not read ${path} (${error?.code ?? "unreadable"}); ${START_REMEDY}` };
|
|
50
|
+
}
|
|
51
|
+
let status;
|
|
52
|
+
try {
|
|
53
|
+
status = JSON.parse(contents);
|
|
54
|
+
} catch {
|
|
55
|
+
status = null;
|
|
56
|
+
}
|
|
57
|
+
const hostKey = status?.host_key;
|
|
58
|
+
if (typeof hostKey !== "string" || !HOST_KEY_PATTERN.test(hostKey)) {
|
|
59
|
+
return { ok: false, code: "HELPER_STATUS_MALFORMED", path, message: `${path} names no well-formed host_key; ${START_REMEDY}` };
|
|
60
|
+
}
|
|
61
|
+
const checkedAt = status.checked_at;
|
|
62
|
+
if (!Number.isFinite(checkedAt)) {
|
|
63
|
+
return { ok: true, host_key: hostKey, checked_at: null, age_seconds: null, fresh: false, path };
|
|
64
|
+
}
|
|
65
|
+
const checkedAtMs = (checkedAt + APPLE_REFERENCE_EPOCH_SECONDS) * 1000;
|
|
66
|
+
const ageSeconds = Math.round((nowMs - checkedAtMs) / 1000);
|
|
67
|
+
return {
|
|
68
|
+
ok: true,
|
|
69
|
+
host_key: hostKey,
|
|
70
|
+
checked_at: new Date(checkedAtMs).toISOString(),
|
|
71
|
+
age_seconds: ageSeconds,
|
|
72
|
+
fresh: ageSeconds >= -60 && ageSeconds <= HELPER_STATUS_FRESH_SECONDS,
|
|
73
|
+
path,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const HOST_USAGE = "Usage: bb host key [--json]";
|
|
78
|
+
|
|
79
|
+
export function hostHelp() {
|
|
80
|
+
return `${HOST_USAGE}
|
|
81
|
+
|
|
82
|
+
Prints this Mac's managed-container host key from the BotBuddy Helper's status
|
|
83
|
+
file, whatever the file's age: the key is this machine's static identity. A file
|
|
84
|
+
older than ${HELPER_STATUS_FRESH_SECONDS} s still prints the key (exit 0) with a freshness warning on
|
|
85
|
+
stderr. Exit 4 when the file is missing or names no host key.
|
|
86
|
+
--json prints {host_key, checked_at, age_seconds, fresh, path}.`;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** `bb host key [--json]` → exit code. */
|
|
90
|
+
export async function cmdHost(args, {
|
|
91
|
+
log = (line) => console.log(line), errorLog = (line) => console.error(line),
|
|
92
|
+
read = () => readHelperHostKey(),
|
|
93
|
+
} = {}) {
|
|
94
|
+
const [sub, ...rest] = args;
|
|
95
|
+
if (sub === "help" || sub === "--help" || sub === "-h") { log(hostHelp()); return 0; }
|
|
96
|
+
const json = rest.includes("--json");
|
|
97
|
+
const unknown = rest.filter((a) => a !== "--json");
|
|
98
|
+
if (sub !== "key" || unknown.length) { errorLog(HOST_USAGE); return 4; }
|
|
99
|
+
const r = read();
|
|
100
|
+
if (!r.ok) {
|
|
101
|
+
errorLog(`bb host key: ${r.code}: ${r.message}`);
|
|
102
|
+
return 4;
|
|
103
|
+
}
|
|
104
|
+
if (json) {
|
|
105
|
+
log(JSON.stringify({ host_key: r.host_key, checked_at: r.checked_at, age_seconds: r.age_seconds, fresh: r.fresh, path: r.path }));
|
|
106
|
+
} else {
|
|
107
|
+
log(r.host_key);
|
|
108
|
+
}
|
|
109
|
+
if (!r.fresh) {
|
|
110
|
+
const age = r.age_seconds == null ? "has no checked_at" : `is ${r.age_seconds} s old`;
|
|
111
|
+
errorLog(`bb host key: warning: the Helper status file ${age} (fresh is ≤ ${HELPER_STATUS_FRESH_SECONDS} s); the key is still this Mac's. ` +
|
|
112
|
+
`If the Helper is not running, check \`launchctl print gui/$(id -u)/${HELPER_WORKER_LABEL}\` and the worker log.`);
|
|
113
|
+
}
|
|
114
|
+
return 0;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// ── bb doctor: duplicate Helper workers and stale Helper locks ───────────────
|
|
118
|
+
|
|
119
|
+
function defaultRun(cmd, args) {
|
|
120
|
+
const r = spawnSync(cmd, args, { encoding: "utf8", timeout: 5000 });
|
|
121
|
+
return { status: r.error ? null : r.status, stdout: r.stdout ?? "" };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** `launchctl list <label>` → pid number, null (loaded, not running), or undefined (unreadable). */
|
|
125
|
+
export function parseLaunchctlListPid(result) {
|
|
126
|
+
if (!result || result.status !== 0) return undefined;
|
|
127
|
+
const m = /"PID"\s*=\s*(\d+)\s*;/.exec(result.stdout);
|
|
128
|
+
return m ? Number(m[1]) : null;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** `ps -axo pid=,comm=` → the pids of `botbuddyhelper-worker` processes. */
|
|
132
|
+
export function parseWorkerPids(stdout) {
|
|
133
|
+
const pids = [];
|
|
134
|
+
for (const line of String(stdout).split("\n")) {
|
|
135
|
+
const m = /^\s*(\d+)\s+(.+?)\s*$/.exec(line);
|
|
136
|
+
if (m && basename(m[2]) === HELPER_WORKER_BINARY) pids.push(Number(m[1]));
|
|
137
|
+
}
|
|
138
|
+
return pids.sort((a, b) => a - b);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** `lsof -F pcn` → Helper lock files held open by a process that is not a Helper process. */
|
|
142
|
+
export function parseStaleLocks(stdout) {
|
|
143
|
+
const stale = [];
|
|
144
|
+
let pid = null; let command = null;
|
|
145
|
+
for (const line of String(stdout).split("\n")) {
|
|
146
|
+
const tag = line[0]; const value = line.slice(1);
|
|
147
|
+
if (tag === "p") { pid = Number(value); command = null; }
|
|
148
|
+
else if (tag === "c") command = value;
|
|
149
|
+
else if (tag === "n") {
|
|
150
|
+
const file = basename(value);
|
|
151
|
+
if (HELPER_LOCK_FILES.includes(file) && !/^botbuddyhelper/i.test(command ?? "")) {
|
|
152
|
+
stale.push({ file, pid, command });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return stale;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Read-only census of Helper worker processes against launchd's own job PID.
|
|
161
|
+
* A duplicate is a `botbuddyhelper-worker` launchd is not running. The launchd
|
|
162
|
+
* PID is read before and after `ps`; if launchd's answer is unreadable or
|
|
163
|
+
* changed (a tick started mid-check), nothing is called a duplicate.
|
|
164
|
+
*/
|
|
165
|
+
export function inspectHelperWorkers({ run = defaultRun, home = homedir(), platformFn = platform } = {}) {
|
|
166
|
+
if (platformFn() !== "darwin") return null;
|
|
167
|
+
const before = parseLaunchctlListPid(run("launchctl", ["list", HELPER_WORKER_LABEL]));
|
|
168
|
+
const workers = parseWorkerPids(run("ps", ["-axo", "pid=,comm="]).stdout);
|
|
169
|
+
const after = parseLaunchctlListPid(run("launchctl", ["list", HELPER_WORKER_LABEL]));
|
|
170
|
+
const lockPaths = HELPER_LOCK_FILES.map((f) => join(helperSupportDir(home), f));
|
|
171
|
+
const lsof = run("lsof", ["+c", "0", "-F", "pcn", ...lockPaths]);
|
|
172
|
+
const staleLocks = parseStaleLocks(lsof.stdout);
|
|
173
|
+
const base = { launchd_pid: after ?? null, workers, duplicates: [], stale_locks: staleLocks };
|
|
174
|
+
if (before === undefined || after === undefined) {
|
|
175
|
+
return { status: "unknown", ...base, reason: `launchd job ${HELPER_WORKER_LABEL} is not readable in this session` };
|
|
176
|
+
}
|
|
177
|
+
if (before !== after) return { status: "unsettled", ...base, reason: "launchd started a worker tick during the check" };
|
|
178
|
+
const duplicates = workers.filter((pid) => pid !== after);
|
|
179
|
+
return { status: duplicates.length ? "duplicate" : "ok", ...base, duplicates };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function defaultIsAlive(pid) {
|
|
183
|
+
try { process.kill(pid, 0); return true; } catch (error) { return error?.code === "EPERM"; }
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** SIGTERM each duplicate (never launchd's worker); wait up to ~3 s for exit. */
|
|
187
|
+
export async function fixHelperWorkers(report, {
|
|
188
|
+
kill = (pid, signal) => process.kill(pid, signal), isAlive = defaultIsAlive,
|
|
189
|
+
sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
190
|
+
} = {}) {
|
|
191
|
+
const result = { terminated: [], failed: [] };
|
|
192
|
+
if (report?.status !== "duplicate") return result;
|
|
193
|
+
for (const pid of report.duplicates) {
|
|
194
|
+
if (pid === report.launchd_pid) continue;
|
|
195
|
+
try { kill(pid, "SIGTERM"); } catch { /* already gone or not ours; checked below */ }
|
|
196
|
+
}
|
|
197
|
+
for (let i = 0; i < 30 && report.duplicates.some((pid) => isAlive(pid)); i++) await sleep(100);
|
|
198
|
+
for (const pid of report.duplicates) (isAlive(pid) ? result.failed : result.terminated).push(pid);
|
|
199
|
+
return result;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Human lines for `bb doctor`. */
|
|
203
|
+
export function formatHelperWorkersLines(report, fix = null) {
|
|
204
|
+
if (!report) return [];
|
|
205
|
+
const lines = [];
|
|
206
|
+
const launchd = report.launchd_pid ? `launchd ${HELPER_WORKER_LABEL} pid ${report.launchd_pid}` : `launchd ${HELPER_WORKER_LABEL} idle between ticks`;
|
|
207
|
+
if (fix && (fix.terminated.length || fix.failed.length)) {
|
|
208
|
+
// A re-checked report (cmdDoctor attaches `fix`) already lists the survivors.
|
|
209
|
+
const running = report.fix ? report.workers.length : report.workers.length - fix.terminated.length;
|
|
210
|
+
for (const pid of fix.terminated) lines.push(`helper workers: terminated duplicate pid ${pid}; ${running} running (${launchd})`);
|
|
211
|
+
for (const pid of fix.failed) lines.push(`helper workers: could not terminate duplicate pid ${pid}; stop it with \`kill ${pid}\``);
|
|
212
|
+
} else if (report.status === "ok") {
|
|
213
|
+
lines.push(`helper workers: ${report.workers.length} running (${launchd})`);
|
|
214
|
+
} else if (report.status === "duplicate") {
|
|
215
|
+
lines.push(`helper workers: ${report.workers.length} running — duplicate pid ${report.duplicates.join(", ")} (${launchd}); fix: \`bb doctor --fix\` terminates the duplicate`);
|
|
216
|
+
} else {
|
|
217
|
+
lines.push(`helper workers: ${report.workers.length} running — ${report.reason}; not checked for duplicates${report.status === "unsettled" ? " (re-run bb doctor)" : ""}`);
|
|
218
|
+
}
|
|
219
|
+
for (const s of report.stale_locks) {
|
|
220
|
+
lines.push(`helper lock: ${s.file} is held open by pid ${s.pid} (${s.command}), not a Helper process, so the Helper cannot take it; stop pid ${s.pid}`);
|
|
221
|
+
}
|
|
222
|
+
return lines;
|
|
223
|
+
}
|
package/src/wait-core.mjs
CHANGED
|
@@ -31,6 +31,10 @@ export const EXIT = Object.freeze({
|
|
|
31
31
|
// BOT-2330: an exact managed-allocation cleanup failed. The waiter must not
|
|
32
32
|
// report a successful terminal match when the signed Helper outcome failed.
|
|
33
33
|
MANAGED_ALLOCATION_FAILED: 10,
|
|
34
|
+
// BOT-2360: the server ended this wait while the client was still parked on it — the
|
|
35
|
+
// liveness reaper abandoned it, it was cancelled, or its row is gone. The relay
|
|
36
|
+
// refuses the stream with a typed `wait_abandoned` error frame; re-arm from next_cursor.
|
|
37
|
+
ABANDONED: 11,
|
|
34
38
|
});
|
|
35
39
|
|
|
36
40
|
// BOT-2013: the uuid spellings Postgres accepts for a lease id (uuid_in), mirrored by the SQL
|
|
@@ -1756,6 +1760,20 @@ export async function runWaitLoop({
|
|
|
1756
1760
|
if (parsed.error === "unauthorized" || parsed.error === "forbidden") {
|
|
1757
1761
|
return finalize("error", { exitCode: EXIT.AUTH, error: parsed.error });
|
|
1758
1762
|
}
|
|
1763
|
+
// BOT-2360: the relay refuses a stream bound to a wait the server already ended
|
|
1764
|
+
// (prod a0cd7b11 parked 54 min on one). Never reconnect to it: exit typed, with the
|
|
1765
|
+
// confirmed cursor so a re-arm replays everything after it.
|
|
1766
|
+
if (parsed.error === "wait_abandoned") {
|
|
1767
|
+
return finalize("abandoned", {
|
|
1768
|
+
exitCode: EXIT.ABANDONED,
|
|
1769
|
+
error: "wait_abandoned",
|
|
1770
|
+
state: {
|
|
1771
|
+
wait_session_id: typeof parsed.wait_session_id === "string" ? parsed.wait_session_id : waitSessionId ?? null,
|
|
1772
|
+
status: typeof parsed.status === "string" ? parsed.status : null,
|
|
1773
|
+
reason: typeof parsed.reason === "string" ? parsed.reason : null,
|
|
1774
|
+
},
|
|
1775
|
+
});
|
|
1776
|
+
}
|
|
1759
1777
|
return finalize("error", { exitCode: EXIT.INTERNAL, error: parsed.error || "stream_error" });
|
|
1760
1778
|
}
|
|
1761
1779
|
|
package/src/wait.mjs
CHANGED
|
@@ -180,7 +180,12 @@ OUTPUT
|
|
|
180
180
|
8 superseded (wait replaced server-side) · 9 lease lost (a
|
|
181
181
|
lease:id= wait whose lease was reaped before it was granted;
|
|
182
182
|
receipt outcome lease_lost + reap_reason — request a new
|
|
183
|
-
lease)
|
|
183
|
+
lease) · 10 managed-allocation cleanup failed (BOT-2330)
|
|
184
|
+
· 11 abandoned (the server ended this wait while it
|
|
185
|
+
was parked — reaped after 180 s without a live stream,
|
|
186
|
+
or cancelled; receipt outcome abandoned +
|
|
187
|
+
state.status/reason — re-arm with --since <next_cursor>).
|
|
188
|
+
If the receipt has
|
|
184
189
|
reacquire_lock (claim=true), acquire_lock again FIRST and re-run
|
|
185
190
|
only on a queued/busy result (granted = you hold it — just resume);
|
|
186
191
|
otherwise re-run from the receipt's next_cursor. docs/agent-wait.md.
|
|
@@ -548,11 +553,28 @@ export function leaseLostNotice(receipt) {
|
|
|
548
553
|
`(reap_reason=${st.reap_reason ?? "unknown"}) — request a new lease (exit ${EXIT.LEASE_LOST})`;
|
|
549
554
|
}
|
|
550
555
|
|
|
556
|
+
/**
|
|
557
|
+
* BOT-2360: the one stderr line for an `abandoned` receipt (null for any other outcome). The
|
|
558
|
+
* server ended this wait (its liveness reaper or a cancel) while the client was
|
|
559
|
+
* still parked; re-arming from next_cursor replays everything after the last frame consumed.
|
|
560
|
+
*/
|
|
561
|
+
export function waitAbandonedNotice(receipt) {
|
|
562
|
+
if (receipt?.outcome !== "abandoned") return null;
|
|
563
|
+
const st = receipt.state ?? {};
|
|
564
|
+
const since = receipt.next_cursor != null ? ` --since ${receipt.next_cursor}` : "";
|
|
565
|
+
return `bb-wait: wait ${st.wait_session_id ?? "?"} was ended server-side (status=${st.status ?? "abandoned"}, ` +
|
|
566
|
+
`reason=${st.reason ?? "unknown"}) — re-arm the same conditions with${since || " a fresh wait"} (exit ${EXIT.ABANDONED})`;
|
|
567
|
+
}
|
|
568
|
+
|
|
551
569
|
function writeLeaseLostNotice(receipt) {
|
|
552
|
-
const notice = leaseLostNotice(receipt);
|
|
570
|
+
const notice = leaseLostNotice(receipt) ?? waitAbandonedNotice(receipt);
|
|
553
571
|
if (notice) process.stderr.write(`${notice}\n`);
|
|
554
572
|
}
|
|
555
573
|
|
|
574
|
+
// BOT-1228 / BOT-2360: outcomes whose wait_session the server already ended. The client
|
|
575
|
+
// never finalizes them (finalize_wait_session only transitions from active).
|
|
576
|
+
const SERVER_ENDED_OUTCOMES = new Set(["superseded", "abandoned"]);
|
|
577
|
+
|
|
556
578
|
// Record the terminal outcome. Best-effort AND bounded: the receipt has already
|
|
557
579
|
// been written to stdout before this runs, so a slow/failing finalize must never
|
|
558
580
|
// delay the agent's wake or change the exit path. A short abort timeout caps how
|
|
@@ -596,7 +618,7 @@ async function finalizeWait(opts, waitSessionId, receipt, timeoutMs = 5000) {
|
|
|
596
618
|
// A real SSE connection to the relay's spine tail. Returns an async iterable of
|
|
597
619
|
// parsed frames. Server emits `id: <seq>` per signal and replays `seq > since`
|
|
598
620
|
// on connect, so a reconnect never loses an event.
|
|
599
|
-
function makeConnect(opts) {
|
|
621
|
+
export function makeConnect(opts) {
|
|
600
622
|
// BOT-1649: mirror the relay's keepalive expiry roll into the 0600 cache. The
|
|
601
623
|
// server bumps session_token_expires_at on connect and every keepalive, but
|
|
602
624
|
// the cache keeps its register-time deadline — so a wait near the 8 h cap goes
|
|
@@ -633,16 +655,25 @@ function makeConnect(opts) {
|
|
|
633
655
|
// signal tears the stalled socket down immediately, then the read rejects and
|
|
634
656
|
// the stream ends so runWaitLoop reconnects.
|
|
635
657
|
const ac = new AbortController();
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
658
|
+
// BOT-2360: bound the wait for response headers. Unbounded, a connect that hangs on a
|
|
659
|
+
// loaded host or a stuck proxy sits silent past the wait reaper's grace (undici's own
|
|
660
|
+
// headers timeout is 300 s); bounded, the worst silent reconnect stays inside it.
|
|
661
|
+
const connectTimer = setTimeout(() => ac.abort(new Error("relay connect timed out")), waitConnectTimeoutMs());
|
|
662
|
+
let res;
|
|
663
|
+
try {
|
|
664
|
+
res = await fetch(url, {
|
|
665
|
+
headers: {
|
|
666
|
+
Authorization: `Bearer ${opts.token}`,
|
|
667
|
+
"x-agent-api-key": opts.token || "",
|
|
668
|
+
Accept: "text/event-stream",
|
|
669
|
+
// BOT-741: never let a proxy gzip-buffer an SSE stream.
|
|
670
|
+
"Accept-Encoding": "identity",
|
|
671
|
+
},
|
|
672
|
+
signal: ac.signal,
|
|
673
|
+
});
|
|
674
|
+
} finally {
|
|
675
|
+
clearTimeout(connectTimer);
|
|
676
|
+
}
|
|
646
677
|
if (res.status === 401) return errorStream("unauthorized");
|
|
647
678
|
if (res.status === 403) return errorStream("forbidden");
|
|
648
679
|
if (!res.ok || !res.body) throw new Error(`relay responded ${res.status}`);
|
|
@@ -655,7 +686,7 @@ function makeConnect(opts) {
|
|
|
655
686
|
// runs the global bb_wait_linear_feed_lagging RPC) whether the Linear ingest
|
|
656
687
|
// feed is stale. Any failure -> false (non-fatal: the wait keeps its last known
|
|
657
688
|
// degraded state rather than flapping on a probe blip).
|
|
658
|
-
const FEED_LAG_PROBE_TIMEOUT_MS = 10_000;
|
|
689
|
+
export const FEED_LAG_PROBE_TIMEOUT_MS = 10_000;
|
|
659
690
|
|
|
660
691
|
function makeFeedLagProbe(opts) {
|
|
661
692
|
// `issues` = the armed `linear` conditions' target ticket keys. The relay
|
|
@@ -715,9 +746,20 @@ function makeFeedLagProbe(opts) {
|
|
|
715
746
|
// runWaitLoop reconnects from the cursor, which re-bumps wait_sessions.last_seen_at
|
|
716
747
|
// (and, with --heartbeat, agents.last_heartbeat) before the reaper's grace.
|
|
717
748
|
// 70s ≈ 2.3 missed keepalives — long enough to never false-trip on jitter, short
|
|
718
|
-
// enough that the reconnect lands inside the
|
|
749
|
+
// enough that the reconnect lands inside the 180 s last_seen reap window (BOT-2360).
|
|
719
750
|
export const DEFAULT_SSE_IDLE_TIMEOUT_MS = 70_000;
|
|
720
751
|
|
|
752
|
+
// BOT-2360: the longest a (re)connect may wait for the relay's response headers. With the
|
|
753
|
+
// idle watchdog (70 s), the backoff cap (15 s) and the feed-lag probe (10 s) it bounds a
|
|
754
|
+
// healthy client's worst silent gap at 125 s, under the 180 s wait-reaper grace with 55 s
|
|
755
|
+
// to spare (asserted in bot2360-abandoned-wait.test.mjs against the registered cron job).
|
|
756
|
+
export const WAIT_CONNECT_TIMEOUT_MS = 30_000;
|
|
757
|
+
|
|
758
|
+
function waitConnectTimeoutMs() {
|
|
759
|
+
const raw = Number(process.env.BOTBUDDY_WAIT_CONNECT_TIMEOUT_MS);
|
|
760
|
+
return Number.isFinite(raw) && raw > 0 ? raw : WAIT_CONNECT_TIMEOUT_MS;
|
|
761
|
+
}
|
|
762
|
+
|
|
721
763
|
function sseIdleTimeoutMs() {
|
|
722
764
|
const raw = Number(process.env.BOTBUDDY_WAIT_IDLE_TIMEOUT_MS);
|
|
723
765
|
return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_SSE_IDLE_TIMEOUT_MS;
|
|
@@ -1194,7 +1236,7 @@ async function deliverTerminalFinalization(checkpoint) {
|
|
|
1194
1236
|
async function flushPendingFinalization(store, checkpoint) {
|
|
1195
1237
|
if (!checkpoint.cloud_wait_session_id || !checkpoint.terminal_receipt ||
|
|
1196
1238
|
(checkpoint.cloud_finalize_pending === false && checkpoint.cloud_finalized_at)) return checkpoint;
|
|
1197
|
-
if (checkpoint.terminal_receipt.outcome
|
|
1239
|
+
if (SERVER_ENDED_OUTCOMES.has(checkpoint.terminal_receipt.outcome)) {
|
|
1198
1240
|
return store.update(checkpoint.local_wait_id, {
|
|
1199
1241
|
cloud_finalize_pending: false,
|
|
1200
1242
|
cloud_finalized_at: new Date().toISOString(),
|
|
@@ -1296,6 +1338,7 @@ function exitForRecoveredOutcome(receipt) {
|
|
|
1296
1338
|
if (receipt?.outcome === "cancelled") return EXIT.MATCHED;
|
|
1297
1339
|
if (receipt?.outcome === "timeout") return EXIT.TIMEOUT;
|
|
1298
1340
|
if (receipt?.outcome === "superseded") return EXIT.SUPERSEDED;
|
|
1341
|
+
if (receipt?.outcome === "abandoned") return EXIT.ABANDONED;
|
|
1299
1342
|
if (receipt?.error === "cursor_expired") return EXIT.CURSOR_EXPIRED;
|
|
1300
1343
|
if (receipt?.error === "unauthorized" || receipt?.error === "forbidden") return EXIT.AUTH;
|
|
1301
1344
|
return EXIT.BACKEND;
|
|
@@ -2095,7 +2138,7 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
|
|
|
2095
2138
|
terminal_receipt: terminalReceipt,
|
|
2096
2139
|
exit_code: state.exitCode,
|
|
2097
2140
|
safe_cursor: terminalReceipt.next_cursor ?? state.cursor ?? checkpoint.safe_cursor,
|
|
2098
|
-
cloud_finalize_pending: Boolean(waitSessionId && terminalReceipt.outcome
|
|
2141
|
+
cloud_finalize_pending: Boolean(waitSessionId && !SERVER_ENDED_OUTCOMES.has(terminalReceipt.outcome)),
|
|
2099
2142
|
});
|
|
2100
2143
|
terminalPersistedFromFrame = true;
|
|
2101
2144
|
return;
|
|
@@ -2125,7 +2168,7 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
|
|
|
2125
2168
|
terminal_receipt: terminalReceipt,
|
|
2126
2169
|
exit_code: exitCode,
|
|
2127
2170
|
safe_cursor: terminalReceipt.next_cursor ?? checkpoint.safe_cursor ?? null,
|
|
2128
|
-
cloud_finalize_pending: Boolean(waitSessionId && terminalReceipt.outcome
|
|
2171
|
+
cloud_finalize_pending: Boolean(waitSessionId && !SERVER_ENDED_OUTCOMES.has(terminalReceipt.outcome)),
|
|
2129
2172
|
});
|
|
2130
2173
|
}
|
|
2131
2174
|
emitReceipt(terminalReceipt, { versioned: true });
|
|
@@ -2134,7 +2177,8 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
|
|
|
2134
2177
|
// ended_at/updated_at stamped). Skip the client finalize: the row is already
|
|
2135
2178
|
// terminal (finalize_wait_session only transitions from active, so it'd be a
|
|
2136
2179
|
// no-op) and `superseded` isn't a wait_session_status enum value anyway.
|
|
2137
|
-
|
|
2180
|
+
// BOT-2360: likewise `abandoned` — the relay refused the stream because the row is ended.
|
|
2181
|
+
if (waitSessionId && !SERVER_ENDED_OUTCOMES.has(receipt.outcome)) {
|
|
2138
2182
|
const finalized = await finalizeWait(opts, waitSessionId, terminalReceipt);
|
|
2139
2183
|
try {
|
|
2140
2184
|
checkpoint = await store.update(checkpoint.local_wait_id, finalized.delivered
|