@botbuddy/cli 1.36.3 → 1.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/lane-status.mjs +22 -0
- package/src/stack.mjs +235 -5
- package/src/telemetry-outbox.mjs +30 -5
- package/src/wait-core.mjs +7 -7
package/package.json
CHANGED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// BOT-1319 — report a phase to the BotBuddy lefthook lane wrapper that may be
|
|
2
|
+
// running this command (botbuddy-web `scripts/log-test-run.mjs`, the same file
|
|
3
|
+
// protocol as `scripts/lib/lane-status.mjs`). The wrapper exports
|
|
4
|
+
// $BOTBUDDY_LANE_STATUS_FILE and turns each atomic rewrite into one lifecycle
|
|
5
|
+
// progress event, so a lane parked here for Docker capacity shows as
|
|
6
|
+
// `waiting · queued_for_capacity` instead of as silent test time. Unset outside a
|
|
7
|
+
// wrapped lane → no-op. Never throws.
|
|
8
|
+
|
|
9
|
+
import { renameSync, writeFileSync } from "node:fs";
|
|
10
|
+
|
|
11
|
+
export function writeLaneStatus(status, env = process.env) {
|
|
12
|
+
const path = env.BOTBUDDY_LANE_STATUS_FILE;
|
|
13
|
+
if (!path) return false;
|
|
14
|
+
try {
|
|
15
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
16
|
+
writeFileSync(tmp, JSON.stringify({ ...status, at: new Date().toISOString() }));
|
|
17
|
+
renameSync(tmp, path);
|
|
18
|
+
return true;
|
|
19
|
+
} catch {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
}
|
package/src/stack.mjs
CHANGED
|
@@ -21,12 +21,14 @@
|
|
|
21
21
|
// labelled fallback for when no Helper is available: it runs `supabase start`/
|
|
22
22
|
// `stop` itself and self-activates/finalizes via the fallback MCP tools.
|
|
23
23
|
|
|
24
|
+
import { writeLaneStatus } from "./lane-status.mjs";
|
|
24
25
|
import { spawn, spawnSync } from "child_process";
|
|
25
26
|
import { readFileSync, realpathSync } from "fs";
|
|
26
27
|
import { open, unlink, readFile } from "fs/promises";
|
|
27
28
|
import { tmpdir } from "os";
|
|
28
29
|
import { basename, dirname, join, relative, isAbsolute } from "path";
|
|
29
30
|
import { randomUUID } from "crypto";
|
|
31
|
+
import { fileURLToPath } from "url";
|
|
30
32
|
import { callToolJson, CALL_AUTH_ERROR_CODES } from "./api.mjs";
|
|
31
33
|
import { SERVER_URL, getConfig } from "./config.mjs";
|
|
32
34
|
import { resolveOwnerToken, resolveAgentKey } from "./cli-credentials.mjs";
|
|
@@ -86,6 +88,10 @@ ${bold("up OPTIONS")}
|
|
|
86
88
|
--idle-ttl <seconds> Idle seconds before the reaper STOPS an unused stack (default 1800).
|
|
87
89
|
--timeout <seconds> Max seconds to park for capacity before giving up (default ${DEFAULT_TIMEOUT_SEC}).
|
|
88
90
|
--no-wait If the host is full, print the queue position and exit (don't park).
|
|
91
|
+
--wait-capacity <dur> Opt-in (BOT-2024): on a TRANSIENT HOST_CAPACITY_SOURCE_STALE refusal, park
|
|
92
|
+
on the server's \`bb wait 'capacity:…'\` up to <dur> (90s, 10m, 1h) and
|
|
93
|
+
re-attempt admission once when it matches. Sustained refusals, the
|
|
94
|
+
deadline (exit 2) and a failed wait (its own exit code) never retry.
|
|
89
95
|
--local-exec FALLBACK (no Helper): run 'supabase start' in an ISOLATED --stack-path
|
|
90
96
|
stack and self-activate. Refused at the worktree root (shared stack).
|
|
91
97
|
--docker-context <name> Explicit Docker context for the mandatory local preflight
|
|
@@ -114,6 +120,7 @@ ${bold("run OPTIONS")}
|
|
|
114
120
|
--connection-file P Optional empty file path for the mode-0600 connection JSON.
|
|
115
121
|
--hard-ttl N Bound the child runtime; timeout requests normal cleanup.
|
|
116
122
|
--local-exec Rejected: stack run never starts Docker directly.
|
|
123
|
+
--wait-capacity <dur> Same as up: park a TRANSIENT stale-capacity refusal, then re-admit once.
|
|
117
124
|
|
|
118
125
|
${bold("GLOBAL")}
|
|
119
126
|
--json Pretty-print the receipt (default is one compact JSON line).
|
|
@@ -124,6 +131,13 @@ ${bold("EXIT CODES")}
|
|
|
124
131
|
4 invalid arguments 5 backend/coordination 6 lease failed/reaped
|
|
125
132
|
7 internal error 8 cleanup/reap proof failed
|
|
126
133
|
|
|
134
|
+
${bold("HOST_CAPACITY_SOURCE_STALE")} (BOT-2024)
|
|
135
|
+
HOST_CAPACITY_SOURCE_STALE is a capacity flag, not a Helper diagnosis. Read \`phase\` on the
|
|
136
|
+
receipt. transient → arm the printed \`bb wait 'capacity:…'\` and re-run once it fires
|
|
137
|
+
(or pass --wait-capacity); never report blocked, never ask for a Helper restart.
|
|
138
|
+
sustained (no beacon ever or >= 10 min) → report BLOCKED quoting \`stale_since\`; a Helper
|
|
139
|
+
check/restart is the right ask. A single refusal is never evidence of an outage.
|
|
140
|
+
|
|
127
141
|
${bold("EXAMPLE")}
|
|
128
142
|
# request a per-worktree stack for this batch, run a lane against it, then reap it
|
|
129
143
|
botbuddy stack run --repo botbuddy-web --ticket BOT-1346 -- pnpm test:integration`;
|
|
@@ -175,7 +189,7 @@ export function parseStackArgs(argv) {
|
|
|
175
189
|
slot: null, host: null, repo: null, ticket: null, ticketUrl: null,
|
|
176
190
|
prId: null, prUrl: null, purpose: null, idleTtl: null, stackPath: ".", stackPathExplicit: false,
|
|
177
191
|
timeout: DEFAULT_TIMEOUT_SEC, provisionTimeout: null, reapTimeout: 300, hardTtl: null,
|
|
178
|
-
connectionFile: null, noWait: false, localExec: false,
|
|
192
|
+
connectionFile: null, noWait: false, localExec: false, waitCapacity: null,
|
|
179
193
|
dockerContext: null, dockerEndpoint: null,
|
|
180
194
|
disposition: "destroy", json: false, receiptMaxBytes: DEFAULT_RECEIPT_MAX_BYTES,
|
|
181
195
|
};
|
|
@@ -248,6 +262,14 @@ export function parseStackArgs(argv) {
|
|
|
248
262
|
}
|
|
249
263
|
break;
|
|
250
264
|
}
|
|
265
|
+
case "--wait-capacity": {
|
|
266
|
+
if (need(a, rest[i + 1])) {
|
|
267
|
+
const secs = parseWaitCapacityDuration(rest[++i]);
|
|
268
|
+
if (secs == null) errors.push("--wait-capacity must be a positive duration: <N>, <N>s, <N>m or <N>h");
|
|
269
|
+
else opts.waitCapacity = secs;
|
|
270
|
+
}
|
|
271
|
+
break;
|
|
272
|
+
}
|
|
251
273
|
case "--no-wait": opts.noWait = true; break;
|
|
252
274
|
case "--local-exec": opts.localExec = true; break;
|
|
253
275
|
case "--docker-context": if (need(a, rest[i + 1])) opts.dockerContext = rest[++i]; break;
|
|
@@ -280,6 +302,9 @@ export function parseStackArgs(argv) {
|
|
|
280
302
|
"worktree root's default (shared canonical) project. Point --stack-path at a slot-derived stack directory with its " +
|
|
281
303
|
"own supabase/config.toml (distinct project_id + remapped ports).");
|
|
282
304
|
}
|
|
305
|
+
if (opts.waitCapacity != null && !["up", "run"].includes(command)) {
|
|
306
|
+
errors.push("--wait-capacity is valid only with `stack up` or `stack run`");
|
|
307
|
+
}
|
|
283
308
|
if ((opts.dockerContext || opts.dockerEndpoint) && !(["up", "done"].includes(command) && opts.localExec)) {
|
|
284
309
|
errors.push("--docker-context and --docker-endpoint are valid only with `stack up --local-exec` or `stack done --local-exec`");
|
|
285
310
|
}
|
|
@@ -708,6 +733,124 @@ export function leaseRefusalError(data) {
|
|
|
708
733
|
return data?.message || data?.code || "lease request refused";
|
|
709
734
|
}
|
|
710
735
|
|
|
736
|
+
// ── BOT-2024: phase-aware HOST_CAPACITY_SOURCE_STALE ─────────────────────────
|
|
737
|
+
// The refusal is a capacity FLAG, not a Helper diagnosis. The server classifies it:
|
|
738
|
+
// `sustained` iff the host never beaconed or its last beacon is >= 600 s old, else
|
|
739
|
+
// `transient` (scheduler lag; the flag's own age is informational only). Agents escalated transient lag as an outage and asked the
|
|
740
|
+
// user to restart a healthy Helper (2026-10-03), so the CLI renders the phase.
|
|
741
|
+
export const HOST_CAPACITY_SOURCE_STALE_CODE = "HOST_CAPACITY_SOURCE_STALE";
|
|
742
|
+
const CAPACITY_STALE_RULE =
|
|
743
|
+
"HOST_CAPACITY_SOURCE_STALE is a capacity flag, not a Helper diagnosis. Read `phase`: transient → arm the printed " +
|
|
744
|
+
"`bb wait 'capacity:…'` and re-run once it fires (never report blocked, never ask for a Helper restart); " +
|
|
745
|
+
"sustained (no beacon ever or >= 10 min) → report BLOCKED quoting `stale_since`. A single refusal is never evidence of an outage.";
|
|
746
|
+
|
|
747
|
+
function humanSecs(secs) {
|
|
748
|
+
if (!Number.isFinite(secs)) return "an unknown time";
|
|
749
|
+
if (secs < 120) return `${secs}s`;
|
|
750
|
+
if (secs < 7200) return `${Math.floor(secs / 60)}m`;
|
|
751
|
+
return `${Math.floor(secs / 3600)}h${Math.floor((secs % 3600) / 60)}m`;
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
/** Operator/agent-facing guidance for a stale-capacity refusal (or null for any other code). */
|
|
755
|
+
export function capacityStaleGuidance(data) {
|
|
756
|
+
if (data?.code !== HOST_CAPACITY_SOURCE_STALE_CODE) return null;
|
|
757
|
+
const host = data.host_key ? ` on ${data.host_key}` : "";
|
|
758
|
+
if (data.phase === "transient") {
|
|
759
|
+
const beacon = Number.isFinite(data.beacon_age_secs) ? `the Helper beaconed ${humanSecs(data.beacon_age_secs)} ago` : "no beacon recorded yet";
|
|
760
|
+
return `${yellow("⚠")} stack: capacity flag is stale${host} (${beacon}). Scheduler lag, not a Helper outage. ` +
|
|
761
|
+
"Do NOT report blocked or ask for a Helper restart.\n" +
|
|
762
|
+
(data.wait ? ` Wait for capacity, then re-run: bb wait '${data.wait}' (or pass --wait-capacity <dur>)` : ` ${CAPACITY_STALE_RULE}`);
|
|
763
|
+
}
|
|
764
|
+
if (data.phase === "sustained") {
|
|
765
|
+
const age = Number.isFinite(data.beacon_age_secs) ? `for ${humanSecs(data.beacon_age_secs)}` : "ever recorded";
|
|
766
|
+
return `${yellow("⚠")} stack: no Helper beacon ${age}${host} (stale since ${data.stale_since ?? "unknown"}) — ` +
|
|
767
|
+
"report BLOCKED quoting stale_since; check `launchctl print user/$UID/com.botbuddy.helper.worker` and the worker log, " +
|
|
768
|
+
"or restart BotBuddy Helper.";
|
|
769
|
+
}
|
|
770
|
+
// A pre-BOT-2024 server sends no phase: give the rule, never a restart ask.
|
|
771
|
+
return `${yellow("⚠")} stack: ${CAPACITY_STALE_RULE}`;
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
/** The refusal's phase fields worth keeping on a receipt (empty for any other refusal). */
|
|
775
|
+
export function capacityStaleReceiptFields(data) {
|
|
776
|
+
if (data?.code !== HOST_CAPACITY_SOURCE_STALE_CODE) return {};
|
|
777
|
+
const out = {};
|
|
778
|
+
for (const k of ["phase", "wait", "stale_since", "beacon_age_secs", "stale_for_secs"]) {
|
|
779
|
+
if (data[k] !== undefined) out[k] = data[k];
|
|
780
|
+
}
|
|
781
|
+
return out;
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/** `--wait-capacity <dur>`: `90`, `90s`, `10m`, `1h` → seconds (null when invalid). */
|
|
785
|
+
export function parseWaitCapacityDuration(raw) {
|
|
786
|
+
const m = /^([1-9]\d*)([smh]?)$/.exec(String(raw ?? "").trim());
|
|
787
|
+
if (!m) return null;
|
|
788
|
+
return Number(m[1]) * ({ "": 1, s: 1, m: 60, h: 3600 })[m[2]];
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
const BOTBUDDY_BIN = join(dirname(fileURLToPath(import.meta.url)), "..", "bin", "botbuddy.mjs");
|
|
792
|
+
|
|
793
|
+
/** argv for the park: this CLI's own `bb wait` on the server's condition, bounded by --timeout. */
|
|
794
|
+
export function capacityWaitArgv(condition, timeoutSec) {
|
|
795
|
+
return [process.execPath, BOTBUDDY_BIN, "wait", condition, "--timeout", String(timeoutSec)];
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
/**
|
|
799
|
+
* Park on the server's capacity wait via the real `bb wait` (registration, ranked
|
|
800
|
+
* grants, beacon-stale handling, checkpoint) — never a shell sleep/poll loop. Its
|
|
801
|
+
* receipt is captured; its stderr (arm notices) passes through.
|
|
802
|
+
*/
|
|
803
|
+
export function defaultWaitCapacity(condition, timeoutSec, { signal } = {}) {
|
|
804
|
+
const [cmd, ...args] = capacityWaitArgv(condition, timeoutSec);
|
|
805
|
+
return new Promise((resolve) => {
|
|
806
|
+
// nosemgrep: javascript.lang.security.detect-child-process.detect-child-process -- fixed argv (this CLI's own bin), no shell.
|
|
807
|
+
const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "inherit"], signal });
|
|
808
|
+
let stdout = "";
|
|
809
|
+
child.stdout.on("data", (d) => { stdout += d; });
|
|
810
|
+
const receiptOf = () => {
|
|
811
|
+
const last = stdout.trim().split(/\r?\n/).filter(Boolean).at(-1);
|
|
812
|
+
try { return last ? JSON.parse(last) : null; } catch { return null; }
|
|
813
|
+
};
|
|
814
|
+
child.on("error", (error) => resolve({ exitCode: EXIT.INTERNAL, receipt: null, error: error?.message ?? String(error) }));
|
|
815
|
+
child.on("close", (code) => resolve({ exitCode: code ?? EXIT.INTERNAL, receipt: receiptOf() }));
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
/**
|
|
820
|
+
* BOT-2024: request a lease; on a TRANSIENT stale-capacity refusal with
|
|
821
|
+
* `--wait-capacity`, park on the server's `wait` condition and re-attempt admission
|
|
822
|
+
* exactly once — ONLY when the wait MATCHED (exit 0). Sustained (or phase-less)
|
|
823
|
+
* refusals never park. The deadline returns `capacityTimeout`; any other nonzero wait
|
|
824
|
+
* exit (auth/backend failure, the host-pinned capacity_source_stale exit after its
|
|
825
|
+
* beacon-confirm window, …) is the wait's own terminal outcome and returns
|
|
826
|
+
* `waitFailed` for the caller to propagate typed — retrying there would mask it
|
|
827
|
+
* (Codex #1018 P2).
|
|
828
|
+
*/
|
|
829
|
+
export async function requestWithCapacityWait(request, opts, { waitCapacity = defaultWaitCapacity, log, signal = null } = {}) {
|
|
830
|
+
let req = await request();
|
|
831
|
+
const stale = (r) => r?.ok && r.data && !r.data.success && r.data.code === HOST_CAPACITY_SOURCE_STALE_CODE;
|
|
832
|
+
if (!stale(req)) return { req };
|
|
833
|
+
const guidance = capacityStaleGuidance(req.data);
|
|
834
|
+
if (guidance) log(guidance);
|
|
835
|
+
if (!opts.waitCapacity || req.data.phase !== "transient" || !req.data.wait) return { req };
|
|
836
|
+
log(`${dim("…")} stack: --wait-capacity: parking up to ${opts.waitCapacity}s on bb wait '${req.data.wait}'`);
|
|
837
|
+
const parked = await waitCapacity(req.data.wait, opts.waitCapacity, { signal });
|
|
838
|
+
if (signal?.aborted) return { req, interrupted: true };
|
|
839
|
+
if (parked?.exitCode === EXIT.TIMEOUT) return { req, capacityTimeout: true, waitReceipt: parked.receipt ?? null };
|
|
840
|
+
if (parked?.exitCode !== 0) {
|
|
841
|
+
const exitCode = Number.isInteger(parked?.exitCode) ? parked.exitCode : EXIT.INTERNAL;
|
|
842
|
+
const detail = parked?.receipt?.error ?? parked?.error ?? parked?.receipt?.outcome ?? "no receipt";
|
|
843
|
+
log(`${yellow("⚠")} stack: --wait-capacity: bb wait '${req.data.wait}' exited ${exitCode} (${detail}); not retrying admission.`);
|
|
844
|
+
return { req, waitFailed: { exitCode, receipt: parked?.receipt ?? null, detail } };
|
|
845
|
+
}
|
|
846
|
+
req = await request();
|
|
847
|
+
if (stale(req)) {
|
|
848
|
+
const again = capacityStaleGuidance(req.data);
|
|
849
|
+
if (again) log(again);
|
|
850
|
+
}
|
|
851
|
+
return { req };
|
|
852
|
+
}
|
|
853
|
+
|
|
711
854
|
// The MCP lease endpoint accepts a session token in x-agent-api-key, whereas
|
|
712
855
|
// the event-stream relay requires that same token in its bearer form too. Keep
|
|
713
856
|
// the lease RPC shape unchanged and derive the relay-compatible form only at
|
|
@@ -1355,6 +1498,8 @@ export async function cmdUp(opts, {
|
|
|
1355
1498
|
machineUuidFn = machineUuid,
|
|
1356
1499
|
assertIsolated = validateLocalExecTarget,
|
|
1357
1500
|
refreshAuth = refreshRejectedCacheAuth,
|
|
1501
|
+
reportLaneStatus = writeLaneStatus,
|
|
1502
|
+
waitCapacity = defaultWaitCapacity,
|
|
1358
1503
|
} = {}) {
|
|
1359
1504
|
let slot;
|
|
1360
1505
|
try { slot = deriveSlot(opts); } catch (e) {
|
|
@@ -1437,11 +1582,58 @@ export async function cmdUp(opts, {
|
|
|
1437
1582
|
? emitResult(buildReceipt({ command: "up", outcome: "error", error: req.error || "unauthorized" }), opts, EXIT.AUTH)
|
|
1438
1583
|
: emitResult(buildReceipt({ command: "up", outcome: "error", error: req.error || "request failed" }), opts, EXIT.BACKEND);
|
|
1439
1584
|
}
|
|
1585
|
+
// BOT-2024: phase-aware stale-capacity guidance; --wait-capacity parks a TRANSIENT
|
|
1586
|
+
// refusal on the server's capacity wait and re-admits once.
|
|
1587
|
+
if (req.data && !req.data.success && req.data.code === HOST_CAPACITY_SOURCE_STALE_CODE) {
|
|
1588
|
+
let first = req;
|
|
1589
|
+
// Codex #1018 R4: the park can outlive the pre-wait credential (expired/revoked
|
|
1590
|
+
// while parked; the spawned `bb wait` may itself have renewed the worktree
|
|
1591
|
+
// session). So the post-wait retry RE-LOADS the credential from disk, then takes
|
|
1592
|
+
// the standard refresh-once-on-auth-rejection path. A second rejection falls
|
|
1593
|
+
// through to the typed AUTH exit below — never a loop.
|
|
1594
|
+
const retryAfterWait = async () => {
|
|
1595
|
+
const reloaded = await authProvider();
|
|
1596
|
+
if (reloaded) auth = reloaded;
|
|
1597
|
+
let retried = await call("request_stack_lease", leaseArgs);
|
|
1598
|
+
if (!retried.ok && retried.auth) {
|
|
1599
|
+
const fresh = await refreshAuth(auth, { cwd: process.cwd() });
|
|
1600
|
+
if (fresh) {
|
|
1601
|
+
auth = fresh;
|
|
1602
|
+
retried = await call("request_stack_lease", leaseArgs);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
return retried;
|
|
1606
|
+
};
|
|
1607
|
+
const admission = await requestWithCapacityWait(
|
|
1608
|
+
async () => { if (first) { const r = first; first = null; return r; } return retryAfterWait(); },
|
|
1609
|
+
opts, { waitCapacity, log: (line) => process.stderr.write(`${line}\n`) },
|
|
1610
|
+
);
|
|
1611
|
+
if (admission.capacityTimeout) {
|
|
1612
|
+
return emitResult(buildReceipt({
|
|
1613
|
+
command: "up", outcome: "timeout", code: admission.req.data.code, ...capacityStaleReceiptFields(admission.req.data),
|
|
1614
|
+
wait_exit_code: EXIT.TIMEOUT, wait_receipt: admission.waitReceipt,
|
|
1615
|
+
error: `capacity did not recover within --wait-capacity ${opts.waitCapacity}s (${admission.req.data.code})`, slot,
|
|
1616
|
+
}), opts, EXIT.TIMEOUT);
|
|
1617
|
+
}
|
|
1618
|
+
if (admission.waitFailed) {
|
|
1619
|
+
return emitResult(buildReceipt({
|
|
1620
|
+
command: "up", outcome: "error", code: admission.req.data.code, ...capacityStaleReceiptFields(admission.req.data),
|
|
1621
|
+
wait_exit_code: admission.waitFailed.exitCode, wait_receipt: admission.waitFailed.receipt,
|
|
1622
|
+
error: `--wait-capacity: bb wait exited ${admission.waitFailed.exitCode} (${admission.waitFailed.detail})`, slot,
|
|
1623
|
+
}), opts, admission.waitFailed.exitCode);
|
|
1624
|
+
}
|
|
1625
|
+
req = admission.req;
|
|
1626
|
+
if (!req.ok) {
|
|
1627
|
+
return emitResult(buildReceipt({ command: "up", outcome: "error", error: req.error || "request failed" }), opts, req.auth ? EXIT.AUTH : EXIT.BACKEND);
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1440
1630
|
const d = req.data;
|
|
1441
1631
|
if (!d.success) {
|
|
1442
1632
|
const hint = worktreeRegistrationHint(d.code, auth) ?? stackPathRootRefusalNotice(d.code, d.message);
|
|
1443
1633
|
if (hint) process.stderr.write(`${hint}\n`);
|
|
1444
|
-
return emitResult(buildReceipt({
|
|
1634
|
+
return emitResult(buildReceipt({
|
|
1635
|
+
command: "up", outcome: "error", code: d.code, ...capacityStaleReceiptFields(d), error: leaseRefusalError(d), slot,
|
|
1636
|
+
}), opts, leaseRefusalExit(d.code));
|
|
1445
1637
|
}
|
|
1446
1638
|
let leaseId = d.lease_id;
|
|
1447
1639
|
let state = d.state;
|
|
@@ -1451,6 +1643,9 @@ export async function cmdUp(opts, {
|
|
|
1451
1643
|
const reusedExisting = state === "active";
|
|
1452
1644
|
|
|
1453
1645
|
if (state === "queued") {
|
|
1646
|
+
// BOT-1319: a wrapped lefthook lane records this park as waiting for capacity.
|
|
1647
|
+
const where = [d.queue_position != null ? `queue position ${d.queue_position}` : null, d.host_key ? `on ${d.host_key}` : null].filter(Boolean).join(" ");
|
|
1648
|
+
reportLaneStatus({ phase: "waiting", reason: "queued_for_capacity", detail: where || "host at capacity" });
|
|
1454
1649
|
if (opts.noWait) {
|
|
1455
1650
|
return emitResult(buildReceipt({ command: "up", outcome: "queued", lease_id: leaseId, state, host_key: d.host_key, slot, queued: true, queue_position: d.queue_position, holders: d.holders }), opts, EXIT.QUEUED);
|
|
1456
1651
|
}
|
|
@@ -1461,6 +1656,7 @@ export async function cmdUp(opts, {
|
|
|
1461
1656
|
if (parked.error) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: parked.error }), opts, EXIT.BACKEND);
|
|
1462
1657
|
state = parked.state || "provisioning";
|
|
1463
1658
|
}
|
|
1659
|
+
reportLaneStatus({ phase: "testing", reason: "provisioning" });
|
|
1464
1660
|
|
|
1465
1661
|
// Reach `active`: local-exec provisions itself; otherwise wait for the Helper.
|
|
1466
1662
|
if (state !== "active") {
|
|
@@ -2030,6 +2226,7 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
|
|
|
2030
2226
|
const clock = adapters.clock ?? { setInterval, clearInterval, setTimeout, clearTimeout };
|
|
2031
2227
|
const signals = adapters.signals ?? process;
|
|
2032
2228
|
const provisionTimeout = opts.provisionTimeout ?? opts.timeout;
|
|
2229
|
+
const stderrLine = adapters.stderr ?? ((line) => process.stderr.write(`${line}\n`));
|
|
2033
2230
|
let leaseId = null;
|
|
2034
2231
|
let child = null;
|
|
2035
2232
|
let childResult = null;
|
|
@@ -2176,7 +2373,34 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
|
|
|
2176
2373
|
};
|
|
2177
2374
|
// The retrying caller (api → rpc.call) transparently clears+self-heals+retries the
|
|
2178
2375
|
// FIRST auth rejection among any of request/get/touch/release.
|
|
2179
|
-
|
|
2376
|
+
// BOT-2024: a stale-capacity refusal prints phase-aware guidance; with the opt-in
|
|
2377
|
+
// --wait-capacity a TRANSIENT one parks on the server's capacity wait and re-admits once.
|
|
2378
|
+
activeWaitAbort = new AbortController();
|
|
2379
|
+
const admission = await requestWithCapacityWait(() => api.request(leaseArgs), opts, {
|
|
2380
|
+
waitCapacity: adapters.waitCapacity ?? defaultWaitCapacity, log: stderrLine, signal: activeWaitAbort.signal,
|
|
2381
|
+
});
|
|
2382
|
+
activeWaitAbort = null;
|
|
2383
|
+
const request = admission.req;
|
|
2384
|
+
// Interrupted DURING the capacity park: the refused request created no lease, so
|
|
2385
|
+
// there is nothing to release. (A signal during a request that DID admit falls
|
|
2386
|
+
// through to the existing post-lease interruption branch, which cleans up.)
|
|
2387
|
+
if (admission.interrupted) {
|
|
2388
|
+
return { exitCode: SIGNAL_EXIT[receivedSignal] ?? EXIT.INTERNAL, outcome: "interrupted", code: request?.data?.code, ...capacityStaleReceiptFields(request?.data) };
|
|
2389
|
+
}
|
|
2390
|
+
if (admission.capacityTimeout) {
|
|
2391
|
+
return {
|
|
2392
|
+
exitCode: EXIT.TIMEOUT, outcome: "timeout", code: request.data.code, ...capacityStaleReceiptFields(request.data),
|
|
2393
|
+
wait_exit_code: EXIT.TIMEOUT, wait_receipt: admission.waitReceipt,
|
|
2394
|
+
error: `capacity did not recover within --wait-capacity ${opts.waitCapacity}s (${request.data.code})`,
|
|
2395
|
+
};
|
|
2396
|
+
}
|
|
2397
|
+
if (admission.waitFailed) {
|
|
2398
|
+
return {
|
|
2399
|
+
exitCode: admission.waitFailed.exitCode, outcome: "error", code: request.data.code, ...capacityStaleReceiptFields(request.data),
|
|
2400
|
+
wait_exit_code: admission.waitFailed.exitCode, wait_receipt: admission.waitFailed.receipt,
|
|
2401
|
+
error: `--wait-capacity: bb wait exited ${admission.waitFailed.exitCode} (${admission.waitFailed.detail})`,
|
|
2402
|
+
};
|
|
2403
|
+
}
|
|
2180
2404
|
if (!request?.ok) {
|
|
2181
2405
|
// Codex #928 R23 (P2): `status: null` (callToolJson's fetch-threw branch) means
|
|
2182
2406
|
// NO response was ever received — the request may have reached the backend and
|
|
@@ -2192,8 +2416,11 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
|
|
|
2192
2416
|
if (!request.data?.success) {
|
|
2193
2417
|
const hint = worktreeRegistrationHint(request.data?.code, rpc.auth)
|
|
2194
2418
|
?? stackPathRootRefusalNotice(request.data?.code, request.data?.message);
|
|
2195
|
-
if (hint)
|
|
2196
|
-
return {
|
|
2419
|
+
if (hint) stderrLine(hint);
|
|
2420
|
+
return {
|
|
2421
|
+
exitCode: leaseRefusalExit(request.data?.code), outcome: "error", code: request.data?.code,
|
|
2422
|
+
...capacityStaleReceiptFields(request.data), error: leaseRefusalError(request.data),
|
|
2423
|
+
};
|
|
2197
2424
|
}
|
|
2198
2425
|
leaseId = request.data.lease_id;
|
|
2199
2426
|
if (request.data.reused) {
|
|
@@ -2341,6 +2568,9 @@ async function cmdRun(opts, childArgv) {
|
|
|
2341
2568
|
// BOT-1797: carry the server's typed refusal code (e.g. stack_path_root_refused)
|
|
2342
2569
|
// so automation can branch on it instead of matching the message text.
|
|
2343
2570
|
command: "run", outcome: result.outcome, code: result.code ?? null, lease_id: result.leaseId ?? null,
|
|
2571
|
+
// BOT-2024: a stale-capacity refusal carries its phase + armable wait on the receipt.
|
|
2572
|
+
...capacityStaleReceiptFields(result),
|
|
2573
|
+
...(result.wait_exit_code != null ? { wait_exit_code: result.wait_exit_code, wait_receipt: result.wait_receipt ?? null } : {}),
|
|
2344
2574
|
child_exit_code: result.childExitCode ?? null, cleanup: result.cleanup?.ok ?? null,
|
|
2345
2575
|
fenced: result.fenced ?? false, error: result.error ?? result.cleanup?.error ?? null,
|
|
2346
2576
|
}), opts, result.exitCode);
|
package/src/telemetry-outbox.mjs
CHANGED
|
@@ -323,11 +323,19 @@ export async function createTelemetryOutbox({
|
|
|
323
323
|
try { imported = JSON.parse(await readFile(legacyImportPath, "utf8")); } catch { /* first migration */ }
|
|
324
324
|
let names = [];
|
|
325
325
|
try { names = (await readdir(legacyDirectory)).filter((name) => name.endsWith(".json")); } catch (error) { if (error?.code === "ENOENT") return { imported: 0, skipped: 0 }; throw error; }
|
|
326
|
+
// BOT-1319: the test-ingest reporter archives what it posted under reported/
|
|
327
|
+
// before the next hook can recover it; a summary whose lifecycle was degraded
|
|
328
|
+
// is still eligible there (others under reported/ were already counted).
|
|
329
|
+
let archived = [];
|
|
330
|
+
try { archived = (await readdir(join(legacyDirectory, "reported"))).filter((name) => name.endsWith(".json")); } catch { /* none archived */ }
|
|
331
|
+
const entries = [
|
|
332
|
+
...names.sort().map((name) => ({ name, file: join(legacyDirectory, name), archived: false })),
|
|
333
|
+
...archived.sort().map((name) => ({ name, file: join(legacyDirectory, "reported", name), archived: true })),
|
|
334
|
+
];
|
|
326
335
|
let added = 0;
|
|
327
336
|
let skipped = 0;
|
|
328
|
-
for (const name of
|
|
337
|
+
for (const { name, file, archived: fromArchive } of entries) {
|
|
329
338
|
if (added >= maxReceipts || Date.now() >= deadline) break;
|
|
330
|
-
const file = join(legacyDirectory, name);
|
|
331
339
|
let raw;
|
|
332
340
|
try { raw = await readFile(file, "utf8"); } catch { skipped += 1; continue; }
|
|
333
341
|
const fingerprint = hash(`${name}\0${raw}`);
|
|
@@ -335,13 +343,30 @@ export async function createTelemetryOutbox({
|
|
|
335
343
|
let summary;
|
|
336
344
|
try { summary = JSON.parse(raw); } catch { skipped += 1; continue; }
|
|
337
345
|
if (!summary || typeof summary !== "object" || typeof summary.runner !== "string") { skipped += 1; continue; }
|
|
346
|
+
if (fromArchive && summary.lifecycle?.telemetry !== "degraded") continue;
|
|
347
|
+
// BOT-1319: the lane wrapper (and anything nested in a wrapped lane) stamps
|
|
348
|
+
// `lifecycle` when a canonical execution already carries this run. Importing
|
|
349
|
+
// it too would count the same run twice; record it as seen and move on. A
|
|
350
|
+
// wrapper summary stamped `telemetry: "degraded"` never reached the outbox
|
|
351
|
+
// durably, so it stays eligible and imports like a historical summary.
|
|
352
|
+
const lifecycle = summary.lifecycle;
|
|
353
|
+
if (lifecycle && typeof lifecycle === "object" && (lifecycle.owned_by || lifecycle.telemetry === "durable")) {
|
|
354
|
+
imported[fingerprint] = { name, imported_at: now().toISOString(), lifecycle: true };
|
|
355
|
+
skipped += 1;
|
|
356
|
+
continue;
|
|
357
|
+
}
|
|
338
358
|
const sourceRunId = `legacy-${fingerprint.slice(0, 32)}`;
|
|
339
359
|
const started = typeof summary.started_at === "string" ? summary.started_at : now().toISOString();
|
|
340
360
|
const finished = typeof summary.finished_at === "string" ? summary.finished_at : now().toISOString();
|
|
341
|
-
|
|
342
|
-
const
|
|
361
|
+
// An explicit skip (BOT-1319 wrapper, verdict "skip", exit 0) is never green.
|
|
362
|
+
const isSkip = summary.verdict === "skip";
|
|
363
|
+
const passed = !isSkip && (summary.verdict === "pass" || summary.exit_code === 0);
|
|
364
|
+
const context = { repo: repository, environment: "local", runner: summary.runner, phase: passed || isSkip ? "complete" : "failed", health: "uninstrumented", ...(isSkip ? { skipped: 1 } : {}) };
|
|
365
|
+
const terminal = isSkip
|
|
366
|
+
? { outcome: "canceled", diagnostic: { code: "skipped", category: "skip", message: String(summary.skipped_reason ?? "skipped").slice(0, 500) } }
|
|
367
|
+
: { outcome: passed ? "passed" : "failed" };
|
|
343
368
|
await append({ schema_version: 1, action: "start", producer_kind: "hook", source_run_id: sourceRunId, attempt: 1, producer_event_id: `${sourceRunId}:start`, producer_sequence: 0, occurred_at: started, context });
|
|
344
|
-
await append({ schema_version: 1, action: "terminal", producer_kind: "hook", source_run_id: sourceRunId, attempt: 1, producer_event_id: `${sourceRunId}:terminal`, producer_sequence: 1, occurred_at: finished, context,
|
|
369
|
+
await append({ schema_version: 1, action: "terminal", producer_kind: "hook", source_run_id: sourceRunId, attempt: 1, producer_event_id: `${sourceRunId}:terminal`, producer_sequence: 1, occurred_at: finished, context, ...terminal });
|
|
345
370
|
imported[fingerprint] = { name, imported_at: now().toISOString() };
|
|
346
371
|
added += 1;
|
|
347
372
|
}
|
package/src/wait-core.mjs
CHANGED
|
@@ -1179,13 +1179,13 @@ export async function runWaitLoop({
|
|
|
1179
1179
|
// the primary hysteresis); 0 = exit immediately on the stale signal. Injectable so
|
|
1180
1180
|
// a clock-controlled test can drive it. Not a grammar surface (no new condition name).
|
|
1181
1181
|
//
|
|
1182
|
-
// Default MUST exceed the beacon evaluator's worst-case cadence (Codex P2).
|
|
1183
|
-
// recovery transition
|
|
1184
|
-
// cron,
|
|
1185
|
-
//
|
|
1186
|
-
//
|
|
1187
|
-
//
|
|
1188
|
-
//
|
|
1182
|
+
// Default MUST exceed the beacon evaluator's worst-case cadence (Codex P2). Before
|
|
1183
|
+
// BOT-2024 the recovery transition was emitted only by the `* * * * *` (60s)
|
|
1184
|
+
// eval_container_beacon_staleness cron, so a host that resumed polling right after a
|
|
1185
|
+
// stale emission had its recovery frame arrive up to ~60s later. Since BOT-2024 the
|
|
1186
|
+
// beacon itself (claim_container_job / attested registerHost) emits the recovery, so
|
|
1187
|
+
// the frame now lands on the Helper's next beacon; 90s still covers one ~45-60s
|
|
1188
|
+
// Helper beacon interval plus margin, and an older server's cron-only recovery.
|
|
1189
1189
|
beaconConfirmMs = 90_000,
|
|
1190
1190
|
// BOT-1837: the arm-time notice (if any) to carry into a TIMEOUT receipt's
|
|
1191
1191
|
// `error`, so the receipt itself says what CI was already doing when this wait
|