@botbuddy/cli 1.59.4 → 1.59.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/commands.mjs +81 -0
- package/src/stack.mjs +166 -24
- package/src/test-lane.mjs +16 -5
- package/src/wait-core.mjs +99 -148
- package/src/wait.mjs +73 -10
package/package.json
CHANGED
package/src/commands.mjs
CHANGED
|
@@ -212,6 +212,8 @@ ${bold("MCP CONFIG KEY")}
|
|
|
212
212
|
${bold("TOOLS")}
|
|
213
213
|
help --tools List every BotBuddy tool
|
|
214
214
|
help <tool> Show one tool's arguments
|
|
215
|
+
help test Safe test-lane launch and recovery guidance
|
|
216
|
+
help symptom <name> Diagnose a known block without source-diving
|
|
215
217
|
call <tool> [--key value] Invoke any tool
|
|
216
218
|
call <tool> --json '{...}' Invoke with a JSON arguments object
|
|
217
219
|
call <tool> --pin <slug> Pin the connection tenant (a user token needs one;
|
|
@@ -362,6 +364,11 @@ async function cmdCall(args) {
|
|
|
362
364
|
}
|
|
363
365
|
|
|
364
366
|
async function cmdToolHelp(args) {
|
|
367
|
+
const local = recoveryHelp(args);
|
|
368
|
+
if (local) {
|
|
369
|
+
console.log(local);
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
365
372
|
// `help --tools` (or `help --all`) lists the catalog; `help <tool>` details one.
|
|
366
373
|
const tool = args[0].startsWith("-") ? null : args[0];
|
|
367
374
|
const res = await fetch(discoveryUrlFor(SERVER_URL, tool));
|
|
@@ -373,6 +380,80 @@ async function cmdToolHelp(args) {
|
|
|
373
380
|
console.log(formatDiscovery(await res.json(), { tool }));
|
|
374
381
|
}
|
|
375
382
|
|
|
383
|
+
// Keep the most common lifecycle recoveries local: an agent must be able to
|
|
384
|
+
// discover the safe next action even when the MCP discovery endpoint is down.
|
|
385
|
+
// This deliberately names only stable, already-shipped commands. New lifecycle
|
|
386
|
+
// commands belong in their owning feature ticket before they are documented here.
|
|
387
|
+
export function recoveryHelp(args) {
|
|
388
|
+
const topic = args.map((arg) => String(arg).trim().toLowerCase()).filter(Boolean).join(" ");
|
|
389
|
+
if (topic === "test") return `${bold("bb help test")} — launch tests without losing their evidence
|
|
390
|
+
|
|
391
|
+
${bold("SAFE DEFAULT")}
|
|
392
|
+
1. Run ${cyan("bb doctor --json")} from the target worktree.
|
|
393
|
+
2. Run the repository's configured test command without manually copying a
|
|
394
|
+
session ID or session token unless the harness explicitly requires it.
|
|
395
|
+
3. Keep the command's original result and receipt path. Do not start a second
|
|
396
|
+
full suite merely because telemetry is missing.
|
|
397
|
+
|
|
398
|
+
${bold("WHEN TEST TELEMETRY IS UNAVAILABLE")}
|
|
399
|
+
Run ${cyan("bb help symptom telemetry-unavailable")}. A null test-run ID means
|
|
400
|
+
there is no semantic test-run wait to arm; it is not a passing result.
|
|
401
|
+
|
|
402
|
+
${bold("WAITING")}
|
|
403
|
+
Use ${cyan("bb wait 'test-run:id=<id>' --heartbeat")} only when the launch output
|
|
404
|
+
contains a real test-run ID. Do not replace a missing lifecycle signal with a
|
|
405
|
+
timer or status-polling loop.`;
|
|
406
|
+
|
|
407
|
+
if (topic === "symptom" || topic === "symptoms") return `${bold("bb help symptom <name>")} — supported recovery topics
|
|
408
|
+
|
|
409
|
+
${cyan("telemetry-unavailable")} A test launch has no test-run ID or wait command.
|
|
410
|
+
${cyan("command-run-stuck")} A local command receipt and remote lifecycle disagree.
|
|
411
|
+
|
|
412
|
+
Choose the symptom printed by the command. If neither applies, preserve the
|
|
413
|
+
original command result and receipt, then report the exact error and command ID.`;
|
|
414
|
+
|
|
415
|
+
if (topic === "symptom telemetry-unavailable") return `${bold("Symptom: telemetry unavailable")}
|
|
416
|
+
|
|
417
|
+
${bold("Meaning")}
|
|
418
|
+
The workload may have started, but BotBuddy did not create a test-run record.
|
|
419
|
+
There is no ${cyan("test-run:id")} wait to arm and silence is never a green result.
|
|
420
|
+
|
|
421
|
+
${bold("Safe recovery")}
|
|
422
|
+
1. Do not launch another full suite yet.
|
|
423
|
+
2. Preserve the original ${cyan("command_run_id")} and receipt path from its output.
|
|
424
|
+
3. Read the terminal receipt when the original runner returns. A receipt with a
|
|
425
|
+
terminal status is evidence of that workload's result.
|
|
426
|
+
4. If the receipt is not terminal, keep the original runner result pending;
|
|
427
|
+
do not poll it or use a timer as a completion signal.
|
|
428
|
+
5. After a terminal receipt, fix the launch/authentication cause and rerun the
|
|
429
|
+
smallest affected scope once, with an explicit rerun reason when supported.
|
|
430
|
+
|
|
431
|
+
${bold("Escalation payload")}
|
|
432
|
+
Include CLI version, exact command (with secrets removed), command-run ID,
|
|
433
|
+
receipt status/exit code, and whether a test-run ID was created.`;
|
|
434
|
+
|
|
435
|
+
if (topic === "symptom command-run-stuck") return `${bold("Symptom: command run appears stuck")}
|
|
436
|
+
|
|
437
|
+
${bold("Meaning")}
|
|
438
|
+
A remote lifecycle record can lag or fail to receive a terminal callback. It
|
|
439
|
+
does not prove that the workload is still running.
|
|
440
|
+
|
|
441
|
+
${bold("Safe recovery")}
|
|
442
|
+
1. Preserve the command-run ID and the original result.
|
|
443
|
+
2. Prefer the owning runner's local terminal receipt for the workload verdict.
|
|
444
|
+
3. If the receipt is terminal, do not cancel or duplicate the command solely
|
|
445
|
+
because remote state remains ${cyan("registered")} or ${cyan("running")}.
|
|
446
|
+
4. If the receipt is absent/non-terminal, retain the original runner result;
|
|
447
|
+
do not poll or infer completion from elapsed time.
|
|
448
|
+
5. Report the receipt and remote discrepancy as telemetry evidence.
|
|
449
|
+
|
|
450
|
+
${bold("Boundary")}
|
|
451
|
+
Command-run status is agent-owned. A different agent may be denied access; that
|
|
452
|
+
denial is not evidence that the run is missing or terminal.`;
|
|
453
|
+
|
|
454
|
+
return null;
|
|
455
|
+
}
|
|
456
|
+
|
|
376
457
|
// BOT-1566 C1: a tenant slug as the server accepts it on `?tenant=` — lowercase
|
|
377
458
|
// alphanumerics and hyphens, 2–63 chars, not starting with a hyphen.
|
|
378
459
|
export const TENANT_SLUG_RE = /^[a-z0-9][a-z0-9-]{1,62}$/;
|
package/src/stack.mjs
CHANGED
|
@@ -64,7 +64,16 @@ export const STACK_SCHEMA_VERSION = 1;
|
|
|
64
64
|
*/
|
|
65
65
|
export const CALLBACK_PORT_UNSUPPORTED_REMOTE = "CALLBACK_PORT_UNSUPPORTED_REMOTE";
|
|
66
66
|
export const DEFAULT_RECEIPT_MAX_BYTES = 10240;
|
|
67
|
-
|
|
67
|
+
/**
|
|
68
|
+
* BOT-2400: a capacity park waits for days, not an hour. A full host costs a wait, never a
|
|
69
|
+
* failed run plus a token-expensive re-arm; and when even this deadline passes the lease keeps
|
|
70
|
+
* its queue place (PARK_DEADLINE) instead of being released.
|
|
71
|
+
*/
|
|
72
|
+
export const DEFAULT_TIMEOUT_SEC = 72 * 3600;
|
|
73
|
+
/** BOT-2400: the typed code on a park that reached its deadline with the lease still queued and held. */
|
|
74
|
+
export const PARK_DEADLINE = "PARK_DEADLINE";
|
|
75
|
+
/** A park ended but the owner hold could not be confirmed; the lease is never cancelled. */
|
|
76
|
+
export const PARK_HOLD_UNCONFIRMED = "PARK_HOLD_UNCONFIRMED";
|
|
68
77
|
/**
|
|
69
78
|
* BOT-2154: how long `stack up --local-exec` lets `supabase start` run before it gives up and
|
|
70
79
|
* releases the lease itself (`--provision-timeout` overrides). Deliberately below the server's
|
|
@@ -74,7 +83,10 @@ export const DEFAULT_TIMEOUT_SEC = 3600;
|
|
|
74
83
|
export const LOCAL_EXEC_PROVISION_TIMEOUT_SEC = 900;
|
|
75
84
|
/** BOT-2230: the bounds the control plane enforces on an owner place hold (hold_stack_lease_place). */
|
|
76
85
|
export const HOLD_ON_TIMEOUT_MIN_SEC = 60;
|
|
77
|
-
|
|
86
|
+
/** BOT-2400: raised from 4 h to 72 h (hold_stack_lease_place_for_agent enforces the same bound). */
|
|
87
|
+
export const HOLD_ON_TIMEOUT_MAX_SEC = 72 * 3600;
|
|
88
|
+
/** The bound a pre-BOT-2400 control plane enforces; a refused longer hold is retried at it. */
|
|
89
|
+
export const LEGACY_HOLD_MAX_SEC = 14400;
|
|
78
90
|
/** How often a run renews its stack directory's retention while it still exists
|
|
79
91
|
* (BOT-1798): frequent enough to cover the pre-child queue/provision waits. */
|
|
80
92
|
const RETENTION_REFRESH_MS = 5 * 60 * 1000;
|
|
@@ -99,6 +111,7 @@ ${bold("USAGE")}
|
|
|
99
111
|
botbuddy stack status <lease_id> Show a lease's state + connection
|
|
100
112
|
botbuddy stack touch <lease_id> Bump the idle clock (and report measured use) so the reaper doesn't stop it
|
|
101
113
|
botbuddy stack done <lease_id> Release (done): idle the stack for re-adoption (--reap tears it down)
|
|
114
|
+
botbuddy stack cancel <lease_id> Give a queued lease's place up (= done --reap; BOT-2400)
|
|
102
115
|
botbuddy stack run [options] -- <executable> [args...]
|
|
103
116
|
Own one Helper-backed test/fix batch end-to-end
|
|
104
117
|
|
|
@@ -117,11 +130,17 @@ ${bold("up OPTIONS")}
|
|
|
117
130
|
or use 'stack run', which materializes one for you.
|
|
118
131
|
--purpose <text> Free-text purpose recorded on the lease.
|
|
119
132
|
--idle-ttl <seconds> Idle seconds before the reaper STOPS an unused stack (default 1800).
|
|
120
|
-
--timeout <seconds>
|
|
133
|
+
--timeout <seconds> Seconds to park for capacity (default ${DEFAULT_TIMEOUT_SEC} = 72 h, BOT-2400). At the
|
|
134
|
+
deadline the lease is NOT released: it keeps its queue place (owner hold) and the
|
|
135
|
+
exit-2 receipt says code PARK_DEADLINE, state queued, queue_hold — re-run the same
|
|
136
|
+
command to resume it; \`stack cancel <lease_id>\` gives it up.
|
|
137
|
+
If the hold cannot be confirmed, PARK_HOLD_UNCONFIRMED keeps ownership and
|
|
138
|
+
omits unverified state; inspect \`stack status <lease_id>\` before resuming.
|
|
121
139
|
--no-wait If the host is full, print the queue position and exit (don't park).
|
|
122
140
|
--hold-on-timeout <seconds>
|
|
123
141
|
BOT-2230: when the --timeout park runs out with the lease still queued,
|
|
124
|
-
keep its queue place for <seconds> (60–
|
|
142
|
+
keep its queue place for <seconds> (60–259200; default since BOT-2400: 259200)
|
|
143
|
+
instead of losing it: the
|
|
125
144
|
lease stays queued but is not dispatched, and re-running the same
|
|
126
145
|
\`stack up\` (same agent + slot) before then re-attaches to it. The exit-2
|
|
127
146
|
receipt carries queue_hold {held, until, queue_position}; \`stack done
|
|
@@ -173,6 +192,10 @@ ${bold("run OPTIONS")}
|
|
|
173
192
|
leases that. The identity is STABLE per worktree+slot (BOT-1803), so
|
|
174
193
|
the next run maps to the same directory, project and ports. "." is REFUSED — the worktree root is the shared
|
|
175
194
|
canonical project and the Helper would start/clobber it.
|
|
195
|
+
--timeout <seconds> Same 72 h park as up (BOT-2400): at the deadline the queued lease keeps its
|
|
196
|
+
place (exit 2, code PARK_DEADLINE) and re-running the same \`stack run\`
|
|
197
|
+
re-attaches to it. A slot whose previous lease of yours is still reaping is
|
|
198
|
+
waited out until reaped, then requested again (no exit 5).
|
|
176
199
|
--provision-timeout N Seconds to wait for a physical Helper provision (default: --timeout).
|
|
177
200
|
--reap-timeout N Seconds to wait for signed physical reap proof (default: 300); also bounds
|
|
178
201
|
the wait for an idle lease being re-provisioned/preempted to reach reaped.
|
|
@@ -216,7 +239,8 @@ ${bold("GLOBAL")}
|
|
|
216
239
|
--receipt-max-bytes N Receipt size cap (default ${DEFAULT_RECEIPT_MAX_BYTES}).
|
|
217
240
|
|
|
218
241
|
${bold("EXIT CODES")}
|
|
219
|
-
0 ok/held (or queued+--no-wait) 2 park
|
|
242
|
+
0 ok/held (or queued+--no-wait) 2 park deadline (PARK_DEADLINE: still queued, place kept)
|
|
243
|
+
3 not authenticated
|
|
220
244
|
4 invalid arguments 5 backend/coordination 6 lease failed/reaped
|
|
221
245
|
7 internal error 8 cleanup/reap proof failed
|
|
222
246
|
|
|
@@ -322,7 +346,11 @@ export function deriveSlot({ slot, repo, ticket } = {}, env = process.env) {
|
|
|
322
346
|
*/
|
|
323
347
|
export function parseStackArgs(argv) {
|
|
324
348
|
const errors = [];
|
|
325
|
-
const [
|
|
349
|
+
const [rawCommand, ...rawRest] = argv;
|
|
350
|
+
// BOT-2400: `stack cancel <lease_id>` is the explicit give-up for a queued lease (its place
|
|
351
|
+
// is otherwise kept): exactly `stack done <lease_id> --reap`.
|
|
352
|
+
const cancel = rawCommand === "cancel";
|
|
353
|
+
const command = cancel ? "done" : rawCommand;
|
|
326
354
|
const divider = rawRest.indexOf("--");
|
|
327
355
|
const rest = divider === -1 ? rawRest : rawRest.slice(0, divider);
|
|
328
356
|
const childArgv = divider === -1 ? [] : rawRest.slice(divider + 1);
|
|
@@ -339,7 +367,7 @@ export function parseStackArgs(argv) {
|
|
|
339
367
|
bundleInclude: [],
|
|
340
368
|
disposition: "destroy", json: false, receiptMaxBytes: DEFAULT_RECEIPT_MAX_BYTES,
|
|
341
369
|
// BOT-1803: eager release (reap) instead of the default idle release (done/run).
|
|
342
|
-
reap:
|
|
370
|
+
reap: cancel,
|
|
343
371
|
};
|
|
344
372
|
const positionals = [];
|
|
345
373
|
const need = (name, v) => {
|
|
@@ -575,7 +603,15 @@ export async function holdQueuePlace(call, leaseId, holdSecs) {
|
|
|
575
603
|
let res;
|
|
576
604
|
try { res = await call("hold_stack_lease_place", { lease_id: leaseId, hold_secs: holdSecs }); }
|
|
577
605
|
catch (err) { return { held: false, code: null, error: String(err?.message ?? err) }; }
|
|
578
|
-
|
|
606
|
+
let d = res?.data;
|
|
607
|
+
// BOT-2400: a control plane that predates the 72 h bound refuses a longer hold; keep the
|
|
608
|
+
// place for the longest hold it accepts rather than losing it.
|
|
609
|
+
if (d?.code === "INVALID_HOLD_SECS" && holdSecs > LEGACY_HOLD_MAX_SEC) {
|
|
610
|
+
try { res = await call("hold_stack_lease_place", { lease_id: leaseId, hold_secs: LEGACY_HOLD_MAX_SEC }); }
|
|
611
|
+
catch (err) { return { held: false, code: null, error: String(err?.message ?? err) }; }
|
|
612
|
+
d = res?.data;
|
|
613
|
+
holdSecs = LEGACY_HOLD_MAX_SEC;
|
|
614
|
+
}
|
|
579
615
|
if (res?.ok && d?.success) {
|
|
580
616
|
return { held: true, until: d.queue_hold_until ?? null, queue_position: d.queue_position ?? null, hold_secs: d.hold_secs ?? holdSecs };
|
|
581
617
|
}
|
|
@@ -1188,7 +1224,9 @@ const BOTBUDDY_BIN = join(dirname(fileURLToPath(import.meta.url)), "..", "bin",
|
|
|
1188
1224
|
|
|
1189
1225
|
/** argv for the park: this CLI's own `bb wait` on the server's condition, bounded by --timeout. */
|
|
1190
1226
|
export function capacityWaitArgv(condition, timeoutSec) {
|
|
1191
|
-
|
|
1227
|
+
// BOT-2400: the caller chose this bound with --wait-capacity, so it may be under the floor.
|
|
1228
|
+
return [process.execPath, BOTBUDDY_BIN, "wait", condition, "--timeout", String(timeoutSec),
|
|
1229
|
+
"--allow-short-timeout", "bb stack --wait-capacity park"];
|
|
1192
1230
|
}
|
|
1193
1231
|
|
|
1194
1232
|
/**
|
|
@@ -2441,10 +2479,12 @@ export async function cmdUp(opts, {
|
|
|
2441
2479
|
try { const read = await call("get_stack_lease", { lease_id: leaseId }); if (read?.ok && read.data?.success && read.data.ledger) ledger = read.data.ledger; } catch { /* keep the request-time ledger */ }
|
|
2442
2480
|
for (const line of describeUnitLedger(ledger)) process.stderr.write(`stack: ${line}\n`);
|
|
2443
2481
|
// BOT-2230: keep the queue place for the owner's next attempt instead of losing it.
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2482
|
+
// BOT-2400: by default (`--hold-on-timeout` only shortens it): a park deadline never
|
|
2483
|
+
// gives the place up — only `stack cancel` / `stack done` or the owner's death does.
|
|
2484
|
+
const queueHold = await holdQueuePlace(call, leaseId, opts.holdOnTimeout ?? HOLD_ON_TIMEOUT_MAX_SEC);
|
|
2485
|
+
if (queueHold.held) {
|
|
2486
|
+
process.stderr.write(`stack: lease ${leaseId} keeps queue place ${queueHold.queue_position ?? "?"} until ${queueHold.until} — re-run the same \`stack up\` before then to re-attach (\`stack cancel ${leaseId}\` gives it up)\n`);
|
|
2487
|
+
} else {
|
|
2448
2488
|
process.stderr.write(`${yellow("⚠")} stack: lease ${leaseId} could not keep its queue place (${queueHold.code ?? "error"}: ${queueHold.error})\n`);
|
|
2449
2489
|
}
|
|
2450
2490
|
// BOT-2237: a park that ended inside an unrecovered platform incident never observed admission.
|
|
@@ -2452,7 +2492,11 @@ export async function cmdUp(opts, {
|
|
|
2452
2492
|
const why = blind
|
|
2453
2493
|
? `parked ${opts.timeout}s inside a platform incident (${parked.incident.reason} since ${parked.incident.first_at}), so its admission was never observed`
|
|
2454
2494
|
: `parked ${opts.timeout}s without capacity`;
|
|
2455
|
-
|
|
2495
|
+
// BOT-2400: a held place is PARK_DEADLINE; the BOT-2237 incident code still wins when the
|
|
2496
|
+
// park never observed admission (the place is held either way — see queue_hold).
|
|
2497
|
+
const code = blind ? parked.incident.reason : (queueHold.held ? PARK_DEADLINE : PARK_HOLD_UNCONFIRMED);
|
|
2498
|
+
const detail = queueHold.held ? why : `${why}; queue hold unconfirmed; the lease was not cancelled — inspect \`bb stack status ${leaseId}\` before resuming or explicitly cancelling it`;
|
|
2499
|
+
return emitResult(buildReceipt({ command: "up", outcome: "timeout", code, lease_id: leaseId, ...(queueHold.held ? { state: "queued" } : {}), slot, error: detail, ...(ledger ? { unit_ledger: ledger } : {}), queue_hold: queueHold }), opts, EXIT.TIMEOUT);
|
|
2456
2500
|
}
|
|
2457
2501
|
if (parked.auth) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: "unauthorized on wait stream" }), opts, EXIT.AUTH);
|
|
2458
2502
|
if (parked.error) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: parked.error }), opts, EXIT.BACKEND);
|
|
@@ -3391,7 +3435,12 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3391
3435
|
touch: (leaseId, usage) => rpc.call("touch_stack_lease", { lease_id: leaseId, ...(usage ? { usage } : {}) }),
|
|
3392
3436
|
// BOT-1803: `{ idle: true }` idles an ACTIVE lease instead of reaping it.
|
|
3393
3437
|
release: (leaseId, signal, { idle = false } = {}) => rpc.call("release_stack_lease", { lease_id: leaseId, disposition: "destroy", ...(idle ? { idle: true } : {}) }, { signal }),
|
|
3438
|
+
// BOT-2400: keep a queued lease's place when the park deadline passes.
|
|
3439
|
+
hold: (leaseId, holdSecs) => holdQueuePlace((name, args) => rpc.call(name, args), leaseId, holdSecs),
|
|
3394
3440
|
};
|
|
3441
|
+
// An injected api double without `hold` keeps the pre-BOT-2400 release unless the test also
|
|
3442
|
+
// injected the tool transport (never a real network call from a test).
|
|
3443
|
+
const holdPlace = api.hold ?? (adapters.callTool ? (leaseId, holdSecs) => holdQueuePlace((name, args) => rpc.call(name, args), leaseId, holdSecs) : null);
|
|
3395
3444
|
const measureUsage = adapters.measureUsage ?? measureStackUsage;
|
|
3396
3445
|
const machineUuidFn = adapters.machineUuidFn ?? machineUuid;
|
|
3397
3446
|
const wait = adapters.wait ?? ((leaseId, done, failed, options) => waitForLease(leaseId, done, failed, options));
|
|
@@ -3404,9 +3453,9 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3404
3453
|
const readLeaseQuietly = async (id) => { try { return await api.get?.(id) ?? null; } catch { return null; } };
|
|
3405
3454
|
// BOT-2237: the typed end of a park that cannot re-authenticate — names the lease, never a
|
|
3406
3455
|
// capacity timeout. Exit AUTH: the caller's credential, not the lease, is what failed.
|
|
3407
|
-
const authUnavailableResult = (waited, phase, cleanup) => ({
|
|
3408
|
-
exitCode: EXIT.AUTH, outcome: "error", code: PARK_INCIDENT_REASONS.AUTH, leaseId:
|
|
3409
|
-
error: `${waited.error} while the lease was ${phase}; its server state is unknown to this run — inspect it with \`bb stack status ${
|
|
3456
|
+
const authUnavailableResult = (waited, phase, cleanup, waitedLeaseId = leaseId) => ({
|
|
3457
|
+
exitCode: EXIT.AUTH, outcome: "error", code: PARK_INCIDENT_REASONS.AUTH, leaseId: waitedLeaseId ?? null, cleanup,
|
|
3458
|
+
error: `${waited.error} while the lease was ${phase}; its server state is unknown to this run — inspect it with \`bb stack status ${waitedLeaseId}\` after \`bb doctor --fix\``,
|
|
3410
3459
|
...(parkIncident ? { incident: parkIncident } : {}),
|
|
3411
3460
|
});
|
|
3412
3461
|
const resilientWait = async (leaseIdArg, done, failed, options) => {
|
|
@@ -3716,6 +3765,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3716
3765
|
// id: wait for it to reach `reaped`, then send the SAME request once more (it then
|
|
3717
3766
|
// provisions a fresh lease for the slot).
|
|
3718
3767
|
let request;
|
|
3768
|
+
let idleRetried = false;
|
|
3719
3769
|
for (let attempt = 0; ; attempt++) {
|
|
3720
3770
|
activeWaitAbort = new AbortController();
|
|
3721
3771
|
const admission = await requestWithCapacityWait(() => api.request(leaseArgs), opts, {
|
|
@@ -3756,6 +3806,44 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3756
3806
|
if (request?.status == null) leaseProvisioned = true;
|
|
3757
3807
|
return { exitCode: request?.auth ? EXIT.AUTH : EXIT.BACKEND, outcome: "error", error: request?.error || "lease request failed" };
|
|
3758
3808
|
}
|
|
3809
|
+
// BOT-2400 (item 6): this agent's OWN previous lease on the slot is still `reaping` (a
|
|
3810
|
+
// provision rollback, an idle reap, the previous batch's teardown). It is not somebody's
|
|
3811
|
+
// live batch: park until it is reaped, then send the same request again.
|
|
3812
|
+
const ownReaping = request.data?.success === true && request.data.reused === true &&
|
|
3813
|
+
request.data.state === "reaping" && request.data.lease_id ? request.data.lease_id : null;
|
|
3814
|
+
if (ownReaping && attempt < 3) {
|
|
3815
|
+
stderrLine(`stack: this slot's previous lease ${ownReaping} is still reaping — waiting for it to reach reaped, then requesting again`);
|
|
3816
|
+
activeWaitAbort = new AbortController();
|
|
3817
|
+
const gone = await resilientWait(ownReaping, (st) => st === "reaped", (st) => st === "reap_failed", { timeoutSec: opts.timeout, auth: rpc.auth, signal: activeWaitAbort.signal });
|
|
3818
|
+
activeWaitAbort = null;
|
|
3819
|
+
if (gone?.interrupted || receivedSignal) {
|
|
3820
|
+
// The previous lease's containers may still own these ports until its reap is proven.
|
|
3821
|
+
retainBlock = true;
|
|
3822
|
+
return { exitCode: SIGNAL_EXIT[receivedSignal] ?? EXIT.INTERNAL, outcome: "interrupted", leaseId: ownReaping };
|
|
3823
|
+
}
|
|
3824
|
+
if (gone?.authUnavailable) {
|
|
3825
|
+
// The old lease can still own this stack's ports. Preserve the definitive credential
|
|
3826
|
+
// terminal instead of misclassifying it as a reap timeout, and do not request another lease.
|
|
3827
|
+
retainBlock = true;
|
|
3828
|
+
return authUnavailableResult(gone, "reaping", undefined, ownReaping);
|
|
3829
|
+
}
|
|
3830
|
+
let reaped = Boolean(gone?.woke) && gone.state === "reaped";
|
|
3831
|
+
if (!reaped && !gone?.failed) {
|
|
3832
|
+
const read = await readLeaseQuietly(ownReaping);
|
|
3833
|
+
reaped = Boolean(read?.ok && read.data?.success && read.data.state === "reaped");
|
|
3834
|
+
}
|
|
3835
|
+
if (!reaped) {
|
|
3836
|
+
// Its containers may still hold this stack's ports.
|
|
3837
|
+
retainBlock = true;
|
|
3838
|
+
return {
|
|
3839
|
+
exitCode: gone?.failed ? EXIT.LEASE_FAILED : EXIT.TIMEOUT, outcome: gone?.failed ? "error" : "timeout", leaseId: ownReaping,
|
|
3840
|
+
error: `this slot's previous lease ${ownReaping} did not reach reaped ${gone?.failed ? "(its reap failed)" : `within ${opts.timeout}s`}; ` +
|
|
3841
|
+
`no new lease was requested — check \`bb stack status ${ownReaping}\``,
|
|
3842
|
+
...(parkIncident ? { incident: parkIncident } : {}),
|
|
3843
|
+
};
|
|
3844
|
+
}
|
|
3845
|
+
continue;
|
|
3846
|
+
}
|
|
3759
3847
|
let why = attempt === 0 && request.data?.success === false && request.data?.lease_id ? IDLE_REPROVISION_REASONS[request.data.code] : null;
|
|
3760
3848
|
// BOT-2403: a re-adopted idle stack is handed over only when its edge runtime still sees
|
|
3761
3849
|
// the functions (the gate restarts the runtime once). Still broken → reap it and send the
|
|
@@ -3780,6 +3868,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3780
3868
|
}
|
|
3781
3869
|
}
|
|
3782
3870
|
if (!why) break;
|
|
3871
|
+
idleRetried = true;
|
|
3783
3872
|
const previous = request.data.lease_id;
|
|
3784
3873
|
idleLease.previousLeaseId = previous;
|
|
3785
3874
|
const becoming = why === "preempted" ? "preempted"
|
|
@@ -3833,7 +3922,15 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3833
3922
|
// stored connection and nothing is provisioned, so the run goes straight on to the
|
|
3834
3923
|
// post-activation flow. Any other reuse is somebody's LIVE batch, refused as before.
|
|
3835
3924
|
const reusedIdle = request.data.reused === true && request.data.reused_idle === true;
|
|
3836
|
-
|
|
3925
|
+
// BOT-2400 (item 1): this agent's own QUEUED lease for the slot — the place a previous run
|
|
3926
|
+
// kept at its park deadline. This run holds the stack directory's project lock, and the
|
|
3927
|
+
// control plane bound the lease to this same worktree + stack path (it refuses another), so
|
|
3928
|
+
// no live run can be parked on it: re-attach and keep parking at the same place.
|
|
3929
|
+
const reusedQueued = request.data.reused === true && !reusedIdle && request.data.state === "queued";
|
|
3930
|
+
if (reusedQueued) {
|
|
3931
|
+
stderrLine(`stack: re-attached to this slot's queued lease ${leaseId} — it kept its queue place${request.data.queue_position != null ? ` (position ${request.data.queue_position})` : ""}`);
|
|
3932
|
+
}
|
|
3933
|
+
if (request.data.reused && !reusedIdle && !reusedQueued) {
|
|
3837
3934
|
// With a stable identity a live lease in THIS stack directory is bound to our block.
|
|
3838
3935
|
if (request.data.stack_path != null && request.data.stack_path === execution.stackPath &&
|
|
3839
3936
|
(request.data.worktree_root == null || request.data.worktree_root === execution.worktreeRoot)) retainBlock = true;
|
|
@@ -3877,7 +3974,18 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3877
3974
|
}
|
|
3878
3975
|
}
|
|
3879
3976
|
// BOT-1803: a re-adopted idle remote stack is already running there; nothing to upload.
|
|
3880
|
-
|
|
3977
|
+
// BOT-2400: a queued lease can also be re-attached after PARK_DEADLINE. The server omits
|
|
3978
|
+
// its one-shot URL once the stored bundle exists, so matching its returned path + digest is
|
|
3979
|
+
// the positive proof that this run must retain the queue place and proceed without a PUT.
|
|
3980
|
+
const reusedQueuedBundle = reusedQueued && bundle &&
|
|
3981
|
+
!request.data.bundle_upload_url &&
|
|
3982
|
+
typeof request.data.bundle_path === "string" && request.data.bundle_path.length > 0 &&
|
|
3983
|
+
request.data.bundle_sha256 === bundle.sha256;
|
|
3984
|
+
if (reusedQueuedBundle) {
|
|
3985
|
+
placement.bundle = { sha256: bundle.sha256, bytes: bundle.size };
|
|
3986
|
+
if (bundle.included?.length) placement.bundleIncluded = bundle.included;
|
|
3987
|
+
stderrLine(`stack: re-attached remote lease ${leaseId} already has the matching stack bundle; keeping its queue place without uploading again`);
|
|
3988
|
+
} else if (!reusedIdle) {
|
|
3881
3989
|
if (!bundle || !request.data.bundle_upload_url) {
|
|
3882
3990
|
const cleanup = await requestCleanup();
|
|
3883
3991
|
return {
|
|
@@ -3961,16 +4069,47 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3961
4069
|
const final = await readLeaseQuietly(leaseId);
|
|
3962
4070
|
const ledger = final?.ok && final.data?.success ? final.data.ledger ?? null : null;
|
|
3963
4071
|
for (const line of describeUnitLedger(ledger)) stderrLine(`stack: ${line}`);
|
|
3964
|
-
const cleanup = await requestCleanup();
|
|
3965
4072
|
// BOT-2237: a park that ended inside an unrecovered platform incident never observed the
|
|
3966
4073
|
// lease's admission, so it must not claim the host had no capacity.
|
|
3967
|
-
|
|
4074
|
+
const blind = parkIncident && !parkIncident.recovered;
|
|
4075
|
+
const why = blind
|
|
4076
|
+
? `lease ${leaseId} parked ${opts.timeout}s inside a platform incident (${parkIncident.reason} since ${parkIncident.first_at}), so its admission was never observed`
|
|
4077
|
+
: `parked ${opts.timeout}s without capacity`;
|
|
4078
|
+
// BOT-2400 (item 1): the park deadline never gives the queue place up. Hold the lease
|
|
4079
|
+
// for the owner's next run (which re-attaches to it) and exit PARK_DEADLINE; only
|
|
4080
|
+
// `stack cancel` / `stack done` or the owner's death releases it.
|
|
4081
|
+
const queueHold = holdPlace ? await holdPlace(leaseId, HOLD_ON_TIMEOUT_MAX_SEC) : null;
|
|
4082
|
+
if (queueHold) {
|
|
4083
|
+
// A refused/lost hold reply is not an owner cancellation: it may have
|
|
4084
|
+
// committed before the reply was lost, or the queued lease may have
|
|
4085
|
+
// just been granted. Preserve both lease and conservative port claim.
|
|
4086
|
+
retainBlock = true;
|
|
4087
|
+
const retainSecs = queueHold.held ? queueHold.hold_secs ?? HOLD_ON_TIMEOUT_MAX_SEC : HOLD_ON_TIMEOUT_MAX_SEC;
|
|
4088
|
+
try { refreshRetention(stackDirFor(execution), { idleTtlSecs: retainSecs, idleRetained: true }); } catch { /* advisory */ }
|
|
4089
|
+
try { execution.renew?.({ idleTtlSecs: retainSecs, idleRetained: true }); } catch { /* advisory */ }
|
|
4090
|
+
}
|
|
4091
|
+
if (queueHold?.held) {
|
|
4092
|
+
stderrLine(`stack: lease ${leaseId} keeps queue place ${queueHold.queue_position ?? "?"} until ${queueHold.until ?? "?"} — re-run the same \`stack run\` to resume it (\`bb stack cancel ${leaseId}\` gives it up)`);
|
|
3968
4093
|
return {
|
|
3969
|
-
exitCode: EXIT.TIMEOUT, outcome: "timeout", code: parkIncident.reason
|
|
3970
|
-
error:
|
|
4094
|
+
exitCode: EXIT.TIMEOUT, outcome: "timeout", code: blind ? parkIncident.reason : PARK_DEADLINE, leaseId, state: "queued", queueHold,
|
|
4095
|
+
error: `${why}; lease ${leaseId} is still queued and keeps its place — re-run the same command to resume it`,
|
|
4096
|
+
...incident, ...(ledger ? { ledger } : {}),
|
|
3971
4097
|
};
|
|
3972
4098
|
}
|
|
3973
|
-
|
|
4099
|
+
if (queueHold) {
|
|
4100
|
+
stderrLine(`${yellow("⚠")} stack: lease ${leaseId} queue hold unconfirmed (${queueHold.code ?? "error"}: ${queueHold.error}); the lease was not cancelled — inspect \`bb stack status ${leaseId}\``);
|
|
4101
|
+
return {
|
|
4102
|
+
exitCode: EXIT.TIMEOUT, outcome: "timeout", code: blind ? parkIncident.reason : PARK_HOLD_UNCONFIRMED,
|
|
4103
|
+
leaseId, queueHold,
|
|
4104
|
+
error: `${why}; queue hold unconfirmed; the lease was not cancelled — inspect \`bb stack status ${leaseId}\` before resuming or explicitly cancelling it`,
|
|
4105
|
+
...incident, ...(ledger ? { ledger } : {}), ...(parkReason ? { parkReason } : {}),
|
|
4106
|
+
};
|
|
4107
|
+
}
|
|
4108
|
+
const cleanup = await requestCleanup();
|
|
4109
|
+
return {
|
|
4110
|
+
exitCode: EXIT.TIMEOUT, outcome: "timeout", ...(blind ? { code: parkIncident.reason } : {}), leaseId, error: why, cleanup,
|
|
4111
|
+
...incident, ...(ledger ? { ledger } : {}), ...(queueHold ? { queueHold } : {}), ...(parkReason ? { parkReason } : {}),
|
|
4112
|
+
};
|
|
3974
4113
|
}
|
|
3975
4114
|
if (parked?.failed) { const cleanup = await requestCleanup(); return { exitCode: EXIT.LEASE_FAILED, outcome: "error", leaseId, error: `lease was ${parked.state} while queued (reap_reason=${parked.reap_reason ?? "unknown"})`, cleanup, ...incident }; }
|
|
3976
4115
|
if (parked?.error || parked?.auth) { const cleanup = await requestCleanup(); return { exitCode: parked?.auth ? EXIT.AUTH : EXIT.LEASE_FAILED, outcome: "error", leaseId, error: parked?.error || "lease did not leave queue", cleanup, ...incident }; }
|
|
@@ -4267,6 +4406,9 @@ export function runReceiptFields(result) {
|
|
|
4267
4406
|
// incident (reason, window, attempts, recovered) — never as the lease's outcome.
|
|
4268
4407
|
...(result.incident ? { platform_incident: result.incident } : {}),
|
|
4269
4408
|
...(result.ledger ? { unit_ledger: result.ledger } : {}),
|
|
4409
|
+
// BOT-2400: a PARK_DEADLINE run names the lease's state (still queued) and the place it holds.
|
|
4410
|
+
...(result.state ? { state: result.state } : {}),
|
|
4411
|
+
...(result.queueHold ? { queue_hold: result.queueHold } : {}),
|
|
4270
4412
|
// BOT-2237: whether this run handed its host port block back, or kept it and why.
|
|
4271
4413
|
...(result.portBlock ? { port_block: result.portBlock } : {}),
|
|
4272
4414
|
};
|
package/src/test-lane.mjs
CHANGED
|
@@ -176,18 +176,29 @@ export async function launchTestLane(argv, {
|
|
|
176
176
|
return { exitCode: EXIT_TEST.INVALID, line: JSON.stringify({ outcome: "rejected", errors }) };
|
|
177
177
|
}
|
|
178
178
|
let managedCredential = null;
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
179
|
+
// A session id names a session but is not an API credential. Resolve the
|
|
180
|
+
// worktree's token even when the caller supplied --session-id, so the
|
|
181
|
+
// create/update calls are authenticated and the launched lane has a durable
|
|
182
|
+
// test-run to wait on. Without this, an otherwise valid explicit id could
|
|
183
|
+
// launch a real test with test_run_id:null and no supported wait condition.
|
|
184
|
+
//
|
|
185
|
+
// Test doubles that provide only a synthetic session id retain the old
|
|
186
|
+
// no-network seam. Production, and every caller that supplies a resolver,
|
|
187
|
+
// must resolve the token or fail before launching an untrackable run.
|
|
188
|
+
const mustResolveCredential = !opts.sessionToken
|
|
189
|
+
&& (!opts.sessionId || !!resolveCredential || call === callToolJson);
|
|
190
|
+
if (mustResolveCredential) {
|
|
182
191
|
if (!resolveCredential && call !== callToolJson) {
|
|
183
192
|
process.stderr.write("botbuddy test: --session-id is required (or set BOTBUDDY_AGENT_SESSION_TOKEN)\n");
|
|
184
193
|
return { exitCode: EXIT_TEST.INVALID, line: JSON.stringify({ outcome: "rejected", errors: ["--session-id is required"] }) };
|
|
185
194
|
}
|
|
186
195
|
try {
|
|
187
|
-
const credential = await (resolveCredential ?? resolveAgentSessionCredential)({ argv, env, cwd });
|
|
196
|
+
const credential = await (resolveCredential ?? resolveAgentSessionCredential)({ argv, env, cwd, sessionId: opts.sessionId });
|
|
188
197
|
managedCredential = credential;
|
|
189
198
|
opts.sessionToken = credential.token;
|
|
190
|
-
|
|
199
|
+
// --session-id is a deliberate caller choice. The resolved token proves
|
|
200
|
+
// authority; it must not silently replace the session the caller named.
|
|
201
|
+
opts.sessionId ??= credential.sessionId ?? null;
|
|
191
202
|
} catch (error) {
|
|
192
203
|
const code = error?.code ?? "agent_session_unavailable";
|
|
193
204
|
process.stderr.write(`botbuddy test: ${error?.message ?? code}\n`);
|
package/src/wait-core.mjs
CHANGED
|
@@ -44,11 +44,15 @@ const LEASE_UUID_SPELLING = /^(?:\{(?=.*\}$)|(?!.*\}$))((?:[0-9a-f]{4}-?){7}[0-9
|
|
|
44
44
|
|
|
45
45
|
// BOT-2013: stack_lease frame states that mean the lease is gone, not granted.
|
|
46
46
|
export const LEASE_LOST_STATES = Object.freeze(new Set(["reaping", "reaped", "reap_failed"]));
|
|
47
|
+
// BOT-2400: the lease states a `lease:id=…,state=<s>` wait may target.
|
|
48
|
+
export const LEASE_WAIT_STATES = Object.freeze(["provisioning", "active", "reaped"]);
|
|
47
49
|
|
|
48
50
|
/** The lease-lost detail for a matched lease frame, or null when the frame is a grant. */
|
|
49
51
|
export function leaseLostDetail(signal, condition) {
|
|
50
52
|
if (condition?.type !== "lease" || signal?.signal_type !== "stack_lease") return null;
|
|
51
53
|
const leaseState = signal.payload?.state;
|
|
54
|
+
// BOT-2400: the state a `state=` wait asked for is its match, never a loss.
|
|
55
|
+
if (condition.params?.state != null && leaseState === condition.params.state) return null;
|
|
52
56
|
if (!LEASE_LOST_STATES.has(leaseState)) return null;
|
|
53
57
|
return {
|
|
54
58
|
lease_id: signal.payload?.lease_id ?? condition.params?.id ?? null,
|
|
@@ -205,7 +209,15 @@ const VALIDATORS = {
|
|
|
205
209
|
const m = LEASE_UUID_SPELLING.exec(raw);
|
|
206
210
|
const hex = m ? m[1].replace(/-/g, "") : null;
|
|
207
211
|
const id = hex ? `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` : raw;
|
|
208
|
-
|
|
212
|
+
// BOT-2400 (folds BOT-1938): `state=` waits for that exact lease state, level-triggered by
|
|
213
|
+
// the register-time snapshot. `state=reaped` is the reap wait: a `reaping` lease keeps
|
|
214
|
+
// waiting instead of ending lease_lost, and an already-reaped lease matches at once.
|
|
215
|
+
if (p.state === undefined) return { id };
|
|
216
|
+
const state = String(p.state).trim().toLowerCase();
|
|
217
|
+
if (!LEASE_WAIT_STATES.includes(state)) {
|
|
218
|
+
throw new Error(`lease state must be one of ${LEASE_WAIT_STATES.join(", ")}`);
|
|
219
|
+
}
|
|
220
|
+
return { id, state };
|
|
209
221
|
},
|
|
210
222
|
// BOT-2330: terminal state for one owner/session-bound managed allocation.
|
|
211
223
|
// Both fences are required so a stale wait cannot observe a later reuse.
|
|
@@ -241,9 +253,9 @@ const VALIDATORS = {
|
|
|
241
253
|
// slots. Capacity is host-shared (no tenant), so `policy` is recorded for intent
|
|
242
254
|
// but matching is by free-slot threshold + optional host pin. `slots>=N` parses
|
|
243
255
|
// as the bare key `slots>` (parseParams splits on the first '='); `min_slots=N`
|
|
244
|
-
// is an alias. `stale_grace`
|
|
245
|
-
//
|
|
246
|
-
//
|
|
256
|
+
// is an alias. `stale_grace` is still parsed for older automation, but since BOT-2400 a
|
|
257
|
+
// stale capacity source never ends the wait: it is reported as degraded and the wait
|
|
258
|
+
// keeps parking to its --timeout.
|
|
247
259
|
capacity(p) {
|
|
248
260
|
if (!p.policy) throw new Error("capacity needs policy=<name>");
|
|
249
261
|
const rawMin = p["slots>"] ?? p.min_slots ?? p.slots;
|
|
@@ -825,6 +837,10 @@ export function truncateReceipt(receipt, maxBytes = 10240) {
|
|
|
825
837
|
state: leaseLostState ?? { truncated: true },
|
|
826
838
|
truncated: true,
|
|
827
839
|
...(clone.client ? { client: clone.client } : {}),
|
|
840
|
+
...(clone.degraded?.includes("capacity_source_sustained") ? {
|
|
841
|
+
degraded: ["capacity_source_stale", "capacity_source_sustained"],
|
|
842
|
+
stale_since: clone.stale_since ?? null,
|
|
843
|
+
} : {}),
|
|
828
844
|
};
|
|
829
845
|
for (const key of ["next_cursor", "wait_session_id"]) {
|
|
830
846
|
if (clone[key] == null) continue;
|
|
@@ -840,6 +856,10 @@ export function truncateReceipt(receipt, maxBytes = 10240) {
|
|
|
840
856
|
error: typeof clone.error === "string" ? clone.error.slice(0, 48) : "receipt_truncated",
|
|
841
857
|
truncated: true,
|
|
842
858
|
...(clone.client ? { client: clone.client } : {}),
|
|
859
|
+
...(clone.degraded?.includes("capacity_source_sustained") ? {
|
|
860
|
+
degraded: ["capacity_source_stale", "capacity_source_sustained"],
|
|
861
|
+
stale_since: clone.stale_since ?? null,
|
|
862
|
+
} : {}),
|
|
843
863
|
};
|
|
844
864
|
// BOT-1590: the recovery hint is the whole point of an error receipt, so keep
|
|
845
865
|
// it (trimmed to whatever still fits the cap) rather than dropping it — a
|
|
@@ -1038,7 +1058,17 @@ function conditionMatchesSignal(condition, signal, waitSessionId) {
|
|
|
1038
1058
|
// BOT-1220: recipient-scoped stack-lease transition (queued→provisioning, or
|
|
1039
1059
|
// →active). The relay only delivers it to the lease's owning agent, so any
|
|
1040
1060
|
// stack_lease frame for this lease id is ours to wake on.
|
|
1041
|
-
|
|
1061
|
+
if (signal.signal_type !== "stack_lease" || signal.subject_key !== `lease:${params.id}`) return false;
|
|
1062
|
+
// BOT-2400: with `state=`, only that state matches — or a terminal loss on the way
|
|
1063
|
+
// (reap_failed for a reap wait; any reap state while waiting for provisioning/active),
|
|
1064
|
+
// which finalizeMatch reports as lease_lost. `reaping` en route to `reaped` keeps waiting.
|
|
1065
|
+
if (params.state != null) {
|
|
1066
|
+
const leaseState = signal.payload?.state;
|
|
1067
|
+
if (leaseState === params.state) return true;
|
|
1068
|
+
if (params.state === "reaped") return leaseState === "reap_failed";
|
|
1069
|
+
return LEASE_LOST_STATES.has(leaseState);
|
|
1070
|
+
}
|
|
1071
|
+
return true;
|
|
1042
1072
|
case "managed-allocation": {
|
|
1043
1073
|
const p = signal.payload;
|
|
1044
1074
|
return signal.signal_type === "managed_allocation" && signal.subject_key === `allocation:${params.id}` &&
|
|
@@ -1358,6 +1388,7 @@ export async function runWaitLoop({
|
|
|
1358
1388
|
backoff = (attempt) => nextBackoff(attempt),
|
|
1359
1389
|
sleep = null,
|
|
1360
1390
|
onFrame = null,
|
|
1391
|
+
onCapacitySourceSustained = null,
|
|
1361
1392
|
startedAt: originalStartedAt = null,
|
|
1362
1393
|
// BOT-1259: the wait session's server-resolved tenant (from the register echo). A
|
|
1363
1394
|
// tenanted spine signal only matches a session resolved to that tenant; `null` (the
|
|
@@ -1381,19 +1412,8 @@ export async function runWaitLoop({
|
|
|
1381
1412
|
// On an expired recovery, replay only durable rows created before this original
|
|
1382
1413
|
// deadline; the caller supplies a short transport grace via deadlineMs.
|
|
1383
1414
|
replayOnlyThroughMs = null,
|
|
1384
|
-
// BOT-1148
|
|
1385
|
-
//
|
|
1386
|
-
// capacity_source_stale. Absorbs threshold flap on the client (the server latch is
|
|
1387
|
-
// the primary hysteresis); 0 = exit immediately on the stale signal. Injectable so
|
|
1388
|
-
// a clock-controlled test can drive it. Not a grammar surface (no new condition name).
|
|
1389
|
-
//
|
|
1390
|
-
// Default MUST exceed the beacon evaluator's worst-case cadence (Codex P2). Before
|
|
1391
|
-
// BOT-2024 the recovery transition was emitted only by the `* * * * *` (60s)
|
|
1392
|
-
// eval_container_beacon_staleness cron, so a host that resumed polling right after a
|
|
1393
|
-
// stale emission had its recovery frame arrive up to ~60s later. Since BOT-2024 the
|
|
1394
|
-
// beacon itself (claim_container_job / attested registerHost) emits the recovery, so
|
|
1395
|
-
// the frame now lands on the Helper's next beacon; 90s still covers one ~45-60s
|
|
1396
|
-
// Helper beacon interval plus margin, and an older server's cron-only recovery.
|
|
1415
|
+
// BOT-1148's beacon-stale confirm window. BOT-2400 (item 8): a stale capacity source no
|
|
1416
|
+
// longer ends a wait, so this is accepted only for callers that still pass it (no effect).
|
|
1397
1417
|
beaconConfirmMs = 90_000,
|
|
1398
1418
|
// BOT-1837: the arm-time notice (if any) to carry into a TIMEOUT receipt's
|
|
1399
1419
|
// `error`, so the receipt itself says what CI was already doing when this wait
|
|
@@ -1427,6 +1447,8 @@ export async function runWaitLoop({
|
|
|
1427
1447
|
const pendingReplayBeaconFrames = [];
|
|
1428
1448
|
let pendingReplaySupersession = null;
|
|
1429
1449
|
const degraded = [];
|
|
1450
|
+
const sustainedHosts = new Map();
|
|
1451
|
+
const sustainedNotices = new Map();
|
|
1430
1452
|
// BOT-2351: `reconnects` counts every stream rotation for the receipt (an operator
|
|
1431
1453
|
// still sees churn); `failures` counts CONSECUTIVE failed connects — a connect that
|
|
1432
1454
|
// threw, or a stream that ended before delivering a single frame — and is reset by any
|
|
@@ -1457,11 +1479,9 @@ export async function runWaitLoop({
|
|
|
1457
1479
|
};
|
|
1458
1480
|
|
|
1459
1481
|
const finalize = async (outcome, extra = {}) => {
|
|
1460
|
-
// A terminal receipt supersedes any pending alarm
|
|
1461
|
-
//
|
|
1462
|
-
// ConfirmTimer is defined further down but only reached at runtime, after setup completes.
|
|
1482
|
+
// A terminal receipt supersedes any pending alarm — cancel their timers so none outlives
|
|
1483
|
+
// the wait and holds the event loop open (Codex round-40 P2).
|
|
1463
1484
|
clearPendingAlarmTimers();
|
|
1464
|
-
clearStaleConfirmTimer?.();
|
|
1465
1485
|
// BOT-1147 (Codex P2): re-check Linear feed freshness before EVERY terminal receipt
|
|
1466
1486
|
// (timeout, capacity-stale, signal match, error) via this single finalization path —
|
|
1467
1487
|
// not just the alarm path — so a mixed wait (e.g. `linear` + `pr-review`) that
|
|
@@ -1485,6 +1505,12 @@ export async function runWaitLoop({
|
|
|
1485
1505
|
error: extra.error ?? null,
|
|
1486
1506
|
reacquireLock: extra.reacquireLock ?? null,
|
|
1487
1507
|
});
|
|
1508
|
+
// Keep the server's oldest outstanding outage clock on every terminal path.
|
|
1509
|
+
if (sustainedHosts.size > 0) {
|
|
1510
|
+
receipt.stale_since = [...sustainedHosts.values()]
|
|
1511
|
+
.filter((value) => typeof value === "string" && Number.isFinite(Date.parse(value)))
|
|
1512
|
+
.sort((a, b) => Date.parse(a) - Date.parse(b))[0] ?? null;
|
|
1513
|
+
}
|
|
1488
1514
|
return { receipt: truncateReceipt(receipt, receiptMaxBytes), exitCode: extra.exitCode };
|
|
1489
1515
|
};
|
|
1490
1516
|
|
|
@@ -1529,20 +1555,13 @@ export async function runWaitLoop({
|
|
|
1529
1555
|
.filter((a) => Number.isFinite(a.atMs))
|
|
1530
1556
|
.map((a) => ({ ...a, kind: "match" }));
|
|
1531
1557
|
if (deadlineMs != null) alarms.push({ kind: "timeout", atMs: deadlineMs });
|
|
1532
|
-
// BOT-1065
|
|
1533
|
-
//
|
|
1534
|
-
//
|
|
1535
|
-
//
|
|
1536
|
-
//
|
|
1537
|
-
//
|
|
1538
|
-
|
|
1539
|
-
if (c.type === "capacity" && c.params.staleGraceSec != null) {
|
|
1540
|
-
const staleAtMs = Number.isFinite(c.params.staleDeadlineMs)
|
|
1541
|
-
? c.params.staleDeadlineMs
|
|
1542
|
-
: now() + c.params.staleGraceSec * 1000;
|
|
1543
|
-
alarms.push({ kind: "stale", conditionId: c.id, atMs: staleAtMs });
|
|
1544
|
-
}
|
|
1545
|
-
}
|
|
1558
|
+
// BOT-1065 bounded a capacity wait with a terminal `stale_grace` alarm (exit 5
|
|
1559
|
+
// capacity_source_stale). BOT-2400 (item 8): a stale capacity source HOLDS the wait
|
|
1560
|
+
// instead of ending it, so the grace no longer arms a terminal alarm: the wait keeps parking to its own --timeout and
|
|
1561
|
+
// re-evaluates when the beacon returns (the beacon-stale path below folds the degraded
|
|
1562
|
+
// marker). A Helper worker that is briefly down (ENT-5492, 2026-10-14 22:32→22:47Z) must
|
|
1563
|
+
// cost a longer wait, never a failed wait plus a token-expensive re-arm. `stale_grace=` is
|
|
1564
|
+
// still accepted (and recorded on a checkpoint) for older automation.
|
|
1546
1565
|
const earliest = alarms.length ? alarms.reduce((m, a) => (a.atMs < m.atMs ? a : m)) : null;
|
|
1547
1566
|
// A recovered wait may have alarms that were already due at its original
|
|
1548
1567
|
// deadline. Do not put those into the initial race: the durable replay must
|
|
@@ -1560,12 +1579,6 @@ export async function runWaitLoop({
|
|
|
1560
1579
|
// finalization path), so every terminal receipt gets it — not just the alarm path.
|
|
1561
1580
|
if (!alarm) return null;
|
|
1562
1581
|
if (alarm.kind === "timeout") return finalize("timeout", { exitCode: EXIT.TIMEOUT, error: timeoutError ?? null });
|
|
1563
|
-
if (alarm.kind === "stale") {
|
|
1564
|
-
// Capacity source didn't deliver within the grace: degrade truthfully and
|
|
1565
|
-
// exit with an error receipt (not a silent hang, not a plain timeout).
|
|
1566
|
-
if (!degraded.includes("capacity_source_stale")) degraded.push("capacity_source_stale");
|
|
1567
|
-
return finalize("error", { exitCode: EXIT.BACKEND, error: "capacity_source_stale" });
|
|
1568
|
-
}
|
|
1569
1582
|
const condition = conditions.find((c) => c.id === alarm.conditionId);
|
|
1570
1583
|
return finalize("matched", {
|
|
1571
1584
|
exitCode: EXIT.MATCHED,
|
|
@@ -1640,84 +1653,59 @@ export async function runWaitLoop({
|
|
|
1640
1653
|
}
|
|
1641
1654
|
};
|
|
1642
1655
|
|
|
1643
|
-
// BOT-1148: server-side, beacon-driven capacity staleness.
|
|
1644
|
-
//
|
|
1645
|
-
//
|
|
1646
|
-
//
|
|
1647
|
-
//
|
|
1648
|
-
//
|
|
1649
|
-
//
|
|
1650
|
-
// without re-registration (AC-3), and may itself grant if it carries free_slots.
|
|
1656
|
+
// BOT-1148: server-side, beacon-driven capacity staleness. A container_capacity frame
|
|
1657
|
+
// carrying source_stale:true whose host matches a HOST-PINNED capacity wait folds
|
|
1658
|
+
// capacity_source_stale into `degraded`; a recovery (source_stale:false, or any normal
|
|
1659
|
+
// capacity frame — BOT-2100) clears it and may itself grant. BOT-2400 (item 8): the stale
|
|
1660
|
+
// state is degraded, never terminal — the BOT-1148 confirm-window fast-fail (exit 5) is
|
|
1661
|
+
// gone, so the wait stays armed and re-evaluates when the beacon returns. Un-pinned
|
|
1662
|
+
// capacity waits ignore a single host's beacon (another live host may serve them, AC-4).
|
|
1651
1663
|
const hostPinnedCapacity = conditions.filter((c) => c.type === "capacity" && c.params.host != null);
|
|
1652
|
-
// PER-HOST stale
|
|
1653
|
-
|
|
1654
|
-
// an --any wait pinned to multiple hosts would miss A's fast-fail when B recovers.
|
|
1655
|
-
const staleConfirm = new Map(); // hostKey -> atMs the confirm window elapses
|
|
1656
|
-
let staleConfirmPromise = null; // races the EARLIEST pending confirm across hosts
|
|
1657
|
-
let staleConfirmTimer = null; // handle for that race's real timer, so it can be cancelled
|
|
1658
|
-
// Cancel any pending confirm timer. The race timer is a REAL setTimeout (not the injected
|
|
1659
|
-
// `sleep`) precisely so it is cancellable: re-arming, a host recovery, or a terminal
|
|
1660
|
-
// finalize supersedes the pending confirm, and an ORPHANED confirm timer would otherwise
|
|
1661
|
-
// keep the process (and the checked-in `pnpm test:bb-wait` lane) alive for the full
|
|
1662
|
-
// beaconConfirmMs window — the parent suite ran ~6.5s, this leak made it ~92s (Codex
|
|
1663
|
-
// round-40 P2). Same "real timer, tracked so it can be cleared" discipline as the alarm
|
|
1664
|
-
// timers above.
|
|
1665
|
-
const clearStaleConfirmTimer = () => {
|
|
1666
|
-
if (staleConfirmTimer != null) { clearTimeout(staleConfirmTimer); staleConfirmTimer = null; }
|
|
1667
|
-
};
|
|
1664
|
+
// PER-HOST stale state (Codex P2): host B's recovery never clears host A's marker.
|
|
1665
|
+
const staleHosts = new Set();
|
|
1668
1666
|
const beaconHostKey = (signal) => {
|
|
1669
1667
|
const match = hostPinnedCapacity.find(
|
|
1670
1668
|
(c) => c.params.host === signal.subject_key || c.params.host === signal.payload?.host_key,
|
|
1671
1669
|
);
|
|
1672
1670
|
return match ? match.params.host : null;
|
|
1673
1671
|
};
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
staleConfirmPromise = new Promise((resolve) => {
|
|
1679
|
-
staleConfirmTimer = setTimeout(() => {
|
|
1680
|
-
staleConfirmTimer = null;
|
|
1681
|
-
resolve({ __staleConfirm: true });
|
|
1682
|
-
}, Math.max(0, soonest - now()));
|
|
1683
|
-
});
|
|
1684
|
-
};
|
|
1685
|
-
// Returns "continue" (skip this frame), "fallthrough" (let matchFrame see it), or
|
|
1686
|
-
// "exit" (terminate immediately on a stale signal when beaconConfirmMs<=0).
|
|
1672
|
+
// The frame always goes on to matchFrame afterwards: a raw
|
|
1673
|
+
// `event:type=container_capacity,subject=<host>` escape-hatch condition must still observe
|
|
1674
|
+
// beacon-stale transitions (Codex round-23/24 P2), and the `capacity` matcher refuses to
|
|
1675
|
+
// grant on a source_stale frame, so a stale frame is never a false grant.
|
|
1687
1676
|
const applyBeaconStale = (signal) => {
|
|
1688
|
-
|
|
1677
|
+
if (signalTenantBlocked(signal, sessionTenant)) return;
|
|
1689
1678
|
const hostKey = beaconHostKey(signal);
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1679
|
+
if (hostKey == null) return;
|
|
1680
|
+
if (signal.payload.source_stale === true) {
|
|
1681
|
+
staleHosts.add(hostKey);
|
|
1682
|
+
noteDegraded("capacity_source_stale");
|
|
1683
|
+
if (signal.payload.phase === "sustained") {
|
|
1684
|
+
const since = typeof signal.payload.stale_since === "string" &&
|
|
1685
|
+
Number.isFinite(Date.parse(signal.payload.stale_since)) ? signal.payload.stale_since : null;
|
|
1686
|
+
sustainedHosts.set(hostKey, since);
|
|
1687
|
+
noteDegraded("capacity_source_sustained");
|
|
1688
|
+
if (!sustainedNotices.has(hostKey) || sustainedNotices.get(hostKey) !== since) {
|
|
1689
|
+
sustainedNotices.set(hostKey, since);
|
|
1690
|
+
onCapacitySourceSustained?.({ host_key: hostKey, stale_since: since });
|
|
1691
|
+
}
|
|
1692
|
+
} else if (signal.payload.phase === "transient") {
|
|
1693
|
+
sustainedHosts.delete(hostKey);
|
|
1702
1694
|
}
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
// grant on a source_stale frame (never a false grant), and a host-pinned-only wait
|
|
1708
|
-
// simply finds no match here and keeps waiting on the confirm race as before.
|
|
1709
|
-
return "fallthrough";
|
|
1695
|
+
} else {
|
|
1696
|
+
staleHosts.delete(hostKey);
|
|
1697
|
+
sustainedHosts.delete(hostKey);
|
|
1698
|
+
sustainedNotices.delete(hostKey);
|
|
1710
1699
|
}
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1700
|
+
for (const [marker, active] of [
|
|
1701
|
+
["capacity_source_stale", staleHosts.size > 0],
|
|
1702
|
+
["capacity_source_sustained", sustainedHosts.size > 0],
|
|
1703
|
+
]) {
|
|
1704
|
+
if (!active) {
|
|
1705
|
+
const i = degraded.indexOf(marker);
|
|
1706
|
+
if (i !== -1) degraded.splice(i, 1);
|
|
1707
|
+
}
|
|
1718
1708
|
}
|
|
1719
|
-
armStaleConfirmRace();
|
|
1720
|
-
return "fallthrough";
|
|
1721
1709
|
};
|
|
1722
1710
|
|
|
1723
1711
|
while (true) {
|
|
@@ -1776,7 +1764,6 @@ export async function runWaitLoop({
|
|
|
1776
1764
|
));
|
|
1777
1765
|
}
|
|
1778
1766
|
const racers = [deadlinePromise, pendingNext];
|
|
1779
|
-
if (staleConfirmPromise) racers.push(staleConfirmPromise);
|
|
1780
1767
|
if (preDeadlineProbePromise && !preDeadlineProbeDone) racers.push(preDeadlineProbePromise);
|
|
1781
1768
|
const raced = await Promise.race(racers);
|
|
1782
1769
|
|
|
@@ -1800,24 +1787,6 @@ export async function runWaitLoop({
|
|
|
1800
1787
|
await probeFeedLag();
|
|
1801
1788
|
continue;
|
|
1802
1789
|
}
|
|
1803
|
-
// BOT-1148: the beacon-stale confirm window elapsed. If still stale (no recovery
|
|
1804
|
-
// cleared it), exit fast on the existing code 5; otherwise drop the settled
|
|
1805
|
-
// confirm promise and keep waiting — pendingNext stays in flight, re-raced next
|
|
1806
|
-
// iteration, so no frame is dropped.
|
|
1807
|
-
if (raced && raced.__staleConfirm) {
|
|
1808
|
-
// Any pinned host whose confirm window elapsed while STILL stale → fast-fail.
|
|
1809
|
-
// A recovery from a different host already removed its own entry, so it can't
|
|
1810
|
-
// suppress this one (Codex P2, per-host).
|
|
1811
|
-
const anyDue = [...staleConfirm.values()].some((atMs) => now() >= atMs);
|
|
1812
|
-
if (anyDue) {
|
|
1813
|
-
if (!degraded.includes("capacity_source_stale")) degraded.push("capacity_source_stale");
|
|
1814
|
-
return finalize("error", { exitCode: EXIT.BACKEND, error: "capacity_source_stale" });
|
|
1815
|
-
}
|
|
1816
|
-
// A settled promise whose host recovered before firing: re-arm for the next
|
|
1817
|
-
// pending host (if any) and keep waiting — pendingNext stays in flight.
|
|
1818
|
-
armStaleConfirmRace();
|
|
1819
|
-
continue;
|
|
1820
|
-
}
|
|
1821
1790
|
// raced settled from pendingNext (a frame result) — consume it before the next read.
|
|
1822
1791
|
pendingNext = null;
|
|
1823
1792
|
const { value: frame, done } = raced;
|
|
@@ -1895,7 +1864,6 @@ export async function runWaitLoop({
|
|
|
1895
1864
|
if (frame.event === "replay_complete") {
|
|
1896
1865
|
replayPending = false;
|
|
1897
1866
|
replayFrontier = cursor;
|
|
1898
|
-
let pendingBeaconExit = null;
|
|
1899
1867
|
// A replay terminal is a point-in-time receipt. Beacon state after an
|
|
1900
1868
|
// earlier match/supersession cannot affect that receipt.
|
|
1901
1869
|
let terminalSeq = [
|
|
@@ -1909,13 +1877,7 @@ export async function runWaitLoop({
|
|
|
1909
1877
|
// They are only transport catch-up, never evidence for the terminal
|
|
1910
1878
|
// receipt, so they must not alter stale confirmation state either.
|
|
1911
1879
|
if (!pendingBeaconFrame.beforeReplayDeadline) continue;
|
|
1912
|
-
|
|
1913
|
-
if (disposition === "exit") {
|
|
1914
|
-
pendingBeaconExit = pendingBeaconFrame;
|
|
1915
|
-
terminalSeq = pendingBeaconFrame.seqKey;
|
|
1916
|
-
break;
|
|
1917
|
-
}
|
|
1918
|
-
if (disposition !== "fallthrough" || !pendingBeaconFrame.beforeReplayDeadline) continue;
|
|
1880
|
+
applyBeaconStale(pendingBeaconFrame.signal);
|
|
1919
1881
|
if (signalTenantBlocked(pendingBeaconFrame.signal, sessionTenant)) continue;
|
|
1920
1882
|
const matched = matchFrame(pendingBeaconFrame.frame, conditions, waitSessionId, sessionTenant);
|
|
1921
1883
|
if (!matched) continue;
|
|
@@ -1938,7 +1900,6 @@ export async function runWaitLoop({
|
|
|
1938
1900
|
const replayCandidate = [
|
|
1939
1901
|
pendingMatch && { kind: "match", ...pendingMatch },
|
|
1940
1902
|
pendingReplaySupersession && { kind: "superseded", ...pendingReplaySupersession },
|
|
1941
|
-
pendingBeaconExit && { kind: "beacon_stale", ...pendingBeaconExit },
|
|
1942
1903
|
].filter(Boolean).sort((a, b) => BigInt(a.seqKey) < BigInt(b.seqKey) ? -1 : BigInt(a.seqKey) > BigInt(b.seqKey) ? 1 : 0)[0] ?? null;
|
|
1943
1904
|
if (replayCandidate) {
|
|
1944
1905
|
const candidateAtMs = Date.parse(replayCandidate.signal.created_at);
|
|
@@ -1946,9 +1907,6 @@ export async function runWaitLoop({
|
|
|
1946
1907
|
return fireAlarm(restoredAlarm);
|
|
1947
1908
|
}
|
|
1948
1909
|
if (replayCandidate.kind === "superseded") return finalizeSuperseded(replayCandidate.signal);
|
|
1949
|
-
if (replayCandidate.kind === "beacon_stale") {
|
|
1950
|
-
return finalize("error", { exitCode: EXIT.BACKEND, error: "capacity_source_stale" });
|
|
1951
|
-
}
|
|
1952
1910
|
const terminal = await finalizeMatch(replayCandidate.entry, replayCandidate.signal);
|
|
1953
1911
|
if (onFrame) {
|
|
1954
1912
|
await onFrame(replayCandidate.signal, {
|
|
@@ -2026,15 +1984,8 @@ export async function runWaitLoop({
|
|
|
2026
1984
|
pendingReplayBeaconFrames.push({ frame, signal, seqKey, beforeReplayDeadline });
|
|
2027
1985
|
continue;
|
|
2028
1986
|
}
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
return finalize("error", { exitCode: EXIT.BACKEND, error: "capacity_source_stale" });
|
|
2032
|
-
}
|
|
2033
|
-
if (disposition === "continue") {
|
|
2034
|
-
await checkpointNonterminalFrame();
|
|
2035
|
-
continue;
|
|
2036
|
-
}
|
|
2037
|
-
// "fallthrough": let matchFrame see the frame — a recovery may grant via
|
|
1987
|
+
applyBeaconStale(signal);
|
|
1988
|
+
// Let matchFrame see the frame — a recovery may grant via
|
|
2038
1989
|
// free_slots, and a raw `event` condition may match a stale frame (the
|
|
2039
1990
|
// `capacity` matcher itself never grants on source_stale).
|
|
2040
1991
|
}
|
package/src/wait.mjs
CHANGED
|
@@ -107,6 +107,10 @@ CONDITIONS (TYPE:key=val,key=val — repeat for several; --any wakes on the fir
|
|
|
107
107
|
becoming active — the zero-poll wake for a parked
|
|
108
108
|
\`botbuddy stack up\` (recipient-scoped to the owner).
|
|
109
109
|
Level-triggered: an already-granted lease wakes at once.
|
|
110
|
+
lease:id=<lease_id>,state=<s> BOT-2400: wait for that exact state (provisioning, active,
|
|
111
|
+
reaped). state=reaped is the reap wait: an already-reaped
|
|
112
|
+
lease matches at once and a reaping one keeps waiting
|
|
113
|
+
(only reap_failed ends it, exit 9).
|
|
110
114
|
guidance:request=<post_id> a command-post guidance request being answered
|
|
111
115
|
(scoped to the asking agent; fetch the body with
|
|
112
116
|
get_guidance_response)
|
|
@@ -123,8 +127,9 @@ CONDITIONS (TYPE:key=val,key=val — repeat for several; --any wakes on the fir
|
|
|
123
127
|
capacity:policy=<p>,slots>=<N>[,host=<h>][,stale_grace=<sec>]
|
|
124
128
|
a managed-container host reaching >= N free slots.
|
|
125
129
|
Host-shared; a silently-dead host is detected server-side
|
|
126
|
-
(capacity_source_stale) via a host beacon
|
|
127
|
-
|
|
130
|
+
(capacity_source_stale) via a host beacon. BOT-2400: a stale
|
|
131
|
+
source is DEGRADED, never terminal — the wait stays armed and
|
|
132
|
+
re-evaluates when the beacon returns (stale_grace is ignored).
|
|
128
133
|
Grants are RANKED (BOT-1577): when several waiters
|
|
129
134
|
contend, a freed slot is handed to the highest-scored
|
|
130
135
|
one and its receipt carries grant:{position,tier,score};
|
|
@@ -152,7 +157,13 @@ CONDITIONS (TYPE:key=val,key=val — repeat for several; --any wakes on the fir
|
|
|
152
157
|
|
|
153
158
|
OPTIONS
|
|
154
159
|
--any wake on the first matching condition (default; only mode in v1)
|
|
155
|
-
--timeout <seconds>
|
|
160
|
+
--timeout <seconds> safety cap (default 86400 = 24 h; hard cap 259200 = 72 h). Timeout is exit 2,
|
|
161
|
+
not an error. The timeout is a SAFETY CAP measured in hours or days, never a
|
|
162
|
+
polling interval: every re-arm wakes you and re-reads your whole context. A
|
|
163
|
+
cap under 7200 s on a non-timer wait exits 4 timeout_too_short (BOT-2400).
|
|
164
|
+
--allow-short-timeout <reason>
|
|
165
|
+
accept a cap under 7200 s (a CI job budget, a probe); the reason is recorded
|
|
166
|
+
on the wait. Automation may set $BOTBUDDY_WAIT_SHORT_TIMEOUT_REASON instead.
|
|
156
167
|
--since <seq> resume from a prior receipt's next_cursor (0..9007199254740991; replay what you missed)
|
|
157
168
|
--receipt-max-bytes <n> cap the receipt (512..65536; default 10240; payloads truncate to pointers)
|
|
158
169
|
--heartbeat keep this agent session alive while waiting (so it is not reaped)
|
|
@@ -194,27 +205,55 @@ OUTPUT
|
|
|
194
205
|
|
|
195
206
|
EXAMPLES
|
|
196
207
|
# wait for a Playwright lane to free up, then acquire it yourself
|
|
197
|
-
botbuddy wait 'lock:subtype=playwright_lane,host=Jonos-MBP'
|
|
208
|
+
botbuddy wait 'lock:subtype=playwright_lane,host=Jonos-MBP'
|
|
198
209
|
|
|
199
210
|
# queue for a specific lane and be HANDED it when it frees (no acquire race):
|
|
200
211
|
# acquire_lock (gets queued) → bb-wait claim=true → wake already holding lane 3
|
|
201
|
-
botbuddy wait 'lock:subtype=playwright_lane,host=Jonos-MBP,slot=3,claim=true' --
|
|
212
|
+
botbuddy wait 'lock:subtype=playwright_lane,host=Jonos-MBP,slot=3,claim=true' --heartbeat
|
|
202
213
|
|
|
203
214
|
# wait for a chat reply on any channel, or 20 minutes, whichever first
|
|
204
215
|
botbuddy wait --any 'chat:channel=*' 'timer:duration=1200' --heartbeat
|
|
205
216
|
|
|
206
217
|
# after pushing review fixes, wait for the next review comment or a state change
|
|
207
218
|
botbuddy wait --any 'pr-review:repo=bot-buddy/botbuddy-webapp,pr=425' \\
|
|
208
|
-
'pr-state:repo=bot-buddy/botbuddy-webapp,pr=425'
|
|
219
|
+
'pr-state:repo=bot-buddy/botbuddy-webapp,pr=425'
|
|
209
220
|
|
|
210
221
|
${SETUP_BLOCK}
|
|
211
222
|
`;
|
|
212
223
|
|
|
224
|
+
// BOT-2400: a wait's timeout is a SAFETY CAP measured in hours or days, never a polling
|
|
225
|
+
// interval. Each re-arm wakes the agent and re-reads its whole context (ENT-4850 §18: 15
|
|
226
|
+
// wakes ≈ 50M tokens), so the default is a day, the cap three days, and a cap under two hours
|
|
227
|
+
// on a non-timer wait is refused unless the caller says why (--allow-short-timeout).
|
|
228
|
+
export const DEFAULT_WAIT_TIMEOUT_SEC = 24 * 3600;
|
|
229
|
+
export const MAX_WAIT_TIMEOUT_SEC = 72 * 3600;
|
|
230
|
+
export const MIN_WAIT_TIMEOUT_SEC = 2 * 3600;
|
|
231
|
+
const SHORT_TIMEOUT_REASON_ENV = "BOTBUDDY_WAIT_SHORT_TIMEOUT_REASON";
|
|
232
|
+
const SHORT_TIMEOUT_REASON_MAX = 300;
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* BOT-2400: the short-timeout gate. Returns null when the wait may arm, else the refusal.
|
|
236
|
+
* `timer:`-only waits are exempt (the timer IS the wake); a resume keeps its saved deadline.
|
|
237
|
+
*/
|
|
238
|
+
export function shortTimeoutRefusal({ timeoutSec, conditions, reason, resuming = false }) {
|
|
239
|
+
if (resuming || !(timeoutSec < MIN_WAIT_TIMEOUT_SEC)) return null;
|
|
240
|
+
if (conditions.every((condition) => condition.type === "timer")) return null;
|
|
241
|
+
if (typeof reason === "string" && reason.trim()) return null;
|
|
242
|
+
return {
|
|
243
|
+
error: "timeout_too_short",
|
|
244
|
+
minimum_timeout_seconds: MIN_WAIT_TIMEOUT_SEC,
|
|
245
|
+
recovery: `a wait's --timeout is a safety cap in hours or days, never a polling interval: omit it (default ${DEFAULT_WAIT_TIMEOUT_SEC} s) or pass at least ${MIN_WAIT_TIMEOUT_SEC} s. ` +
|
|
246
|
+
`A caller that genuinely needs a short cap (a CI job budget, a probe) passes --allow-short-timeout "<reason>" (or ${SHORT_TIMEOUT_REASON_ENV}); the reason is recorded on the wait.`,
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
|
|
213
250
|
function parseArgv(argv) {
|
|
214
251
|
const opts = {
|
|
215
252
|
conditions: [],
|
|
216
253
|
mode: "any",
|
|
217
|
-
timeout:
|
|
254
|
+
timeout: DEFAULT_WAIT_TIMEOUT_SEC,
|
|
255
|
+
// BOT-2400: why this wait needs a cap under MIN_WAIT_TIMEOUT_SEC (recorded on the wait).
|
|
256
|
+
shortTimeoutReason: process.env[SHORT_TIMEOUT_REASON_ENV] || null,
|
|
218
257
|
since: null,
|
|
219
258
|
sinceProvided: false,
|
|
220
259
|
receiptMaxBytes: 10240,
|
|
@@ -248,6 +287,7 @@ function parseArgv(argv) {
|
|
|
248
287
|
else if (a === "--any") opts.mode = "any";
|
|
249
288
|
else if (a === "--heartbeat") opts.heartbeat = true;
|
|
250
289
|
else if (a === "--timeout") opts.timeout = Number(optionValue());
|
|
290
|
+
else if (a === "--allow-short-timeout") opts.shortTimeoutReason = optionValue();
|
|
251
291
|
else if (a === "--deadline-at") opts.deadlineAt = optionValue(); // internal recovery argv; never shown as public API
|
|
252
292
|
else if (a === "--since") { opts.sinceProvided = true; opts.since = optionValue(); }
|
|
253
293
|
else if (a === "--receipt-max-bytes") opts.receiptMaxBytes = Number(optionValue());
|
|
@@ -317,6 +357,8 @@ async function registerWait(opts, conditions, deadlineIso) {
|
|
|
317
357
|
// wait will NOT replay itself.
|
|
318
358
|
since: String(opts.since ?? -1),
|
|
319
359
|
...(opts.localWaitId ? { local_wait_id: opts.localWaitId } : {}),
|
|
360
|
+
// BOT-2400: a cap under the floor was asked for on purpose; the relay records why.
|
|
361
|
+
...(opts.shortTimeoutReasonApplied ? { short_timeout_reason: opts.shortTimeoutReasonApplied } : {}),
|
|
320
362
|
...(opts.expectedAgentId ? {
|
|
321
363
|
expected_agent_id: opts.expectedAgentId,
|
|
322
364
|
expected_tenant_id: opts.expectedTenantId ?? null,
|
|
@@ -525,6 +567,10 @@ async function registerWait(opts, conditions, deadlineIso) {
|
|
|
525
567
|
// pr-review/pr-state target. Report-only — surfaced on stderr so a silent
|
|
526
568
|
// edge-triggered park isn't mistaken for "no review yet".
|
|
527
569
|
prReviewSnapshot: Array.isArray(body.pr_review_snapshot) ? body.pr_review_snapshot : null,
|
|
570
|
+
// BOT-2400: how often this session already armed this exact condition set (24 h), and
|
|
571
|
+
// the relay's warning once that reaches 3.
|
|
572
|
+
rearmCount: Number.isInteger(body.rearm_count) ? body.rearm_count : null,
|
|
573
|
+
rearmWarning: typeof body.rearm_warning === "string" ? body.rearm_warning : null,
|
|
528
574
|
// BOT-1837: what CI had already done on the branch at arm time. An ABSENT
|
|
529
575
|
// arm_context (a pre-BOT-1837 relay, or a condition that isn't scope=next
|
|
530
576
|
// with a branch) and an explicit `latest_run: null` both collapse to null —
|
|
@@ -1613,7 +1659,19 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
|
|
|
1613
1659
|
});
|
|
1614
1660
|
process.exit(EXIT.INVALID);
|
|
1615
1661
|
}
|
|
1616
|
-
|
|
1662
|
+
// BOT-2400: refuse a short cap before anything is armed (exit 4, names the floor).
|
|
1663
|
+
const shortRefusal = shortTimeoutRefusal({
|
|
1664
|
+
timeoutSec: opts.timeout, conditions, reason: opts.shortTimeoutReason, resuming: Boolean(opts.resumeLocalWait),
|
|
1665
|
+
});
|
|
1666
|
+
if (shortRefusal) {
|
|
1667
|
+
process.stderr.write(`bb-wait: --timeout ${opts.timeout} is under the ${MIN_WAIT_TIMEOUT_SEC} s floor — ${shortRefusal.recovery}\n`);
|
|
1668
|
+
emitReceipt({ schema_version: 1, outcome: "error", ...shortRefusal });
|
|
1669
|
+
process.exit(EXIT.INVALID);
|
|
1670
|
+
}
|
|
1671
|
+
if (opts.timeout < MIN_WAIT_TIMEOUT_SEC && !conditions.every((condition) => condition.type === "timer") && !opts.resumeLocalWait) {
|
|
1672
|
+
opts.shortTimeoutReasonApplied = String(opts.shortTimeoutReason).trim().slice(0, SHORT_TIMEOUT_REASON_MAX);
|
|
1673
|
+
}
|
|
1674
|
+
const timeoutSec = Math.min(opts.timeout, MAX_WAIT_TIMEOUT_SEC); // BOT-2400: 72 h hard cap
|
|
1617
1675
|
const deadlineMs = opts.deadlineAt == null ? Date.now() + timeoutSec * 1000 : Date.parse(opts.deadlineAt);
|
|
1618
1676
|
if (!Number.isFinite(deadlineMs)) {
|
|
1619
1677
|
emitReceipt({ schema_version: 1, outcome: "error", error: "wait_checkpoint_invalid" });
|
|
@@ -1635,8 +1693,8 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
|
|
|
1635
1693
|
const createdMs = Date.parse(checkpoint.created_at);
|
|
1636
1694
|
const savedDeadlineMs = Date.parse(checkpoint.deadline_at);
|
|
1637
1695
|
if (opts.deadlineAt !== checkpoint.deadline_at || !Number.isFinite(createdMs) ||
|
|
1638
|
-
!Number.isFinite(savedDeadlineMs) || savedDeadlineMs > createdMs +
|
|
1639
|
-
throw new WaitCheckpointError("wait_checkpoint_invalid", "saved wait deadline exceeds its original
|
|
1696
|
+
!Number.isFinite(savedDeadlineMs) || savedDeadlineMs > createdMs + MAX_WAIT_TIMEOUT_SEC * 1000) {
|
|
1697
|
+
throw new WaitCheckpointError("wait_checkpoint_invalid", "saved wait deadline exceeds its original 72-hour cap");
|
|
1640
1698
|
}
|
|
1641
1699
|
const expectedSpecs = resumedSpecs(checkpoint);
|
|
1642
1700
|
if (expectedSpecs.length !== opts.conditions.length ||
|
|
@@ -1897,6 +1955,8 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
|
|
|
1897
1955
|
for (const line of formatPrReviewSnapshotWarnings(reg.prReviewSnapshot)) {
|
|
1898
1956
|
process.stderr.write(`bb-wait: ${line}\n`);
|
|
1899
1957
|
}
|
|
1958
|
+
// BOT-2400: re-arming the same wait again and again burns tokens on every wake.
|
|
1959
|
+
if (reg.rearmWarning) process.stderr.write(`bb-wait: ${reg.rearmWarning}\n`);
|
|
1900
1960
|
// BOT-1837: one line naming the branch's latest run at arm time. Report-only
|
|
1901
1961
|
// (arming before dispatch is the correct flow), and carried into a timeout
|
|
1902
1962
|
// receipt below so the receipt explains the silence too.
|
|
@@ -2161,6 +2221,9 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
|
|
|
2161
2221
|
// BOT-1837: a timeout receipt carries the arm-time CI notice, so a caller
|
|
2162
2222
|
// reading only the receipt learns the run it meant had already finished.
|
|
2163
2223
|
timeoutError: ciArmNotice,
|
|
2224
|
+
onCapacitySourceSustained: ({ host_key, stale_since }) => {
|
|
2225
|
+
process.stderr.write(`bb-wait: BLOCKED capacity_source_sustained on ${host_key}, stale_since=${stale_since ?? "unknown"}; ask for a Helper check/restart (launchctl print user/$UID/com.botbuddy.helper.worker). The wait remains armed.\n`);
|
|
2226
|
+
},
|
|
2164
2227
|
// BOT-1259: fail-closed tenant scope for the central guard, plus a debug sink
|
|
2165
2228
|
// (opt-in via BB_WAIT_DEBUG) that surfaces each fail-closed drop without turning
|
|
2166
2229
|
// it into an error.
|