@botbuddy/cli 1.46.1 → 1.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/stack-isolation.mjs +72 -11
- package/src/stack-usage.mjs +111 -0
- package/src/stack.mjs +332 -85
package/package.json
CHANGED
package/src/stack-isolation.mjs
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
// Supply Guard's `scripts/local-supabase-isolated.sh`, ported here because the
|
|
21
21
|
// published CLI cannot import repo scripts and stays Node-stdlib only.
|
|
22
22
|
import { createHash, randomUUID } from "node:crypto";
|
|
23
|
-
import { cpSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
23
|
+
import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
24
24
|
import { homedir, hostname } from "node:os";
|
|
25
25
|
import { basename, dirname, isAbsolute, join, relative, sep } from "node:path";
|
|
26
26
|
import { lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
|
|
@@ -72,6 +72,11 @@ export function defaultPortRegistryDir() {
|
|
|
72
72
|
const LEASED_STACK_RETENTION_GRACE_MS = 30 * 60 * 1000;
|
|
73
73
|
/** The idle TTL a lease gets when the caller names none (`bb stack up --idle-ttl`). */
|
|
74
74
|
const DEFAULT_IDLE_TTL_SECS = 1800;
|
|
75
|
+
/** BOT-1803: how long an IDLE lease's stack directory and port block stay reserved
|
|
76
|
+
* after its run ended. The stack is still running (the lease is `active` + idle), so
|
|
77
|
+
* another worktree must not take its ports; the next run on the same slot re-adopts it
|
|
78
|
+
* and renews the claim. Far above any idle TTL, below "forever". */
|
|
79
|
+
export const LEASED_STACK_IDLE_RETENTION_SECS = 24 * 60 * 60;
|
|
75
80
|
|
|
76
81
|
|
|
77
82
|
// TOML decimal integers may carry a leading sign (`+56321`); radix forms may not.
|
|
@@ -251,7 +256,12 @@ function identityTag(ticket, slot) {
|
|
|
251
256
|
* and the file lock then reads as reclaimable — so the refresh could swap the
|
|
252
257
|
* `supabase/functions` tree under a batch that was still running. A run's lease
|
|
253
258
|
* is released with `destroy`, so a stable identity bought no volume reuse anyway.
|
|
254
|
-
* Omit `nonce` for a deterministic per-slot identity
|
|
259
|
+
* Omit `nonce` for a deterministic per-slot identity — since BOT-1803 that is what
|
|
260
|
+
* `bb stack run` uses: a run's lease goes IDLE (still running) instead of being reaped,
|
|
261
|
+
* and the next run on the same worktree+slot re-adopts that exact stack, which needs
|
|
262
|
+
* the same directory, project id and ports. Concurrent runs on one slot are still
|
|
263
|
+
* refused by the per-project lock, and the server refuses a second LIVE lease on the
|
|
264
|
+
* slot; a nonce remains for tests.
|
|
255
265
|
*/
|
|
256
266
|
export function leasedStackIdentity({ worktreeRoot, slot, ticket, nonce } = {}) {
|
|
257
267
|
// JSON-encode the parts so no path/slot/nonce combination can alias another
|
|
@@ -626,6 +636,9 @@ function collectFinishedStacks(stacksDir, { keep, ticket, slot, readdir, remove,
|
|
|
626
636
|
// and fall back to the conservative default age.
|
|
627
637
|
const retainUntil = Number.isFinite(marker?.retain_until) ? marker.retain_until : null;
|
|
628
638
|
if (retainUntil === null && !isOwnStackFamily(entry, { ticket, slot, marker })) continue;
|
|
639
|
+
// BOT-1803: an IDLE lease's stack is still running from this directory — never collect it
|
|
640
|
+
// on a deadline (a run on its slot re-adopts or re-provisions it).
|
|
641
|
+
if (marker?.idle_retained === true) continue;
|
|
629
642
|
if (now() < (retainUntil ?? stat(join(stacksDir, entry)).mtimeMs + LEASED_STACK_GC_MIN_AGE_MS)) continue;
|
|
630
643
|
if (stackDirIsClaimed(entry, { read, alive, host, marker })) continue;
|
|
631
644
|
remove(join(stacksDir, entry), { recursive: true, force: true });
|
|
@@ -646,15 +659,19 @@ function collectFinishedStacks(stacksDir, { keep, ticket, slot, readdir, remove,
|
|
|
646
659
|
* Returns false (never throws) when there is no marker to refresh — an explicitly
|
|
647
660
|
* prepared `--stack-path` is the caller's to manage, not ours to annotate.
|
|
648
661
|
*/
|
|
649
|
-
export function refreshLeasedStackRetention(stackDir, { idleTtlSecs, now = Date.now, read = readFileSync, write = writeFileSync, rename = renameSync, portRegistryDir = null } = {}) {
|
|
662
|
+
export function refreshLeasedStackRetention(stackDir, { idleTtlSecs, idleRetained = false, now = Date.now, read = readFileSync, write = writeFileSync, rename = renameSync, portRegistryDir = null } = {}) {
|
|
650
663
|
const marker = readStackMarker(stackDir, read);
|
|
651
664
|
if (!marker) return false;
|
|
652
665
|
const ttlMs = Math.max(0, Number.isFinite(idleTtlSecs) ? idleTtlSecs : DEFAULT_IDLE_TTL_SECS) * 1000;
|
|
653
666
|
const at = now();
|
|
667
|
+
// BOT-1803: `idle_retained` marks a stack whose lease went idle (still running, kept
|
|
668
|
+
// for re-adoption); a renewal from a live batch clears it.
|
|
669
|
+
const { idle_retained: _wasIdle, released: _wasReleased, ...base } = marker;
|
|
654
670
|
const renewed = JSON.stringify({
|
|
655
|
-
...
|
|
671
|
+
...base,
|
|
656
672
|
last_seen_at: at,
|
|
657
673
|
retain_until: at + Math.max(LEASED_STACK_GC_MIN_AGE_MS, ttlMs + LEASED_STACK_RETENTION_GRACE_MS),
|
|
674
|
+
...(idleRetained ? { idle_retained: true } : {}),
|
|
658
675
|
});
|
|
659
676
|
try {
|
|
660
677
|
atomicWrite(join(stackDir, LEASED_STACK_MARKER), renewed, { write, rename });
|
|
@@ -684,6 +701,34 @@ export function refreshLeasedStackRetention(stackDir, { idleTtlSecs, now = Date.
|
|
|
684
701
|
return true;
|
|
685
702
|
}
|
|
686
703
|
|
|
704
|
+
/**
|
|
705
|
+
* BOT-1803 (Codex #1160 P1): may `stack run` re-materialize the STABLE per-slot stack
|
|
706
|
+
* directory? Only when no batch can still be using it: no marker yet, a lease that went idle
|
|
707
|
+
* cleanly (`idle_retained`), or a batch whose reap was proven (`released`). Any other marker
|
|
708
|
+
* may belong to a batch whose owner process died while its detached child and containers
|
|
709
|
+
* still run — re-copying `supabase/` under those bind mounts is exactly what the
|
|
710
|
+
* per-invocation identity (BOT-1798) prevented, so such a run gets a fresh identity instead.
|
|
711
|
+
*/
|
|
712
|
+
export function stableRunStackReusable(worktreeRoot, dirName, { read = readFileSync } = {}) {
|
|
713
|
+
if (typeof dirName !== "string" || !dirName) return true;
|
|
714
|
+
const marker = readStackMarker(join(worktreeRoot, LEASED_STACKS_DIR, dirName), read);
|
|
715
|
+
if (!marker) return true;
|
|
716
|
+
return marker.idle_retained === true || marker.released === true;
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
/** BOT-1803: record a PROVEN reap on the stack's marker, so the next run may reuse the directory. */
|
|
720
|
+
export function markLeasedStackReleased(stackDir, { read = readFileSync, write = writeFileSync, rename = renameSync } = {}) {
|
|
721
|
+
const marker = readStackMarker(stackDir, read);
|
|
722
|
+
if (!marker) return false;
|
|
723
|
+
const { idle_retained: _idle, ...base } = marker;
|
|
724
|
+
try {
|
|
725
|
+
atomicWrite(join(stackDir, LEASED_STACK_MARKER), JSON.stringify({ ...base, released: true }), { write, rename });
|
|
726
|
+
return true;
|
|
727
|
+
} catch {
|
|
728
|
+
return false;
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
|
|
687
732
|
/** This stack's ownership/retention marker, or null when it has none. */
|
|
688
733
|
function readStackMarker(stackDir, read) {
|
|
689
734
|
try {
|
|
@@ -702,7 +747,7 @@ function readStackMarker(stackDir, read) {
|
|
|
702
747
|
* them and the second Helper fails on a bind error after provisioning starts.
|
|
703
748
|
* A block counts as taken while its stack is retained or its lock owner is alive.
|
|
704
749
|
*/
|
|
705
|
-
function occupiedPortBlocks(registryDir, { keep, readdir, read, now, alive, host, stale = null }) {
|
|
750
|
+
function occupiedPortBlocks(registryDir, { keep, readdir, read, now, alive, host, stale = null, exists = existsSync }) {
|
|
706
751
|
const taken = new Map(); // portBase -> { projectId, createdAt }
|
|
707
752
|
let entries;
|
|
708
753
|
try { entries = readdir(registryDir); } catch { return taken; }
|
|
@@ -712,7 +757,12 @@ function occupiedPortBlocks(registryDir, { keep, readdir, read, now, alive, host
|
|
|
712
757
|
try { claim = JSON.parse(read(join(registryDir, entry), "utf8")); } catch { continue; }
|
|
713
758
|
const portBase = Number(claim?.port_base);
|
|
714
759
|
if (!Number.isInteger(portBase)) continue;
|
|
715
|
-
|
|
760
|
+
// BOT-1803: an IDLE lease's stack keeps running with no process renewing its claim, and
|
|
761
|
+
// the lease has no time ceiling — so its claim stands while its stack directory exists
|
|
762
|
+
// (until a run on the same slot re-adopts or re-provisions it, or the worktree is gone),
|
|
763
|
+
// never on a fixed deadline.
|
|
764
|
+
const retained = (Number.isFinite(claim.retain_until) && now() < claim.retain_until)
|
|
765
|
+
|| (claim.idle_retained === true && typeof claim.stack_dir === "string" && exists(claim.stack_dir));
|
|
716
766
|
const ownerAlive = Number.isInteger(claim.owner_pid) && claim.owner_pid > 0
|
|
717
767
|
&& (!claim.host || claim.host === host()) && alive(claim.owner_pid);
|
|
718
768
|
if (!retained && !ownerAlive) {
|
|
@@ -923,7 +973,7 @@ export function reservePortBlocks(ports, { registryDir = defaultPortRegistryDir(
|
|
|
923
973
|
|
|
924
974
|
mkdir(registryDir, { recursive: true });
|
|
925
975
|
const claimed = [];
|
|
926
|
-
const record = (block, at = now(), ttlSecs = owner?.idleTtlSecs) => JSON.stringify({
|
|
976
|
+
const record = (block, at = now(), ttlSecs = owner?.idleTtlSecs, idleRetained = false) => JSON.stringify({
|
|
927
977
|
schema: 1,
|
|
928
978
|
project_id: owner?.projectId ?? null,
|
|
929
979
|
invocation_token: invocationToken,
|
|
@@ -935,6 +985,7 @@ export function reservePortBlocks(ports, { registryDir = defaultPortRegistryDir(
|
|
|
935
985
|
retain_until: at + Math.max(LEASED_STACK_GC_MIN_AGE_MS, (Number(ttlSecs) || DEFAULT_IDLE_TTL_SECS) * 1000 + LEASED_STACK_RETENTION_GRACE_MS),
|
|
936
986
|
owner_pid: process.pid,
|
|
937
987
|
host: host(),
|
|
988
|
+
...(idleRetained ? { idle_retained: true } : {}),
|
|
938
989
|
});
|
|
939
990
|
try {
|
|
940
991
|
withAllocationLock(registryDir, { writeExclusive, read, remove, now, alive, host, sleep }, () => {
|
|
@@ -959,7 +1010,11 @@ export function reservePortBlocks(ports, { registryDir = defaultPortRegistryDir(
|
|
|
959
1010
|
// Only THIS invocation's own reservation is not a peer.
|
|
960
1011
|
let existing = null;
|
|
961
1012
|
try { existing = JSON.parse(read(portClaimPath(registryDir, block), "utf8")); } catch { /* unreadable */ }
|
|
962
|
-
|
|
1013
|
+
// BOT-1803: a claim kept for an IDLE lease of this same prepared project is the
|
|
1014
|
+
// stack this run is about to re-adopt (or re-provision) on the same slot — the
|
|
1015
|
+
// project lock already serializes runs of one project, so take it over.
|
|
1016
|
+
const idleOwnProject = existing?.idle_retained === true && owner?.projectId != null && existing.project_id === owner.projectId;
|
|
1017
|
+
if (existing && (existing.invocation_token === invocationToken || idleOwnProject)) write(portClaimPath(registryDir, block), record(block));
|
|
963
1018
|
else {
|
|
964
1019
|
const busy = new Error(`refusing to lease this stack: port block ${block} was claimed by another run.`);
|
|
965
1020
|
busy.code = "stack_ports_busy";
|
|
@@ -981,13 +1036,13 @@ export function reservePortBlocks(ports, { registryDir = defaultPortRegistryDir(
|
|
|
981
1036
|
/** Keep the reservation alive while the batch runs (Codex #928 R21): a
|
|
982
1037
|
* prepared `--stack-path` has no directory marker, so nothing else renews it,
|
|
983
1038
|
* and an aged-out claim could be reclaimed under a still-live stack. */
|
|
984
|
-
renew: ({ idleTtlSecs, now: at = now } = {}) => {
|
|
1039
|
+
renew: ({ idleTtlSecs, idleRetained = false, now: at = now } = {}) => {
|
|
985
1040
|
let renewed = false;
|
|
986
1041
|
for (const block of claimed) {
|
|
987
1042
|
try {
|
|
988
1043
|
const current = JSON.parse(read(portClaimPath(registryDir, block), "utf8"));
|
|
989
1044
|
if (current?.invocation_token !== invocationToken) continue;
|
|
990
|
-
write(portClaimPath(registryDir, block), record(block, at(), idleTtlSecs ?? owner?.idleTtlSecs));
|
|
1045
|
+
write(portClaimPath(registryDir, block), record(block, at(), idleTtlSecs ?? owner?.idleTtlSecs, idleRetained));
|
|
991
1046
|
renewed = true;
|
|
992
1047
|
} catch { /* advisory: never fence a healthy batch */ }
|
|
993
1048
|
}
|
|
@@ -1356,8 +1411,14 @@ export function materializeRunStackDir(cwd, { slot, ticket, nonce, idleTtlSecs }
|
|
|
1356
1411
|
// allocation lock (Codex #928 R16): the scan only REPORTS stale reservations, and
|
|
1357
1412
|
// reclaiming one plus taking the block happen as one serialized step, so a
|
|
1358
1413
|
// slower peer can never delete the winner's fresh claim.
|
|
1414
|
+
// BOT-1803: a stable (nonce-less) identity re-materializes the SAME stack each run, so
|
|
1415
|
+
// it keeps the block it already holds (an idle lease's stack is still bound to it, and
|
|
1416
|
+
// the deterministic stack digest covers the remapped ports) instead of drifting back to
|
|
1417
|
+
// the hash-preferred block whenever that one has freed up.
|
|
1418
|
+
const previous = dirExisted ? readStackMarker(stackDir, read) : null;
|
|
1419
|
+
const previousBlock = previous?.project_id === identity.projectId && Number.isInteger(previous?.port_base) ? previous.port_base : null;
|
|
1359
1420
|
const portBase = withAllocationLock(registryDir, { writeExclusive, read, remove, now, alive, host, sleep }, () => {
|
|
1360
|
-
let preferred = identity.portBase;
|
|
1421
|
+
let preferred = previousBlock ?? identity.portBase;
|
|
1361
1422
|
for (let attempt = 0; attempt < LEASED_PORT_SLOTS; attempt++) {
|
|
1362
1423
|
const stale = new Set();
|
|
1363
1424
|
const taken = occupiedPortBlocks(registryDir, { keep: identity.projectId, readdir, read, now, alive, host, stale });
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
// BOT-1803 (AC-12) — the client-side USE signal for a leased stack.
|
|
2
|
+
//
|
|
3
|
+
// The server judges a lease idle only from observations. For a CO-LOCATED stack
|
|
4
|
+
// (execution_mode `local` / `local_exec`) the coding machine can read the stack's
|
|
5
|
+
// own Postgres: `docker exec supabase_db_<project_id> psql … pg_stat_activity` says
|
|
6
|
+
// whether any client is connected and working right now, and when a client last
|
|
7
|
+
// changed state. `bb stack touch` and `bb stack run`'s heartbeat send that as
|
|
8
|
+
// `usage: { client_connections, last_client_query_at }`.
|
|
9
|
+
//
|
|
10
|
+
// Fail toward "no observation": a remote lease, an unusable project id, a Docker/psql
|
|
11
|
+
// failure or a slow answer (> USAGE_TIMEOUT_MS) yields `null`, and the caller then
|
|
12
|
+
// sends NO `usage` at all — the server never judges a lease idle from an observation
|
|
13
|
+
// that did not happen. Nothing here reads or prints a secret: the query runs as the
|
|
14
|
+
// container's local `postgres` role over its Unix socket.
|
|
15
|
+
//
|
|
16
|
+
// Dependency-free (Node stdlib only), like the rest of the published CLI.
|
|
17
|
+
import { execFile } from "child_process";
|
|
18
|
+
|
|
19
|
+
/** How long one measurement may take before it counts as no observation. */
|
|
20
|
+
export const USAGE_TIMEOUT_MS = 5_000;
|
|
21
|
+
|
|
22
|
+
/** Supabase's own service roles: their connections are the platform, not a client. */
|
|
23
|
+
export const SUPABASE_INTERNAL_ROLES = Object.freeze([
|
|
24
|
+
"supabase_admin", "supabase_auth_admin", "supabase_storage_admin", "supabase_realtime_admin",
|
|
25
|
+
"supabase_replication_admin", "supabase_functions_admin", "supabase_read_only_user", "pgbouncer", "authenticator",
|
|
26
|
+
]);
|
|
27
|
+
|
|
28
|
+
const ROLE_LIST = SUPABASE_INTERNAL_ROLES.map((r) => `'${r}'`).join(", ");
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Two values, `|`-separated (psql -tA):
|
|
32
|
+
* 1. client_connections — client backends doing work now (state <> 'idle'), excluding
|
|
33
|
+
* this probe and every internal role except `authenticator`: a PostgREST request in
|
|
34
|
+
* flight is use now (Codex #1160 P1); its pooled sessions sitting idle are not.
|
|
35
|
+
* 2. last_client_query_at — the latest state change of any client backend, counting
|
|
36
|
+
* `authenticator` (PostgREST traffic IS use), ISO-8601 UTC; empty when none.
|
|
37
|
+
*/
|
|
38
|
+
export const USAGE_SQL =
|
|
39
|
+
"SELECT " +
|
|
40
|
+
"(SELECT count(*) FROM pg_stat_activity WHERE backend_type = 'client backend' AND pid <> pg_backend_pid() " +
|
|
41
|
+
`AND (coalesce(usename, '') NOT IN (${ROLE_LIST}) OR usename = 'authenticator') AND state <> 'idle'), ` +
|
|
42
|
+
"(SELECT to_char(max(state_change) AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM pg_stat_activity " +
|
|
43
|
+
"WHERE backend_type = 'client backend' AND pid <> pg_backend_pid() " +
|
|
44
|
+
`AND (coalesce(usename, '') NOT IN (${ROLE_LIST}) OR usename = 'authenticator'))`;
|
|
45
|
+
|
|
46
|
+
const CO_LOCATED_MODES = new Set(["local", "local_exec"]);
|
|
47
|
+
// `supabase_db_<id>` is named after the Supabase project id; refuse anything else
|
|
48
|
+
// rather than build a container name (an argv element) out of it.
|
|
49
|
+
const PROJECT_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,62}$/;
|
|
50
|
+
|
|
51
|
+
/** The Docker CLI selector for the engine running the stack ([] = the CLI's current context). */
|
|
52
|
+
export function dockerSelectorArgs(target) {
|
|
53
|
+
if (typeof target?.context === "string" && target.context) return ["--context", target.context];
|
|
54
|
+
const endpoint = target?.resolved_endpoint ?? target?.endpoint;
|
|
55
|
+
if (typeof endpoint === "string" && endpoint) return ["-H", endpoint];
|
|
56
|
+
return [];
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** The full `docker` argv for one measurement of `projectId`'s database. */
|
|
60
|
+
export function usageDockerArgs(projectId, target = null) {
|
|
61
|
+
return [...dockerSelectorArgs(target), "exec", `supabase_db_${projectId}`, "psql", "-U", "postgres", "-d", "postgres", "-tAc", USAGE_SQL];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Parse psql's `-tA` answer (`<count>|<iso or empty>`); null when it is not that shape. */
|
|
65
|
+
export function parseUsageOutput(text) {
|
|
66
|
+
const line = String(text ?? "").split(/\r?\n/).map((l) => l.trim()).find((l) => l.length > 0);
|
|
67
|
+
if (!line) return null;
|
|
68
|
+
const parts = line.split("|");
|
|
69
|
+
if (parts.length !== 2) return null;
|
|
70
|
+
const [countRaw, atRaw] = parts.map((p) => p.trim());
|
|
71
|
+
if (!/^\d+$/.test(countRaw)) return null;
|
|
72
|
+
if (atRaw && Number.isNaN(Date.parse(atRaw))) return null;
|
|
73
|
+
return { client_connections: Number(countRaw), last_client_query_at: atRaw || null };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Default runner: `docker <args>` with a hard timeout; never a shell. */
|
|
77
|
+
export function runDockerUsage(args, { timeoutMs = USAGE_TIMEOUT_MS } = {}) {
|
|
78
|
+
return new Promise((resolve) => {
|
|
79
|
+
// nosemgrep: javascript.lang.security.detect-child-process.detect-child-process -- fixed executable, validated argv, no shell.
|
|
80
|
+
execFile("docker", args, { timeout: timeoutMs, killSignal: "SIGKILL", maxBuffer: 64 * 1024, windowsHide: true }, (error, stdout) => {
|
|
81
|
+
resolve({ code: error ? (typeof error.code === "number" ? error.code : 1) : 0, stdout: String(stdout ?? ""), timedOut: Boolean(error?.killed) });
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Measure a lease's client use. `lease` is a `get_stack_lease` answer (or the same shape):
|
|
88
|
+
* `{ execution_mode, connection: { project_id, botbuddy_docker_target? } }`.
|
|
89
|
+
* Returns `{ client_connections, last_client_query_at }`, or null for no observation.
|
|
90
|
+
* Never throws.
|
|
91
|
+
*/
|
|
92
|
+
export async function measureStackUsage(lease, { run = runDockerUsage, timeoutMs = USAGE_TIMEOUT_MS } = {}) {
|
|
93
|
+
let timer = null;
|
|
94
|
+
try {
|
|
95
|
+
if (!CO_LOCATED_MODES.has(lease?.execution_mode)) return null;
|
|
96
|
+
const projectId = lease?.connection?.project_id;
|
|
97
|
+
if (typeof projectId !== "string" || !PROJECT_ID_RE.test(projectId)) return null;
|
|
98
|
+
const args = usageDockerArgs(projectId, lease.connection.botbuddy_docker_target ?? null);
|
|
99
|
+
// Not unref'd: the timer is the only thing that settles a hung measurement, and it is
|
|
100
|
+
// cleared in `finally` the moment the answer lands (an unref'd timer let the event loop end
|
|
101
|
+
// with the race still pending on Linux CI).
|
|
102
|
+
const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve(null), timeoutMs); });
|
|
103
|
+
const answer = await Promise.race([Promise.resolve().then(() => run(args, { timeoutMs })), timeout]);
|
|
104
|
+
if (!answer || answer.code !== 0 || answer.timedOut) return null;
|
|
105
|
+
return parseUsageOutput(answer.stdout);
|
|
106
|
+
} catch {
|
|
107
|
+
return null;
|
|
108
|
+
} finally {
|
|
109
|
+
if (timer) clearTimeout(timer);
|
|
110
|
+
}
|
|
111
|
+
}
|
package/src/stack.mjs
CHANGED
|
@@ -6,10 +6,16 @@
|
|
|
6
6
|
// saturated; return an ACTIVE lease receipt.
|
|
7
7
|
// botbuddy stack status <id> → the lease's current state + connection.
|
|
8
8
|
// botbuddy stack touch <id> → bump the idle clock (before a long silent step).
|
|
9
|
-
// botbuddy stack done <id> → release (`done`):
|
|
9
|
+
// botbuddy stack done <id> → release (`done`): idle the stack (BOT-1803; --reap tears it down).
|
|
10
10
|
//
|
|
11
11
|
// The canonical batch flow is: stack up → test → break → fix → test → stack done.
|
|
12
12
|
//
|
|
13
|
+
// BOT-1803 (JIT idle leases): a finished batch's lease goes IDLE instead of being reaped —
|
|
14
|
+
// still `active`, stack running, capacity still held — so the next batch on the same
|
|
15
|
+
// worktree+slot re-adopts it with no provision. The control plane preempts an idle lease
|
|
16
|
+
// only when a queued lease needs the capacity (volumes kept), and re-provisions it when the
|
|
17
|
+
// stack's content digest changed.
|
|
18
|
+
//
|
|
13
19
|
// Design contract (matches bb-wait):
|
|
14
20
|
// * Dependency-free (Node stdlib only) so an explicit `npm exec --package` invocation cold-starts fast.
|
|
15
21
|
// * Exactly ONE single-line JSON receipt to stdout; all diagnostics go to stderr.
|
|
@@ -36,7 +42,8 @@ import { AGENT_KEY_RE, readAgentSessionTokenEnv } from "./agent-key.mjs";
|
|
|
36
42
|
import { resolveAgentSessionCredential, selfHealAgentSession } from "./agent-session.mjs";
|
|
37
43
|
import { runDockerCommand, runDockerWorkflow, ADMITTED_DOCKER_VALIDATIONS } from "./docker-hygiene.mjs";
|
|
38
44
|
import { acquireStackLock, lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
|
|
39
|
-
import { assertLeasedStackIsolated, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
|
|
45
|
+
import { LEASED_STACK_IDLE_RETENTION_SECS, assertLeasedStackIsolated, markLeasedStackReleased, stableRunStackReusable, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
|
|
46
|
+
import { measureStackUsage } from "./stack-usage.mjs";
|
|
40
47
|
import { machineUuid } from "./machine-id.mjs";
|
|
41
48
|
import { DEFAULT_BUNDLE_MAX_BYTES, buildStackBundle, uploadStackBundle, validateBundleIncludePath } from "./stack-bundle.mjs";
|
|
42
49
|
import { assertSshTarget, establishSshTunnel } from "./stack-tunnel.mjs";
|
|
@@ -87,8 +94,8 @@ export const STACK_HELP = `${bold("botbuddy stack")} — one command for a batch
|
|
|
87
94
|
${bold("USAGE")}
|
|
88
95
|
botbuddy stack up [options] Request a lease; park if the host is full; hold it active
|
|
89
96
|
botbuddy stack status <lease_id> Show a lease's state + connection
|
|
90
|
-
botbuddy stack touch <lease_id> Bump the idle clock so the reaper doesn't stop it
|
|
91
|
-
botbuddy stack done <lease_id> Release (done):
|
|
97
|
+
botbuddy stack touch <lease_id> Bump the idle clock (and report measured use) so the reaper doesn't stop it
|
|
98
|
+
botbuddy stack done <lease_id> Release (done): idle the stack for re-adoption (--reap tears it down)
|
|
92
99
|
botbuddy stack run [options] -- <executable> [args...]
|
|
93
100
|
Own one Helper-backed test/fix batch end-to-end
|
|
94
101
|
|
|
@@ -138,8 +145,14 @@ ${bold("up OPTIONS")}
|
|
|
138
145
|
SSH forwards; --remote is refused here — use \`stack run --remote\`.
|
|
139
146
|
|
|
140
147
|
${bold("done OPTIONS")}
|
|
141
|
-
|
|
148
|
+
(default) BOT-1803: an ACTIVE lease goes IDLE — the stack keeps running and its
|
|
149
|
+
capacity stays held, so the next \`stack run\`/\`stack up\` on this slot re-adopts
|
|
150
|
+
it without provisioning. Receipt outcome "idle" + idle_since, idle_reason.
|
|
151
|
+
A queued/provisioning/reaping/reaped lease is released exactly as --reap.
|
|
152
|
+
--reap Release eagerly: reap (tear down) the stack now, as before BOT-1803.
|
|
153
|
+
--stop With --reap: keep volumes (cheap re-provision next batch). Default: destroy.
|
|
142
154
|
--local-exec FALLBACK (no Helper): run 'supabase stop' locally and self-finalize.
|
|
155
|
+
Implies --reap (a local-exec stack is never left idle).
|
|
143
156
|
--docker-context <name> Explicit Docker context used by the matching local-exec up
|
|
144
157
|
(OrbStack or Docker Desktop; must match the engine that provisioned the lease).
|
|
145
158
|
--docker-endpoint <uri> Explicit Docker endpoint instead of --docker-context.
|
|
@@ -152,10 +165,16 @@ ${bold("run OPTIONS")}
|
|
|
152
165
|
--stack-path <relative> Optional PREPARED isolated stack dir. Omit it (the norm): run
|
|
153
166
|
materializes .botbuddy/stacks/<project_id>/ from this worktree's
|
|
154
167
|
supabase/ tree with a distinct project_id + remapped ports and
|
|
155
|
-
leases that.
|
|
168
|
+
leases that. The identity is STABLE per worktree+slot (BOT-1803), so
|
|
169
|
+
the next run maps to the same directory, project and ports. "." is REFUSED — the worktree root is the shared
|
|
156
170
|
canonical project and the Helper would start/clobber it.
|
|
157
171
|
--provision-timeout N Seconds to wait for a physical Helper provision (default: --timeout).
|
|
158
|
-
--reap-timeout N Seconds to wait for signed physical reap proof (default: 300)
|
|
172
|
+
--reap-timeout N Seconds to wait for signed physical reap proof (default: 300); also bounds
|
|
173
|
+
the wait for an idle lease being re-provisioned/preempted to reach reaped.
|
|
174
|
+
--reap Reap the stack when the child ends (as before BOT-1803). Default: the lease
|
|
175
|
+
goes IDLE (no reap wait; receipt cleanup {idle, idle_since}) and the stack
|
|
176
|
+
directory + port block stay reserved (24 h, renewed on re-adoption).
|
|
177
|
+
Interrupts, --hard-ttl, a fenced heartbeat or a lost tunnel always reap.
|
|
159
178
|
--connection-file P Optional empty file path for the mode-0600 connection JSON.
|
|
160
179
|
--hard-ttl N Bound the child runtime; timeout requests normal cleanup.
|
|
161
180
|
--local-exec Rejected: stack run never starts Docker directly.
|
|
@@ -196,6 +215,21 @@ ${bold("EXIT CODES")}
|
|
|
196
215
|
4 invalid arguments 5 backend/coordination 6 lease failed/reaped
|
|
197
216
|
7 internal error 8 cleanup/reap proof failed
|
|
198
217
|
|
|
218
|
+
${bold("IDLE LEASES")} (BOT-1803)
|
|
219
|
+
stack run sends stack_digest (the deterministic sha256 of the materialised stack bundle, also
|
|
220
|
+
with --local-only; omitted when it cannot be built) and the control plane answers with one of:
|
|
221
|
+
reused_idle the slot's idle lease is re-adopted: no provision; receipt reused "idle"
|
|
222
|
+
IDLE_LEASE_REPROVISIONING / LEASE_PREEMPTED
|
|
223
|
+
the idle lease is being reaped (content changed / capacity reclaimed): run
|
|
224
|
+
waits for it to reach reaped (--reap-timeout), re-sends the same request
|
|
225
|
+
once, and the receipt says reprovisioned "digest_mismatch" | "preempted";
|
|
226
|
+
no reap in time → exit 6
|
|
227
|
+
4 STACK_EXECUTION_MODE_MISMATCH the idle lease is in another mode/host (it stays idle)
|
|
228
|
+
While the child runs (and on \`stack touch\`) a co-located lease's measured use is sent as
|
|
229
|
+
usage {client_connections, last_client_query_at} (docker exec … psql pg_stat_activity, 5 s cap;
|
|
230
|
+
no usage at all when it cannot be read). A lease preempted under a running child ends it:
|
|
231
|
+
exit 6, code LEASE_PREEMPTED, volumes_kept true; \`stack touch\` exits 6 with outcome "preempted".
|
|
232
|
+
|
|
199
233
|
${bold("REMOTE LEASES")} (BOT-2142, stack run)
|
|
200
234
|
The receipt carries execution_mode (local|remote), execution_host_key, requester_host,
|
|
201
235
|
transport (none|ssh-forward), and for a remote lease forwards [{name, remote, local}] + bundle.
|
|
@@ -295,6 +329,8 @@ export function parseStackArgs(argv) {
|
|
|
295
329
|
// default exclusions (`--bundle-include`, repeatable).
|
|
296
330
|
bundleInclude: [],
|
|
297
331
|
disposition: "destroy", json: false, receiptMaxBytes: DEFAULT_RECEIPT_MAX_BYTES,
|
|
332
|
+
// BOT-1803: eager release (reap) instead of the default idle release (done/run).
|
|
333
|
+
reap: false,
|
|
298
334
|
};
|
|
299
335
|
const positionals = [];
|
|
300
336
|
const need = (name, v) => {
|
|
@@ -413,6 +449,7 @@ export function parseStackArgs(argv) {
|
|
|
413
449
|
case "--docker-context": if (need(a, rest[i + 1])) opts.dockerContext = rest[++i]; break;
|
|
414
450
|
case "--docker-endpoint": if (need(a, rest[i + 1])) opts.dockerEndpoint = rest[++i]; break;
|
|
415
451
|
case "--stop": opts.disposition = "stop"; break;
|
|
452
|
+
case "--reap": opts.reap = true; break;
|
|
416
453
|
case "--destroy": opts.disposition = "destroy"; break;
|
|
417
454
|
case "--json": opts.json = true; break;
|
|
418
455
|
default:
|
|
@@ -462,6 +499,9 @@ export function parseStackArgs(argv) {
|
|
|
462
499
|
"worktree root's default (shared canonical) project. Point --stack-path at a slot-derived stack directory with its " +
|
|
463
500
|
"own supabase/config.toml (distinct project_id + remapped ports).");
|
|
464
501
|
}
|
|
502
|
+
if (opts.reap && !["done", "run"].includes(command)) {
|
|
503
|
+
errors.push("--reap is valid only with `stack done` or `stack run`");
|
|
504
|
+
}
|
|
465
505
|
if (opts.waitCapacity != null && !["up", "run"].includes(command)) {
|
|
466
506
|
errors.push("--wait-capacity is valid only with `stack up` or `stack run`");
|
|
467
507
|
}
|
|
@@ -642,15 +682,17 @@ export function resolveStackPath(cwd = process.cwd(), stackPath = ".") {
|
|
|
642
682
|
*
|
|
643
683
|
* Fail closed: whatever produced the path, a `run` lease at `"."` is refused.
|
|
644
684
|
*
|
|
645
|
-
*
|
|
646
|
-
*
|
|
647
|
-
*
|
|
648
|
-
*
|
|
649
|
-
*
|
|
650
|
-
*
|
|
651
|
-
*
|
|
652
|
-
*
|
|
653
|
-
*
|
|
685
|
+
* BOT-1803: the identity is STABLE per (worktree, slot) — no per-invocation nonce —
|
|
686
|
+
* because a finished run's lease now goes IDLE with its stack still running, and the
|
|
687
|
+
* next run on the slot re-adopts that exact stack (same directory, project id and
|
|
688
|
+
* ports, so the deterministic stack digest is comparable across runs). Concurrency is
|
|
689
|
+
* unchanged: the per-project lock is held for the whole batch, so a second concurrent
|
|
690
|
+
* run on the same slot is refused (stack_slot_busy), and the server refuses a second
|
|
691
|
+
* LIVE lease on the slot. (BOT-1798's per-invocation identity, Codex #928 R3/R5,
|
|
692
|
+
* guarded a batch whose OWNER process died while its detached child still ran; a run
|
|
693
|
+
* re-materializing over such a batch is the residual risk of the stable identity, and
|
|
694
|
+
* the server's live-lease refusal then stops it before any Helper acts.)
|
|
695
|
+
* `opts.stackNonce` still forces a distinct identity (tests).
|
|
654
696
|
*/
|
|
655
697
|
async function claimProjectLock(projectId, lock, subject) {
|
|
656
698
|
try {
|
|
@@ -665,7 +707,7 @@ async function claimProjectLock(projectId, lock, subject) {
|
|
|
665
707
|
}
|
|
666
708
|
}
|
|
667
709
|
|
|
668
|
-
export async function resolveRunExecution(cwd, opts, { materialize = materializeRunStackDir, lock = acquireStackLock, identity = leasedStackIdentity, reserveBlocks = reservePortBlocks, releaseClaim = releaseLeasedStackClaim, portRegistryDir = null } = {}) {
|
|
710
|
+
export async function resolveRunExecution(cwd, opts, { materialize = materializeRunStackDir, lock = acquireStackLock, identity = leasedStackIdentity, reserveBlocks = reservePortBlocks, releaseClaim = releaseLeasedStackClaim, stableReusable = stableRunStackReusable, markReleased = markLeasedStackReleased, portRegistryDir = null } = {}) {
|
|
669
711
|
const refuseRoot = (execution) => {
|
|
670
712
|
if (execution.stackPath === ".") {
|
|
671
713
|
throw new Error(
|
|
@@ -728,8 +770,12 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
|
|
|
728
770
|
};
|
|
729
771
|
}
|
|
730
772
|
const worktreeRoot = realpathSync(cwd);
|
|
731
|
-
//
|
|
732
|
-
|
|
773
|
+
// BOT-1803: a stable per-(worktree, slot) identity unless a test pins a nonce — and only
|
|
774
|
+
// when no batch can still be using that directory (Codex #1160 P1): a marker left by a
|
|
775
|
+
// batch that neither went idle nor proved its reap may belong to a live detached child,
|
|
776
|
+
// so this run takes a fresh per-invocation identity instead (BOT-1798's protection).
|
|
777
|
+
const stableDir = identity({ worktreeRoot, slot: opts.slot, ticket: opts.ticket, nonce: null }).dirName;
|
|
778
|
+
const nonce = opts.stackNonce ?? (stableReusable(worktreeRoot, stableDir) ? null : randomUUID());
|
|
733
779
|
const { projectId } = identity({ worktreeRoot, slot: opts.slot, ticket: opts.ticket, nonce });
|
|
734
780
|
const held = await claimProjectLock(projectId, lock, `this batch's isolated stack (${projectId})`);
|
|
735
781
|
try {
|
|
@@ -751,6 +797,8 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
|
|
|
751
797
|
release: ({ reaped = false } = {}) => {
|
|
752
798
|
if (reaped) {
|
|
753
799
|
try { releaseClaim(stackDirFor(execution), portRegistryDir ? { registryDir: portRegistryDir } : undefined); } catch { /* advisory */ }
|
|
800
|
+
// BOT-1803: the stack is provably gone, so the next run may reuse this directory.
|
|
801
|
+
try { markReleased(stackDirFor(execution)); } catch { /* advisory */ }
|
|
754
802
|
}
|
|
755
803
|
held.release();
|
|
756
804
|
},
|
|
@@ -2421,6 +2469,8 @@ export async function cmdUp(opts, {
|
|
|
2421
2469
|
command: "up", outcome: "active", lease_id: leaseId, state: "active",
|
|
2422
2470
|
host_key: g.host_key, slot: g.slot, connection: g.connection,
|
|
2423
2471
|
idle_ttl_secs: g.idle_ttl_secs, resource_name: g.resource_name,
|
|
2472
|
+
// BOT-1803: the slot's IDLE lease was re-adopted (already running; nothing provisioned).
|
|
2473
|
+
...(d.reused_idle === true ? { reused: "idle" } : {}),
|
|
2424
2474
|
...(localPreflight ? { preflight: localPreflight } : {}),
|
|
2425
2475
|
}), opts, EXIT.OK);
|
|
2426
2476
|
}
|
|
@@ -2453,16 +2503,55 @@ export async function cmdStatus(leaseId, opts, {
|
|
|
2453
2503
|
...(g.reap_requested_at ? { reap_requested_at: g.reap_requested_at } : {}),
|
|
2454
2504
|
...(g.reap_failed_at ? { reap_failed: true, reap_failed_at: g.reap_failed_at } : {}),
|
|
2455
2505
|
...(g.state === "reaping" && g.error ? { error: g.error } : {}),
|
|
2506
|
+
// BOT-1803: an idle lease is `active` with idle_since; display_state says "idle".
|
|
2507
|
+
...idleStatusFields(g),
|
|
2456
2508
|
}), opts, EXIT.OK);
|
|
2457
2509
|
}
|
|
2458
2510
|
|
|
2459
|
-
|
|
2511
|
+
/** BOT-1803: the idle/use fields a get_stack_lease answer carries (absent on older servers). */
|
|
2512
|
+
function idleStatusFields(g) {
|
|
2513
|
+
const out = {};
|
|
2514
|
+
for (const key of ["display_state", "idle", "idle_since", "idle_reason", "last_observed_use_at", "usage_observed_at", "usage_observed_source", "reap_reason", "preempted_for_lease_id"]) {
|
|
2515
|
+
if (g?.[key] !== undefined && g?.[key] !== null) out[key] = g[key];
|
|
2516
|
+
}
|
|
2517
|
+
return out;
|
|
2518
|
+
}
|
|
2519
|
+
|
|
2520
|
+
/** BOT-1803: is a touch/get answer the control plane's "this lease was preempted"? */
|
|
2521
|
+
export function isPreemptedAnswer(d) {
|
|
2522
|
+
return d?.code === "LEASE_PREEMPTED" || (d?.success === false && d?.reap_reason === "preempted");
|
|
2523
|
+
}
|
|
2524
|
+
|
|
2525
|
+
/**
|
|
2526
|
+
* BOT-1803: `stack touch` bumps the idle clock and, for a co-located lease, reports the
|
|
2527
|
+
* measured client use (AC-12). Reading the lease first (for its project + Docker target)
|
|
2528
|
+
* is best-effort: without it the touch simply carries no usage. A preempted lease is a
|
|
2529
|
+
* typed receipt and exit 6 — the keeper exit Supply Guard's heartbeat daemon sees (AC-11).
|
|
2530
|
+
*/
|
|
2531
|
+
export async function cmdTouch(leaseId, opts, { rpc = retryingStackCall(), emitResult = emit, measureUsage = measureStackUsage } = {}) {
|
|
2460
2532
|
const auth = await rpc.init();
|
|
2461
|
-
if (!auth) return
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2533
|
+
if (!auth) return emitResult(buildReceipt({ command: "touch", outcome: "error", lease_id: leaseId, error: "not authenticated — run `botbuddy login`" }), opts, EXIT.AUTH);
|
|
2534
|
+
let usage = null;
|
|
2535
|
+
try {
|
|
2536
|
+
const got = await rpc.call("get_stack_lease", { lease_id: leaseId });
|
|
2537
|
+
if (got?.ok && got.data?.success && got.data.state === "active") usage = await measureUsage(got.data);
|
|
2538
|
+
} catch { usage = null; }
|
|
2539
|
+
const r = await rpc.call("touch_stack_lease", { lease_id: leaseId, ...(usage ? { usage } : {}) });
|
|
2540
|
+
if (!r.ok) return emitResult(buildReceipt({ command: "touch", outcome: "error", lease_id: leaseId, error: r.error }), opts, r.auth ? EXIT.AUTH : EXIT.BACKEND);
|
|
2541
|
+
if (isPreemptedAnswer(r.data)) {
|
|
2542
|
+
return emitResult(buildReceipt({
|
|
2543
|
+
command: "touch", outcome: "preempted", code: "LEASE_PREEMPTED", lease_id: leaseId, volumes_kept: true,
|
|
2544
|
+
state: r.data.state ?? null, reap_reason: r.data.reap_reason ?? "preempted",
|
|
2545
|
+
error: "the lease was preempted (its idle capacity went to a queued lease); the stack was stopped with its volumes kept — request a new lease",
|
|
2546
|
+
}), opts, EXIT.LEASE_FAILED);
|
|
2547
|
+
}
|
|
2548
|
+
if (!r.data.success) return emitResult(buildReceipt({ command: "touch", outcome: "error", lease_id: leaseId, code: r.data.code, error: r.data.code }), opts, EXIT.BACKEND);
|
|
2549
|
+
return emitResult(buildReceipt({
|
|
2550
|
+
command: "touch", outcome: "touched", lease_id: leaseId, last_used_at: r.data.last_used_at,
|
|
2551
|
+
...(r.data.idle !== undefined ? { idle: r.data.idle } : {}),
|
|
2552
|
+
...(r.data.idle_since !== undefined ? { idle_since: r.data.idle_since } : {}),
|
|
2553
|
+
usage_recorded: r.data.usage_recorded === true,
|
|
2554
|
+
}), opts, EXIT.OK);
|
|
2466
2555
|
}
|
|
2467
2556
|
|
|
2468
2557
|
export async function cmdDone(leaseId, opts, {
|
|
@@ -2607,9 +2696,21 @@ export async function cmdDone(leaseId, opts, {
|
|
|
2607
2696
|
}
|
|
2608
2697
|
}
|
|
2609
2698
|
|
|
2610
|
-
|
|
2699
|
+
// BOT-1803: the default `done` idles an ACTIVE lease (stack kept running for the next
|
|
2700
|
+
// batch on this slot). `--reap` is today's eager release, and `--local-exec` implies it:
|
|
2701
|
+
// its self-teardown below (and every hermetic lane / Supply Guard stop) needs the reap.
|
|
2702
|
+
const idleRelease = !opts.reap && !opts.localExec;
|
|
2703
|
+
const r = await call("release_stack_lease", { lease_id: leaseId, disposition: opts.disposition, ...(idleRelease ? { idle: true } : {}) });
|
|
2611
2704
|
if (!r.ok) return emitResult(buildReceipt({ command: "done", outcome: "error", lease_id: leaseId, error: r.error }), opts, r.auth ? EXIT.AUTH : EXIT.BACKEND);
|
|
2612
2705
|
if (!r.data.success) return emitResult(buildReceipt({ command: "done", outcome: "error", lease_id: leaseId, code: r.data.code, error: r.data.code }), opts, EXIT.BACKEND);
|
|
2706
|
+
if (idleRelease && r.data.idle === true) {
|
|
2707
|
+
return emitResult(buildReceipt({
|
|
2708
|
+
command: "done", outcome: "idle", lease_id: leaseId, state: r.data.state ?? "active",
|
|
2709
|
+
idle: true, idle_since: r.data.idle_since ?? null, idle_reason: r.data.idle_reason ?? null,
|
|
2710
|
+
...(r.data.noop === true ? { noop: true } : {}),
|
|
2711
|
+
}), opts, EXIT.OK);
|
|
2712
|
+
}
|
|
2713
|
+
// Any other answer (the lease was not active) is exactly today's release.
|
|
2613
2714
|
let state = r.data.state;
|
|
2614
2715
|
// BOT-1887 (Codex #972 P1): a lease that is ALREADY reaping (an idle reap that
|
|
2615
2716
|
// established `stop`, or a retried `destroy`) keeps its in-flight teardown contract —
|
|
@@ -2686,6 +2787,8 @@ export async function cmdDone(leaseId, opts, {
|
|
|
2686
2787
|
}
|
|
2687
2788
|
|
|
2688
2789
|
const SIGNAL_EXIT = Object.freeze({ SIGINT: 130, SIGTERM: 143, SIGHUP: 129 });
|
|
2790
|
+
/** BOT-1803: request answers meaning "this slot's idle lease is being reaped; re-request once it is". */
|
|
2791
|
+
const IDLE_REPROVISION_REASONS = Object.freeze({ IDLE_LEASE_REPROVISIONING: "digest_mismatch", LEASE_PREEMPTED: "preempted" });
|
|
2689
2792
|
|
|
2690
2793
|
function safeConnectionPath(requested, leaseId) {
|
|
2691
2794
|
if (requested) {
|
|
@@ -2823,10 +2926,12 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
|
|
|
2823
2926
|
// BOT-2099 (Codex #1057 R4): a platform blip recorded by ANY lease wait stays on EVERY
|
|
2824
2927
|
// terminal result (interrupted, unreadable, thrown, …), not only the paths that remember it.
|
|
2825
2928
|
// BOT-2142: where the lease ran (and how it was reached) is on every terminal result too.
|
|
2826
|
-
|
|
2929
|
+
// BOT-1803: whether this run re-adopted / re-provisioned an idle lease, and its stack digest.
|
|
2930
|
+
const incidentBox = { incident: null, execution: null, idleLease: null };
|
|
2827
2931
|
const result = await runStackLifecycleOnce(opts, childArgv, adapters, incidentBox);
|
|
2828
2932
|
const withIncident = incidentBox.incident && !result?.incident ? { ...result, incident: incidentBox.incident } : result;
|
|
2829
|
-
|
|
2933
|
+
const withExecution = incidentBox.execution && !withIncident?.execution ? { ...withIncident, execution: incidentBox.execution } : withIncident;
|
|
2934
|
+
return incidentBox.idleLease && !withExecution?.idleLease ? { ...withExecution, idleLease: incidentBox.idleLease } : withExecution;
|
|
2830
2935
|
}
|
|
2831
2936
|
|
|
2832
2937
|
async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
@@ -2851,9 +2956,12 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
2851
2956
|
const api = adapters.api ?? {
|
|
2852
2957
|
request: (args) => rpc.call("request_stack_lease", args),
|
|
2853
2958
|
get: (leaseId) => rpc.call("get_stack_lease", { lease_id: leaseId }),
|
|
2854
|
-
|
|
2855
|
-
|
|
2959
|
+
// BOT-1803: a heartbeat carries the measured client use when there is one.
|
|
2960
|
+
touch: (leaseId, usage) => rpc.call("touch_stack_lease", { lease_id: leaseId, ...(usage ? { usage } : {}) }),
|
|
2961
|
+
// BOT-1803: `{ idle: true }` idles an ACTIVE lease instead of reaping it.
|
|
2962
|
+
release: (leaseId, signal, { idle = false } = {}) => rpc.call("release_stack_lease", { lease_id: leaseId, disposition: "destroy", ...(idle ? { idle: true } : {}) }, { signal }),
|
|
2856
2963
|
};
|
|
2964
|
+
const measureUsage = adapters.measureUsage ?? measureStackUsage;
|
|
2857
2965
|
const machineUuidFn = adapters.machineUuidFn ?? machineUuid;
|
|
2858
2966
|
const wait = adapters.wait ?? ((leaseId, done, failed, options) => waitForLease(leaseId, done, failed, options));
|
|
2859
2967
|
// BOT-2099: the queue park and the provision wait survive auth/availability blips.
|
|
@@ -2918,6 +3026,14 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
2918
3026
|
let tunnelLost = null;
|
|
2919
3027
|
const placement = { mode: null, hostKey: null, requesterHost: null, transport: "none", forwards: null, bundle: null, placementReason: null, considered: null };
|
|
2920
3028
|
incidentBox.execution = placement;
|
|
3029
|
+
// BOT-1803: idle-lease bookkeeping for the receipt.
|
|
3030
|
+
const idleLease = { reused: null, reprovisioned: null, previousLeaseId: null, stackDigest: null };
|
|
3031
|
+
incidentBox.idleLease = idleLease;
|
|
3032
|
+
// BOT-1803: set when this run's port block must stay reserved although nothing of ours
|
|
3033
|
+
// was provisioned (the slot's idle/live stack in THIS directory is still bound to it).
|
|
3034
|
+
let retainBlock = false;
|
|
3035
|
+
// BOT-1803: a heartbeat touch found the lease preempted under the running child.
|
|
3036
|
+
let preempted = null;
|
|
2921
3037
|
const closeTunnel = async () => {
|
|
2922
3038
|
if (!tunnel) return;
|
|
2923
3039
|
const open = tunnel;
|
|
@@ -2925,7 +3041,10 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
2925
3041
|
try { await open.close(); } catch { /* best effort: the process is gone either way */ }
|
|
2926
3042
|
};
|
|
2927
3043
|
|
|
2928
|
-
|
|
3044
|
+
// BOT-1803: `idle` (only the normal end of a run) asks the server to idle the ACTIVE lease
|
|
3045
|
+
// — no reap job, no signed-reap wait. Any answer that is not an idle one (the lease was
|
|
3046
|
+
// not active) continues into today's signed-reap wait unchanged.
|
|
3047
|
+
const requestCleanup = ({ idle = false } = {}) => {
|
|
2929
3048
|
// A signal can arrive while the lease-request RPC is in flight. There is
|
|
2930
3049
|
// nothing to release yet, so do not cache this no-op; once the RPC returns
|
|
2931
3050
|
// the interruption branch must perform the real signed cleanup.
|
|
@@ -2937,13 +3056,16 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
2937
3056
|
const cleanupTimer = clock.setTimeout(() => cleanupAbort.abort("reap-timeout"), opts.reapTimeout * 1000);
|
|
2938
3057
|
let release;
|
|
2939
3058
|
try {
|
|
2940
|
-
release = await api.release(leaseId, cleanupAbort.signal);
|
|
3059
|
+
release = idle ? await api.release(leaseId, cleanupAbort.signal, { idle: true }) : await api.release(leaseId, cleanupAbort.signal);
|
|
2941
3060
|
} finally {
|
|
2942
3061
|
clock.clearTimeout(cleanupTimer);
|
|
2943
3062
|
}
|
|
2944
3063
|
if (!release?.ok || !release?.data?.success) {
|
|
2945
3064
|
return cleanupResult = { ok: false, error: release?.error || release?.data?.code || "release failed" };
|
|
2946
3065
|
}
|
|
3066
|
+
if (idle && release.data.idle === true) {
|
|
3067
|
+
return cleanupResult = { ok: true, idle: true, state: release.data.state ?? "active", idle_since: release.data.idle_since ?? null, idle_reason: release.data.idle_reason ?? null };
|
|
3068
|
+
}
|
|
2947
3069
|
if (release.data.state === "reaped") return cleanupResult = { ok: true, state: "reaped" };
|
|
2948
3070
|
// BOT-1822 (Codex P2 round 2, extended round 3): `reap_failed` is synthetic
|
|
2949
3071
|
// (never a real DB state), so a Helper's failed receipt landing in the split
|
|
@@ -3062,18 +3184,34 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3062
3184
|
let executionRequest = opts.execution ?? "auto";
|
|
3063
3185
|
let bundle = null;
|
|
3064
3186
|
const bundleInclude = opts.bundleInclude ?? [];
|
|
3187
|
+
const buildRunBundle = ({ digestOnly = false } = {}) => {
|
|
3188
|
+
// BOT-2162: ship the worktree's BOT-2107 readiness declaration with the stack, so the
|
|
3189
|
+
// serving Helper runs the canonical bind-RPC probe for a remote lease too.
|
|
3190
|
+
const declaration = join(execution.worktreeRoot, ".botbuddy", "config.json");
|
|
3191
|
+
let hasDeclaration = false;
|
|
3192
|
+
try { hasDeclaration = existsSync(declaration) && lstatSync(declaration).isFile(); } catch { hasDeclaration = false; }
|
|
3193
|
+
return buildBundle(stackDirFor(execution), {
|
|
3194
|
+
// A digest that ships nothing has no upload cap (the archive is only hashed).
|
|
3195
|
+
maxBytes: digestOnly ? Number.MAX_SAFE_INTEGER : (opts.bundleMaxBytes ?? DEFAULT_BUNDLE_MAX_BYTES),
|
|
3196
|
+
...(digestOnly ? { rawLimitBytes: Number.MAX_SAFE_INTEGER } : {}),
|
|
3197
|
+
...(hasDeclaration ? { extraFiles: [{ rel: ".botbuddy/config.json", abs: declaration }] } : {}),
|
|
3198
|
+
...(bundleInclude.length ? { include: bundleInclude } : {}),
|
|
3199
|
+
});
|
|
3200
|
+
};
|
|
3201
|
+
// BOT-1803: the stack digest — the deterministic bundle sha256 of the materialised stack —
|
|
3202
|
+
// goes on EVERY request so the control plane re-adopts this slot's idle lease only when
|
|
3203
|
+
// its content is unchanged. --local-only ships nothing, so there it is computed for the
|
|
3204
|
+
// digest alone (no upload cap). Fail closed (Codex #1160 P1): every run request says
|
|
3205
|
+
// `require_stack_digest`, so when no digest could be computed the control plane re-provisions
|
|
3206
|
+
// an idle lease instead of handing back a stack whose content it cannot prove current.
|
|
3207
|
+
let stackDigest = null;
|
|
3208
|
+
let digestError = null;
|
|
3209
|
+
if (executionRequest === "local") {
|
|
3210
|
+
try { stackDigest = buildRunBundle({ digestOnly: true })?.sha256 ?? null; } catch (error) { digestError = error; }
|
|
3211
|
+
}
|
|
3065
3212
|
if (executionRequest !== "local") {
|
|
3066
3213
|
try {
|
|
3067
|
-
|
|
3068
|
-
// serving Helper runs the canonical bind-RPC probe for a remote lease too.
|
|
3069
|
-
const declaration = join(execution.worktreeRoot, ".botbuddy", "config.json");
|
|
3070
|
-
let hasDeclaration = false;
|
|
3071
|
-
try { hasDeclaration = existsSync(declaration) && lstatSync(declaration).isFile(); } catch { hasDeclaration = false; }
|
|
3072
|
-
bundle = buildBundle(stackDirFor(execution), {
|
|
3073
|
-
maxBytes: opts.bundleMaxBytes ?? DEFAULT_BUNDLE_MAX_BYTES,
|
|
3074
|
-
...(hasDeclaration ? { extraFiles: [{ rel: ".botbuddy/config.json", abs: declaration }] } : {}),
|
|
3075
|
-
...(bundleInclude.length ? { include: bundleInclude } : {}),
|
|
3076
|
-
});
|
|
3214
|
+
bundle = buildRunBundle();
|
|
3077
3215
|
for (const entry of bundle.included ?? []) {
|
|
3078
3216
|
stderrLine(`stack: the bundle includes ${entry.path} (sha256 ${entry.sha256.slice(0, 12)}…; keys: ${entry.keys.join(", ") || "none"})`);
|
|
3079
3217
|
}
|
|
@@ -3092,6 +3230,17 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3092
3230
|
stderrLine(`${yellow("⚠")} stack: could not build the remote-lease bundle (${message}); requesting a co-located lease (execution local) instead.`);
|
|
3093
3231
|
}
|
|
3094
3232
|
}
|
|
3233
|
+
if (!stackDigest && !bundle?.sha256 && !digestError) {
|
|
3234
|
+
// `auto` fell back to local above because its (capped) bundle failed: hash it uncapped.
|
|
3235
|
+
try { stackDigest = buildRunBundle({ digestOnly: true })?.sha256 ?? null; } catch (error) { digestError = error; }
|
|
3236
|
+
}
|
|
3237
|
+
stackDigest = stackDigest ?? bundle?.sha256 ?? null;
|
|
3238
|
+
if (typeof stackDigest !== "string" || !/^[0-9a-f]{64}$/.test(stackDigest)) stackDigest = null;
|
|
3239
|
+
if (!stackDigest) {
|
|
3240
|
+
stderrLine(`${yellow("⚠")} stack: could not compute the stack digest (${String(digestError?.message ?? digestError ?? "no digest")}); ` +
|
|
3241
|
+
"an idle stack on this slot will be re-provisioned rather than reused.");
|
|
3242
|
+
}
|
|
3243
|
+
idleLease.stackDigest = stackDigest;
|
|
3095
3244
|
const leaseArgs = {
|
|
3096
3245
|
slot, host_key: opts.host || undefined, repo: opts.repo, ticket_id: opts.ticket,
|
|
3097
3246
|
ticket_url: opts.ticketUrl || undefined, pr_id: opts.prId || undefined, pr_url: opts.prUrl || undefined,
|
|
@@ -3100,6 +3249,9 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3100
3249
|
machine_uuid: hardwareUuid,
|
|
3101
3250
|
execution: executionRequest,
|
|
3102
3251
|
...(bundle ? { bundle_sha256: bundle.sha256 } : {}),
|
|
3252
|
+
...(stackDigest ? { stack_digest: stackDigest } : {}),
|
|
3253
|
+
// BOT-1803 (Codex #1160 P1): a run never re-adopts an idle stack it cannot prove current.
|
|
3254
|
+
require_stack_digest: true,
|
|
3103
3255
|
// BOT-2269: what the bundle carries beyond the defaults (paths, sha256, key NAMES).
|
|
3104
3256
|
...(bundle?.included?.length ? { bundle_included: bundle.included } : {}),
|
|
3105
3257
|
...(opts.callbackPort != null ? { callback_port: opts.callbackPort } : {}),
|
|
@@ -3108,61 +3260,111 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3108
3260
|
// FIRST auth rejection among any of request/get/touch/release.
|
|
3109
3261
|
// BOT-2024: a stale-capacity refusal prints phase-aware guidance; with the opt-in
|
|
3110
3262
|
// --wait-capacity a TRANSIENT one parks on the server's capacity wait and re-admits once.
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3263
|
+
// BOT-1803: an idle lease on this slot that the control plane is re-provisioning (its
|
|
3264
|
+
// stack digest changed) or that lost a race to a preemption answers with the OLD lease
|
|
3265
|
+
// id: wait for it to reach `reaped`, then send the SAME request once more (it then
|
|
3266
|
+
// provisions a fresh lease for the slot).
|
|
3267
|
+
let request;
|
|
3268
|
+
for (let attempt = 0; ; attempt++) {
|
|
3269
|
+
activeWaitAbort = new AbortController();
|
|
3270
|
+
const admission = await requestWithCapacityWait(() => api.request(leaseArgs), opts, {
|
|
3271
|
+
waitCapacity: adapters.waitCapacity ?? defaultWaitCapacity, log: stderrLine, signal: activeWaitAbort.signal,
|
|
3272
|
+
});
|
|
3273
|
+
activeWaitAbort = null;
|
|
3274
|
+
request = admission.req;
|
|
3275
|
+
|
|
3276
|
+
// Interrupted DURING the capacity park: the refused request created no lease, so
|
|
3277
|
+
// there is nothing to release. (A signal during a request that DID admit falls
|
|
3278
|
+
// through to the existing post-lease interruption branch, which cleans up.)
|
|
3279
|
+
if (admission.interrupted) {
|
|
3280
|
+
return { exitCode: SIGNAL_EXIT[receivedSignal] ?? EXIT.INTERNAL, outcome: "interrupted", code: request?.data?.code, ...capacityStaleReceiptFields(request?.data) };
|
|
3281
|
+
}
|
|
3282
|
+
if (admission.capacityTimeout) {
|
|
3283
|
+
return {
|
|
3284
|
+
exitCode: EXIT.TIMEOUT, outcome: "timeout", code: request.data.code, ...capacityStaleReceiptFields(request.data),
|
|
3285
|
+
wait_exit_code: EXIT.TIMEOUT, wait_receipt: admission.waitReceipt,
|
|
3286
|
+
error: `capacity did not recover within --wait-capacity ${opts.waitCapacity}s (${request.data.code})`,
|
|
3287
|
+
};
|
|
3288
|
+
}
|
|
3289
|
+
if (admission.waitFailed) {
|
|
3290
|
+
return {
|
|
3291
|
+
exitCode: admission.waitFailed.exitCode, outcome: "error", code: request.data.code, ...capacityStaleReceiptFields(request.data),
|
|
3292
|
+
wait_exit_code: admission.waitFailed.exitCode, wait_receipt: admission.waitFailed.receipt,
|
|
3293
|
+
error: `--wait-capacity: bb wait exited ${admission.waitFailed.exitCode} (${admission.waitFailed.detail})`,
|
|
3294
|
+
};
|
|
3295
|
+
}
|
|
3296
|
+
if (!request?.ok) {
|
|
3297
|
+
// Codex #928 R23 (P2): `status: null` (callToolJson's fetch-threw branch) means
|
|
3298
|
+
// NO response was ever received — the request may have reached the backend and
|
|
3299
|
+
// created a lease we simply never heard back about, so a Helper could already be
|
|
3300
|
+
// provisioning it. Any OTHER failure carries a real HTTP status: the backend was
|
|
3301
|
+
// reached and definitely refused (or errored) before creating anything of ours.
|
|
3302
|
+
// Only the definite case is safe to treat as unprovisioned and free the port
|
|
3303
|
+
// claim immediately below; an ambiguous transport failure retains it instead — it
|
|
3304
|
+
// still expires on its own TTL if nothing was actually provisioned.
|
|
3305
|
+
if (request?.status == null) leaseProvisioned = true;
|
|
3306
|
+
return { exitCode: request?.auth ? EXIT.AUTH : EXIT.BACKEND, outcome: "error", error: request?.error || "lease request failed" };
|
|
3307
|
+
}
|
|
3308
|
+
const why = attempt === 0 && request.data?.success === false && request.data?.lease_id ? IDLE_REPROVISION_REASONS[request.data.code] : null;
|
|
3309
|
+
if (!why) break;
|
|
3310
|
+
const previous = request.data.lease_id;
|
|
3311
|
+
idleLease.previousLeaseId = previous;
|
|
3312
|
+
stderrLine(`stack: this slot's idle lease ${previous} is being ${why === "preempted" ? "preempted" : "re-provisioned (the stack content changed)"} — waiting for it to reach reaped, then requesting again`);
|
|
3313
|
+
activeWaitAbort = new AbortController();
|
|
3314
|
+
const gone = await resilientWait(previous, (st) => st === "reaped", (st) => st === "reap_failed", { timeoutSec: opts.reapTimeout, auth: rpc.auth, signal: activeWaitAbort.signal });
|
|
3315
|
+
activeWaitAbort = null;
|
|
3316
|
+
if (gone?.interrupted || receivedSignal) {
|
|
3317
|
+
return { exitCode: SIGNAL_EXIT[receivedSignal] ?? EXIT.INTERNAL, outcome: "interrupted", code: request.data.code };
|
|
3318
|
+
}
|
|
3319
|
+
let reaped = Boolean(gone?.woke) && gone.state === "reaped";
|
|
3320
|
+
if (!reaped && !gone?.failed) {
|
|
3321
|
+
// The same missed-wake recheck as the provision and reap waits (BOT-1822/1827).
|
|
3322
|
+
const read = await readLeaseQuietly(previous);
|
|
3323
|
+
reaped = Boolean(read?.ok && read.data?.success && read.data.state === "reaped");
|
|
3324
|
+
}
|
|
3325
|
+
if (!reaped) {
|
|
3326
|
+
// Its containers may still hold this stack's ports.
|
|
3327
|
+
retainBlock = true;
|
|
3328
|
+
return {
|
|
3329
|
+
exitCode: EXIT.LEASE_FAILED, outcome: "error", code: request.data.code,
|
|
3330
|
+
error: `this slot's idle lease ${previous} did not reach reaped ${gone?.failed ? "(its reap failed)" : `within ${opts.reapTimeout}s`} ` +
|
|
3331
|
+
`after ${why}; no new lease was requested — check \`bb stack status ${previous}\` and re-run once it is reaped`,
|
|
3332
|
+
...(parkIncident ? { incident: parkIncident } : {}),
|
|
3333
|
+
};
|
|
3334
|
+
}
|
|
3335
|
+
idleLease.reprovisioned = why;
|
|
3148
3336
|
}
|
|
3149
3337
|
if (!request.data?.success) {
|
|
3150
3338
|
const hint = worktreeRegistrationHint(request.data?.code, rpc.auth)
|
|
3151
3339
|
?? stackPathRootRefusalNotice(request.data?.code, request.data?.message)
|
|
3152
3340
|
?? remoteRefusalNotice(request.data?.code);
|
|
3153
3341
|
if (hint) stderrLine(hint);
|
|
3342
|
+
// BOT-1803: the slot's idle lease stays idle (another mode/host); keep its block.
|
|
3343
|
+
if (request.data?.code === "STACK_EXECUTION_MODE_MISMATCH" && request.data?.lease_id) retainBlock = true;
|
|
3154
3344
|
return {
|
|
3155
3345
|
exitCode: leaseRefusalExit(request.data?.code), outcome: "error", code: request.data?.code,
|
|
3156
3346
|
...capacityStaleReceiptFields(request.data), error: leaseRefusalError(request.data),
|
|
3157
3347
|
};
|
|
3158
3348
|
}
|
|
3159
3349
|
leaseId = request.data.lease_id;
|
|
3160
|
-
|
|
3350
|
+
// BOT-1803: `reused_idle` — the slot's IDLE lease was re-adopted: it is active with its
|
|
3351
|
+
// stored connection and nothing is provisioned, so the run goes straight on to the
|
|
3352
|
+
// post-activation flow. Any other reuse is somebody's LIVE batch, refused as before.
|
|
3353
|
+
const reusedIdle = request.data.reused === true && request.data.reused_idle === true;
|
|
3354
|
+
if (request.data.reused && !reusedIdle) {
|
|
3355
|
+
// With a stable identity a live lease in THIS stack directory is bound to our block.
|
|
3356
|
+
if (request.data.stack_path != null && request.data.stack_path === execution.stackPath &&
|
|
3357
|
+
(request.data.worktree_root == null || request.data.worktree_root === execution.worktreeRoot)) retainBlock = true;
|
|
3161
3358
|
return { exitCode: EXIT.BACKEND, outcome: "error", leaseId, error: "a live lease already exists for this agent and stack slot; wait for that batch to finish instead of sharing its stack" };
|
|
3162
3359
|
}
|
|
3163
3360
|
|
|
3164
3361
|
leaseProvisioned = true; // from here a Helper may act on this lease
|
|
3165
3362
|
let state = request.data.state;
|
|
3363
|
+
if (reusedIdle) {
|
|
3364
|
+
idleLease.reused = "idle";
|
|
3365
|
+
state = "active";
|
|
3366
|
+
stderrLine(`stack: re-adopted idle lease ${leaseId}${request.data.host_key ? ` on ${request.data.host_key}` : ""} — the stack is already running; nothing to provision`);
|
|
3367
|
+
}
|
|
3166
3368
|
// BOT-2142: where the server placed the lease. A server that predates BOT-2145
|
|
3167
3369
|
// sends no execution_mode, and its lease is co-located.
|
|
3168
3370
|
const remoteLease = request.data.execution_mode === "remote";
|
|
@@ -3192,6 +3394,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3192
3394
|
return { exitCode: EXIT.BACKEND, outcome: "error", code: error.code ?? "REMOTE_UNREACHABLE", leaseId, error: error.message, cleanup };
|
|
3193
3395
|
}
|
|
3194
3396
|
}
|
|
3397
|
+
// BOT-1803: a re-adopted idle remote stack is already running there; nothing to upload.
|
|
3398
|
+
if (!reusedIdle) {
|
|
3195
3399
|
if (!bundle || !request.data.bundle_upload_url) {
|
|
3196
3400
|
const cleanup = await requestCleanup();
|
|
3197
3401
|
return {
|
|
@@ -3217,6 +3421,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3217
3421
|
placement.bundle = { sha256: bundle.sha256, bytes: bundle.size };
|
|
3218
3422
|
if (bundle.included?.length) placement.bundleIncluded = bundle.included;
|
|
3219
3423
|
stderrLine(`stack: lease ${leaseId} is remote on ${placement.hostKey ?? "?"} — uploaded the ${bundle.size}-byte stack bundle (sha256 ${bundle.sha256.slice(0, 12)}…)`);
|
|
3424
|
+
}
|
|
3220
3425
|
} else if (bundle && state === "queued" && request.data.bundle_upload_url) {
|
|
3221
3426
|
// BOT-2207: the lease queued HERE but may still move: every minute the control plane
|
|
3222
3427
|
// re-offers it to the owner's other hosts, and it can only move with its bundle in
|
|
@@ -3389,9 +3594,23 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3389
3594
|
// last activity, not its creation, so a batch that outlives the default window
|
|
3390
3595
|
// and then loses its owner process is never collected while it is still running.
|
|
3391
3596
|
keepAlive();
|
|
3597
|
+
// BOT-1803 (AC-12): a co-located lease's heartbeat carries the measured client use; a
|
|
3598
|
+
// remote one (or an unreadable measurement) sends no usage at all.
|
|
3599
|
+
const usageLease = { ...current.data, execution_mode: current.data.execution_mode ?? (servedRemote ? "remote" : "local") };
|
|
3392
3600
|
heartbeat = clock.setInterval(async () => {
|
|
3393
3601
|
if (fencing || childResult) return;
|
|
3394
|
-
|
|
3602
|
+
let usage = null;
|
|
3603
|
+
if (!servedRemote) { try { usage = await measureUsage(usageLease); } catch { usage = null; } }
|
|
3604
|
+
if (fencing || childResult) return;
|
|
3605
|
+
const touched = usage ? await api.touch(leaseId, usage) : await api.touch(leaseId);
|
|
3606
|
+
if (touched?.ok && isPreemptedAnswer(touched.data)) {
|
|
3607
|
+
// BOT-1803 (AC-11): the control plane reclaimed this lease's capacity. Stop the
|
|
3608
|
+
// child and end typed — never the generic lost-lease wording.
|
|
3609
|
+
preempted = { state: touched.data.state ?? "reaping", reap_reason: touched.data.reap_reason ?? "preempted" };
|
|
3610
|
+
fencing = true;
|
|
3611
|
+
stopChild("SIGTERM", EXIT.LEASE_FAILED);
|
|
3612
|
+
return;
|
|
3613
|
+
}
|
|
3395
3614
|
if (!touched?.ok || !touched?.data?.success) {
|
|
3396
3615
|
fencing = true;
|
|
3397
3616
|
stopChild("SIGTERM", EXIT.LEASE_FAILED);
|
|
@@ -3407,10 +3626,29 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3407
3626
|
if (retention) { clock.clearInterval(retention); retention = null; }
|
|
3408
3627
|
if (hardTimer) clock.clearTimeout(hardTimer);
|
|
3409
3628
|
await closeTunnel();
|
|
3410
|
-
|
|
3629
|
+
// BOT-1803: a run that ended on its own (the child exited, success or not) leaves its
|
|
3630
|
+
// lease IDLE for the next run on this slot — no reap wait. `--reap`, a signal, the
|
|
3631
|
+
// hard TTL, a fenced/preempted heartbeat and a lost tunnel keep the eager signed reap.
|
|
3632
|
+
const idleOnEnd = !opts.reap && !receivedSignal && !fencing && !tunnelLost && forcedStopCode == null;
|
|
3633
|
+
const cleanup = await requestCleanup({ idle: idleOnEnd });
|
|
3634
|
+
if (cleanup?.idle) {
|
|
3635
|
+
// The stack is still running: keep its directory and host port block reserved well
|
|
3636
|
+
// past any idle TTL so no other worktree takes them; the next run renews the claim.
|
|
3637
|
+
try { refreshRetention(stackDirFor(execution), { idleTtlSecs: LEASED_STACK_IDLE_RETENTION_SECS, idleRetained: true }); } catch { /* advisory */ }
|
|
3638
|
+
try { execution.renew?.({ idleTtlSecs: LEASED_STACK_IDLE_RETENTION_SECS, idleRetained: true }); } catch { /* advisory */ }
|
|
3639
|
+
stderrLine(`stack: lease ${leaseId} is idle${cleanup.idle_since ? ` since ${cleanup.idle_since}` : ""} — its stack keeps running for the next run on this slot (\`bb stack done ${leaseId} --reap\` tears it down)`);
|
|
3640
|
+
}
|
|
3411
3641
|
if (tunnelLost && !receivedSignal) {
|
|
3412
3642
|
return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: childResult.code, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) });
|
|
3413
3643
|
}
|
|
3644
|
+
if (preempted && !receivedSignal) {
|
|
3645
|
+
return {
|
|
3646
|
+
exitCode: EXIT.LEASE_FAILED, outcome: "failed", code: "LEASE_PREEMPTED", volumesKept: true, leaseId, childExitCode: childResult.code, cleanup, fenced: true,
|
|
3647
|
+
error: `lease ${leaseId} was preempted while the child ran (its capacity went to a queued lease); the child was stopped and the stack ` +
|
|
3648
|
+
"stopped with its volumes kept — re-run the batch to get a new lease",
|
|
3649
|
+
...(parkIncident ? { incident: parkIncident } : {}),
|
|
3650
|
+
};
|
|
3651
|
+
}
|
|
3414
3652
|
const exitCode = receivedSignal ? SIGNAL_EXIT[receivedSignal] : forcedStopCode ?? (childResult.code !== 0 ? childResult.code : !cleanup.ok ? EXIT.CLEANUP_FAILED : 0);
|
|
3415
3653
|
return { exitCode, outcome: exitCode === 0 ? "completed" : "failed", leaseId, childExitCode: childResult.code, cleanup, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) };
|
|
3416
3654
|
} catch (error) {
|
|
@@ -3436,8 +3674,10 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3436
3674
|
// A request the backend refused, or one that only re-attached to somebody
|
|
3437
3675
|
// else's live lease, provisioned nothing of ours — its block must go back at
|
|
3438
3676
|
// once, or repeated refusals would exhaust the host (Codex #928 R21 P2).
|
|
3439
|
-
|
|
3440
|
-
|
|
3677
|
+
// BOT-1803: an IDLE lease (cleanupResult.state "active") is not a reap — its stack still
|
|
3678
|
+
// runs, so its block stays reserved; so does one this run must not free (retainBlock).
|
|
3679
|
+
const provenReap = !retainBlock && (!leaseProvisioned
|
|
3680
|
+
|| (cleanupResult?.ok === true && (cleanupResult.state === "reaped" || cleanupResult.state == null)));
|
|
3441
3681
|
execution?.release?.({ reaped: provenReap });
|
|
3442
3682
|
for (const [signal, handler] of signalHandlers) signals.off?.(signal, handler);
|
|
3443
3683
|
}
|
|
@@ -3452,7 +3692,14 @@ export function runReceiptFields(result) {
|
|
|
3452
3692
|
// BOT-2024: a stale-capacity refusal carries its phase + armable wait on the receipt.
|
|
3453
3693
|
...capacityStaleReceiptFields(result),
|
|
3454
3694
|
...(result.wait_exit_code != null ? { wait_exit_code: result.wait_exit_code, wait_receipt: result.wait_receipt ?? null } : {}),
|
|
3455
|
-
child_exit_code: result.childExitCode ?? null,
|
|
3695
|
+
child_exit_code: result.childExitCode ?? null,
|
|
3696
|
+
// BOT-1803: an idle release is `cleanup: { idle: true, idle_since }` (nothing was reaped).
|
|
3697
|
+
cleanup: result.cleanup?.idle ? { idle: true, idle_since: result.cleanup.idle_since ?? null } : result.cleanup?.ok ?? null,
|
|
3698
|
+
...(result.volumesKept ? { volumes_kept: true } : {}),
|
|
3699
|
+
...(result.idleLease?.reused ? { reused: result.idleLease.reused } : {}),
|
|
3700
|
+
...(result.idleLease?.reprovisioned ? { reprovisioned: result.idleLease.reprovisioned } : {}),
|
|
3701
|
+
...(result.idleLease?.previousLeaseId ? { previous_lease_id: result.idleLease.previousLeaseId } : {}),
|
|
3702
|
+
...(result.idleLease?.stackDigest ? { stack_digest: result.idleLease.stackDigest } : {}),
|
|
3456
3703
|
...(result.parkReason ? { park_reason: result.parkReason } : {}),
|
|
3457
3704
|
// BOT-2142: where the lease ran and how the child reached it.
|
|
3458
3705
|
execution_mode: result.execution?.mode ?? null,
|