@botbuddy/cli 1.53.1 → 1.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/stack-isolation.mjs +113 -27
- package/src/stack-mount-health.mjs +94 -0
- package/src/stack-usage.mjs +22 -1
- package/src/stack.mjs +112 -16
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@botbuddy/cli",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "BotBuddy
|
|
3
|
+
"version": "1.55.0",
|
|
4
|
+
"description": "BotBuddy \u2014 Swarm coordination CLI for multi-agent workflows",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
7
|
"bb": "./bin/botbuddy.mjs",
|
package/src/stack-isolation.mjs
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
// Supply Guard's `scripts/local-supabase-isolated.sh`, ported here because the
|
|
21
21
|
// published CLI cannot import repo scripts and stays Node-stdlib only.
|
|
22
22
|
import { createHash, randomUUID } from "node:crypto";
|
|
23
|
-
import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
23
|
+
import { copyFileSync, cpSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, readlinkSync, realpathSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
24
24
|
import { homedir, hostname } from "node:os";
|
|
25
25
|
import { basename, dirname, isAbsolute, join, relative, sep } from "node:path";
|
|
26
26
|
import { lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
|
|
@@ -1108,7 +1108,7 @@ function selectPortBase(preferred, taken) {
|
|
|
1108
1108
|
* (BOT-1798, Codex #928 R4 P1). The refresh prunes and copies BEFORE
|
|
1109
1109
|
* `resolveStackPath` could notice an escape, so a stale or hostile symlink at
|
|
1110
1110
|
* `.botbuddy/`, `.botbuddy/stacks/`, `<project_id>/` or its `supabase/` would
|
|
1111
|
-
* make
|
|
1111
|
+
* make the refresh's deletions remove data OUTSIDE the worktree. Every existing component of
|
|
1112
1112
|
* the path must therefore resolve to itself; the first one that does not is
|
|
1113
1113
|
* fatal, and a component that does not exist yet ends the walk (nothing below it
|
|
1114
1114
|
* can exist either). `worktreeRoot` is already canonical (realpath of the cwd).
|
|
@@ -1137,12 +1137,76 @@ function assertCanonicalTarget(worktreeRoot, relPath, realpath) {
|
|
|
1137
1137
|
* worktree cannot linger in the leased stack (and be applied by `supabase start`).
|
|
1138
1138
|
* The stack's own CLI scratch state (SKIP_ENTRIES) is left alone. A not-yet-created
|
|
1139
1139
|
* directory is simply nothing to prune. */
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1140
|
+
// BOT-2403 — the stack is refreshed IN PLACE. The edge runtime bind-mounts
|
|
1141
|
+
// `<stack>/supabase/functions` (a directory) and each BOT-1828 escaping import (single
|
|
1142
|
+
// files), and a bind mount pins the inode it resolved at container start. The old
|
|
1143
|
+
// prune-then-cpSync refresh removed and recreated every one of them on each run (also a
|
|
1144
|
+
// refused or queued one, which materializes before its lease request), so a live or idle
|
|
1145
|
+
// stack's runtime was left holding deleted directories: 0 functions, every function 503
|
|
1146
|
+
// BOOT_ERROR after re-adoption. Here an existing directory is kept and recursed into, an
|
|
1147
|
+
// unchanged file is not touched, a changed file is rewritten in place (copyFileSync keeps
|
|
1148
|
+
// the inode; cpSync onto an existing file does not), and only entries the source no longer
|
|
1149
|
+
// has — or whose type changed — are removed (rsync --delete semantics, as before).
|
|
1150
|
+
|
|
1151
|
+
/** True when two regular files have identical bytes. */
|
|
1152
|
+
function sameFileContent(a, b, { read, lstat }) {
|
|
1153
|
+
try {
|
|
1154
|
+
if (lstat(a).size !== lstat(b).size) return false;
|
|
1155
|
+
return Buffer.compare(read(a), read(b)) === 0;
|
|
1156
|
+
} catch {
|
|
1157
|
+
return false;
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
/** Bring one source entry to `dest` in place; `copy` only ever creates what is missing. */
|
|
1162
|
+
function syncEntryInPlace(source, dest, ops) {
|
|
1163
|
+
const { lstat, readlink, remove, copy, copyFile } = ops;
|
|
1164
|
+
const src = lstat(source);
|
|
1165
|
+
let existing = null;
|
|
1166
|
+
try { existing = lstat(dest); } catch { /* missing */ }
|
|
1167
|
+
if (src.isDirectory()) {
|
|
1168
|
+
if (existing?.isDirectory()) return syncTreeInPlace(source, dest, ops);
|
|
1169
|
+
if (existing) remove(dest, { recursive: true, force: true });
|
|
1170
|
+
copy(source, dest, { recursive: true, force: true, verbatimSymlinks: true, filter: (path) => !SKIP_ENTRIES.has(basename(path)) });
|
|
1171
|
+
return;
|
|
1172
|
+
}
|
|
1173
|
+
if (src.isSymbolicLink()) {
|
|
1174
|
+
if (existing?.isSymbolicLink() && readlink(dest) === readlink(source)) return;
|
|
1175
|
+
if (existing) remove(dest, { recursive: true, force: true });
|
|
1176
|
+
copy(source, dest, { force: true, verbatimSymlinks: true });
|
|
1177
|
+
return;
|
|
1178
|
+
}
|
|
1179
|
+
if (existing?.isFile()) {
|
|
1180
|
+
if (!sameFileContent(source, dest, ops)) copyFile(source, dest);
|
|
1181
|
+
return;
|
|
1182
|
+
}
|
|
1183
|
+
if (existing) remove(dest, { recursive: true, force: true });
|
|
1184
|
+
copy(source, dest, { force: true });
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
/**
|
|
1188
|
+
* Make `dest` mirror `source` without replacing anything that already matches. `keep`
|
|
1189
|
+
* names top-level `dest` entries that are not the source's to manage (the Supabase CLI's
|
|
1190
|
+
* scratch state, the rewritten config.toml); `SKIP_ENTRIES` are never copied.
|
|
1191
|
+
*/
|
|
1192
|
+
function syncTreeInPlace(source, dest, ops, { keep = new Set() } = {}) {
|
|
1193
|
+
const { readdir, remove, mkdir } = ops;
|
|
1194
|
+
mkdir(dest, { recursive: true });
|
|
1195
|
+
const wanted = new Set(readdir(source).filter((entry) => !SKIP_ENTRIES.has(entry) && !keep.has(entry)));
|
|
1196
|
+
for (const entry of readdir(dest)) {
|
|
1197
|
+
if (keep.has(entry) || wanted.has(entry)) continue;
|
|
1198
|
+
remove(join(dest, entry), { recursive: true, force: true });
|
|
1199
|
+
}
|
|
1200
|
+
for (const entry of wanted) syncEntryInPlace(join(source, entry), join(dest, entry), ops);
|
|
1201
|
+
}
|
|
1202
|
+
|
|
1203
|
+
/** The inode of the stack's functions directory, or null when it has none. */
|
|
1204
|
+
function functionsInode(stackDir, lstat) {
|
|
1205
|
+
try {
|
|
1206
|
+
const st = lstat(join(stackDir, "supabase", "functions"));
|
|
1207
|
+
return st.isDirectory() ? Number(st.ino) : null;
|
|
1208
|
+
} catch {
|
|
1209
|
+
return null;
|
|
1146
1210
|
}
|
|
1147
1211
|
}
|
|
1148
1212
|
|
|
@@ -1247,7 +1311,7 @@ function listMatchingFiles(dir, { exts = [], names = [] }, { readdir, stat }) {
|
|
|
1247
1311
|
* file that does not exist, naming the importing file and the specifier. Returns the
|
|
1248
1312
|
* sorted list of worktree-relative paths copied (possibly empty).
|
|
1249
1313
|
*/
|
|
1250
|
-
function copyEscapingImports(worktreeRoot, stackDir, { read, mkdir, copy, readdir, stat, realpath }) {
|
|
1314
|
+
function copyEscapingImports(worktreeRoot, stackDir, { read, mkdir, copy, readdir, stat, realpath, sync = null }) {
|
|
1251
1315
|
const functionsDir = join(worktreeRoot, "supabase", "functions");
|
|
1252
1316
|
const visited = new Set(); // worktree-relative paths already queued or copied
|
|
1253
1317
|
const queue = [];
|
|
@@ -1333,7 +1397,10 @@ function copyEscapingImports(worktreeRoot, stackDir, { read, mkdir, copy, readdi
|
|
|
1333
1397
|
// whose text to scan for further imports.
|
|
1334
1398
|
let isDir = false;
|
|
1335
1399
|
try { isDir = stat(item.absPath).isDirectory(); } catch { /* copy() will report it */ }
|
|
1336
|
-
|
|
1400
|
+
// BOT-2403: each copied file is a single-file bind mount of a running edge runtime, so
|
|
1401
|
+
// it is refreshed in place (same inode), never replaced.
|
|
1402
|
+
if (sync) syncEntryInPlace(item.absPath, dest, sync);
|
|
1403
|
+
else copy(item.absPath, dest, isDir ? { recursive: true, force: true } : { force: true });
|
|
1337
1404
|
extraFiles.push(item.relPath);
|
|
1338
1405
|
if (!isDir && ESCAPING_IMPORT_SCAN_EXTENSIONS.some((ext) => item.relPath.endsWith(ext))) {
|
|
1339
1406
|
scanForImports(item.absPath, item.relPath);
|
|
@@ -1354,9 +1421,11 @@ function copyEscapingImports(worktreeRoot, stackDir, { read, mkdir, copy, readdi
|
|
|
1354
1421
|
* symlink that resolves on the host does not resolve inside Docker (the same
|
|
1355
1422
|
* reason Supply Guard's isolated-stack script materializes function sources).
|
|
1356
1423
|
* Copying every run also keeps the leased stack in step with the worktree — and
|
|
1357
|
-
* the copy
|
|
1424
|
+
* the copy mirrors the worktree (rsync --delete semantics, like the hermetic lane), so a
|
|
1358
1425
|
* migration or function deleted in the worktree is not resurrected by the leased
|
|
1359
|
-
* stack. Only the Supabase CLI's own scratch state survives the refresh.
|
|
1426
|
+
* stack. Only the Supabase CLI's own scratch state survives the refresh. BOT-2403: the
|
|
1427
|
+
* refresh is IN PLACE — a running (or idle) stack's edge runtime bind-mounts the
|
|
1428
|
+
* functions directory and the escaping-import files, so they are never replaced.
|
|
1360
1429
|
*/
|
|
1361
1430
|
export function materializeRunStackDir(cwd, { slot, ticket, nonce, idleTtlSecs } = {}, io = {}) {
|
|
1362
1431
|
const read = io.readFile ?? readFileSync;
|
|
@@ -1369,6 +1438,14 @@ export function materializeRunStackDir(cwd, { slot, ticket, nonce, idleTtlSecs }
|
|
|
1369
1438
|
const now = io.now ?? Date.now;
|
|
1370
1439
|
const rename = io.rename ?? renameSync;
|
|
1371
1440
|
const resolveRoot = io.realpath ?? realpathSync;
|
|
1441
|
+
const lstat = io.lstat ?? lstatSync;
|
|
1442
|
+
const warn = io.warn ?? (() => {});
|
|
1443
|
+
// BOT-2403: the in-place refresh's file operations (`copy` only creates what is missing).
|
|
1444
|
+
const sync = {
|
|
1445
|
+
read, mkdir, remove, copy, readdir, lstat,
|
|
1446
|
+
readlink: io.readlink ?? readlinkSync,
|
|
1447
|
+
copyFile: io.copyFile ?? copyFileSync,
|
|
1448
|
+
};
|
|
1372
1449
|
|
|
1373
1450
|
const worktreeRoot = resolveRoot(cwd);
|
|
1374
1451
|
const sourceDir = join(worktreeRoot, "supabase");
|
|
@@ -1412,7 +1489,7 @@ export function materializeRunStackDir(cwd, { slot, ticket, nonce, idleTtlSecs }
|
|
|
1412
1489
|
// stable per-slot one, so the project id is exactly as specific (and keeps a
|
|
1413
1490
|
// refresh idempotent).
|
|
1414
1491
|
const invocationToken = nonce ? String(nonce) : identity.projectId;
|
|
1415
|
-
const marker = (portBase, extraFiles = []) => JSON.stringify({
|
|
1492
|
+
const marker = (portBase, extraFiles = [], functionsIno = null) => JSON.stringify({
|
|
1416
1493
|
schema: 1,
|
|
1417
1494
|
project_id: identity.projectId,
|
|
1418
1495
|
invocation_token: invocationToken,
|
|
@@ -1428,6 +1505,9 @@ export function materializeRunStackDir(cwd, { slot, ticket, nonce, idleTtlSecs }
|
|
|
1428
1505
|
// outside supabase/ — recorded so an operator can see why this stack dir has
|
|
1429
1506
|
// (e.g.) a src/ directory, instead of guessing at a `supabase start` failure.
|
|
1430
1507
|
extra_files: extraFiles,
|
|
1508
|
+
// BOT-2403: the functions directory a running edge runtime bind-mounts; a later run
|
|
1509
|
+
// warns when another writer replaced it (the container would keep the old one).
|
|
1510
|
+
functions_inode: functionsIno,
|
|
1431
1511
|
});
|
|
1432
1512
|
const registryDir = io.portRegistryDir ?? defaultPortRegistryDir();
|
|
1433
1513
|
const writeExclusive = io.writeExclusive ?? ((path, data) => writeFileSync(path, data, { flag: "wx" }));
|
|
@@ -1490,29 +1570,35 @@ export function materializeRunStackDir(cwd, { slot, ticket, nonce, idleTtlSecs }
|
|
|
1490
1570
|
const { text } = rewriteLeasedStackConfig(rootConfig, { projectId: identity.projectId, portBase });
|
|
1491
1571
|
assertLeasedStackIsolated(rootConfig, text);
|
|
1492
1572
|
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1573
|
+
// BOT-2403: a replaced functions directory strands a running edge runtime's bind mount.
|
|
1574
|
+
// This refresh never replaces it; another writer may have, so say so once.
|
|
1575
|
+
const recordedIno = previous?.project_id === identity.projectId && Number.isInteger(previous?.functions_inode) ? previous.functions_inode : null;
|
|
1576
|
+
const currentIno = functionsInode(stackDir, lstat);
|
|
1577
|
+
if (recordedIno !== null && currentIno !== null && currentIno !== recordedIno) {
|
|
1578
|
+
warn(`stack: warning: ${stackPath}/supabase/functions was replaced since this stack was materialized (inode ${recordedIno} → ${currentIno}); ` +
|
|
1579
|
+
"a running edge runtime still sees the old directory. Update a leased stack directory in place — never rm + recreate it.");
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
// BOT-2142: symlinks stay verbatim (relative), never rewritten to absolute links INTO
|
|
1583
|
+
// the source worktree, which a remote bundle must refuse (Supply Guard's
|
|
1584
|
+
// supabase/CLAUDE.md -> AGENTS.md).
|
|
1585
|
+
syncTreeInPlace(sourceDir, join(stackDir, "supabase"), sync, { keep: new Set([...SKIP_ENTRIES, "config.toml"]) });
|
|
1586
|
+
let currentConfig = null;
|
|
1587
|
+
try { currentConfig = read(join(stackDir, "supabase", "config.toml"), "utf8"); } catch { /* first materialization */ }
|
|
1588
|
+
if (currentConfig !== text) write(join(stackDir, "supabase", "config.toml"), text);
|
|
1504
1589
|
|
|
1505
1590
|
// BOT-1828: bring along any file an edge function imports OUTSIDE supabase/
|
|
1506
1591
|
// (Supply Guard's admin-api-keys/index.ts pulls in ../../../src/lib/…) —
|
|
1507
1592
|
// `supabase start` resolves function imports when it starts the edge runtime, so
|
|
1508
1593
|
// without this the Helper fails deep inside `supabase start` with a raw ENOENT.
|
|
1509
|
-
const extraFiles = copyEscapingImports(worktreeRoot, stackDir, { read, mkdir, copy, readdir, stat, realpath: resolveRoot });
|
|
1594
|
+
const extraFiles = copyEscapingImports(worktreeRoot, stackDir, { read, mkdir, copy, readdir, stat, realpath: resolveRoot, sync });
|
|
1510
1595
|
|
|
1511
1596
|
// Re-state the claim after the copy: the marker records who owns this directory,
|
|
1512
1597
|
// which port block it holds, and how long a later run must keep it (Codex #928
|
|
1513
1598
|
// R7/R12). It lives beside `supabase/`, which the refresh replaces wholesale.
|
|
1514
|
-
|
|
1515
|
-
atomicWrite(
|
|
1599
|
+
const functionsIno = functionsInode(stackDir, lstat);
|
|
1600
|
+
atomicWrite(join(stackDir, LEASED_STACK_MARKER), marker(portBase, extraFiles, functionsIno), { write, rename });
|
|
1601
|
+
atomicWrite(portClaimPath(registryDir, portBase), marker(portBase, extraFiles, functionsIno), { write, rename });
|
|
1516
1602
|
return stackPath;
|
|
1517
1603
|
} catch (error) {
|
|
1518
1604
|
try { remove(portClaimPath(registryDir, portBase), { force: true }); } catch { /* best effort */ }
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// BOT-2403 — a re-adopted idle stack is never handed over with a stale functions mount.
|
|
2
|
+
//
|
|
3
|
+
// The edge runtime (`supabase_edge_runtime_<project>`) bind-mounts the stack's
|
|
4
|
+
// `supabase/functions` DIRECTORY. A bind mount pins the directory it resolved at
|
|
5
|
+
// container start: if that directory is later removed and recreated (a new inode — what
|
|
6
|
+
// `bb stack run`'s materializer did before BOT-2403, and what any rm + recreate writer
|
|
7
|
+
// does), the container keeps the deleted directory and lists 0 entries, so every function
|
|
8
|
+
// answers 503 BOOT_ERROR while the host directory looks complete. Re-adoption proves the
|
|
9
|
+
// files' CONTENT (the stack digest), not that the running container still sees them.
|
|
10
|
+
//
|
|
11
|
+
// So before a re-adopted CO-LOCATED stack is reported ready, `bb stack run` checks the
|
|
12
|
+
// mount from inside the container. Empty while the host directory is not → restart the
|
|
13
|
+
// edge runtime once (a restart resolves the bind source again) → still empty → the caller
|
|
14
|
+
// re-provisions. A remote lease is skipped: re-adoption uploads no bundle, so the Helper's
|
|
15
|
+
// copy of the stack directory is never rewritten under a running container. A stack with
|
|
16
|
+
// no functions, or with no edge runtime container (`No such container`), is skipped too; any
|
|
17
|
+
// other inspection failure fails closed.
|
|
18
|
+
//
|
|
19
|
+
// Dependency-free (Node stdlib only); docker is run with a fixed argv, never a shell.
|
|
20
|
+
import { execFile } from "node:child_process";
|
|
21
|
+
import { readdirSync } from "node:fs";
|
|
22
|
+
import { join } from "node:path";
|
|
23
|
+
import { dockerSelectorArgs } from "./stack-usage.mjs";
|
|
24
|
+
|
|
25
|
+
export const MOUNT_CHECK_TIMEOUT_MS = 10_000;
|
|
26
|
+
export const RUNTIME_RESTART_TIMEOUT_MS = 60_000;
|
|
27
|
+
const CO_LOCATED_MODES = new Set(["local", "local_exec"]);
|
|
28
|
+
const PROJECT_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,62}$/;
|
|
29
|
+
|
|
30
|
+
/** Default runner: `docker <args>` with a hard timeout; never a shell. */
|
|
31
|
+
export function runDocker(args, { timeoutMs = MOUNT_CHECK_TIMEOUT_MS } = {}) {
|
|
32
|
+
return new Promise((resolve) => {
|
|
33
|
+
// nosemgrep: javascript.lang.security.detect-child-process.detect-child-process -- fixed executable, validated argv, no shell.
|
|
34
|
+
execFile("docker", args, { timeout: timeoutMs, killSignal: "SIGKILL", maxBuffer: 4 * 1024 * 1024, windowsHide: true }, (error, stdout, stderr) => {
|
|
35
|
+
resolve({ code: error ? (typeof error.code === "number" ? error.code : 1) : 0, stdout: String(stdout ?? ""), stderr: String(stderr ?? ""), timedOut: Boolean(error?.killed) });
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** The container path the functions directory is mounted at, or null (from `docker inspect … .Mounts`). */
|
|
41
|
+
export function functionsMountDestination(mountsJson, hostFunctionsDir) {
|
|
42
|
+
let mounts;
|
|
43
|
+
try { mounts = JSON.parse(String(mountsJson ?? "").trim()); } catch { return null; }
|
|
44
|
+
if (!Array.isArray(mounts)) return null;
|
|
45
|
+
const mount = mounts.find((m) => m?.Type === "bind" && m?.Source === hostFunctionsDir);
|
|
46
|
+
return typeof mount?.Destination === "string" && mount.Destination ? mount.Destination : null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Check (and once repair) the functions mount of a re-adopted idle lease.
|
|
51
|
+
* `lease` is the re-adoption answer: `{ execution_mode, connection: { project_id, botbuddy_docker_target? } }`.
|
|
52
|
+
* Returns `{ ok: true, skipped? , restarted? }` or `{ ok: false, restarted, reason }`. Never throws.
|
|
53
|
+
*/
|
|
54
|
+
export async function checkReadoptedStackMounts(lease, stackDir, { run = runDocker, listHost = (dir) => readdirSync(dir), log = () => {} } = {}) {
|
|
55
|
+
if (!CO_LOCATED_MODES.has(lease?.execution_mode)) return { ok: true, skipped: "remote" };
|
|
56
|
+
const projectId = lease?.connection?.project_id;
|
|
57
|
+
if (typeof projectId !== "string" || !PROJECT_ID_RE.test(projectId)) return { ok: true, skipped: "no_project_id" };
|
|
58
|
+
if (typeof stackDir !== "string" || !stackDir) return { ok: true, skipped: "no_stack_dir" };
|
|
59
|
+
const hostFunctions = join(stackDir, "supabase", "functions");
|
|
60
|
+
let hostEntries;
|
|
61
|
+
try { hostEntries = listHost(hostFunctions); } catch { return { ok: true, skipped: "no_functions" }; }
|
|
62
|
+
if (!Array.isArray(hostEntries) || hostEntries.length === 0) return { ok: true, skipped: "no_functions" };
|
|
63
|
+
|
|
64
|
+
const docker = dockerSelectorArgs(lease.connection.botbuddy_docker_target ?? null);
|
|
65
|
+
const container = `supabase_edge_runtime_${projectId}`;
|
|
66
|
+
const inspect = await run([...docker, "container", "inspect", "--format", "{{json .Mounts}}", container]);
|
|
67
|
+
if (inspect.code !== 0) {
|
|
68
|
+
// Only a definitively absent runtime (edge_runtime disabled) is skipped. Any other
|
|
69
|
+
// failure — daemon/context down, permissions, a timeout — fails closed (Codex #1191 P2):
|
|
70
|
+
// the caller re-provisions rather than hand over a stack nobody could check.
|
|
71
|
+
// Real counterpart (Docker Desktop, 2026-10-16): "Error response from daemon: No such container: <name>".
|
|
72
|
+
if (!inspect.timedOut && /No such (container|object)/i.test(inspect.stderr ?? "")) return { ok: true, skipped: "no_edge_runtime" };
|
|
73
|
+
return { ok: false, restarted: false, reason: `could not inspect ${container} (exit ${inspect.code}${inspect.timedOut ? ", timed out" : ""})` };
|
|
74
|
+
}
|
|
75
|
+
const destination = functionsMountDestination(inspect.stdout, hostFunctions);
|
|
76
|
+
if (!destination) return { ok: false, restarted: false, reason: `${container} does not bind-mount ${hostFunctions}` };
|
|
77
|
+
|
|
78
|
+
const mountedEntries = async () => {
|
|
79
|
+
const answer = await run([...docker, "exec", container, "ls", "-A", destination]);
|
|
80
|
+
if (answer.code !== 0) return null;
|
|
81
|
+
return answer.stdout.split(/\r?\n/).filter((line) => line.length > 0).length;
|
|
82
|
+
};
|
|
83
|
+
const first = await mountedEntries();
|
|
84
|
+
if (first) return { ok: true, restarted: false };
|
|
85
|
+
|
|
86
|
+
log(`stack: the re-adopted stack's edge runtime sees ${first === null ? "no readable" : "an empty"} functions mount (${hostEntries.length} entries on the host) — restarting ${container} once`);
|
|
87
|
+
const restart = await run([...docker, "restart", container], { timeoutMs: RUNTIME_RESTART_TIMEOUT_MS });
|
|
88
|
+
if (restart.code !== 0) {
|
|
89
|
+
return { ok: false, restarted: false, reason: `docker restart ${container} failed (exit ${restart.code}${restart.timedOut ? ", timed out" : ""})` };
|
|
90
|
+
}
|
|
91
|
+
const second = await mountedEntries();
|
|
92
|
+
if (second) return { ok: true, restarted: true };
|
|
93
|
+
return { ok: false, restarted: true, reason: `${container} still sees ${second === null ? "no readable" : "an empty"} functions mount after a restart` };
|
|
94
|
+
}
|
package/src/stack-usage.mjs
CHANGED
|
@@ -27,10 +27,29 @@ export const SUPABASE_INTERNAL_ROLES = Object.freeze([
|
|
|
27
27
|
|
|
28
28
|
const ROLE_LIST = SUPABASE_INTERNAL_ROLES.map((r) => `'${r}'`).join(", ");
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* BOT-2402: Supabase's own internal database. Logflare (`supabase_analytics_<project>`)
|
|
32
|
+
* writes its log tables there as `postgres` every ~10 s, so on a stack with analytics
|
|
33
|
+
* enabled every observation looked like fresh client use. Nothing a client does to the
|
|
34
|
+
* project happens there; the Helper census excludes it too (helper 0.2.39).
|
|
35
|
+
*/
|
|
36
|
+
export const SUPABASE_INTERNAL_DATABASE = "_supabase";
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* BOT-2402: a pg_cron job run is a libpq connection the scheduler opens to its own server
|
|
40
|
+
* (unix socket or loopback) as the job owner, usually `postgres`, every minute. It is the
|
|
41
|
+
* stack's own schedule, not a client (the Helper census rule, BOT-2219). A backend that only
|
|
42
|
+
* borrows the name from another address is still a client.
|
|
43
|
+
*/
|
|
44
|
+
const NOT_CRON_RUN = "NOT (coalesce(application_name, '') = 'pg_cron' AND (client_addr IS NULL OR host(client_addr) IN ('127.0.0.1', '::1')))";
|
|
45
|
+
const NOT_PLATFORM = `datname IS DISTINCT FROM '${SUPABASE_INTERNAL_DATABASE}' AND ${NOT_CRON_RUN}`;
|
|
46
|
+
|
|
30
47
|
/**
|
|
31
48
|
* Two values, `|`-separated (psql -tA):
|
|
32
49
|
* 1. client_connections — client backends doing work now (state <> 'idle'), excluding
|
|
33
|
-
* this probe
|
|
50
|
+
* this probe, anything on the internal `_supabase` database, pg_cron's own job runs
|
|
51
|
+
* (BOT-2402) and every
|
|
52
|
+
* internal role except `authenticator`: a PostgREST request in
|
|
34
53
|
* flight is use now (Codex #1160 P1); its pooled sessions sitting idle are not.
|
|
35
54
|
* 2. last_client_query_at — the latest state change of any client backend, counting
|
|
36
55
|
* `authenticator` (PostgREST traffic IS use), ISO-8601 UTC; empty when none.
|
|
@@ -38,9 +57,11 @@ const ROLE_LIST = SUPABASE_INTERNAL_ROLES.map((r) => `'${r}'`).join(", ");
|
|
|
38
57
|
export const USAGE_SQL =
|
|
39
58
|
"SELECT " +
|
|
40
59
|
"(SELECT count(*) FROM pg_stat_activity WHERE backend_type = 'client backend' AND pid <> pg_backend_pid() " +
|
|
60
|
+
`AND ${NOT_PLATFORM} ` +
|
|
41
61
|
`AND (coalesce(usename, '') NOT IN (${ROLE_LIST}) OR usename = 'authenticator') AND state <> 'idle'), ` +
|
|
42
62
|
"(SELECT to_char(max(state_change) AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM pg_stat_activity " +
|
|
43
63
|
"WHERE backend_type = 'client backend' AND pid <> pg_backend_pid() " +
|
|
64
|
+
`AND ${NOT_PLATFORM} ` +
|
|
44
65
|
`AND (coalesce(usename, '') NOT IN (${ROLE_LIST}) OR usename = 'authenticator'))`;
|
|
45
66
|
|
|
46
67
|
const CO_LOCATED_MODES = new Set(["local", "local_exec"]);
|
package/src/stack.mjs
CHANGED
|
@@ -44,6 +44,7 @@ import { runDockerCommand, runDockerWorkflow, ADMITTED_DOCKER_VALIDATIONS } from
|
|
|
44
44
|
import { acquireStackLock, lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
|
|
45
45
|
import { LEASED_STACK_IDLE_RETENTION_SECS, assertLeasedStackIsolated, leasedStackClaimHeld, markLeasedStackReleased, stableRunStackReusable, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
|
|
46
46
|
import { measureStackUsage } from "./stack-usage.mjs";
|
|
47
|
+
import { checkReadoptedStackMounts } from "./stack-mount-health.mjs";
|
|
47
48
|
import { machineUuid } from "./machine-id.mjs";
|
|
48
49
|
import { DEFAULT_BUNDLE_MAX_BYTES, buildStackBundle, uploadStackBundle, validateBundleIncludePath } from "./stack-bundle.mjs";
|
|
49
50
|
import { assertSshTarget, establishSshTunnel } from "./stack-tunnel.mjs";
|
|
@@ -218,7 +219,11 @@ ${bold("EXIT CODES")}
|
|
|
218
219
|
${bold("IDLE LEASES")} (BOT-1803)
|
|
219
220
|
stack run sends stack_digest (the deterministic sha256 of the materialised stack bundle, also
|
|
220
221
|
with --local-only; omitted when it cannot be built) and the control plane answers with one of:
|
|
221
|
-
reused_idle the slot's idle lease is re-adopted: no provision; receipt reused "idle"
|
|
222
|
+
reused_idle the slot's idle lease is re-adopted: no provision; receipt reused "idle".
|
|
223
|
+
A co-located one is handed over only when its edge runtime still sees the
|
|
224
|
+
functions (BOT-2403): an empty mount restarts the runtime once; still empty →
|
|
225
|
+
the lease is reaped and the request re-sent once (reprovisioned
|
|
226
|
+
"unhealthy_mount"); receipt readopt_health {ok, restarted|skipped, reason}
|
|
222
227
|
IDLE_LEASE_REPROVISIONING / LEASE_PREEMPTED
|
|
223
228
|
the idle lease is being reaped (content changed / capacity reclaimed): run
|
|
224
229
|
waits for it to reach reaped (--reap-timeout), re-sends the same request
|
|
@@ -586,6 +591,7 @@ const CONSIDERED_VERDICT_TEXT = {
|
|
|
586
591
|
HOST_CAPACITY_WAIT: () => "Docker pressure full",
|
|
587
592
|
HOST_CAPACITY_BUDGET_FULL: () => "slot budget full",
|
|
588
593
|
HOST_CAPACITY_INVENTORY_UNKNOWN: () => "unknown Docker resources in its census",
|
|
594
|
+
HOST_PRESSURE: () => "host under pressure",
|
|
589
595
|
TENANT_RESERVATION_FULL: () => "tenant reservation full",
|
|
590
596
|
SLOT_HELD: () => "this slot is already leased there",
|
|
591
597
|
HOST_HELPER_OUTDATED: () => "Helper predates remote leases (protocol < 4)",
|
|
@@ -605,6 +611,33 @@ export function describeConsidered(considered) {
|
|
|
605
611
|
});
|
|
606
612
|
}
|
|
607
613
|
|
|
614
|
+
/**
|
|
615
|
+
* BOT-1630: the typed park reason of a queued lease, or null. The server holds a host under
|
|
616
|
+
* pressure (`docker info` failing, low memory, engine footprint) with verdict HOST_PRESSURE, and
|
|
617
|
+
* a Helper that hits it mid-provision defers the lease back to the queue with that reason.
|
|
618
|
+
*/
|
|
619
|
+
export function queuedParkReason(d) {
|
|
620
|
+
if (!d || typeof d !== "object") return null;
|
|
621
|
+
const ledger = d.ledger && typeof d.ledger === "object" ? d.ledger : null;
|
|
622
|
+
if (ledger?.verdict === "HOST_PRESSURE" || ledger?.capacity?.code === "HOST_PRESSURE") return "host_pressure";
|
|
623
|
+
const here = Array.isArray(d.considered) ? d.considered.find((c) => c?.host_key === d.host_key) : null;
|
|
624
|
+
if (here?.verdict_code === "HOST_PRESSURE") return "host_pressure";
|
|
625
|
+
if (d.provision_defer_reason === "HOST_PRESSURE") return "host_pressure";
|
|
626
|
+
return null;
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/** BOT-1630: "docker_unresponsive (docker_command_failed:docker info:-2; clears …)" per binding dimension. */
|
|
630
|
+
export function describePressure(d) {
|
|
631
|
+
const reasons = d?.ledger?.capacity?.pressure?.reasons;
|
|
632
|
+
if (!Array.isArray(reasons) || reasons.length === 0) return null;
|
|
633
|
+
return reasons.filter((r) => r && typeof r.dimension === "string").map((r) => {
|
|
634
|
+
const parts = [];
|
|
635
|
+
if (r.observed != null) parts.push(r.limit != null ? `${r.observed} vs limit ${r.limit}` : String(r.observed));
|
|
636
|
+
if (typeof r.clears_at === "string") parts.push(`clears ${r.clears_at}`);
|
|
637
|
+
return parts.length ? `${r.dimension} (${parts.join("; ")})` : r.dimension;
|
|
638
|
+
}).join(", ") || null;
|
|
639
|
+
}
|
|
640
|
+
|
|
608
641
|
export function describeUnitLedger(ledger) {
|
|
609
642
|
if (!ledger || typeof ledger !== "object") return [];
|
|
610
643
|
const cap = ledger.capacity ?? {};
|
|
@@ -707,7 +740,7 @@ async function claimProjectLock(projectId, lock, subject) {
|
|
|
707
740
|
}
|
|
708
741
|
}
|
|
709
742
|
|
|
710
|
-
export async function resolveRunExecution(cwd, opts, { materialize = materializeRunStackDir, lock = acquireStackLock, identity = leasedStackIdentity, reserveBlocks = reservePortBlocks, releaseClaim = releaseLeasedStackClaim, claimHeld = leasedStackClaimHeld, stableReusable = stableRunStackReusable, markReleased = markLeasedStackReleased, portRegistryDir = null } = {}) {
|
|
743
|
+
export async function resolveRunExecution(cwd, opts, { materialize = materializeRunStackDir, lock = acquireStackLock, identity = leasedStackIdentity, reserveBlocks = reservePortBlocks, releaseClaim = releaseLeasedStackClaim, claimHeld = leasedStackClaimHeld, stableReusable = stableRunStackReusable, markReleased = markLeasedStackReleased, portRegistryDir = null, warn = (line) => process.stderr.write(`${line}\n`) } = {}) {
|
|
711
744
|
const refuseRoot = (execution) => {
|
|
712
745
|
if (execution.stackPath === ".") {
|
|
713
746
|
throw new Error(
|
|
@@ -785,7 +818,8 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
|
|
|
785
818
|
const execution = refuseRoot(resolveStackPath(cwd, materialize(
|
|
786
819
|
cwd,
|
|
787
820
|
{ slot: opts.slot, repo: opts.repo, ticket: opts.ticket, nonce, idleTtlSecs: opts.idleTtl ?? undefined },
|
|
788
|
-
|
|
821
|
+
// BOT-2403: a functions directory another writer replaced under this stack is reported.
|
|
822
|
+
{ ...(portRegistryDir ? { portRegistryDir } : {}), warn },
|
|
789
823
|
)));
|
|
790
824
|
return {
|
|
791
825
|
...execution,
|
|
@@ -2865,6 +2899,8 @@ export async function cmdDone(leaseId, opts, {
|
|
|
2865
2899
|
const SIGNAL_EXIT = Object.freeze({ SIGINT: 130, SIGTERM: 143, SIGHUP: 129 });
|
|
2866
2900
|
/** BOT-1803: request answers meaning "this slot's idle lease is being reaped; re-request once it is". */
|
|
2867
2901
|
const IDLE_REPROVISION_REASONS = Object.freeze({ IDLE_LEASE_REPROVISIONING: "digest_mismatch", LEASE_PREEMPTED: "preempted" });
|
|
2902
|
+
// BOT-2403: the re-adopted idle stack failed the functions-mount gate and was reaped.
|
|
2903
|
+
const READOPT_UNHEALTHY_REASON = "unhealthy_mount";
|
|
2868
2904
|
|
|
2869
2905
|
function safeConnectionPath(requested, leaseId) {
|
|
2870
2906
|
if (requested) {
|
|
@@ -3093,6 +3129,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3093
3129
|
// stack directory; the default is the real materializing resolver.
|
|
3094
3130
|
const resolveExecution = adapters.resolveExecution ?? ((runOpts) => resolveRunExecution(process.cwd(), runOpts));
|
|
3095
3131
|
const refreshRetention = adapters.refreshRetention ?? refreshLeasedStackRetention;
|
|
3132
|
+
// BOT-2403: a re-adopted co-located idle stack must see its functions before it is handed over.
|
|
3133
|
+
const readoptHealth = adapters.readoptHealth ?? ((lease, stackDir) => checkReadoptedStackMounts(lease, stackDir, { log: stderrLine }));
|
|
3096
3134
|
// BOT-2142: remote-lease seams (bundle build/upload, SSH forwards). Production uses the
|
|
3097
3135
|
// real builders; the fake-ssh tests swap only the ssh binary (BOTBUDDY_STACK_SSH_BIN works too).
|
|
3098
3136
|
const buildBundle = adapters.buildBundle ?? buildStackBundle;
|
|
@@ -3124,7 +3162,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3124
3162
|
const placement = { mode: null, hostKey: null, requesterHost: null, transport: "none", forwards: null, bundle: null, placementReason: null, considered: null };
|
|
3125
3163
|
incidentBox.execution = placement;
|
|
3126
3164
|
// BOT-1803: idle-lease bookkeeping for the receipt.
|
|
3127
|
-
const idleLease = { reused: null, reprovisioned: null, previousLeaseId: null, stackDigest: null };
|
|
3165
|
+
const idleLease = { reused: null, reprovisioned: null, previousLeaseId: null, stackDigest: null, readoptHealth: null };
|
|
3128
3166
|
incidentBox.idleLease = idleLease;
|
|
3129
3167
|
// BOT-1803: set when this run's port block must stay reserved although nothing of ours
|
|
3130
3168
|
// was provisioned (the slot's idle/live stack in THIS directory is still bound to it).
|
|
@@ -3407,16 +3445,43 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3407
3445
|
if (request?.status == null) leaseProvisioned = true;
|
|
3408
3446
|
return { exitCode: request?.auth ? EXIT.AUTH : EXIT.BACKEND, outcome: "error", error: request?.error || "lease request failed" };
|
|
3409
3447
|
}
|
|
3410
|
-
|
|
3448
|
+
let why = attempt === 0 && request.data?.success === false && request.data?.lease_id ? IDLE_REPROVISION_REASONS[request.data.code] : null;
|
|
3449
|
+
// BOT-2403: a re-adopted idle stack is handed over only when its edge runtime still sees
|
|
3450
|
+
// the functions (the gate restarts the runtime once). Still broken → reap it and send the
|
|
3451
|
+
// same request once more, exactly like a digest re-provision.
|
|
3452
|
+
if (!why && attempt === 0 && request.data?.success === true && request.data.reused === true && request.data.reused_idle === true) {
|
|
3453
|
+
let health;
|
|
3454
|
+
try { health = await readoptHealth(request.data, stackDirFor(execution)); } catch (error) { health = { ok: false, reason: String(error?.message ?? error) }; }
|
|
3455
|
+
idleLease.readoptHealth = health;
|
|
3456
|
+
if (!health?.ok) {
|
|
3457
|
+
const broken = request.data.lease_id;
|
|
3458
|
+
stderrLine(`stack: re-adopted idle lease ${broken} is not healthy (${health?.reason ?? "unknown"}) — reaping it and provisioning a fresh stack`);
|
|
3459
|
+
const released = await api.release(broken);
|
|
3460
|
+
if (!released?.ok || released.data?.success === false) {
|
|
3461
|
+
retainBlock = true;
|
|
3462
|
+
return {
|
|
3463
|
+
exitCode: EXIT.LEASE_FAILED, outcome: "error", code: "READOPT_UNHEALTHY", leaseId: broken,
|
|
3464
|
+
error: `re-adopted idle lease ${broken} is not healthy (${health?.reason ?? "unknown"}) and its reap could not be requested ` +
|
|
3465
|
+
`(${released?.data?.code ?? released?.error ?? "release failed"}); run \`bb stack done ${broken} --reap\` and re-run`,
|
|
3466
|
+
};
|
|
3467
|
+
}
|
|
3468
|
+
why = READOPT_UNHEALTHY_REASON;
|
|
3469
|
+
}
|
|
3470
|
+
}
|
|
3411
3471
|
if (!why) break;
|
|
3412
3472
|
const previous = request.data.lease_id;
|
|
3413
3473
|
idleLease.previousLeaseId = previous;
|
|
3414
|
-
|
|
3474
|
+
const becoming = why === "preempted" ? "preempted"
|
|
3475
|
+
: why === READOPT_UNHEALTHY_REASON ? "reaped (its edge runtime lost the functions mount)" : "re-provisioned (the stack content changed)";
|
|
3476
|
+
stderrLine(`stack: this slot's idle lease ${previous} is being ${becoming} — waiting for it to reach reaped, then requesting again`);
|
|
3415
3477
|
activeWaitAbort = new AbortController();
|
|
3416
3478
|
const gone = await resilientWait(previous, (st) => st === "reaped", (st) => st === "reap_failed", { timeoutSec: opts.reapTimeout, auth: rpc.auth, signal: activeWaitAbort.signal });
|
|
3417
3479
|
activeWaitAbort = null;
|
|
3418
3480
|
if (gone?.interrupted || receivedSignal) {
|
|
3419
|
-
|
|
3481
|
+
// Codex #1191 P2: the old lease's reap is not proven, so its containers may still hold
|
|
3482
|
+
// this stack's ports — keep the block (never double-booked by the next run).
|
|
3483
|
+
retainBlock = true;
|
|
3484
|
+
return { exitCode: SIGNAL_EXIT[receivedSignal] ?? EXIT.INTERNAL, outcome: "interrupted", code: request.data.code ?? "READOPT_UNHEALTHY" };
|
|
3420
3485
|
}
|
|
3421
3486
|
let reaped = Boolean(gone?.woke) && gone.state === "reaped";
|
|
3422
3487
|
if (!reaped && !gone?.failed) {
|
|
@@ -3428,7 +3493,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3428
3493
|
// Its containers may still hold this stack's ports.
|
|
3429
3494
|
retainBlock = true;
|
|
3430
3495
|
return {
|
|
3431
|
-
exitCode: EXIT.LEASE_FAILED, outcome: "error", code: request.data.code,
|
|
3496
|
+
exitCode: EXIT.LEASE_FAILED, outcome: "error", code: request.data.code ?? "READOPT_UNHEALTHY",
|
|
3432
3497
|
error: `this slot's idle lease ${previous} did not reach reaped ${gone?.failed ? "(its reap failed)" : `within ${opts.reapTimeout}s`} ` +
|
|
3433
3498
|
`after ${why}; no new lease was requested — check \`bb stack status ${previous}\` and re-run once it is reaped`,
|
|
3434
3499
|
...(parkIncident ? { incident: parkIncident } : {}),
|
|
@@ -3547,16 +3612,28 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3547
3612
|
const cleanup = await requestCleanup();
|
|
3548
3613
|
return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup };
|
|
3549
3614
|
}
|
|
3615
|
+
// BOT-1630: ONE park budget (--timeout) across every park. A Helper that finds the host under
|
|
3616
|
+
// pressure mid-provision defers the lease back to `queued` (its place kept); run then parks
|
|
3617
|
+
// again against the same deadline instead of timing out on the provision clock.
|
|
3618
|
+
const parkDeadlineMs = (adapters.now ?? Date.now)() + opts.timeout * 1000;
|
|
3619
|
+
let parkView = request.data;
|
|
3620
|
+
let parkReason = null;
|
|
3621
|
+
for (;;) {
|
|
3550
3622
|
if (state === "queued") {
|
|
3551
3623
|
// BOT-2019: say where the lease sits (server dispatch rank, or why it has none) before parking.
|
|
3552
|
-
stderrLine(`stack: lease ${leaseId} queued — ${describeQueuePosition(
|
|
3553
|
-
for (const line of describeUnitLedger(
|
|
3554
|
-
for (const line of describeConsidered(
|
|
3624
|
+
stderrLine(`stack: lease ${leaseId} queued — ${describeQueuePosition(parkView)}`);
|
|
3625
|
+
for (const line of describeUnitLedger(parkView.ledger)) stderrLine(`stack: ${line}`);
|
|
3626
|
+
for (const line of describeConsidered(parkView.considered)) stderrLine(`stack: ${line}`);
|
|
3627
|
+
parkReason = queuedParkReason(parkView) ?? parkReason;
|
|
3628
|
+
if (parkReason === "host_pressure") {
|
|
3629
|
+
const why = describePressure(parkView);
|
|
3630
|
+
stderrLine(`stack: parked — park_reason host_pressure${why ? `: ${why}` : ""}; the lease keeps its queue place and starts when the host's pressure clears`);
|
|
3631
|
+
}
|
|
3555
3632
|
activeWaitAbort = new AbortController();
|
|
3556
3633
|
// BOT-2013: every reap state (reaping / reaped / reap_failed) means the queued lease is lost.
|
|
3557
3634
|
// BOT-2099: an auth/availability blip is retried inside the park until its deadline;
|
|
3558
3635
|
// it never releases a queued lease or becomes the lease's outcome.
|
|
3559
|
-
const parked = await resilientWait(leaseId, (s) => s !== "queued" && s != null, isReaped, { timeoutSec: opts.timeout, auth: rpc.auth, signal: activeWaitAbort.signal });
|
|
3636
|
+
const parked = await resilientWait(leaseId, (s) => s !== "queued" && s != null, isReaped, { timeoutSec: opts.timeout, deadlineMs: parkDeadlineMs, auth: rpc.auth, signal: activeWaitAbort.signal });
|
|
3560
3637
|
activeWaitAbort = null;
|
|
3561
3638
|
const incident = parkIncident ? { incident: parkIncident } : {};
|
|
3562
3639
|
if (parked?.interrupted || receivedSignal) { const cleanup = await requestCleanup(); return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup, ...incident }; }
|
|
@@ -3578,7 +3655,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3578
3655
|
error: `lease ${leaseId} parked ${opts.timeout}s inside a platform incident (${parkIncident.reason} since ${parkIncident.first_at}), so its admission was never observed`,
|
|
3579
3656
|
};
|
|
3580
3657
|
}
|
|
3581
|
-
return { exitCode: EXIT.TIMEOUT, outcome: "timeout", leaseId, error: `parked ${opts.timeout}s without capacity`, cleanup, ...incident, ...(ledger ? { ledger } : {}) };
|
|
3658
|
+
return { exitCode: EXIT.TIMEOUT, outcome: "timeout", leaseId, error: `parked ${opts.timeout}s without capacity`, cleanup, ...incident, ...(ledger ? { ledger } : {}), ...(parkReason ? { parkReason } : {}) };
|
|
3582
3659
|
}
|
|
3583
3660
|
if (parked?.failed) { const cleanup = await requestCleanup(); return { exitCode: EXIT.LEASE_FAILED, outcome: "error", leaseId, error: `lease was ${parked.state} while queued (reap_reason=${parked.reap_reason ?? "unknown"})`, cleanup, ...incident }; }
|
|
3584
3661
|
if (parked?.error || parked?.auth) { const cleanup = await requestCleanup(); return { exitCode: parked?.auth ? EXIT.AUTH : EXIT.LEASE_FAILED, outcome: "error", leaseId, error: parked?.error || "lease did not leave queue", cleanup, ...incident }; }
|
|
@@ -3593,17 +3670,33 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3593
3670
|
// provisionTimeout before the belt-and-braces recheck below ever notices
|
|
3594
3671
|
// the lease is no longer provisioning. isReaped (== "reaping" || "reaped")
|
|
3595
3672
|
// already covers both; cmdUp's equivalent waits use it for this reason.
|
|
3596
|
-
|
|
3673
|
+
// BOT-1630: `queued` also ends this wait — a deferred provision (HOST_PRESSURE, capacity).
|
|
3674
|
+
const active = await resilientWait(leaseId, (s) => s === "active" || s === "queued", isReaped, { timeoutSec: provisionTimeout, auth: rpc.auth, signal: activeWaitAbort.signal });
|
|
3597
3675
|
activeWaitAbort = null;
|
|
3598
3676
|
if (active?.interrupted || receivedSignal) { const cleanup = await requestCleanup(); return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup }; }
|
|
3599
3677
|
if (active?.authUnavailable) { const cleanup = await requestCleanup(); return authUnavailableResult(active, "provisioning", cleanup); }
|
|
3678
|
+
if (active?.state === "queued" && !active.timeout && !active.failed && !active.error && !active.auth) {
|
|
3679
|
+
const read = await readLeaseQuietly(leaseId);
|
|
3680
|
+
parkView = read?.ok && read.data?.success ? read.data : { host_key: placement.hostKey };
|
|
3681
|
+
const why = parkView.provision_defer_reason ?? "host capacity";
|
|
3682
|
+
stderrLine(`stack: lease ${leaseId} was deferred back to the queue while provisioning (${why}) — its place is kept; parking again`);
|
|
3683
|
+
state = "queued";
|
|
3684
|
+
continue;
|
|
3685
|
+
}
|
|
3600
3686
|
if (active?.timeout) {
|
|
3601
3687
|
// BOT-1822 belt-and-braces: the zero-poll wait can still miss a live
|
|
3602
3688
|
// `stack_lease` signal (an older server before this fix, a relay hiccup,
|
|
3603
3689
|
// BOT-1816's class of dropped wakes) even though the lease genuinely went
|
|
3604
|
-
// active. One direct read before declaring
|
|
3605
|
-
// a
|
|
3690
|
+
// active or was deferred back to queued. One direct read before declaring
|
|
3691
|
+
// a timeout avoids reaping a healthy stack or destroying a kept queue place.
|
|
3606
3692
|
const recheck = await api.get(leaseId);
|
|
3693
|
+
if (recheck?.ok && recheck.data?.success && recheck.data.state === "queued") {
|
|
3694
|
+
parkView = recheck.data;
|
|
3695
|
+
const why = parkView.provision_defer_reason ?? "host capacity";
|
|
3696
|
+
stderrLine(`stack: lease ${leaseId} was deferred back to the queue while provisioning (${why}) — its place is kept; parking again`);
|
|
3697
|
+
state = "queued";
|
|
3698
|
+
continue;
|
|
3699
|
+
}
|
|
3607
3700
|
if (!(recheck?.ok && recheck.data?.success && recheck.data.state === "active")) {
|
|
3608
3701
|
const cleanup = await requestCleanup();
|
|
3609
3702
|
return { exitCode: EXIT.TIMEOUT, outcome: "timeout", leaseId, error: `physical provision did not reach active in ${provisionTimeout}s`, cleanup, ...(parkIncident ? { incident: parkIncident } : {}) };
|
|
@@ -3621,6 +3714,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
|
|
|
3621
3714
|
return { exitCode: active?.auth ? EXIT.AUTH : EXIT.LEASE_FAILED, outcome: "error", leaseId, error: active?.error || `lease ${active?.state ?? "failed"} before reaching active${why}`, cleanup, ...(parkIncident ? { incident: parkIncident } : {}) };
|
|
3622
3715
|
}
|
|
3623
3716
|
}
|
|
3717
|
+
break;
|
|
3718
|
+
}
|
|
3624
3719
|
const current = await api.get(leaseId);
|
|
3625
3720
|
if (!current?.ok || !current.data?.success || current.data.state !== "active" || !current.data.connection || typeof current.data.connection !== "object") {
|
|
3626
3721
|
const cleanup = await requestCleanup();
|
|
@@ -3834,6 +3929,7 @@ export function runReceiptFields(result) {
|
|
|
3834
3929
|
...(result.idleLease?.reprovisioned ? { reprovisioned: result.idleLease.reprovisioned } : {}),
|
|
3835
3930
|
...(result.idleLease?.previousLeaseId ? { previous_lease_id: result.idleLease.previousLeaseId } : {}),
|
|
3836
3931
|
...(result.idleLease?.stackDigest ? { stack_digest: result.idleLease.stackDigest } : {}),
|
|
3932
|
+
...(result.idleLease?.readoptHealth ? { readopt_health: result.idleLease.readoptHealth } : {}),
|
|
3837
3933
|
...(result.parkReason ? { park_reason: result.parkReason } : {}),
|
|
3838
3934
|
// BOT-2142: where the lease ran and how the child reached it.
|
|
3839
3935
|
execution_mode: result.execution?.mode ?? null,
|