@botbuddy/cli 1.40.13 → 1.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@botbuddy/cli",
3
- "version": "1.40.13",
3
+ "version": "1.41.0",
4
4
  "description": "BotBuddy — Swarm coordination CLI for multi-agent workflows",
5
5
  "type": "module",
6
6
  "bin": {
@@ -0,0 +1,296 @@
1
+ // BOT-2142 — the remote-lease stack bundle.
2
+ //
3
+ // A remote lease (BOT-2145 `execution: "remote"|"auto"`) is provisioned on a host
4
+ // that does not have this worktree, so `bb stack run` ships the materialised
5
+ // isolated stack directory (`.botbuddy/stacks/<project_id>/`, BOT-1798) as one
6
+ // gzip'd tar. The control plane pins it by digest (`bundle_sha256`, sent with the
7
+ // lease request) and hands back a signed upload URL (`bundle_upload_url`); the
8
+ // serving Helper downloads it and verifies the digest before provisioning
9
+ // (BOT-2143).
10
+ //
11
+ // Determinism is the contract: the digest must be a pure function of the tree's
12
+ // relative paths, contents and executable bits, so a retried request for the same
13
+ // tree re-sends the same `bundle_sha256`. The archive is therefore written here
14
+ // (ustar + PAX long names, sorted entries, mtime/uid/gid 0, modes normalised to
15
+ // 0644/0755, gzip header OS byte fixed) rather than by the platform `tar`, whose
16
+ // output differs between bsdtar and GNU tar and embeds mtimes and owners.
17
+ //
18
+ // Dependency-free (Node stdlib only), like the rest of `bb stack`.
19
+
20
+ import { createHash } from "node:crypto";
21
+ import { lstatSync, readdirSync, readFileSync, realpathSync, statSync } from "node:fs";
22
+ import { isAbsolute, join, relative, sep } from "node:path";
23
+ import { gunzipSync, gzipSync } from "node:zlib";
24
+
25
+ /** Default compressed-size cap (BOT-2142 §2): a real stack is a few MB. */
26
+ export const DEFAULT_BUNDLE_MAX_BYTES = 50 * 1024 * 1024;
27
+
28
+ /** Directory/file names never shipped, at any depth: dependency trees and the
29
+ * Supabase CLI's own scratch state (the serving host rebuilds both). */
30
+ const EXCLUDED_NAMES = new Set(["node_modules", ".temp", ".branches"]);
31
+ /** Root-only: the retention marker records local pids, hosts and paths (BOT-1798). */
32
+ const EXCLUDED_ROOT_FILES = new Set([".bb-stack.json"]);
33
+ /** Conventionally UNTRACKED dotenv files hold real provider secrets; committed
34
+ * ones (`.env.example`, `.env.test`, `.env.integration`, …) are test config and ship. */
35
+ const SECRET_DOTENV = /^\.env(?:\.local|\..+\.local)?$/;
36
+
37
+ function bundleError(code, message) {
38
+ const error = new Error(message);
39
+ error.code = code;
40
+ return error;
41
+ }
42
+
43
+ /** True when a stack-relative POSIX path (file or directory) is never bundled. */
44
+ export function isExcludedBundlePath(relPath) {
45
+ const parts = String(relPath).split("/").filter(Boolean);
46
+ if (parts.length === 0) return false;
47
+ const name = parts[parts.length - 1];
48
+ if (parts.length === 1 && EXCLUDED_ROOT_FILES.has(name)) return true;
49
+ if (parts.some((part) => EXCLUDED_NAMES.has(part))) return true;
50
+ return SECRET_DOTENV.test(name);
51
+ }
52
+
53
+ // ── ustar writer ─────────────────────────────────────────────────────────────
54
+
55
+ const BLOCK = 512;
56
+
57
+ function octal(value, width) {
58
+ // width includes the trailing NUL
59
+ return value.toString(8).padStart(width - 1, "0") + "\0";
60
+ }
61
+
62
+ function header(name, { size = 0, mode = 0o644, type = "0" } = {}) {
63
+ const block = Buffer.alloc(BLOCK, 0);
64
+ block.write(name, 0, 100, "utf8");
65
+ block.write(octal(mode, 8), 100, "ascii");
66
+ block.write(octal(0, 8), 108, "ascii"); // uid
67
+ block.write(octal(0, 8), 116, "ascii"); // gid
68
+ block.write(octal(size, 12), 124, "ascii");
69
+ block.write(octal(0, 12), 136, "ascii"); // mtime
70
+ block.write(" ", 148, "ascii"); // checksum placeholder
71
+ block.write(type, 156, "ascii");
72
+ block.write("ustar\0", 257, "ascii");
73
+ block.write("00", 263, "ascii");
74
+ let sum = 0;
75
+ for (const byte of block) sum += byte;
76
+ block.write(`${sum.toString(8).padStart(6, "0")}\0 `, 148, "ascii");
77
+ return block;
78
+ }
79
+
80
+ function padding(size) {
81
+ const rest = size % BLOCK;
82
+ return rest === 0 ? Buffer.alloc(0) : Buffer.alloc(BLOCK - rest, 0);
83
+ }
84
+
85
+ /** One PAX `path=` record; its decimal length prefix counts itself. */
86
+ function paxPathRecord(path) {
87
+ const body = ` path=${path}\n`;
88
+ let length = Buffer.byteLength(body) + 1;
89
+ while (String(length).length + Buffer.byteLength(body) !== length) length = String(length).length + Buffer.byteLength(body);
90
+ return Buffer.from(`${length}${body}`, "utf8");
91
+ }
92
+
93
+ function entryBlocks(name, opts, body = null) {
94
+ const blocks = [];
95
+ if (Buffer.byteLength(name) > 100) {
96
+ const pax = paxPathRecord(name);
97
+ blocks.push(header("PaxHeader/entry", { size: pax.length, type: "x" }), pax, padding(pax.length));
98
+ }
99
+ // An over-long name is carried by the PAX record; the ustar field keeps a
100
+ // deterministic truncated stand-in.
101
+ blocks.push(header(Buffer.from(name).subarray(0, 100).toString("utf8"), opts));
102
+ if (body) blocks.push(body, padding(body.length));
103
+ return blocks;
104
+ }
105
+
106
+ // ── tree walk ────────────────────────────────────────────────────────────────
107
+
108
+ function toPosix(rel) {
109
+ return sep === "/" ? rel : rel.split(sep).join("/");
110
+ }
111
+
112
+ /**
113
+ * Collect the bundle's entries in byte-wise sorted order. A symlink that resolves
114
+ * INSIDE the stack directory is stored as the file/dir it points at (the serving
115
+ * host's Docker bind mounts cannot follow a link anyway); one resolving outside
116
+ * it is refused, so a bundle can never carry a file from elsewhere on this machine.
117
+ */
118
+ function collect(root, rootReal, relDir, out, excluded, seenDirs = new Set([rootReal])) {
119
+ const absDir = relDir ? join(root, relDir) : root;
120
+ const names = readdirSync(absDir).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
121
+ for (const name of names) {
122
+ const rel = relDir ? `${relDir}/${name}` : name;
123
+ if (isExcludedBundlePath(rel)) { excluded.push(rel); continue; }
124
+ const abs = join(root, rel);
125
+ let info = lstatSync(abs);
126
+ if (info.isSymbolicLink()) {
127
+ let target;
128
+ try { target = realpathSync(abs); } catch { throw bundleError("BUNDLE_UNSAFE_LINK", `refusing to bundle ${rel}: it is a dangling symlink`); }
129
+ const inside = relative(rootReal, target);
130
+ if (inside === "" || inside === ".." || inside.startsWith(`..${sep}`) || isAbsolute(inside)) {
131
+ throw bundleError("BUNDLE_UNSAFE_LINK", `refusing to bundle ${rel}: the symlink resolves outside the stack directory (${target})`);
132
+ }
133
+ // Codex #1077 P1: the exclusion rules apply to what a link POINTS AT too, or an
134
+ // innocently named link (config-link -> .env) would ship an excluded secret.
135
+ if (isExcludedBundlePath(toPosix(inside))) { excluded.push(rel); continue; }
136
+ info = statSync(abs);
137
+ }
138
+ if (info.isDirectory()) {
139
+ // A directory reached twice (a symlink to an ancestor or a sibling) would loop
140
+ // or duplicate the tree; refuse it rather than guess which copy is meant.
141
+ const real = realpathSync(abs);
142
+ if (seenDirs.has(real)) throw bundleError("BUNDLE_UNSAFE_LINK", `refusing to bundle ${rel}: it is a symlink to a directory already in the bundle (a cycle or duplicate)`);
143
+ out.push({ rel: `${rel}/`, type: "dir" });
144
+ collect(root, rootReal, rel, out, excluded, new Set([...seenDirs, real]));
145
+ } else if (info.isFile()) {
146
+ out.push({ rel, type: "file", abs, size: info.size, executable: (info.mode & 0o111) !== 0 });
147
+ }
148
+ // sockets, fifos and devices are not stack inputs; skip them silently
149
+ }
150
+ }
151
+
152
+ /**
153
+ * Build the deterministic bundle of a materialised stack directory.
154
+ * @returns {{ bytes: Buffer, sha256: string, size: number, files: number, excluded: string[] }}
155
+ */
156
+ /**
157
+ * `extraFiles` ([{ rel, abs }]) adds files from OUTSIDE the stack dir at a fixed relative
158
+ * path, ahead of the tree (deterministic). BOT-2162: the worktree's `.botbuddy/config.json`
159
+ * — the BOT-2107 readiness declaration the serving Helper reads at
160
+ * `<bundle root>/.botbuddy/config.json` — so a remote BotBuddy lease runs the same
161
+ * canonical bind-RPC readiness probe as a local one. Each must be a regular file (not a
162
+ * symlink) with a safe relative name; one the stack dir already carries is not duplicated.
163
+ */
164
+ export function buildStackBundle(stackDir, { maxBytes = DEFAULT_BUNDLE_MAX_BYTES, rawLimitBytes = null, readFile = readFileSync, extraFiles = [] } = {}) {
165
+ const root = realpathSync(stackDir);
166
+ try {
167
+ if (!statSync(join(root, "supabase", "config.toml")).isFile()) throw new Error("not a file");
168
+ } catch {
169
+ throw bundleError("BUNDLE_NOT_A_STACK", `refusing to bundle ${root}: it has no supabase/config.toml, so it is not a materialised stack directory`);
170
+ }
171
+ const entries = [];
172
+ const excluded = [];
173
+ collect(root, root, "", entries, excluded);
174
+ const present = new Set(entries.map((e) => e.rel));
175
+ const extras = [];
176
+ for (const { rel, abs } of extraFiles) {
177
+ const parts = String(rel).split("/");
178
+ if (!rel || rel.startsWith("/") || parts.some((p) => p === "" || p === "." || p === "..") || isExcludedBundlePath(rel)) {
179
+ throw bundleError("BUNDLE_UNSAFE_LINK", `refusing to add ${rel} to the bundle: it is not a safe relative path`);
180
+ }
181
+ if (present.has(rel)) continue;
182
+ const info = lstatSync(abs);
183
+ if (!info.isFile()) throw bundleError("BUNDLE_UNSAFE_LINK", `refusing to add ${rel} to the bundle: ${abs} is not a regular file`);
184
+ for (let i = 1; i < parts.length; i++) {
185
+ const dir = `${parts.slice(0, i).join("/")}/`;
186
+ if (!present.has(dir)) { present.add(dir); extras.push({ rel: dir, type: "dir" }); }
187
+ }
188
+ present.add(rel);
189
+ extras.push({ rel, type: "file", abs, size: info.size, executable: false });
190
+ }
191
+ entries.unshift(...extras);
192
+ // Raw bytes are bounded too, so a pathological tree fails fast instead of
193
+ // exhausting memory before the compressed cap can be checked.
194
+ const rawLimit = rawLimitBytes ?? Math.max(maxBytes * 16, 64 * 1024 * 1024);
195
+ const chunks = [];
196
+ let raw = 0;
197
+ let files = 0;
198
+ const tooLarge = (what) => bundleError("BUNDLE_TOO_LARGE",
199
+ `the stack bundle for ${root} is ${what}, over the ${maxBytes} byte cap. A materialised stack is normally a few MB: ` +
200
+ "look for a large file copied into supabase/ (dumps, fixtures, a stray node_modules is already excluded), " +
201
+ "or raise the cap with --bundle-max-mb <n>.");
202
+ for (const entry of entries) {
203
+ const blocks = entry.type === "dir"
204
+ ? entryBlocks(toPosix(entry.rel), { mode: 0o755, type: "5" })
205
+ : (() => {
206
+ // Codex #1077 P2: refuse from the stat size BEFORE allocating the file, so one
207
+ // huge file is a BUNDLE_TOO_LARGE error, never an out-of-memory kill.
208
+ if (raw + entry.size > rawLimit) throw tooLarge(`more than ${rawLimit} bytes before compression (at ${entry.rel}, ${entry.size} bytes)`);
209
+ const body = readFile(entry.abs);
210
+ files += 1;
211
+ return entryBlocks(toPosix(entry.rel), { size: body.length, mode: entry.executable ? 0o755 : 0o644, type: "0" }, body);
212
+ })();
213
+ for (const block of blocks) { raw += block.length; chunks.push(block); }
214
+ if (raw > rawLimit) throw tooLarge(`more than ${rawLimit} bytes before compression`);
215
+ }
216
+ chunks.push(Buffer.alloc(BLOCK * 2, 0)); // end-of-archive marker
217
+ const bytes = gzipSync(Buffer.concat(chunks), { level: 9 });
218
+ bytes[9] = 3; // gzip OS byte: "Unix" on every platform, so the digest is portable
219
+ if (bytes.length > maxBytes) throw tooLarge(`${bytes.length} bytes compressed`);
220
+ return { bytes, sha256: createHash("sha256").update(bytes).digest("hex"), size: bytes.length, files, excluded };
221
+ }
222
+
223
+ /** Entry names in archive order (PAX long names resolved). For tests and diagnostics. */
224
+ export function bundleEntryNames(tgz) {
225
+ const tar = gunzipSync(tgz);
226
+ const names = [];
227
+ let pending = null;
228
+ for (let offset = 0; offset + BLOCK <= tar.length;) {
229
+ const block = tar.subarray(offset, offset + BLOCK);
230
+ if (block.every((byte) => byte === 0)) break;
231
+ const size = parseInt(block.subarray(124, 135).toString("ascii"), 8) || 0;
232
+ const type = String.fromCharCode(block[156]);
233
+ const body = tar.subarray(offset + BLOCK, offset + BLOCK + size);
234
+ if (type === "x") {
235
+ const match = /\d+ path=([^\n]*)\n/.exec(body.toString("utf8"));
236
+ pending = match ? match[1] : null;
237
+ } else {
238
+ names.push(pending ?? block.subarray(0, 100).toString("utf8").replace(/\0.*$/s, ""));
239
+ pending = null;
240
+ }
241
+ offset += BLOCK + Math.ceil(size / BLOCK) * BLOCK;
242
+ }
243
+ return names;
244
+ }
245
+
246
+ // ── upload ───────────────────────────────────────────────────────────────────
247
+
248
+ /** The signed upload URL must be https, or plain http only on loopback (a local
249
+ * control plane). Anything else is refused before a byte leaves this machine. */
250
+ export function assertBundleUploadUrl(url) {
251
+ let parsed;
252
+ try { parsed = new URL(String(url ?? "")); } catch { parsed = null; }
253
+ const loopback = parsed && ["127.0.0.1", "localhost", "[::1]"].includes(parsed.hostname);
254
+ if (!parsed || !(parsed.protocol === "https:" || (parsed.protocol === "http:" && loopback))) {
255
+ throw bundleError("BUNDLE_UPLOAD_URL_INVALID", "the control plane returned an unusable bundle_upload_url (it must be https, or http on loopback)");
256
+ }
257
+ return parsed;
258
+ }
259
+
260
+ /** Redact a URL's query (the signed token) from any text we might print. */
261
+ function redact(text, url) {
262
+ let out = String(text ?? "");
263
+ try {
264
+ const parsed = new URL(url);
265
+ if (parsed.search) out = out.split(parsed.search).join("?<redacted>");
266
+ for (const value of parsed.searchParams.values()) if (value) out = out.split(value).join("<redacted>");
267
+ } catch { /* nothing to redact */ }
268
+ return out;
269
+ }
270
+
271
+ /**
272
+ * PUT the bundle to the signed Storage upload URL (`stack-bundles/<tenant>/<lease>.tgz`).
273
+ * Never throws: returns { ok, status, error? }. The signed token never appears in the error.
274
+ */
275
+ export async function uploadStackBundle(url, bundle, { fetchImpl = fetch, signal = undefined } = {}) {
276
+ try {
277
+ assertBundleUploadUrl(url);
278
+ } catch (error) {
279
+ return { ok: false, status: null, error: error.message };
280
+ }
281
+ let response;
282
+ try {
283
+ response = await fetchImpl(url, {
284
+ method: "PUT",
285
+ headers: { "content-type": "application/gzip", "x-upsert": "false" },
286
+ body: bundle.bytes,
287
+ ...(signal ? { signal } : {}),
288
+ });
289
+ } catch (error) {
290
+ return { ok: false, status: null, error: `bundle upload failed: ${redact(error?.message ?? error, url)}` };
291
+ }
292
+ if (response?.ok) return { ok: true, status: response.status };
293
+ let detail = "";
294
+ try { detail = (await response.text()).slice(0, 200); } catch { /* no body */ }
295
+ return { ok: false, status: response?.status ?? null, error: `bundle upload was refused (HTTP ${response?.status ?? "?"})${detail ? `: ${redact(detail, url)}` : ""}` };
296
+ }
@@ -1408,6 +1408,10 @@ export function materializeRunStackDir(cwd, { slot, ticket, nonce, idleTtlSecs }
1408
1408
  copy(sourceDir, join(stackDir, "supabase"), {
1409
1409
  recursive: true,
1410
1410
  force: true,
1411
+ // BOT-2142: keep relative symlinks relative (cpSync's default rewrites them to
1412
+ // absolute links INTO the source worktree, which a remote bundle must refuse —
1413
+ // Supply Guard's supabase/CLAUDE.md -> AGENTS.md).
1414
+ verbatimSymlinks: true,
1411
1415
  filter: (source) => !SKIP_ENTRIES.has(basename(source)),
1412
1416
  });
1413
1417
  write(join(stackDir, "supabase", "config.toml"), text);
@@ -0,0 +1,322 @@
1
+ // BOT-2142 — SSH port forwards for a remote stack lease.
2
+ //
3
+ // A remote lease (BOT-2145) is served by another host the caller's owner owns. Its
4
+ // stack publishes on that host, and the control plane advertises ONLY loopback
5
+ // ports plus the host's operator-set `ssh_target` (BOT-2146: a LAN address is
6
+ // never returned). The caller reaches it the way the BOT-2021 spine proved: one
7
+ // `ssh -N` carrying a `-L` per stack port (+29 ms per new connection), and — when
8
+ // the caller runs a callback listener the stack's edge functions call through
9
+ // `host.docker.internal` (Supply Guard's TLS probe / PDF worker on 4600) — a
10
+ // reverse `-R <port>:127.0.0.1:<port>`.
11
+ //
12
+ // Contract:
13
+ // * every forward binds 127.0.0.1 on BOTH ends; nothing leaves loopback;
14
+ // * a free local port keeps the remote number, a busy one gets an ephemeral
15
+ // port (the remap table is returned and lands on the receipt);
16
+ // * `ExitOnForwardFailure=yes` makes a lost bind race a clean exit, retried on
17
+ // remapped ports; `BatchMode=yes` means ssh never prompts (no agent hang);
18
+ // * `ServerAlive*` turns a dead link (caller asleep, BOT-1980) into an exit the
19
+ // CLI sees as `tunnel_lost`, instead of a hang the child reads as product
20
+ // failures (BB hop 12 of the spine).
21
+ //
22
+ // Dependency-free (Node stdlib only).
23
+
24
+ import { spawn, spawnSync } from "node:child_process";
25
+ import { connect, createServer } from "node:net";
26
+
27
+ export const SSH_FORWARD_OPTIONS = Object.freeze([
28
+ "-N", "-T",
29
+ "-o", "BatchMode=yes",
30
+ "-o", "ExitOnForwardFailure=yes",
31
+ "-o", "ServerAliveInterval=15",
32
+ "-o", "ServerAliveCountMax=3",
33
+ // A user's ControlMaster config would multiplex the forwards through a master
34
+ // this process does not own, so its exit could no longer mean "tunnel lost".
35
+ "-o", "ControlMaster=no",
36
+ "-o", "ControlPath=none",
37
+ ]);
38
+
39
+ export const DEFAULT_TUNNEL_READY_MS = 20_000;
40
+ const LOOPBACK = new Set(["127.0.0.1", "localhost", "[::1]", "::1"]);
41
+
42
+ function tunnelError(code, message, extra = {}) {
43
+ const error = new Error(message);
44
+ error.code = code;
45
+ Object.assign(error, extra);
46
+ return error;
47
+ }
48
+
49
+ /** An ssh destination we are willing to put on argv: an alias or [user@]host, or
50
+ * ssh://[user@]host[:port] — never something ssh could read as an option or a shell could
51
+ * split. A port is accepted ONLY in the ssh:// form: OpenSSH reads `mini:2222` as a host
52
+ * literally named "mini:2222" on port 22 (Codex #1077 P2). */
53
+ const SSH_USER = "[A-Za-z0-9_][A-Za-z0-9._%+-]*";
54
+ const SSH_HOST = "(?:[A-Za-z0-9_][A-Za-z0-9._%-]*|\\[[0-9A-Fa-f:.%a-z]+\\])";
55
+ const SSH_TARGET_RE = new RegExp(`^(?:(?:${SSH_USER}@)?${SSH_HOST}|ssh://(?:${SSH_USER}@)?${SSH_HOST}(?::\\d{1,5})?)$`);
56
+
57
+ /**
58
+ * The serving host's SSH destination, validated. Missing → REMOTE_UNREACHABLE with
59
+ * the operator instruction (the CLI cannot invent one; BOT-2142 edge case).
60
+ */
61
+ export function assertSshTarget(target, hostKey = null) {
62
+ const host = hostKey ? `serving host ${hostKey}` : "the serving host";
63
+ if (target == null || target === "") {
64
+ throw tunnelError("REMOTE_UNREACHABLE",
65
+ `${host} has no ssh_target, so this machine cannot reach its stack. An operator must set the host's SSH destination ` +
66
+ "(container_hosts.ssh_target, e.g. the `ssh mini` alias this machine already trusts; BOT-2174) — or re-run with --local-only.");
67
+ }
68
+ if (typeof target !== "string" || !SSH_TARGET_RE.test(target)) {
69
+ throw tunnelError("REMOTE_UNREACHABLE", `${host} advertises an ssh_target that is not a plain ssh destination ([user@]host, or ` +
70
+ "ssh://[user@]host:port when it needs a port); refusing to pass it to ssh");
71
+ }
72
+ return target;
73
+ }
74
+
75
+ function validPort(value) {
76
+ const n = Number(value);
77
+ return Number.isInteger(n) && n > 0 && n <= 65535 ? n : null;
78
+ }
79
+
80
+ function urlPort(url) {
81
+ if (typeof url !== "string") return null;
82
+ const match = /^[a-z][a-z0-9+.-]*:\/\/(?:[^@/]*@)?(?:\[[^\]]+\]|[^:/?#]+):(\d+)/i.exec(url.trim());
83
+ return match ? validPort(match[1]) : null;
84
+ }
85
+
86
+ /** Every stack port the child may touch: api + db always, then `connection.ports[]`. */
87
+ export function remotePortsFromConnection(connection) {
88
+ const ports = [];
89
+ const add = (name, raw) => {
90
+ const port = validPort(raw);
91
+ if (port == null) throw tunnelError("REMOTE_CONNECTION_INVALID", `the remote lease connection has an invalid port for ${name}: ${raw}`);
92
+ if (!ports.some((p) => p.port === port)) ports.push({ name, port });
93
+ };
94
+ const api = urlPort(connection?.api_url);
95
+ if (api == null) throw tunnelError("REMOTE_CONNECTION_INVALID", "the remote lease connection has no api_url with an explicit port");
96
+ add("api", api);
97
+ const db = validPort(connection?.db_port) ?? urlPort(connection?.db_url);
98
+ if (db != null) add("db", db);
99
+ for (const entry of Array.isArray(connection?.ports) ? connection.ports : []) {
100
+ add(String(entry?.name ?? "port"), entry?.port);
101
+ }
102
+ return ports;
103
+ }
104
+
105
+ /** True when 127.0.0.1:<port> can be bound right now. */
106
+ export function isLocalPortFree(port) {
107
+ return new Promise((resolve) => {
108
+ const server = createServer();
109
+ server.once("error", () => resolve(false));
110
+ server.listen({ host: "127.0.0.1", port, exclusive: true }, () => server.close(() => resolve(true)));
111
+ });
112
+ }
113
+
114
+ /** An ephemeral port the kernel considers free on 127.0.0.1. */
115
+ export function pickFreeLocalPort() {
116
+ return new Promise((resolve, reject) => {
117
+ const server = createServer();
118
+ server.once("error", reject);
119
+ server.listen({ host: "127.0.0.1", port: 0, exclusive: true }, () => {
120
+ const { port } = server.address();
121
+ server.close(() => resolve(port));
122
+ });
123
+ });
124
+ }
125
+
126
+ /**
127
+ * Decide each forward's local port: the remote number when it is free (and not in
128
+ * `avoid`), otherwise a fresh ephemeral port. Returns the remap table.
129
+ */
130
+ export async function planForwards(remotePorts, { isFree = isLocalPortFree, pickFree = pickFreeLocalPort, avoid = new Set() } = {}) {
131
+ const used = new Set();
132
+ const plan = [];
133
+ for (const { name, port } of remotePorts) {
134
+ let local = null;
135
+ if (!avoid.has(port) && !used.has(port) && await isFree(port)) local = port;
136
+ for (let tries = 0; local == null && tries < 20; tries++) {
137
+ const candidate = await pickFree();
138
+ if (!used.has(candidate) && !avoid.has(candidate)) local = candidate;
139
+ }
140
+ if (local == null) throw tunnelError("REMOTE_UNREACHABLE", `could not find a free local port to forward ${name} (${port})`);
141
+ used.add(local);
142
+ plan.push({ name, remote: port, local, remapped: local !== port });
143
+ }
144
+ return plan;
145
+ }
146
+
147
+ /** `ssh` argv (without the binary) for the planned forwards. */
148
+ export function buildSshForwardArgv({ sshTarget, forwards, callbackPort = null }) {
149
+ const argv = [...SSH_FORWARD_OPTIONS];
150
+ for (const f of forwards) argv.push("-L", `127.0.0.1:${f.local}:127.0.0.1:${f.remote}`);
151
+ if (callbackPort != null) argv.push("-R", `${callbackPort}:127.0.0.1:${callbackPort}`);
152
+ argv.push("--", sshTarget);
153
+ return argv;
154
+ }
155
+
156
+ /**
157
+ * The connection the CHILD sees: every loopback URL/port the forwards cover is
158
+ * rewritten to its local port. project_id, keys and everything else are kept.
159
+ * The server's object is not mutated.
160
+ */
161
+ export function rewriteConnectionForForwards(connection, forwards) {
162
+ const byRemote = new Map(forwards.map((f) => [f.remote, f.local]));
163
+ const rewriteUrl = (value) => value.replace(
164
+ /^([a-z][a-z0-9+.-]*:\/\/(?:[^@/]*@)?)(127\.0\.0\.1|localhost|\[::1\]):(\d+)/i,
165
+ (whole, prefix, _host, port) => (byRemote.has(Number(port)) ? `${prefix}127.0.0.1:${byRemote.get(Number(port))}` : whole),
166
+ );
167
+ const out = {};
168
+ for (const [key, value] of Object.entries(connection ?? {})) {
169
+ out[key] = typeof value === "string" ? rewriteUrl(value) : value;
170
+ }
171
+ const db = validPort(connection?.db_port);
172
+ if (db != null && byRemote.has(db)) out.db_port = byRemote.get(db);
173
+ out.ports = forwards.map((f) => ({ name: f.name, port: f.local, remote_port: f.remote }));
174
+ out.transport = "ssh-forward";
175
+ return out;
176
+ }
177
+
178
+ function probeConnect(port, timeoutMs = 500) {
179
+ return new Promise((resolve) => {
180
+ const socket = connect({ host: "127.0.0.1", port });
181
+ const done = (ok) => { socket.destroy(); resolve(ok); };
182
+ socket.setTimeout(timeoutMs, () => done(false));
183
+ socket.once("connect", () => done(true));
184
+ socket.once("error", () => done(false));
185
+ });
186
+ }
187
+
188
+ const BIND_RACE = /address already in use|cannot listen|could not request local forwarding/i;
189
+
190
+ /**
191
+ * Spawn ONE ssh carrying every forward and resolve once each local listener accepts.
192
+ * Rejects (killing ssh) if ssh exits first or readiness times out.
193
+ */
194
+ export async function openSshForwards({
195
+ sshBin = "ssh", argv, forwards, sshTarget, readyTimeoutMs = DEFAULT_TUNNEL_READY_MS,
196
+ spawnImpl = spawn, env = process.env, connectProbe = probeConnect, sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
197
+ }) {
198
+ // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process -- fixed ssh argv; target validated by assertSshTarget; no shell.
199
+ const child = spawnImpl(sshBin, argv, { stdio: ["ignore", "ignore", "pipe"], env });
200
+ let stderr = "";
201
+ child.stderr?.on("data", (chunk) => { stderr = (stderr + chunk.toString("utf8")).slice(-2048); });
202
+ let exited = null;
203
+ let closing = false;
204
+ const listeners = [];
205
+ const exitPromise = new Promise((resolve) => {
206
+ const finish = (code, signal) => {
207
+ if (exited) return;
208
+ exited = { code: code ?? null, signal: signal ?? null };
209
+ resolve(exited);
210
+ if (!closing) for (const fn of listeners) fn({ ...exited, stderr: stderr.trim() });
211
+ };
212
+ child.once("exit", finish);
213
+ child.once("error", (error) => { stderr += `\n${error.message}`; finish(null, null); });
214
+ });
215
+
216
+ const close = async () => {
217
+ if (exited) return exited;
218
+ closing = true;
219
+ try { child.kill("SIGTERM"); } catch { /* gone */ }
220
+ const timer = setTimeout(() => { try { child.kill("SIGKILL"); } catch { /* gone */ } }, 3_000);
221
+ timer.unref?.();
222
+ const result = await exitPromise;
223
+ clearTimeout(timer);
224
+ return result;
225
+ };
226
+
227
+ const deadline = Date.now() + readyTimeoutMs;
228
+ const pending = new Set(forwards.map((f) => f.local));
229
+ while (pending.size) {
230
+ if (exited) break;
231
+ for (const port of [...pending]) if (await connectProbe(port)) pending.delete(port);
232
+ if (!pending.size || exited) break;
233
+ if (Date.now() > deadline) break;
234
+ await sleep(100);
235
+ }
236
+ if (exited || pending.size) {
237
+ const timedOut = !exited;
238
+ await close();
239
+ const tail = stderr.trim();
240
+ throw tunnelError("REMOTE_UNREACHABLE",
241
+ `ssh ${sshTarget} could not open the stack forwards${timedOut ? ` within ${Math.round(readyTimeoutMs / 1000)}s` : ` (exit ${exited?.code ?? "?"})`}` +
242
+ `${tail ? `: ${tail}` : ""}. Check that \`ssh ${sshTarget}\` works non-interactively from this machine (BatchMode: a key, no password prompt).`,
243
+ { bindRace: BIND_RACE.test(tail), sshStderr: tail });
244
+ }
245
+ return {
246
+ pid: child.pid,
247
+ argv,
248
+ exited: exitPromise,
249
+ onExit: (fn) => { listeners.push(fn); if (exited && !closing) fn({ ...exited, stderr: stderr.trim() }); },
250
+ stderrTail: () => stderr.trim(),
251
+ close,
252
+ };
253
+ }
254
+
255
+ /**
256
+ * Codex #1077 P1: `-R` asks for a loopback listener on the host, but an sshd with
257
+ * `GatewayPorts yes` silently binds it on every interface, exposing this machine's
258
+ * callback service to the host's network. The CLI cannot read sshd_config, so after the
259
+ * forward is up it asks the host which address the listener really holds and refuses
260
+ * anything but loopback (fail closed: no lsof, no listener, unreadable → refused).
261
+ */
262
+ export function verifyCallbackLoopback({ sshBin, sshTarget, port, env = process.env, run = spawnSync }) {
263
+ const fail = (why) => tunnelError("CALLBACK_NOT_LOOPBACK",
264
+ `the reverse forward for --callback-port ${port} on ${sshTarget} could not be proven loopback-only (${why}). ` +
265
+ "An sshd with GatewayPorts yes binds -R forwards on every interface; set GatewayPorts no (or clientspecified) on the host, " +
266
+ "or run without --callback-port.");
267
+ // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process -- fixed argv; target validated; integer port.
268
+ const r = run(sshBin, [
269
+ "-T", "-o", "BatchMode=yes", "-o", "ControlMaster=no", "-o", "ControlPath=none", "--", sshTarget,
270
+ `lsof -nP -iTCP:${Number(port)} -sTCP:LISTEN -Fn`,
271
+ ], { encoding: "utf8", env, timeout: 15_000 });
272
+ if (r.error || r.status !== 0) {
273
+ const tail = String(r.stderr || r.error?.message || "").trim().split("\n").slice(-1)[0];
274
+ // lsof exits 1 with no output when nothing listens.
275
+ if (r.status === 1 && !String(r.stdout || "").trim()) throw fail(`no listener on the host's port ${port}`);
276
+ throw fail(`could not run lsof on the host (exit ${r.status ?? "?"}${tail ? `: ${tail}` : ""})`);
277
+ }
278
+ const addresses = String(r.stdout || "").split("\n").filter((l) => l.startsWith("n")).map((l) => l.slice(1).trim());
279
+ if (!addresses.length) throw fail(`no listener on the host's port ${port}`);
280
+ const exposed = addresses.filter((a) => !(a.startsWith("127.") || a.startsWith("[::1]:") || a.startsWith("localhost:")));
281
+ if (exposed.length) throw fail(`the host listens on ${exposed.join(", ")}`);
282
+ return addresses;
283
+ }
284
+
285
+ /**
286
+ * Plan, spawn and verify the forwards for an ACTIVE remote lease.
287
+ * @returns {{ forwards, connection, argv, attempts, onExit, close, pid }}
288
+ */
289
+ export async function establishSshTunnel({
290
+ connection, sshTarget, hostKey = null, callbackPort = null, sshBin = null, env = process.env,
291
+ attempts = 3, readyTimeoutMs = DEFAULT_TUNNEL_READY_MS, isFree = isLocalPortFree, pickFree = pickFreeLocalPort, spawnImpl = spawn,
292
+ runRemote = spawnSync,
293
+ }) {
294
+ const target = assertSshTarget(sshTarget, hostKey);
295
+ const remotePorts = remotePortsFromConnection(connection);
296
+ const bin = sshBin || env.BOTBUDDY_STACK_SSH_BIN || "ssh";
297
+ let avoid = new Set();
298
+ let lastError = null;
299
+ for (let attempt = 1; attempt <= attempts; attempt++) {
300
+ const forwards = await planForwards(remotePorts, { isFree, pickFree, avoid });
301
+ const argv = buildSshForwardArgv({ sshTarget: target, forwards, callbackPort });
302
+ try {
303
+ const handle = await openSshForwards({ sshBin: bin, argv, forwards, sshTarget: target, readyTimeoutMs, spawnImpl, env });
304
+ if (callbackPort != null) {
305
+ try {
306
+ verifyCallbackLoopback({ sshBin: bin, sshTarget: target, port: callbackPort, env, run: runRemote });
307
+ } catch (error) {
308
+ await handle.close();
309
+ throw error;
310
+ }
311
+ }
312
+ return { ...handle, forwards, connection: rewriteConnectionForForwards(connection, forwards), attempts: attempt };
313
+ } catch (error) {
314
+ lastError = error;
315
+ // Only a lost local bind race is worth another try, on fresh ports; an auth or
316
+ // host failure would fail the same way again.
317
+ if (!error?.bindRace) throw error;
318
+ avoid = new Set([...avoid, ...forwards.map((f) => f.local), ...remotePorts.map((p) => p.port)]);
319
+ }
320
+ }
321
+ throw lastError;
322
+ }
package/src/stack.mjs CHANGED
@@ -23,7 +23,7 @@
23
23
 
24
24
  import { writeLaneStatus } from "./lane-status.mjs";
25
25
  import { spawn, spawnSync } from "child_process";
26
- import { readFileSync, realpathSync } from "fs";
26
+ import { existsSync, lstatSync, readFileSync, realpathSync } from "fs";
27
27
  import { open, unlink, readFile } from "fs/promises";
28
28
  import { homedir, tmpdir } from "os";
29
29
  import { basename, dirname, join, relative, isAbsolute } from "path";
@@ -39,10 +39,25 @@ import { runDockerCommand, runDockerWorkflow, ADMITTED_DOCKER_VALIDATIONS } from
39
39
  import { acquireStackLock, lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
40
40
  import { assertLeasedStackIsolated, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
41
41
  import { machineUuid } from "./machine-id.mjs";
42
+ import { DEFAULT_BUNDLE_MAX_BYTES, buildStackBundle, uploadStackBundle } from "./stack-bundle.mjs";
43
+ import { assertSshTarget, establishSshTunnel } from "./stack-tunnel.mjs";
42
44
  import { ENGINE_PUBLISHES_BEYOND_LOOPBACK, engineSocketIsLocal, formatPublishIp, isLoopbackIp, publishIpFix, resolvePublishIp } from "./docker-publish-ip.mjs";
43
45
  import { bold, dim, yellow } from "./utils.mjs";
44
46
 
45
47
  export const STACK_SCHEMA_VERSION = 1;
48
+
49
+ /**
50
+ * Codex #1077 P1: an sshd with `GatewayPorts yes` binds a `-R` forward on every interface,
51
+ * and the post-forward lsof check (`verifyCallbackLoopback`) runs only AFTER the listener
52
+ * exists — so the caller's callback service could be reachable from the host's network for
53
+ * a few seconds. Until a pre-forward probe lands (BOT-2184), a REMOTE lease refuses
54
+ * --callback-port outright (fail closed). Co-located leases are unaffected.
55
+ */
56
+ export const CALLBACK_PORT_UNSUPPORTED_REMOTE = "CALLBACK_PORT_UNSUPPORTED_REMOTE";
57
+ const CALLBACK_PORT_REMOTE_REFUSAL =
58
+ `${CALLBACK_PORT_UNSUPPORTED_REMOTE}: --callback-port is not supported on a remote lease yet — the -R listener ` +
59
+ "could be exposed on the serving host's network (sshd GatewayPorts yes) before it is checked. Tracked in BOT-2184 " +
60
+ "(pre-forward GatewayPorts probe). Use --local-only for a run that needs a callback port.";
46
61
  export const DEFAULT_RECEIPT_MAX_BYTES = 10240;
47
62
  export const DEFAULT_TIMEOUT_SEC = 3600;
48
63
  /** How often a run renews its stack directory's retention while it still exists
@@ -102,6 +117,9 @@ ${bold("up OPTIONS")}
102
117
  --docker-context <name> Explicit Docker context for the mandatory local preflight
103
118
  (an allowlisted engine: OrbStack or Docker Desktop, e.g. desktop-linux).
104
119
  --docker-endpoint <uri> Explicit Docker endpoint instead of --docker-context.
120
+ --local-only Always the case for up: it asks for a co-located host (execution "local").
121
+ up exits once the lease is active, so it cannot hold a remote lease's
122
+ SSH forwards; --remote is refused here — use \`stack run --remote\`.
105
123
 
106
124
  ${bold("done OPTIONS")}
107
125
  --stop Keep volumes (cheap re-provision next batch). Default: destroy.
@@ -126,6 +144,22 @@ ${bold("run OPTIONS")}
126
144
  --hard-ttl N Bound the child runtime; timeout requests normal cleanup.
127
145
  --local-exec Rejected: stack run never starts Docker directly.
128
146
  --wait-capacity <dur> Same as up: park a TRANSIENT stale-capacity refusal, then re-admit once.
147
+ --remote Require a REMOTE lease (BOT-2142): another host this owner owns serves the
148
+ stack. run uploads the materialised stack as a digest-pinned bundle, waits
149
+ for active, forwards every stack port over one \`ssh -N -L\` to the host's
150
+ ssh_target (transport ssh-forward), and gives the child a connection file
151
+ on the LOCAL ports (same numbers when free, else remapped; see "forwards").
152
+ --local-only Require a co-located host (execution "local"); no bundle is built.
153
+ Default (neither flag): execution "auto" — co-located if it admits now,
154
+ else a remote host with capacity; the bundle digest is sent up front.
155
+ --callback-port N The caller runs a listener the stack calls back through
156
+ host.docker.internal:N (e.g. 4600): sent as callback_port. Co-located
157
+ leases only for now: a REMOTE lease refuses it (exit 4
158
+ CALLBACK_PORT_UNSUPPORTED_REMOTE) until the pre-forward GatewayPorts
159
+ probe lands (BOT-2184).
160
+ --bundle-max-mb N Remote-lease bundle size cap in MB (default 50, compressed). The bundle
161
+ excludes .bb-stack.json, node_modules, supabase/.temp|.branches and
162
+ untracked-secret dotenv files (.env, .env.local, .env.*.local).
129
163
 
130
164
  ${bold("GLOBAL")}
131
165
  --json Pretty-print the receipt (default is one compact JSON line).
@@ -136,6 +170,28 @@ ${bold("EXIT CODES")}
136
170
  4 invalid arguments 5 backend/coordination 6 lease failed/reaped
137
171
  7 internal error 8 cleanup/reap proof failed
138
172
 
173
+ ${bold("REMOTE LEASES")} (BOT-2142, stack run)
174
+ The receipt carries execution_mode (local|remote), execution_host_key, requester_host,
175
+ transport (none|ssh-forward), and for a remote lease forwards [{name, remote, local}] + bundle.
176
+ The child also gets BOTBUDDY_STACK_EXECUTION_MODE and BOTBUDDY_STACK_TRANSPORT.
177
+ 5 REMOTE_UNREACHABLE the host has no ssh_target (an operator must set it), or
178
+ \`ssh <target>\` could not open the forwards (BatchMode: key auth only)
179
+ 5 REMOTE_UNSUPPORTED --remote, but the control plane placed the lease locally (pre-remote server)
180
+ 5 HOST_NO_SSH_TARGET the (pinned) remote host has no ssh_target — operator: set container_hosts.ssh_target
181
+ 5 HOST_HELPER_OUTDATED the (pinned) remote host's Helper predates protocol 4 — upgrade that Helper
182
+ 5 NO_HELPER_HOST no co-located host admits and no owned host qualifies as remote
183
+ 4 STACK_EXECUTION_MODE_MISMATCH the slot already holds a lease in another mode/host — stack done it first
184
+ (the four codes above create NO lease)
185
+ 5 BUNDLE_UPLOAD_FAILED the signed bundle upload failed (the lease is released first)
186
+ 5 CALLBACK_NOT_LOOPBACK (reserved for BOT-2184) the host's -R listener is not loopback-only
187
+ (sshd GatewayPorts yes) or could not be checked with lsof
188
+ 4 BUNDLE_TOO_LARGE --remote and the bundle is over --bundle-max-mb
189
+ 4 CALLBACK_PORT_UNSUPPORTED_REMOTE --callback-port on a remote lease (refused until BOT-2184;
190
+ under auto the remote lease is released first)
191
+ 6 TUNNEL_LOST the forward exited mid-child: the child is stopped, the lease released,
192
+ park_reason tunnel_lost (its failures were transport, not product)
193
+ Every remote failure after the lease exists still runs \`stack done\` and closes the forwards.
194
+
139
195
  ${bold("HOST_CAPACITY_SOURCE_STALE")} (BOT-2024)
140
196
  HOST_CAPACITY_SOURCE_STALE is a capacity flag, not a Helper diagnosis. Read \`phase\` on the
141
197
  receipt. transient → arm the printed \`bb wait 'capacity:…'\` and re-run once it fires
@@ -196,6 +252,8 @@ export function parseStackArgs(argv) {
196
252
  timeout: DEFAULT_TIMEOUT_SEC, provisionTimeout: null, reapTimeout: 300, hardTtl: null,
197
253
  connectionFile: null, noWait: false, claimOnly: false, localExec: false, waitCapacity: null,
198
254
  dockerContext: null, dockerEndpoint: null,
255
+ // BOT-2142: execution placement (null = command default), remote-lease callback + bundle cap.
256
+ execution: null, remote: false, localOnly: false, callbackPort: null, bundleMaxBytes: null,
199
257
  disposition: "destroy", json: false, receiptMaxBytes: DEFAULT_RECEIPT_MAX_BYTES,
200
258
  };
201
259
  const positionals = [];
@@ -275,6 +333,24 @@ export function parseStackArgs(argv) {
275
333
  }
276
334
  break;
277
335
  }
336
+ case "--remote": opts.remote = true; break;
337
+ case "--local-only": opts.localOnly = true; break;
338
+ case "--callback-port": {
339
+ if (need(a, rest[i + 1])) {
340
+ const n = Number(rest[++i]);
341
+ if (!Number.isInteger(n) || n <= 0 || n > 65535) errors.push("--callback-port must be an integer port (1-65535)");
342
+ else opts.callbackPort = n;
343
+ }
344
+ break;
345
+ }
346
+ case "--bundle-max-mb": {
347
+ if (need(a, rest[i + 1])) {
348
+ const n = Number(rest[++i]);
349
+ if (!Number.isInteger(n) || n <= 0 || n > 1024) errors.push("--bundle-max-mb must be an integer number of MB (1-1024)");
350
+ else opts.bundleMaxBytes = n * 1024 * 1024;
351
+ }
352
+ break;
353
+ }
278
354
  case "--no-wait": opts.noWait = true; break;
279
355
  case "--claim-only": opts.claimOnly = true; break;
280
356
  case "--local-exec": opts.localExec = true; break;
@@ -288,6 +364,24 @@ export function parseStackArgs(argv) {
288
364
  else positionals.push(a);
289
365
  }
290
366
  }
367
+ // BOT-2142: where the lease may be served. `run` defaults to the server's `auto`;
368
+ // `up` exits once the lease is active, so it can never hold a remote lease's SSH
369
+ // forwards and always asks for a co-located host.
370
+ if (opts.remote && opts.localOnly) errors.push("--remote and --local-only are mutually exclusive");
371
+ if (opts.remote && opts.localExec) {
372
+ errors.push("--remote cannot be combined with --local-exec: a local-exec stack is started on THIS machine");
373
+ }
374
+ if ((opts.remote || opts.localOnly) && !["up", "run"].includes(command)) {
375
+ errors.push("--remote and --local-only are valid only with `stack up` or `stack run`");
376
+ } else if (command === "up" && opts.remote) {
377
+ errors.push("stack up cannot hold a remote lease's SSH forwards (it exits once the lease is active); use `stack run --remote -- <command>`");
378
+ }
379
+ if ((opts.callbackPort != null || opts.bundleMaxBytes != null) && command !== "run") {
380
+ errors.push("--callback-port and --bundle-max-mb are valid only with `stack run`");
381
+ } else if (opts.remote && opts.callbackPort != null) {
382
+ errors.push(CALLBACK_PORT_REMOTE_REFUSAL);
383
+ }
384
+ opts.execution = opts.remote ? "remote" : opts.localOnly ? "local" : null;
291
385
  if (typeof opts.stackPath !== "string" || !opts.stackPath.trim() || opts.stackPath.startsWith("/") ||
292
386
  opts.stackPath.split("/").some((part) => part === "..")) {
293
387
  errors.push("--stack-path must be a non-empty relative path without '..'");
@@ -787,9 +881,35 @@ export function leaseRefusalExit(code) {
787
881
  return code === STACK_PATH_ROOT_REFUSED_CODE || code === "STACK_EXECUTION_MODE_MISMATCH" ? EXIT.INVALID : EXIT.BACKEND;
788
882
  }
789
883
 
790
- /** The refusal text for a receipt: the server's message, else its code. */
884
+ /**
885
+ * BOT-2142: the remote-lease host-selection refusals of BOT-2145 (#1087) arrive as a bare
886
+ * `{ success: false, code }`. Name each one with the fix, so an agent never has to guess
887
+ * what a code means. Every one of them created NO lease.
888
+ */
889
+ export const REMOTE_REFUSAL_GUIDANCE = Object.freeze({
890
+ HOST_NO_SSH_TARGET: "the remote host has no ssh_target, so this machine cannot forward to its stack. An operator must set " +
891
+ "container_hosts.ssh_target for that host (e.g. the `ssh mini` alias this machine already trusts; BOT-2174). " +
892
+ "No lease was created; re-run with --local-only meanwhile.",
893
+ HOST_HELPER_OUTDATED: "the remote host's Helper is too old to serve a remote lease (it must speak protocol 4, which " +
894
+ "provisions from the uploaded bundle; BOT-2143). Upgrade the Helper on that host. No lease was created; re-run with --local-only meanwhile.",
895
+ NO_HELPER_HOST: "no host can serve this lease: no co-located Helper admits it and no other host you own qualifies as remote " +
896
+ "(an owned, device-consented, beacon-fresh Helper on protocol 4 with an operator-set ssh_target). No lease was created.",
897
+ HOST_NOT_OWNED: "the pinned host is not a device-consented Helper host your owner owns. No lease was created.",
898
+ HOST_STALE: "the pinned remote host's Helper beacon is stale. No lease was created; retry once it beacons, or drop --host.",
899
+ HOST_NOT_REMOTE: "--remote pinned this machine's own host; drop --host or --remote. No lease was created.",
900
+ BUNDLE_REQUIRED: "the server picked a remote host but this request carried no bundle digest. No lease was created; " +
901
+ "re-run (the bundle is built automatically unless it failed — see the warning above) or use --local-only.",
902
+ });
903
+
904
+ /** The refusal text for a receipt: the server's message, else the CLI's guidance, else the code. */
791
905
  export function leaseRefusalError(data) {
792
- return data?.message || data?.code || "lease request refused";
906
+ return data?.message || REMOTE_REFUSAL_GUIDANCE[data?.code] || data?.code || "lease request refused";
907
+ }
908
+
909
+ /** One stderr line naming a remote-lease refusal (null for every other code). */
910
+ export function remoteRefusalNotice(code) {
911
+ const guidance = REMOTE_REFUSAL_GUIDANCE[code];
912
+ return guidance ? `${yellow("⚠")} stack: ${code}: ${guidance}` : null;
793
913
  }
794
914
 
795
915
  // ── BOT-2024: phase-aware HOST_CAPACITY_SOURCE_STALE ─────────────────────────
@@ -1823,6 +1943,9 @@ export async function cmdUp(opts, {
1823
1943
  stack_path: execution.stackPath,
1824
1944
  worktree_root: execution.worktreeRoot,
1825
1945
  machine_uuid: hardwareUuid,
1946
+ // BOT-2142: `up` returns once the lease is active and cannot keep SSH forwards
1947
+ // open for a remote host, so it always asks for a co-located one (BOT-2145).
1948
+ execution: "local",
1826
1949
  // BOT-1926: fence the provision job from the Helper server-side. Without it a
1827
1950
  // polling Helper wins the request→reserve race, provisions the stack itself and
1828
1951
  // records a targetless connection that `done --local-exec` cannot release.
@@ -2511,6 +2634,22 @@ export async function materializeLeasedTestConfig(worktreeRoot, leaseId, connect
2511
2634
  return { envFile, stackConfig, edgeMountRoot };
2512
2635
  }
2513
2636
 
2637
+ /**
2638
+ * BOT-2142: the SSH forward to a remote lease's host exited while the child ran. The
2639
+ * child was stopped (its failures would otherwise read as product failures, BOT-2021
2640
+ * spine hop 12) and the lease released; the outcome is a lost lease, exit 6, with
2641
+ * the BOT-1754-style `park_reason`.
2642
+ */
2643
+ function tunnelLostResult(lost, { leaseId, cleanup, childExitCode, ...rest }) {
2644
+ const how = lost.signal ? `signal ${lost.signal}` : `exit ${lost.code ?? "?"}`;
2645
+ return {
2646
+ exitCode: EXIT.LEASE_FAILED, outcome: "failed", code: "TUNNEL_LOST", parkReason: "tunnel_lost", leaseId, childExitCode, cleanup,
2647
+ error: `the SSH forward to ${lost.target ?? "the serving host"} ended (${how}) while the child was running, so the child was stopped and the lease released` +
2648
+ `${lost.stderr ? ` — ssh: ${lost.stderr.split("\n").slice(-2).join(" ")}` : ""}. Re-run the batch; if it repeats, check the link to the host.`,
2649
+ ...rest,
2650
+ };
2651
+ }
2652
+
2514
2653
  /**
2515
2654
  * The command-owned lifecycle. The optional adapters are intentionally narrow so
2516
2655
  * its queue, heartbeat, process-group, and cleanup contract is spawn-testable
@@ -2519,9 +2658,11 @@ export async function materializeLeasedTestConfig(worktreeRoot, leaseId, connect
2519
2658
  export async function runStackLifecycle(opts, childArgv, adapters = {}) {
2520
2659
  // BOT-2099 (Codex #1057 R4): a platform blip recorded by ANY lease wait stays on EVERY
2521
2660
  // terminal result (interrupted, unreadable, thrown, …), not only the paths that remember it.
2522
- const incidentBox = { incident: null };
2661
+ // BOT-2142: where the lease ran (and how it was reached) is on every terminal result too.
2662
+ const incidentBox = { incident: null, execution: null };
2523
2663
  const result = await runStackLifecycleOnce(opts, childArgv, adapters, incidentBox);
2524
- return incidentBox.incident && !result?.incident ? { ...result, incident: incidentBox.incident } : result;
2664
+ const withIncident = incidentBox.incident && !result?.incident ? { ...result, incident: incidentBox.incident } : result;
2665
+ return incidentBox.execution && !withIncident?.execution ? { ...withIncident, execution: incidentBox.execution } : withIncident;
2525
2666
  }
2526
2667
 
2527
2668
  async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
@@ -2583,6 +2724,11 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2583
2724
  // stack directory; the default is the real materializing resolver.
2584
2725
  const resolveExecution = adapters.resolveExecution ?? ((runOpts) => resolveRunExecution(process.cwd(), runOpts));
2585
2726
  const refreshRetention = adapters.refreshRetention ?? refreshLeasedStackRetention;
2727
+ // BOT-2142: remote-lease seams (bundle build/upload, SSH forwards). Production uses the
2728
+ // real builders; the fake-ssh tests swap only the ssh binary (BOTBUDDY_STACK_SSH_BIN works too).
2729
+ const buildBundle = adapters.buildBundle ?? buildStackBundle;
2730
+ const uploadBundle = adapters.uploadBundle ?? uploadStackBundle;
2731
+ const openTunnel = adapters.openTunnel ?? ((args) => establishSshTunnel(args));
2586
2732
  const clock = adapters.clock ?? { setInterval, clearInterval, setTimeout, clearTimeout };
2587
2733
  const signals = adapters.signals ?? process;
2588
2734
  const provisionTimeout = opts.provisionTimeout ?? opts.timeout;
@@ -2603,6 +2749,17 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2603
2749
  let forcedStopCode = null;
2604
2750
  let fencing = false;
2605
2751
  let activeWaitAbort = null;
2752
+ // BOT-2142: the remote lease's forwards, and whether they died under a running child.
2753
+ let tunnel = null;
2754
+ let tunnelLost = null;
2755
+ const placement = { mode: null, hostKey: null, requesterHost: null, transport: "none", forwards: null, bundle: null };
2756
+ incidentBox.execution = placement;
2757
+ const closeTunnel = async () => {
2758
+ if (!tunnel) return;
2759
+ const open = tunnel;
2760
+ tunnel = null;
2761
+ try { await open.close(); } catch { /* best effort: the process is gone either way */ }
2762
+ };
2606
2763
 
2607
2764
  const requestCleanup = () => {
2608
2765
  // A signal can arrive while the lease-request RPC is in flight. There is
@@ -2734,12 +2891,44 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2734
2891
  keepAlive();
2735
2892
  retention = clock.setInterval(keepAlive, RETENTION_REFRESH_MS);
2736
2893
  retention?.unref?.();
2894
+ // BOT-2142: `auto` (the default) may be served by another host this owner owns, and
2895
+ // the client cannot know which one the server picks, so the bundle digest goes up
2896
+ // front for both `auto` and `remote` (BOT-2145 refuses a remote pick without it).
2897
+ // `--local-only` never ships anything.
2898
+ let executionRequest = opts.execution ?? "auto";
2899
+ let bundle = null;
2900
+ if (executionRequest !== "local") {
2901
+ try {
2902
+ // BOT-2162: ship the worktree's BOT-2107 readiness declaration with the stack, so the
2903
+ // serving Helper runs the canonical bind-RPC probe for a remote lease too.
2904
+ const declaration = join(execution.worktreeRoot, ".botbuddy", "config.json");
2905
+ let hasDeclaration = false;
2906
+ try { hasDeclaration = existsSync(declaration) && lstatSync(declaration).isFile(); } catch { hasDeclaration = false; }
2907
+ bundle = buildBundle(stackDirFor(execution), {
2908
+ maxBytes: opts.bundleMaxBytes ?? DEFAULT_BUNDLE_MAX_BYTES,
2909
+ ...(hasDeclaration ? { extraFiles: [{ rel: ".botbuddy/config.json", abs: declaration }] } : {}),
2910
+ });
2911
+ } catch (error) {
2912
+ const message = String(error?.message ?? error);
2913
+ if (executionRequest === "remote") {
2914
+ const fixable = typeof error?.code === "string" && error.code.startsWith("BUNDLE_");
2915
+ return { exitCode: fixable ? EXIT.INVALID : EXIT.INTERNAL, outcome: "error", code: error?.code ?? "BUNDLE_FAILED", error: message };
2916
+ }
2917
+ // Codex #1077 P2: without a digest, `auto` could still pick a remote host and be
2918
+ // refused BUNDLE_REQUIRED instead of queueing here, so ask for this machine.
2919
+ executionRequest = "local";
2920
+ stderrLine(`${yellow("⚠")} stack: could not build the remote-lease bundle (${message}); requesting a co-located lease (execution local) instead.`);
2921
+ }
2922
+ }
2737
2923
  const leaseArgs = {
2738
2924
  slot, host_key: opts.host || undefined, repo: opts.repo, ticket_id: opts.ticket,
2739
2925
  ticket_url: opts.ticketUrl || undefined, pr_id: opts.prId || undefined, pr_url: opts.prUrl || undefined,
2740
2926
  purpose: opts.purpose || "stack run", idle_ttl_secs: opts.idleTtl ?? undefined,
2741
2927
  stack_path: execution.stackPath, worktree_root: execution.worktreeRoot,
2742
2928
  machine_uuid: hardwareUuid,
2929
+ execution: executionRequest,
2930
+ ...(bundle ? { bundle_sha256: bundle.sha256 } : {}),
2931
+ ...(opts.callbackPort != null ? { callback_port: opts.callbackPort } : {}),
2743
2932
  };
2744
2933
  // The retrying caller (api → rpc.call) transparently clears+self-heals+retries the
2745
2934
  // FIRST auth rejection among any of request/get/touch/release.
@@ -2785,7 +2974,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2785
2974
  }
2786
2975
  if (!request.data?.success) {
2787
2976
  const hint = worktreeRegistrationHint(request.data?.code, rpc.auth)
2788
- ?? stackPathRootRefusalNotice(request.data?.code, request.data?.message);
2977
+ ?? stackPathRootRefusalNotice(request.data?.code, request.data?.message)
2978
+ ?? remoteRefusalNotice(request.data?.code);
2789
2979
  if (hint) stderrLine(hint);
2790
2980
  return {
2791
2981
  exitCode: leaseRefusalExit(request.data?.code), outcome: "error", code: request.data?.code,
@@ -2799,6 +2989,63 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2799
2989
 
2800
2990
  leaseProvisioned = true; // from here a Helper may act on this lease
2801
2991
  let state = request.data.state;
2992
+ // BOT-2142: where the server placed the lease. A server that predates BOT-2145
2993
+ // sends no execution_mode, and its lease is co-located.
2994
+ const remoteLease = request.data.execution_mode === "remote";
2995
+ placement.mode = request.data.execution_mode ?? null;
2996
+ placement.hostKey = request.data.host_key ?? null;
2997
+ placement.requesterHost = request.data.requester_host ?? null;
2998
+ if (executionRequest === "remote" && !remoteLease) {
2999
+ const cleanup = await requestCleanup();
3000
+ return {
3001
+ exitCode: EXIT.BACKEND, outcome: "error", code: "REMOTE_UNSUPPORTED", leaseId, cleanup,
3002
+ error: `--remote was requested but the control plane placed lease ${leaseId} as ${request.data.execution_mode ?? "a co-located (pre-remote) lease"}; ` +
3003
+ "it was released. The server may predate remote leases — retry without --remote, or after it is upgraded.",
3004
+ };
3005
+ }
3006
+ let sshTarget = request.data.ssh_target ?? request.data.connection?.ssh_target ?? null;
3007
+ if (remoteLease && opts.callbackPort != null) {
3008
+ // Codex #1077 P1 / BOT-2184: `auto` placed this run remote; release before any upload
3009
+ // or forward rather than open an -R listener that may be exposed.
3010
+ const cleanup = await requestCleanup();
3011
+ return { exitCode: EXIT.INVALID, outcome: "error", code: CALLBACK_PORT_UNSUPPORTED_REMOTE, leaseId, cleanup, error: CALLBACK_PORT_REMOTE_REFUSAL };
3012
+ }
3013
+ if (remoteLease) {
3014
+ // Fail fast when the server SAYS the host has no SSH destination (the key is
3015
+ // present but empty); a server that only reports it on the active read is
3016
+ // checked again once the lease is active.
3017
+ const advertised = request.data.ssh_target !== undefined || request.data.connection?.ssh_target !== undefined;
3018
+ if (advertised) {
3019
+ try { assertSshTarget(sshTarget, placement.hostKey); } catch (error) {
3020
+ const cleanup = await requestCleanup();
3021
+ return { exitCode: EXIT.BACKEND, outcome: "error", code: error.code ?? "REMOTE_UNREACHABLE", leaseId, error: error.message, cleanup };
3022
+ }
3023
+ }
3024
+ if (!bundle || !request.data.bundle_upload_url) {
3025
+ const cleanup = await requestCleanup();
3026
+ return {
3027
+ exitCode: EXIT.BACKEND, outcome: "error", code: bundle ? "BUNDLE_UPLOAD_URL_MISSING" : "BUNDLE_REQUIRED", leaseId, cleanup,
3028
+ error: bundle
3029
+ ? `the control plane placed lease ${leaseId} on ${placement.hostKey ?? "a remote host"} without a bundle_upload_url; it was released`
3030
+ : `lease ${leaseId} was placed on ${placement.hostKey ?? "a remote host"} but this run has no stack bundle; it was released`,
3031
+ };
3032
+ }
3033
+ // Codex #1077 P1: a stalled PUT must not outlive a SIGINT/SIGTERM — the signal
3034
+ // handler aborts the active wait, and that now includes the upload.
3035
+ activeWaitAbort = new AbortController();
3036
+ const uploaded = await uploadBundle(request.data.bundle_upload_url, bundle, { signal: activeWaitAbort.signal });
3037
+ activeWaitAbort = null;
3038
+ if (receivedSignal) {
3039
+ const cleanup = await requestCleanup();
3040
+ return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup };
3041
+ }
3042
+ if (!uploaded?.ok) {
3043
+ const cleanup = await requestCleanup();
3044
+ return { exitCode: EXIT.BACKEND, outcome: "error", code: "BUNDLE_UPLOAD_FAILED", leaseId, cleanup, error: uploaded?.error || "bundle upload failed" };
3045
+ }
3046
+ placement.bundle = { sha256: bundle.sha256, bytes: bundle.size };
3047
+ stderrLine(`stack: lease ${leaseId} is remote on ${placement.hostKey ?? "?"} — uploaded the ${bundle.size}-byte stack bundle (sha256 ${bundle.sha256.slice(0, 12)}…)`);
3048
+ }
2802
3049
  if (receivedSignal) {
2803
3050
  const cleanup = await requestCleanup();
2804
3051
  return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup };
@@ -2874,13 +3121,36 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2874
3121
  return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup };
2875
3122
  }
2876
3123
 
2877
- connectionFile = await writeConnection(safeConnectionPath(opts.connectionFile, leaseId), current.data.connection);
3124
+ placement.hostKey = current.data.host_key ?? placement.hostKey;
3125
+ placement.mode = current.data.execution_mode ?? placement.mode ?? (remoteLease ? "remote" : null);
3126
+ let childConnection = current.data.connection;
3127
+ if (remoteLease) {
3128
+ // BOT-2142: forward every stack port over ONE ssh and hand the child the local ports.
3129
+ sshTarget = current.data.connection?.ssh_target ?? current.data.ssh_target ?? sshTarget;
3130
+ try {
3131
+ tunnel = await openTunnel({ connection: current.data.connection, sshTarget, hostKey: placement.hostKey, callbackPort: opts.callbackPort ?? null });
3132
+ } catch (error) {
3133
+ const cleanup = await requestCleanup();
3134
+ return { exitCode: EXIT.BACKEND, outcome: "error", code: error?.code ?? "REMOTE_UNREACHABLE", leaseId, error: String(error?.message ?? error), cleanup };
3135
+ }
3136
+ placement.transport = "ssh-forward";
3137
+ placement.forwards = tunnel.forwards.map(({ name, remote, local }) => ({ name, remote, local }));
3138
+ for (const f of tunnel.forwards.filter((x) => x.remapped)) stderrLine(`stack: forward ${f.name} remapped: local ${f.local} → ${placement.hostKey ?? "remote"} ${f.remote}`);
3139
+ childConnection = tunnel.connection;
3140
+ const forwardTarget = sshTarget;
3141
+ tunnel.onExit((exit) => {
3142
+ if (childResult) return;
3143
+ tunnelLost = { ...exit, target: forwardTarget };
3144
+ stopChild("SIGTERM", EXIT.LEASE_FAILED);
3145
+ });
3146
+ }
3147
+ connectionFile = await writeConnection(safeConnectionPath(opts.connectionFile, leaseId), childConnection);
2878
3148
  if (receivedSignal) {
2879
3149
  const cleanup = await requestCleanup();
2880
3150
  return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup };
2881
3151
  }
2882
3152
  if (isDenoIntegrationLane(childArgv)) {
2883
- leasedTestConfig = await materializeTestConfig(execution.worktreeRoot, leaseId, current.data.connection, { edgeMountRoot: stackDirFor(execution) });
3153
+ leasedTestConfig = await materializeTestConfig(execution.worktreeRoot, leaseId, childConnection, { edgeMountRoot: stackDirFor(execution) });
2884
3154
  }
2885
3155
  if (receivedSignal) {
2886
3156
  const cleanup = await requestCleanup();
@@ -2890,6 +3160,10 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2890
3160
  ...process.env,
2891
3161
  BOTBUDDY_STACK_LEASE_ID: leaseId,
2892
3162
  BOTBUDDY_STACK_CONNECTION_FILE: connectionFile,
3163
+ // BOT-2142: lets a lane's own checks (e.g. the port-owner preflight) know the
3164
+ // listeners are SSH forwards to another host, not this machine's Docker.
3165
+ BOTBUDDY_STACK_EXECUTION_MODE: remoteLease ? "remote" : (placement.mode ?? "local"),
3166
+ BOTBUDDY_STACK_TRANSPORT: placement.transport,
2893
3167
  ...(leasedTestConfig ? {
2894
3168
  BB_INTEGRATION_ENV_FILE: leasedTestConfig.envFile,
2895
3169
  BB_STACK_CONFIG: leasedTestConfig.stackConfig,
@@ -2903,6 +3177,12 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2903
3177
  // there would fail on a missing package.json AFTER the stack was provisioned.
2904
3178
  // The child therefore always runs in the (canonicalised) worktree root.
2905
3179
  const childCwd = execution.worktreeRoot;
3180
+ if (tunnelLost) {
3181
+ // The forwards died between being verified and the child starting.
3182
+ await closeTunnel();
3183
+ const cleanup = await requestCleanup();
3184
+ return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: null });
3185
+ }
2906
3186
  child = startChild(childArgv, childEnv, childCwd);
2907
3187
  leaseIdleTtlSecs = current.data.idle_ttl_secs ?? opts.idleTtl ?? 1800;
2908
3188
  const cadenceMs = Math.max(1_000, Math.floor(leaseIdleTtlSecs * 1000 / 3));
@@ -2927,7 +3207,11 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2927
3207
  if (heartbeat) clock.clearInterval(heartbeat);
2928
3208
  if (retention) { clock.clearInterval(retention); retention = null; }
2929
3209
  if (hardTimer) clock.clearTimeout(hardTimer);
3210
+ await closeTunnel();
2930
3211
  const cleanup = await requestCleanup();
3212
+ if (tunnelLost && !receivedSignal) {
3213
+ return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: childResult.code, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) });
3214
+ }
2931
3215
  const exitCode = receivedSignal ? SIGNAL_EXIT[receivedSignal] : forcedStopCode ?? (childResult.code !== 0 ? childResult.code : !cleanup.ok ? EXIT.CLEANUP_FAILED : 0);
2932
3216
  return { exitCode, outcome: exitCode === 0 ? "completed" : "failed", leaseId, childExitCode: childResult.code, cleanup, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) };
2933
3217
  } catch (error) {
@@ -2942,6 +3226,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2942
3226
  if (heartbeat) clock.clearInterval(heartbeat);
2943
3227
  if (retention) clock.clearInterval(retention);
2944
3228
  if (hardTimer) clock.clearTimeout(hardTimer);
3229
+ // BOT-2142: forwards never outlive the batch, whichever path ended it.
3230
+ await closeTunnel();
2945
3231
  if (connectionFile) await removeConnection(connectionFile);
2946
3232
  if (leasedTestConfig) await Promise.all([removeConnection(leasedTestConfig.envFile), removeConnection(leasedTestConfig.stackConfig)]);
2947
3233
  // Release the materialization lock only once the batch (and its signed reap)
@@ -2968,6 +3254,14 @@ async function cmdRun(opts, childArgv) {
2968
3254
  ...capacityStaleReceiptFields(result),
2969
3255
  ...(result.wait_exit_code != null ? { wait_exit_code: result.wait_exit_code, wait_receipt: result.wait_receipt ?? null } : {}),
2970
3256
  child_exit_code: result.childExitCode ?? null, cleanup: result.cleanup?.ok ?? null,
3257
+ ...(result.parkReason ? { park_reason: result.parkReason } : {}),
3258
+ // BOT-2142: where the lease ran and how the child reached it.
3259
+ execution_mode: result.execution?.mode ?? null,
3260
+ execution_host_key: result.execution?.hostKey ?? null,
3261
+ requester_host: result.execution?.requesterHost ?? null,
3262
+ transport: result.execution?.transport ?? "none",
3263
+ ...(result.execution?.forwards ? { forwards: result.execution.forwards } : {}),
3264
+ ...(result.execution?.bundle ? { bundle: result.execution.bundle } : {}),
2971
3265
  fenced: result.fenced ?? false, error: result.error ?? result.cleanup?.error ?? null,
2972
3266
  // BOT-2099: a platform auth/availability blip during the park is reported as an
2973
3267
  // incident (reason, window, attempts, recovered) — never as the lease's outcome.