@botbuddy/cli 1.45.7 → 1.47.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@botbuddy/cli",
3
- "version": "1.45.7",
3
+ "version": "1.47.0",
4
4
  "description": "BotBuddy — Swarm coordination CLI for multi-agent workflows",
5
5
  "type": "module",
6
6
  "bin": {
package/src/run.mjs CHANGED
@@ -15,7 +15,9 @@ import { fileURLToPath } from "node:url";
15
15
  import { callToolJson } from "./api.mjs";
16
16
  import { parseLaneEvents, laneCaseCounts, flushLaneCases, buildLaneSummary, laneSummaryFilename } from "./test-lane-events.mjs";
17
17
  import { AGENT_KEY_RE, readAgentSessionTokenEnv, legacyAgentAliasHint } from "./agent-key.mjs";
18
- import { clearAgentState, isRejectedCachedMcpSession, resolveAgentSessionCredential } from "./agent-session.mjs";
18
+ import { hasAgentSessionRuntime, isRejectedCachedMcpSession, resolveAgentSessionCredential, selfHealAgentSession } from "./agent-session.mjs";
19
+ import { isAgentStateFresh, readAgentState, worktreeRoot } from "./agent-state.mjs";
20
+ import { readAgentBinding } from "./wait-profile.mjs";
19
21
  import { createTelemetryOutbox } from "./telemetry-outbox.mjs";
20
22
  import { deliverExecutionEvent } from "./telemetry-delivery.mjs";
21
23
  import { attestTelemetryCredential, loadTelemetryIdentity } from "./telemetry-config.mjs";
@@ -231,49 +233,82 @@ function workerInvocation(context) {
231
233
  return [process.execPath, [bin, "run", "--worker", context]];
232
234
  }
233
235
 
236
+ function sessionFailure(code) { return Object.assign(new Error(code), { code }); }
237
+
238
+ // A selected ID is metadata, not a credential. Require positive private-cache
239
+ // authority before resolving it; never let the API's ambient owner key stand in.
240
+ async function resolveRunCredential({ cwd, env, opts, resolveCredential }) {
241
+ const cached = await readAgentState(cwd);
242
+ const explicitId = opts.sessionId;
243
+ const validateBinding = async (identity) => {
244
+ const bound = await readAgentBinding(cwd);
245
+ if (!bound?.tenant || identity?.tenant !== bound.tenant || !identity?.worktree
246
+ || await worktreeRoot(identity.worktree) !== await worktreeRoot(cwd)) {
247
+ throw sessionFailure("run_session_binding_mismatch");
248
+ }
249
+ };
250
+ if (explicitId && !cached) throw sessionFailure("run_session_authority_missing");
251
+ if (cached) {
252
+ if (explicitId && cached.session_id !== explicitId) throw sessionFailure("run_session_mismatch");
253
+ await validateBinding(cached);
254
+ if (explicitId && !isAgentStateFresh(cached, new Date())) throw sessionFailure("run_session_expired");
255
+ } else if (await hasAgentSessionRuntime({ cwd })) {
256
+ throw sessionFailure("run_session_authority_missing");
257
+ }
258
+ const credential = await (resolveCredential ?? resolveAgentSessionCredential)({ cwd, env,
259
+ sessionId: explicitId, flagToken: null });
260
+ // The resolver reads again. A concurrent replacement must satisfy the same
261
+ // binding and explicit ID rather than silently adopting unrelated authority.
262
+ if (credential.source === "cache") {
263
+ await validateBinding(credential.identity);
264
+ if (explicitId && credential.identity?.sessionId !== explicitId) throw sessionFailure("run_session_mismatch");
265
+ }
266
+ return credential;
267
+ }
268
+
269
+ function rejectedSessionReceipt(error, runId) {
270
+ const code = error?.code ?? "agent_session_unavailable";
271
+ const exitCode = ["session_mint_failed", "session_mint_invalid"].includes(code) ? EXIT.BACKEND : EXIT.AUTH;
272
+ return { exitCode, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "rejected",
273
+ ...(runId ? { run_id: runId } : {}), error: code,
274
+ recovery: code === "client_key_required"
275
+ ? "run `bb login` to install the durable client key, then retry"
276
+ : "run `bb doctor --fix` and select the current worktree session, then retry",
277
+ exit_code: exitCode } };
278
+ }
279
+
234
280
  export async function launchRun(argv, {
235
281
  cwd = process.cwd(), spawnImpl = spawn, call = null, childEnv = null, testRun = null,
236
- resolveCredential = null, env = process.env,
282
+ resolveCredential = null, renewCredential = selfHealAgentSession, env = process.env,
237
283
  } = {}) {
238
- let effectiveArgv = argv;
239
- let parsed = parseRunArgs(effectiveArgv, env);
284
+ const parsed = parseRunArgs(argv, env);
240
285
  let managedCredential = null;
241
286
  // A local test double owns its credential boundary; production invocations
242
287
  // self-heal when the session credential was not supplied by flag or env.
243
288
  // Keep all other usage failures local and deterministic before minting.
244
289
  const onlyMissingIdentity = parsed.errors.length === 1 && parsed.errors[0].includes("session-id");
245
- if (!parsed.opts.foreground && !parsed.opts.sessionId && !parsed.opts.sessionToken
246
- && onlyMissingIdentity && (resolveCredential || call === null)) {
290
+ if (!parsed.opts.foreground && !parsed.opts.sessionToken
291
+ && (parsed.errors.length === 0 || onlyMissingIdentity)
292
+ && (resolveCredential || call === null || call === callToolJson)) {
247
293
  try {
248
- const credential = await (resolveCredential ?? resolveAgentSessionCredential)({ argv, env, cwd });
294
+ const credential = await resolveRunCredential({ cwd, env, opts: parsed.opts, resolveCredential });
249
295
  managedCredential = credential;
250
- effectiveArgv = ["--agent-session-token", credential.token, ...argv];
251
- parsed = parseRunArgs(effectiveArgv, env);
296
+ parsed.opts.sessionToken = credential.token;
297
+ if (!SESSION_TOKEN_RE.test(credential.token)) throw sessionFailure("session_mint_invalid");
298
+ if (onlyMissingIdentity) parsed.errors.length = 0;
252
299
  } catch (error) {
253
- return {
254
- exitCode: EXIT.AUTH,
255
- receipt: {
256
- schema_version: RUN_SCHEMA_VERSION,
257
- outcome: "rejected",
258
- error: error?.code ?? "agent_session_unavailable",
259
- recovery: error?.code === "client_key_required"
260
- ? "run `bb login` to install the durable client key, then retry"
261
- : "run `bb doctor --fix` to repair agent authentication, then retry",
262
- exit_code: EXIT.AUTH,
263
- },
264
- };
300
+ return rejectedSessionReceipt(error);
265
301
  }
266
302
  }
267
303
  const { opts, command, errors } = parsed;
268
304
  if (errors.length) return { exitCode: EXIT.INVALID, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "rejected", errors, exit_code: EXIT.INVALID } };
269
305
  // BOT-1572 (AC-8): when a session token is present it is the credential — pin it
270
306
  // so register_command authenticates as the session's agent and derives session_id
271
- // from the token. Tests inject `call` directly and bypass this.
272
- const effectiveCall = call ?? (opts.sessionToken
273
- ? (t, a) => callToolJson(t, a, { auth: { "x-agent-api-key": opts.sessionToken } })
274
- : callToolJson);
307
+ // from the token. Read the current token on each call, including after renewal.
308
+ const effectiveCall = (t, a) => (call ?? callToolJson)(t, a,
309
+ opts.sessionToken ? { auth: { "x-agent-api-key": opts.sessionToken } } : {});
275
310
  const runId = randomUUID();
276
- const registerArgs = {
311
+ const registerArgs = () => ({
277
312
  // Omit session_id entirely when only a token is present — the backend derives
278
313
  // it. A plain id is still sent (and honoured) when supplied.
279
314
  ...(opts.sessionId ? { session_id: opts.sessionId } : {}),
@@ -281,18 +316,22 @@ export async function launchRun(argv, {
281
316
  command_hash: commandHash(command), environment_hash: environmentHash(cwd, opts.environment),
282
317
  environment: opts.environment, command_kind: opts.kind, expected_duration_seconds: opts.expectedDuration,
283
318
  rerun_reason: opts.rerunReason,
284
- };
285
- let registration = await effectiveCall("register_command", registerArgs);
319
+ });
320
+ let registration = await effectiveCall("register_command", registerArgs());
286
321
  // A cached session can be revoked server-side before its local expiry. The
287
322
  // managed cache is ours to replace once; explicit flag/env credentials belong
288
323
  // to the caller and are never silently rotated.
289
324
  if (isRejectedCachedMcpSession(registration, managedCredential?.source, 0)) {
290
- await clearAgentState(cwd);
291
- const credential = await (resolveCredential ?? resolveAgentSessionCredential)({ argv, env, cwd });
292
- managedCredential = credential;
293
- opts.sessionToken = credential.token;
294
- opts.sessionId = credential.sessionId ?? opts.sessionId;
295
- registration = await effectiveCall("register_command", registerArgs);
325
+ try {
326
+ const credential = await renewCredential({ cwd, env, force: true, rejectedCredential: {
327
+ token: managedCredential.token, identity: managedCredential.identity,
328
+ } });
329
+ managedCredential = credential;
330
+ opts.sessionToken = credential.token;
331
+ opts.sessionId = credential.identity?.sessionId ?? credential.sessionId;
332
+ if (!SESSION_TOKEN_RE.test(opts.sessionToken) || !opts.sessionId) throw sessionFailure("session_mint_invalid");
333
+ registration = await effectiveCall("register_command", registerArgs());
334
+ } catch (error) { return rejectedSessionReceipt(error, runId); }
296
335
  }
297
336
  if (!registration.ok || registration.isError || !registration.data?.accepted || !registration.data?.run_credential) {
298
337
  return { exitCode: EXIT.BACKEND, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "rejected", run_id: runId, error: registration.data?.error ?? registration.error ?? "durable_owner_not_established", exit_code: EXIT.BACKEND } };
@@ -310,7 +349,10 @@ export async function launchRun(argv, {
310
349
  await effectiveCall("update_command_run", { run_id: runId, run_credential: registration.data.run_credential, status: "owner_lost", invalidated_reason: `launcher_failed:${error instanceof Error ? error.message : String(error)}` });
311
350
  return { exitCode: EXIT.INTERNAL, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "owner_lost", run_id: runId, exit_code: EXIT.INTERNAL } };
312
351
  }
313
- return { exitCode: EXIT.OK, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "launched", run_id: runId, session_id: opts.sessionId, durable_owner: "detached_worker", receipt_path: receipt, reentry: "update_command_run publishes the terminal callback for this run_id", exit_code: EXIT.OK } };
352
+ const knownSession = opts.sessionId ?? managedCredential?.identity?.sessionId ?? managedCredential?.sessionId;
353
+ return { exitCode: EXIT.OK, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "launched", run_id: runId,
354
+ ...(knownSession ? { session_id: knownSession } : { session_binding: "server_derived" }),
355
+ durable_owner: "detached_worker", receipt_path: receipt, reentry: "update_command_run publishes the terminal callback for this run_id", exit_code: EXIT.OK } };
314
356
  }
315
357
 
316
358
  export async function runWorker(contextFile, { spawnImpl = spawn, call = callToolJson } = {}) {
@@ -20,7 +20,7 @@
20
20
  // Supply Guard's `scripts/local-supabase-isolated.sh`, ported here because the
21
21
  // published CLI cannot import repo scripts and stays Node-stdlib only.
22
22
  import { createHash, randomUUID } from "node:crypto";
23
- import { cpSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
23
+ import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
24
24
  import { homedir, hostname } from "node:os";
25
25
  import { basename, dirname, isAbsolute, join, relative, sep } from "node:path";
26
26
  import { lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
@@ -72,6 +72,11 @@ export function defaultPortRegistryDir() {
72
72
  const LEASED_STACK_RETENTION_GRACE_MS = 30 * 60 * 1000;
73
73
  /** The idle TTL a lease gets when the caller names none (`bb stack up --idle-ttl`). */
74
74
  const DEFAULT_IDLE_TTL_SECS = 1800;
75
+ /** BOT-1803: how long an IDLE lease's stack directory and port block stay reserved
76
+ * after its run ended. The stack is still running (the lease is `active` + idle), so
77
+ * another worktree must not take its ports; the next run on the same slot re-adopts it
78
+ * and renews the claim. Far above any idle TTL, below "forever". */
79
+ export const LEASED_STACK_IDLE_RETENTION_SECS = 24 * 60 * 60;
75
80
 
76
81
 
77
82
  // TOML decimal integers may carry a leading sign (`+56321`); radix forms may not.
@@ -251,7 +256,12 @@ function identityTag(ticket, slot) {
251
256
  * and the file lock then reads as reclaimable — so the refresh could swap the
252
257
  * `supabase/functions` tree under a batch that was still running. A run's lease
253
258
  * is released with `destroy`, so a stable identity bought no volume reuse anyway.
254
- * Omit `nonce` for a deterministic per-slot identity (tests, diagnostics).
259
+ * Omit `nonce` for a deterministic per-slot identity — since BOT-1803 that is what
260
+ * `bb stack run` uses: a run's lease goes IDLE (still running) instead of being reaped,
261
+ * and the next run on the same worktree+slot re-adopts that exact stack, which needs
262
+ * the same directory, project id and ports. Concurrent runs on one slot are still
263
+ * refused by the per-project lock, and the server refuses a second LIVE lease on the
264
+ * slot; a nonce remains for tests.
255
265
  */
256
266
  export function leasedStackIdentity({ worktreeRoot, slot, ticket, nonce } = {}) {
257
267
  // JSON-encode the parts so no path/slot/nonce combination can alias another
@@ -626,6 +636,9 @@ function collectFinishedStacks(stacksDir, { keep, ticket, slot, readdir, remove,
626
636
  // and fall back to the conservative default age.
627
637
  const retainUntil = Number.isFinite(marker?.retain_until) ? marker.retain_until : null;
628
638
  if (retainUntil === null && !isOwnStackFamily(entry, { ticket, slot, marker })) continue;
639
+ // BOT-1803: an IDLE lease's stack is still running from this directory — never collect it
640
+ // on a deadline (a run on its slot re-adopts or re-provisions it).
641
+ if (marker?.idle_retained === true) continue;
629
642
  if (now() < (retainUntil ?? stat(join(stacksDir, entry)).mtimeMs + LEASED_STACK_GC_MIN_AGE_MS)) continue;
630
643
  if (stackDirIsClaimed(entry, { read, alive, host, marker })) continue;
631
644
  remove(join(stacksDir, entry), { recursive: true, force: true });
@@ -646,15 +659,19 @@ function collectFinishedStacks(stacksDir, { keep, ticket, slot, readdir, remove,
646
659
  * Returns false (never throws) when there is no marker to refresh — an explicitly
647
660
  * prepared `--stack-path` is the caller's to manage, not ours to annotate.
648
661
  */
649
- export function refreshLeasedStackRetention(stackDir, { idleTtlSecs, now = Date.now, read = readFileSync, write = writeFileSync, rename = renameSync, portRegistryDir = null } = {}) {
662
+ export function refreshLeasedStackRetention(stackDir, { idleTtlSecs, idleRetained = false, now = Date.now, read = readFileSync, write = writeFileSync, rename = renameSync, portRegistryDir = null } = {}) {
650
663
  const marker = readStackMarker(stackDir, read);
651
664
  if (!marker) return false;
652
665
  const ttlMs = Math.max(0, Number.isFinite(idleTtlSecs) ? idleTtlSecs : DEFAULT_IDLE_TTL_SECS) * 1000;
653
666
  const at = now();
667
+ // BOT-1803: `idle_retained` marks a stack whose lease went idle (still running, kept
668
+ // for re-adoption); a renewal from a live batch clears it.
669
+ const { idle_retained: _wasIdle, released: _wasReleased, ...base } = marker;
654
670
  const renewed = JSON.stringify({
655
- ...marker,
671
+ ...base,
656
672
  last_seen_at: at,
657
673
  retain_until: at + Math.max(LEASED_STACK_GC_MIN_AGE_MS, ttlMs + LEASED_STACK_RETENTION_GRACE_MS),
674
+ ...(idleRetained ? { idle_retained: true } : {}),
658
675
  });
659
676
  try {
660
677
  atomicWrite(join(stackDir, LEASED_STACK_MARKER), renewed, { write, rename });
@@ -684,6 +701,34 @@ export function refreshLeasedStackRetention(stackDir, { idleTtlSecs, now = Date.
684
701
  return true;
685
702
  }
686
703
 
704
+ /**
705
+ * BOT-1803 (Codex #1160 P1): may `stack run` re-materialize the STABLE per-slot stack
706
+ * directory? Only when no batch can still be using it: no marker yet, a lease that went idle
707
+ * cleanly (`idle_retained`), or a batch whose reap was proven (`released`). Any other marker
708
+ * may belong to a batch whose owner process died while its detached child and containers
709
+ * still run — re-copying `supabase/` under those bind mounts is exactly what the
710
+ * per-invocation identity (BOT-1798) prevented, so such a run gets a fresh identity instead.
711
+ */
712
+ export function stableRunStackReusable(worktreeRoot, dirName, { read = readFileSync } = {}) {
713
+ if (typeof dirName !== "string" || !dirName) return true;
714
+ const marker = readStackMarker(join(worktreeRoot, LEASED_STACKS_DIR, dirName), read);
715
+ if (!marker) return true;
716
+ return marker.idle_retained === true || marker.released === true;
717
+ }
718
+
719
+ /** BOT-1803: record a PROVEN reap on the stack's marker, so the next run may reuse the directory. */
720
+ export function markLeasedStackReleased(stackDir, { read = readFileSync, write = writeFileSync, rename = renameSync } = {}) {
721
+ const marker = readStackMarker(stackDir, read);
722
+ if (!marker) return false;
723
+ const { idle_retained: _idle, ...base } = marker;
724
+ try {
725
+ atomicWrite(join(stackDir, LEASED_STACK_MARKER), JSON.stringify({ ...base, released: true }), { write, rename });
726
+ return true;
727
+ } catch {
728
+ return false;
729
+ }
730
+ }
731
+
687
732
  /** This stack's ownership/retention marker, or null when it has none. */
688
733
  function readStackMarker(stackDir, read) {
689
734
  try {
@@ -702,7 +747,7 @@ function readStackMarker(stackDir, read) {
702
747
  * them and the second Helper fails on a bind error after provisioning starts.
703
748
  * A block counts as taken while its stack is retained or its lock owner is alive.
704
749
  */
705
- function occupiedPortBlocks(registryDir, { keep, readdir, read, now, alive, host, stale = null }) {
750
+ function occupiedPortBlocks(registryDir, { keep, readdir, read, now, alive, host, stale = null, exists = existsSync }) {
706
751
  const taken = new Map(); // portBase -> { projectId, createdAt }
707
752
  let entries;
708
753
  try { entries = readdir(registryDir); } catch { return taken; }
@@ -712,7 +757,12 @@ function occupiedPortBlocks(registryDir, { keep, readdir, read, now, alive, host
712
757
  try { claim = JSON.parse(read(join(registryDir, entry), "utf8")); } catch { continue; }
713
758
  const portBase = Number(claim?.port_base);
714
759
  if (!Number.isInteger(portBase)) continue;
715
- const retained = Number.isFinite(claim.retain_until) && now() < claim.retain_until;
760
+ // BOT-1803: an IDLE lease's stack keeps running with no process renewing its claim, and
761
+ // the lease has no time ceiling — so its claim stands while its stack directory exists
762
+ // (until a run on the same slot re-adopts or re-provisions it, or the worktree is gone),
763
+ // never on a fixed deadline.
764
+ const retained = (Number.isFinite(claim.retain_until) && now() < claim.retain_until)
765
+ || (claim.idle_retained === true && typeof claim.stack_dir === "string" && exists(claim.stack_dir));
716
766
  const ownerAlive = Number.isInteger(claim.owner_pid) && claim.owner_pid > 0
717
767
  && (!claim.host || claim.host === host()) && alive(claim.owner_pid);
718
768
  if (!retained && !ownerAlive) {
@@ -923,7 +973,7 @@ export function reservePortBlocks(ports, { registryDir = defaultPortRegistryDir(
923
973
 
924
974
  mkdir(registryDir, { recursive: true });
925
975
  const claimed = [];
926
- const record = (block, at = now(), ttlSecs = owner?.idleTtlSecs) => JSON.stringify({
976
+ const record = (block, at = now(), ttlSecs = owner?.idleTtlSecs, idleRetained = false) => JSON.stringify({
927
977
  schema: 1,
928
978
  project_id: owner?.projectId ?? null,
929
979
  invocation_token: invocationToken,
@@ -935,6 +985,7 @@ export function reservePortBlocks(ports, { registryDir = defaultPortRegistryDir(
935
985
  retain_until: at + Math.max(LEASED_STACK_GC_MIN_AGE_MS, (Number(ttlSecs) || DEFAULT_IDLE_TTL_SECS) * 1000 + LEASED_STACK_RETENTION_GRACE_MS),
936
986
  owner_pid: process.pid,
937
987
  host: host(),
988
+ ...(idleRetained ? { idle_retained: true } : {}),
938
989
  });
939
990
  try {
940
991
  withAllocationLock(registryDir, { writeExclusive, read, remove, now, alive, host, sleep }, () => {
@@ -959,7 +1010,11 @@ export function reservePortBlocks(ports, { registryDir = defaultPortRegistryDir(
959
1010
  // Only THIS invocation's own reservation is not a peer.
960
1011
  let existing = null;
961
1012
  try { existing = JSON.parse(read(portClaimPath(registryDir, block), "utf8")); } catch { /* unreadable */ }
962
- if (existing && existing.invocation_token === invocationToken) write(portClaimPath(registryDir, block), record(block));
1013
+ // BOT-1803: a claim kept for an IDLE lease of this same prepared project is the
1014
+ // stack this run is about to re-adopt (or re-provision) on the same slot — the
1015
+ // project lock already serializes runs of one project, so take it over.
1016
+ const idleOwnProject = existing?.idle_retained === true && owner?.projectId != null && existing.project_id === owner.projectId;
1017
+ if (existing && (existing.invocation_token === invocationToken || idleOwnProject)) write(portClaimPath(registryDir, block), record(block));
963
1018
  else {
964
1019
  const busy = new Error(`refusing to lease this stack: port block ${block} was claimed by another run.`);
965
1020
  busy.code = "stack_ports_busy";
@@ -981,13 +1036,13 @@ export function reservePortBlocks(ports, { registryDir = defaultPortRegistryDir(
981
1036
  /** Keep the reservation alive while the batch runs (Codex #928 R21): a
982
1037
  * prepared `--stack-path` has no directory marker, so nothing else renews it,
983
1038
  * and an aged-out claim could be reclaimed under a still-live stack. */
984
- renew: ({ idleTtlSecs, now: at = now } = {}) => {
1039
+ renew: ({ idleTtlSecs, idleRetained = false, now: at = now } = {}) => {
985
1040
  let renewed = false;
986
1041
  for (const block of claimed) {
987
1042
  try {
988
1043
  const current = JSON.parse(read(portClaimPath(registryDir, block), "utf8"));
989
1044
  if (current?.invocation_token !== invocationToken) continue;
990
- write(portClaimPath(registryDir, block), record(block, at(), idleTtlSecs ?? owner?.idleTtlSecs));
1045
+ write(portClaimPath(registryDir, block), record(block, at(), idleTtlSecs ?? owner?.idleTtlSecs, idleRetained));
991
1046
  renewed = true;
992
1047
  } catch { /* advisory: never fence a healthy batch */ }
993
1048
  }
@@ -1356,8 +1411,14 @@ export function materializeRunStackDir(cwd, { slot, ticket, nonce, idleTtlSecs }
1356
1411
  // allocation lock (Codex #928 R16): the scan only REPORTS stale reservations, and
1357
1412
  // reclaiming one plus taking the block happen as one serialized step, so a
1358
1413
  // slower peer can never delete the winner's fresh claim.
1414
+ // BOT-1803: a stable (nonce-less) identity re-materializes the SAME stack each run, so
1415
+ // it keeps the block it already holds (an idle lease's stack is still bound to it, and
1416
+ // the deterministic stack digest covers the remapped ports) instead of drifting back to
1417
+ // the hash-preferred block whenever that one has freed up.
1418
+ const previous = dirExisted ? readStackMarker(stackDir, read) : null;
1419
+ const previousBlock = previous?.project_id === identity.projectId && Number.isInteger(previous?.port_base) ? previous.port_base : null;
1359
1420
  const portBase = withAllocationLock(registryDir, { writeExclusive, read, remove, now, alive, host, sleep }, () => {
1360
- let preferred = identity.portBase;
1421
+ let preferred = previousBlock ?? identity.portBase;
1361
1422
  for (let attempt = 0; attempt < LEASED_PORT_SLOTS; attempt++) {
1362
1423
  const stale = new Set();
1363
1424
  const taken = occupiedPortBlocks(registryDir, { keep: identity.projectId, readdir, read, now, alive, host, stale });
@@ -0,0 +1,111 @@
1
+ // BOT-1803 (AC-12) — the client-side USE signal for a leased stack.
2
+ //
3
+ // The server judges a lease idle only from observations. For a CO-LOCATED stack
4
+ // (execution_mode `local` / `local_exec`) the coding machine can read the stack's
5
+ // own Postgres: `docker exec supabase_db_<project_id> psql … pg_stat_activity` says
6
+ // whether any client is connected and working right now, and when a client last
7
+ // changed state. `bb stack touch` and `bb stack run`'s heartbeat send that as
8
+ // `usage: { client_connections, last_client_query_at }`.
9
+ //
10
+ // Fail toward "no observation": a remote lease, an unusable project id, a Docker/psql
11
+ // failure or a slow answer (> USAGE_TIMEOUT_MS) yields `null`, and the caller then
12
+ // sends NO `usage` at all — the server never judges a lease idle from an observation
13
+ // that did not happen. Nothing here reads or prints a secret: the query runs as the
14
+ // container's local `postgres` role over its Unix socket.
15
+ //
16
+ // Dependency-free (Node stdlib only), like the rest of the published CLI.
17
+ import { execFile } from "child_process";
18
+
19
+ /** How long one measurement may take before it counts as no observation. */
20
+ export const USAGE_TIMEOUT_MS = 5_000;
21
+
22
+ /** Supabase's own service roles: their connections are the platform, not a client. */
23
+ export const SUPABASE_INTERNAL_ROLES = Object.freeze([
24
+ "supabase_admin", "supabase_auth_admin", "supabase_storage_admin", "supabase_realtime_admin",
25
+ "supabase_replication_admin", "supabase_functions_admin", "supabase_read_only_user", "pgbouncer", "authenticator",
26
+ ]);
27
+
28
+ const ROLE_LIST = SUPABASE_INTERNAL_ROLES.map((r) => `'${r}'`).join(", ");
29
+
30
+ /**
31
+ * Two values, `|`-separated (psql -tA):
32
+ * 1. client_connections — client backends doing work now (state <> 'idle'), excluding
33
+ * this probe and every internal role except `authenticator`: a PostgREST request in
34
+ * flight is use now (Codex #1160 P1); its pooled sessions sitting idle are not.
35
+ * 2. last_client_query_at — the latest state change of any client backend, counting
36
+ * `authenticator` (PostgREST traffic IS use), ISO-8601 UTC; empty when none.
37
+ */
38
+ export const USAGE_SQL =
39
+ "SELECT " +
40
+ "(SELECT count(*) FROM pg_stat_activity WHERE backend_type = 'client backend' AND pid <> pg_backend_pid() " +
41
+ `AND (coalesce(usename, '') NOT IN (${ROLE_LIST}) OR usename = 'authenticator') AND state <> 'idle'), ` +
42
+ "(SELECT to_char(max(state_change) AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') FROM pg_stat_activity " +
43
+ "WHERE backend_type = 'client backend' AND pid <> pg_backend_pid() " +
44
+ `AND (coalesce(usename, '') NOT IN (${ROLE_LIST}) OR usename = 'authenticator'))`;
45
+
46
+ const CO_LOCATED_MODES = new Set(["local", "local_exec"]);
47
+ // `supabase_db_<id>` is named after the Supabase project id; refuse anything else
48
+ // rather than build a container name (an argv element) out of it.
49
+ const PROJECT_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,62}$/;
50
+
51
+ /** The Docker CLI selector for the engine running the stack ([] = the CLI's current context). */
52
+ export function dockerSelectorArgs(target) {
53
+ if (typeof target?.context === "string" && target.context) return ["--context", target.context];
54
+ const endpoint = target?.resolved_endpoint ?? target?.endpoint;
55
+ if (typeof endpoint === "string" && endpoint) return ["-H", endpoint];
56
+ return [];
57
+ }
58
+
59
+ /** The full `docker` argv for one measurement of `projectId`'s database. */
60
+ export function usageDockerArgs(projectId, target = null) {
61
+ return [...dockerSelectorArgs(target), "exec", `supabase_db_${projectId}`, "psql", "-U", "postgres", "-d", "postgres", "-tAc", USAGE_SQL];
62
+ }
63
+
64
+ /** Parse psql's `-tA` answer (`<count>|<iso or empty>`); null when it is not that shape. */
65
+ export function parseUsageOutput(text) {
66
+ const line = String(text ?? "").split(/\r?\n/).map((l) => l.trim()).find((l) => l.length > 0);
67
+ if (!line) return null;
68
+ const parts = line.split("|");
69
+ if (parts.length !== 2) return null;
70
+ const [countRaw, atRaw] = parts.map((p) => p.trim());
71
+ if (!/^\d+$/.test(countRaw)) return null;
72
+ if (atRaw && Number.isNaN(Date.parse(atRaw))) return null;
73
+ return { client_connections: Number(countRaw), last_client_query_at: atRaw || null };
74
+ }
75
+
76
+ /** Default runner: `docker <args>` with a hard timeout; never a shell. */
77
+ export function runDockerUsage(args, { timeoutMs = USAGE_TIMEOUT_MS } = {}) {
78
+ return new Promise((resolve) => {
79
+ // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process -- fixed executable, validated argv, no shell.
80
+ execFile("docker", args, { timeout: timeoutMs, killSignal: "SIGKILL", maxBuffer: 64 * 1024, windowsHide: true }, (error, stdout) => {
81
+ resolve({ code: error ? (typeof error.code === "number" ? error.code : 1) : 0, stdout: String(stdout ?? ""), timedOut: Boolean(error?.killed) });
82
+ });
83
+ });
84
+ }
85
+
86
+ /**
87
+ * Measure a lease's client use. `lease` is a `get_stack_lease` answer (or the same shape):
88
+ * `{ execution_mode, connection: { project_id, botbuddy_docker_target? } }`.
89
+ * Returns `{ client_connections, last_client_query_at }`, or null for no observation.
90
+ * Never throws.
91
+ */
92
+ export async function measureStackUsage(lease, { run = runDockerUsage, timeoutMs = USAGE_TIMEOUT_MS } = {}) {
93
+ let timer = null;
94
+ try {
95
+ if (!CO_LOCATED_MODES.has(lease?.execution_mode)) return null;
96
+ const projectId = lease?.connection?.project_id;
97
+ if (typeof projectId !== "string" || !PROJECT_ID_RE.test(projectId)) return null;
98
+ const args = usageDockerArgs(projectId, lease.connection.botbuddy_docker_target ?? null);
99
+ // Not unref'd: the timer is the only thing that settles a hung measurement, and it is
100
+ // cleared in `finally` the moment the answer lands (an unref'd timer let the event loop end
101
+ // with the race still pending on Linux CI).
102
+ const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve(null), timeoutMs); });
103
+ const answer = await Promise.race([Promise.resolve().then(() => run(args, { timeoutMs })), timeout]);
104
+ if (!answer || answer.code !== 0 || answer.timedOut) return null;
105
+ return parseUsageOutput(answer.stdout);
106
+ } catch {
107
+ return null;
108
+ } finally {
109
+ if (timer) clearTimeout(timer);
110
+ }
111
+ }
package/src/stack.mjs CHANGED
@@ -6,10 +6,16 @@
6
6
  // saturated; return an ACTIVE lease receipt.
7
7
  // botbuddy stack status <id> → the lease's current state + connection.
8
8
  // botbuddy stack touch <id> → bump the idle clock (before a long silent step).
9
- // botbuddy stack done <id> → release (`done`): reap the stack.
9
+ // botbuddy stack done <id> → release (`done`): idle the stack (BOT-1803; --reap tears it down).
10
10
  //
11
11
  // The canonical batch flow is: stack up → test → break → fix → test → stack done.
12
12
  //
13
+ // BOT-1803 (JIT idle leases): a finished batch's lease goes IDLE instead of being reaped —
14
+ // still `active`, stack running, capacity still held — so the next batch on the same
15
+ // worktree+slot re-adopts it with no provision. The control plane preempts an idle lease
16
+ // only when a queued lease needs the capacity (volumes kept), and re-provisions it when the
17
+ // stack's content digest changed.
18
+ //
13
19
  // Design contract (matches bb-wait):
14
20
  // * Dependency-free (Node stdlib only) so an explicit `npm exec --package` invocation cold-starts fast.
15
21
  // * Exactly ONE single-line JSON receipt to stdout; all diagnostics go to stderr.
@@ -36,7 +42,8 @@ import { AGENT_KEY_RE, readAgentSessionTokenEnv } from "./agent-key.mjs";
36
42
  import { resolveAgentSessionCredential, selfHealAgentSession } from "./agent-session.mjs";
37
43
  import { runDockerCommand, runDockerWorkflow, ADMITTED_DOCKER_VALIDATIONS } from "./docker-hygiene.mjs";
38
44
  import { acquireStackLock, lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
39
- import { assertLeasedStackIsolated, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
45
+ import { LEASED_STACK_IDLE_RETENTION_SECS, assertLeasedStackIsolated, markLeasedStackReleased, stableRunStackReusable, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
46
+ import { measureStackUsage } from "./stack-usage.mjs";
40
47
  import { machineUuid } from "./machine-id.mjs";
41
48
  import { DEFAULT_BUNDLE_MAX_BYTES, buildStackBundle, uploadStackBundle, validateBundleIncludePath } from "./stack-bundle.mjs";
42
49
  import { assertSshTarget, establishSshTunnel } from "./stack-tunnel.mjs";
@@ -87,8 +94,8 @@ export const STACK_HELP = `${bold("botbuddy stack")} — one command for a batch
87
94
  ${bold("USAGE")}
88
95
  botbuddy stack up [options] Request a lease; park if the host is full; hold it active
89
96
  botbuddy stack status <lease_id> Show a lease's state + connection
90
- botbuddy stack touch <lease_id> Bump the idle clock so the reaper doesn't stop it
91
- botbuddy stack done <lease_id> Release (done): tear the stack down
97
+ botbuddy stack touch <lease_id> Bump the idle clock (and report measured use) so the reaper doesn't stop it
98
+ botbuddy stack done <lease_id> Release (done): idle the stack for re-adoption (--reap tears it down)
92
99
  botbuddy stack run [options] -- <executable> [args...]
93
100
  Own one Helper-backed test/fix batch end-to-end
94
101
 
@@ -138,8 +145,14 @@ ${bold("up OPTIONS")}
138
145
  SSH forwards; --remote is refused here — use \`stack run --remote\`.
139
146
 
140
147
  ${bold("done OPTIONS")}
141
- --stop Keep volumes (cheap re-provision next batch). Default: destroy.
148
+ (default) BOT-1803: an ACTIVE lease goes IDLE — the stack keeps running and its
149
+ capacity stays held, so the next \`stack run\`/\`stack up\` on this slot re-adopts
150
+ it without provisioning. Receipt outcome "idle" + idle_since, idle_reason.
151
+ A queued/provisioning/reaping/reaped lease is released exactly as --reap.
152
+ --reap Release eagerly: reap (tear down) the stack now, as before BOT-1803.
153
+ --stop With --reap: keep volumes (cheap re-provision next batch). Default: destroy.
142
154
  --local-exec FALLBACK (no Helper): run 'supabase stop' locally and self-finalize.
155
+ Implies --reap (a local-exec stack is never left idle).
143
156
  --docker-context <name> Explicit Docker context used by the matching local-exec up
144
157
  (OrbStack or Docker Desktop; must match the engine that provisioned the lease).
145
158
  --docker-endpoint <uri> Explicit Docker endpoint instead of --docker-context.
@@ -152,10 +165,16 @@ ${bold("run OPTIONS")}
152
165
  --stack-path <relative> Optional PREPARED isolated stack dir. Omit it (the norm): run
153
166
  materializes .botbuddy/stacks/<project_id>/ from this worktree's
154
167
  supabase/ tree with a distinct project_id + remapped ports and
155
- leases that. "." is REFUSED — the worktree root is the shared
168
+ leases that. The identity is STABLE per worktree+slot (BOT-1803), so
169
+ the next run maps to the same directory, project and ports. "." is REFUSED — the worktree root is the shared
156
170
  canonical project and the Helper would start/clobber it.
157
171
  --provision-timeout N Seconds to wait for a physical Helper provision (default: --timeout).
158
- --reap-timeout N Seconds to wait for signed physical reap proof (default: 300).
172
+ --reap-timeout N Seconds to wait for signed physical reap proof (default: 300); also bounds
173
+ the wait for an idle lease being re-provisioned/preempted to reach reaped.
174
+ --reap Reap the stack when the child ends (as before BOT-1803). Default: the lease
175
+ goes IDLE (no reap wait; receipt cleanup {idle, idle_since}) and the stack
176
+ directory + port block stay reserved (24 h, renewed on re-adoption).
177
+ Interrupts, --hard-ttl, a fenced heartbeat or a lost tunnel always reap.
159
178
  --connection-file P Optional empty file path for the mode-0600 connection JSON.
160
179
  --hard-ttl N Bound the child runtime; timeout requests normal cleanup.
161
180
  --local-exec Rejected: stack run never starts Docker directly.
@@ -196,6 +215,21 @@ ${bold("EXIT CODES")}
196
215
  4 invalid arguments 5 backend/coordination 6 lease failed/reaped
197
216
  7 internal error 8 cleanup/reap proof failed
198
217
 
218
+ ${bold("IDLE LEASES")} (BOT-1803)
219
+ stack run sends stack_digest (the deterministic sha256 of the materialised stack bundle, also
220
+ with --local-only; omitted when it cannot be built) and the control plane answers with one of:
221
+ reused_idle the slot's idle lease is re-adopted: no provision; receipt reused "idle"
222
+ IDLE_LEASE_REPROVISIONING / LEASE_PREEMPTED
223
+ the idle lease is being reaped (content changed / capacity reclaimed): run
224
+ waits for it to reach reaped (--reap-timeout), re-sends the same request
225
+ once, and the receipt says reprovisioned "digest_mismatch" | "preempted";
226
+ no reap in time → exit 6
227
+ 4 STACK_EXECUTION_MODE_MISMATCH the idle lease is in another mode/host (it stays idle)
228
+ While the child runs (and on \`stack touch\`) a co-located lease's measured use is sent as
229
+ usage {client_connections, last_client_query_at} (docker exec … psql pg_stat_activity, 5 s cap;
230
+ no usage at all when it cannot be read). A lease preempted under a running child ends it:
231
+ exit 6, code LEASE_PREEMPTED, volumes_kept true; \`stack touch\` exits 6 with outcome "preempted".
232
+
199
233
  ${bold("REMOTE LEASES")} (BOT-2142, stack run)
200
234
  The receipt carries execution_mode (local|remote), execution_host_key, requester_host,
201
235
  transport (none|ssh-forward), and for a remote lease forwards [{name, remote, local}] + bundle.
@@ -295,6 +329,8 @@ export function parseStackArgs(argv) {
295
329
  // default exclusions (`--bundle-include`, repeatable).
296
330
  bundleInclude: [],
297
331
  disposition: "destroy", json: false, receiptMaxBytes: DEFAULT_RECEIPT_MAX_BYTES,
332
+ // BOT-1803: eager release (reap) instead of the default idle release (done/run).
333
+ reap: false,
298
334
  };
299
335
  const positionals = [];
300
336
  const need = (name, v) => {
@@ -413,6 +449,7 @@ export function parseStackArgs(argv) {
413
449
  case "--docker-context": if (need(a, rest[i + 1])) opts.dockerContext = rest[++i]; break;
414
450
  case "--docker-endpoint": if (need(a, rest[i + 1])) opts.dockerEndpoint = rest[++i]; break;
415
451
  case "--stop": opts.disposition = "stop"; break;
452
+ case "--reap": opts.reap = true; break;
416
453
  case "--destroy": opts.disposition = "destroy"; break;
417
454
  case "--json": opts.json = true; break;
418
455
  default:
@@ -462,6 +499,9 @@ export function parseStackArgs(argv) {
462
499
  "worktree root's default (shared canonical) project. Point --stack-path at a slot-derived stack directory with its " +
463
500
  "own supabase/config.toml (distinct project_id + remapped ports).");
464
501
  }
502
+ if (opts.reap && !["done", "run"].includes(command)) {
503
+ errors.push("--reap is valid only with `stack done` or `stack run`");
504
+ }
465
505
  if (opts.waitCapacity != null && !["up", "run"].includes(command)) {
466
506
  errors.push("--wait-capacity is valid only with `stack up` or `stack run`");
467
507
  }
@@ -642,15 +682,17 @@ export function resolveStackPath(cwd = process.cwd(), stackPath = ".") {
642
682
  *
643
683
  * Fail closed: whatever produced the path, a `run` lease at `"."` is refused.
644
684
  *
645
- * Each invocation gets its OWN stack directory and project id (BOT-1798, Codex
646
- * #928 R3/R5): a second `run` on the same worktree+slot must never delete and
647
- * recopy the `supabase/functions` tree a live batch's edge runtime is
648
- * bind-mounting — and that batch stays live even when its OWNER process is gone
649
- * (the child is detached, and the containers and server lease survive to the idle
650
- * TTL), so no liveness check on this side could make a shared directory safe. The
651
- * per-project lock is still taken and held for the whole batch, so the directory
652
- * is visibly claimed while it is in use; a run's lease is released with `destroy`,
653
- * so nothing is lost by not reusing the previous invocation's volume.
685
+ * BOT-1803: the identity is STABLE per (worktree, slot) — no per-invocation nonce —
686
+ * because a finished run's lease now goes IDLE with its stack still running, and the
687
+ * next run on the slot re-adopts that exact stack (same directory, project id and
688
+ * ports, so the deterministic stack digest is comparable across runs). Concurrency is
689
+ * unchanged: the per-project lock is held for the whole batch, so a second concurrent
690
+ * run on the same slot is refused (stack_slot_busy), and the server refuses a second
691
+ * LIVE lease on the slot. (BOT-1798's per-invocation identity, Codex #928 R3/R5,
692
+ * guarded a batch whose OWNER process died while its detached child still ran; a run
693
+ * re-materializing over such a batch is the residual risk of the stable identity, and
694
+ * the server's live-lease refusal then stops it before any Helper acts.)
695
+ * `opts.stackNonce` still forces a distinct identity (tests).
654
696
  */
655
697
  async function claimProjectLock(projectId, lock, subject) {
656
698
  try {
@@ -665,7 +707,7 @@ async function claimProjectLock(projectId, lock, subject) {
665
707
  }
666
708
  }
667
709
 
668
- export async function resolveRunExecution(cwd, opts, { materialize = materializeRunStackDir, lock = acquireStackLock, identity = leasedStackIdentity, reserveBlocks = reservePortBlocks, releaseClaim = releaseLeasedStackClaim, portRegistryDir = null } = {}) {
710
+ export async function resolveRunExecution(cwd, opts, { materialize = materializeRunStackDir, lock = acquireStackLock, identity = leasedStackIdentity, reserveBlocks = reservePortBlocks, releaseClaim = releaseLeasedStackClaim, stableReusable = stableRunStackReusable, markReleased = markLeasedStackReleased, portRegistryDir = null } = {}) {
669
711
  const refuseRoot = (execution) => {
670
712
  if (execution.stackPath === ".") {
671
713
  throw new Error(
@@ -728,8 +770,12 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
728
770
  };
729
771
  }
730
772
  const worktreeRoot = realpathSync(cwd);
731
- // One nonce per invocation, shared by the identity and the materialization below.
732
- const nonce = opts.stackNonce ?? randomUUID();
773
+ // BOT-1803: a stable per-(worktree, slot) identity unless a test pins a nonce — and only
774
+ // when no batch can still be using that directory (Codex #1160 P1): a marker left by a
775
+ // batch that neither went idle nor proved its reap may belong to a live detached child,
776
+ // so this run takes a fresh per-invocation identity instead (BOT-1798's protection).
777
+ const stableDir = identity({ worktreeRoot, slot: opts.slot, ticket: opts.ticket, nonce: null }).dirName;
778
+ const nonce = opts.stackNonce ?? (stableReusable(worktreeRoot, stableDir) ? null : randomUUID());
733
779
  const { projectId } = identity({ worktreeRoot, slot: opts.slot, ticket: opts.ticket, nonce });
734
780
  const held = await claimProjectLock(projectId, lock, `this batch's isolated stack (${projectId})`);
735
781
  try {
@@ -751,6 +797,8 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
751
797
  release: ({ reaped = false } = {}) => {
752
798
  if (reaped) {
753
799
  try { releaseClaim(stackDirFor(execution), portRegistryDir ? { registryDir: portRegistryDir } : undefined); } catch { /* advisory */ }
800
+ // BOT-1803: the stack is provably gone, so the next run may reuse this directory.
801
+ try { markReleased(stackDirFor(execution)); } catch { /* advisory */ }
754
802
  }
755
803
  held.release();
756
804
  },
@@ -2421,6 +2469,8 @@ export async function cmdUp(opts, {
2421
2469
  command: "up", outcome: "active", lease_id: leaseId, state: "active",
2422
2470
  host_key: g.host_key, slot: g.slot, connection: g.connection,
2423
2471
  idle_ttl_secs: g.idle_ttl_secs, resource_name: g.resource_name,
2472
+ // BOT-1803: the slot's IDLE lease was re-adopted (already running; nothing provisioned).
2473
+ ...(d.reused_idle === true ? { reused: "idle" } : {}),
2424
2474
  ...(localPreflight ? { preflight: localPreflight } : {}),
2425
2475
  }), opts, EXIT.OK);
2426
2476
  }
@@ -2453,16 +2503,55 @@ export async function cmdStatus(leaseId, opts, {
2453
2503
  ...(g.reap_requested_at ? { reap_requested_at: g.reap_requested_at } : {}),
2454
2504
  ...(g.reap_failed_at ? { reap_failed: true, reap_failed_at: g.reap_failed_at } : {}),
2455
2505
  ...(g.state === "reaping" && g.error ? { error: g.error } : {}),
2506
+ // BOT-1803: an idle lease is `active` with idle_since; display_state says "idle".
2507
+ ...idleStatusFields(g),
2456
2508
  }), opts, EXIT.OK);
2457
2509
  }
2458
2510
 
2459
- async function cmdTouch(leaseId, opts, { rpc = retryingStackCall() } = {}) {
2511
+ /** BOT-1803: the idle/use fields a get_stack_lease answer carries (absent on older servers). */
2512
+ function idleStatusFields(g) {
2513
+ const out = {};
2514
+ for (const key of ["display_state", "idle", "idle_since", "idle_reason", "last_observed_use_at", "usage_observed_at", "usage_observed_source", "reap_reason", "preempted_for_lease_id"]) {
2515
+ if (g?.[key] !== undefined && g?.[key] !== null) out[key] = g[key];
2516
+ }
2517
+ return out;
2518
+ }
2519
+
2520
+ /** BOT-1803: is a touch/get answer the control plane's "this lease was preempted"? */
2521
+ export function isPreemptedAnswer(d) {
2522
+ return d?.code === "LEASE_PREEMPTED" || (d?.success === false && d?.reap_reason === "preempted");
2523
+ }
2524
+
2525
+ /**
2526
+ * BOT-1803: `stack touch` bumps the idle clock and, for a co-located lease, reports the
2527
+ * measured client use (AC-12). Reading the lease first (for its project + Docker target)
2528
+ * is best-effort: without it the touch simply carries no usage. A preempted lease is a
2529
+ * typed receipt and exit 6 — the keeper exit Supply Guard's heartbeat daemon sees (AC-11).
2530
+ */
2531
+ export async function cmdTouch(leaseId, opts, { rpc = retryingStackCall(), emitResult = emit, measureUsage = measureStackUsage } = {}) {
2460
2532
  const auth = await rpc.init();
2461
- if (!auth) return emit(buildReceipt({ command: "touch", outcome: "error", lease_id: leaseId, error: "not authenticated — run `botbuddy login`" }), opts, EXIT.AUTH);
2462
- const r = await rpc.call("touch_stack_lease", { lease_id: leaseId });
2463
- if (!r.ok) return emit(buildReceipt({ command: "touch", outcome: "error", lease_id: leaseId, error: r.error }), opts, r.auth ? EXIT.AUTH : EXIT.BACKEND);
2464
- if (!r.data.success) return emit(buildReceipt({ command: "touch", outcome: "error", lease_id: leaseId, code: r.data.code, error: r.data.code }), opts, EXIT.BACKEND);
2465
- return emit(buildReceipt({ command: "touch", outcome: "touched", lease_id: leaseId, last_used_at: r.data.last_used_at }), opts, EXIT.OK);
2533
+ if (!auth) return emitResult(buildReceipt({ command: "touch", outcome: "error", lease_id: leaseId, error: "not authenticated — run `botbuddy login`" }), opts, EXIT.AUTH);
2534
+ let usage = null;
2535
+ try {
2536
+ const got = await rpc.call("get_stack_lease", { lease_id: leaseId });
2537
+ if (got?.ok && got.data?.success && got.data.state === "active") usage = await measureUsage(got.data);
2538
+ } catch { usage = null; }
2539
+ const r = await rpc.call("touch_stack_lease", { lease_id: leaseId, ...(usage ? { usage } : {}) });
2540
+ if (!r.ok) return emitResult(buildReceipt({ command: "touch", outcome: "error", lease_id: leaseId, error: r.error }), opts, r.auth ? EXIT.AUTH : EXIT.BACKEND);
2541
+ if (isPreemptedAnswer(r.data)) {
2542
+ return emitResult(buildReceipt({
2543
+ command: "touch", outcome: "preempted", code: "LEASE_PREEMPTED", lease_id: leaseId, volumes_kept: true,
2544
+ state: r.data.state ?? null, reap_reason: r.data.reap_reason ?? "preempted",
2545
+ error: "the lease was preempted (its idle capacity went to a queued lease); the stack was stopped with its volumes kept — request a new lease",
2546
+ }), opts, EXIT.LEASE_FAILED);
2547
+ }
2548
+ if (!r.data.success) return emitResult(buildReceipt({ command: "touch", outcome: "error", lease_id: leaseId, code: r.data.code, error: r.data.code }), opts, EXIT.BACKEND);
2549
+ return emitResult(buildReceipt({
2550
+ command: "touch", outcome: "touched", lease_id: leaseId, last_used_at: r.data.last_used_at,
2551
+ ...(r.data.idle !== undefined ? { idle: r.data.idle } : {}),
2552
+ ...(r.data.idle_since !== undefined ? { idle_since: r.data.idle_since } : {}),
2553
+ usage_recorded: r.data.usage_recorded === true,
2554
+ }), opts, EXIT.OK);
2466
2555
  }
2467
2556
 
2468
2557
  export async function cmdDone(leaseId, opts, {
@@ -2607,9 +2696,21 @@ export async function cmdDone(leaseId, opts, {
2607
2696
  }
2608
2697
  }
2609
2698
 
2610
- const r = await call("release_stack_lease", { lease_id: leaseId, disposition: opts.disposition });
2699
+ // BOT-1803: the default `done` idles an ACTIVE lease (stack kept running for the next
2700
+ // batch on this slot). `--reap` is today's eager release, and `--local-exec` implies it:
2701
+ // its self-teardown below (and every hermetic lane / Supply Guard stop) needs the reap.
2702
+ const idleRelease = !opts.reap && !opts.localExec;
2703
+ const r = await call("release_stack_lease", { lease_id: leaseId, disposition: opts.disposition, ...(idleRelease ? { idle: true } : {}) });
2611
2704
  if (!r.ok) return emitResult(buildReceipt({ command: "done", outcome: "error", lease_id: leaseId, error: r.error }), opts, r.auth ? EXIT.AUTH : EXIT.BACKEND);
2612
2705
  if (!r.data.success) return emitResult(buildReceipt({ command: "done", outcome: "error", lease_id: leaseId, code: r.data.code, error: r.data.code }), opts, EXIT.BACKEND);
2706
+ if (idleRelease && r.data.idle === true) {
2707
+ return emitResult(buildReceipt({
2708
+ command: "done", outcome: "idle", lease_id: leaseId, state: r.data.state ?? "active",
2709
+ idle: true, idle_since: r.data.idle_since ?? null, idle_reason: r.data.idle_reason ?? null,
2710
+ ...(r.data.noop === true ? { noop: true } : {}),
2711
+ }), opts, EXIT.OK);
2712
+ }
2713
+ // Any other answer (the lease was not active) is exactly today's release.
2613
2714
  let state = r.data.state;
2614
2715
  // BOT-1887 (Codex #972 P1): a lease that is ALREADY reaping (an idle reap that
2615
2716
  // established `stop`, or a retried `destroy`) keeps its in-flight teardown contract —
@@ -2686,6 +2787,8 @@ export async function cmdDone(leaseId, opts, {
2686
2787
  }
2687
2788
 
2688
2789
  const SIGNAL_EXIT = Object.freeze({ SIGINT: 130, SIGTERM: 143, SIGHUP: 129 });
2790
+ /** BOT-1803: request answers meaning "this slot's idle lease is being reaped; re-request once it is". */
2791
+ const IDLE_REPROVISION_REASONS = Object.freeze({ IDLE_LEASE_REPROVISIONING: "digest_mismatch", LEASE_PREEMPTED: "preempted" });
2689
2792
 
2690
2793
  function safeConnectionPath(requested, leaseId) {
2691
2794
  if (requested) {
@@ -2823,10 +2926,12 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
2823
2926
  // BOT-2099 (Codex #1057 R4): a platform blip recorded by ANY lease wait stays on EVERY
2824
2927
  // terminal result (interrupted, unreadable, thrown, …), not only the paths that remember it.
2825
2928
  // BOT-2142: where the lease ran (and how it was reached) is on every terminal result too.
2826
- const incidentBox = { incident: null, execution: null };
2929
+ // BOT-1803: whether this run re-adopted / re-provisioned an idle lease, and its stack digest.
2930
+ const incidentBox = { incident: null, execution: null, idleLease: null };
2827
2931
  const result = await runStackLifecycleOnce(opts, childArgv, adapters, incidentBox);
2828
2932
  const withIncident = incidentBox.incident && !result?.incident ? { ...result, incident: incidentBox.incident } : result;
2829
- return incidentBox.execution && !withIncident?.execution ? { ...withIncident, execution: incidentBox.execution } : withIncident;
2933
+ const withExecution = incidentBox.execution && !withIncident?.execution ? { ...withIncident, execution: incidentBox.execution } : withIncident;
2934
+ return incidentBox.idleLease && !withExecution?.idleLease ? { ...withExecution, idleLease: incidentBox.idleLease } : withExecution;
2830
2935
  }
2831
2936
 
2832
2937
  async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
@@ -2851,9 +2956,12 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2851
2956
  const api = adapters.api ?? {
2852
2957
  request: (args) => rpc.call("request_stack_lease", args),
2853
2958
  get: (leaseId) => rpc.call("get_stack_lease", { lease_id: leaseId }),
2854
- touch: (leaseId) => rpc.call("touch_stack_lease", { lease_id: leaseId }),
2855
- release: (leaseId, signal) => rpc.call("release_stack_lease", { lease_id: leaseId, disposition: "destroy" }, { signal }),
2959
+ // BOT-1803: a heartbeat carries the measured client use when there is one.
2960
+ touch: (leaseId, usage) => rpc.call("touch_stack_lease", { lease_id: leaseId, ...(usage ? { usage } : {}) }),
2961
+ // BOT-1803: `{ idle: true }` idles an ACTIVE lease instead of reaping it.
2962
+ release: (leaseId, signal, { idle = false } = {}) => rpc.call("release_stack_lease", { lease_id: leaseId, disposition: "destroy", ...(idle ? { idle: true } : {}) }, { signal }),
2856
2963
  };
2964
+ const measureUsage = adapters.measureUsage ?? measureStackUsage;
2857
2965
  const machineUuidFn = adapters.machineUuidFn ?? machineUuid;
2858
2966
  const wait = adapters.wait ?? ((leaseId, done, failed, options) => waitForLease(leaseId, done, failed, options));
2859
2967
  // BOT-2099: the queue park and the provision wait survive auth/availability blips.
@@ -2918,6 +3026,14 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2918
3026
  let tunnelLost = null;
2919
3027
  const placement = { mode: null, hostKey: null, requesterHost: null, transport: "none", forwards: null, bundle: null, placementReason: null, considered: null };
2920
3028
  incidentBox.execution = placement;
3029
+ // BOT-1803: idle-lease bookkeeping for the receipt.
3030
+ const idleLease = { reused: null, reprovisioned: null, previousLeaseId: null, stackDigest: null };
3031
+ incidentBox.idleLease = idleLease;
3032
+ // BOT-1803: set when this run's port block must stay reserved although nothing of ours
3033
+ // was provisioned (the slot's idle/live stack in THIS directory is still bound to it).
3034
+ let retainBlock = false;
3035
+ // BOT-1803: a heartbeat touch found the lease preempted under the running child.
3036
+ let preempted = null;
2921
3037
  const closeTunnel = async () => {
2922
3038
  if (!tunnel) return;
2923
3039
  const open = tunnel;
@@ -2925,7 +3041,10 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2925
3041
  try { await open.close(); } catch { /* best effort: the process is gone either way */ }
2926
3042
  };
2927
3043
 
2928
- const requestCleanup = () => {
3044
+ // BOT-1803: `idle` (only the normal end of a run) asks the server to idle the ACTIVE lease
3045
+ // — no reap job, no signed-reap wait. Any answer that is not an idle one (the lease was
3046
+ // not active) continues into today's signed-reap wait unchanged.
3047
+ const requestCleanup = ({ idle = false } = {}) => {
2929
3048
  // A signal can arrive while the lease-request RPC is in flight. There is
2930
3049
  // nothing to release yet, so do not cache this no-op; once the RPC returns
2931
3050
  // the interruption branch must perform the real signed cleanup.
@@ -2937,13 +3056,16 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
2937
3056
  const cleanupTimer = clock.setTimeout(() => cleanupAbort.abort("reap-timeout"), opts.reapTimeout * 1000);
2938
3057
  let release;
2939
3058
  try {
2940
- release = await api.release(leaseId, cleanupAbort.signal);
3059
+ release = idle ? await api.release(leaseId, cleanupAbort.signal, { idle: true }) : await api.release(leaseId, cleanupAbort.signal);
2941
3060
  } finally {
2942
3061
  clock.clearTimeout(cleanupTimer);
2943
3062
  }
2944
3063
  if (!release?.ok || !release?.data?.success) {
2945
3064
  return cleanupResult = { ok: false, error: release?.error || release?.data?.code || "release failed" };
2946
3065
  }
3066
+ if (idle && release.data.idle === true) {
3067
+ return cleanupResult = { ok: true, idle: true, state: release.data.state ?? "active", idle_since: release.data.idle_since ?? null, idle_reason: release.data.idle_reason ?? null };
3068
+ }
2947
3069
  if (release.data.state === "reaped") return cleanupResult = { ok: true, state: "reaped" };
2948
3070
  // BOT-1822 (Codex P2 round 2, extended round 3): `reap_failed` is synthetic
2949
3071
  // (never a real DB state), so a Helper's failed receipt landing in the split
@@ -3062,18 +3184,34 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
3062
3184
  let executionRequest = opts.execution ?? "auto";
3063
3185
  let bundle = null;
3064
3186
  const bundleInclude = opts.bundleInclude ?? [];
3187
+ const buildRunBundle = ({ digestOnly = false } = {}) => {
3188
+ // BOT-2162: ship the worktree's BOT-2107 readiness declaration with the stack, so the
3189
+ // serving Helper runs the canonical bind-RPC probe for a remote lease too.
3190
+ const declaration = join(execution.worktreeRoot, ".botbuddy", "config.json");
3191
+ let hasDeclaration = false;
3192
+ try { hasDeclaration = existsSync(declaration) && lstatSync(declaration).isFile(); } catch { hasDeclaration = false; }
3193
+ return buildBundle(stackDirFor(execution), {
3194
+ // A digest that ships nothing has no upload cap (the archive is only hashed).
3195
+ maxBytes: digestOnly ? Number.MAX_SAFE_INTEGER : (opts.bundleMaxBytes ?? DEFAULT_BUNDLE_MAX_BYTES),
3196
+ ...(digestOnly ? { rawLimitBytes: Number.MAX_SAFE_INTEGER } : {}),
3197
+ ...(hasDeclaration ? { extraFiles: [{ rel: ".botbuddy/config.json", abs: declaration }] } : {}),
3198
+ ...(bundleInclude.length ? { include: bundleInclude } : {}),
3199
+ });
3200
+ };
3201
+ // BOT-1803: the stack digest — the deterministic bundle sha256 of the materialised stack —
3202
+ // goes on EVERY request so the control plane re-adopts this slot's idle lease only when
3203
+ // its content is unchanged. --local-only ships nothing, so there it is computed for the
3204
+ // digest alone (no upload cap). Fail closed (Codex #1160 P1): every run request says
3205
+ // `require_stack_digest`, so when no digest could be computed the control plane re-provisions
3206
+ // an idle lease instead of handing back a stack whose content it cannot prove current.
3207
+ let stackDigest = null;
3208
+ let digestError = null;
3209
+ if (executionRequest === "local") {
3210
+ try { stackDigest = buildRunBundle({ digestOnly: true })?.sha256 ?? null; } catch (error) { digestError = error; }
3211
+ }
3065
3212
  if (executionRequest !== "local") {
3066
3213
  try {
3067
- // BOT-2162: ship the worktree's BOT-2107 readiness declaration with the stack, so the
3068
- // serving Helper runs the canonical bind-RPC probe for a remote lease too.
3069
- const declaration = join(execution.worktreeRoot, ".botbuddy", "config.json");
3070
- let hasDeclaration = false;
3071
- try { hasDeclaration = existsSync(declaration) && lstatSync(declaration).isFile(); } catch { hasDeclaration = false; }
3072
- bundle = buildBundle(stackDirFor(execution), {
3073
- maxBytes: opts.bundleMaxBytes ?? DEFAULT_BUNDLE_MAX_BYTES,
3074
- ...(hasDeclaration ? { extraFiles: [{ rel: ".botbuddy/config.json", abs: declaration }] } : {}),
3075
- ...(bundleInclude.length ? { include: bundleInclude } : {}),
3076
- });
3214
+ bundle = buildRunBundle();
3077
3215
  for (const entry of bundle.included ?? []) {
3078
3216
  stderrLine(`stack: the bundle includes ${entry.path} (sha256 ${entry.sha256.slice(0, 12)}…; keys: ${entry.keys.join(", ") || "none"})`);
3079
3217
  }
@@ -3092,6 +3230,17 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
3092
3230
  stderrLine(`${yellow("⚠")} stack: could not build the remote-lease bundle (${message}); requesting a co-located lease (execution local) instead.`);
3093
3231
  }
3094
3232
  }
3233
+ if (!stackDigest && !bundle?.sha256 && !digestError) {
3234
+ // `auto` fell back to local above because its (capped) bundle failed: hash it uncapped.
3235
+ try { stackDigest = buildRunBundle({ digestOnly: true })?.sha256 ?? null; } catch (error) { digestError = error; }
3236
+ }
3237
+ stackDigest = stackDigest ?? bundle?.sha256 ?? null;
3238
+ if (typeof stackDigest !== "string" || !/^[0-9a-f]{64}$/.test(stackDigest)) stackDigest = null;
3239
+ if (!stackDigest) {
3240
+ stderrLine(`${yellow("⚠")} stack: could not compute the stack digest (${String(digestError?.message ?? digestError ?? "no digest")}); ` +
3241
+ "an idle stack on this slot will be re-provisioned rather than reused.");
3242
+ }
3243
+ idleLease.stackDigest = stackDigest;
3095
3244
  const leaseArgs = {
3096
3245
  slot, host_key: opts.host || undefined, repo: opts.repo, ticket_id: opts.ticket,
3097
3246
  ticket_url: opts.ticketUrl || undefined, pr_id: opts.prId || undefined, pr_url: opts.prUrl || undefined,
@@ -3100,6 +3249,9 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
3100
3249
  machine_uuid: hardwareUuid,
3101
3250
  execution: executionRequest,
3102
3251
  ...(bundle ? { bundle_sha256: bundle.sha256 } : {}),
3252
+ ...(stackDigest ? { stack_digest: stackDigest } : {}),
3253
+ // BOT-1803 (Codex #1160 P1): a run never re-adopts an idle stack it cannot prove current.
3254
+ require_stack_digest: true,
3103
3255
  // BOT-2269: what the bundle carries beyond the defaults (paths, sha256, key NAMES).
3104
3256
  ...(bundle?.included?.length ? { bundle_included: bundle.included } : {}),
3105
3257
  ...(opts.callbackPort != null ? { callback_port: opts.callbackPort } : {}),
@@ -3108,61 +3260,111 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
3108
3260
  // FIRST auth rejection among any of request/get/touch/release.
3109
3261
  // BOT-2024: a stale-capacity refusal prints phase-aware guidance; with the opt-in
3110
3262
  // --wait-capacity a TRANSIENT one parks on the server's capacity wait and re-admits once.
3111
- activeWaitAbort = new AbortController();
3112
- const admission = await requestWithCapacityWait(() => api.request(leaseArgs), opts, {
3113
- waitCapacity: adapters.waitCapacity ?? defaultWaitCapacity, log: stderrLine, signal: activeWaitAbort.signal,
3114
- });
3115
- activeWaitAbort = null;
3116
- const request = admission.req;
3117
- // Interrupted DURING the capacity park: the refused request created no lease, so
3118
- // there is nothing to release. (A signal during a request that DID admit falls
3119
- // through to the existing post-lease interruption branch, which cleans up.)
3120
- if (admission.interrupted) {
3121
- return { exitCode: SIGNAL_EXIT[receivedSignal] ?? EXIT.INTERNAL, outcome: "interrupted", code: request?.data?.code, ...capacityStaleReceiptFields(request?.data) };
3122
- }
3123
- if (admission.capacityTimeout) {
3124
- return {
3125
- exitCode: EXIT.TIMEOUT, outcome: "timeout", code: request.data.code, ...capacityStaleReceiptFields(request.data),
3126
- wait_exit_code: EXIT.TIMEOUT, wait_receipt: admission.waitReceipt,
3127
- error: `capacity did not recover within --wait-capacity ${opts.waitCapacity}s (${request.data.code})`,
3128
- };
3129
- }
3130
- if (admission.waitFailed) {
3131
- return {
3132
- exitCode: admission.waitFailed.exitCode, outcome: "error", code: request.data.code, ...capacityStaleReceiptFields(request.data),
3133
- wait_exit_code: admission.waitFailed.exitCode, wait_receipt: admission.waitFailed.receipt,
3134
- error: `--wait-capacity: bb wait exited ${admission.waitFailed.exitCode} (${admission.waitFailed.detail})`,
3135
- };
3136
- }
3137
- if (!request?.ok) {
3138
- // Codex #928 R23 (P2): `status: null` (callToolJson's fetch-threw branch) means
3139
- // NO response was ever received — the request may have reached the backend and
3140
- // created a lease we simply never heard back about, so a Helper could already be
3141
- // provisioning it. Any OTHER failure carries a real HTTP status: the backend was
3142
- // reached and definitely refused (or errored) before creating anything of ours.
3143
- // Only the definite case is safe to treat as unprovisioned and free the port
3144
- // claim immediately below; an ambiguous transport failure retains it instead — it
3145
- // still expires on its own TTL if nothing was actually provisioned.
3146
- if (request?.status == null) leaseProvisioned = true;
3147
- return { exitCode: request?.auth ? EXIT.AUTH : EXIT.BACKEND, outcome: "error", error: request?.error || "lease request failed" };
3263
+ // BOT-1803: an idle lease on this slot that the control plane is re-provisioning (its
3264
+ // stack digest changed) or that lost a race to a preemption answers with the OLD lease
3265
+ // id: wait for it to reach `reaped`, then send the SAME request once more (it then
3266
+ // provisions a fresh lease for the slot).
3267
+ let request;
3268
+ for (let attempt = 0; ; attempt++) {
3269
+ activeWaitAbort = new AbortController();
3270
+ const admission = await requestWithCapacityWait(() => api.request(leaseArgs), opts, {
3271
+ waitCapacity: adapters.waitCapacity ?? defaultWaitCapacity, log: stderrLine, signal: activeWaitAbort.signal,
3272
+ });
3273
+ activeWaitAbort = null;
3274
+ request = admission.req;
3275
+
3276
+ // Interrupted DURING the capacity park: the refused request created no lease, so
3277
+ // there is nothing to release. (A signal during a request that DID admit falls
3278
+ // through to the existing post-lease interruption branch, which cleans up.)
3279
+ if (admission.interrupted) {
3280
+ return { exitCode: SIGNAL_EXIT[receivedSignal] ?? EXIT.INTERNAL, outcome: "interrupted", code: request?.data?.code, ...capacityStaleReceiptFields(request?.data) };
3281
+ }
3282
+ if (admission.capacityTimeout) {
3283
+ return {
3284
+ exitCode: EXIT.TIMEOUT, outcome: "timeout", code: request.data.code, ...capacityStaleReceiptFields(request.data),
3285
+ wait_exit_code: EXIT.TIMEOUT, wait_receipt: admission.waitReceipt,
3286
+ error: `capacity did not recover within --wait-capacity ${opts.waitCapacity}s (${request.data.code})`,
3287
+ };
3288
+ }
3289
+ if (admission.waitFailed) {
3290
+ return {
3291
+ exitCode: admission.waitFailed.exitCode, outcome: "error", code: request.data.code, ...capacityStaleReceiptFields(request.data),
3292
+ wait_exit_code: admission.waitFailed.exitCode, wait_receipt: admission.waitFailed.receipt,
3293
+ error: `--wait-capacity: bb wait exited ${admission.waitFailed.exitCode} (${admission.waitFailed.detail})`,
3294
+ };
3295
+ }
3296
+ if (!request?.ok) {
3297
+ // Codex #928 R23 (P2): `status: null` (callToolJson's fetch-threw branch) means
3298
+ // NO response was ever received — the request may have reached the backend and
3299
+ // created a lease we simply never heard back about, so a Helper could already be
3300
+ // provisioning it. Any OTHER failure carries a real HTTP status: the backend was
3301
+ // reached and definitely refused (or errored) before creating anything of ours.
3302
+ // Only the definite case is safe to treat as unprovisioned and free the port
3303
+ // claim immediately below; an ambiguous transport failure retains it instead — it
3304
+ // still expires on its own TTL if nothing was actually provisioned.
3305
+ if (request?.status == null) leaseProvisioned = true;
3306
+ return { exitCode: request?.auth ? EXIT.AUTH : EXIT.BACKEND, outcome: "error", error: request?.error || "lease request failed" };
3307
+ }
3308
+ const why = attempt === 0 && request.data?.success === false && request.data?.lease_id ? IDLE_REPROVISION_REASONS[request.data.code] : null;
3309
+ if (!why) break;
3310
+ const previous = request.data.lease_id;
3311
+ idleLease.previousLeaseId = previous;
3312
+ stderrLine(`stack: this slot's idle lease ${previous} is being ${why === "preempted" ? "preempted" : "re-provisioned (the stack content changed)"} — waiting for it to reach reaped, then requesting again`);
3313
+ activeWaitAbort = new AbortController();
3314
+ const gone = await resilientWait(previous, (st) => st === "reaped", (st) => st === "reap_failed", { timeoutSec: opts.reapTimeout, auth: rpc.auth, signal: activeWaitAbort.signal });
3315
+ activeWaitAbort = null;
3316
+ if (gone?.interrupted || receivedSignal) {
3317
+ return { exitCode: SIGNAL_EXIT[receivedSignal] ?? EXIT.INTERNAL, outcome: "interrupted", code: request.data.code };
3318
+ }
3319
+ let reaped = Boolean(gone?.woke) && gone.state === "reaped";
3320
+ if (!reaped && !gone?.failed) {
3321
+ // The same missed-wake recheck as the provision and reap waits (BOT-1822/1827).
3322
+ const read = await readLeaseQuietly(previous);
3323
+ reaped = Boolean(read?.ok && read.data?.success && read.data.state === "reaped");
3324
+ }
3325
+ if (!reaped) {
3326
+ // Its containers may still hold this stack's ports.
3327
+ retainBlock = true;
3328
+ return {
3329
+ exitCode: EXIT.LEASE_FAILED, outcome: "error", code: request.data.code,
3330
+ error: `this slot's idle lease ${previous} did not reach reaped ${gone?.failed ? "(its reap failed)" : `within ${opts.reapTimeout}s`} ` +
3331
+ `after ${why}; no new lease was requested — check \`bb stack status ${previous}\` and re-run once it is reaped`,
3332
+ ...(parkIncident ? { incident: parkIncident } : {}),
3333
+ };
3334
+ }
3335
+ idleLease.reprovisioned = why;
3148
3336
  }
3149
3337
  if (!request.data?.success) {
3150
3338
  const hint = worktreeRegistrationHint(request.data?.code, rpc.auth)
3151
3339
  ?? stackPathRootRefusalNotice(request.data?.code, request.data?.message)
3152
3340
  ?? remoteRefusalNotice(request.data?.code);
3153
3341
  if (hint) stderrLine(hint);
3342
+ // BOT-1803: the slot's idle lease stays idle (another mode/host); keep its block.
3343
+ if (request.data?.code === "STACK_EXECUTION_MODE_MISMATCH" && request.data?.lease_id) retainBlock = true;
3154
3344
  return {
3155
3345
  exitCode: leaseRefusalExit(request.data?.code), outcome: "error", code: request.data?.code,
3156
3346
  ...capacityStaleReceiptFields(request.data), error: leaseRefusalError(request.data),
3157
3347
  };
3158
3348
  }
3159
3349
  leaseId = request.data.lease_id;
3160
- if (request.data.reused) {
3350
+ // BOT-1803: `reused_idle` — the slot's IDLE lease was re-adopted: it is active with its
3351
+ // stored connection and nothing is provisioned, so the run goes straight on to the
3352
+ // post-activation flow. Any other reuse is somebody's LIVE batch, refused as before.
3353
+ const reusedIdle = request.data.reused === true && request.data.reused_idle === true;
3354
+ if (request.data.reused && !reusedIdle) {
3355
+ // With a stable identity a live lease in THIS stack directory is bound to our block.
3356
+ if (request.data.stack_path != null && request.data.stack_path === execution.stackPath &&
3357
+ (request.data.worktree_root == null || request.data.worktree_root === execution.worktreeRoot)) retainBlock = true;
3161
3358
  return { exitCode: EXIT.BACKEND, outcome: "error", leaseId, error: "a live lease already exists for this agent and stack slot; wait for that batch to finish instead of sharing its stack" };
3162
3359
  }
3163
3360
 
3164
3361
  leaseProvisioned = true; // from here a Helper may act on this lease
3165
3362
  let state = request.data.state;
3363
+ if (reusedIdle) {
3364
+ idleLease.reused = "idle";
3365
+ state = "active";
3366
+ stderrLine(`stack: re-adopted idle lease ${leaseId}${request.data.host_key ? ` on ${request.data.host_key}` : ""} — the stack is already running; nothing to provision`);
3367
+ }
3166
3368
  // BOT-2142: where the server placed the lease. A server that predates BOT-2145
3167
3369
  // sends no execution_mode, and its lease is co-located.
3168
3370
  const remoteLease = request.data.execution_mode === "remote";
@@ -3192,6 +3394,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
3192
3394
  return { exitCode: EXIT.BACKEND, outcome: "error", code: error.code ?? "REMOTE_UNREACHABLE", leaseId, error: error.message, cleanup };
3193
3395
  }
3194
3396
  }
3397
+ // BOT-1803: a re-adopted idle remote stack is already running there; nothing to upload.
3398
+ if (!reusedIdle) {
3195
3399
  if (!bundle || !request.data.bundle_upload_url) {
3196
3400
  const cleanup = await requestCleanup();
3197
3401
  return {
@@ -3217,6 +3421,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
3217
3421
  placement.bundle = { sha256: bundle.sha256, bytes: bundle.size };
3218
3422
  if (bundle.included?.length) placement.bundleIncluded = bundle.included;
3219
3423
  stderrLine(`stack: lease ${leaseId} is remote on ${placement.hostKey ?? "?"} — uploaded the ${bundle.size}-byte stack bundle (sha256 ${bundle.sha256.slice(0, 12)}…)`);
3424
+ }
3220
3425
  } else if (bundle && state === "queued" && request.data.bundle_upload_url) {
3221
3426
  // BOT-2207: the lease queued HERE but may still move: every minute the control plane
3222
3427
  // re-offers it to the owner's other hosts, and it can only move with its bundle in
@@ -3389,9 +3594,23 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
3389
3594
  // last activity, not its creation, so a batch that outlives the default window
3390
3595
  // and then loses its owner process is never collected while it is still running.
3391
3596
  keepAlive();
3597
+ // BOT-1803 (AC-12): a co-located lease's heartbeat carries the measured client use; a
3598
+ // remote one (or an unreadable measurement) sends no usage at all.
3599
+ const usageLease = { ...current.data, execution_mode: current.data.execution_mode ?? (servedRemote ? "remote" : "local") };
3392
3600
  heartbeat = clock.setInterval(async () => {
3393
3601
  if (fencing || childResult) return;
3394
- const touched = await api.touch(leaseId);
3602
+ let usage = null;
3603
+ if (!servedRemote) { try { usage = await measureUsage(usageLease); } catch { usage = null; } }
3604
+ if (fencing || childResult) return;
3605
+ const touched = usage ? await api.touch(leaseId, usage) : await api.touch(leaseId);
3606
+ if (touched?.ok && isPreemptedAnswer(touched.data)) {
3607
+ // BOT-1803 (AC-11): the control plane reclaimed this lease's capacity. Stop the
3608
+ // child and end typed — never the generic lost-lease wording.
3609
+ preempted = { state: touched.data.state ?? "reaping", reap_reason: touched.data.reap_reason ?? "preempted" };
3610
+ fencing = true;
3611
+ stopChild("SIGTERM", EXIT.LEASE_FAILED);
3612
+ return;
3613
+ }
3395
3614
  if (!touched?.ok || !touched?.data?.success) {
3396
3615
  fencing = true;
3397
3616
  stopChild("SIGTERM", EXIT.LEASE_FAILED);
@@ -3407,10 +3626,29 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
3407
3626
  if (retention) { clock.clearInterval(retention); retention = null; }
3408
3627
  if (hardTimer) clock.clearTimeout(hardTimer);
3409
3628
  await closeTunnel();
3410
- const cleanup = await requestCleanup();
3629
+ // BOT-1803: a run that ended on its own (the child exited, success or not) leaves its
3630
+ // lease IDLE for the next run on this slot — no reap wait. `--reap`, a signal, the
3631
+ // hard TTL, a fenced/preempted heartbeat and a lost tunnel keep the eager signed reap.
3632
+ const idleOnEnd = !opts.reap && !receivedSignal && !fencing && !tunnelLost && forcedStopCode == null;
3633
+ const cleanup = await requestCleanup({ idle: idleOnEnd });
3634
+ if (cleanup?.idle) {
3635
+ // The stack is still running: keep its directory and host port block reserved well
3636
+ // past any idle TTL so no other worktree takes them; the next run renews the claim.
3637
+ try { refreshRetention(stackDirFor(execution), { idleTtlSecs: LEASED_STACK_IDLE_RETENTION_SECS, idleRetained: true }); } catch { /* advisory */ }
3638
+ try { execution.renew?.({ idleTtlSecs: LEASED_STACK_IDLE_RETENTION_SECS, idleRetained: true }); } catch { /* advisory */ }
3639
+ stderrLine(`stack: lease ${leaseId} is idle${cleanup.idle_since ? ` since ${cleanup.idle_since}` : ""} — its stack keeps running for the next run on this slot (\`bb stack done ${leaseId} --reap\` tears it down)`);
3640
+ }
3411
3641
  if (tunnelLost && !receivedSignal) {
3412
3642
  return tunnelLostResult(tunnelLost, { leaseId, cleanup, childExitCode: childResult.code, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) });
3413
3643
  }
3644
+ if (preempted && !receivedSignal) {
3645
+ return {
3646
+ exitCode: EXIT.LEASE_FAILED, outcome: "failed", code: "LEASE_PREEMPTED", volumesKept: true, leaseId, childExitCode: childResult.code, cleanup, fenced: true,
3647
+ error: `lease ${leaseId} was preempted while the child ran (its capacity went to a queued lease); the child was stopped and the stack ` +
3648
+ "stopped with its volumes kept — re-run the batch to get a new lease",
3649
+ ...(parkIncident ? { incident: parkIncident } : {}),
3650
+ };
3651
+ }
3414
3652
  const exitCode = receivedSignal ? SIGNAL_EXIT[receivedSignal] : forcedStopCode ?? (childResult.code !== 0 ? childResult.code : !cleanup.ok ? EXIT.CLEANUP_FAILED : 0);
3415
3653
  return { exitCode, outcome: exitCode === 0 ? "completed" : "failed", leaseId, childExitCode: childResult.code, cleanup, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) };
3416
3654
  } catch (error) {
@@ -3436,8 +3674,10 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
3436
3674
  // A request the backend refused, or one that only re-attached to somebody
3437
3675
  // else's live lease, provisioned nothing of ours — its block must go back at
3438
3676
  // once, or repeated refusals would exhaust the host (Codex #928 R21 P2).
3439
- const provenReap = !leaseProvisioned
3440
- || (cleanupResult?.ok === true && (cleanupResult.state === "reaped" || cleanupResult.state == null));
3677
+ // BOT-1803: an IDLE lease (cleanupResult.state "active") is not a reap — its stack still
3678
+ // runs, so its block stays reserved; so does one this run must not free (retainBlock).
3679
+ const provenReap = !retainBlock && (!leaseProvisioned
3680
+ || (cleanupResult?.ok === true && (cleanupResult.state === "reaped" || cleanupResult.state == null)));
3441
3681
  execution?.release?.({ reaped: provenReap });
3442
3682
  for (const [signal, handler] of signalHandlers) signals.off?.(signal, handler);
3443
3683
  }
@@ -3452,7 +3692,14 @@ export function runReceiptFields(result) {
3452
3692
  // BOT-2024: a stale-capacity refusal carries its phase + armable wait on the receipt.
3453
3693
  ...capacityStaleReceiptFields(result),
3454
3694
  ...(result.wait_exit_code != null ? { wait_exit_code: result.wait_exit_code, wait_receipt: result.wait_receipt ?? null } : {}),
3455
- child_exit_code: result.childExitCode ?? null, cleanup: result.cleanup?.ok ?? null,
3695
+ child_exit_code: result.childExitCode ?? null,
3696
+ // BOT-1803: an idle release is `cleanup: { idle: true, idle_since }` (nothing was reaped).
3697
+ cleanup: result.cleanup?.idle ? { idle: true, idle_since: result.cleanup.idle_since ?? null } : result.cleanup?.ok ?? null,
3698
+ ...(result.volumesKept ? { volumes_kept: true } : {}),
3699
+ ...(result.idleLease?.reused ? { reused: result.idleLease.reused } : {}),
3700
+ ...(result.idleLease?.reprovisioned ? { reprovisioned: result.idleLease.reprovisioned } : {}),
3701
+ ...(result.idleLease?.previousLeaseId ? { previous_lease_id: result.idleLease.previousLeaseId } : {}),
3702
+ ...(result.idleLease?.stackDigest ? { stack_digest: result.idleLease.stackDigest } : {}),
3456
3703
  ...(result.parkReason ? { park_reason: result.parkReason } : {}),
3457
3704
  // BOT-2142: where the lease ran and how the child reached it.
3458
3705
  execution_mode: result.execution?.mode ?? null,