@botbuddy/cli 1.50.1 → 1.51.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botbuddy/cli",
-  "version": "1.50.1",
+  "version": "1.51.0",
   "description": "BotBuddy — Swarm coordination CLI for multi-agent workflows",
   "type": "module",
   "bin": {

package/src/agent-session.mjs

@@ -261,7 +261,7 @@ export async function selfHealAgentSession({
         || (current.worktree && await worktreeRoot(current.worktree) !== registrationWorktree)
         || (current.agent_session_token === rejectedCredential.token
           && (current.session_id !== identity.sessionId || (current.session_token_generation ?? null) !== identity.generation))) {
-        throw missing("session_mint_invalid", "The cached session no longer matches the rejected agent, tenant, worktree or session incarnation.");
+        throw missing("session_identity_mismatch", "The cached session no longer matches the rejected agent, tenant, worktree or session incarnation.");
       }
       if (current.agent_session_token !== rejectedCredential.token && isFresh(current, now())) {
         return withIdentity({ token: current.agent_session_token, sessionId: sessionId ?? current.session_id, agentId: current.agent_id, source: "cache", ticketSignals: null, stackLeaseCapability: null }, current);
@@ -368,14 +368,17 @@ export async function selfHealAgentSession({
       throw missing("session_mint_invalid", "register_agent did not return an agent_id and session_token");
     }
     if (renewal && (data.agent_id !== current.agent_id
-      || data.tenant_id !== bound.tenant
+      || (data.tenant_id !== undefined && data.tenant_id !== bound.tenant)
       || (data.tenant !== undefined && data.tenant !== bound.tenant)
-      || (data.worktree !== undefined && data.worktree !== registrationWorktree)
+      || (data.worktree !== undefined && data.worktree !== registrationWorktree))) {
+      throw missing("session_identity_mismatch", "register_agent returned a replacement for a different agent, tenant or worktree.");
+    }
+    if (renewal && (data.tenant_id !== bound.tenant
       || !AGENT_KEY_RE.test(data.session_token)
       || data.session_token === current.agent_session_token
       || typeof data.session_id !== "string" || !data.session_id
       || (data.session_ttl_seconds !== undefined && (!Number.isFinite(Number(data.session_ttl_seconds)) || Number(data.session_ttl_seconds) <= 0)))) {
-      throw missing("session_mint_invalid", "register_agent returned an invalid replacement session or a different agent, tenant or worktree.");
+      throw missing("session_mint_invalid", "register_agent returned an invalid replacement session.");
     }
     const issued = now();
     const ttlSeconds = Number.isFinite(Number(data.session_ttl_seconds)) ? Number(data.session_ttl_seconds) : SESSION_TTL_MS / 1000;

package/src/stack-isolation.mjs

@@ -945,6 +945,26 @@ export function releaseLeasedStackClaim(stackDir, { registryDir = defaultPortReg
   }
 }
 
+/**
+ * BOT-2237 (Codex #1168 P2): whether THIS stack's own port-block claim is still on disk —
+ * the verified result a receipt reports after a release attempt (a failed unlink leaves it).
+ */
+export function leasedStackClaimHeld(stackDir, { registryDir = defaultPortRegistryDir(), io = {} } = {}) {
+  const read = io.readFile ?? readFileSync;
+  // `readStackMarker` deliberately normalizes failures to null for best-effort
+  // collection. A release receipt needs the stricter contract: only ENOENT proves
+  // this stack has no marker/claim; unreadable or corrupt marker data is unknown.
+  let marker = null;
+  try { marker = JSON.parse(read(join(stackDir, LEASED_STACK_MARKER), "utf8")); } catch (error) { return error?.code !== "ENOENT"; }
+  if (!marker || typeof marker !== "object" || !Number.isInteger(marker.port_base)) return true;
+  let claim = null;
+  // Codex #1168 R2 P2: only a PROVEN absence (ENOENT) counts as released; an unreadable or
+  // corrupt claim file is unverifiable, so it is reported as still held.
+  try { claim = JSON.parse(read(portClaimPath(registryDir, marker.port_base), "utf8")); } catch (error) { return error?.code !== "ENOENT"; }
+  const owner = marker.invocation_token ?? marker.project_id;
+  return Boolean(claim) && (claim.invocation_token ?? claim.project_id) === owner;
+}
+
 /**
  * Reserve the blocks an EXPLICIT `--stack-path` occupies (BOT-1798, Codex #928
  * R18). Its project lock only serializes runs that share a project id; two
@@ -1048,13 +1068,18 @@ export function reservePortBlocks(ports, { registryDir = defaultPortRegistryDir(
       }
       return renewed;
     },
+    /** Returns whether none of this invocation's claims remain (BOT-2237: a verified result). */
     release: () => {
+      let allReleased = true;
       for (const block of claimed) {
-        try {
-          const current = JSON.parse(read(portClaimPath(registryDir, block), "utf8"));
-          if (current?.invocation_token === invocationToken) remove(portClaimPath(registryDir, block), { force: true });
-        } catch { /* gone or superseded: never unlink blindly */ }
+        let current = null;
+        // Codex #1168 R2 P2: a claim that is gone (ENOENT) is released; one that cannot be read
+        // or parsed is unverifiable — never unlinked blindly, and never reported as released.
+        try { current = JSON.parse(read(portClaimPath(registryDir, block), "utf8")); } catch (error) { if (error?.code !== "ENOENT") allReleased = false; continue; }
+        if (current?.invocation_token !== invocationToken) continue; // superseded: never unlink blindly
+        try { remove(portClaimPath(registryDir, block), { force: true }); } catch { allReleased = false; }
       }
+      return allReleased;
     },
   };
 }

package/src/stack.mjs

@@ -42,7 +42,7 @@ import { AGENT_KEY_RE, readAgentSessionTokenEnv } from "./agent-key.mjs";
 import { resolveAgentSessionCredential, selfHealAgentSession } from "./agent-session.mjs";
 import { runDockerCommand, runDockerWorkflow, ADMITTED_DOCKER_VALIDATIONS } from "./docker-hygiene.mjs";
 import { acquireStackLock, lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
-import { LEASED_STACK_IDLE_RETENTION_SECS, assertLeasedStackIsolated, markLeasedStackReleased, stableRunStackReusable, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
+import { LEASED_STACK_IDLE_RETENTION_SECS, assertLeasedStackIsolated, leasedStackClaimHeld, markLeasedStackReleased, stableRunStackReusable, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
 import { measureStackUsage } from "./stack-usage.mjs";
 import { machineUuid } from "./machine-id.mjs";
 import { DEFAULT_BUNDLE_MAX_BYTES, buildStackBundle, uploadStackBundle, validateBundleIncludePath } from "./stack-bundle.mjs";
@@ -707,7 +707,7 @@ async function claimProjectLock(projectId, lock, subject) {
   }
 }
 
-export async function resolveRunExecution(cwd, opts, { materialize = materializeRunStackDir, lock = acquireStackLock, identity = leasedStackIdentity, reserveBlocks = reservePortBlocks, releaseClaim = releaseLeasedStackClaim, stableReusable = stableRunStackReusable, markReleased = markLeasedStackReleased, portRegistryDir = null } = {}) {
+export async function resolveRunExecution(cwd, opts, { materialize = materializeRunStackDir, lock = acquireStackLock, identity = leasedStackIdentity, reserveBlocks = reservePortBlocks, releaseClaim = releaseLeasedStackClaim, claimHeld = leasedStackClaimHeld, stableReusable = stableRunStackReusable, markReleased = markLeasedStackReleased, portRegistryDir = null } = {}) {
   const refuseRoot = (execution) => {
     if (execution.stackPath === ".") {
       throw new Error(
@@ -763,9 +763,12 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
       // A prepared directory carries no retention marker, so its reservation is
       // renewed here on the batch's own beat (Codex #928 R21 P2).
       renew: (options) => ports.renew(options),
+      // BOT-2237: returns the VERIFIED disposition the receipt reports.
       release: ({ reaped = false } = {}) => {
-        if (reaped) ports.release();
+        let released = false;
+        if (reaped) { try { released = ports.release() !== false; } catch { released = false; } }
         held.release();
+        return { released };
       },
     };
   }
@@ -794,13 +797,19 @@ export async function resolveRunExecution(cwd, opts, { materialize = materialize
       // or running containers, so the reservation must stand until its retention
       // expires. The project lock is always released — it guards this directory,
       // which no other invocation can target.
+      // BOT-2237 (Codex #1168 P2): returns the VERIFIED disposition — whether this stack's
+      // claim is gone afterwards — so a failed unlink is never reported as released.
       release: ({ reaped = false } = {}) => {
+        const registry = portRegistryDir ? { registryDir: portRegistryDir } : undefined;
+        let released = false;
         if (reaped) {
-          try { releaseClaim(stackDirFor(execution), portRegistryDir ? { registryDir: portRegistryDir } : undefined); } catch { /* advisory */ }
+          try { releaseClaim(stackDirFor(execution), registry); } catch { /* verified below */ }
+          try { released = !claimHeld(stackDirFor(execution), registry); } catch { released = false; }
           // BOT-1803: the stack is provably gone, so the next run may reuse this directory.
           try { markReleased(stackDirFor(execution)); } catch { /* advisory */ }
         }
         held.release();
+        return { released };
       },
     };
   } catch (error) {
@@ -947,16 +956,22 @@ export function retryingStackCall({
   // (runStackLifecycle short-circuits its own await) skip init() — avoiding an extra
   // await point before the first RPC is dispatched.
   let auth = seedAuth;
-  let refreshed = false;
+  // One renewal per rotation: disarmed by a renewal, re-armed only once a call succeeds again.
+  // BOT-2237: a long `stack run` can see its session rotated more than once (a sibling
+  // `bb doctor --fix`/`bb wait` self-heal); a renewed credential that WORKED and is later
+  // rejected is a new rotation, not the same failure — so it may renew again. A renewal
+  // whose credential is itself rejected stays disarmed (never a per-call mint loop).
+  let renewalArmed = true;
   return {
     async init() { if (auth === undefined) auth = await authProvider(); return auth; },
     async call(name, args, callOptions = {}) {
       let r = await callTool(name, args, { ...callOptions, auth });
-      if (!refreshed && !r?.ok && r?.auth) {
-        refreshed = true;
+      if (renewalArmed && !r?.ok && r?.auth) {
+        renewalArmed = false;
         const fresh = await refresh(auth, { cwd: cwd() });
         if (fresh) { auth = fresh; r = await callTool(name, args, { ...callOptions, auth }); }
       }
+      if (r?.ok) renewalArmed = true;
       return r;
     },
     get auth() { return auth; },
@@ -970,7 +985,7 @@ export function retryingStackCall({
 // taxonomy so automation sees AUTH/BACKEND, not a CLI-defect INTERNAL. Credential
 // problems are AUTH (3); a transport/server mint failure is BACKEND (5); anything
 // unrecognized stays INTERNAL (7).
-const AUTH_ERROR_CODES = new Set(["client_key_required", "client_key_unavailable", "tenant_binding_required", "session_token_malformed", "session_mint_unauthorized"]);
+const AUTH_ERROR_CODES = new Set(["client_key_required", "client_key_unavailable", "tenant_binding_required", "session_token_malformed", "session_mint_unauthorized", "session_identity_mismatch"]);
 // BOT-1750 (Codex R11 P2): `session_mint_invalid` is an incompatible/malformed SERVER
 // response, not a credential problem — classify it with the transport/server failures.
 const BACKEND_ERROR_CODES = new Set(["session_mint_failed", "session_mint_invalid"]);
@@ -1384,16 +1399,31 @@ export async function waitForLease(leaseId, isDone, isFailed, { timeoutSec, auth
 }
 
 /**
- * BOT-2099 — the platform-incident reasons a park can record. A park NEVER ends with an
- * auth or mint error as its outcome: an outage is retried for the park's whole remaining
- * deadline and, if it outlasts it, the park ends as an ordinary `timeout` that carries
- * this incident (who to blame is the platform, not the lease).
+ * BOT-2099 — the platform-incident reasons a park can record. A park never ends with a raw
+ * auth or mint error as its outcome. A transport outage is retried for the park's whole
+ * remaining deadline and, if it outlasts it, ends as a `timeout` that carries this incident.
+ * An auth failure is bounded tighter (BOT-2237, PARK_AUTH_BUDGET_MS / DEFINITIVE_REAUTH_CODES)
+ * and ends typed as `platform_auth_unavailable`, naming the lease.
  */
 export const PARK_INCIDENT_REASONS = Object.freeze({
   AUTH: "platform_auth_unavailable",       // 401/403 on the wait, or the session re-mint rejected
   UNREACHABLE: "platform_unreachable",     // transport failure / 5xx / malformed relay response
 });
 export const PARK_BACKOFF = Object.freeze({ initialMs: 2_000, maxMs: 60_000 });
+/**
+ * BOT-2237: how long a park keeps retrying an UNRECOVERED auth failure before it ends with a
+ * typed `platform_auth_unavailable`. A short server-side auth blip (BOT-2099 hop 7, about a
+ * minute) still recovers inside it; a session that cannot be re-minted no longer sits out the
+ * whole park deadline — the server's queued-lease liveness (BOT-2013) reaps an unwatched lease
+ * well before an hour-long park ends, so waiting longer only reports a phantom capacity timeout.
+ */
+export const PARK_AUTH_BUDGET_MS = 5 * 60_000;
+// BOT-2237: re-mint refusals that no retry can fix — a locally proven worktree identity fence,
+// or durable credentials that are absent. A malformed mint response is a backend blip and gets
+// the regular auth budget; it is not evidence that the lease changed owner.
+const DEFINITIVE_REAUTH_CODES = new Set([
+  "session_identity_mismatch", "client_key_required", "tenant_binding_required", "session_token_malformed", "agent_tenant_immutable",
+]);
 
 function abortableSleep(ms, signal) {
   return new Promise((resolve) => {
@@ -1433,10 +1463,11 @@ export async function reauthenticateParkedLease(current, kind, {
  * head of the queue over a one-second 401 (hop 6) and ending every parked lease in a
  * server-side auth outage with `session_mint_unauthorized` (hop 7). Here such a result is
  * retried with bounded exponential backoff (PARK_BACKOFF) until the park's OWN deadline:
- * re-authenticate on an auth failure (`reauth`), re-register the wait (a fresh registration
- * replays the lease's current state), and — after a recovery — read the lease directly
- * (`probe`) so a grant or reap that landed during the outage is never missed. Mint errors
- * thrown by `reauth` are attempts, not outcomes. The result is waitForLease's shape plus
+ * re-authenticate on an auth failure (`reauth`), re-register the wait, and — after a
+ * recovery — read the lease directly (`probe`) so a grant or reap that landed during the
+ * outage is never missed (registration does NOT replay a lease's current state). Mint errors
+ * thrown by `reauth` are attempts, except a definitive refusal or an exhausted auth budget
+ * (BOT-2237), which end the park with `{ authUnavailable, code, error }`. The result is waitForLease's shape plus
  * `credential` (the headers that finally worked) and, when any blip occurred, `incident`.
  */
 export async function parkOnLease(leaseId, isDone, isFailed, {
@@ -1448,14 +1479,18 @@ export async function parkOnLease(leaseId, isDone, isFailed, {
   // retryingStackCall) owns the credential; the park re-reads it before every attempt so a
   // credential minted during a probe is never overwritten by an older one.
   syncCredential = null,
+  authBudgetMs = PARK_AUTH_BUDGET_MS,
 } = {}) {
   const deadline = deadlineMs ?? now() + timeoutSec * 1000;
   let credential = auth;
   let incident = null;
   let delay = backoff.initialMs;
+  // BOT-2237: when the current unrecovered run of AUTH failures began (null = none open).
+  let authFailingSince = null;
   const note = (kind, detail, code = null) => {
     const at = new Date(now()).toISOString();
     if (!incident) incident = { reason: null, first_at: at, attempts: 0, recovered: false };
+    if (kind === "auth" && authFailingSince == null) authFailingSince = now();
     incident.recovered = false; // a new failure re-opens a recovered incident
     incident.attempts += 1;
     incident.last_at = at;
@@ -1464,6 +1499,15 @@ export async function parkOnLease(leaseId, isDone, isFailed, {
     incident.last_error_code = code;
   };
   const done = (result) => ({ ...result, credential, ...(incident ? { incident } : {}) });
+  const recovered = () => { if (incident) incident.recovered = true; authFailingSince = null; delay = backoff.initialMs; };
+  // BOT-2237: the park cannot re-establish a principal that may observe this lease. End it
+  // typed, naming the lease, instead of parking blind until the deadline and then calling the
+  // outage a capacity timeout.
+  const authUnavailable = (code, detail) => done({
+    authUnavailable: true, code: code ?? "unauthorized",
+    error: `lease ${leaseId}: the parked wait lost its agent session and could not re-authenticate (${code ?? "unauthorized"}: ${String(detail).slice(0, 200)})`,
+  });
+  const authBudgetSpent = () => authFailingSince != null && now() - authFailingSince >= authBudgetMs;
   for (;;) {
     if (signal?.aborted) return done({ interrupted: true });
     const remainingMs = deadline - now();
@@ -1483,13 +1527,14 @@ export async function parkOnLease(leaseId, isDone, isFailed, {
     if (!result?.auth && !result?.error) {
       // Codex #1057 R5: any attempt that did not fail on auth/transport (a match, a reap, or an
       // ordinary timeout on a healthy stream) means the platform recovered.
-      if (incident) incident.recovered = true;
+      recovered();
       // A wait that ends on its own timer is a real timeout; never the outage's last word.
       if (result?.timeout && incident && now() < deadline) continue;
       return incident ? done(result) : result;
     }
     const kind = result.auth ? "auth" : "transport";
     note(kind, result.auth ? `unauthorized (${result.phase ?? "stream"})` : result.error, result.auth ? "unauthorized" : null);
+    if (kind === "auth" && authBudgetSpent()) return authUnavailable(incident.last_error_code, incident.last_error);
     log(`${yellow("⚠")} stack: lease ${leaseId} wait interrupted by a platform ${kind === "auth" ? "auth" : "availability"} blip (${incident.last_error}); still parked — retrying in ${Math.round(delay / 1000)}s (attempt ${incident.attempts}).`);
     // Back off (bounded) before re-authenticating, then retry; never past the deadline.
     for (;;) {
@@ -1504,17 +1549,25 @@ export async function parkOnLease(leaseId, isDone, isFailed, {
         break;
       } catch (err) {
         note("auth", err?.message ?? err, err?.code ?? null);
+        if (DEFINITIVE_REAUTH_CODES.has(err?.code) || authBudgetSpent()) return authUnavailable(err?.code ?? null, err?.message ?? err);
         log(`${yellow("⚠")} stack: lease ${leaseId} re-authentication failed (${err?.code ?? err?.message ?? err}); still parked — retrying in ${Math.round(delay / 1000)}s.`);
       }
     }
     if (probe) {
+      let read = null;
       try {
-        const read = await probe(credential);
+        read = await probe(credential);
         if (syncCredential) credential = syncCredential() ?? credential;
-        const state = read?.state ?? null;
-        if (state != null && isFailed(state)) { incident.recovered = true; return done({ failed: true, state, ...(read?.reap_reason ? { reap_reason: read.reap_reason } : {}) }); }
-        if (state != null && isDone(state)) { incident.recovered = true; return done({ woke: true, state }); }
       } catch { /* the next wait attempt reports the failure */ }
+      // BOT-2237 (lease f0607db4): a re-minted credential that is not the lease's owner can never
+      // observe it — re-registering the wait would park blind until the deadline.
+      if (read?.ownerLost) return authUnavailable(read.code ?? "NOT_LEASE_OWNER", read.message ?? "the re-authenticated agent does not own this lease");
+      const state = read?.state ?? null;
+      // Codex #1168 P1: an authenticated read of the lease proves the outage is over, so a later,
+      // independent rotation starts a fresh auth budget instead of inheriting this one.
+      if (state != null) recovered();
+      if (state != null && isFailed(state)) return done({ failed: true, state, ...(read?.reap_reason ? { reap_reason: read.reap_reason } : {}) });
+      if (state != null && isDone(state)) return done({ woke: true, state });
     }
   }
 }
@@ -2215,7 +2268,16 @@ export async function cmdUp(opts, {
   const resilientWait = async (isDone, isFailed) => {
     const result = await parkOnLease(leaseId, isDone, isFailed, {
       timeoutSec: opts.timeout, auth, wait: waitFn, reauth, ...(sleep ? { sleep } : {}), ...(now ? { now } : {}),
-      probe: async () => { const read = await call("get_stack_lease", { lease_id: leaseId }); return read?.ok && read.data?.success ? read.data : null; },
+      // Codex #1168 R2 P1: probe with the credential the park just re-minted — `auth`/`call`
+      // are only updated after the park returns, and an old-credential read would 401 and
+      // miss a grant or reap that landed during the outage.
+      probe: async (credential) => {
+        const read = await callTool("get_stack_lease", { lease_id: leaseId }, { auth: credential ?? auth });
+        if (read?.ok && read.data?.success) return read.data;
+        // BOT-2237: a re-authenticated principal that does not own the lease can never observe it.
+        if (read?.ok && read.data?.code === "NOT_LEASE_OWNER") return { ownerLost: true, code: "NOT_LEASE_OWNER", message: read.data.message ?? null };
+        return null;
+      },
     });
     if (result.credential && result.credential !== auth) {
       auth = result.credential;
@@ -2227,6 +2289,11 @@ export async function cmdUp(opts, {
   // Codex #1057 R2: a blip that occurred during any wait stays on EVERY terminal receipt
   // (recovered or not), so recoveries are not lost from the incident record.
   let upIncident = null;
+  // BOT-2237: a park that cannot re-authenticate ends typed, naming the lease.
+  const emitAuthUnavailable = (r) => emitResult(buildReceipt({
+    command: "up", outcome: "error", code: PARK_INCIDENT_REASONS.AUTH, lease_id: leaseId, slot,
+    error: `${r.error}; its server state is unknown to this command — inspect it with \`bb stack status ${leaseId}\` after \`bb doctor --fix\``,
+  }), opts, EXIT.AUTH);
   const emitPlain = emitResult;
   emitResult = (receipt, o, code) => emitPlain(upIncident && !receipt.platform_incident ? { ...receipt, platform_incident: upIncident } : receipt, o, code);
   // BOT-1711 (Codex R13): request_stack_lease returned an EXISTING active lease (reused),
@@ -2247,6 +2314,7 @@ export async function cmdUp(opts, {
     // Park zero-poll until the lease leaves the queue (granted in BOT-1187 order).
     // BOT-2013: a lease reaped while queued (orphaned_agent, …) is lost, not "left the queue".
     const parked = await resilientWait(nonQueued, isReaped);
+    if (parked.authUnavailable) return emitAuthUnavailable(parked);
     if (parked.failed) {
       return emitResult(buildReceipt({
         command: "up", outcome: "error", lease_id: leaseId, state: parked.state, slot,
@@ -2266,7 +2334,12 @@ export async function cmdUp(opts, {
       } else if (queueHold) {
         process.stderr.write(`${yellow("⚠")} stack: lease ${leaseId} could not keep its queue place (${queueHold.code ?? "error"}: ${queueHold.error})\n`);
       }
-      return emitResult(buildReceipt({ command: "up", outcome: "timeout", lease_id: leaseId, state: "queued", slot, error: `parked ${opts.timeout}s without capacity`, ...(ledger ? { unit_ledger: ledger } : {}), ...(queueHold ? { queue_hold: queueHold } : {}) }), opts, EXIT.TIMEOUT);
+      // BOT-2237: a park that ended inside an unrecovered platform incident never observed admission.
+      const blind = parked.incident && !parked.incident.recovered;
+      const why = blind
+        ? `parked ${opts.timeout}s inside a platform incident (${parked.incident.reason} since ${parked.incident.first_at}), so its admission was never observed`
+        : `parked ${opts.timeout}s without capacity`;
+      return emitResult(buildReceipt({ command: "up", outcome: "timeout", ...(blind ? { code: parked.incident.reason } : {}), lease_id: leaseId, state: "queued", slot, error: why, ...(ledger ? { unit_ledger: ledger } : {}), ...(queueHold ? { queue_hold: queueHold } : {}) }), opts, EXIT.TIMEOUT);
     }
     if (parked.auth) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: "unauthorized on wait stream" }), opts, EXIT.AUTH);
     if (parked.error) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: parked.error }), opts, EXIT.BACKEND);
@@ -2367,6 +2440,7 @@ export async function cmdUp(opts, {
         if (helperClaimed || helperAlreadyActive) {
           if (!helperAlreadyActive) {
             const active = await resilientWait(isActive, isReaped);
+            if (active.authUnavailable) return emitAuthUnavailable(active);
             if (active.timeout) return emitResult(buildReceipt({ command: "up", outcome: "timeout", lease_id: leaseId, state, slot, error: `a Helper claimed the provision job but it did not reach active in ${opts.timeout}s`, ...(active.incident ? { platform_incident: active.incident } : {}) }), opts, EXIT.TIMEOUT);
             if (active.auth) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: "unauthorized on wait stream" }), opts, EXIT.AUTH);
             if (active.failed) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, state: active.state, error: "lease was reaped before it became active" }), opts, EXIT.LEASE_FAILED);
@@ -2421,6 +2495,7 @@ export async function cmdUp(opts, {
       }
     } else {
       const active = await resilientWait(isActive, isReaped);
+      if (active.authUnavailable) return emitAuthUnavailable(active);
       if (active.timeout) return emitResult(buildReceipt({ command: "up", outcome: "timeout", lease_id: leaseId, state, slot, error: `provisioning did not reach active in ${opts.timeout}s (Helper may be down — retry with --local-exec)`, ...(active.incident ? { platform_incident: active.incident } : {}) }), opts, EXIT.TIMEOUT);
       if (active.auth) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: "unauthorized on wait stream" }), opts, EXIT.AUTH);
       if (active.failed) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, state: active.state, error: "lease was reaped before it became active" }), opts, EXIT.LEASE_FAILED);
@@ -2926,12 +3001,20 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
   // BOT-2099 (Codex #1057 R4): a platform blip recorded by ANY lease wait stays on EVERY
   // terminal result (interrupted, unreadable, thrown, …), not only the paths that remember it.
   // BOT-2142: where the lease ran (and how it was reached) is on every terminal result too.
+  // A receipt always says what happened to a port-block claim. Before allocation
+  // no claim exists, which is a safe, explicit disposition rather than omission.
   // BOT-1803: whether this run re-adopted / re-provisioned an idle lease, and its stack digest.
-  const incidentBox = { incident: null, execution: null, idleLease: null };
-  const result = await runStackLifecycleOnce(opts, childArgv, adapters, incidentBox);
+  const incidentBox = { incident: null, execution: null, idleLease: null, portBlock: { released: false, reason: "no port-block claim was created before stack allocation" } };
+  let result;
+  try {
+    result = await runStackLifecycleOnce(opts, childArgv, adapters, incidentBox);
+  } catch (error) {
+    result = { exitCode: classifyStackAuthError(error), outcome: "error", error: String(error?.message ?? error) };
+  }
   const withIncident = incidentBox.incident && !result?.incident ? { ...result, incident: incidentBox.incident } : result;
   const withExecution = incidentBox.execution && !withIncident?.execution ? { ...withIncident, execution: incidentBox.execution } : withIncident;
-  return incidentBox.idleLease && !withExecution?.idleLease ? { ...withExecution, idleLease: incidentBox.idleLease } : withExecution;
+  const withIdle = incidentBox.idleLease && !withExecution?.idleLease ? { ...withExecution, idleLease: incidentBox.idleLease } : withExecution;
+  return incidentBox.portBlock && !withIdle?.portBlock ? { ...withIdle, portBlock: incidentBox.portBlock } : withIdle;
 }
 
 async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
@@ -2947,7 +3030,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
   // Resolve the initial credential with the same short-circuit as before (an injected
   // auth skips the network resolve), then seed the retrying caller so no extra await
   // point is introduced before the first RPC.
-  const initialAuth = adapters.auth ?? await stackAuthHeader();
+  const initialAuth = adapters.auth ?? await (adapters.authProvider ?? stackAuthHeader)();
   const rpc = retryingStackCall({
     callTool: adapters.callTool ?? callToolJson,
     refresh: refreshAuth,
@@ -2971,6 +3054,13 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
   let parkIncident = null;
   // BOT-2099: a best-effort diagnostic read (ledger, persisted Helper reason) never throws.
   const readLeaseQuietly = async (id) => { try { return await api.get?.(id) ?? null; } catch { return null; } };
+  // BOT-2237: the typed end of a park that cannot re-authenticate — names the lease, never a
+  // capacity timeout. Exit AUTH: the caller's credential, not the lease, is what failed.
+  const authUnavailableResult = (waited, phase, cleanup) => ({
+    exitCode: EXIT.AUTH, outcome: "error", code: PARK_INCIDENT_REASONS.AUTH, leaseId: leaseId ?? null, cleanup,
+    error: `${waited.error} while the lease was ${phase}; its server state is unknown to this run — inspect it with \`bb stack status ${leaseId}\` after \`bb doctor --fix\``,
+    ...(parkIncident ? { incident: parkIncident } : {}),
+  });
   const resilientWait = async (leaseIdArg, done, failed, options) => {
     const result = await parkOnLease(leaseIdArg, done, failed, {
       ...options, wait,
@@ -2979,7 +3069,13 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
       // credential the RPC layer itself refreshed during the probe is never overwritten.
       reauth: async (_current, kind) => { rpc.adopt(await reauth(rpc.auth, kind)); return rpc.auth; },
       syncCredential: () => rpc.auth,
-      probe: async () => { const read = await api.get(leaseIdArg); return read?.ok && read.data?.success ? read.data : null; },
+      probe: async () => {
+        const read = await api.get(leaseIdArg);
+        if (read?.ok && read.data?.success) return read.data;
+        // BOT-2237: the server answered and says this principal does not own the lease.
+        if (read?.ok && read.data?.code === "NOT_LEASE_OWNER") return { ownerLost: true, code: "NOT_LEASE_OWNER", message: read.data.message ?? null };
+        return null;
+      },
       ...(adapters.sleep ? { sleep: adapters.sleep } : {}),
       ...(adapters.now ? { now: adapters.now } : {}),
       log: stderrLine,
@@ -3118,7 +3214,12 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
     const postCheck = durableReapFailure(recheck);
     if (postCheck) return cleanupResult = postCheck;
     return cleanupResult = { ok: false, error: reaped?.timeout ? `signed reap did not arrive within ${opts.reapTimeout}s` : (reaped?.error || "signed reap was not proven") };
-    })();
+    })().catch((error) => {
+      // BOT-2237: a release/read whose session renewal THROWS (e.g. `session_mint_invalid`
+      // once the cache names another agent) is a failed cleanup of THIS lease — never an
+      // exception that escapes the lifecycle and loses the lease id from the receipt.
+      return cleanupResult = { ok: false, error: `cleanup could not run: ${error?.code ? `${error.code}: ` : ""}${error?.message ?? error}` };
+    });
     return cleanupPromise;
   };
 
@@ -3458,14 +3559,25 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
       activeWaitAbort = null;
       const incident = parkIncident ? { incident: parkIncident } : {};
       if (parked?.interrupted || receivedSignal) { const cleanup = await requestCleanup(); return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup, ...incident }; }
+      if (parked?.authUnavailable) {
+        const cleanup = await requestCleanup();
+        return authUnavailableResult(parked, "queued", cleanup);
+      }
       if (parked?.timeout) {
         // BOT-2099: say who held the units for the whole park, read before the release.
         const final = await readLeaseQuietly(leaseId);
         const ledger = final?.ok && final.data?.success ? final.data.ledger ?? null : null;
         for (const line of describeUnitLedger(ledger)) stderrLine(`stack: ${line}`);
         const cleanup = await requestCleanup();
-        const during = parkIncident && !parkIncident.recovered ? ` (platform incident: ${parkIncident.reason} since ${parkIncident.first_at})` : "";
-        return { exitCode: EXIT.TIMEOUT, outcome: "timeout", leaseId, error: `parked ${opts.timeout}s without capacity${during}`, cleanup, ...incident, ...(ledger ? { ledger } : {}) };
+        // BOT-2237: a park that ended inside an unrecovered platform incident never observed the
+        // lease's admission, so it must not claim the host had no capacity.
+        if (parkIncident && !parkIncident.recovered) {
+          return {
+            exitCode: EXIT.TIMEOUT, outcome: "timeout", code: parkIncident.reason, leaseId, cleanup, ...incident, ...(ledger ? { ledger } : {}),
+            error: `lease ${leaseId} parked ${opts.timeout}s inside a platform incident (${parkIncident.reason} since ${parkIncident.first_at}), so its admission was never observed`,
+          };
+        }
+        return { exitCode: EXIT.TIMEOUT, outcome: "timeout", leaseId, error: `parked ${opts.timeout}s without capacity`, cleanup, ...incident, ...(ledger ? { ledger } : {}) };
       }
       if (parked?.failed) { const cleanup = await requestCleanup(); return { exitCode: EXIT.LEASE_FAILED, outcome: "error", leaseId, error: `lease was ${parked.state} while queued (reap_reason=${parked.reap_reason ?? "unknown"})`, cleanup, ...incident }; }
       if (parked?.error || parked?.auth) { const cleanup = await requestCleanup(); return { exitCode: parked?.auth ? EXIT.AUTH : EXIT.LEASE_FAILED, outcome: "error", leaseId, error: parked?.error || "lease did not leave queue", cleanup, ...incident }; }
@@ -3483,6 +3595,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
       const active = await resilientWait(leaseId, (s) => s === "active", isReaped, { timeoutSec: provisionTimeout, auth: rpc.auth, signal: activeWaitAbort.signal });
       activeWaitAbort = null;
       if (active?.interrupted || receivedSignal) { const cleanup = await requestCleanup(); return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup }; }
+      if (active?.authUnavailable) { const cleanup = await requestCleanup(); return authUnavailableResult(active, "provisioning", cleanup); }
       if (active?.timeout) {
         // BOT-1822 belt-and-braces: the zero-poll wait can still miss a live
         // `stack_lease` signal (an older server before this fix, a relay hiccup,
@@ -3602,7 +3715,10 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
       let usage = null;
       if (!servedRemote) { try { usage = await measureUsage(usageLease); } catch { usage = null; } }
       if (fencing || childResult) return;
-      const touched = usage ? await api.touch(leaseId, usage) : await api.touch(leaseId);
+      // BOT-2237: a touch whose renewal throws is a failed heartbeat (fence), never an
+      // unhandled rejection from a timer callback.
+      let touched;
+      try { touched = usage ? await api.touch(leaseId, usage) : await api.touch(leaseId); } catch (error) { touched = { ok: false, error: String(error?.message ?? error) }; }
       if (touched?.ok && isPreemptedAnswer(touched.data)) {
         // BOT-1803 (AC-11): the control plane reclaimed this lease's capacity. Stop the
         // child and end typed — never the generic lost-lease wording.
@@ -3678,7 +3794,21 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
     // runs, so its block stays reserved; so does one this run must not free (retainBlock).
     const provenReap = !retainBlock && (!leaseProvisioned
       || (cleanupResult?.ok === true && (cleanupResult.state === "reaped" || cleanupResult.state == null)));
-    execution?.release?.({ reaped: provenReap });
+    if (execution?.release) {
+      // BOT-2237 (AC-3): every terminal path says what happened to the host port-block claim,
+      // from the release's VERIFIED result (Codex #1168 P2), never from the reap alone.
+      let disposition = null;
+      let releaseError = null;
+      try { disposition = execution.release({ reaped: provenReap }); } catch (error) { releaseError = error; }
+      const retained = "; the claim is kept until its retention expires";
+      incidentBox.portBlock = !provenReap
+        ? { released: false, reason: retainBlock
+          ? `a lease on this stack slot may still hold its ports (idle, unreaped, or another live batch)${retained} so its stack is never double-booked`
+          : `reap not proven (${cleanupResult?.error ?? "no cleanup ran"})${retained} so a stack the Helper may still run is never double-booked` }
+        : releaseError || disposition?.released === false
+          ? { released: false, reason: `the reap was proven but the claim could not be removed (${releaseError?.message ?? "it is still on disk"})${retained}` }
+          : { released: true };
+    }
     for (const [signal, handler] of signalHandlers) signals.off?.(signal, handler);
   }
 }
@@ -3718,6 +3848,8 @@ export function runReceiptFields(result) {
     // incident (reason, window, attempts, recovered) — never as the lease's outcome.
     ...(result.incident ? { platform_incident: result.incident } : {}),
     ...(result.ledger ? { unit_ledger: result.ledger } : {}),
+    // BOT-2237: whether this run handed its host port block back, or kept it and why.
+    ...(result.portBlock ? { port_block: result.portBlock } : {}),
   };
 }
 

package/src/wait-core.mjs

@@ -1241,7 +1241,9 @@ const FEED_PROBE_BUDGET_MS = 2000;
  *   - reconnects when the stream ends, resuming from the last seen seq, and
  *     de-duplicates a repeated seq across the reconnect (exactly-once)
  *   - maps server error frames to AUTH / CURSOR_EXPIRED
- *   - gives up with BACKEND after the reconnect budget is exhausted
+ *   - gives up with BACKEND after `maxReconnects` CONSECUTIVE failed connects
+ *     (BOT-2351: any delivered frame resets the count; healthy rotations never
+ *     exhaust it, so only the wall deadline bounds a healthy wait)
  *
  * @returns {Promise<{receipt: object, exitCode: number}>}
  */
@@ -1340,8 +1342,14 @@ export async function runWaitLoop({
   const pendingReplayBeaconFrames = [];
   let pendingReplaySupersession = null;
   const degraded = [];
+  // BOT-2351: `reconnects` counts every stream rotation for the receipt (an operator
+  // still sees churn); `failures` counts CONSECUTIVE failed connects — a connect that
+  // threw, or a stream that ended before delivering a single frame — and is reset by any
+  // delivered frame. Only `failures` is bounded by maxReconnects, so a healthy wait whose
+  // relay recycles its stream every few minutes is ended only by its match or its deadline.
   let reconnects = 0;
   let failures = 0;
+  const noteBackendUnreachable = () => { if (!degraded.includes("backend_unreachable")) degraded.push("backend_unreachable"); };
 
   // Internal one-shot alarms (the deadline, the pre-deadline feed probe, reconnect backoff)
   // are REAL timers via the default sleep. Each exists only to resolve a race — once the loop
@@ -1649,9 +1657,9 @@ export async function runWaitLoop({
       stream = await connect(connectCursor);
     } catch (err) {
       failures += 1;
-      degraded.push("backend_unreachable");
+      noteBackendUnreachable();
       if (failures > maxReconnects) {
-        return finalize("error", { exitCode: EXIT.BACKEND, error: String(err && err.message || err) });
+        return finalize("error", { exitCode: EXIT.BACKEND, error: `reconnect budget exhausted: ${String(err && err.message || err)}` });
       }
       const waitMs = backoff(failures);
       const raced = await Promise.race([
@@ -1664,6 +1672,8 @@ export async function runWaitLoop({
 
     const iterator = stream[Symbol.asyncIterator]();
     let streamEnded = false;
+    // BOT-2351: whether this connection delivered anything at all (see `failures`).
+    let deliveredFrame = false;
     // BOT-1148 (Codex P2): the pending iterator.next() is HOISTED across loop
     // iterations. When a non-frame racer wins (a beacon-stale confirm that a recovery
     // then cancels), we must NOT abandon the in-flight read and start a fresh one —
@@ -1714,6 +1724,9 @@ export async function runWaitLoop({
         break;
       }
       if (!frame) continue;
+      // BOT-2351: the relay answered on this connection, so the backend is reachable —
+      // the consecutive-failure budget starts over.
+      if (!deliveredFrame) { deliveredFrame = true; failures = 0; }
 
       if (frame.event === "connected") {
         let connected = {};
@@ -1968,12 +1981,17 @@ export async function runWaitLoop({
       await checkpointNonterminalFrame();
     }
 
-    // Stream ended without a match: reconnect from the cursor.
+    // Stream ended without a match: reconnect from the cursor. A stream that delivered
+    // frames was a healthy rotation (the relay recycles streams every few minutes); one
+    // that ended before any frame is a failed connect and counts toward the budget.
     reconnects += 1;
-    if (reconnects > maxReconnects) {
-      return finalize("error", { exitCode: EXIT.BACKEND, error: "reconnect budget exhausted" });
+    if (!deliveredFrame) {
+      failures += 1;
+      if (failures > maxReconnects) {
+        return finalize("error", { exitCode: EXIT.BACKEND, error: "reconnect budget exhausted" });
+      }
     }
-    const waitMs = backoff(reconnects);
+    const waitMs = backoff(Math.max(1, failures));
     const raced = await Promise.race([deadlinePromise, sleepFn(waitMs).then(() => ({ __retry: true }))]);
     if (raced && raced.__deadline) return fireReplayGraceExpired();
   }