@botbuddy/cli 1.40.9 → 1.40.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botbuddy/cli",
-  "version": "1.40.9",
+  "version": "1.40.10",
   "description": "BotBuddy — Swarm coordination CLI for multi-agent workflows",
   "type": "module",
   "bin": {

package/src/stack.mjs

@@ -352,6 +352,30 @@ export function describeQueuePosition(d) {
 }
 
 /** BOT-2019: the queue fields a server returned for a queued lease, for a receipt. */
+/**
+ * BOT-2099: the human lines for the server's unit ledger (`stack_lease_unit_ledger`) — who
+ * holds the host's capacity units and since when — printed whenever a lease parks, so a
+ * starved lease names its holders instead of only its position. Other tenants' leases are
+ * tenant + state + since; unleased stacks are 5-character prefixes (the server redacts).
+ */
+export function describeUnitLedger(ledger) {
+  if (!ledger || typeof ledger !== "object") return [];
+  const cap = ledger.capacity ?? {};
+  const lines = [`host ${ledger.host_key ?? "?"}: ${ledger.capacity_units ?? "?"} units — verdict ${ledger.verdict ?? "unknown"}`
+    + ` (free ${cap.free_slots ?? "?"}, budget free ${cap.budget_free_slots ?? "?"}, unleased ${cap.observed_unmanaged_projects ?? "?"})`];
+  for (const l of Array.isArray(ledger.leases) ? ledger.leases : []) {
+    const who = l.lease_id ? `lease ${l.lease_id}` : `${l.tenant_id ?? "?"} lease`;
+    const what = [l.ticket_id, l.slot, l.project].filter(Boolean).join(" ");
+    const flags = [l.reap_reason ? `reap_reason ${l.reap_reason}` : null, l.no_containers === true ? "no containers" : null].filter(Boolean);
+    lines.push(`  ${who} ${l.state} since ${l.since ?? "?"}${what ? ` [${what}]` : ""}${flags.length ? ` (${flags.join(", ")})` : ""}`);
+  }
+  for (const u of Array.isArray(ledger.unmanaged) ? ledger.unmanaged : []) {
+    lines.push(`  unleased ${u.project_prefix ?? "?"} since ${u.since ?? "unknown"}${u.allowlisted ? " (allowlisted)" : ""}`);
+  }
+  if (ledger.permits) lines.push(`  managed-container permits: ${ledger.permits}`);
+  return lines;
+}
+
 function queueReceiptFields(d) {
   return {
     queue_position: d?.queue_position ?? null,
@@ -679,6 +703,9 @@ export function retryingStackCall({
       return r;
     },
     get auth() { return auth; },
+    // BOT-2099: a park that re-authenticated through an outage hands its fresh
+    // credential back, so every later RPC (get/touch/release) uses it.
+    adopt(fresh) { if (fresh) auth = fresh; },
   };
 }
 
@@ -919,7 +946,7 @@ async function registerLeaseWait(leaseId, timeoutSec, auth, signal, fetchImpl =
     body: JSON.stringify({ action: "register", conditions: [{ type: "lease", params: { id: leaseId } }], deadline, mode: "any" }),
     signal,
   });
-  if (!res.ok) throw new Error(`register responded ${res.status}`);
+  if (!res.ok) throw Object.assign(new Error(`register responded ${res.status}`), { status: res.status });
   const body = await res.json();
   return { waitSessionId: body.wait_session_id ?? null, cursorStart: body.cursor_start ?? null };
 }
@@ -987,6 +1014,15 @@ export async function waitForLease(leaseId, isDone, isFailed, { timeoutSec, auth
       const { result, status } = resultForAbortReason(ac.signal.reason);
       return finish(result, status);
     }
+    // BOT-2099 (hop 6, 2026-10-06 01:54Z): a registration rejected by an auth or
+    // availability blip is RETRIED by the park (parkOnLease), never downgraded to a
+    // live-only park that is invisible on /waits and keeps nothing alive server-side.
+    // Only a definite client-side rejection (another 4xx) keeps the old live-only path.
+    const status = err?.status ?? null;
+    if (status === 401 || status === 403) return finish({ auth: true, phase: "register" }, "error");
+    if (status == null || status === 408 || status === 429 || status >= 500) {
+      return finish({ error: `wait registration: ${err?.message ?? err}`, phase: "register" }, "error");
+    }
     process.stderr.write(`${yellow("⚠")} stack: wait registration failed (${err?.message ?? err}); the lease will not appear on /waits — parking live-only.\n`);
   }
   const url = new URL(eventStreamBase());
@@ -1005,7 +1041,7 @@ export async function waitForLease(leaseId, isDone, isFailed, { timeoutSec, auth
     return finish({ error: `sse connect: ${err?.message ?? err}` }, "error");
   }
   if (res.status === 401 || res.status === 403) return finish({ auth: true }, "error");
-  if (!res.ok || !res.body) return finish({ error: `sse responded ${res.status}` }, "error");
+  if (!res.ok || !res.body) return finish({ error: `sse responded ${res.status}`, status: res.status }, "error");
   const decoder = new TextDecoder();
   let buf = "";
   // BOT-1827: record a match/failure locally and `break` out of BOTH loops
@@ -1064,6 +1100,149 @@ export async function waitForLease(leaseId, isDone, isFailed, { timeoutSec, auth
   return finish({ timeout: true }, "timeout");
 }
 
+/**
+ * BOT-2099 — the platform-incident reasons a park can record. A park NEVER ends with an
+ * auth or mint error as its outcome: an outage is retried for the park's whole remaining
+ * deadline and, if it outlasts it, the park ends as an ordinary `timeout` that carries
+ * this incident (who to blame is the platform, not the lease).
+ */
+export const PARK_INCIDENT_REASONS = Object.freeze({
+  AUTH: "platform_auth_unavailable",       // 401/403 on the wait, or the session re-mint rejected
+  UNREACHABLE: "platform_unreachable",     // transport failure / 5xx / malformed relay response
+});
+export const PARK_BACKOFF = Object.freeze({ initialMs: 2_000, maxMs: 60_000 });
+
+function abortableSleep(ms, signal) {
+  return new Promise((resolve) => {
+    if (signal?.aborted) return resolve();
+    // Deliberately REFERENCED (Codex #1057 P1): while a park backs off, this timer may be the
+    // only live handle; unreferencing it would let Node exit before the retry, the cleanup
+    // or the receipt, leaving the lease behind.
+    const timer = setTimeout(() => { signal?.removeEventListener?.("abort", onAbort); resolve(); }, Math.max(0, ms));
+    const onAbort = () => { clearTimeout(timer); resolve(); };
+    signal?.addEventListener?.("abort", onAbort, { once: true });
+  });
+}
+
+/**
+ * BOT-2099: re-establish the credential a parked lease waits with. On a transport failure
+ * the credential is fine — keep it. On an auth rejection, first re-read (another process in
+ * this worktree may already have re-minted, and minting again would supersede it), then
+ * clear the per-worktree cache and self-heal once. Only a well-formed agent SESSION token
+ * replaces a session credential: the operator fallback is a different principal that
+ * cannot see the lease (BOT-1750 R8), so it is never adopted. Throws the mint error, which
+ * the park treats as one more failed attempt.
+ */
+export async function reauthenticateParkedLease(current, kind, {
+  cwd = process.cwd(), authProvider = stackAuthHeader, clear = clearAgentState,
+} = {}) {
+  if (kind !== "auth") return current;
+  const source = current?.__source ?? null;
+  if (source !== "cache" && source !== "minted") return current;
+  const isFreshSession = (h) => (h?.__source === "cache" || h?.__source === "minted")
+    && typeof h["x-agent-api-key"] === "string" && AGENT_KEY_RE.test(h["x-agent-api-key"])
+    && h["x-agent-api-key"] !== current["x-agent-api-key"];
+  const reread = await authProvider();
+  if (isFreshSession(reread)) return reread;
+  await clear(cwd);
+  const minted = await authProvider();
+  return isFreshSession(minted) ? minted : current;
+}
+
+/**
+ * BOT-2099: park on `lease:<id>` and SURVIVE control-plane auth/availability blips.
+ *
+ * `waitForLease` ends on the first 401/403, transport error or failed registration. Before
+ * this, every caller turned that into the lease's outcome — releasing a queued lease at the
+ * head of the queue over a one-second 401 (hop 6) and ending every parked lease in a
+ * server-side auth outage with `session_mint_unauthorized` (hop 7). Here such a result is
+ * retried with bounded exponential backoff (PARK_BACKOFF) until the park's OWN deadline:
+ * re-authenticate on an auth failure (`reauth`), re-register the wait (a fresh registration
+ * replays the lease's current state), and — after a recovery — read the lease directly
+ * (`probe`) so a grant or reap that landed during the outage is never missed. Mint errors
+ * thrown by `reauth` are attempts, not outcomes. The result is waitForLease's shape plus
+ * `credential` (the headers that finally worked) and, when any blip occurred, `incident`.
+ */
+export async function parkOnLease(leaseId, isDone, isFailed, {
+  timeoutSec, deadlineMs = null, auth, signal = null,
+  wait = waitForLease, reauth = (current, kind) => reauthenticateParkedLease(current, kind),
+  probe = null, sleep = (ms) => abortableSleep(ms, signal), now = () => Date.now(),
+  backoff = PARK_BACKOFF, log = (line) => process.stderr.write(`${line}\n`),
+  // Codex #1057 R3: a caller whose RPC layer can itself re-mint (stack run's
+  // retryingStackCall) owns the credential; the park re-reads it before every attempt so a
+  // credential minted during a probe is never overwritten by an older one.
+  syncCredential = null,
+} = {}) {
+  const deadline = deadlineMs ?? now() + timeoutSec * 1000;
+  let credential = auth;
+  let incident = null;
+  let delay = backoff.initialMs;
+  const note = (kind, detail, code = null) => {
+    const at = new Date(now()).toISOString();
+    if (!incident) incident = { reason: null, first_at: at, attempts: 0, recovered: false };
+    incident.recovered = false; // a new failure re-opens a recovered incident
+    incident.attempts += 1;
+    incident.last_at = at;
+    incident.reason = kind === "auth" ? PARK_INCIDENT_REASONS.AUTH : PARK_INCIDENT_REASONS.UNREACHABLE;
+    incident.last_error = String(detail).slice(0, 300);
+    incident.last_error_code = code;
+  };
+  const done = (result) => ({ ...result, credential, ...(incident ? { incident } : {}) });
+  for (;;) {
+    if (signal?.aborted) return done({ interrupted: true });
+    const remainingMs = deadline - now();
+    if (remainingMs <= 0) return done({ timeout: true });
+    if (syncCredential) credential = syncCredential() ?? credential;
+    let result;
+    try {
+      result = await wait(leaseId, isDone, isFailed, { timeoutSec: Math.ceil(remainingMs / 1000), deadlineMs: deadline, auth: credential, signal });
+    } catch (err) {
+      result = { error: `lease wait: ${err?.message ?? err}` };
+    }
+    // Codex #1057 R6: a permanent client error (400/404/… — a route, protocol or config
+    // mismatch) is not a blip: fail at once. Only auth, transport (no status), 408/429 and
+    // 5xx are retried.
+    const permanent = result?.error && result.status != null && result.status !== 408 && result.status !== 429 && result.status < 500;
+    if (permanent) return incident ? done(result) : result;
+    if (!result?.auth && !result?.error) {
+      // Codex #1057 R5: any attempt that did not fail on auth/transport (a match, a reap, or an
+      // ordinary timeout on a healthy stream) means the platform recovered.
+      if (incident) incident.recovered = true;
+      // A wait that ends on its own timer is a real timeout; never the outage's last word.
+      if (result?.timeout && incident && now() < deadline) continue;
+      return incident ? done(result) : result;
+    }
+    const kind = result.auth ? "auth" : "transport";
+    note(kind, result.auth ? `unauthorized (${result.phase ?? "stream"})` : result.error, result.auth ? "unauthorized" : null);
+    log(`${yellow("⚠")} stack: lease ${leaseId} wait interrupted by a platform ${kind === "auth" ? "auth" : "availability"} blip (${incident.last_error}); still parked — retrying in ${Math.round(delay / 1000)}s (attempt ${incident.attempts}).`);
+    // Back off (bounded) before re-authenticating, then retry; never past the deadline.
+    for (;;) {
+      await sleep(Math.min(delay, Math.max(0, deadline - now())));
+      delay = Math.min(delay * 2, backoff.maxMs);
+      if (signal?.aborted) return done({ interrupted: true });
+      if (now() >= deadline) return done({ timeout: true });
+      if (kind !== "auth") break; // a transport failure keeps its (valid) credential
+      try {
+        if (syncCredential) credential = syncCredential() ?? credential;
+        credential = (await reauth(credential, kind)) ?? credential;
+        break;
+      } catch (err) {
+        note("auth", err?.message ?? err, err?.code ?? null);
+        log(`${yellow("⚠")} stack: lease ${leaseId} re-authentication failed (${err?.code ?? err?.message ?? err}); still parked — retrying in ${Math.round(delay / 1000)}s.`);
+      }
+    }
+    if (probe) {
+      try {
+        const read = await probe(credential);
+        if (syncCredential) credential = syncCredential() ?? credential;
+        const state = read?.state ?? null;
+        if (state != null && isFailed(state)) { incident.recovered = true; return done({ failed: true, state, ...(read?.reap_reason ? { reap_reason: read.reap_reason } : {}) }); }
+        if (state != null && isDone(state)) { incident.recovered = true; return done({ woke: true, state }); }
+      } catch { /* the next wait attempt reports the failure */ }
+    }
+  }
+}
+
 const isActive = (s) => s === "active";
 const nonQueued = (s) => s != null && s !== "queued";
 // BOT-1822 (Codex round 3 P2): `reap_failed` is a distinct terminal wake
@@ -1534,6 +1713,9 @@ export async function cmdUp(opts, {
   refreshAuth = refreshRejectedCacheAuth,
   reportLaneStatus = writeLaneStatus,
   waitCapacity = defaultWaitCapacity,
+  reauth = (current, kind) => reauthenticateParkedLease(current, kind),
+  sleep = null,
+  now = null,
 } = {}) {
   let slot;
   try { slot = deriveSlot(opts); } catch (e) {
@@ -1671,6 +1853,25 @@ export async function cmdUp(opts, {
   }
   let leaseId = d.lease_id;
   let state = d.state;
+  // BOT-2099: every lease wait in `stack up` (queue park, Helper provision, Helper takeover)
+  // survives auth/availability blips (parkOnLease) and adopts the re-minted credential.
+  const resilientWait = async (isDone, isFailed) => {
+    const result = await parkOnLease(leaseId, isDone, isFailed, {
+      timeoutSec: opts.timeout, auth, wait: waitFn, reauth, ...(sleep ? { sleep } : {}), ...(now ? { now } : {}),
+      probe: async () => { const read = await call("get_stack_lease", { lease_id: leaseId }); return read?.ok && read.data?.success ? read.data : null; },
+    });
+    if (result.credential && result.credential !== auth) {
+      auth = result.credential;
+      call = (name, args, callOptions = {}) => callTool(name, args, { ...callOptions, auth });
+    }
+    if (result.incident) upIncident = result.incident;
+    return result;
+  };
+  // Codex #1057 R2: a blip that occurred during any wait stays on EVERY terminal receipt
+  // (recovered or not), so recoveries are not lost from the incident record.
+  let upIncident = null;
+  const emitPlain = emitResult;
+  emitResult = (receipt, o, code) => emitPlain(upIncident && !receipt.platform_incident ? { ...receipt, platform_incident: upIncident } : receipt, o, code);
   // BOT-1711 (Codex R13): request_stack_lease returned an EXISTING active lease (reused),
   // rather than a freshly minted queued/provisioning one. A reused lease was not provisioned
   // in this invocation, so its recorded metadata cannot be trusted for a local-exec target.
@@ -1682,12 +1883,13 @@ export async function cmdUp(opts, {
     const where = describeQueuePosition(d);
     reportLaneStatus({ phase: "waiting", reason: "queued_for_capacity", detail: where || "host at capacity" });
     process.stderr.write(`stack: lease ${leaseId} queued — ${where}\n`);
+    for (const line of describeUnitLedger(d.ledger)) process.stderr.write(`stack: ${line}\n`);
     if (opts.noWait) {
-      return emitResult(buildReceipt({ command: "up", outcome: "queued", lease_id: leaseId, state, host_key: d.host_key, slot, queued: true, ...queueReceiptFields(d), holders: d.holders }), opts, EXIT.QUEUED);
+      return emitResult(buildReceipt({ command: "up", outcome: "queued", lease_id: leaseId, state, host_key: d.host_key, slot, queued: true, ...queueReceiptFields(d), holders: d.holders, ...(d.ledger ? { unit_ledger: d.ledger } : {}) }), opts, EXIT.QUEUED);
     }
     // Park zero-poll until the lease leaves the queue (granted in BOT-1187 order).
     // BOT-2013: a lease reaped while queued (orphaned_agent, …) is lost, not "left the queue".
-    const parked = await waitFn(leaseId, nonQueued, isReaped, { timeoutSec: opts.timeout, auth });
+    const parked = await resilientWait(nonQueued, isReaped);
     if (parked.failed) {
       return emitResult(buildReceipt({
         command: "up", outcome: "error", lease_id: leaseId, state: parked.state, slot,
@@ -1695,7 +1897,13 @@ export async function cmdUp(opts, {
         error: `lease was ${parked.state} while queued (reap_reason=${parked.reap_reason ?? "unknown"}) — request a new lease`,
       }), opts, EXIT.LEASE_FAILED);
     }
-    if (parked.timeout) return emitResult(buildReceipt({ command: "up", outcome: "timeout", lease_id: leaseId, state: "queued", slot, error: `parked ${opts.timeout}s without capacity` }), opts, EXIT.TIMEOUT);
+    if (parked.timeout) {
+      // Codex #1057 R2: say who held the units at the deadline (re-read; best effort).
+      let ledger = d.ledger ?? null;
+      try { const read = await call("get_stack_lease", { lease_id: leaseId }); if (read?.ok && read.data?.success && read.data.ledger) ledger = read.data.ledger; } catch { /* keep the request-time ledger */ }
+      for (const line of describeUnitLedger(ledger)) process.stderr.write(`stack: ${line}\n`);
+      return emitResult(buildReceipt({ command: "up", outcome: "timeout", lease_id: leaseId, state: "queued", slot, error: `parked ${opts.timeout}s without capacity`, ...(ledger ? { unit_ledger: ledger } : {}) }), opts, EXIT.TIMEOUT);
+    }
     if (parked.auth) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: "unauthorized on wait stream" }), opts, EXIT.AUTH);
     if (parked.error) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: parked.error }), opts, EXIT.BACKEND);
     state = parked.state || "provisioning";
@@ -1793,8 +2001,8 @@ export async function cmdUp(opts, {
         const helperAlreadyActive = reserved.data?.code === "LEASE_NOT_PROVISIONING" && reserved.data?.state === "active";
         if (helperClaimed || helperAlreadyActive) {
           if (!helperAlreadyActive) {
-            const active = await waitFn(leaseId, isActive, isReaped, { timeoutSec: opts.timeout, auth });
-            if (active.timeout) return emitResult(buildReceipt({ command: "up", outcome: "timeout", lease_id: leaseId, state, slot, error: `a Helper claimed the provision job but it did not reach active in ${opts.timeout}s` }), opts, EXIT.TIMEOUT);
+            const active = await resilientWait(isActive, isReaped);
+            if (active.timeout) return emitResult(buildReceipt({ command: "up", outcome: "timeout", lease_id: leaseId, state, slot, error: `a Helper claimed the provision job but it did not reach active in ${opts.timeout}s`, ...(active.incident ? { platform_incident: active.incident } : {}) }), opts, EXIT.TIMEOUT);
             if (active.auth) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: "unauthorized on wait stream" }), opts, EXIT.AUTH);
             if (active.failed) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, state: active.state, error: "lease was reaped before it became active" }), opts, EXIT.LEASE_FAILED);
             if (active.error) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: active.error }), opts, EXIT.BACKEND);
@@ -1832,8 +2040,8 @@ export async function cmdUp(opts, {
         }
       }
     } else {
-      const active = await waitFn(leaseId, isActive, isReaped, { timeoutSec: opts.timeout, auth });
-      if (active.timeout) return emitResult(buildReceipt({ command: "up", outcome: "timeout", lease_id: leaseId, state, slot, error: `provisioning did not reach active in ${opts.timeout}s (Helper may be down — retry with --local-exec)` }), opts, EXIT.TIMEOUT);
+      const active = await resilientWait(isActive, isReaped);
+      if (active.timeout) return emitResult(buildReceipt({ command: "up", outcome: "timeout", lease_id: leaseId, state, slot, error: `provisioning did not reach active in ${opts.timeout}s (Helper may be down — retry with --local-exec)`, ...(active.incident ? { platform_incident: active.incident } : {}) }), opts, EXIT.TIMEOUT);
       if (active.auth) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: "unauthorized on wait stream" }), opts, EXIT.AUTH);
       if (active.failed) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, state: active.state, error: "lease was reaped before it became active" }), opts, EXIT.LEASE_FAILED);
       if (active.error) return emitResult(buildReceipt({ command: "up", outcome: "error", lease_id: leaseId, error: active.error }), opts, EXIT.BACKEND);
@@ -1900,11 +2108,14 @@ export async function cmdStatus(leaseId, opts, {
   // ticket it is scored under; a null position is reported, never a crash.
   const queued = g.state === "queued";
   if (queued) stderr(`stack: lease ${leaseId} queued — ${describeQueuePosition(g)}`);
+  if (queued) for (const line of describeUnitLedger(g.ledger)) stderr(`stack: ${line}`);
   return emitResult(buildReceipt({
     command: "status", outcome: g.state, lease_id: leaseId, state: g.state,
     host_key: g.host_key, slot: g.slot, connection: g.connection,
     ...(queued ? queueReceiptFields(g) : { queue_position: g.queue_position }),
     idle_ttl_secs: g.idle_ttl_secs, queued: g.queued,
+    // BOT-2099 (Codex #1057 R3): machine callers get the unit ledger of a queued lease too.
+    ...(queued && g.ledger ? { unit_ledger: g.ledger } : {}),
     // BOT-1960: a lease fenced in `reaping` says since when, and — once the server gave up
     // retrying its failed reap — that it is reap_failed and why.
     ...(g.reap_requested_at ? { reap_requested_at: g.reap_requested_at } : {}),
@@ -2257,6 +2468,14 @@ export async function materializeLeasedTestConfig(worktreeRoot, leaseId, connect
  * without Docker or a live BotBuddy service.
  */
 export async function runStackLifecycle(opts, childArgv, adapters = {}) {
+  // BOT-2099 (Codex #1057 R4): a platform blip recorded by ANY lease wait stays on EVERY
+  // terminal result (interrupted, unreadable, thrown, …), not only the paths that remember it.
+  const incidentBox = { incident: null };
+  const result = await runStackLifecycleOnce(opts, childArgv, adapters, incidentBox);
+  return incidentBox.incident && !result?.incident ? { ...result, incident: incidentBox.incident } : result;
+}
+
+async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
   // BOT-1750 (Codex R9/R11 P2): `stack run` is a lease-creating entry point AND a
   // long-lived one (queue → provision → child → heartbeat → cleanup). A cached token
   // can be rotated server-side at ANY of those points, so EVERY RPC — not just the
@@ -2283,6 +2502,29 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
   };
   const machineUuidFn = adapters.machineUuidFn ?? machineUuid;
   const wait = adapters.wait ?? ((leaseId, done, failed, options) => waitForLease(leaseId, done, failed, options));
+  // BOT-2099: the queue park and the provision wait survive auth/availability blips.
+  // Production re-authenticates through the self-heal path; an injected-auth test double
+  // without an explicit `reauth` keeps its credential (never re-mints a fake one).
+  const reauth = adapters.reauth ?? (adapters.auth ? async (current) => current : (current, kind) => reauthenticateParkedLease(current, kind));
+  let parkIncident = null;
+  // BOT-2099: a best-effort diagnostic read (ledger, persisted Helper reason) never throws.
+  const readLeaseQuietly = async (id) => { try { return await api.get?.(id) ?? null; } catch { return null; } };
+  const resilientWait = async (leaseIdArg, done, failed, options) => {
+    const result = await parkOnLease(leaseIdArg, done, failed, {
+      ...options, wait,
+      // The RPC layer (retryingStackCall) is the single owner of the credential: a re-mint
+      // is adopted there at once, and the park re-reads it before each attempt — so a
+      // credential the RPC layer itself refreshed during the probe is never overwritten.
+      reauth: async (_current, kind) => { rpc.adopt(await reauth(rpc.auth, kind)); return rpc.auth; },
+      syncCredential: () => rpc.auth,
+      probe: async () => { const read = await api.get(leaseIdArg); return read?.ok && read.data?.success ? read.data : null; },
+      ...(adapters.sleep ? { sleep: adapters.sleep } : {}),
+      ...(adapters.now ? { now: adapters.now } : {}),
+      log: stderrLine,
+    });
+    if (result?.incident) { parkIncident = result.incident; incidentBox.incident = result.incident; }
+    return result;
+  };
   // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process -- validated executable + argv only; shell is never used.
   const startChild = adapters.startChild ?? ((argv, env, cwd) => spawn(argv[0], argv.slice(1), { cwd, env, stdio: "inherit", detached: true }));
   const writeConnection = adapters.writeConnection ?? writeConnectionFile;
@@ -2343,9 +2585,18 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
     // `stack_leases.error` (which `get_stack_lease` already returns) holds the
     // most recent failure message regardless of which live signal was or
     // wasn't seen.
-    const durableReapFailure = (read) => (read?.ok && read.data?.success && read.data.state === "reaping" && read.data.error)
-      ? { ok: false, error: `the lease is still reaping and may still hold containers — ${read.data.error}` }
-      : null;
+    // BOT-2099 (hop 4, lease 14dffd23): `error` on a `reaping` lease is NOT by itself a
+    // failed reap — a provision the Helper rolled back is `reaping / provision_failed`
+    // with the PROVISION failure in `error`, and its reap is still to come. A reap has
+    // failed only when the server says so: `reap_failed_at` (BOT-1960) or the latest
+    // reap_stack job `failed` (`last_reap_outcome`, BOT-2099). A server too old to
+    // return `last_reap_outcome` keeps the previous conservative reading.
+    const durableReapFailure = (read) => {
+      if (!(read?.ok && read.data?.success && read.data.state === "reaping" && read.data.error)) return null;
+      const d = read.data;
+      const reapFailed = d.reap_failed_at != null || d.last_reap_outcome === "failed" || !("last_reap_outcome" in d);
+      return reapFailed ? { ok: false, error: `the lease is still reaping and may still hold containers — ${d.error}` } : null;
+    };
     // BOT-1822 (Codex round 3 P2): check BEFORE opening the wait, not only after
     // it times out — a failure already persisted must be reported at once
     // instead of burning the entire --reap-timeout waiting for a wake that will
@@ -2357,7 +2608,8 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
     // state — instead of leaving this wait to park until --reap-timeout on a reap
     // that already failed. Report that specific outcome instead of the generic
     // "was not proven" timeout message.
-    const reaped = await wait(leaseId, (state) => state === "reaped", (state) => state === "reap_failed", { timeoutSec: opts.reapTimeout, auth: rpc.auth });
+    // BOT-2099 (Codex #1057 R6): the signed-reap wait survives auth/availability blips too.
+    const reaped = await resilientWait(leaseId, (state) => state === "reaped", (state) => state === "reap_failed", { timeoutSec: opts.reapTimeout, auth: rpc.auth });
     if (reaped?.woke || reaped?.state === "reaped") return cleanupResult = { ok: true, state: "reaped" };
     if (reaped?.failed) return cleanupResult = { ok: false, error: "the signed reap failed and the lease may still hold containers (fenced) — see stack hygiene" };
     // Belt-and-braces: the same race can still slip between the pre-check above
@@ -2505,14 +2757,26 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
     if (state === "queued") {
       // BOT-2019: say where the lease sits (server dispatch rank, or why it has none) before parking.
       stderrLine(`stack: lease ${leaseId} queued — ${describeQueuePosition(request.data)}`);
+      for (const line of describeUnitLedger(request.data.ledger)) stderrLine(`stack: ${line}`);
       activeWaitAbort = new AbortController();
       // BOT-2013: every reap state (reaping / reaped / reap_failed) means the queued lease is lost.
-      const parked = await wait(leaseId, (s) => s !== "queued" && s != null, isReaped, { timeoutSec: opts.timeout, auth: rpc.auth, signal: activeWaitAbort.signal });
+      // BOT-2099: an auth/availability blip is retried inside the park until its deadline;
+      // it never releases a queued lease or becomes the lease's outcome.
+      const parked = await resilientWait(leaseId, (s) => s !== "queued" && s != null, isReaped, { timeoutSec: opts.timeout, auth: rpc.auth, signal: activeWaitAbort.signal });
       activeWaitAbort = null;
-      if (parked?.interrupted || receivedSignal) { const cleanup = await requestCleanup(); return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup }; }
-      if (parked?.timeout) { const cleanup = await requestCleanup(); return { exitCode: EXIT.TIMEOUT, outcome: "timeout", leaseId, error: `parked ${opts.timeout}s without capacity`, cleanup }; }
-      if (parked?.failed) { const cleanup = await requestCleanup(); return { exitCode: EXIT.LEASE_FAILED, outcome: "error", leaseId, error: `lease was ${parked.state} while queued (reap_reason=${parked.reap_reason ?? "unknown"})`, cleanup }; }
-      if (parked?.error || parked?.auth) { const cleanup = await requestCleanup(); return { exitCode: parked?.auth ? EXIT.AUTH : EXIT.LEASE_FAILED, outcome: "error", leaseId, error: parked?.error || "lease did not leave queue", cleanup }; }
+      const incident = parkIncident ? { incident: parkIncident } : {};
+      if (parked?.interrupted || receivedSignal) { const cleanup = await requestCleanup(); return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup, ...incident }; }
+      if (parked?.timeout) {
+        // BOT-2099: say who held the units for the whole park, read before the release.
+        const final = await readLeaseQuietly(leaseId);
+        const ledger = final?.ok && final.data?.success ? final.data.ledger ?? null : null;
+        for (const line of describeUnitLedger(ledger)) stderrLine(`stack: ${line}`);
+        const cleanup = await requestCleanup();
+        const during = parkIncident && !parkIncident.recovered ? ` (platform incident: ${parkIncident.reason} since ${parkIncident.first_at})` : "";
+        return { exitCode: EXIT.TIMEOUT, outcome: "timeout", leaseId, error: `parked ${opts.timeout}s without capacity${during}`, cleanup, ...incident, ...(ledger ? { ledger } : {}) };
+      }
+      if (parked?.failed) { const cleanup = await requestCleanup(); return { exitCode: EXIT.LEASE_FAILED, outcome: "error", leaseId, error: `lease was ${parked.state} while queued (reap_reason=${parked.reap_reason ?? "unknown"})`, cleanup, ...incident }; }
+      if (parked?.error || parked?.auth) { const cleanup = await requestCleanup(); return { exitCode: parked?.auth ? EXIT.AUTH : EXIT.LEASE_FAILED, outcome: "error", leaseId, error: parked?.error || "lease did not leave queue", cleanup, ...incident }; }
       state = parked.state;
     }
     if (state !== "active") {
@@ -2524,7 +2788,7 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
       // provisionTimeout before the belt-and-braces recheck below ever notices
       // the lease is no longer provisioning. isReaped (== "reaping" || "reaped")
       // already covers both; cmdUp's equivalent waits use it for this reason.
-      const active = await wait(leaseId, (s) => s === "active", isReaped, { timeoutSec: provisionTimeout, auth: rpc.auth, signal: activeWaitAbort.signal });
+      const active = await resilientWait(leaseId, (s) => s === "active", isReaped, { timeoutSec: provisionTimeout, auth: rpc.auth, signal: activeWaitAbort.signal });
       activeWaitAbort = null;
       if (active?.interrupted || receivedSignal) { const cleanup = await requestCleanup(); return { exitCode: SIGNAL_EXIT[receivedSignal], outcome: "interrupted", leaseId, cleanup }; }
       if (active?.timeout) {
@@ -2536,10 +2800,20 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
         const recheck = await api.get(leaseId);
         if (!(recheck?.ok && recheck.data?.success && recheck.data.state === "active")) {
           const cleanup = await requestCleanup();
-          return { exitCode: EXIT.TIMEOUT, outcome: "timeout", leaseId, error: `physical provision did not reach active in ${provisionTimeout}s`, cleanup };
+          return { exitCode: EXIT.TIMEOUT, outcome: "timeout", leaseId, error: `physical provision did not reach active in ${provisionTimeout}s`, cleanup, ...(parkIncident ? { incident: parkIncident } : {}) };
+        }
+      }
+      if (active?.failed || active?.error || active?.auth) {
+        // BOT-2099 (hops 3–4): name the Helper's reason (e.g. `schema readiness rejected:
+        // canonical_probe`) — it is persisted on the lease as `error` — instead of only the state.
+        let why = "";
+        if (active?.failed) {
+          const read = await readLeaseQuietly(leaseId);
+          if (read?.ok && read.data?.success && typeof read.data.error === "string" && read.data.error) why = ` — ${read.data.error}`;
         }
+        const cleanup = await requestCleanup();
+        return { exitCode: active?.auth ? EXIT.AUTH : EXIT.LEASE_FAILED, outcome: "error", leaseId, error: active?.error || `lease ${active?.state ?? "failed"} before reaching active${why}`, cleanup, ...(parkIncident ? { incident: parkIncident } : {}) };
       }
-      if (active?.failed || active?.error || active?.auth) { const cleanup = await requestCleanup(); return { exitCode: active?.auth ? EXIT.AUTH : EXIT.LEASE_FAILED, outcome: "error", leaseId, error: active?.error || `lease ${active?.state ?? "failed"} before reaching active`, cleanup }; }
     }
     const current = await api.get(leaseId);
     if (!current?.ok || !current.data?.success || current.data.state !== "active" || !current.data.connection || typeof current.data.connection !== "object") {
@@ -2606,7 +2880,7 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
     if (hardTimer) clock.clearTimeout(hardTimer);
     const cleanup = await requestCleanup();
     const exitCode = receivedSignal ? SIGNAL_EXIT[receivedSignal] : forcedStopCode ?? (childResult.code !== 0 ? childResult.code : !cleanup.ok ? EXIT.CLEANUP_FAILED : 0);
-    return { exitCode, outcome: exitCode === 0 ? "completed" : "failed", leaseId, childExitCode: childResult.code, cleanup, fenced: fencing };
+    return { exitCode, outcome: exitCode === 0 ? "completed" : "failed", leaseId, childExitCode: childResult.code, cleanup, fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) };
   } catch (error) {
     const cleanup = await requestCleanup();
     // BOT-1750 (Codex R10 P2): a thrown auth/self-heal failure (e.g. a refresh that
@@ -2646,6 +2920,10 @@ async function cmdRun(opts, childArgv) {
     ...(result.wait_exit_code != null ? { wait_exit_code: result.wait_exit_code, wait_receipt: result.wait_receipt ?? null } : {}),
     child_exit_code: result.childExitCode ?? null, cleanup: result.cleanup?.ok ?? null,
     fenced: result.fenced ?? false, error: result.error ?? result.cleanup?.error ?? null,
+    // BOT-2099: a platform auth/availability blip during the park is reported as an
+    // incident (reason, window, attempts, recovered) — never as the lease's outcome.
+    ...(result.incident ? { platform_incident: result.incident } : {}),
+    ...(result.ledger ? { unit_ledger: result.ledger } : {}),
   }), opts, result.exitCode);
 }