@botbuddy/cli 1.45.4 → 1.45.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botbuddy/cli",
-  "version": "1.45.4",
+  "version": "1.45.7",
   "description": "BotBuddy — Swarm coordination CLI for multi-agent workflows",
   "type": "module",
   "bin": {

package/src/stack-tunnel.mjs

@@ -37,6 +37,19 @@ export const SSH_FORWARD_OPTIONS = Object.freeze([
 ]);
 
 export const DEFAULT_TUNNEL_READY_MS = 20_000;
+
+/**
+ * The host-side bind every reverse forward asks for: IPv4 loopback ONLY (Codex #1134 P1). With no
+ * bind address sshd binds `localhost` on both address families and reports the forward granted
+ * when EITHER binds, so two sessions could each hold one family of the same port (captured
+ * 2026-10-13, OpenSSH_10.3p1). One address is one socket: the forward is granted whole or refused
+ * whole, and any other listener on the port is another process's. An sshd with `GatewayPorts yes`
+ * still binds every interface, which the probe and the post-forward check refuse.
+ */
+export const REVERSE_BIND = "127.0.0.1";
+const REVERSE_BIND_RE = REVERSE_BIND.split(".").join("\\.");
+/** Listeners on the host's `port` that are not the one address a reverse forward of ours binds. */
+const foreignListeners = (addresses, port) => addresses.filter((a) => a !== `${REVERSE_BIND}:${Number(port)}`);
 const LOOPBACK = new Set(["127.0.0.1", "localhost", "[::1]", "::1"]);
 
 function tunnelError(code, message, extra = {}) {
@@ -146,9 +159,11 @@ export async function planForwards(remotePorts, { isFree = isLocalPortFree, pick
 
 /** `ssh` argv (without the binary) for the planned forwards. */
 export function buildSshForwardArgv({ sshTarget, forwards, callbackPort = null }) {
-  const argv = [...SSH_FORWARD_OPTIONS];
+  // `-v` only with a callback forward: OpenSSH then logs whether the host GRANTED this session's
+  // `-R` (see openSshForwards `confirmRemoteForward`).
+  const argv = [...(callbackPort != null ? ["-v"] : []), ...SSH_FORWARD_OPTIONS];
   for (const f of forwards) argv.push("-L", `127.0.0.1:${f.local}:127.0.0.1:${f.remote}`);
-  if (callbackPort != null) argv.push("-R", `${callbackPort}:127.0.0.1:${callbackPort}`);
+  if (callbackPort != null) argv.push("-R", `${REVERSE_BIND}:${callbackPort}:127.0.0.1:${callbackPort}`);
   argv.push("--", sshTarget);
   return argv;
 }
@@ -186,19 +201,41 @@ function probeConnect(port, timeoutMs = 500) {
 }
 
 const BIND_RACE = /address already in use|cannot listen|could not request local forwarding/i;
+// What `ssh -v` adds to stderr that is not an error (captured 2026-10-13: OpenSSH_10.3p1 on
+// macOS, and OpenSSH_9.6p1 on Linux, which prints its version banner with no debug prefix).
+const SSH_VERBOSE_CHATTER = /^(debug\d: |Authenticated to |Warning: Permanently added |OpenSSH_[\w.]+[ ,])/;
 
 /**
  * Spawn ONE ssh carrying every forward and resolve once each local listener accepts.
  * Rejects (killing ssh) if ssh exits first or readiness times out.
+ *
+ * `confirmRemoteForward` (a host port; argv must carry `-v`): also wait for OpenSSH's
+ * `remote forward success for: listen 127.0.0.1:<port>` from THIS session (Codex #1134 P1). The `-L`
+ * listeners are up before the host answers the `-R` request, so without it a session whose
+ * `-R` the host refused looks ready until its exit is delivered.
  */
 export async function openSshForwards({
   sshBin = "ssh", argv, forwards, sshTarget, readyTimeoutMs = DEFAULT_TUNNEL_READY_MS,
   spawnImpl = spawn, env = process.env, connectProbe = probeConnect, sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+  confirmRemoteForward = null,
 }) {
   // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process -- fixed ssh argv; target validated by assertSshTarget; no shell.
   const child = spawnImpl(sshBin, argv, { stdio: ["ignore", "ignore", "pipe"], env });
+  const verbose = argv.includes("-v");
+  const granted = confirmRemoteForward == null ? null : new RegExp(`^debug1: remote forward success for: listen ${REVERSE_BIND_RE}:${Number(confirmRemoteForward)}\\b`);
+  let remoteConfirmed = granted == null;
   let stderr = "";
-  child.stderr?.on("data", (chunk) => { stderr = (stderr + chunk.toString("utf8")).slice(-2048); });
+  let partial = "";
+  child.stderr?.on("data", (chunk) => {
+    const lines = (partial + chunk.toString("utf8")).split("\n");
+    partial = lines.pop();
+    for (const line of lines) {
+      if (granted?.test(line)) remoteConfirmed = true;
+      if (verbose && SSH_VERBOSE_CHATTER.test(line)) continue;
+      stderr = `${stderr}${line}\n`.slice(-2048);
+    }
+  });
+  const stderrText = () => `${stderr}${verbose && SSH_VERBOSE_CHATTER.test(partial) ? "" : partial}`.trim();
   let exited = null;
   let closing = false;
   const listeners = [];
@@ -207,10 +244,10 @@ export async function openSshForwards({
       if (exited) return;
       exited = { code: code ?? null, signal: signal ?? null };
       resolve(exited);
-      if (!closing) for (const fn of listeners) fn({ ...exited, stderr: stderr.trim() });
+      if (!closing) for (const fn of listeners) fn({ ...exited, stderr: stderrText() });
     };
     child.once("exit", finish);
-    child.once("error", (error) => { stderr += `\n${error.message}`; finish(null, null); });
+    child.once("error", (error) => { stderr += `\n${error.message}\n`; finish(null, null); });
   });
 
   const close = async () => {
@@ -226,17 +263,22 @@ export async function openSshForwards({
 
   const deadline = Date.now() + readyTimeoutMs;
   const pending = new Set(forwards.map((f) => f.local));
-  while (pending.size) {
+  while (pending.size || !remoteConfirmed) {
     if (exited) break;
     for (const port of [...pending]) if (await connectProbe(port)) pending.delete(port);
-    if (!pending.size || exited) break;
+    if ((!pending.size && remoteConfirmed) || exited) break;
     if (Date.now() > deadline) break;
     await sleep(100);
   }
-  if (exited || pending.size) {
+  if (exited || pending.size || !remoteConfirmed) {
     const timedOut = !exited;
     await close();
-    const tail = stderr.trim();
+    const tail = stderrText();
+    if (timedOut && !pending.size) {
+      throw tunnelError("REMOTE_UNREACHABLE",
+        `${sshTarget} never confirmed the reverse forward for --callback-port ${confirmRemoteForward} within ${Math.round(readyTimeoutMs / 1000)}s` +
+          `${tail ? `: ${tail}` : ""}, so it was not trusted and the tunnel was closed.`, { sshStderr: tail });
+    }
     throw tunnelError("REMOTE_UNREACHABLE",
       `ssh ${sshTarget} could not open the stack forwards${timedOut ? ` within ${Math.round(readyTimeoutMs / 1000)}s` : ` (exit ${exited?.code ?? "?"})`}` +
         `${tail ? `: ${tail}` : ""}. Check that \`ssh ${sshTarget}\` works non-interactively from this machine (BatchMode: a key, no password prompt).`,
@@ -246,8 +288,8 @@ export async function openSshForwards({
     pid: child.pid,
     argv,
     exited: exitPromise,
-    onExit: (fn) => { listeners.push(fn); if (exited && !closing) fn({ ...exited, stderr: stderr.trim() }); },
-    stderrTail: () => stderr.trim(),
+    onExit: (fn) => { listeners.push(fn); if (exited && !closing) fn({ ...exited, stderr: stderrText() }); },
+    stderrTail: stderrText,
     close,
   };
 }
@@ -259,11 +301,12 @@ export async function openSshForwards({
  * forward is up it asks the host which address the listener really holds and refuses
  * anything but loopback (fail closed: no lsof, no listener, unreadable → refused).
  */
-export function verifyCallbackLoopback({ sshBin, sshTarget, port, env = process.env, run = spawnSync }) {
-  const fail = (why) => tunnelError("CALLBACK_NOT_LOOPBACK",
-    `the reverse forward for --callback-port ${port} on ${sshTarget} could not be proven loopback-only (${why}). ` +
+export function verifyCallbackLoopback({ sshBin, sshTarget, port, env = process.env, run = spawnSync, label = null }) {
+  const what = label ?? `the reverse forward for --callback-port ${port}`;
+  const fail = (why, extra = {}) => tunnelError("CALLBACK_NOT_LOOPBACK",
+    `${what} on ${sshTarget} could not be proven loopback-only (${why}). ` +
       "An sshd with GatewayPorts yes binds -R forwards on every interface; set GatewayPorts no (or clientspecified) on the host, " +
-      "or run without --callback-port.");
+      "or run without --callback-port.", extra);
   // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process -- fixed argv; target validated; integer port.
   const r = run(sshBin, [
     "-T", "-o", "BatchMode=yes", "-o", "ControlMaster=no", "-o", "ControlPath=none", "--", sshTarget,
@@ -272,16 +315,145 @@ export function verifyCallbackLoopback({ sshBin, sshTarget, port, env = process.
   if (r.error || r.status !== 0) {
     const tail = String(r.stderr || r.error?.message || "").trim().split("\n").slice(-1)[0];
     // lsof exits 1 with no output when nothing listens.
-    if (r.status === 1 && !String(r.stdout || "").trim()) throw fail(`no listener on the host's port ${port}`);
+    if (r.status === 1 && !String(r.stdout || "").trim()) throw fail(`no listener on the host's port ${port}`, { noListener: true });
     throw fail(`could not run lsof on the host (exit ${r.status ?? "?"}${tail ? `: ${tail}` : ""})`);
   }
   const addresses = String(r.stdout || "").split("\n").filter((l) => l.startsWith("n")).map((l) => l.slice(1).trim());
-  if (!addresses.length) throw fail(`no listener on the host's port ${port}`);
+  if (!addresses.length) throw fail(`no listener on the host's port ${port}`, { noListener: true });
   const exposed = addresses.filter((a) => !(a.startsWith("127.") || a.startsWith("[::1]:") || a.startsWith("localhost:")));
-  if (exposed.length) throw fail(`the host listens on ${exposed.join(", ")}`);
+  if (exposed.length) throw fail(`the host listens on ${exposed.join(", ")}`, { exposed, listeners: addresses });
   return addresses;
 }
 
+
+/** A random throwaway port in the dynamic range (49152–65535) for the GatewayPorts probe. */
+export function randomProbePort(random = Math.random) {
+  return 49152 + Math.floor(random() * (65535 - 49152 + 1));
+}
+
+const REMOTE_FORWARD_BUSY = /remote port forwarding failed|address already in use|cannot listen/i;
+export const DEFAULT_PROBE_READY_MS = 10_000;
+
+/**
+ * Codex #1134 P1: the probe's LOCAL destination is a port this process holds for the whole probe —
+ * a listener that closes every connection without a byte — so no other local service can bind it
+ * while a (possibly wildcard) probe listener on the host forwards there. Resolves { port, close }.
+ */
+export function holdSilentLocalPort() {
+  return new Promise((resolve, reject) => {
+    const server = createServer((socket) => socket.destroy());
+    server.once("error", reject);
+    server.listen({ host: "127.0.0.1", port: 0, exclusive: true }, () => {
+      resolve({ port: server.address().port, close: () => new Promise((done) => server.close(() => done())) });
+    });
+  });
+}
+
+/**
+ * BOT-2184: prove the host keeps `-R` listeners on loopback BEFORE the real callback forward
+ * exists. Opens a short-lived `ssh -N -R P:127.0.0.1:<held local port>` for a throwaway host
+ * port P (the local end is a listener this process holds that answers nothing, so nothing is
+ * exposed even if sshd binds it wildcard), asks
+ * the host which address P really holds (`verifyCallbackLoopback` with `port: P`), and kills the
+ * probe. Fail closed: an exposed bind, lsof missing, or a listener that never appears is
+ * CALLBACK_NOT_LOOPBACK; a busy P is retried once on another port, then refused.
+ */
+export async function probeGatewayPorts({
+  sshBin = "ssh", sshTarget, env = process.env, spawnImpl = spawn, run = spawnSync, pickProbePort = randomProbePort,
+  holdLocalPort = holdSilentLocalPort, readyTimeoutMs = DEFAULT_PROBE_READY_MS, attempts = 2,
+  sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+}) {
+  const fail = (why) => tunnelError("CALLBACK_NOT_LOOPBACK",
+    `the GatewayPorts probe on ${sshTarget} could not prove reverse forwards stay loopback-only (${why}), so no -R for ` +
+      "--callback-port was opened. An sshd with GatewayPorts yes binds -R forwards on every interface; set GatewayPorts no " +
+      "(or clientspecified) on the host, or run without --callback-port.");
+  const tried = new Set();
+  let busyTail = "";
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    let port = pickProbePort();
+    for (let i = 0; tried.has(port) && i < 20; i++) port = pickProbePort();
+    tried.add(port);
+    // P must be free on the host BEFORE the probe, or a listener seen afterwards could be someone
+    // else's (a loopback one would pass a host that would expose ours). Taken → another port.
+    try {
+      verifyCallbackLoopback({ sshBin, sshTarget, port, env, run, label: `the GatewayPorts probe (host port ${port})` });
+      busyTail = `host port ${port} already has a listener`;
+      continue;
+    } catch (error) {
+      // A listener that was there before the probe, on ANY address, says nothing about
+      // GatewayPorts (Codex #1134 P2): the port is busy, probe another one.
+      if (error?.exposed) { busyTail = `host port ${port} already has a listener (${error.listeners.join(", ")})`; continue; }
+      if (!error?.noListener) throw error;
+    }
+    const held = await holdLocalPort();
+    const closedLocal = held.port;
+    // `-v`: OpenSSH logs `remote forward success for: listen 127.0.0.1:<P>` once the host GRANTED this
+    // forward (Codex #1134 P1): only then can a listener on P be this probe's, not a process that
+    // took P after the free-port check.
+    const argv = [
+      "-v", "-N", "-T", "-o", "BatchMode=yes", "-o", "ExitOnForwardFailure=yes", "-o", "ControlMaster=no", "-o", "ControlPath=none",
+      "-R", `${REVERSE_BIND}:${port}:127.0.0.1:${closedLocal}`, "--", sshTarget,
+    ];
+    // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process -- fixed ssh argv; target validated by assertSshTarget; integer ports; no shell.
+    const child = spawnImpl(sshBin, argv, { stdio: ["ignore", "ignore", "pipe"], env });
+    let stderr = "";
+    let exited = null;
+    let confirmed = false;
+    const granted = new RegExp(`remote forward success for: listen ${REVERSE_BIND_RE}:${port}\\b`);
+    child.stderr?.on("data", (chunk) => {
+      const text = stderr.slice(-256) + chunk.toString("utf8");
+      if (granted.test(text)) confirmed = true;
+      stderr = (stderr + chunk.toString("utf8")).slice(-2048);
+    });
+    const exitPromise = new Promise((resolve) => {
+      child.once("exit", (code, signal) => { exited = { code, signal }; resolve(exited); });
+      child.once("error", (error) => { stderr += `\n${error.message}`; exited = { code: null, signal: null }; resolve(exited); });
+    });
+    try {
+      const deadline = Date.now() + readyTimeoutMs;
+      for (;;) {
+        if (exited) {
+          const tail = stderr.trim().split("\n").filter((l) => !l.startsWith("debug1:")).slice(-3).join(" | ");
+          if (REMOTE_FORWARD_BUSY.test(stderr)) { busyTail = tail; break; }
+          throw fail(`the probe forward exited (${exited.code ?? exited.signal ?? "?"})${tail ? `: ${tail}` : ""}`);
+        }
+        if (!confirmed) {
+          if (Date.now() > deadline) throw fail(`the host never confirmed the probe forward on port ${port} within ${Math.round(readyTimeoutMs / 1000)}s`);
+          await sleep(100);
+          continue;
+        }
+        try {
+          const addresses = verifyCallbackLoopback({ sshBin, sshTarget, port, env, run, label: `the GatewayPorts probe (host port ${port})` });
+          // The probe asked for one address; anything else on P is another process's, so this
+          // port proves nothing about the host. Probe another one.
+          const foreign = foreignListeners(addresses, port);
+          if (foreign.length) { busyTail = `host port ${port} also has a listener that is not this probe's (${foreign.join(", ")})`; break; }
+          return { port, attempts: attempt };
+        } catch (error) {
+          if (!error?.noListener) {
+            // An exposed bind or no lsof: refuse (never retried). The message names the probe port.
+            throw error;
+          }
+        }
+        if (exited) continue;
+        if (Date.now() > deadline) throw fail(`the probe listener on host port ${port} never appeared within ${Math.round(readyTimeoutMs / 1000)}s`);
+        await sleep(150);
+      }
+    } finally {
+      if (!exited) {
+        try { child.kill("SIGTERM"); } catch { /* gone */ }
+        const timer = setTimeout(() => { try { child.kill("SIGKILL"); } catch { /* gone */ } }, 3_000);
+        timer.unref?.();
+        await exitPromise;
+        clearTimeout(timer);
+      }
+      // Released only after the probe forward is gone.
+      await held.close();
+    }
+  }
+  throw fail(`the host refused ${attempts} probe ports: ${busyTail}`);
+}
+
 /**
  * Plan, spawn and verify the forwards for an ACTIVE remote lease.
  * @returns {{ forwards, connection, argv, attempts, onExit, close, pid }}
@@ -289,21 +461,54 @@ export function verifyCallbackLoopback({ sshBin, sshTarget, port, env = process.
 export async function establishSshTunnel({
   connection, sshTarget, hostKey = null, callbackPort = null, sshBin = null, env = process.env,
   attempts = 3, readyTimeoutMs = DEFAULT_TUNNEL_READY_MS, isFree = isLocalPortFree, pickFree = pickFreeLocalPort, spawnImpl = spawn,
-  runRemote = spawnSync,
+  runRemote = spawnSync, probe = probeGatewayPorts, probeReadyTimeoutMs = DEFAULT_PROBE_READY_MS,
 }) {
   const target = assertSshTarget(sshTarget, hostKey);
   const remotePorts = remotePortsFromConnection(connection);
   const bin = sshBin || env.BOTBUDDY_STACK_SSH_BIN || "ssh";
-  let avoid = new Set();
+  // BOT-2184: prove the host keeps -R listeners on loopback BEFORE the real callback forward
+  // exists, so the caller's callback service is never reachable from the host's network.
+  // The post-forward check below stays as a second line.
+  if (callbackPort != null) {
+    await probe({ sshBin: bin, sshTarget: target, env, run: runRemote, spawnImpl, readyTimeoutMs: probeReadyTimeoutMs });
+    // Codex #1134 P1: the REAL callback port must be free on the host, or the post-forward check
+    // could accept another process's loopback listener and the stack's callbacks would reach it.
+    let holders = null;
+    try {
+      holders = verifyCallbackLoopback({ sshBin: bin, sshTarget: target, port: callbackPort, env, run: runRemote });
+    } catch (error) {
+      // No -R of ours exists yet, so a listener on any address is someone else's: busy, not a
+      // GatewayPorts verdict.
+      if (error?.exposed) holders = error.listeners;
+      else if (!error?.noListener) throw error;
+    }
+    if (holders) {
+      throw tunnelError("CALLBACK_PORT_BUSY",
+        `--callback-port ${callbackPort} is already listening on ${target} (${holders.join(", ")}): another run or service holds it, ` +
+          "so the stack's callbacks could reach that process instead of this machine. Pick another --callback-port, or wait for it to free.");
+    }
+  }
+  // Codex #1134 P2: a remapped stack forward must never take the callback port locally; ssh
+  // would bind it before the caller's callback service and the -R traffic would loop into it.
+  let avoid = new Set(callbackPort != null ? [Number(callbackPort)] : []);
   let lastError = null;
   for (let attempt = 1; attempt <= attempts; attempt++) {
     const forwards = await planForwards(remotePorts, { isFree, pickFree, avoid });
     const argv = buildSshForwardArgv({ sshTarget: target, forwards, callbackPort });
     try {
-      const handle = await openSshForwards({ sshBin: bin, argv, forwards, sshTarget: target, readyTimeoutMs, spawnImpl, env });
+      // Codex #1134 P1: the tunnel is ready only once the host GRANTED this session's -R; the
+      // post-forward check below then reads a listener that can only be this session's.
+      const handle = await openSshForwards({ sshBin: bin, argv, forwards, sshTarget: target, readyTimeoutMs, spawnImpl, env, confirmRemoteForward: callbackPort });
       if (callbackPort != null) {
         try {
-          verifyCallbackLoopback({ sshBin: bin, sshTarget: target, port: callbackPort, env, run: runRemote });
+          const addresses = verifyCallbackLoopback({ sshBin: bin, sshTarget: target, port: callbackPort, env, run: runRemote });
+          // This session holds exactly 127.0.0.1:<port>; any other listener on the port is not ours.
+          const foreign = foreignListeners(addresses, callbackPort);
+          if (foreign.length) {
+            throw tunnelError("CALLBACK_PORT_BUSY",
+              `--callback-port ${callbackPort} on ${target} also has a listener that is not this run's (${foreign.join(", ")}): another run or service shares the port, ` +
+                "so the stack's callbacks could reach that process instead of this machine. Pick another --callback-port, or wait for it to free.");
+          }
         } catch (error) {
           await handle.close();
           throw error;
@@ -311,6 +516,13 @@ export async function establishSshTunnel({
       }
       return { ...handle, forwards, connection: rewriteConnectionForForwards(connection, forwards), attempts: attempt };
     } catch (error) {
+      // Another run took the callback port between the free-port check and this session's -R:
+      // the host refused ours, so nothing of this run listens there.
+      if (callbackPort != null && new RegExp(`remote port forwarding failed for listen port ${Number(callbackPort)}\\b`).test(error?.sshStderr ?? "")) {
+        throw tunnelError("CALLBACK_PORT_BUSY",
+          `--callback-port ${callbackPort} was taken on ${target} before this run's reverse forward could bind it (ssh: ${error.sshStderr.split("\n").slice(-1)[0]}): ` +
+            "another run or service holds it, so the stack's callbacks could reach that process instead of this machine. Pick another --callback-port, or wait for it to free.");
+      }
       lastError = error;
       // Only a lost local bind race is worth another try, on fresh ports; an auth or
       // host failure would fail the same way again.

package/src/stack.mjs

@@ -33,8 +33,7 @@ import { callToolJson, CALL_AUTH_ERROR_CODES } from "./api.mjs";
 import { SERVER_URL, getConfig } from "./config.mjs";
 import { resolveOwnerToken, resolveAgentKey } from "./cli-credentials.mjs";
 import { AGENT_KEY_RE, readAgentSessionTokenEnv } from "./agent-key.mjs";
-import { resolveAgentSessionCredential } from "./agent-session.mjs";
-import { clearAgentState } from "./agent-state.mjs";
+import { resolveAgentSessionCredential, selfHealAgentSession } from "./agent-session.mjs";
 import { runDockerCommand, runDockerWorkflow, ADMITTED_DOCKER_VALIDATIONS } from "./docker-hygiene.mjs";
 import { acquireStackLock, lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
 import { assertLeasedStackIsolated, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
@@ -47,17 +46,13 @@ import { bold, dim, yellow } from "./utils.mjs";
 export const STACK_SCHEMA_VERSION = 1;
 
 /**
- * Codex #1077 P1: an sshd with `GatewayPorts yes` binds a `-R` forward on every interface,
- * and the post-forward lsof check (`verifyCallbackLoopback`) runs only AFTER the listener
- * exists — so the caller's callback service could be reachable from the host's network for
- * a few seconds. Until a pre-forward probe lands (BOT-2184), a REMOTE lease refuses
- * --callback-port outright (fail closed). Co-located leases are unaffected.
+ * Codex #1077 P1 / BOT-2184: an sshd with `GatewayPorts yes` binds a `-R` forward on every
+ * interface. A remote lease with --callback-port therefore runs the pre-forward GatewayPorts
+ * probe (`probeGatewayPorts` in stack-tunnel.mjs) before the real `-R` exists, and refuses
+ * with exit 5 CALLBACK_NOT_LOOPBACK when the host would expose it. The code below is kept
+ * only so older automation that matched on it still resolves; no path emits it any more.
  */
 export const CALLBACK_PORT_UNSUPPORTED_REMOTE = "CALLBACK_PORT_UNSUPPORTED_REMOTE";
-const CALLBACK_PORT_REMOTE_REFUSAL =
-  `${CALLBACK_PORT_UNSUPPORTED_REMOTE}: --callback-port is not supported on a remote lease yet — the -R listener ` +
-  "could be exposed on the serving host's network (sshd GatewayPorts yes) before it is checked. Tracked in BOT-2184 " +
-  "(pre-forward GatewayPorts probe). Use --local-only for a run that needs a callback port.";
 export const DEFAULT_RECEIPT_MAX_BYTES = 10240;
 export const DEFAULT_TIMEOUT_SEC = 3600;
 /**
@@ -174,10 +169,11 @@ ${bold("run OPTIONS")}
                          Default (neither flag): execution "auto" — co-located if it admits now,
                          else a remote host with capacity; the bundle digest is sent up front.
   --callback-port N      The caller runs a listener the stack calls back through
-                         host.docker.internal:N (e.g. 4600): sent as callback_port. Co-located
-                         leases only for now: a REMOTE lease refuses it (exit 4
-                         CALLBACK_PORT_UNSUPPORTED_REMOTE) until the pre-forward GatewayPorts
-                         probe lands (BOT-2184).
+                         host.docker.internal:N (e.g. 4600): sent as callback_port. On a REMOTE
+                         lease run first probes the host with a throwaway reverse forward and opens
+                         -R 127.0.0.1:N:127.0.0.1:N only when the host keeps it on loopback (sshd
+                         GatewayPorts no/clientspecified); otherwise exit 5 CALLBACK_NOT_LOOPBACK
+                         (BOT-2184).
   --bundle-max-mb N      Remote-lease bundle size cap in MB (default 50, compressed). The bundle
                          excludes .bb-stack.json, node_modules, supabase/.temp|.branches and
                          untracked-secret dotenv files (.env, .env.local, .env.*.local).
@@ -218,14 +214,17 @@ ${bold("REMOTE LEASES")} (BOT-2142, stack run)
   4 STACK_EXECUTION_MODE_MISMATCH  the slot already holds a lease in another mode/host — stack done it first
   (the four codes above create NO lease)
   5 BUNDLE_UPLOAD_FAILED  the signed bundle upload failed (the lease is released first)
-  5 CALLBACK_NOT_LOOPBACK (reserved for BOT-2184) the host's -R listener is not loopback-only
-                          (sshd GatewayPorts yes) or could not be checked with lsof
+  5 CALLBACK_NOT_LOOPBACK --callback-port: the GatewayPorts probe (before any real -R) or the
+                          post-forward check found the host's -R listener not loopback-only
+                          (sshd GatewayPorts yes), or could not check it (no lsof, no listener)
+  5 CALLBACK_PORT_BUSY    --callback-port is already listening on the serving host (another run or
+                          service), the host refused this run's -R because another run took the
+                          port first, or another listener shares the port; this run holds no -R —
+                          pick another port or wait
   4 BUNDLE_TOO_LARGE      --remote and the bundle is over --bundle-max-mb
   4 BUNDLE_SECRET_ENV     a --bundle-include file holds a live-credential-shaped value (names the key)
   4 BUNDLE_INCLUDE_*      a --bundle-include path is missing, not a file, too large, or excluded
                           (BUNDLE_UNSAFE_LINK: it resolves outside the stack); no lease is created
-  4 CALLBACK_PORT_UNSUPPORTED_REMOTE  --callback-port on a remote lease (refused until BOT-2184;
-                          under auto the remote lease is released first)
   6 TUNNEL_LOST           the forward exited mid-child: the child is stopped, the lease released,
                           park_reason tunnel_lost (its failures were transport, not product)
   Every remote failure after the lease exists still runs \`stack done\` and closes the forwards.
@@ -435,8 +434,6 @@ export function parseStackArgs(argv) {
   }
   if ((opts.callbackPort != null || opts.bundleMaxBytes != null) && command !== "run") {
     errors.push("--callback-port and --bundle-max-mb are valid only with `stack run`");
-  } else if (opts.remote && opts.callbackPort != null) {
-    errors.push(CALLBACK_PORT_REMOTE_REFUSAL);
   }
   if (opts.bundleInclude.length && command !== "run") {
     errors.push("--bundle-include is valid only with `stack run` (only run builds a remote-lease bundle)");
@@ -830,7 +827,7 @@ export async function stackAuthHeader({
     // and silently authenticate as the OAuth/MCP-key principal, which is the exact
     // principal switch the catch above guards against.
     if (typeof credential.token === "string" && AGENT_KEY_RE.test(credential.token)) {
-      return withSource({ "x-agent-api-key": credential.token }, credential.source);
+      return withSource({ "x-agent-api-key": credential.token }, credential.source, credential.identity);
     }
     throw Object.assign(new Error(`resolved agent session token has an unexpected shape (source: ${credential.source ?? "unknown"}); re-run \`bb doctor --fix\``), { code: "session_token_malformed" });
   }
@@ -850,37 +847,39 @@ export async function stackAuthHeader({
 }
 
 // BOT-1750 (Codex R7 P2): carry the credential SOURCE on the returned headers as a
-// NON-enumerable property, so callers can clear+self-heal+retry once on a
+// NON-enumerable property, so callers can renew+retry once on a
 // server-rejected cached session (parity with run/test/pw/wait) without the source
 // ever leaking into the spread fetch headers.
-function withSource(headers, source) {
+function withSource(headers, source, identity = null) {
   Object.defineProperty(headers, "__source", { value: source ?? null, enumerable: false });
+  Object.defineProperty(headers, "__identity", { value: identity, enumerable: false });
   return headers;
 }
 
 /**
  * BOT-1750 (Codex R7 P2): a locally-fresh cache whose token was revoked server-side
  * (e.g. a concurrent re-registration rotated the session) is rejected at the lease
- * RPC. Mirror the other consumers: when the FIRST auth came from the per-worktree
- * cache and the RPC returns an auth rejection, clear the cache and self-heal once,
- * then let the caller retry with the fresh headers. Returns the fresh headers, or
- * null when no retry applies (not cache-sourced, or the refresh could not authenticate).
+ * RPC. Renew once without deleting the cache or losing lease ownership identity.
+ * The shared lock re-reads and verifies the cache; a concurrent renewal wins.
+ * Failure leaves the existing cache untouched and propagates its typed error.
  */
 export async function refreshRejectedCacheAuth(auth, {
-  cwd = process.cwd(), authProvider = stackAuthHeader, clear = clearAgentState,
+  cwd = process.cwd(), renew = selfHealAgentSession,
 } = {}) {
   if (auth?.__source !== "cache") return null;
-  await clear(cwd);
-  const fresh = await authProvider();
-  // BOT-1750 (Codex R8 P2): retry ONLY with a genuinely self-healed session. If the
-  // client key or binding vanished after clearing the cache, authProvider falls
-  // through to the OAuth/MCP operator credential — a DIFFERENT principal whose token
-  // also differs, so a bare "different token" check would retry as the wrong agent.
-  // Require source "minted" AND a well-formed, changed session token.
-  if (fresh?.__source !== "minted") return null;
-  const token = fresh["x-agent-api-key"];
+  return renewStackSession(auth, { cwd, renew });
+}
+
+async function renewStackSession(auth, { cwd, renew }) {
+  const credential = await renew({ cwd, force: true,
+    rejectedCredential: { token: auth["x-agent-api-key"], identity: auth.__identity } });
+  // BOT-1750 (Codex R8 P2): the renewal adapter must never yield an operator
+  // principal. "cache" is permitted
+  // only for a newer same-identity session verified inside the renewal lock.
+  if (credential?.source !== "minted" && credential?.source !== "cache") return null;
+  const token = credential.token;
   if (typeof token !== "string" || !AGENT_KEY_RE.test(token) || token === auth["x-agent-api-key"]) return null;
-  return fresh;
+  return withSource({ "x-agent-api-key": token }, credential.source, credential.identity);
 }
 
 /**
@@ -888,8 +887,8 @@ export async function refreshRejectedCacheAuth(auth, {
  * server-rejected cached session ONCE — the owner-scoped management commands
  * (`status`/`touch`/`done`) need the same recovery as the lease-creating paths, or a
  * revoked-but-locally-fresh cache leaves an active lease unreleasable. `init()`
- * resolves auth; `call()` runs the RPC and, on the FIRST auth rejection, clears +
- * self-heals + retries with the fresh credential (every later `call` reuses it).
+ * resolves auth; `call()` runs the RPC and, on the FIRST auth rejection, renews +
+ * retries with the fresh credential (every later `call` reuses it).
  */
 export function retryingStackCall({
   callTool = callToolJson, authProvider = stackAuthHeader,
@@ -1364,25 +1363,18 @@ function abortableSleep(ms, signal) {
  * BOT-2099: re-establish the credential a parked lease waits with. On a transport failure
  * the credential is fine — keep it. On an auth rejection, first re-read (another process in
  * this worktree may already have re-minted, and minting again would supersede it), then
- * clear the per-worktree cache and self-heal once. Only a well-formed agent SESSION token
+ * renew under the shared cache lock without removing recovery identity. Only a well-formed agent SESSION token
  * replaces a session credential: the operator fallback is a different principal that
  * cannot see the lease (BOT-1750 R8), so it is never adopted. Throws the mint error, which
  * the park treats as one more failed attempt.
  */
 export async function reauthenticateParkedLease(current, kind, {
-  cwd = process.cwd(), authProvider = stackAuthHeader, clear = clearAgentState,
+  cwd = process.cwd(), renew = selfHealAgentSession,
 } = {}) {
   if (kind !== "auth") return current;
   const source = current?.__source ?? null;
   if (source !== "cache" && source !== "minted") return current;
-  const isFreshSession = (h) => (h?.__source === "cache" || h?.__source === "minted")
-    && typeof h["x-agent-api-key"] === "string" && AGENT_KEY_RE.test(h["x-agent-api-key"])
-    && h["x-agent-api-key"] !== current["x-agent-api-key"];
-  const reread = await authProvider();
-  if (isFreshSession(reread)) return reread;
-  await clear(cwd);
-  const minted = await authProvider();
-  return isFreshSession(minted) ? minted : current;
+  return (await renewStackSession(current, { cwd, renew })) ?? current;
 }
 
 /**
@@ -2484,7 +2476,7 @@ export async function cmdDone(leaseId, opts, {
 } = {}) {
   // BOT-1750 (Codex R10 P2): `done` must recover a server-rejected cached session on
   // its own — otherwise an active lease can't be released until the cache is manually
-  // repaired. The retrying caller clears+self-heals+retries the FIRST auth rejection.
+  // repaired. The retrying caller renews+retries the FIRST auth rejection.
   const rpc = retryingStackCall({ callTool, authProvider, refresh: refreshAuth });
   const auth = await rpc.init();
   if (!auth) return emitResult(buildReceipt({ command: "done", outcome: "error", lease_id: leaseId, error: "not authenticated — run `botbuddy login`" }), opts, EXIT.AUTH);
@@ -2841,7 +2833,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
   // BOT-1750 (Codex R9/R11 P2): `stack run` is a lease-creating entry point AND a
   // long-lived one (queue → provision → child → heartbeat → cleanup). A cached token
   // can be rotated server-side at ANY of those points, so EVERY RPC — not just the
-  // initial request — must be able to clear + self-heal + retry once. Route them all
+  // initial request — must be able to renew + retry once. Route them all
   // through one retryingStackCall so a single self-heal re-authenticates the rest
   // (heartbeat, get, cleanup release), instead of fencing a healthy child or leaking
   // an unreleasable lease. Retry only when we own the credential or a test provides an
@@ -3112,7 +3104,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
       ...(bundle?.included?.length ? { bundle_included: bundle.included } : {}),
       ...(opts.callbackPort != null ? { callback_port: opts.callbackPort } : {}),
     };
-    // The retrying caller (api → rpc.call) transparently clears+self-heals+retries the
+    // The retrying caller (api → rpc.call) transparently renews+retries the
     // FIRST auth rejection among any of request/get/touch/release.
     // BOT-2024: a stale-capacity refusal prints phase-aware guidance; with the opt-in
     // --wait-capacity a TRANSIENT one parks on the server's capacity wait and re-admits once.
@@ -3189,12 +3181,6 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
       };
     }
     let sshTarget = request.data.ssh_target ?? request.data.connection?.ssh_target ?? null;
-    if (remoteLease && opts.callbackPort != null) {
-      // Codex #1077 P1 / BOT-2184: `auto` placed this run remote; release before any upload
-      // or forward rather than open an -R listener that may be exposed.
-      const cleanup = await requestCleanup();
-      return { exitCode: EXIT.INVALID, outcome: "error", code: CALLBACK_PORT_UNSUPPORTED_REMOTE, leaseId, cleanup, error: CALLBACK_PORT_REMOTE_REFUSAL };
-    }
     if (remoteLease) {
       // Fail fast when the server SAYS the host has no SSH destination (the key is
       // present but empty); a server that only reports it on the active read is
@@ -3335,10 +3321,6 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
     const servedRemote = remoteLease || current.data.execution_mode === "remote";
     if (servedRemote && !remoteLease) {
       stderrLine(`stack: lease ${leaseId} moved to ${placement.hostKey ?? "another host"} while queued (${placement.placementReason ?? "moved"}) — forwarding to it`);
-      if (opts.callbackPort != null) {
-        const cleanup = await requestCleanup();
-        return { exitCode: EXIT.INVALID, outcome: "error", code: CALLBACK_PORT_UNSUPPORTED_REMOTE, leaseId, cleanup, error: CALLBACK_PORT_REMOTE_REFUSAL };
-      }
     }
     let childConnection = current.data.connection;
     if (servedRemote) {