@botbuddy/cli 1.64.5 → 1.64.7

package/daemon-release/darwin-arm64.json

@@ -1,7 +1,7 @@
 {
-  "release_version": "1.64.5",
-  "source_commit": "74649b2c740e68076a3d2753d52c39baa5dd6041",
-  "source_sha256": "7f55b0dc6c779a3bb000adaf31bcb8df85f8b58db4c2e58c3f24ba118ce7f869",
+  "release_version": "1.64.7",
+  "source_commit": "52077eb9393ab47e2a35b6ddba00a9a62955ca07",
+  "source_sha256": "9ac2dd0f2b02bdb4eac636e402b63620881c126c760d60b5313122323830b18a",
   "target": "darwin-arm64",
-  "sha256": "40982758f67eb18207e820823b3f28d0d1ee35bd8c84a70d22262a4a0ce306d8"
+  "sha256": "7ab6917ca6dd35571ff58b30e3baa08c387b7395a61e8e75d833e86ba379bfea"
 }

package/daemon-release/linux-x64.json

@@ -1,7 +1,7 @@
 {
-  "release_version": "1.64.5",
-  "source_commit": "74649b2c740e68076a3d2753d52c39baa5dd6041",
-  "source_sha256": "7f55b0dc6c779a3bb000adaf31bcb8df85f8b58db4c2e58c3f24ba118ce7f869",
+  "release_version": "1.64.7",
+  "source_commit": "52077eb9393ab47e2a35b6ddba00a9a62955ca07",
+  "source_sha256": "9ac2dd0f2b02bdb4eac636e402b63620881c126c760d60b5313122323830b18a",
   "target": "linux-x64",
-  "sha256": "a48a8775314b124b3fad95e5a2aabd44cf94d26af57bafe7ab01fe4d7aec6724"
+  "sha256": "0752327535efa03af48f49c97eca74b0df6050b46f16ef44ccc0ebdf042169d6"
 }

package/daemon-release/manifest.json

@@ -1,21 +1,21 @@
 {
   "schema_version": 1,
-  "release_version": "1.64.5",
-  "source_commit": "74649b2c740e68076a3d2753d52c39baa5dd6041",
-  "source_sha256": "7f55b0dc6c779a3bb000adaf31bcb8df85f8b58db4c2e58c3f24ba118ce7f869",
+  "release_version": "1.64.7",
+  "source_commit": "52077eb9393ab47e2a35b6ddba00a9a62955ca07",
+  "source_sha256": "9ac2dd0f2b02bdb4eac636e402b63620881c126c760d60b5313122323830b18a",
   "targets": {
     "darwin-arm64": {
       "goos": "darwin",
       "goarch": "arm64",
       "binary": "daemon/bin/darwin-arm64/botbuddy-daemon",
-      "sha256": "40982758f67eb18207e820823b3f28d0d1ee35bd8c84a70d22262a4a0ce306d8",
+      "sha256": "7ab6917ca6dd35571ff58b30e3baa08c387b7395a61e8e75d833e86ba379bfea",
       "mode": "0755"
     },
     "linux-x64": {
       "goos": "linux",
       "goarch": "amd64",
       "binary": "daemon/bin/linux-x64/botbuddy-daemon",
-      "sha256": "a48a8775314b124b3fad95e5a2aabd44cf94d26af57bafe7ab01fe4d7aec6724",
+      "sha256": "0752327535efa03af48f49c97eca74b0df6050b46f16ef44ccc0ebdf042169d6",
       "mode": "0755"
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botbuddy/cli",
-  "version": "1.64.5",
+  "version": "1.64.7",
   "description": "BotBuddy \u2014 Swarm coordination CLI for multi-agent workflows",
   "type": "module",
   "bin": {

package/src/stack-native-callback.mjs

@@ -81,18 +81,24 @@ function callbackReadiness(value,callbackPort){
 
 // Renewal cadence and acknowledgement budgets. The first renewal also runs the relay's
 // consuming-namespace probe (validation + docker exec), so it gets the larger budget.
-const GENERATION_MS=30_000,RENEW_INTERVAL_MS=10_000,FIRST_ACK_MS=15_000,RENEW_ACK_MS=6_000,STATUS_DEADLINE_MS=5_000,TRANSIENT_RETRY_MS=1_000;
-// A transient status read is retried after TRANSIENT_RETRY_MS only while that retry (read +
-// acknowledgement) can still land inside the current 30 s generation; otherwise the route closes.
-const TRANSIENT_GRACE_MS=GENERATION_MS-TRANSIENT_RETRY_MS-STATUS_DEADLINE_MS-RENEW_ACK_MS;
-export async function openNativeCallbackTransport({current,leaseId,hostKey,callbackPort,requesterMachineId,sshTarget,readCurrent,sshBin='ssh',env=process.env,run=execFile,spawnImpl=spawn,verifyRuntime=verifyInstalledCallbackRuntime,openGateway=openCallbackGateway,renewIntervalMs=RENEW_INTERVAL_MS,firstAckMs=FIRST_ACK_MS,renewAckMs=RENEW_ACK_MS,statusDeadlineMs=STATUS_DEADLINE_MS,transientRetryMs=TRANSIENT_RETRY_MS,transientGraceMs=TRANSIENT_GRACE_MS}){
+const GENERATION_MS=30_000,RENEW_INTERVAL_MS=10_000,FIRST_ACK_MS=15_000,RENEW_ACK_MS=6_000,STATUS_DEADLINE_MS=5_000,REFUSAL_RETRY_MS=1_000;
+// A definitive answer stops the child only if one re-read repeats it, and that re-read (read +
+// acknowledgement) must still land inside the current 30 s generation.
+const REFUSAL_GRACE_MS=GENERATION_MS-REFUSAL_RETRY_MS-STATUS_DEADLINE_MS-RENEW_ACK_MS;
+// BOT-2957: the longest the route may live on its last positive answer while the control plane
+// does not answer, counted from when that read was sent. A control-plane stall must not kill a
+// healthy run, so the requester keeps renewing the 30 s generation for this long; this number is
+// also the longest a callback revoked during such an outage can keep running. The relay and the
+// gateway keep their own 30 s generation, so a requester that dies still loses the route at once.
+export const CALLBACK_STALE_PROOF_MAX_MS=180_000;
+export async function openNativeCallbackTransport({current,leaseId,hostKey,callbackPort,requesterMachineId,sshTarget,readCurrent,sshBin='ssh',env=process.env,run=execFile,spawnImpl=spawn,verifyRuntime=verifyInstalledCallbackRuntime,openGateway=openCallbackGateway,renewIntervalMs=RENEW_INTERVAL_MS,firstAckMs=FIRST_ACK_MS,renewAckMs=RENEW_ACK_MS,statusDeadlineMs=STATUS_DEADLINE_MS,refusalRetryMs=REFUSAL_RETRY_MS,refusalGraceMs=REFUSAL_GRACE_MS,staleProofMaxMs=CALLBACK_STALE_PROOF_MAX_MS,onOutage=()=>{}}){
  if(typeof readCurrent!=='function')fail('CALLBACK_AUTHORIZATION_LOST','current callback status reader required');
  const routeGeneration=randomUUID(),options={leaseId,hostKey,callbackPort,requesterMachineId,routeGeneration};
  const binding=currentCallbackBinding(current,options),snapshot=JSON.stringify(binding),target=assertSshTarget(sshTarget),d=binding.descriptor;
  await verifyRuntime({descriptor:d,sshTarget:target,sshBin,env,run});
  const capability=randomBytes(32).toString('hex'),readinessNonce=randomBytes(16).toString('hex'),directory='/tmp/bb-callback-'+randomUUID(),socket=directory+'/callback.sock';
  const sshArgs=['-T','-o','BatchMode=yes','-o','ControlMaster=no','-o','ControlPath=none','--',target];
- let gateway,child,terminal,refreshTimer,statusAbort,pendingReady,pendingAck,closed=false,refreshing=false,cleanupPromise,exitResult;const exits=new Set();
+ let gateway,child,terminal,refreshTimer,retryTimer,staleTimer,statusAbort,pendingReady,pendingAck,closed=false,refreshing=false,cleanupPromise,exitResult;const exits=new Set();
  const reportExit=exit=>{exitResult=exit;for(const listener of exits)listener(exit)};
  const remote=async command=>{try{return await run(sshBin,[...sshArgs,command],{env,timeout:15_000,maxBuffer:4096})}catch{fail('CALLBACK_REMOTE_OPERATION_FAILED','owned callback operation failed');}};
  const retire=async()=>{
@@ -102,7 +108,7 @@ export async function openNativeCallbackTransport({current,leaseId,hostKey,callb
   await remote(command);
  };
  const close=()=>cleanupPromise??=(async()=>{
-  closed=true;clearInterval(refreshTimer);statusAbort?.abort();pendingReady?.reject(Error('callback closed'));pendingAck?.reject(Error('callback closed'));await gateway?.close();
+  closed=true;clearInterval(refreshTimer);clearTimeout(retryTimer);clearTimeout(staleTimer);statusAbort?.abort();pendingReady?.reject(Error('callback closed'));pendingAck?.reject(Error('callback closed'));await gateway?.close();
   if(child){child.stdin.end();const term=setTimeout(()=>child.kill('SIGTERM'),3000);const kill=setTimeout(()=>child.kill('SIGKILL'),6000);term.unref();kill.unref();try{await terminal}finally{clearTimeout(term);clearTimeout(kill)}}
   try{await retire()}catch{fail('CALLBACK_CLEANUP_UNCERTAIN','owned callback cleanup could not be verified');}
  })();
@@ -130,33 +136,55 @@ export async function openNativeCallbackTransport({current,leaseId,hostKey,callb
    child.stdin.write(JSON.stringify({binding,capability,socket_path:socket,readiness_nonce:readinessNonce})+'\n');
   });
   const callbackUrl=callbackReadiness(ready,callbackPort);
-  gateway.onClose(()=>{if(!closed){reportExit({reason:'callback_gateway_closed',code:'CALLBACK_GATEWAY_EXPIRED'});void close().catch(()=>{})}});
-  let renewedAt=null,refusalPending=false;
+  let renewedAt=null,refusal=null,provenAt=null,provenWall=null,lastReadFailure=null;
+  const outage={episodes:0,failed_reads:0,longest_ms:0,last_failure:null,recovered:0};let outageActive=false;
+  // BOT-2957: both clocks, because a sleeping requester stops one of them on some platforms.
+  const staleFor=()=>provenAt===null?Infinity:Math.max(performance.now()-provenAt,Date.now()-provenWall);
+  const noteOutage=active=>{outage.longest_ms=Math.max(outage.longest_ms,Math.round(staleFor()));if(active&&!outageActive)outage.episodes++;else if(!active&&outageActive)outage.recovered++;outageActive=active;onOutage({...outage});};
+  // Ends the route without a positive answer. A pending refusal keeps its own name.
+  const stopUnproven=()=>{
+   if(closed)return;
+   const lost=refusal?{code:refusal.code??'CALLBACK_AUTHORIZATION_LOST',authorization_reason:boundedAuthorizationReason(refusal.authorizationReason)??'refused'}:{code:'CALLBACK_STATUS_UNREADABLE',authorization_reason:boundedAuthorizationReason('status_unreadable:'+(lastReadFailure??'no_answer'))??'status_unreadable'};
+   if(outageActive)noteOutage(true);reportExit({reason:'callback_authorization_lost',...lost});void close().catch(()=>{});
+  };
+  const proven=(atMonotonic,atWall)=>{provenAt=atMonotonic;provenWall=atWall;refusal=null;clearTimeout(staleTimer);staleTimer=setTimeout(stopUnproven,Math.max(0,staleProofMaxMs-(performance.now()-atMonotonic)));staleTimer.unref();};
+  const rereadSoon=()=>{clearTimeout(retryTimer);retryTimer=setTimeout(()=>{void refresh()},refusalRetryMs);retryTimer.unref();};
+  gateway.onClose(()=>{if(closed)return;if(staleFor()>=staleProofMaxMs){stopUnproven();return;}reportExit({reason:'callback_gateway_closed',code:'CALLBACK_GATEWAY_EXPIRED'});void close().catch(()=>{})});
   const refresh=async(first=false)=>{
-   if(closed||refreshing)return;refreshing=true;
+   if(closed||refreshing)return;refreshing=true;clearTimeout(retryTimer);
+   // BOT-2957: every failure below names the step it failed in.
+   let step='status_read';
    try{
-    statusAbort=new AbortController();let statusTimeout,renewed;
+    statusAbort=new AbortController();let statusTimeout,renewed=binding;const readAt=performance.now(),readWall=Date.now();
     try{
-     const status=await new Promise((resolve,reject)=>{statusTimeout=setTimeout(()=>{statusAbort.abort();reject(Object.assign(Error('callback status deadline'),{transient:true}));},statusDeadlineMs);statusTimeout.unref();Promise.resolve().then(()=>readCurrent(statusAbort.signal)).then(resolve,reject).finally(()=>clearTimeout(statusTimeout));});
-     if(closed)return;renewed=currentCallbackBinding(status,options);
+     const status=await new Promise((resolve,reject)=>{statusTimeout=setTimeout(()=>{statusAbort.abort();reject(Object.assign(Error('callback status deadline'),{transient:true,readFailure:'deadline'}));},statusDeadlineMs);statusTimeout.unref();Promise.resolve().then(()=>readCurrent(statusAbort.signal)).then(resolve,reject).finally(()=>clearTimeout(statusTimeout));});
+     if(closed)return;step='status_answer';renewed=currentCallbackBinding(status,options);
      if(JSON.stringify(renewed)!==snapshot)refuse(tokenReason('binding_changed',changedBindingField(binding,renewed)),'callback context changed');
-     refusalPending=false;
+     if(outageActive)noteOutage(false);proven(readAt,readWall);lastReadFailure=null;
     }catch(error){
-     // A failed status READ is not an authorization answer: keep the live generation (relay and
-     // gateway still expire it at 30 s) and retry on the next tick while inside the grace window.
-     if(!first&&error?.transient===true&&renewedAt!==null&&performance.now()-renewedAt<transientGraceMs){const retry=setTimeout(()=>{void refresh()},transientRetryMs);retry.unref();return;}
-     if(error?.transient===true&&!error.code&&!first)error.code='CALLBACK_STATUS_UNREADABLE';
-     // BOT-2880: a definitive answer stops the child only if one re-read repeats it, while the live
-     // generation can still carry that re-read. The opening renewal and an expired grace fail closed.
-     if(!first&&error?.transient!==true&&!refusalPending&&renewedAt!==null&&performance.now()-renewedAt<transientGraceMs){refusalPending=true;const retry=setTimeout(()=>{void refresh()},transientRetryMs);retry.unref();return;}
-     throw error;
+     // The opening renewal never rides on anything: it needs a fresh positive answer.
+     if(first){if(error?.transient===true&&!error.authorizationReason)error.authorizationReason=boundedAuthorizationReason('status_unreadable:'+(error.readFailure??'no_answer'))??'status_unreadable';throw error;}
+     if(error?.transient===true){
+      lastReadFailure=typeof error.readFailure==='string'?error.readFailure:'no_answer';
+      // An answered refusal is never extended on stale proof: its re-read has only the live generation.
+      if(refusal){if(performance.now()-renewedAt<refusalGraceMs){rereadSoon();return;}stopUnproven();return;}
+      // BOT-2957: an unanswered read is not an authorization answer. Renew on the last positive
+      // answer, only until the bound since that read, and never on any read after a refusal.
+      if(staleFor()>=staleProofMaxMs){stopUnproven();return;}
+      outage.failed_reads++;outage.last_failure=lastReadFailure;noteOutage(true);
+     }else{
+      // BOT-2880: a definitive answer stops the child only if one re-read repeats it, while the live
+      // generation can still carry that re-read. An expired grace fails closed.
+      if(!refusal&&performance.now()-renewedAt<refusalGraceMs){refusal=error;rereadSoon();return;}
+      throw error;
+     }
     }
-    const sentAt=performance.now();
+    step='renewal_ack';const sentAt=performance.now();
     await new Promise((resolve,reject)=>{const timeout=setTimeout(()=>reject(Object.assign(Error('callback renewal acknowledgement deadline'),first?{}:{code:'CALLBACK_RENEWAL_ACK_TIMEOUT',transport:true})),first?firstAckMs:renewAckMs);timeout.unref();pendingAck={resolve:value=>{clearTimeout(timeout);resolve(value)},reject:error=>{clearTimeout(timeout);reject(error)}};child.stdin.write(JSON.stringify({binding:renewed,renew:true})+'\n');});
-    if(closed)return;renewedAt=sentAt;gateway.refresh(Object.freeze({snapshot}),30_000-(performance.now()-sentAt));
+    if(closed)return;renewedAt=sentAt;step='gateway_refresh';gateway.refresh(Object.freeze({snapshot}),Math.min(30_000-(performance.now()-sentAt),staleProofMaxMs-staleFor()));
    }
    // The first renewal carries the consuming-namespace probe: its uncoded failure is readiness.
-   catch(error){if(!closed){const authorizationReason=boundedAuthorizationReason(error?.authorizationReason);reportExit({reason:error?.transport===true?'callback_renewal_unacknowledged':'callback_authorization_lost',code:error?.code??(first?'CALLBACK_NOT_READY':'CALLBACK_AUTHORIZATION_LOST'),...(authorizationReason?{authorization_reason:authorizationReason}:{})});void close().catch(()=>{});}}
+   catch(error){if(!closed){const authorizationReason=boundedAuthorizationReason(error?.authorizationReason)??(error?.transport===true?null:'renewal_failed:'+step);reportExit({reason:error?.transport===true?'callback_renewal_unacknowledged':'callback_authorization_lost',code:error?.code??(first?'CALLBACK_NOT_READY':'CALLBACK_AUTHORIZATION_LOST'),...(authorizationReason?{authorization_reason:authorizationReason}:{})});void close().catch(()=>{});}}
    finally{refreshing=false}
   };
   await refresh(true);if(closed||!gateway.isLive())throw Object.assign(Error('callback route not ready after readiness'),{code:exitResult?.code??'CALLBACK_NOT_READY',...(exitResult?.authorization_reason?{authorizationReason:exitResult.authorization_reason}:{})});

package/src/stack.mjs

@@ -53,7 +53,7 @@ import { checkReadoptedStackMounts } from "./stack-mount-health.mjs";
 import { machineUuid } from "./machine-id.mjs";
 import { DEFAULT_BUNDLE_MAX_BYTES, buildStackBundle, uploadStackBundle, validateBundleIncludePath } from "./stack-bundle.mjs";
 import { assertSshTarget, establishSshTunnel } from "./stack-tunnel.mjs";
-import { NATIVE_CALLBACK_TRANSPORT, boundedAuthorizationReason, openNativeCallbackTransport } from "./stack-native-callback.mjs";
+import { CALLBACK_STALE_PROOF_MAX_MS, NATIVE_CALLBACK_TRANSPORT, boundedAuthorizationReason, openNativeCallbackTransport } from "./stack-native-callback.mjs";
 import { FORWARD_ATTESTATION_ENV, startForwardAttestation, readForwardAttestation } from "./stack-forward-attestation.mjs";
 import { stackServicesForForwards } from "./stack-service-discovery.mjs";
 import { ENGINE_PUBLISHES_BEYOND_LOOPBACK, engineSocketIsLocal, formatPublishIp, isLoopbackIp, publishIpFix, resolvePublishIp } from "./docker-publish-ip.mjs";
@@ -381,7 +381,13 @@ ${bold("REMOTE LEASES")} (BOT-2142, stack run)
                           A refusal must repeat on one re-read before it stops the child (BOT-2880), and
                           callback_authorization_reason names the refused check, e.g. requester_host_disabled
                           (the requester machine's own Docker host was set to disabled: use maintenance),
-                          lease_state:reaping, binding_changed:descriptor.producer_pid
+                          lease_state:reaping, binding_changed:descriptor.producer_pid.
+                          A control plane that does not answer is not a refusal (BOT-2957): the route rides on
+                          its last positive answer for up to 180 s. Past that, callback_cause is
+                          CALLBACK_STATUS_UNREADABLE and the reason names the failed read, e.g.
+                          status_unreadable:deadline, status_unreadable:http_503, status_unreadable:rpc_error.
+                          callback_outage on the receipt counts the gaps a run rode through.
+  6 LEASE_HEARTBEAT_UNANSWERED  every heartbeat touch went unanswered for 180 s; child stopped, lease reaped
   5 CALLBACK_PORT_BUSY    --callback-port is already listening on the serving host (another run or
                           service), the host refused this run's -R because another run took the
                           port first, or another listener shares the port; this run holds no -R —
@@ -408,6 +414,11 @@ ${bold("EXAMPLE")}
   botbuddy stack run --repo botbuddy-web --ticket BOT-1346 -- pnpm test:integration`;
 
 const STACK_COMMANDS = new Set(["up", "run", "status", "touch", "done", "cancel"]);
+// BOT-2957: a heartbeat touch that goes unanswered is retried this soon, each with its own deadline,
+// for up to CALLBACK_STALE_PROOF_MAX_MS. Past that the run stops with this code.
+export const STACK_HEARTBEAT_UNANSWERED_CODE = "LEASE_HEARTBEAT_UNANSWERED";
+const HEARTBEAT_RETRY_MS = 15_000;
+const HEARTBEAT_TOUCH_DEADLINE_MS = 15_000;
 
 /** Scoped help is assembled from the canonical text, so flags cannot drift. */
 export function stackCommandHelp(command) {
@@ -1249,6 +1260,23 @@ async function renewStackSession(auth, { cwd, renew }) {
   return withSource({ "x-agent-api-key": token }, credential.source, credential.identity);
 }
 
+// BOT-2957: how long one call waits on a credential renewal, shared or its own. The renewal itself
+// takes no signal, so without this a stalled one would hold cleanup past --reap-timeout. The call
+// then reports no answer (never a refusal), and the renewal keeps running for whoever comes next.
+const RENEWAL_WAIT_MS = 30_000;
+const RENEWAL_UNFINISHED = Symbol("renewal unfinished");
+function untilRenewalWaitEnds(renewal, signal, waitMs) {
+  if (signal?.aborted) return Promise.resolve(RENEWAL_UNFINISHED);
+  return new Promise((resolve, reject) => {
+    let timer = null;
+    const done = (settle, value) => { clearTimeout(timer); signal?.removeEventListener?.("abort", stop); settle(value); };
+    const stop = () => done(resolve, RENEWAL_UNFINISHED);
+    signal?.addEventListener?.("abort", stop, { once: true });
+    timer = setTimeout(stop, waitMs);
+    renewal.then((value) => done(resolve, value), (error) => done(reject, error));
+  });
+}
+
 /**
  * BOT-1750 (Codex R10 P2): a stack RPC caller that transparently recovers from a
  * server-rejected cached session ONCE — the owner-scoped management commands
@@ -1260,7 +1288,7 @@ async function renewStackSession(auth, { cwd, renew }) {
 export function retryingStackCall({
   callTool = callToolJson, authProvider = stackAuthHeader,
   refresh = refreshRejectedCacheAuth, cwd = () => process.cwd(),
-  auth: seedAuth,
+  auth: seedAuth, renewalWaitMs = RENEWAL_WAIT_MS,
 } = {}) {
   // A pre-resolved `auth` seed lets a caller that already resolved the credential
   // (runStackLifecycle short-circuits its own await) skip init() — avoiding an extra
@@ -1272,13 +1300,30 @@ export function retryingStackCall({
   // rejected is a new rotation, not the same failure — so it may renew again. A renewal
   // whose credential is itself rejected stays disarmed (never a per-call mint loop).
   let renewalArmed = true;
+  let pendingRenewal = null;
   return {
     async init() { if (auth === undefined) auth = await authProvider(); return auth; },
     async call(name, args, callOptions = {}) {
       let r = await callTool(name, args, { ...callOptions, auth });
-      if (renewalArmed && !r?.ok && r?.auth) {
-        renewalArmed = false;
-        const fresh = await refresh(auth, { cwd: cwd() });
+      if (!r?.ok && r?.auth && (renewalArmed || pendingRenewal)) {
+        // BOT-2957: a 401 while a renewal is still running says nothing yet: wait for that renewal
+        // (the caller's own deadline bounds the wait). Reading it as an answer fenced a healthy run.
+        if (!pendingRenewal) {
+          renewalArmed = false;
+          const rejected = auth;
+          const renewal = (async () => {
+            // A renewal the backend could not answer was not refused: keep it available, or the
+            // first read after an outage is a dead credential's 401 that reads as a refusal.
+            // The renewal adopts its own result, because every caller may have stopped waiting by then.
+            try { const renewed = await refresh(rejected, { cwd: cwd() }); if (renewed) auth = renewed; return renewed; }
+            catch (error) { if (classifyStackAuthError(error) === EXIT.BACKEND) renewalArmed = true; throw error; }
+          })();
+          pendingRenewal = renewal;
+          const settled = () => { if (pendingRenewal === renewal) pendingRenewal = null; };
+          renewal.then(settled, settled);
+        }
+        const fresh = await untilRenewalWaitEnds(pendingRenewal, callOptions.signal, renewalWaitMs);
+        if (fresh === RENEWAL_UNFINISHED) return { ok: false, status: null, error: "transport: credential renewal still running" };
         if (fresh) { auth = fresh; r = await callTool(name, args, { ...callOptions, auth }); }
       }
       if (r?.ok) renewalArmed = true;
@@ -3756,7 +3801,8 @@ export async function runStackLifecycle(opts, childArgv, adapters = {}) {
   const withIdle = incidentBox.idleLease && !withExecution?.idleLease ? { ...withExecution, idleLease: incidentBox.idleLease } : withExecution;
   const withPortBlock = incidentBox.portBlock && !withIdle?.portBlock ? { ...withIdle, portBlock: incidentBox.portBlock } : withIdle;
   const withWebUrl = incidentBox.webUrl && withPortBlock && !withPortBlock.webUrl ? { ...withPortBlock, webUrl: incidentBox.webUrl } : withPortBlock;
-  const final = incidentBox.stopSignal && withWebUrl ? { ...withWebUrl, stopSignal: incidentBox.stopSignal } : withWebUrl;
+  const withStopSignal = incidentBox.stopSignal && withWebUrl ? { ...withWebUrl, stopSignal: incidentBox.stopSignal } : withWebUrl;
+  const final = incidentBox.callbackOutage && withStopSignal ? { ...withStopSignal, callbackOutage: incidentBox.callbackOutage } : withStopSignal;
   // BOT-2467: non-enumerable, so the receipt fields and result comparisons never see it.
   if (final && typeof final === "object" && incidentBox.outcomeChannel) {
     Object.defineProperty(final, OUTCOME_CHANNEL, { value: incidentBox.outcomeChannel, enumerable: false });
@@ -3795,7 +3841,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
     getAdmission: (args) => rpc.call("get_stack_lease_admission", args),
     get: (leaseId, { signal } = {}) => rpc.call("get_stack_lease", { lease_id: leaseId }, { signal }),
     // BOT-1803: a heartbeat carries the measured client use when there is one.
-    touch: (leaseId, usage) => rpc.call("touch_stack_lease", { lease_id: leaseId, ...(usage ? { usage } : {}) }),
+    touch: (leaseId, usage, { signal } = {}) => rpc.call("touch_stack_lease", { lease_id: leaseId, ...(usage ? { usage } : {}) }, { signal }),
     // BOT-1803: `{ idle: true }` idles an ACTIVE lease instead of reaping it.
     release: (leaseId, signal, { idle = false } = {}) => rpc.call("release_stack_lease", { lease_id: leaseId, disposition: "destroy", ...(idle ? { idle: true } : {}) }, { signal }),
     // BOT-2400: keep a queued lease's place when the park deadline passes.
@@ -3871,6 +3917,8 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
   let childResult = null;
   let connectionFile = null;
   let heartbeat = null;
+  let heartbeatRetry = null;
+  let heartbeatUnanswered = false;
   let retention = null;
   let hardTimer = null;
   let cleanupResult = null;
@@ -4915,17 +4963,13 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
       if (useNativeCallback) {
         try {
           nativeCallback = await openCallback({ current: current.data, leaseId, hostKey: placement.hostKey, callbackPort: opts.callbackPort, requesterMachineId: machineUuidFn(), sshTarget,
-            readCurrent: async (signal) => { let observed; try { observed = await api.get(leaseId, { signal }); } catch (error) { throw Object.assign(new Error(`current callback status unreadable (${error?.message ?? error})`), { transient: true }); } if (!observed?.ok) {
-              // Only an unanswered read is transient (transport, 5xx, 408/429, unreadable body). A 401/403
-              // that survived the auth refresh or a typed JSON-RPC refusal is a definitive answer.
-              // BOT-2847: the shared classification (stack-release-retry.mjs) the cleanup release retries on.
-              const transient = observed == null || isTransientStackRpcResult(observed);
-              throw Object.assign(new Error(transient ? "current callback status unreadable" : "current callback authorization refused"), transient ? { transient: true } : { code: "CALLBACK_AUTHORIZATION_LOST", authorizationReason: refusedRpcReason(observed) });
-            }
-            // BOT-2790 AC-6 (BOT-2828): a backend RPC_ERROR (e.g. a statement timeout) is an unanswered read.
-            if (isTransientStackRpcResult(observed)) throw Object.assign(new Error("current callback status unreadable (RPC_ERROR)"), { transient: true });
-            if (observed.data?.success !== true) throw Object.assign(new Error("current callback authorization unavailable"), { code: "CALLBACK_AUTHORIZATION_LOST", authorizationReason: refusedRpcReason(observed) });
-            return observed.data; },
+            readCurrent: (signal) => readCallbackStatus(api, leaseId, signal),
+            onOutage: (outage) => {
+              const previous = incidentBox.callbackOutage;
+              incidentBox.callbackOutage = outage;
+              if (outage.episodes > (previous?.episodes ?? 0)) stderrLine(`stack: the control plane is not answering (${outage.last_failure}); the callback route stays on its last positive answer for up to ${CALLBACK_STALE_PROOF_MAX_MS / 1000} s`);
+              if (outage.recovered > (previous?.recovered ?? 0)) stderrLine("stack: the control plane answered again; the callback route stayed up");
+            },
           });
           nativeCallback.onExit((exit) => { if (childResult) return; callbackLost = exit; fencing = true; stopChild("SIGTERM", EXIT.LEASE_FAILED); });
           placement.callbackTransport = NATIVE_CALLBACK_TRANSPORT;
@@ -5036,35 +5080,80 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
     // BOT-1803 (AC-12): a co-located lease's heartbeat carries the measured client use; a
     // remote one (or an unreadable measurement) sends no usage at all.
     const usageLease = { ...current.data, execution_mode: current.data.execution_mode ?? (servedRemote ? "remote" : "local") };
-    heartbeat = clock.setInterval(async () => {
-      if (fencing || childResult) return;
-      let usage = null;
-      if (!servedRemote) { try { usage = await measureUsage(usageLease); } catch { usage = null; } }
-      if (fencing || childResult) return;
-      // BOT-2237: a touch whose renewal throws is a failed heartbeat (fence), never an
-      // unhandled rejection from a timer callback.
-      let touched;
-      try { touched = usage ? await api.touch(leaseId, usage) : await api.touch(leaseId); } catch (error) { touched = { ok: false, error: String(error?.message ?? error) }; }
-      if (touched?.ok && isPreemptedAnswer(touched.data)) {
-        // BOT-1803 (AC-11): the control plane reclaimed this lease's capacity. Stop the
-        // child and end typed — never the generic lost-lease wording.
-        preempted = { state: touched.data.state ?? "reaping", reap_reason: touched.data.reap_reason ?? "preempted" };
-        fencing = true;
-        stopChild("SIGTERM", EXIT.LEASE_FAILED);
-        return;
-      }
-      if (!touched?.ok || !touched?.data?.success) {
-        fencing = true;
-        stopChild("SIGTERM", EXIT.LEASE_FAILED);
-        return;
+    const heartbeatNow = adapters.now ?? Date.now;
+    let heartbeating = false;
+    let touchUnansweredSince = null;
+    const touchLease = async () => {
+      if (fencing || childResult || heartbeating) return;
+      heartbeating = true;
+      try {
+        let usage = null;
+        if (!servedRemote) { try { usage = await measureUsage(usageLease); } catch { usage = null; } }
+        if (fencing || childResult) return;
+        // BOT-2237: a touch whose renewal throws is a failed heartbeat (fence), never an
+        // unhandled rejection from a timer callback.
+        let touched;
+        let unanswered = false;
+        const touchAbort = new AbortController();
+        let touchTimer = null;
+        // The deadline ends the whole operation, not only its fetch: a credential renewal inside the
+        // touch takes no signal, and a touch that never settled would stop every later heartbeat.
+        const touchDeadline = new Promise((resolve) => {
+          touchTimer = clock.setTimeout(() => { touchAbort.abort("touch-deadline"); resolve({ ok: false, status: null, error: "transport: touch deadline" }); }, HEARTBEAT_TOUCH_DEADLINE_MS);
+          touchTimer?.unref?.();
+        });
+        try {
+          const touching = Promise.resolve(api.touch(leaseId, usage ?? undefined, { signal: touchAbort.signal }));
+          touching.catch(() => {}); // a late failure after the deadline has nobody left to hear it
+          touched = await Promise.race([touching, touchDeadline]);
+          unanswered = isTransientStackRpcResult(touched);
+        } catch (error) {
+          touched = { ok: false, error: String(error?.message ?? error) };
+          unanswered = error?.code === "session_mint_failed";
+        } finally {
+          clock.clearTimeout(touchTimer);
+        }
+        if (fencing || childResult) return;
+        if (touched?.ok && isPreemptedAnswer(touched.data)) {
+          // BOT-1803 (AC-11): the control plane reclaimed this lease's capacity. Stop the
+          // child and end typed — never the generic lost-lease wording.
+          preempted = { state: touched.data.state ?? "reaping", reap_reason: touched.data.reap_reason ?? "preempted" };
+          fencing = true;
+          stopChild("SIGTERM", EXIT.LEASE_FAILED);
+          return;
+        }
+        if (!touched?.ok || !touched?.data?.success) {
+          // BOT-2957: an unanswered touch says nothing about the lease, and one of them used to
+          // stop a healthy child during a control-plane stall. Only a real answer fences at once.
+          if (unanswered) {
+            touchUnansweredSince ??= heartbeatNow();
+            if (heartbeatNow() - touchUnansweredSince < CALLBACK_STALE_PROOF_MAX_MS) {
+              clock.clearTimeout(heartbeatRetry);
+              heartbeatRetry = clock.setTimeout(() => touchLease(), HEARTBEAT_RETRY_MS);
+              heartbeatRetry?.unref?.();
+              return;
+            }
+            heartbeatUnanswered = true;
+          }
+          fencing = true;
+          stopChild("SIGTERM", EXIT.LEASE_FAILED);
+          return;
+        }
+        touchUnansweredSince = null;
+        clock.clearTimeout(heartbeatRetry);
+        heartbeatRetry = null;
+        keepAlive();
+      } finally {
+        heartbeating = false;
       }
-      keepAlive();
-    }, cadenceMs);
+    };
+    heartbeat = clock.setInterval(touchLease, cadenceMs);
     heartbeat.unref?.();
     if (opts.hardTtl) hardTimer = clock.setTimeout(() => stopChild("SIGTERM", EXIT.TIMEOUT), opts.hardTtl * 1000);
     hardTimer?.unref?.();
     childResult = await waitForChild(child);
     if (heartbeat) clock.clearInterval(heartbeat);
+    if (heartbeatRetry) clock.clearTimeout(heartbeatRetry);
     if (retention) { clock.clearInterval(retention); retention = null; }
     if (hardTimer) clock.clearTimeout(hardTimer);
     await closeTunnel();
@@ -5101,7 +5190,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
       };
     }
     const exitCode = receivedSignal ? SIGNAL_EXIT[receivedSignal] : callbackCleanupError ? EXIT.CLEANUP_FAILED : forcedStopCode ?? (childResult.code !== 0 ? childResult.code : callbackCleanupError || !cleanup.ok ? EXIT.CLEANUP_FAILED : 0);
-    return { exitCode, outcome: exitCode === 0 ? "completed" : "failed", leaseId, childExitCode: childResult.code, cleanup, ...(callbackCleanupError ? { code: callbackCleanupError } : {}), fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) };
+    return { exitCode, outcome: exitCode === 0 ? "completed" : "failed", leaseId, childExitCode: childResult.code, cleanup, ...(callbackCleanupError ? { code: callbackCleanupError } : heartbeatUnanswered ? { code: STACK_HEARTBEAT_UNANSWERED_CODE } : {}), fenced: fencing, ...(parkIncident ? { incident: parkIncident } : {}) };
   } catch (error) {
     const cleanup = await requestCleanup();
     // BOT-1750 (Codex R10 P2): a thrown auth/self-heal failure (e.g. a refresh that
@@ -5203,6 +5292,8 @@ export function runReceiptFields(result) {
     ...(result.callbackCause ? { callback_cause: result.callbackCause } : {}),
     // BOT-2880: which check refused a CALLBACK_AUTHORIZATION_LOST (a bounded token, never a value).
     ...(result.callbackAuthorizationReason ? { callback_authorization_reason: result.callbackAuthorizationReason } : {}),
+    // BOT-2957: the control-plane gaps this run's callback route rode through (never on a run that had none).
+    ...(result.callbackOutage ? { callback_outage: result.callbackOutage } : {}),
     // BOT-2780: the reserved requester origin generated auth links name (never a credential).
     ...(result.execution?.requesterOrigin ? { requester_origin: result.execution.requesterOrigin } : {}),
     // BOT-2788: "previous" (an idle stack's origin re-reserved), "previous_port_busy" or "new".
@@ -5261,6 +5352,44 @@ export async function cmdRun(opts, childArgv, {
  * receipt deliberately does not carry (which wait a timeout ended and the original
  * lease fence). Never emitted.
  */
+// BOT-2957: what the callback status read could not get, as a bounded token for
+// callback_authorization_reason (status_unreadable:<token>).
+function callbackReadFailure(observed) {
+  if (observed == null) return "no_answer";
+  if (typeof observed.status === "number" && observed.status >= 300) return `http_${observed.status}`;
+  if (observed.ok && observed.data?.code === "RPC_ERROR") return "rpc_error";
+  if (observed.error === "invalid_json") return "invalid_json";
+  if (!observed.ok && observed.status == null) return "transport";
+  return "malformed_answer";
+}
+const unansweredCallbackRead = (failure) => Object.assign(new Error(`current callback status unreadable (${failure})`), { transient: true, readFailure: failure });
+const refusedCallbackRead = (observed) => Object.assign(new Error("current callback authorization refused"), { code: "CALLBACK_AUTHORIZATION_LOST", authorizationReason: refusedRpcReason(observed) });
+
+// One callback status read. It throws one of two things. Only an answer is a refusal: a 401/403
+// that survived the auth refresh, a typed JSON-RPC or tool refusal, or a credential that cannot be
+// renewed. Everything else is unanswered (transport, 5xx/408/429, a backend RPC_ERROR such as a
+// statement timeout, an unreadable or untyped body; BOT-2790 AC-6, BOT-2847, BOT-2957).
+export async function readCallbackStatus(api, leaseId, signal) {
+  let observed;
+  try {
+    observed = await api.get(leaseId, { signal });
+  } catch (error) {
+    const authClass = classifyStackAuthError(error);
+    if (authClass === EXIT.AUTH) throw refusedCallbackRead({ code: error.code });
+    throw unansweredCallbackRead(authClass === EXIT.BACKEND ? "session_mint" : "transport");
+  }
+  if (observed == null || isTransientStackRpcResult(observed)) throw unansweredCallbackRead(callbackReadFailure(observed));
+  if (!observed.ok) {
+    if (observed.auth || observed.code) throw refusedCallbackRead(observed);
+    throw unansweredCallbackRead(callbackReadFailure(observed));
+  }
+  if (observed.data?.success !== true) {
+    if (typeof observed.data?.code === "string" && observed.data.code) throw refusedCallbackRead(observed);
+    throw unansweredCallbackRead(callbackReadFailure(observed));
+  }
+  return observed.data;
+}
+
 // BOT-2880: a definitive RPC refusal of the callback status read names itself by the backend's
 // typed code or its HTTP status. The shared bound drops anything that is not a plain token.
 function refusedRpcReason(observed) {

package/src/staging-pool/errors.mjs

@@ -155,7 +155,7 @@ export const ERROR_CODES = Object.freeze({
     "launchctl kickstart -k gui/$(id -u)/{label}"),
   mint_backing_off: entry("human", "repair",
     "Staging sign-in for {persona} keeps failing, so the job is backing off.",
-    "The last mint_failed line in {log} names it: password_rejected or no_verified_totp_factor = seed drift; password_source_* = the password env file; totp_source_failed = the botbuddy CLI login.",
+    "The last mint_failed line in {log} names it: password_rejected or no_verified_totp_factor = seed drift; mint_retries_exhausted = a staging Auth outage, so wait for the next attempt; password_source_* = the password env file; totp_source_failed = the botbuddy CLI login.",
     "grep mint_failed {log} | tail -1   # then run bb help symptom with the code that line names"),
   last_tick_failed: entry("human", "repair",
     "The pool job's last tick had sign-in (mint) failures.",
@@ -285,6 +285,10 @@ export const ERROR_CODES = Object.freeze({
     "Staging Auth rejected the persona password.",
     "Seed drift: the staging persona was re-seeded with another password, or the env file is stale.",
     "bb staging-pool mint-one --persona {persona}   # after re-running the staging persona seed or updating the env file"),
+  password_grant_failed: entry("human", "repair",
+    "Staging Auth answered the first sign-in step with an unexpected status, which is not a credential refusal.",
+    "A blocked request, a banned or unconfirmed user, or a GoTrue change; {log} has the status and error code.",
+    "grep mint_failed {log} | tail -1   # then: bb staging-pool mint-one --persona {persona}"),
   no_verified_totp_factor: entry("human", "repair",
     "The persona has no verified TOTP factor on staging.",
     "Seed drift: the staging seed job has not enrolled the factor.",
@@ -303,7 +307,7 @@ export const ERROR_CODES = Object.freeze({
     "grep mint_failed {log} | tail -1   # report this code"),
   mint_retries_exhausted: entry("human", "repair",
     "The sign-in gave up after its retries.",
-    "Transient Auth or network failures; {log} names the last one.",
+    "A staging Auth outage (for example a Cloudflare 52x while staging restarts) or network failures; {log} names the last one.",
     "grep mint_failed {log} | tail -1   # then: bb staging-pool mint-one --persona {persona}"),
   not_aal2: entry("human", "repair",
     "MFA verify returned a session that is not AAL2.",

package/src/staging-pool/signin-gotrue-totp.mjs

@@ -25,7 +25,9 @@ export class SignInError extends Error {
   }
 }
 
-const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
+// 520-524 are Cloudflare's own answers while the origin is down or unreachable (BOT-2947):
+// Auth never saw the request or never answered it, so they say nothing about the credentials.
+const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524]);
 const REVOKED_ERROR_CODES = new Set(["session_not_found", "bad_jwt", "refresh_token_not_found"]);
 
 function describeFailure(response) {
@@ -75,6 +77,7 @@ export function createGoTrueTotpSignIn({
   const isTransient = (response) => response.status === 0 || RETRYABLE_STATUSES.has(response.status);
   const isRevoked = (response) =>
     (response.status === 401 || response.status === 403) && REVOKED_ERROR_CODES.has(response.body?.error_code);
+  const isCredentialRefusal = (response) => response.status === 400 && response.body?.error_code === "invalid_credentials";
 
   function assertAal2ForPersona(storedSession, persona) {
     const claims = decodeJwtClaims(storedSession.access_token);
@@ -108,7 +111,12 @@ export function createGoTrueTotpSignIn({
           await retry(`password grant ${describeFailure(password)}`);
           continue;
         }
-        throw new SignInError("password_rejected", `Password grant refused: ${describeFailure(password)}.`, {
+        if (isCredentialRefusal(password)) {
+          throw new SignInError("password_rejected", `Password grant refused: ${describeFailure(password)}.`, {
+            status: password.status,
+          });
+        }
+        throw new SignInError("password_grant_failed", `Password grant failed: ${describeFailure(password)}.`, {
           status: password.status,
         });
       }