@botbuddy/cli 1.63.29 → 1.63.31

package/daemon-release/darwin-arm64.json

@@ -1,7 +1,7 @@
 {
-  "release_version": "1.63.29",
-  "source_commit": "732c555dd44141d581f96959ed590007b3ec09de",
-  "source_sha256": "c2f0fc90169b4580f2d8918fb1100b853e0663a65fe470dc399c2a33787d7639",
+  "release_version": "1.63.31",
+  "source_commit": "3f7fe8fd0dfb2a66d2d1eedd7fa1a0e6852190a8",
+  "source_sha256": "acb5cf30dc9a9ecbfcddbbcae39c4c280a7da9c3f1186f9bafc232e1bebdb645",
   "target": "darwin-arm64",
-  "sha256": "05fd954131e610edd04ff1ee4ee4a0e37047f3ab56bbb0d4fd5fc3cee82bcd86"
+  "sha256": "f0b5a4a0d1ee1d059dbad7c14f7f9bf99e9d1665e23e79a9a46dcdc0e8b65e8d"
 }

package/daemon-release/linux-x64.json

@@ -1,7 +1,7 @@
 {
-  "release_version": "1.63.29",
-  "source_commit": "732c555dd44141d581f96959ed590007b3ec09de",
-  "source_sha256": "c2f0fc90169b4580f2d8918fb1100b853e0663a65fe470dc399c2a33787d7639",
+  "release_version": "1.63.31",
+  "source_commit": "3f7fe8fd0dfb2a66d2d1eedd7fa1a0e6852190a8",
+  "source_sha256": "acb5cf30dc9a9ecbfcddbbcae39c4c280a7da9c3f1186f9bafc232e1bebdb645",
   "target": "linux-x64",
-  "sha256": "1b3d42884f57dd70d3946137b704009d42ee027e2ca060b2e4df70325c52c5b7"
+  "sha256": "1bb884654f83bbfd9c9083e6cd777297600fd473f7e22e8572610f002c433057"
 }

package/daemon-release/manifest.json

@@ -1,21 +1,21 @@
 {
   "schema_version": 1,
-  "release_version": "1.63.29",
-  "source_commit": "732c555dd44141d581f96959ed590007b3ec09de",
-  "source_sha256": "c2f0fc90169b4580f2d8918fb1100b853e0663a65fe470dc399c2a33787d7639",
+  "release_version": "1.63.31",
+  "source_commit": "3f7fe8fd0dfb2a66d2d1eedd7fa1a0e6852190a8",
+  "source_sha256": "acb5cf30dc9a9ecbfcddbbcae39c4c280a7da9c3f1186f9bafc232e1bebdb645",
   "targets": {
     "darwin-arm64": {
       "goos": "darwin",
       "goarch": "arm64",
       "binary": "daemon/bin/darwin-arm64/botbuddy-daemon",
-      "sha256": "05fd954131e610edd04ff1ee4ee4a0e37047f3ab56bbb0d4fd5fc3cee82bcd86",
+      "sha256": "f0b5a4a0d1ee1d059dbad7c14f7f9bf99e9d1665e23e79a9a46dcdc0e8b65e8d",
       "mode": "0755"
     },
     "linux-x64": {
       "goos": "linux",
       "goarch": "amd64",
       "binary": "daemon/bin/linux-x64/botbuddy-daemon",
-      "sha256": "1b3d42884f57dd70d3946137b704009d42ee027e2ca060b2e4df70325c52c5b7",
+      "sha256": "1bb884654f83bbfd9c9083e6cd777297600fd473f7e22e8572610f002c433057",
       "mode": "0755"
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botbuddy/cli",
-  "version": "1.63.29",
+  "version": "1.63.31",
   "description": "BotBuddy \u2014 Swarm coordination CLI for multi-agent workflows",
   "type": "module",
   "bin": {

package/src/api.mjs

@@ -530,7 +530,9 @@ export async function callToolJson(toolName, args = {}, {
   const text = payload.result?.content?.map((c) => c.text).filter(Boolean).join("\n") ?? "";
   let data;
   try { data = text ? JSON.parse(text) : {}; } catch { data = { raw: text }; }
-  return { ok: true, data, isError: payload.result?.isError === true };
+  // BOT-2847 (Codex #1434): keep the HTTP status, so a gateway 5xx whose body is JSON (but not a
+  // JSON-RPC answer) is not mistaken for a definite tool answer by a retrying caller.
+  return { ok: true, data, isError: payload.result?.isError === true, status: res.status };
 }
 
 export async function listResources({ fetchImpl = fetch, auth = null, tenant = null, resolvePin = resolveDefaultTenantPin, log = (line) => console.log(line) } = {}) {

package/src/stack-release-retry.mjs

@@ -0,0 +1,81 @@
+// BOT-2847: a transient backend failure on a stack lease release is retried, never leaked.
+//
+// The classification is BOT-2790 AC-6's (BOT-2828), shared with the native callback's
+// status read: an UNANSWERED call is transient — a backend `RPC_ERROR` (for example a
+// statement timeout), a transport failure, an HTTP 5xx/408/429 or an unreadable body. A
+// 401/403 that survived the auth refresh, a typed JSON-RPC refusal or a typed tool refusal
+// (`NOT_LEASE_OWNER`, `LEASE_NOT_FOUND`, ...) is a definite answer and is never retried.
+//
+// Retrying a release is safe: `release_stack_lease` is idempotent (a duplicate release is a
+// no-op, an already-idle lease answers idle again), and an `RPC_ERROR` means the server's
+// transaction rolled back.
+
+/** Backoff between release attempts: 8 attempts within about 60 s. */
+export const RELEASE_RETRY_DELAYS_MS = Object.freeze([1000, 2000, 4000, 8000, 15000, 15000, 15000]);
+/** The wall-clock budget for one release's retries (the delays alone add up to it). */
+export const RELEASE_RETRY_BUDGET_MS = 60_000;
+/** Backoff for the handoff call made once the release's retries run out. */
+export const RELEASE_HANDOFF_DELAYS_MS = Object.freeze([2000, 4000]);
+/** The deadline for the handoff (its own: the release's deadline may already have passed). */
+export const RELEASE_HANDOFF_TIMEOUT_MS = 20_000;
+/** `cleanup.code` when the release never got an answer and the lease was handed off instead. */
+export const CLEANUP_RETRY_EXHAUSTED = "CLEANUP_RETRY_EXHAUSTED";
+
+/** True when a `callToolJson`-shaped result is an unanswered call (BOT-2790 AC-6). */
+export function isTransientStackRpcResult(result) {
+  if (result == null || typeof result !== "object") return false;
+  const status = result.status;
+  const unansweredStatus = typeof status === "number" && (status >= 500 || status === 408 || status === 429);
+  // A gateway 5xx/408/429 whose body happens to be JSON still parses as `ok` (Codex #1434).
+  if (result.ok) return unansweredStatus || (result.data?.success !== true && result.data?.code === "RPC_ERROR");
+  if (result.auth) return false;
+  return status == null || unansweredStatus || result.error === "invalid_json";
+}
+
+/** One short line naming why a stack RPC failed, for a receipt. */
+export function describeStackRpcFailure(result) {
+  if (result == null || typeof result !== "object") return "no response";
+  if (result.ok) {
+    const code = result.data?.code ?? (typeof result.status === "number" && result.status >= 300 ? `HTTP ${result.status}` : "failed");
+    const message = typeof result.data?.message === "string" && result.data.message ? `: ${result.data.message}` : "";
+    return `${code}${message}`.slice(0, 160);
+  }
+  if (result.error === "invalid_json") return `unreadable response (HTTP ${result.status ?? "?"})`;
+  if (result.status != null && result.status >= 300) return `HTTP ${result.status}${result.error ? ` ${result.error}` : ""}`.slice(0, 160);
+  return String(result.error ?? "no response").slice(0, 160);
+}
+
+/** A real-timer sleep that ends early when `signal` aborts. */
+export function abortableSleep(ms, signal) {
+  return new Promise((resolve) => {
+    if (signal?.aborted) { resolve(); return; }
+    let timer = null;
+    const done = () => { clearTimeout(timer); signal?.removeEventListener?.("abort", done); resolve(); };
+    timer = setTimeout(done, ms);
+    signal?.addEventListener?.("abort", done, { once: true });
+  });
+}
+
+/**
+ * Run `invoke` until it returns something other than a transient failure, backing off
+ * between attempts. Stops when the delays or `budgetMs` run out, or when `signal` aborts.
+ * A thrown error propagates unchanged (the caller owns that path).
+ */
+export async function callWithTransientRetry(invoke, {
+  delaysMs = RELEASE_RETRY_DELAYS_MS, budgetMs = RELEASE_RETRY_BUDGET_MS,
+  sleep = abortableSleep, now = Date.now, signal,
+} = {}) {
+  const started = now();
+  let attempts = 0;
+  let result;
+  for (;;) {
+    attempts += 1;
+    result = await invoke();
+    if (!isTransientStackRpcResult(result)) return { result, attempts, exhausted: false, elapsedMs: now() - started };
+    const delay = delaysMs[attempts - 1];
+    if (delay == null || signal?.aborted || now() - started + delay > budgetMs) break;
+    await sleep(delay, signal);
+    if (signal?.aborted) break;
+  }
+  return { result, attempts, exhausted: true, elapsedMs: now() - started };
+}

package/src/stack.mjs

@@ -45,6 +45,7 @@ import { runDockerCommand, runDockerWorkflow, ADMITTED_DOCKER_VALIDATIONS } from
 import { acquireStackLock, lockPathForProject, projectIdFromConfig } from "./stack-file-lock.mjs";
 import { LEASED_STACK_IDLE_RETENTION_SECS, assertFreshRunStackAvailable, idleRequesterOriginPort, recordRequesterOrigin, stableRunStackIdle, recordAdmissionRequest, bindLeasedStackClaim, recoverLeasedStackClaim, hasLocalLeaseClaim, leasedClaimContext, assertLeasedStackIsolated, leasedStackClaimHeld, markLeasedStackReleased, stableRunStackReusable, leasedStackIdentity, materializeRunStackDir, portMapInConfig, refreshLeasedStackRetention, releaseLeasedStackClaim, reservePortBlocks } from "./stack-isolation.mjs";
 import { reserveRequesterOrigin } from "./stack-requester-origin.mjs";
+import { CLEANUP_RETRY_EXHAUSTED, RELEASE_HANDOFF_DELAYS_MS, RELEASE_HANDOFF_TIMEOUT_MS, callWithTransientRetry, describeStackRpcFailure, isTransientStackRpcResult } from "./stack-release-retry.mjs";
 import { identifiedAdmissionAck, identifiedAdmissionDelivery, identifiedAdmissionRecovery } from "./stack-identified-admission.mjs";
 import { patchRequesterAuthOrigin } from "./stack-requester-origin-config.mjs";
 import { measureStackUsage } from "./stack-usage.mjs";
@@ -3278,6 +3279,7 @@ export async function cmdDone(leaseId, opts, {
   waitFn = waitForLease,
   recoverClaim = recoverLocalLeaseClaim,
   localClaimExists = hasLocalLeaseClaim,
+  releaseRetry = {},
 } = {}) {
   // BOT-1750 (Codex R10 P2): `done` must recover a server-rejected cached session on
   // its own — otherwise an active lease can't be released until the cache is manually
@@ -3420,9 +3422,14 @@ export async function cmdDone(leaseId, opts, {
   // batch on this slot). `--reap` is today's eager release, and `--local-exec` implies it:
   // its self-teardown below (and every hermetic lane / Supply Guard stop) needs the reap.
   const idleRelease = !opts.reap && !opts.localExec;
-  const r = await call("release_stack_lease", { lease_id: leaseId, disposition: opts.disposition, ...(idleRelease ? { idle: true } : {}) });
-  if (!r.ok) return emitResult(buildReceipt({ command: "done", outcome: "error", lease_id: leaseId, error: r.error }), opts, r.auth ? EXIT.AUTH : EXIT.BACKEND);
-  if (!r.data.success) return emitResult(buildReceipt({ command: "done", outcome: "error", lease_id: leaseId, code: r.data.code, error: r.data.code }), opts, EXIT.BACKEND);
+  // BOT-2847: an unanswered release (RPC_ERROR, transport, 5xx) is retried for about 60 s; a
+  // definite refusal is reported at once. An exhausted retry names the cause and the attempts.
+  const released = await callWithTransientRetry(
+    () => call("release_stack_lease", { lease_id: leaseId, disposition: opts.disposition, ...(idleRelease ? { idle: true } : {}) }), releaseRetry);
+  const r = released.result;
+  const unanswered = released.exhausted ? `${describeStackRpcFailure(r)} (no answer in ${released.attempts} attempts over ${Math.round(released.elapsedMs / 1000)}s)` : null;
+  if (!r.ok) return emitResult(buildReceipt({ command: "done", outcome: "error", lease_id: leaseId, error: unanswered ?? r.error }), opts, r.auth ? EXIT.AUTH : EXIT.BACKEND);
+  if (!r.data.success) return emitResult(buildReceipt({ command: "done", outcome: "error", lease_id: leaseId, code: r.data.code, error: unanswered ?? r.data.code }), opts, EXIT.BACKEND);
   if (idleRelease && r.data.idle === true) {
     return emitResult(buildReceipt({
       command: "done", outcome: "idle", lease_id: leaseId, state: r.data.state ?? "active",
@@ -3904,6 +3911,53 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
     }
   };
 
+  // BOT-2847: the release's retry policy (tests inject an instant `sleep`).
+  const cleanupRetry = adapters.cleanupRetry ?? {};
+  // BOT-1803: an idle lease's stack is still running — keep its directory and host port block
+  // reserved well past any idle TTL so no other worktree takes them; the next run renews the
+  // claim. BOT-2847: an idle handoff after an unanswered release needs the same retention.
+  const retainIdleStack = () => {
+    if (!execution) return;
+    try { refreshRetention(stackDirFor(execution), { idleTtlSecs: LEASED_STACK_IDLE_RETENTION_SECS, idleRetained: true }); } catch { /* advisory */ }
+    try { execution.renew?.({ idleTtlSecs: LEASED_STACK_IDLE_RETENTION_SECS, idleRetained: true }); } catch { /* advisory */ }
+  };
+  // BOT-2847 AC-2: the release never got an answer, so the lease must not stay active. Hand it
+  // off through the other disposition, a different server path: an eager release that failed
+  // goes idle (reclaimable — a queued lease preempts it, and it is reaped once its owner is
+  // gone); an idle release that failed goes to the reaper. Neither is a proven cleanup, so the
+  // result stays failed (CLEANUP_FAILED) and the receipt names the cause and the handoff.
+  const handOffUnreleasedLease = async (idle, attempt, releaseAs) => {
+    const cause = describeStackRpcFailure(attempt.result);
+    const secs = Math.round(attempt.elapsedMs / 1000);
+    const handoffAbort = new AbortController();
+    const handoffTimer = clock.setTimeout(() => handoffAbort.abort("handoff-timeout"), cleanupRetry.handoffTimeoutMs ?? RELEASE_HANDOFF_TIMEOUT_MS);
+    let handoff;
+    try {
+      handoff = await callWithTransientRetry(releaseAs(!idle, handoffAbort.signal), {
+        ...cleanupRetry, delaysMs: cleanupRetry.handoffDelaysMs ?? RELEASE_HANDOFF_DELAYS_MS, signal: handoffAbort.signal,
+      });
+    } catch (error) {
+      handoff = { result: { ok: false, error: String(error?.message ?? error) } };
+    } finally {
+      clock.clearTimeout(handoffTimer);
+    }
+    const answer = handoff.result;
+    const accepted = answer?.ok === true && answer.data?.success === true;
+    const handedTo = !accepted ? null : answer.data.idle === true ? "idle" : "reaper";
+    // Codex #1434 P2: the idle stack keeps running, so its directory and port claim must not expire.
+    if (handedTo === "idle") retainIdleStack();
+    const unanswered = `release_stack_lease${idle ? " (idle)" : ""} got no answer in ${attempt.attempts} attempts over ${secs}s (${cause})`;
+    const outcome = handedTo === "idle"
+      ? "; the lease was handed off idle instead (reclaimable: a queued lease preempts it, and it is reaped once its owner is gone)"
+      : handedTo === "reaper"
+        ? `; the lease was handed to the reaper instead (state ${answer.data.state ?? "unknown"}), its signed reap not awaited`
+        : `; the ${idle ? "reaper" : "idle"} handoff also failed (${describeStackRpcFailure(answer)}), so the lease may still be active — release it with \`bb stack done ${leaseId} --reap\``;
+    return {
+      ok: false, code: CLEANUP_RETRY_EXHAUSTED, error: `${unanswered}${outcome}`,
+      handoff: { handed_to: handedTo, state: accepted ? answer.data.state ?? null : null, release_attempts: attempt.attempts, handoff_attempts: handoff.attempts ?? 1 },
+    };
+  };
+
   // BOT-1803: `idle` (only the normal end of a run) asks the server to idle the ACTIVE lease
   // — no reap job, no signed-reap wait. Any answer that is not an idle one (the lease was
   // not active) continues into today's signed-reap wait unchanged.
@@ -3918,12 +3972,18 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
     if (!leaseId) return cleanupResult = { ok: true, state: null };
     const cleanupAbort = new AbortController();
     const cleanupTimer = clock.setTimeout(() => cleanupAbort.abort("reap-timeout"), opts.reapTimeout * 1000);
+    const releaseAs = (asIdle, signal) => () => (asIdle ? api.release(leaseId, signal, { idle: true }) : api.release(leaseId, signal));
     let release;
+    let attempt;
     try {
-      release = idle ? await api.release(leaseId, cleanupAbort.signal, { idle: true }) : await api.release(leaseId, cleanupAbort.signal);
+      // BOT-2847: an unanswered release (RPC_ERROR, transport, 5xx) is retried with backoff for
+      // about 60 s; a definite refusal is returned at once.
+      attempt = await callWithTransientRetry(releaseAs(idle, cleanupAbort.signal), { ...cleanupRetry, signal: cleanupAbort.signal });
+      release = attempt.result;
     } finally {
       clock.clearTimeout(cleanupTimer);
     }
+    if (attempt.exhausted) return cleanupResult = await handOffUnreleasedLease(idle, attempt, releaseAs);
     if (!release?.ok || !release?.data?.success) {
       return cleanupResult = { ok: false, error: release?.error || release?.data?.code || "release failed" };
     }
@@ -4839,11 +4899,12 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
             readCurrent: async (signal) => { let observed; try { observed = await api.get(leaseId, { signal }); } catch (error) { throw Object.assign(new Error(`current callback status unreadable (${error?.message ?? error})`), { transient: true }); } if (!observed?.ok) {
               // Only an unanswered read is transient (transport, 5xx, 408/429, unreadable body). A 401/403
               // that survived the auth refresh or a typed JSON-RPC refusal is a definitive answer.
-              const transient = !observed?.auth && (observed?.status == null || observed.status >= 500 || observed.status === 408 || observed.status === 429 || observed.error === "invalid_json");
+              // BOT-2847: the shared classification (stack-release-retry.mjs) the cleanup release retries on.
+              const transient = observed == null || isTransientStackRpcResult(observed);
               throw Object.assign(new Error(transient ? "current callback status unreadable" : "current callback authorization refused"), transient ? { transient: true } : { code: "CALLBACK_AUTHORIZATION_LOST" });
             }
             // BOT-2790 AC-6 (BOT-2828): a backend RPC_ERROR (e.g. a statement timeout) is an unanswered read.
-            if (observed.data?.success !== true && observed.data?.code === "RPC_ERROR") throw Object.assign(new Error("current callback status unreadable (RPC_ERROR)"), { transient: true });
+            if (isTransientStackRpcResult(observed)) throw Object.assign(new Error("current callback status unreadable (RPC_ERROR)"), { transient: true });
             if (observed.data?.success !== true) throw Object.assign(new Error("current callback authorization unavailable"), { code: "CALLBACK_AUTHORIZATION_LOST" });
             return observed.data; },
           });
@@ -4992,10 +5053,7 @@ async function runStackLifecycleOnce(opts, childArgv, adapters, incidentBox) {
     const idleOnEnd = !opts.reap && !receivedSignal && !fencing && !tunnelLost && !callbackCleanupError && forcedStopCode == null;
     const cleanup = await requestCleanup({ idle: idleOnEnd });
     if (cleanup?.idle) {
-      // The stack is still running: keep its directory and host port block reserved well
-      // past any idle TTL so no other worktree takes them; the next run renews the claim.
-      try { refreshRetention(stackDirFor(execution), { idleTtlSecs: LEASED_STACK_IDLE_RETENTION_SECS, idleRetained: true }); } catch { /* advisory */ }
-      try { execution.renew?.({ idleTtlSecs: LEASED_STACK_IDLE_RETENTION_SECS, idleRetained: true }); } catch { /* advisory */ }
+      retainIdleStack();
       stderrLine(`stack: lease ${leaseId} is idle${cleanup.idle_since ? ` since ${cleanup.idle_since}` : ""} — its stack keeps running for the next run on this slot (\`bb stack done ${leaseId} --reap\` tears it down)`);
     }
     if (originLost && !tunnelLost && !receivedSignal) {
@@ -5094,6 +5152,8 @@ export function runReceiptFields(result) {
     cleanup: result.cleanup?.idle ? { idle: true, idle_since: result.cleanup.idle_since ?? null } : result.cleanup?.ok ?? null,
     cleanup_code: result.cleanup?.code ?? null,
     cleanup_error: result.cleanup?.ok === false ? result.cleanup.error ?? "cleanup was not proven" : null,
+    // BOT-2847: an unanswered release names where the lease was handed off (idle | reaper | null).
+    ...(result.cleanup?.handoff ? { cleanup_handoff: result.cleanup.handoff } : {}),
     ...(result.leaseId && result.cleanup?.ok === false ? { recovery: {
       status: `bb stack status ${result.leaseId}`,
       wait: `bb wait 'lease:id=${result.leaseId},state=reaped' --heartbeat`,