@botbuddy/cli 1.47.1 → 1.48.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botbuddy/cli",
-  "version": "1.47.1",
+  "version": "1.48.0",
   "description": "BotBuddy — Swarm coordination CLI for multi-agent workflows",
   "type": "module",
   "bin": {

package/src/wait-core.mjs

@@ -31,6 +31,10 @@ export const EXIT = Object.freeze({
   // BOT-2330: an exact managed-allocation cleanup failed. The waiter must not
   // report a successful terminal match when the signed Helper outcome failed.
   MANAGED_ALLOCATION_FAILED: 10,
+  // BOT-2360: the server ended this wait while the client was still parked on it — the
+  // liveness reaper abandoned it, it was cancelled, or its row is gone. The relay
+  // refuses the stream with a typed `wait_abandoned` error frame; re-arm from next_cursor.
+  ABANDONED: 11,
 });
 
 // BOT-2013: the uuid spellings Postgres accepts for a lease id (uuid_in), mirrored by the SQL
@@ -1756,6 +1760,20 @@ export async function runWaitLoop({
         if (parsed.error === "unauthorized" || parsed.error === "forbidden") {
           return finalize("error", { exitCode: EXIT.AUTH, error: parsed.error });
         }
+        // BOT-2360: the relay refuses a stream bound to a wait the server already ended
+        // (prod a0cd7b11 parked 54 min on one). Never reconnect to it: exit typed, with the
+        // confirmed cursor so a re-arm replays everything after it.
+        if (parsed.error === "wait_abandoned") {
+          return finalize("abandoned", {
+            exitCode: EXIT.ABANDONED,
+            error: "wait_abandoned",
+            state: {
+              wait_session_id: typeof parsed.wait_session_id === "string" ? parsed.wait_session_id : waitSessionId ?? null,
+              status: typeof parsed.status === "string" ? parsed.status : null,
+              reason: typeof parsed.reason === "string" ? parsed.reason : null,
+            },
+          });
+        }
         return finalize("error", { exitCode: EXIT.INTERNAL, error: parsed.error || "stream_error" });
       }
 

package/src/wait.mjs

@@ -180,7 +180,12 @@ OUTPUT
               8 superseded (wait replaced server-side) · 9 lease lost (a
                 lease:id= wait whose lease was reaped before it was granted;
                 receipt outcome lease_lost + reap_reason — request a new
-                lease). If the receipt has
+                lease) · 10 managed-allocation cleanup failed (BOT-2330)
+                · 11 abandoned (the server ended this wait while it
+                was parked — reaped after 180 s without a live stream,
+                or cancelled; receipt outcome abandoned +
+                state.status/reason — re-arm with --since <next_cursor>).
+                If the receipt has
                 reacquire_lock (claim=true), acquire_lock again FIRST and re-run
                 only on a queued/busy result (granted = you hold it — just resume);
                 otherwise re-run from the receipt's next_cursor. docs/agent-wait.md.
@@ -548,11 +553,28 @@ export function leaseLostNotice(receipt) {
     `(reap_reason=${st.reap_reason ?? "unknown"}) — request a new lease (exit ${EXIT.LEASE_LOST})`;
 }
 
+/**
+ * BOT-2360: the one stderr line for an `abandoned` receipt (null for any other outcome). The
+ * server ended this wait (its liveness reaper or a cancel) while the client was
+ * still parked; re-arming from next_cursor replays everything after the last frame consumed.
+ */
+export function waitAbandonedNotice(receipt) {
+  if (receipt?.outcome !== "abandoned") return null;
+  const st = receipt.state ?? {};
+  const since = receipt.next_cursor != null ? ` --since ${receipt.next_cursor}` : "";
+  return `bb-wait: wait ${st.wait_session_id ?? "?"} was ended server-side (status=${st.status ?? "abandoned"}, ` +
+    `reason=${st.reason ?? "unknown"}) — re-arm the same conditions with${since || " a fresh wait"} (exit ${EXIT.ABANDONED})`;
+}
+
 function writeLeaseLostNotice(receipt) {
-  const notice = leaseLostNotice(receipt);
+  const notice = leaseLostNotice(receipt) ?? waitAbandonedNotice(receipt);
   if (notice) process.stderr.write(`${notice}\n`);
 }
 
+// BOT-1228 / BOT-2360: outcomes whose wait_session the server already ended. The client
+// never finalizes them (finalize_wait_session only transitions from active).
+const SERVER_ENDED_OUTCOMES = new Set(["superseded", "abandoned"]);
+
 // Record the terminal outcome. Best-effort AND bounded: the receipt has already
 // been written to stdout before this runs, so a slow/failing finalize must never
 // delay the agent's wake or change the exit path. A short abort timeout caps how
@@ -596,7 +618,7 @@ async function finalizeWait(opts, waitSessionId, receipt, timeoutMs = 5000) {
 // A real SSE connection to the relay's spine tail. Returns an async iterable of
 // parsed frames. Server emits `id: <seq>` per signal and replays `seq > since`
 // on connect, so a reconnect never loses an event.
-function makeConnect(opts) {
+export function makeConnect(opts) {
   // BOT-1649: mirror the relay's keepalive expiry roll into the 0600 cache. The
   // server bumps session_token_expires_at on connect and every keepalive, but
   // the cache keeps its register-time deadline — so a wait near the 8 h cap goes
@@ -633,16 +655,25 @@ function makeConnect(opts) {
     // signal tears the stalled socket down immediately, then the read rejects and
     // the stream ends so runWaitLoop reconnects.
     const ac = new AbortController();
-    const res = await fetch(url, {
-      headers: {
-        Authorization: `Bearer ${opts.token}`,
-        "x-agent-api-key": opts.token || "",
-        Accept: "text/event-stream",
-        // BOT-741: never let a proxy gzip-buffer an SSE stream.
-        "Accept-Encoding": "identity",
-      },
-      signal: ac.signal,
-    });
+    // BOT-2360: bound the wait for response headers. Unbounded, a connect that hangs on a
+    // loaded host or a stuck proxy sits silent past the wait reaper's grace (undici's own
+    // headers timeout is 300 s); bounded, the worst silent reconnect stays inside it.
+    const connectTimer = setTimeout(() => ac.abort(new Error("relay connect timed out")), waitConnectTimeoutMs());
+    let res;
+    try {
+      res = await fetch(url, {
+        headers: {
+          Authorization: `Bearer ${opts.token}`,
+          "x-agent-api-key": opts.token || "",
+          Accept: "text/event-stream",
+          // BOT-741: never let a proxy gzip-buffer an SSE stream.
+          "Accept-Encoding": "identity",
+        },
+        signal: ac.signal,
+      });
+    } finally {
+      clearTimeout(connectTimer);
+    }
     if (res.status === 401) return errorStream("unauthorized");
     if (res.status === 403) return errorStream("forbidden");
     if (!res.ok || !res.body) throw new Error(`relay responded ${res.status}`);
@@ -655,7 +686,7 @@ function makeConnect(opts) {
 // runs the global bb_wait_linear_feed_lagging RPC) whether the Linear ingest
 // feed is stale. Any failure -> false (non-fatal: the wait keeps its last known
 // degraded state rather than flapping on a probe blip).
-const FEED_LAG_PROBE_TIMEOUT_MS = 10_000;
+export const FEED_LAG_PROBE_TIMEOUT_MS = 10_000;
 
 function makeFeedLagProbe(opts) {
   // `issues` = the armed `linear` conditions' target ticket keys. The relay
@@ -715,9 +746,20 @@ function makeFeedLagProbe(opts) {
 // runWaitLoop reconnects from the cursor, which re-bumps wait_sessions.last_seen_at
 // (and, with --heartbeat, agents.last_heartbeat) before the reaper's grace.
 // 70s ≈ 2.3 missed keepalives — long enough to never false-trip on jitter, short
-// enough that the reconnect lands inside the 90s last_seen reap window.
+// enough that the reconnect lands inside the 180 s last_seen reap window (BOT-2360).
 export const DEFAULT_SSE_IDLE_TIMEOUT_MS = 70_000;
 
+// BOT-2360: the longest a (re)connect may wait for the relay's response headers. With the
+// idle watchdog (70 s), the backoff cap (15 s) and the feed-lag probe (10 s) it bounds a
+// healthy client's worst silent gap at 125 s, under the 180 s wait-reaper grace with 55 s
+// to spare (asserted in bot2360-abandoned-wait.test.mjs against the registered cron job).
+export const WAIT_CONNECT_TIMEOUT_MS = 30_000;
+
+function waitConnectTimeoutMs() {
+  const raw = Number(process.env.BOTBUDDY_WAIT_CONNECT_TIMEOUT_MS);
+  return Number.isFinite(raw) && raw > 0 ? raw : WAIT_CONNECT_TIMEOUT_MS;
+}
+
 function sseIdleTimeoutMs() {
   const raw = Number(process.env.BOTBUDDY_WAIT_IDLE_TIMEOUT_MS);
   return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_SSE_IDLE_TIMEOUT_MS;
@@ -1194,7 +1236,7 @@ async function deliverTerminalFinalization(checkpoint) {
 async function flushPendingFinalization(store, checkpoint) {
   if (!checkpoint.cloud_wait_session_id || !checkpoint.terminal_receipt ||
       (checkpoint.cloud_finalize_pending === false && checkpoint.cloud_finalized_at)) return checkpoint;
-  if (checkpoint.terminal_receipt.outcome === "superseded") {
+  if (SERVER_ENDED_OUTCOMES.has(checkpoint.terminal_receipt.outcome)) {
     return store.update(checkpoint.local_wait_id, {
       cloud_finalize_pending: false,
       cloud_finalized_at: new Date().toISOString(),
@@ -1296,6 +1338,7 @@ function exitForRecoveredOutcome(receipt) {
   if (receipt?.outcome === "cancelled") return EXIT.MATCHED;
   if (receipt?.outcome === "timeout") return EXIT.TIMEOUT;
   if (receipt?.outcome === "superseded") return EXIT.SUPERSEDED;
+  if (receipt?.outcome === "abandoned") return EXIT.ABANDONED;
   if (receipt?.error === "cursor_expired") return EXIT.CURSOR_EXPIRED;
   if (receipt?.error === "unauthorized" || receipt?.error === "forbidden") return EXIT.AUTH;
   return EXIT.BACKEND;
@@ -2095,7 +2138,7 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
             terminal_receipt: terminalReceipt,
             exit_code: state.exitCode,
             safe_cursor: terminalReceipt.next_cursor ?? state.cursor ?? checkpoint.safe_cursor,
-            cloud_finalize_pending: Boolean(waitSessionId && terminalReceipt.outcome !== "superseded"),
+            cloud_finalize_pending: Boolean(waitSessionId && !SERVER_ENDED_OUTCOMES.has(terminalReceipt.outcome)),
           });
           terminalPersistedFromFrame = true;
           return;
@@ -2125,7 +2168,7 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
         terminal_receipt: terminalReceipt,
         exit_code: exitCode,
         safe_cursor: terminalReceipt.next_cursor ?? checkpoint.safe_cursor ?? null,
-        cloud_finalize_pending: Boolean(waitSessionId && terminalReceipt.outcome !== "superseded"),
+        cloud_finalize_pending: Boolean(waitSessionId && !SERVER_ENDED_OUTCOMES.has(terminalReceipt.outcome)),
       });
     }
     emitReceipt(terminalReceipt, { versioned: true });
@@ -2134,7 +2177,8 @@ export async function runWait(argv, { recoveryLocalWaitId = null } = {}) {
     // ended_at/updated_at stamped). Skip the client finalize: the row is already
     // terminal (finalize_wait_session only transitions from active, so it'd be a
     // no-op) and `superseded` isn't a wait_session_status enum value anyway.
-    if (waitSessionId && receipt.outcome !== "superseded") {
+    // BOT-2360: likewise `abandoned` — the relay refused the stream because the row is ended.
+    if (waitSessionId && !SERVER_ENDED_OUTCOMES.has(receipt.outcome)) {
       const finalized = await finalizeWait(opts, waitSessionId, terminalReceipt);
       try {
         checkpoint = await store.update(checkpoint.local_wait_id, finalized.delivered