@botbuddy/cli 1.29.2 → 1.29.4

package/src/wait-core.mjs

@@ -1023,6 +1023,7 @@ export async function runWaitLoop({
   backoff = (attempt) => nextBackoff(attempt),
   sleep = null,
   onFrame = null,
+  startedAt: originalStartedAt = null,
   // BOT-1259: the wait session's server-resolved tenant (from the register echo). A
   // tenanted spine signal only matches a session resolved to that tenant; `null` (the
   // session's tenant could not be resolved) fails closed, and `undefined` (the caller
@@ -1042,6 +1043,9 @@ export async function runWaitLoop({
   // clear on recovery). Injected so the CLI wires the real freshness query and a
   // test drives it clock-controlled; omitted (null) = no probing.
   feedLagProbe = null,
+  // On an expired recovery, replay only durable rows created before this original
+  // deadline; the caller supplies a short transport grace via deadlineMs.
+  replayOnlyThroughMs = null,
   // BOT-1148: after a beacon-driven source_stale:true transition on a HOST-PINNED
   // capacity wait, wait this long for a recovery (source_stale:false) before exiting
   // capacity_source_stale. Absorbs threshold flap on the client (the server latch is
@@ -1057,13 +1061,28 @@ export async function runWaitLoop({
   // frame. 90s = 60s cron + margin.
   beaconConfirmMs = 90_000,
 }) {
-  const startedAt = nowIso();
+  const startedAt = typeof originalStartedAt === "string" && Number.isFinite(Date.parse(originalStartedAt))
+    ? originalStartedAt : nowIso();
   const seen = new Set();
   // An omitted --since arms from the current high-water mark (live only): the
   // relay must NOT replay history, or a brand-new wait would immediately match a
   // long-past lock release. Only an explicit --since replays (cursor != null).
   const sinceParsed = normalizeSince(since);
   let cursor = sinceParsed.kind === "ok" ? sinceParsed.value : null;
+  // The last cursor whose replay range has been confirmed by replay_complete.
+  // Live signals can overtake lower replay rows, so `cursor` is unsafe as a
+  // reconnect cursor until that marker arrives.
+  let replayFrontier = cursor;
+  // A new relay advertises replay_complete in its connected frame. Until that
+  // handshake arrives, retain legacy first-match behavior for an older relay
+  // that cannot certify a replay frontier.
+  let replayPending = false;
+  // A replay can be interleaved with newer live rows. Hold matching rows until
+  // replay_complete confirms the range, then choose the earliest eligible match.
+  // This prevents a terminal receipt from either skipping lower replay matches or
+  // replaying its own match on a chained wait.
+  const pendingReplayMatches = [];
+  let pendingReplaySupersession = null;
   const degraded = [];
   let reconnects = 0;
   let failures = 0;
@@ -1109,13 +1128,34 @@ export async function runWaitLoop({
       degraded,
       matched: extra.matched ?? [],
       state: extra.state ?? { conditions: conditions.map((c) => ({ condition_id: c.id, type: c.type })) },
-      nextCursor: cursor,
+      // A live frame can overtake an unconfirmed replay backlog. Never publish
+      // that speculative cursor in a terminal receipt: chained waits must resume
+      // from the confirmed replay frontier until replay_complete promotes it.
+      nextCursor: replayPending ? replayFrontier : cursor,
       error: extra.error ?? null,
       reacquireLock: extra.reacquireLock ?? null,
     });
     return { receipt: truncateReceipt(receipt, receiptMaxBytes), exitCode: extra.exitCode };
   };
 
+  const finalizeSuperseded = async (signal) => {
+    // Re-arm from cursor_start rather than this high sequence. A matching row can
+    // have been allocated earlier and arrive after the supersession on a replay.
+    const cs = signal.payload?.cursor_start;
+    if (cs != null && /^\d+$/.test(String(cs))) cursor = String(cs);
+    const reacquire = conditions
+      .filter((c) => c.type === "lock" && c.params.claim)
+      .map((c) => ({ subtype: c.params.subtype, host: c.params.host, slot: c.params.slot }));
+    return finalize("superseded", {
+      exitCode: EXIT.SUPERSEDED,
+      state: {
+        superseded: true,
+        status: typeof signal.payload?.status === "string" ? signal.payload.status : null,
+      },
+      reacquireLock: reacquire,
+    });
+  };
+
   // Fold the timer conditions and the --timeout cap into one earliest alarm.
   // A timer condition firing is a MATCH (exit 0); the --timeout cap is a
   // TIMEOUT (exit 2). With --any the earliest of the two always wins.
@@ -1131,27 +1171,40 @@ export async function runWaitLoop({
   // (degraded) on an EXISTING exit code (5), never a new numeric code (AC 3).
   for (const c of conditions) {
     if (c.type === "capacity" && c.params.staleGraceSec != null) {
-      alarms.push({ kind: "stale", conditionId: c.id, atMs: now() + c.params.staleGraceSec * 1000 });
+      const staleAtMs = Number.isFinite(c.params.staleDeadlineMs)
+        ? c.params.staleDeadlineMs
+        : now() + c.params.staleGraceSec * 1000;
+      alarms.push({ kind: "stale", conditionId: c.id, atMs: staleAtMs });
     }
   }
   const earliest = alarms.length ? alarms.reduce((m, a) => (a.atMs < m.atMs ? a : m)) : null;
-
-  const fireAlarm = async () => {
+  // A recovered wait may have alarms that were already due at its original
+  // deadline. Do not put those into the initial race: the durable replay must
+  // establish whether an earlier spine match won. `deadlineMs` is the caller's
+  // short transport grace, so it remains eligible to bound a stuck replay.
+  const replayGraceAlarm = replayOnlyThroughMs == null ? earliest : alarms
+    .filter((alarm) => alarm.atMs > replayOnlyThroughMs)
+    .reduce((m, alarm) => (m == null || alarm.atMs < m.atMs ? alarm : m), null);
+  const restoredAlarm = replayOnlyThroughMs == null ? null : alarms
+    .filter((alarm) => alarm.atMs <= replayOnlyThroughMs)
+    .reduce((m, alarm) => (m == null || alarm.atMs < m.atMs ? alarm : m), null);
+
+  const fireAlarm = async (alarm = earliest) => {
     // The terminal feed-freshness reprobe now lives in finalize() (the single
     // finalization path), so every terminal receipt gets it — not just the alarm path.
-    if (!earliest) return null;
-    if (earliest.kind === "timeout") return finalize("timeout", { exitCode: EXIT.TIMEOUT });
-    if (earliest.kind === "stale") {
+    if (!alarm) return null;
+    if (alarm.kind === "timeout") return finalize("timeout", { exitCode: EXIT.TIMEOUT });
+    if (alarm.kind === "stale") {
       // Capacity source didn't deliver within the grace: degrade truthfully and
       // exit with an error receipt (not a silent hang, not a plain timeout).
       if (!degraded.includes("capacity_source_stale")) degraded.push("capacity_source_stale");
       return finalize("error", { exitCode: EXIT.BACKEND, error: "capacity_source_stale" });
     }
-    const condition = conditions.find((c) => c.id === earliest.conditionId);
+    const condition = conditions.find((c) => c.id === alarm.conditionId);
     return finalize("matched", {
       exitCode: EXIT.MATCHED,
       matched: [{
-        condition_id: earliest.conditionId,
+        condition_id: alarm.conditionId,
         signal_type: "timer",
         seq: null,
         subject_key: condition?.params?.deadline ?? `+${condition?.params?.duration}s`,
@@ -1160,10 +1213,18 @@ export async function runWaitLoop({
     });
   };
 
+  // The recovery grace bounds one invocation's transport work; it must never
+  // decide the original logical wait. The caller leaves this typed result out
+  // of the durable terminal checkpoint so `bb wait resume` can try the same
+  // replay again from its confirmed cursor.
+  const fireReplayGraceExpired = async () => replayOnlyThroughMs != null
+    ? finalize("error", { exitCode: EXIT.BACKEND, error: "replay_incomplete" })
+    : fireAlarm();
+
   // A single promise that resolves when the earliest alarm passes.
-  const deadlinePromise = earliest == null
+  const deadlinePromise = replayGraceAlarm == null
     ? new Promise(() => {}) // no alarm: only a pushed signal can wake us
-    : sleepFn(Math.max(0, earliest.atMs - now())).then(() => ({ __deadline: true }));
+    : sleepFn(Math.max(0, replayGraceAlarm.atMs - now())).then(() => ({ __deadline: true }));
 
   // BOT-1147 AC-3: only probe when a linear condition is armed and a probe is
   // wired — every other wait type is unaffected.
@@ -1178,8 +1239,8 @@ export async function runWaitLoop({
   // Fires at most once (a single pre-deadline probe, not a re-arming timer, so an
   // injected instant `sleep` in tests can't spin the race).
   let preDeadlineProbeDone = false;
-  const preDeadlineProbePromise = (feedLagActive && earliest != null)
-    ? sleepFn(Math.max(0, earliest.atMs - now() - FEED_PROBE_BUDGET_MS)).then(() => ({ __feedProbe: true }))
+  const preDeadlineProbePromise = (feedLagActive && replayGraceAlarm != null)
+    ? sleepFn(Math.max(0, replayGraceAlarm.atMs - now() - FEED_PROBE_BUDGET_MS)).then(() => ({ __feedProbe: true }))
     : null;
   // The specific issue key(s) the armed `linear` conditions target. Threaded into
   // the probe so it scopes freshness to those issues' workspace(s) rather than the
@@ -1294,7 +1355,9 @@ export async function runWaitLoop({
   };
 
   while (true) {
-    if (earliest != null && now() >= earliest.atMs) {
+    // Expired recovery must first replay durable pre-deadline rows. A restored
+    // timer or capacity-stale alarm may otherwise win before that replay begins.
+    if (replayOnlyThroughMs == null && earliest != null && now() >= earliest.atMs) {
       return fireAlarm();
     }
 
@@ -1302,9 +1365,16 @@ export async function runWaitLoop({
     // and a recovered one clears — reflected in the terminal receipt / wait_session.
     await probeFeedLag();
 
+    // Every reconnect starts a new replay/live overlap window. A live signal can
+    // overtake the replay backlog, then the transport can die before its marker;
+    // reconnect from the last confirmed frontier so lower replay matches remain
+    // eligible instead of skipping straight to that live signal's cursor.
+    const connectCursor = replayFrontier;
+    const reconnectingIncompleteReplay = replayPending;
+    replayPending = replayOnlyThroughMs != null && connectCursor != null;
     let stream;
     try {
-      stream = await connect(cursor);
+      stream = await connect(connectCursor);
     } catch (err) {
       failures += 1;
       degraded.push("backend_unreachable");
@@ -1316,7 +1386,7 @@ export async function runWaitLoop({
         deadlinePromise,
         sleepFn(waitMs).then(() => ({ __retry: true })),
       ]);
-      if (raced && raced.__deadline) return fireAlarm();
+      if (raced && raced.__deadline) return fireReplayGraceExpired();
       continue;
     }
 
@@ -1336,9 +1406,7 @@ export async function runWaitLoop({
       if (preDeadlineProbePromise && !preDeadlineProbeDone) racers.push(preDeadlineProbePromise);
       const raced = await Promise.race(racers);
 
-      if (raced && raced.__deadline) {
-        return fireAlarm();
-      }
+      if (raced && raced.__deadline) return fireReplayGraceExpired();
       // BOT-1147 (Codex round-27 P2): the pre-deadline feed probe fired. Fold a
       // mid-wait stall/recovery into `degraded` before the imminent timeout receipt,
       // then keep waiting — pendingNext stays in flight (hoisted), re-raced next
@@ -1375,6 +1443,28 @@ export async function runWaitLoop({
       }
       if (!frame) continue;
 
+      if (frame.event === "connected") {
+        let connected = {};
+        try { connected = JSON.parse(frame.data); } catch { /* legacy relay */ }
+        // Old relays emit connected without this capability and never send the
+        // marker. Preserve their established first-match behavior rather than
+        // buffering a live signal to timeout during a rolling deployment.
+        if (replayOnlyThroughMs != null && connectCursor != null && connected.replay_complete_marker !== true) {
+          return finalize("error", { exitCode: EXIT.BACKEND, error: "replay_marker_unsupported" });
+        }
+        if (connectCursor != null && connected.replay_complete_marker !== true && reconnectingIncompleteReplay) {
+          // The connection that established this replay boundary died before its
+          // marker. Restart legacy semantics from the last confirmed cursor: do
+          // not retain buffered/seen rows that would discard the replayed match.
+          pendingReplayMatches.length = 0;
+          pendingReplaySupersession = null;
+          seen.clear();
+          cursor = replayFrontier;
+        }
+        replayPending = connectCursor != null && connected.replay_complete_marker === true;
+        continue;
+      }
+
       // BOT-1067: a degraded marker frame (e.g. the relay fell back to bounded
       // re-query polling during a Realtime outage) is folded into the receipt's
       // degraded array. The wait CONTINUES — delivery is just late and flagged —
@@ -1400,13 +1490,60 @@ export async function runWaitLoop({
         return finalize("error", { exitCode: EXIT.INTERNAL, error: parsed.error || "stream_error" });
       }
 
+      if (frame.event === "replay_complete") {
+        replayPending = false;
+        replayFrontier = cursor;
+        const pendingMatch = pendingReplayMatches
+          .sort((a, b) => BigInt(a.seqKey) < BigInt(b.seqKey) ? -1 : BigInt(a.seqKey) > BigInt(b.seqKey) ? 1 : 0)[0];
+        const replayCandidate = pendingMatch &&
+          (!pendingReplaySupersession || BigInt(pendingMatch.seqKey) < BigInt(pendingReplaySupersession.seqKey))
+          ? { kind: "match", ...pendingMatch }
+          : pendingReplaySupersession ? { kind: "superseded", ...pendingReplaySupersession } : null;
+        if (replayCandidate) {
+          const candidateAtMs = Date.parse(replayCandidate.signal.created_at);
+          if (restoredAlarm != null && Number.isFinite(candidateAtMs) && restoredAlarm.atMs < candidateAtMs) {
+            return fireAlarm(restoredAlarm);
+          }
+          if (replayCandidate.kind === "superseded") return finalizeSuperseded(replayCandidate.signal);
+          const terminal = await finalize("matched", {
+            exitCode: EXIT.MATCHED,
+            matched: [replayCandidate.entry],
+          });
+          if (onFrame) {
+            await onFrame(replayCandidate.signal, {
+              cursor: replayCandidate.seqKey,
+              terminalReceipt: terminal.receipt,
+              exitCode: terminal.exitCode,
+            });
+          }
+          return terminal;
+        }
+        if (replayOnlyThroughMs != null) return fireAlarm();
+        if (onFrame) await onFrame(null, { cursor, replayComplete: true });
+        continue;
+      }
       const signal = decodeSignal(frame);
       if (!signal || signal.seq == null) continue;
       const seqKey = String(signal.seq);
+      // Replayed rows can arrive after a higher live row. Keep the highest
+      // observed sequence so replay_complete can safely promote the whole
+      // confirmed range, even when that live row is de-duplicated below.
+      if (cursor == null || BigInt(seqKey) > BigInt(cursor)) cursor = seqKey;
+      if (!replayPending) replayFrontier = cursor;
       if (seen.has(seqKey)) continue; // exactly-once across reconnects
       seen.add(seqKey);
-      cursor = seqKey;
-      if (onFrame) onFrame(signal);
+      const checkpointNonterminalFrame = async () => {
+        if (!replayPending && onFrame) await onFrame(signal, { cursor: seqKey });
+      };
+      const beforeReplayDeadline = replayOnlyThroughMs == null || (
+        typeof signal.created_at === "string" &&
+        Number.isFinite(Date.parse(signal.created_at)) &&
+        Date.parse(signal.created_at) <= replayOnlyThroughMs
+      );
+      // Do not checkpoint the replay cursor yet. A crash after a pre-match
+      // checkpoint would skip this very frame on recovery. Non-matches become
+      // durable only after matching is complete; a match carries its terminal
+      // receipt in the same checkpoint callback below (BOT-1656).
 
       // BOT-1249: a `wait_superseded` spine signal targeting THIS registration is
       // terminal (exit 8). reconcile_waits_to_canonical() emits it — in seq order,
@@ -1430,43 +1567,11 @@ export async function runWaitLoop({
         waitSessionId != null &&
         signal.payload && signal.payload.wait_session_id === waitSessionId
       ) {
-        // Re-arm the re-registration from the wait's ORIGINAL arm cursor (cursor_start),
-        // NOT this supersession's seq. `seq` is allocated at INSERT, not COMMIT, and a
-        // live frame can overtake the replay backlog — so a supersession can be delivered
-        // to us AHEAD of a match that was actually eligible (a chat/pr/ci signal for a
-        // mixed `any` wait, or a canonical signal that arrived during the alias window).
-        // If we re-armed from the supersession's (higher) seq we'd skip that match on
-        // re-registration. cursor_start is the seq this wait armed from, so re-arming here
-        // gives the re-registered wait EXACTLY the coverage the original registration had
-        // — strictly better than re-arming from the last-seen (higher) cursor, which is
-        // all a normal same-wait reconnect gets. (The one residual gap — a match whose row
-        // was allocated a seq BELOW cursor_start but committed after registration read
-        // max(seq) — is the pre-existing BOT-989 register-after-emit race that any
-        // reconnect shares; it is out of scope here and tracked separately. We do NOT
-        // re-arm from 0, which would spuriously re-match stale pre-arm signals.) Falls
-        // back to the current cursor if the emitter didn't stamp cursor_start (defensive —
-        // reconcile always does).
-        const cs = signal.payload.cursor_start;
-        if (cs != null && /^\d+$/.test(String(cs))) cursor = String(cs);
-        // A superseded CLAIM wait lost its grantable queue entry: reconcile_waits_to_
-        // canonical() dequeues the queued resource_locks row before abandoning the
-        // session. A plain re-run re-registers the wait_session but does NOT recreate
-        // that queue row, so grant_free_lock would have nothing to promote and the
-        // re-armed claim would hang to --timeout. Surface the lock(s) to re-acquire so
-        // the caller re-queues (acquire_lock) BEFORE re-parking (BOT-1228 Codex round-3
-        // P1). Broadcast waits need no queue entry, so this is empty for them.
-        const reacquire = conditions
-          .filter((c) => c.type === "lock" && c.params.claim)
-          .map((c) => ({ subtype: c.params.subtype, host: c.params.host, slot: c.params.slot }));
-        return finalize("superseded", {
-          exitCode: EXIT.SUPERSEDED,
-          state: {
-            superseded: true,
-            status: typeof signal.payload.status === "string" ? signal.payload.status : null,
-          },
-          // Top-level (survives receipt truncation) — see buildReceipt.
-          reacquireLock: reacquire,
-        });
+        if (replayPending) {
+          if (beforeReplayDeadline) pendingReplaySupersession ??= { signal, seqKey };
+          continue;
+        }
+        return finalizeSuperseded(signal);
       }
 
       // BOT-1148: a beacon-driven capacity staleness transition is handled before the
@@ -1477,7 +1582,10 @@ export async function runWaitLoop({
         if (disposition === "exit") {
           return finalize("error", { exitCode: EXIT.BACKEND, error: "capacity_source_stale" });
         }
-        if (disposition === "continue") continue;
+        if (disposition === "continue") {
+          await checkpointNonterminalFrame();
+          continue;
+        }
         // "fallthrough": let matchFrame see the frame — a recovery may grant via
         // free_slots, and a raw `event` condition may match a stale frame (the
         // `capacity` matcher itself never grants on source_stale).
@@ -1491,11 +1599,12 @@ export async function runWaitLoop({
         if (debug) {
           debug(`bb-wait: dropped ${signal.signal_type} signal (seq ${signal.seq}) — tenant ${signal.tenant_id} ≠ session tenant ${sessionTenant ?? "(unresolved)"} [BOT-1259 fail-closed]`);
         }
+        await checkpointNonterminalFrame();
         continue;
       }
 
       const matched = matchFrame(frame, conditions, waitSessionId, sessionTenant);
-      if (matched) {
+      if (matched && beforeReplayDeadline) {
         const entry = {
           condition_id: matched.id,
           signal_type: signal.signal_type,
@@ -1507,11 +1616,26 @@ export async function runWaitLoop({
         // BOT-1577: a ranked capacity grant records this wait's position/tier/score.
         const grant = capacityGrantForReceipt(signal, waitSessionId);
         if (grant) entry.grant = grant;
-        return finalize("matched", {
+        if (replayPending) {
+          pendingReplayMatches.push({ entry, seqKey, signal });
+          continue;
+        }
+        const terminal = await finalize("matched", {
           exitCode: EXIT.MATCHED,
           matched: [entry],
         });
+        if (onFrame) {
+          await onFrame(signal, {
+            cursor: seqKey,
+            terminalReceipt: terminal.receipt,
+            exitCode: terminal.exitCode,
+          });
+        }
+        return terminal;
       }
+      // The frame was completely evaluated and cannot be an eligible terminal
+      // result. It is now safe to advance the persisted replay floor.
+      await checkpointNonterminalFrame();
     }
 
     // Stream ended without a match: reconnect from the cursor.
@@ -1521,6 +1645,6 @@ export async function runWaitLoop({
     }
     const waitMs = backoff(reconnects);
     const raced = await Promise.race([deadlinePromise, sleepFn(waitMs).then(() => ({ __retry: true }))]);
-    if (raced && raced.__deadline) return fireAlarm();
+    if (raced && raced.__deadline) return fireReplayGraceExpired();
   }
 }