mixdog 0.7.1

package/src/agent/orchestrator/bridge-retry.mjs

@@ -0,0 +1,220 @@
+/**
+ * bridge-retry — provider-agnostic auto-retry for detached bridge workers.
+ *
+ * Wraps a runFn (returns askSession result) with bounded auto-retry when the
+ * failure is a TRANSIENT NETWORK fault (Axis R). STALL failures do NOT retry.
+ * Fresh session per attempt — no message-history carry-over.
+ *
+ * Cap: NETWORK_MAX_ATTEMPTS total tries for network (default 3 = 2 retries),
+ * STALL_MAX_ATTEMPTS = 1 (0 retries). Cap is in-memory only; dispatch-persist
+ * stores no attempt counter, so a supervisor restart resets the attempt index
+ * to startAttempt (caller-supplied).
+ */
+
+// STALL auto-retry stays removed by decision: a stall retry re-issued against
+// the SAME provider only re-poked a slow/wedged endpoint and raced a manual
+// re-dispatch into duplicate results; the 300s first-byte stall window
+// (stall-policy.mjs DEFAULT_STALL_FIRST_BYTE_ABORT_S) is the stall backstop.
+// NETWORK-transient retry is restored separately (Axis R, below) — safe because
+// a connect failure never reaches the provider, so it cannot duplicate.
+import { isContextOverflowError } from './providers/retry-classifier.mjs';
+
+// Axis R — bounded NETWORK-transient retry restored; STALL retry stays OFF.
+// A connect/socket failure means the request never reached the provider (no
+// tool emission, no partial result) → a fresh-session retry cannot duplicate,
+// so network errors are safe to retry. A stall is mid-stream (already emitted)
+// and re-poking a wedged endpoint risks duplicates → stall stays at 0 retries.
+const NETWORK_MAX_ATTEMPTS = (() => {
+  const v = parseInt(process.env.MIXDOG_BRIDGE_NETWORK_MAX_ATTEMPTS, 10);
+  return Number.isFinite(v) ? Math.min(Math.max(v, 1), 5) : 3;
+})();
+const STALL_MAX_ATTEMPTS = 1; // intentional: stall auto-retry disabled (0 retries)
+
+// Short backoff between network-transient retries. The socket already paid the
+// connect timeout, so keep pauses small; index by attempt, clamp at the tail.
+const NETWORK_RETRY_BACKOFF_MS = [0, 500, 1000, 2000, 4000];
+
+// Small whitelist of Node.js / undici error codes that represent transient
+// network conditions safe to retry with a fresh session.  Kept as a flat Set
+// so the lookup is O(1) and the full table is visible in one place.
+const TRANSIENT_NETWORK_CODES = new Set([
+  'ECONNRESET',
+  'ECONNREFUSED',
+  'ETIMEDOUT',
+  'ENOTFOUND',
+  'EAI_AGAIN',
+  // undici (Node 18+ fetch) emits its own timeout / socket-teardown codes.
+  // Each is an explicit transient signal from the HTTP client — not a
+  // heuristic — and is safe to retry against a fresh session.
+  'UND_ERR_HEADERS_TIMEOUT',
+  'UND_ERR_BODY_TIMEOUT',
+  'UND_ERR_CONNECT_TIMEOUT',
+  'UND_ERR_SOCKET',
+]);
+
+// Transient WS close codes (abnormal close / server restart / overload).
+const TRANSIENT_WS_CODES = new Set([1006, 1011, 1012, 4000]);
+
+// Transient HTTP statuses: timeout (408), gateway faults (502/503/504).
+// 500 (Internal Server Error) is excluded — almost always an application-layer
+// problem (prompt invalid, tool schema mismatch) where retry amplifies load and
+// can double-bill tokens without changing the outcome. 502/503/504 are network
+// edge faults safe to retry with a fresh session.
+const TRANSIENT_HTTP_STATUS = new Set([408, 502, 503, 504]);
+
+/**
+ * Returns true when `err` represents a transient network or provider fault
+ * that is safe to retry with a fresh session.  Walks err.cause and
+ * err.response.data up to depth 2 so wrapped errors (axios / fetch / sdk)
+ * are detected without unbounded recursion.
+ *
+ * Classification order (first match wins):
+ *   1. err.retryable === true  — provider explicitly marks the error retryable.
+ *   2. err.code in TRANSIENT_NETWORK_CODES  — raw socket / DNS transient.
+ *   3. err.wsCloseCode in TRANSIENT_WS_CODES  — WS abnormal / server restart.
+ *   4. err.httpStatus or err.response.status in TRANSIENT_HTTP_STATUS  — gateway fault.
+ * @param {unknown} err
+ * @returns {boolean}
+ */
+export function isTransientNetworkError(err, _depth = 0) {
+  if (!err || _depth > 2) return false;
+  if (err.retryable === true) return true;
+  if (err.code != null && TRANSIENT_NETWORK_CODES.has(err.code)) return true;
+  if (err.wsCloseCode != null && TRANSIENT_WS_CODES.has(err.wsCloseCode)) return true;
+  if (err.wsCloseCode === 1000) {
+    const msg = (err instanceof Error ? err.message : String(err)) || '';
+    if (/before response\.completed/i.test(msg)) return true;
+  }
+  if (err.httpStatus != null && TRANSIENT_HTTP_STATUS.has(err.httpStatus)) return true;
+  if (err.response?.status != null && TRANSIENT_HTTP_STATUS.has(err.response.status)) return true;
+  if (err.cause != null) return isTransientNetworkError(err.cause, _depth + 1);
+  if (err.response?.data != null) return isTransientNetworkError(err.response.data, _depth + 1);
+  return false;
+}
+
+/**
+ * Returns true when `err` represents a bridge-stall-watchdog abort
+ * (no-first-byte, no-SSE-delta, tool-runtime-exceeded). The watchdog at
+ * src/agent/bridge-stall-watchdog.mjs constructs an Error with
+ * name='BridgeStallAbortError' and message='bridge stall watchdog: <N>s',
+ * then passes the message string to closeSession() so SessionClosedError's
+ * structured `reason` field (manager.mjs:1412) carries the same marker.
+ *
+ * Detection covers three surfaces in priority order:
+ *   1. err.name === 'BridgeStallAbortError'  — provider rejected the
+ *      underlying promise with signal.reason verbatim before the race
+ *      rejector fired.
+ *   2. err.reason starts with 'bridge stall watchdog'  — SessionClosedError
+ *      from post-loop validation (manager.mjs:1412) where closeReason was
+ *      threaded through closeSession().
+ *   3. err.cause walked one level for wrapped abort errors.
+ *
+ * The race-rejector path (manager.mjs:1232) loses the marker (closes with
+ * a generic 'aborted during call' reason, reason=null). For that case the
+ * caller supplies an `isStallAbort` predicate (e.g. `stallWatch.fired()`).
+ *
+ * @param {unknown} err
+ * @returns {boolean}
+ */
+export function isRetryableStall(err, _depth = 0) {
+  if (!err || _depth > 2) return false;
+  if (err.name === 'BridgeStallAbortError') return true;
+  const reasonStr = typeof err.reason === 'string' ? err.reason : '';
+  if (reasonStr.startsWith('bridge stall watchdog')) return true;
+  if (err.cause != null && err.cause !== err) return isRetryableStall(err.cause, _depth + 1);
+  return false;
+}
+/**
+ * Run `runFn(attempt)` with automatic retry on transient network errors.
+ *
+ * @param {object} opts
+ * @param {string}   opts.role        - role label (for stderr logging)
+ * @param {string}   opts.jobId       - jobId (for stderr logging)
+ * @param {number}  [opts.startAttempt=0] - initial attempt counter (from persisted state)
+ * @param {(attempt: number) => Promise<*>} opts.runFn
+ *   Called with the current attempt index.  Must return the askSession result
+ *   or throw.  A fresh session must be created by the caller inside runFn
+ *   when attempt > 0 (the wrapper does NOT manage session lifecycle).
+ * @param {AbortSignal} [opts.externalSignal=null]
+ *   Caller's abort signal; checked at every attempt boundary so request
+ *   cancellation surfaces immediately instead of paying another provider call.
+ * @param {() => boolean} [opts.isStallAbort=null]
+ *   Optional predicate the caller can wire to `stallWatch.fired()` so the
+ *   wrapper can identify a stall abort even when the race rejector
+ *   (_api_call_with_interrupt in manager.mjs:1232) strips the structured
+ *   closeReason from the SessionClosedError that surfaces here.
+ * @returns {Promise<{ result: *, attempt: number }>}
+ *   Resolves with the successful result and the attempt index that succeeded.
+ *   Rejects with the last error when all attempts are exhausted or the error
+ *   is non-recoverable.
+ */
+export async function runWithDispatchRetry({ role, jobId, startAttempt = 0, runFn, externalSignal = null, isStallAbort = null }) {
+  let attempt = typeof startAttempt === 'number' && startAttempt >= 0 ? startAttempt : 0;
+
+  while (attempt < NETWORK_MAX_ATTEMPTS) {
+    // Abort fence at every attempt boundary — uninterruptible retry loops are
+    // a request-cancellation invariant violation; if the caller's signal aborts
+    // mid-retry, surface immediately instead of paying another provider call.
+    if (externalSignal?.aborted) {
+      throw new Error('[bridge-retry] aborted by caller signal');
+    }
+    try {
+      const result = await runFn(attempt);
+      return { result, attempt };
+    } catch (err) {
+      const nextAttempt = attempt + 1;
+      // Stall classification — checked BEFORE network classification because a
+      // BridgeStallAbortError can technically carry a transient-looking cause
+      // (e.g. underlying socket close after the watchdog abort propagated).
+      // The watchdog is the authoritative signal; do not let the cause walk
+      // re-classify it as a network retry.
+      // Context-window-exceeded is a deterministic refusal: re-issuing the
+      // SAME oversized input against a fresh session (this wrapper carries no
+      // history) cannot change the outcome. The trim-and-retry fix lives in
+      // the agent loop's send path (loop.mjs), which shrinks the transcript
+      // before re-sending. Never classify it as a retryable transient here.
+      const contextOverflow = isContextOverflowError(err);
+      const stalled =
+        !contextOverflow
+        && (isRetryableStall(err)
+        || (typeof isStallAbort === 'function' && (() => {
+          try { return !!isStallAbort(); } catch { return false; }
+        })()));
+      const networkTransient = !contextOverflow && !stalled && isTransientNetworkError(err);
+
+      // Network retries up to NETWORK_MAX_ATTEMPTS (safe: no duplicate, request
+      // never reached the provider). Stall does NOT retry (STALL_MAX_ATTEMPTS=1
+      // → 0 retries) — intentional decision preserved.
+      const networkCanRetry = networkTransient && nextAttempt < NETWORK_MAX_ATTEMPTS;
+      const stallCanRetry = stalled && nextAttempt < STALL_MAX_ATTEMPTS;
+      if (networkCanRetry || stallCanRetry) {
+        const reasonLabel = stalled ? 'stall' : 'network';
+        const waitMs = networkCanRetry
+          ? NETWORK_RETRY_BACKOFF_MS[Math.min(attempt, NETWORK_RETRY_BACKOFF_MS.length - 1)]
+          : 0;
+        const waitedSeconds = (waitMs / 1000).toFixed(1);
+        const msg = (err instanceof Error ? err.message : String(err)) || '';
+        try {
+          process.stderr.write(
+            `[bridge-retry] role=${role} job=${jobId} attempt=${attempt} reason=${reasonLabel} waitedSeconds=${waitedSeconds} error="${msg.slice(0, 160)}"\n` +
+            `[bridge-retry] role=${role} job=${jobId} → retrying as attempt=${nextAttempt}\n`,
+          );
+        } catch { /* best-effort */ }
+        if (waitMs > 0) {
+          await new Promise((r) => setTimeout(r, waitMs));
+        }
+        attempt = nextAttempt;
+        continue;
+      }
+
+      // Non-recoverable, stall (0 retries), or network cap reached — surface the
+      // original error. A stall lands here on first occurrence; the outer catch
+      // in src/agent/index.mjs observes stallWatch.fired() and routes through
+      // the abort path.
+      throw err;
+    }
+  }
+
+  // Unreachable, but TypeScript / linters are happier.
+  throw new Error('[bridge-retry] exhausted attempts without resolution');
+}