mixdog 0.7.17 → 0.8.0

package/src/agent/orchestrator/bridge-retry.mjs

@@ -1,220 +0,0 @@
-/**
- * bridge-retry — provider-agnostic auto-retry for detached bridge workers.
- *
- * Wraps a runFn (returns askSession result) with bounded auto-retry when the
- * failure is a TRANSIENT NETWORK fault (Axis R). STALL failures do NOT retry.
- * Fresh session per attempt — no message-history carry-over.
- *
- * Cap: NETWORK_MAX_ATTEMPTS total tries for network (default 3 = 2 retries),
- * STALL_MAX_ATTEMPTS = 1 (0 retries). Cap is in-memory only; dispatch-persist
- * stores no attempt counter, so a supervisor restart resets the attempt index
- * to startAttempt (caller-supplied).
- */
-
-// STALL auto-retry stays removed by decision: a stall retry re-issued against
-// the SAME provider only re-poked a slow/wedged endpoint and raced a manual
-// re-dispatch into duplicate results; the 300s first-byte stall window
-// (stall-policy.mjs DEFAULT_STALL_FIRST_BYTE_ABORT_S) is the stall backstop.
-// NETWORK-transient retry is restored separately (Axis R, below) — safe because
-// a connect failure never reaches the provider, so it cannot duplicate.
-import { isContextOverflowError } from './providers/retry-classifier.mjs';
-
-// Axis R — bounded NETWORK-transient retry restored; STALL retry stays OFF.
-// A connect/socket failure means the request never reached the provider (no
-// tool emission, no partial result) → a fresh-session retry cannot duplicate,
-// so network errors are safe to retry. A stall is mid-stream (already emitted)
-// and re-poking a wedged endpoint risks duplicates → stall stays at 0 retries.
-const NETWORK_MAX_ATTEMPTS = (() => {
-  const v = parseInt(process.env.MIXDOG_BRIDGE_NETWORK_MAX_ATTEMPTS, 10);
-  return Number.isFinite(v) ? Math.min(Math.max(v, 1), 5) : 3;
-})();
-const STALL_MAX_ATTEMPTS = 1; // intentional: stall auto-retry disabled (0 retries)
-
-// Short backoff between network-transient retries. The socket already paid the
-// connect timeout, so keep pauses small; index by attempt, clamp at the tail.
-const NETWORK_RETRY_BACKOFF_MS = [0, 500, 1000, 2000, 4000];
-
-// Small whitelist of Node.js / undici error codes that represent transient
-// network conditions safe to retry with a fresh session.  Kept as a flat Set
-// so the lookup is O(1) and the full table is visible in one place.
-const TRANSIENT_NETWORK_CODES = new Set([
-  'ECONNRESET',
-  'ECONNREFUSED',
-  'ETIMEDOUT',
-  'ENOTFOUND',
-  'EAI_AGAIN',
-  // undici (Node 18+ fetch) emits its own timeout / socket-teardown codes.
-  // Each is an explicit transient signal from the HTTP client — not a
-  // heuristic — and is safe to retry against a fresh session.
-  'UND_ERR_HEADERS_TIMEOUT',
-  'UND_ERR_BODY_TIMEOUT',
-  'UND_ERR_CONNECT_TIMEOUT',
-  'UND_ERR_SOCKET',
-]);
-
-// Transient WS close codes (abnormal close / server restart / overload).
-const TRANSIENT_WS_CODES = new Set([1006, 1011, 1012, 4000]);
-
-// Transient HTTP statuses: timeout (408), gateway faults (502/503/504).
-// 500 (Internal Server Error) is excluded — almost always an application-layer
-// problem (prompt invalid, tool schema mismatch) where retry amplifies load and
-// can double-bill tokens without changing the outcome. 502/503/504 are network
-// edge faults safe to retry with a fresh session.
-const TRANSIENT_HTTP_STATUS = new Set([408, 502, 503, 504]);
-
-/**
- * Returns true when `err` represents a transient network or provider fault
- * that is safe to retry with a fresh session.  Walks err.cause and
- * err.response.data up to depth 2 so wrapped errors (axios / fetch / sdk)
- * are detected without unbounded recursion.
- *
- * Classification order (first match wins):
- *   1. err.retryable === true  — provider explicitly marks the error retryable.
- *   2. err.code in TRANSIENT_NETWORK_CODES  — raw socket / DNS transient.
- *   3. err.wsCloseCode in TRANSIENT_WS_CODES  — WS abnormal / server restart.
- *   4. err.httpStatus or err.response.status in TRANSIENT_HTTP_STATUS  — gateway fault.
- * @param {unknown} err
- * @returns {boolean}
- */
-export function isTransientNetworkError(err, _depth = 0) {
-  if (!err || _depth > 2) return false;
-  if (err.retryable === true) return true;
-  if (err.code != null && TRANSIENT_NETWORK_CODES.has(err.code)) return true;
-  if (err.wsCloseCode != null && TRANSIENT_WS_CODES.has(err.wsCloseCode)) return true;
-  if (err.wsCloseCode === 1000) {
-    const msg = (err instanceof Error ? err.message : String(err)) || '';
-    if (/before response\.completed/i.test(msg)) return true;
-  }
-  if (err.httpStatus != null && TRANSIENT_HTTP_STATUS.has(err.httpStatus)) return true;
-  if (err.response?.status != null && TRANSIENT_HTTP_STATUS.has(err.response.status)) return true;
-  if (err.cause != null) return isTransientNetworkError(err.cause, _depth + 1);
-  if (err.response?.data != null) return isTransientNetworkError(err.response.data, _depth + 1);
-  return false;
-}
-
-/**
- * Returns true when `err` represents a bridge-stall-watchdog abort
- * (no-first-byte, no-SSE-delta, tool-runtime-exceeded). The watchdog at
- * src/agent/bridge-stall-watchdog.mjs constructs an Error with
- * name='BridgeStallAbortError' and message='bridge stall watchdog: <N>s',
- * then passes the message string to closeSession() so SessionClosedError's
- * structured `reason` field (manager.mjs:1412) carries the same marker.
- *
- * Detection covers three surfaces in priority order:
- *   1. err.name === 'BridgeStallAbortError'  — provider rejected the
- *      underlying promise with signal.reason verbatim before the race
- *      rejector fired.
- *   2. err.reason starts with 'bridge stall watchdog'  — SessionClosedError
- *      from post-loop validation (manager.mjs:1412) where closeReason was
- *      threaded through closeSession().
- *   3. err.cause walked one level for wrapped abort errors.
- *
- * The race-rejector path (manager.mjs:1232) loses the marker (closes with
- * a generic 'aborted during call' reason, reason=null). For that case the
- * caller supplies an `isStallAbort` predicate (e.g. `stallWatch.fired()`).
- *
- * @param {unknown} err
- * @returns {boolean}
- */
-export function isRetryableStall(err, _depth = 0) {
-  if (!err || _depth > 2) return false;
-  if (err.name === 'BridgeStallAbortError') return true;
-  const reasonStr = typeof err.reason === 'string' ? err.reason : '';
-  if (reasonStr.startsWith('bridge stall watchdog')) return true;
-  if (err.cause != null && err.cause !== err) return isRetryableStall(err.cause, _depth + 1);
-  return false;
-}
-/**
- * Run `runFn(attempt)` with automatic retry on transient network errors.
- *
- * @param {object} opts
- * @param {string}   opts.role        - role label (for stderr logging)
- * @param {string}   opts.jobId       - jobId (for stderr logging)
- * @param {number}  [opts.startAttempt=0] - initial attempt counter (from persisted state)
- * @param {(attempt: number) => Promise<*>} opts.runFn
- *   Called with the current attempt index.  Must return the askSession result
- *   or throw.  A fresh session must be created by the caller inside runFn
- *   when attempt > 0 (the wrapper does NOT manage session lifecycle).
- * @param {AbortSignal} [opts.externalSignal=null]
- *   Caller's abort signal; checked at every attempt boundary so request
- *   cancellation surfaces immediately instead of paying another provider call.
- * @param {() => boolean} [opts.isStallAbort=null]
- *   Optional predicate the caller can wire to `stallWatch.fired()` so the
- *   wrapper can identify a stall abort even when the race rejector
- *   (_api_call_with_interrupt in manager.mjs:1232) strips the structured
- *   closeReason from the SessionClosedError that surfaces here.
- * @returns {Promise<{ result: *, attempt: number }>}
- *   Resolves with the successful result and the attempt index that succeeded.
- *   Rejects with the last error when all attempts are exhausted or the error
- *   is non-recoverable.
- */
-export async function runWithDispatchRetry({ role, jobId, startAttempt = 0, runFn, externalSignal = null, isStallAbort = null }) {
-  let attempt = typeof startAttempt === 'number' && startAttempt >= 0 ? startAttempt : 0;
-
-  while (attempt < NETWORK_MAX_ATTEMPTS) {
-    // Abort fence at every attempt boundary — uninterruptible retry loops are
-    // a request-cancellation invariant violation; if the caller's signal aborts
-    // mid-retry, surface immediately instead of paying another provider call.
-    if (externalSignal?.aborted) {
-      throw new Error('[bridge-retry] aborted by caller signal');
-    }
-    try {
-      const result = await runFn(attempt);
-      return { result, attempt };
-    } catch (err) {
-      const nextAttempt = attempt + 1;
-      // Stall classification — checked BEFORE network classification because a
-      // BridgeStallAbortError can technically carry a transient-looking cause
-      // (e.g. underlying socket close after the watchdog abort propagated).
-      // The watchdog is the authoritative signal; do not let the cause walk
-      // re-classify it as a network retry.
-      // Context-window-exceeded is a deterministic refusal: re-issuing the
-      // SAME oversized input against a fresh session (this wrapper carries no
-      // history) cannot change the outcome. The trim-and-retry fix lives in
-      // the agent loop's send path (loop.mjs), which shrinks the transcript
-      // before re-sending. Never classify it as a retryable transient here.
-      const contextOverflow = isContextOverflowError(err);
-      const stalled =
-        !contextOverflow
-        && (isRetryableStall(err)
-        || (typeof isStallAbort === 'function' && (() => {
-          try { return !!isStallAbort(); } catch { return false; }
-        })()));
-      const networkTransient = !contextOverflow && !stalled && isTransientNetworkError(err);
-
-      // Network retries up to NETWORK_MAX_ATTEMPTS (safe: no duplicate, request
-      // never reached the provider). Stall does NOT retry (STALL_MAX_ATTEMPTS=1
-      // → 0 retries) — intentional decision preserved.
-      const networkCanRetry = networkTransient && nextAttempt < NETWORK_MAX_ATTEMPTS;
-      const stallCanRetry = stalled && nextAttempt < STALL_MAX_ATTEMPTS;
-      if (networkCanRetry || stallCanRetry) {
-        const reasonLabel = stalled ? 'stall' : 'network';
-        const waitMs = networkCanRetry
-          ? NETWORK_RETRY_BACKOFF_MS[Math.min(attempt, NETWORK_RETRY_BACKOFF_MS.length - 1)]
-          : 0;
-        const waitedSeconds = (waitMs / 1000).toFixed(1);
-        const msg = (err instanceof Error ? err.message : String(err)) || '';
-        try {
-          process.stderr.write(
-            `[bridge-retry] role=${role} job=${jobId} attempt=${attempt} reason=${reasonLabel} waitedSeconds=${waitedSeconds} error="${msg.slice(0, 160)}"\n` +
-            `[bridge-retry] role=${role} job=${jobId} → retrying as attempt=${nextAttempt}\n`,
-          );
-        } catch { /* best-effort */ }
-        if (waitMs > 0) {
-          await new Promise((r) => setTimeout(r, waitMs));
-        }
-        attempt = nextAttempt;
-        continue;
-      }
-
-      // Non-recoverable, stall (0 retries), or network cap reached — surface the
-      // original error. A stall lands here on first occurrence; the outer catch
-      // in src/agent/index.mjs observes stallWatch.fired() and routes through
-      // the abort path.
-      throw err;
-    }
-  }
-
-  // Unreachable, but TypeScript / linters are happier.
-  throw new Error('[bridge-retry] exhausted attempts without resolution');
-}