@srouterhq/server 0.0.0-stage → 0.9.0

package/dist/lib/fallback-loop.js

@@ -0,0 +1,1336 @@
+import { soonestBudgetResetAt } from '../services/key-budget.js';
+import { recordRateLimitHit, recordModelFailure, recordSuccess, hasOtherUsableKey, routableKeyIdsForModel, formatResetEta } from '../services/router.js';
+import { safeHeaderValue } from './header-value.js';
+import { recordRequest, recordTokens, setCooldown, getActiveCooldownsForKeys, getCooldownDecisionForLimit, getSoonestCooldownExpiry, getPaymentRequiredCooldownMs, getModelForbiddenCooldownMs, learnLimitFromError, } from '../services/ratelimit.js';
+import { isRetryableError, isRateLimitSignal, isKeyAuthError, isClientAbortError, isHedgeAbortError, isDailyQuotaExhaustedError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isAccountSuspendedError, isProviderBadRequestError, isProviderDegradedError, isProviderLevelError, isContextTooLargeError, isTimeoutErrorText, isStreamTruncatedError, } from './error-classify.js';
+import { sanitizeProviderErrorMessage, summarizeAttemptError } from './error-redaction.js';
+import { benchForTools, clearToolRejections, noteToolRejection, toolCapabilityKey } from './tool-capability.js';
+import { parseProviderReportedSize } from './provider-size-parser.js';
+import { recordRouteTrace } from './route-live.js';
+import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';
+import { noteModelRetirementSignal } from '../services/model-retirement.js';
+import { getDb, getSetting } from '../db/index.js';
+import { newBreaker, recordBreakerFailure } from './guardrails.js';
+import { getRequestTrace, newRequestTrace, runWithRequestTrace } from './attempt-trace.js';
+import { logRequest, persistRequestAttempts } from './request-log.js';
+import { withKeyProxy } from './proxy.js';
+import { getEndpointTimeBudgetMs } from './ttfb-budget.js';
+import { learnOutputCapFromError, learnedOutputCap } from './output-cap.js';
+// Every surface caps failover hops at the same number.
+export const FALLBACK_MAX_RETRIES = 20;
+// ── Model-level failure benching ─────────────────────────────────────────────
+// A model that keeps failing upstream (401/429/5xx/empty-stream/timeout) must
+// not keep being picked by auto-routing: every dead-end attempt wastes seconds
+// of user-visible latency. The per-key cooldown above benches the *key*; this
+// is the model-level counterpart — a sliding window of failures *across keys*
+// that benches the whole MODEL once the streak is convincing. The window counts
+// across keys, so the bench must span keys too: benching only the key that
+// happened to fail last leaves every sibling key serving the same sick model
+// until each one trips its own streak. Recovery stays automatic: the bench is
+// 'heuristic', the only source the cooldown-probe job may clear early, and it
+// re-validates each benched key once half the bench has been served (the probe
+// exercises the credential, not the model, so a model still sick behind a good
+// key simply re-trips the streak). Client-cancels never reach
+// recordRetryableFailure (the loop returns on isClientAbortError before this
+// bookkeeping), so they don't count.
+export const MODEL_FAILURE_WINDOW_MS = 15 * 60 * 1000; // sliding window: 15 min
+export const MODEL_FAILURE_THRESHOLD = 3; // failures within window
+export const MODEL_FAILURE_COOLDOWN_MS = 10 * 60 * 1000; // bench duration: 10 min
+const modelFailureTimestamps = new Map(); // model_db_id → times
+/** Record one retryable upstream failure for a model. Once the sliding window
+ *  holds ≥ MODEL_FAILURE_THRESHOLD failures, bench the model on EVERY key that
+ *  can route to it so the model sinks out of routing until upstream heals, then
+ *  reset the counter (one bench per streak). */
+function noteModelFailure(route, now) {
+    const window = (modelFailureTimestamps.get(route.modelDbId) ?? [])
+        .filter(t => now - t < MODEL_FAILURE_WINDOW_MS);
+    window.push(now);
+    modelFailureTimestamps.set(route.modelDbId, window);
+    if (window.length < MODEL_FAILURE_THRESHOLD)
+        return;
+    // Fall back to the failing key alone when the model's key set can't be read
+    // (model row gone, DB unavailable) — a narrower bench beats none.
+    const keyIds = routableKeyIdsForModel(route.modelDbId);
+    const targets = keyIds.length > 0 ? keyIds : [route.keyId];
+    const benchUntil = now + MODEL_FAILURE_COOLDOWN_MS;
+    const active = getActiveCooldownsForKeys(targets, now);
+    for (const keyId of targets) {
+        // Never SHORTEN an existing bench: a provider-stated reset or an escalated
+        // ladder step outlasting this window knows more than this heuristic does,
+        // and overwriting it would also drop its non-probeable provenance.
+        const current = active.get(keyId)
+            ?.find(c => c.platform === route.platform && c.modelId === route.modelId);
+        if (current && current.expiresAtMs >= benchUntil)
+            continue;
+        setCooldown(route.platform, route.modelId, keyId, MODEL_FAILURE_COOLDOWN_MS, 'heuristic');
+    }
+    modelFailureTimestamps.delete(route.modelDbId);
+}
+/** A served request is the strongest counter-evidence: clear the failure
+ *  window so a recovered model is not benched for a stale streak. */
+function clearModelFailure(route) {
+    modelFailureTimestamps.delete(route.modelDbId);
+}
+/** Bench one key on every enabled model of its platform it can route to (the
+ *  route's own model is benched by the caller). For KEY-level verdicts only —
+ *  an account suspension, an empty balance — never for a model-level one.
+ *  Also rules the key out on those models for the rest of THIS request via
+ *  state.skipKeys, so the loop's next hop cannot land on the same account.
+ *  Never throws: an unreadable catalog leaves the narrower per-route bench in
+ *  place.
+ */
+function benchKeyAcrossPlatform(route, durationMs, source, state, reason) {
+    let modelIds = [];
+    try {
+        modelIds = getDb().prepare('SELECT model_id FROM models WHERE platform = ? AND enabled = 1 AND (key_id IS NULL OR key_id = ?)').all(route.platform, route.keyId).map(r => r.model_id);
+    }
+    catch (dbErr) {
+        console.warn(`[FallbackLoop] could not list ${route.platform} models to bench key ${route.keyId}: ${dbErr?.message ?? dbErr}`);
+        return;
+    }
+    let benched = 0;
+    for (const modelId of modelIds) {
+        if (modelId === route.modelId)
+            continue;
+        setCooldown(route.platform, modelId, route.keyId, durationMs, source);
+        state.skipKeys.add(`${route.platform}:${modelId}:${route.keyId}`);
+        benched++;
+    }
+    console.warn(`[FallbackLoop] ${route.platform} key ${route.keyId} ${reason}; benched it on ${benched + 1} model(s) for ${Math.round(durationMs / 60_000)}min`);
+}
+// ── Wall-clock retry budget ──────────────────────────────────────────────────
+// Serial failover has no time bound of its own: the observed worst case was a
+// 38.8s TTFB over 11 attempts, and the theoretical worst is maxRetries x the
+// per-attempt HTTP timeout. The budget is checked before STARTING each retry
+// and, when abortInFlight is available, while waiting for its first byte.
+// Successful endpoint TTFB history can widen the configured base budget.
+// The first attempt always runs, and so does the FIRST retry: when
+// attempt 0 alone consumes the whole budget (a slow-failing model), refusing
+// attempt 1 would make failover structurally impossible for exactly the
+// requests that need it (#751). The budget stops attempts >= 2 only.
+// 0 disables the budget entirely.
+// Precedence mirrors the response cache: the settings-table value wins when
+// present (runtime-tunable), then the env var, then the default.
+export const DEFAULT_FALLBACK_TIME_BUDGET_MS = 45_000;
+// Share of the wall-clock budget an aborted attempt must have been silent for
+// before the hedge abort counts as provider health rather than bad luck. An
+// attempt late in a ladder inherits only the scraps of the budget its
+// predecessors left, and killing it proves nothing about that route; one that
+// owned most of the window and still sent no first byte is stalled. Kept a
+// fraction rather than a constant because the budget IS the declared patience
+// — raising FALLBACK_TIME_BUDGET_MS should raise the evidence bar with it.
+export const HEDGE_BENCH_MIN_SILENT_FRACTION = 0.75;
+export const FALLBACK_TIME_BUDGET_SETTING = 'fallback_time_budget_ms';
+export function getFallbackTimeBudgetMs() {
+    let stored;
+    try {
+        stored = getSetting(FALLBACK_TIME_BUDGET_SETTING);
+    }
+    catch {
+        stored = undefined; // DB not ready — never throw on the proxy hot path
+    }
+    const candidates = [stored, process.env.FALLBACK_TIME_BUDGET_MS];
+    for (const raw of candidates) {
+        if (raw === undefined || raw.trim() === '')
+            continue;
+        const n = Number(raw);
+        if (Number.isFinite(n) && n >= 0)
+            return n;
+    }
+    return DEFAULT_FALLBACK_TIME_BUDGET_MS;
+}
+/** Total for the next dispatch. Input-only observations still need output space. */
+export function fallbackRoutingTokens(state, estimatedTotal, outputReserve) {
+    return Math.max(estimatedTotal, state.observedTotalTokens ?? 0, state.observedInputTokens == null ? 0 : state.observedInputTokens + Math.max(0, outputReserve));
+}
+export function newFallbackState() {
+    return { skipKeys: new Set(), skipModels: new Set(), skipPlatforms: new Set(), modelNotFoundPlatforms: new Map() };
+}
+// DISTINCT model_not_found hops on one platform, within one request, before the
+// whole platform is ruled out: a healthy catalog never misses three different
+// models in a single failover chain — a stale/broken one does it routinely
+// (NavyAI: 24 models tried in one session, 0 ok, #1218). 3 keeps one fluke
+// removal from condemning a provider.
+export const MODEL_NOT_FOUND_PLATFORM_LIMIT = 3;
+// Milliseconds until the next UTC midnight — when most providers' daily free
+// allocations reset. Floored at one minute so a hit seconds before midnight
+// still records a real bench instead of a no-op.
+export function msUntilNextUtcMidnight(now = Date.now()) {
+    const d = new Date(now);
+    const next = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate() + 1);
+    return Math.max(next - now, 60_000);
+}
+const pacificDate = new Intl.DateTimeFormat('en-US', {
+    timeZone: 'America/Los_Angeles', year: 'numeric', month: 'numeric', day: 'numeric',
+});
+const pacificHour = new Intl.DateTimeFormat('en-US', {
+    timeZone: 'America/Los_Angeles', hour: 'numeric', hourCycle: 'h23',
+});
+/** Gemini daily quotas reset at midnight Pacific, including DST transitions.
+ * https://ai.google.dev/gemini-api/docs/rate-limits */
+export function msUntilNextPacificMidnight(now = Date.now()) {
+    const parts = pacificDate.formatToParts(now);
+    const part = (name) => Number(parts.find(p => p.type === name).value);
+    // 08:00 UTC on the next Pacific date is either midnight (PST) or 01:00
+    // (PDT). Read the offset on that date, not now: DST days have 23/25 hours.
+    const candidate = Date.UTC(part('year'), part('month') - 1, part('day') + 1, 8);
+    const midnight = candidate - Number(pacificHour.format(candidate)) * 3_600_000;
+    return Math.max(midnight - now, 60_000);
+}
+/**
+ * The one true cooldown-duration selection after a retryable upstream failure:
+ *   - 402 out-of-credits  → a full day (PAYMENT_REQUIRED_COOLDOWN_MS)
+ *   - 403 model-not-on-tier → a full day (MODEL_FORBIDDEN_COOLDOWN_MS), because a
+ *     tier/subscription gate won't clear on the next minute window (issue #256).
+ *     Both honour the operator's cooldown ceiling (#952): relay endpoints emit
+ *     these for transient reasons, and a day is then far too long.
+ *   - a 429 that says the DAILY free allocation is spent (Cloudflare "used up
+ *     your daily free allocation of 10,000 neurons") → benched until the next
+ *     UTC midnight, like the 402 path. The old transient 90s cooldown made the
+ *     router re-pick a dead-for-the-day provider all day long. An explicit
+ *     provider Retry-After wins over the midnight heuristic: rolling daily
+ *     windows (Groq RPD "try again in 7m12s" with a Retry-After header) reset
+ *     well before midnight, and the provider knows its own reset time best.
+ *     Gemini daily violations instead wait until midnight Pacific; a shorter
+ *     RetryInfo can refer to a simultaneous per-minute violation.
+ *   - anything else → the transient/daily escalation ladder, honoring the
+ *     provider's Retry-After as a floor (getCooldownDurationForLimit).
+ */
+export function cooldownForError(route, err) {
+    return cooldownDecisionForError(route, err).durationMs;
+}
+/**
+ * cooldownForError plus the provenance tag the cooldown-probe recovery job
+ * keys off (see CooldownSource in services/ratelimit.ts): 402 → 'credit' and
+ * 403 → 'tier' (a key-validation probe passing proves nothing about credits or
+ * tier, so those are never probed); a daily-quota bench → 'authoritative' (the
+ * expiry is the provider's own reset time, whether from Retry-After or the
+ * UTC-midnight convention — a fact, not a guess); everything else defers to
+ * getCooldownDecisionForLimit, which tags 'authoritative' only when an explicit
+ * Retry-After actually determined the expiry.
+ */
+export function cooldownDecisionForError(route, err) {
+    if (isPaymentRequiredError(err))
+        return { durationMs: getPaymentRequiredCooldownMs(), source: 'credit' };
+    // Before the model-forbidden check: a suspended account is a 403 too, but
+    // it is the KEY that is out, for as long as an empty balance would be — and
+    // under the same operator ceiling (#952).
+    if (isAccountSuspendedError(err))
+        return { durationMs: getPaymentRequiredCooldownMs(), source: 'credit' };
+    if (isModelAccessForbiddenError(err))
+        return { durationMs: getModelForbiddenCooldownMs(), source: 'tier' };
+    if (isDailyQuotaExhaustedError(err)) {
+        if (route.platform === 'google') {
+            // RetryInfo can describe a simultaneous per-minute violation. It cannot
+            // reopen a daily allowance before the Pacific reset (#1339).
+            return { durationMs: Math.max(msUntilNextPacificMidnight(), err?.retryAfterMs ?? 0), source: 'authoritative' };
+        }
+        return { durationMs: err?.retryAfterMs ?? msUntilNextUtcMidnight(), source: 'authoritative' };
+    }
+    return getCooldownDecisionForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err?.retryAfterMs, 
+    // Only a real 429/rate-limit error may feed the null-limits exhaustion
+    // heuristic; a timeout or 5xx is retryable but says nothing about quota,
+    // so it stays on the short transient bench instead of the ladder (#592).
+    { quotaSignal: isRateLimitSignal(err) });
+}
+// ── Truncated-stream streak (#1218) ──────────────────────────────────────────
+// A stream that answers 200, sends partial SSE, then dies without [DONE] /
+// finish_reason ("stream ended unexpectedly") is retried normally — one
+// truncation is common on flaky free gateways. But the same route producing
+// them back-to-back is a sick edge: bench it briefly so the ladder stops
+// re-paying the round trip. Success on the route resets the streak.
+export const TRUNCATION_STREAK_LIMIT = 3;
+// 5 min vs the ordinary 90s transient bench: the streak must add REAL distance
+// over the default or it changes nothing — the escalation ladder already gives
+// a single truncation 90s.
+export const TRUNCATION_BENCH_MS = 5 * 60 * 1000;
+const truncationStreaks = new Map(); // "platform:modelId:keyId"
+export function resetTruncationStreaks() {
+    truncationStreaks.clear();
+}
+// ── Empty-completion streak (issue #751) ─────────────────────────────────────
+// The skipBench exemption below assumes an empty 'length' completion is
+// REQUEST behavior (this turn's max_tokens spent on hidden reasoning). A
+// model+key that produces them consecutively is broken, not unlucky — with an
+// unconditional exemption it stayed penalty-free forever while failing every
+// request routed to it. At the streak limit the exemption lifts and the
+// failure takes the normal cooldown/penalty/limit-learning path (and counts
+// toward the breaker). Format violations (skipModelForRequest) never accrue:
+// they are model behavior for THIS request's response_format, not key health.
+export const EMPTY_COMPLETION_STREAK_LIMIT = 3;
+const emptyCompletionStreaks = new Map(); // "platform:modelId:keyId"
+export function resetEmptyCompletionStreaks() {
+    emptyCompletionStreaks.clear();
+}
+/** Drop every model's sliding failure window. Module state outlives a test
+ *  case, so without this a case inherits the previous one's failure counts and
+ *  trips the threshold early — clearing `rate_limit_cooldowns` alone does not
+ *  reach it. */
+export function resetModelFailureWindows() {
+    modelFailureTimestamps.clear();
+}
+/** Operator clear (#952): drop every model's failure window and report how many
+ *  models were mid-streak. The benches those streaks produced are ordinary
+ *  cooldown rows and go with clearAllCooldowns. */
+export function clearModelFailureWindows() {
+    const count = modelFailureTimestamps.size;
+    modelFailureTimestamps.clear();
+    return count;
+}
+// Advance (or break) the streak for this failure and report whether the
+// skipBench exemption still holds. Called exactly once per retryable failure,
+// from recordRetryableFailure; the loop reuses its returned decision for the
+// breaker so the two consumers can never disagree.
+function consumeSkipBenchExemption(route, err) {
+    const key = `${route.platform}:${route.modelId}:${route.keyId}`;
+    if (err?.skipBench !== true) {
+        // A normally-penalized failure breaks the streak: the cooldown ladder is
+        // already handling whatever is wrong with this model+key.
+        emptyCompletionStreaks.delete(key);
+        return false;
+    }
+    if (err?.skipModelForRequest === true)
+        return true;
+    const streak = (emptyCompletionStreaks.get(key) ?? 0) + 1;
+    emptyCompletionStreaks.set(key, streak);
+    return streak < EMPTY_COMPLETION_STREAK_LIMIT;
+}
+/**
+ * Apply the full per-key failure bookkeeping shared by every surface after a
+ * retryable failure:
+ *   - rule out the WHOLE model for the rest of the request on a 404 (removed
+ *     upstream) or 403 (off this key's tier) — a sibling key would fail it the
+ *     same way (PR #111 / issue #256);
+ *   - bench this model+key via cooldownForError;
+ *   - demote the model in the scorer ONLY when the failure exhausted it — i.e.
+ *     no sibling key can still serve it (#454 gate). skipKeys already contains
+ *     the just-failed key here, preserving #479's "count budget across keys"
+ *     semantics: hasOtherUsableKey excludes both the failed key and skipKeys;
+ *   - learn a provider-reported ceiling (e.g. a Groq 413 "TPM: Limit 30000")
+ *     from the error body so the next pre-check fails over before the 413.
+ *
+ * Reasoning-truncation exemption: an error thrown with `skipBench: true` (a
+ * reasoning model that spent the whole max_tokens budget on hidden reasoning,
+ * finish_reason 'length') still fails over — the key is skipped for THIS
+ * request — but is NOT a provider-health signal, so no cooldown, no model
+ * penalty, and no limit-learning are recorded. Benching those was costing
+ * healthy models a 90s cooldown + a scorer penalty per truncated turn. The
+ * exemption is streak-bounded (#751): from the EMPTY_COMPLETION_STREAK_LIMITth
+ * consecutive empty completion on the same model+key it stops applying, until
+ * a success (or a normally-penalized failure) resets the streak.
+ *
+ * Returns whether the skipBench exemption held for this failure, so the loop
+ * can keep the breaker in lockstep with the bench decision.
+ *
+ * Callers add the just-failed key to skipKeys via this function (do not pre-add).
+ */
+export function recordRetryableFailure(route, err, state, now = Date.now()) {
+    // `skipModelForRequest: true` = the failure is MODEL behavior, not key
+    // state (ignored response_format, JSON truncated at max_tokens): a sibling
+    // key would reproduce it exactly, so rule out the whole model for this
+    // request instead of burning one failover hop per key.
+    // Context-too-large is MODEL-level too: a sibling key serves the same model
+    // with the same context window (and, for Groq-style per-key TPM 413s, the
+    // same tier ceiling), so it would reject the same request identically.
+    if (isModelNotFoundError(err) || isModelAccessForbiddenError(err) || isContextTooLargeError(err) || err?.skipModelForRequest === true) {
+        state.skipModels.add(route.modelDbId);
+    }
+    // A stale CATALOG (not a single dead model) answers model-not-found on every
+    // sibling model in a row: NavyAI served 24 models over one session, 0 ok,
+    // each re-paying a 2.5–10s round trip (#1218). The per-model skip above
+    // can't see the pattern — each miss is a different model. Count DISTINCT
+    // model_not_found hops per platform within one request; from the threshold,
+    // rule the whole platform out for the rest of the request: the catalog (or
+    // the key's access to it) is broken, and the next PROVIDER is the better
+    // hop. Same request-scoped lifetime as skipModels (#111/#256 semantics).
+    // Custom relays are exempt: every relay shares the one platform id 'custom'
+    // (#651), so three misses spread over three different relays would rule out
+    // every healthy relay too. Their misses stay per-model.
+    if (isModelNotFoundError(err) && !route.endpointScope) {
+        const seen = state.modelNotFoundPlatforms.get(route.platform) ?? new Set();
+        seen.add(route.modelDbId);
+        state.modelNotFoundPlatforms.set(route.platform, seen);
+        if (seen.size >= MODEL_NOT_FOUND_PLATFORM_LIMIT) {
+            state.skipPlatforms.add(route.platform);
+        }
+    }
+    // A model-level 404/410 that says the model is GONE (not merely missing right
+    // now) outlives this request: persist it once the evidence is strong enough,
+    // or the retired model burns a fallback slot on every request forever (#634).
+    // The trace object identifies the request, so one request's failover across
+    // sibling keys counts as the single observation it is.
+    noteModelRetirementSignal(route, err, getRequestTrace());
+    // Provider-reported request size (#507): if the just-failed attempt was a
+    // size-related rejection that names the request's real token count, latch it
+    // onto the request state. The route() closure on the next attempt picks it
+    // up and feeds it as the routing estimate, so the existing tpm_limit /
+    // context_window gates in router.ts skip low-ceiling models on retry
+    // instead of letting them fire and 413 again. Only fires on a size-class
+    // error — the parser is conservative and returns null for opaque bodies
+    // (every non-supported platform, the bare "Request Entity Too Large"
+    // Groq shape, and the github limit-only case), so behavior is unchanged
+    // when there's nothing to latch. Max wins so a smaller later reading of
+    // the same request can't drop the estimate.
+    if (isContextTooLargeError(err) || isProviderBadRequestError(err)) {
+        const reported = parseProviderReportedSize(route.platform, err?.message);
+        if (reported != null) {
+            const field = reported.kind === 'input' ? 'observedInputTokens' : 'observedTotalTokens';
+            state[field] = Math.max(state[field] ?? 0, reported.tokens);
+        }
+    }
+    state.skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
+    // #788: provider-level failures (5xx / timeout / transport / degraded) mean
+    // the PROVIDER is sick, not this key — every key AND every model of that
+    // platform would fail identically. Rule out the whole platform for this
+    // request so the loop moves to the NEXT provider instead of burning one
+    // failover hop per key. Key-scoped failures (auth/quota) stay on the
+    // single-key path, and the per-key cooldown below is still the only thing
+    // that outlives the request.
+    if (isProviderLevelError(err)) {
+        state.skipPlatforms.add(route.platform);
+    }
+    // Too big for this model is a fact about the REQUEST, not the model's
+    // health: it still serves every smaller request. Skip it for this request
+    // (above) and learn the reported ceiling so the router sizes it out next
+    // time, but no cooldown and no penalty. Benching it let one oversized agent
+    // turn (Claude Code ships ~16k tokens of tool schemas) sink every small-TPM
+    // model it touched, so later ordinary requests found the pool rate limited.
+    if (isContextTooLargeError(err)) {
+        learnLimitFromError(route.modelDbId, err);
+        return false;
+    }
+    // A max_tokens above this model's output ceiling (Claude Code asks for
+    // 128000; Groq gpt-oss and Ollama Cloud's Nemotron stop at 65536) is the
+    // request's shape, not the model's health and not missing tool support, so
+    // it stays off the cooldown, penalty and tool-rejection books. A ceiling
+    // learned just now makes the same route usable again at once (the next
+    // attempt is clamped to it); one that was already applied and still got
+    // rejected rules the model out for this request.
+    const capBefore = learnedOutputCap(route.platform, route.modelId);
+    const ceiling = learnOutputCapFromError(route, err);
+    if (ceiling != null) {
+        if (capBefore == null || ceiling < capBefore) {
+            state.skipKeys.delete(`${route.platform}:${route.modelId}:${route.keyId}`);
+        }
+        else {
+            state.skipModels.add(route.modelDbId);
+        }
+        return false;
+    }
+    if (consumeSkipBenchExemption(route, err))
+        return true;
+    const decision = cooldownDecisionForError(route, err);
+    setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
+    // A truncated stream (200 + partial SSE, no [DONE]/finish_reason) on the
+    // SAME route repeatedly is a sick edge, not bad luck — but a single one is
+    // common enough on flaky free gateways that benching on first sight would
+    // over-fire. Streak-bounded like empty completions (#751): N truncations in
+    // a row on this platform+model+key bench the route for one cooldown window;
+    // a success resets the streak (recordUpstreamSuccess).
+    if (isStreamTruncatedError(err)) {
+        const tk = `${route.platform}:${route.modelId}:${route.keyId}`;
+        const streak = (truncationStreaks.get(tk) ?? 0) + 1;
+        if (streak >= TRUNCATION_STREAK_LIMIT) {
+            truncationStreaks.delete(tk);
+            setCooldown(route.platform, route.modelId, route.keyId, TRUNCATION_BENCH_MS, 'heuristic');
+            console.warn(`[FallbackLoop] ${route.platform} ${route.modelId} key ${route.keyId}: ${streak} consecutive truncated streams — benching the route for ${Math.round(TRUNCATION_BENCH_MS / 1000)}s`);
+        }
+        else {
+            truncationStreaks.set(tk, streak);
+        }
+    }
+    else {
+        // Any other failure class on this route breaks the truncation streak: the
+        // cooldown ladder is already handling whatever that is.
+        truncationStreaks.delete(`${route.platform}:${route.modelId}:${route.keyId}`);
+    }
+    // A 400 "bad request" answer to a request that carries `tools` is how a
+    // model that does not really support tool calling shows up (#1230). The 90s
+    // transient cooldown above forgets it, so remember it per model+endpoint;
+    // after a few distinct requests the router tries that model last for tool
+    // requests (lib/tool-capability.ts). Only the final classification counts:
+    // context-too-large, model-not-found and degraded 400s are other problems.
+    if (state.wantsTools === true && classifyAttemptError(err) === 'provider_bad_request') {
+        const tk = toolCapabilityKey(route.platform, route.modelId, route.endpointScope);
+        const seen = (state.toolRejects ??= new Set());
+        if (!seen.has(tk)) {
+            seen.add(tk);
+            noteToolRejection(tk, now);
+        }
+    }
+    // A suspended ACCOUNT fails every model behind the key identically. The
+    // per-route cooldown above benches only the model that happened to be
+    // tried, so the next request picks the platform's next model and pays the
+    // same round trip again — on a platform with a large catalog that is the
+    // whole failover budget, every request, until each model has tripped
+    // separately. Bench the key across the platform's catalog in one go, and
+    // rule the platform out for the rest of this request: a sibling key would
+    // be a different account, but with the budget already spent on this one
+    // the next PROVIDER is the better hop.
+    if (isAccountSuspendedError(err)) {
+        state.skipPlatforms.add(route.platform);
+        benchKeyAcrossPlatform(route, decision.durationMs, decision.source, state, 'reports the account suspended');
+    }
+    else if (isPaymentRequiredError(err)) {
+        // A 402 is the ACCOUNT's balance, not one model's (#1239: one broke
+        // HuggingFace key answered three different models with 402 in a single
+        // request, three hops spent rediscovering the same empty wallet). Bench
+        // the key on every model of the platform for the credit window. Unlike a
+        // suspension the platform itself stays in play: a sibling key is a
+        // different wallet and may well have credits, so only THIS key is out.
+        benchKeyAcrossPlatform(route, decision.durationMs, decision.source, state, 'is out of credits (402)');
+    }
+    // Model-level failure benching: a model failing across keys (or repeatedly on
+    // one key) must sink out of routing instead of being re-picked every request.
+    noteModelFailure(route, now);
+    // Model-level penalty only when no sibling key can still serve (#454).
+    if (!hasOtherUsableKey(route.modelDbId, route.keyId, state.skipKeys)) {
+        // Hard limit signals (429/402) carry the heavier demotion; ordinary
+        // upstream failures (5xx/timeout/empty stream) get a lighter one so the
+        // model sinks gradually instead of being banished on a single blip.
+        if (isRateLimitSignal(err)) {
+            recordRateLimitHit(route.modelDbId);
+        }
+        else {
+            recordModelFailure(route.modelDbId);
+        }
+    }
+    learnLimitFromError(route.modelDbId, err);
+    return false;
+}
+// ── Upstream 401 handling (key-fatal, not request-fatal) ─────────────────────
+// A 401 means THIS key is bad, not this model or this request. The old behavior
+// (non-retryable → 502) stranded the provider's healthy sibling key and every
+// other provider in the chain, and left the bad key in rotation failing traffic
+// until the next 5-minute health cycle. Now the loop skips the key, benches the
+// model+key long enough to cover the health cycle, and kicks an immediate
+// targeted revalidation so a confirmed-bad key flips to status 'invalid' (and
+// out of routing) within seconds instead of minutes.
+export const AUTH_FAILURE_COOLDOWN_MS = 5 * 60 * 1000;
+// Dedupe window for the fire-and-forget revalidation: many concurrent requests
+// hitting the same bad key must not stampede the provider's validate endpoint.
+const REVALIDATION_DEDUPE_MS = 30_000;
+const lastRevalidation = new Map();
+function triggerKeyRevalidation(platform, keyId) {
+    const now = Date.now();
+    const last = lastRevalidation.get(keyId) ?? 0;
+    if (now - last < REVALIDATION_DEDUPE_MS)
+        return;
+    lastRevalidation.set(keyId, now);
+    console.warn(`[FallbackLoop] Upstream 401 from ${platform} key ${keyId}; revalidating it now instead of waiting for the health cycle`);
+    void checkKeyHealth(keyId).catch(err => {
+        console.error(`[FallbackLoop] Immediate revalidation of key ${keyId} failed:`, err?.message);
+    });
+}
+/**
+ * Bookkeeping for an auth-fatal (401 / invalid key) attempt: skip the key for
+ * this request, bench the model+key for the health-cycle window, and start an
+ * immediate revalidation. Deliberately NO model penalty and NO limit-learning —
+ * a bad key says nothing about the model's health.
+ */
+export function recordAuthFailure(route, state) {
+    state.skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
+    setCooldown(route.platform, route.modelId, route.keyId, AUTH_FAILURE_COOLDOWN_MS);
+    triggerKeyRevalidation(route.platform, route.keyId);
+}
+/**
+ * The success-side accounting every surface runs after a completed attempt:
+ * count the request + its tokens against the model+key's rate-limit windows and
+ * clear the model's 429 penalty. `rateLimitTokens` is whatever the surface metered
+ * (the provider's usage.total_tokens for non-stream, an estimate for stream).
+ */
+export function recordUpstreamSuccess(route, rateLimitTokens, state) {
+    // A tool-carrying request that was served proves two things (#1230): this
+    // model does handle tools, and the request itself was well-formed, so the
+    // models that answered it with a 400 earlier in this chain are the ones at
+    // fault. Defer those for tool requests without waiting for more evidence.
+    if (state?.wantsTools === true) {
+        const winner = toolCapabilityKey(route.platform, route.modelId, route.endpointScope);
+        clearToolRejections(winner);
+        for (const tk of state.toolRejects ?? [])
+            if (tk !== winner)
+                benchForTools(tk);
+    }
+    recordRequest(route.platform, route.modelId, route.keyId);
+    recordTokens(route.platform, route.modelId, route.keyId, rateLimitTokens);
+    recordSuccess(route.modelDbId);
+    // A served request proves the model+key can complete: the empty-completion
+    // streak (#751) starts over.
+    emptyCompletionStreaks.delete(`${route.platform}:${route.modelId}:${route.keyId}`);
+    // A served request is the strongest evidence the route's edge is alive:
+    // the truncated-stream streak (#1218) starts over too.
+    truncationStreaks.delete(`${route.platform}:${route.modelId}:${route.keyId}`);
+    // A served request is the strongest possible evidence the model works, so
+    // clear any model-level failure streak that could bench it later.
+    clearModelFailure(route);
+    // A served request is the strongest possible evidence the key works, so clear
+    // any stale 'error' status left by an earlier transport blip instead of waiting
+    // for the next health pass to make the key routable again.
+    markKeyHealthyFromRequest(route.keyId);
+}
+export function classifyAttemptError(err) {
+    if (isKeyAuthError(err))
+        return 'auth';
+    if (isPaymentRequiredError(err))
+        return 'out_of_credits';
+    if (isDailyQuotaExhaustedError(err))
+        return 'daily_quota_exhausted';
+    if (isModelNotFoundError(err))
+        return 'model_not_found';
+    if (isModelAccessForbiddenError(err))
+        return 'forbidden';
+    // Before the bad-request check: OpenAI-compat context errors arrive as
+    // "API error 400: This model's maximum context length is …", which the
+    // generic provider_bad_request rule would otherwise swallow.
+    if (isContextTooLargeError(err))
+        return 'context_too_large';
+    // A DEGRADED-function 400 (NVIDIA NIM, #522) is provider health, not request
+    // shape — keep it out of provider_bad_request so the trail reads honestly.
+    if (isProviderDegradedError(err))
+        return 'upstream_error';
+    if (isProviderBadRequestError(err))
+        return 'provider_bad_request';
+    const msg = (err?.message ?? '').toLowerCase();
+    if (msg.includes('empty completion'))
+        return 'empty_completion';
+    if (msg.includes('ignored response_format') || msg.includes('truncated json'))
+        return 'format_ignored';
+    // Before the generic classes so the trail names the real cause rather than
+    // booking a schema violation as a bare 'error'.
+    if (msg.includes('invalid tool arguments'))
+        return 'invalid_tool_arguments';
+    if (isTimeoutErrorText(msg))
+        return 'timeout';
+    if (msg.includes('429') || msg.includes('rate limit') || msg.includes('too many requests') || msg.includes('quota'))
+        return 'rate_limited';
+    const status = typeof err?.status === 'number' ? err.status : 0;
+    if (status >= 500 || msg.includes('500') || msg.includes('502') || msg.includes('503') || msg.includes('unavailable') || msg.includes('internal server error'))
+        return 'upstream_error';
+    return 'error';
+}
+const TRAIL_MAX_SHOWN = 10;
+const TRAIL_HEADER_MAX_LENGTH = 1024;
+// ── Detailed failover trace header (opt-in) ──────────────────────────────────
+// X-Fallback-Trail already tells a caller WHICH hops burned and WHY, but not
+// how long they cost. That is the part an agent cannot reconstruct: a request
+// answered in 40s reads identically whether one provider stalled for 39s or
+// four failed fast. The per-hop timings and the redacted provider message are
+// already collected for the request_attempts table; this exposes them on the
+// response so the caller sees them without a dashboard round trip.
+//
+// Off by default. It widens what an already-loopback-ish surface reveals —
+// hop timings plus provider error text — so it is opt-in like the discovery
+// aliases, not something a default install starts emitting. Precedence matches
+// the failover budget above: settings-table value, then env var, then off.
+export const EXPOSE_FALLBACK_DETAIL_SETTING = 'expose_fallback_detail_header';
+// Ten hops of "platform/model keyN=class t=…+…ms msg=…" with a capped message
+// each. Kept well under the 8 KB total header budget that proxies commonly
+// enforce, since this rides alongside X-Fallback-Trail's own 1 KB.
+const DETAIL_HEADER_MAX_LENGTH = 2048;
+// summarizeAttemptError caps at 200; the header wants a tighter budget so ten
+// hops still fit. The full text remains in request_attempts either way.
+const DETAIL_SUMMARY_MAX_LENGTH = 120;
+export function isFallbackDetailHeaderEnabled() {
+    let stored;
+    try {
+        stored = getSetting(EXPOSE_FALLBACK_DETAIL_SETTING);
+    }
+    catch {
+        stored = undefined; // DB not ready — never throw on the proxy hot path
+    }
+    for (const raw of [stored, process.env.FALLBACK_DETAIL_HEADER]) {
+        if (raw === undefined || raw.trim() === '')
+            continue;
+        const value = raw.trim().toLowerCase();
+        return value === '1' || value === 'true';
+    }
+    return false;
+}
+/**
+ * One `platform/model keyN=outcome t=<start>+<duration>ms msg=<summary>` segment
+ * per hop, `; `-joined — the same leading shape as X-Fallback-Trail so the two
+ * headers line up when read together.
+ *
+ * The summary is the already-redacted `errorSummary`, never `err.message`. Its
+ * semicolons become commas because `; ` is the record separator; nothing else
+ * is escaped, which keeps the value readable, and `safeHeaderValue` handles any
+ * non-ASCII on the way out.
+ */
+export function formatAttemptDetail(records) {
+    const shown = records.slice(0, TRAIL_MAX_SHOWN).map(r => {
+        const parts = [
+            `${r.platform}/${r.modelId}`,
+            `key${r.keyOrdinal}=${r.outcome}`,
+            `t=${r.startOffsetMs}+${r.durationMs}ms`,
+        ];
+        if (r.errorSummary) {
+            parts.push(`msg=${r.errorSummary.slice(0, DETAIL_SUMMARY_MAX_LENGTH).replace(/;/g, ',')}`);
+        }
+        return parts.join(' ');
+    });
+    const extra = records.length - shown.length;
+    return shown.join('; ') + (extra > 0 ? `; +${extra} more` : '');
+}
+export function formatAttemptTrail(attempts) {
+    const shown = attempts
+        .slice(0, TRAIL_MAX_SHOWN)
+        .map(a => `${a.platform}/${a.modelId} key${a.keyOrdinal}: ${a.errorClass}`);
+    const extra = attempts.length - shown.length;
+    return shown.join('; ') + (extra > 0 ? `; +${extra} more` : '');
+}
+/**
+ * Set the failover diagnostics headers every surface stamps on its responses:
+ * X-Fallback-Attempts (how many hops failed before this response) and
+ * X-Fallback-Trail (what each hop was and why it failed). Until now the trail
+ * only reached clients inside exhaustion error MESSAGES — a request that
+ * eventually succeeded gave no hint that it burned five hops first, which is
+ * exactly the case an operator wants to notice. Values go through
+ * safeHeaderValue so a non-ASCII or control-laden model id can neither inject
+ * header lines nor make Node reject the response outright (#619).
+ *
+ * When EXPOSE_FALLBACK_DETAIL_SETTING is on, X-Fallback-Detail joins them with
+ * per-hop timings and the redacted provider message. Every caller runs inside
+ * the request's AsyncLocalStorage scope, so the trace is readable here without
+ * threading it through all five surfaces.
+ *
+ * Note what the detail header can and cannot contain: at flush time the trace
+ * holds exactly the hops that already FAILED, each with final timings. The hop
+ * currently being served is recorded only after dispatch returns — after
+ * res.json(), or after the whole stream has finished — so its duration is not
+ * knowable while headers are still open, on either path.
+ */
+export function setFallbackHeaders(res, failedAttempts, trail) {
+    if (failedAttempts > 0)
+        res.setHeader('X-Fallback-Attempts', String(failedAttempts));
+    if (trail && trail.length > 0) {
+        const value = trail
+            .slice(0, TRAIL_MAX_SHOWN)
+            .map(a => `${a.platform}/${a.modelId} key${a.keyOrdinal}=${a.errorClass}`)
+            .join('; ') + (trail.length > TRAIL_MAX_SHOWN ? `; +${trail.length - TRAIL_MAX_SHOWN} more` : '');
+        // Ten hops of "platform/model keyN=class" outgrow the default cap, so the
+        // trail gets its own budget.
+        res.setHeader('X-Fallback-Trail', safeHeaderValue(value, TRAIL_HEADER_MAX_LENGTH));
+    }
+    // Checked before the setting so the overwhelmingly common no-failover request
+    // never pays for a settings read.
+    const records = getRequestTrace()?.records;
+    if (records && records.length > 0 && isFallbackDetailHeaderEnabled()) {
+        res.setHeader('X-Fallback-Detail', safeHeaderValue(formatAttemptDetail(records), DETAIL_HEADER_MAX_LENGTH));
+    }
+}
+/**
+ * The OpenAI-compatible `error` object for an exhaustion body — shared by every
+ * OpenAI-shaped surface so the wire shape (message/type/code/retryAtMs) cannot
+ * drift between them.
+ */
+export function exhaustionErrorPayload(body) {
+    const payload = {
+        message: body.message,
+        type: body.type,
+    };
+    if (body.code)
+        payload.code = body.code;
+    if (body.retryAtMs != null)
+        payload.retryAtMs = body.retryAtMs;
+    return payload;
+}
+/**
+ * Stamp the standard retry headers for an exhaustion body: a Retry-After of
+ * ceil((retryAtMs - now) / 1000) seconds when the body carries a concrete
+ * retry time. Callers must only invoke this before headers are flushed (i.e.
+ * never on a committed SSE stream).
+ */
+export function setExhaustionHeaders(res, body, now = Date.now()) {
+    if (body.retryAtMs == null)
+        return;
+    res.setHeader('Retry-After', String(Math.max(0, Math.ceil((body.retryAtMs - now) / 1000))));
+}
+// Attempt classes that mean "this candidate is unavailable until a KNOWN time"
+// — a rate-limit window, a benched daily allocation, an out-of-credits key
+// (day bench), or a tier-forbidden model (day bench). When EVERY attempt is in
+// this family the pool recovers by itself, so the honest exhaustion is a 429
+// with a concrete retry time. Anything else in the mix (a 5xx, a timeout, a
+// vanished model) means waiting is not a promise, so the mixed case renders a
+// 502 instead.
+const UNAVAILABLE_UNTIL_KNOWN_TIME = new Set([
+    'rate_limited',
+    'daily_quota_exhausted',
+    'out_of_credits',
+    'forbidden',
+]);
+/**
+ * The shared exhaustion response body — the single failure-kind → terminal-
+ * status ladder every surface renders. Aggregated over the per-attempt failure
+ * classes (most-specific first):
+ *   - Every attempt failed auth (401/invalid key) → 502 provider_error saying the
+ *     PROVIDER keys are bad — distinct from a rate-limit exhaustion, and never
+ *     'authentication_error' (which would wrongly blame the CLIENT's key).
+ *   - Every attempt died on a context/prompt-too-large rejection → 413: no
+ *     candidate can fit this request; retrying cannot help, shrinking it can.
+ *   - Every attempt got a model-not-found/gone from its provider → 404: the
+ *     model has been removed upstream everywhere we route it. (No 410 — the
+ *     catalog keeps no removal tombstones, so "verifiably existed before" is
+ *     not determinable here.)
+ *   - A chain that died on a DEGRADED-function 400 (NVIDIA NIM, #522) → 503:
+ *     provider capacity, not a bad request.
+ *   - A request every routed provider rejected as invalid → 400
+ *     invalid_request_error, not a misleading rate-limit exhaustion.
+ *   - Circuit-breaker stop → 503 (the pool looks unhealthy; retry later).
+ *   - Every attempt unavailable-until-known-time (rate limits, benched
+ *     quotas/credits/tiers) → 429 rate_limit_error with `retryAtMs` (soonest
+ *     cooldown expiry) for a matching Retry-After header.
+ *   - Mixed/other upstream failures (5xx, timeouts, transport errors) → 502
+ *     provider_error: the UPSTREAMS failed. Never 500 — that status is
+ *     reserved for our own bugs.
+ * All bodies carry the attempt trail (what was tried, per attempt) and, where
+ * meaningful, the soonest-cooldown-reset hint.
+ */
+export function exhaustedRetryError(lastError, maxRetries, ctx) {
+    const safeLastError = sanitizeProviderErrorMessage(lastError?.message);
+    const attempts = ctx?.attempts ?? [];
+    const trail = attempts.length > 0 ? ` Attempt trail: ${formatAttemptTrail(attempts)}.` : '';
+    const budgetNote = ctx?.timedOut
+        ? ` (stopped early: retry time budget ${Math.round((ctx.budgetMs ?? 0) / 1000)}s exceeded — one failover hop is always allowed, and past that the budget stops starting further retries and cancels an attempt still waiting on its first byte; raise FALLBACK_TIME_BUDGET_MS or the fallback_time_budget_ms setting to allow a longer failover chain)`
+        : '';
+    const everyAttempt = (cls) => attempts.length > 0 && attempts.every(a => (cls instanceof Set ? cls.has(a.errorClass) : a.errorClass === cls));
+    if (everyAttempt('auth')) {
+        return {
+            kind: 'auth',
+            status: 502,
+            type: 'provider_error',
+            code: 'provider_authentication_failed',
+            message: `All ${attempts.length} attempted provider key(s) failed authentication${budgetNote}. ` +
+                'The configured upstream API key(s) look invalid or expired; they are being revalidated now and will be marked invalid automatically. ' +
+                `Check the provider keys in the dashboard.${trail} Last error: ${safeLastError}`,
+        };
+    }
+    // Aggregate diagnoses before the lastError-shape ones below: when EVERY
+    // attempt failed the same way, that unanimity is stronger evidence than the
+    // shape of whichever error happened to come last.
+    if (everyAttempt('context_too_large')) {
+        return {
+            kind: 'context_too_large',
+            status: 413,
+            type: 'invalid_request_error',
+            code: 'context_length_exceeded',
+            message: `The request is too large for every routed candidate: all ${attempts.length} attempt(s) were rejected as ` +
+                `over the model's context/size limit${budgetNote}. Retrying will not help — reduce the prompt/history size or ` +
+                `enable a larger-context model.${trail} Last error: ${safeLastError}`,
+        };
+    }
+    if (everyAttempt('model_not_found')) {
+        return {
+            kind: 'model_not_found',
+            status: 404,
+            type: 'invalid_request_error',
+            code: 'model_not_found',
+            message: `Every routed provider reports the model as not found or removed upstream (${attempts.length} attempt(s))` +
+                `${budgetNote}. The catalog entry looks stale; pick another model or call /v1/models for the available list.` +
+                `${trail} Last error: ${safeLastError}`,
+        };
+    }
+    // A chain that died on a DEGRADED-function 400 (NVIDIA NIM, #522) is a
+    // provider-capacity outage, not a bad request: render 503 so clients retry
+    // later instead of "fixing" a request that was never broken.
+    if (isProviderDegradedError(lastError)) {
+        return {
+            kind: 'unavailable',
+            status: 503,
+            type: 'service_unavailable',
+            code: 'provider_degraded',
+            message: `The routed provider reported the model's hosted deployment as temporarily degraded${budgetNote}. ` +
+                `This is a provider-side condition; retry later or route to another provider.${trail} Last error: ${safeLastError}`,
+        };
+    }
+    // 400 only when the trail agrees with the last error: EVERY attempt was a
+    // provider-side request rejection (or the legacy no-trail shape). A mixed
+    // trail — stale models, an empty wallet, a rate limit, and one bad-request
+    // hop that happened to come last — is not evidence the caller's request is
+    // invalid, and rendering it as one sent users debugging their own payload
+    // (#1239: 5× provider_bad_request + 3× out_of_credits read as "All routed
+    // providers rejected the request as invalid"). Those fall through to the
+    // mixed 502 below, whose message names the class breakdown.
+    if (isProviderBadRequestError(lastError) && (attempts.length === 0 || everyAttempt('provider_bad_request'))) {
+        return {
+            kind: 'bad_request',
+            status: 400,
+            type: 'invalid_request_error',
+            code: 'provider_rejected_request',
+            message: `All routed providers rejected the request as invalid${budgetNote}.${trail} Last error: ${safeLastError}`,
+        };
+    }
+    // Circuit-breaker guardrail stop. Checked after the aggregate and bad-request
+    // diagnoses, which are more specific about WHY the pool is failing.
+    if (ctx?.breakerFails) {
+        const breakerEta = formatResetEta(getSoonestCooldownExpiry());
+        const breakerEtaNote = breakerEta ? ` Soonest cooldown reset ${breakerEta}.` : '';
+        return {
+            kind: 'unavailable',
+            status: 503,
+            type: 'service_unavailable',
+            code: 'upstream_unhealthy',
+            message: `Failover stopped early by the circuit-breaker guardrail: ${ctx.breakerFails} consecutive upstream ` +
+                `failure${ctx.breakerFails === 1 ? '' : 's'} (max_consecutive_upstream_fails). The enabled pool looks ` +
+                `unhealthy right now, so the remaining candidates were skipped instead of burning quota on them.` +
+                `${breakerEtaNote}${trail} Last error: ${safeLastError}`,
+        };
+    }
+    // 429 only when EVERY candidate is unavailable until a known time (or the
+    // caller gave us no per-attempt classes to aggregate — the legacy shape).
+    if (attempts.length === 0 || everyAttempt(UNAVAILABLE_UNTIL_KNOWN_TIME)) {
+        const attemptCount = attempts.length > 0 ? attempts.length : maxRetries;
+        const scope = attemptCount == null
+            ? 'All models rate-limited'
+            : `All models rate-limited after ${attemptCount} attempt${attemptCount === 1 ? '' : 's'}`;
+        const retryAtMs = getSoonestCooldownExpiry() ?? undefined;
+        const eta = formatResetEta(retryAtMs);
+        const etaNote = eta ? ` Soonest cooldown reset ${eta}.` : '';
+        return {
+            kind: 'rate_limit',
+            status: 429,
+            type: 'rate_limit_error',
+            code: 'rate_limit_exceeded',
+            ...(retryAtMs != null ? { retryAtMs } : {}),
+            message: `${scope}${budgetNote}.${etaNote}${trail} Last error: ${safeLastError}`,
+        };
+    }
+    // Mixed/other upstream failures (5xx, timeouts, transport errors, or a mix of
+    // those with rate limits): the upstream pool failed us, so say 502 — not 429
+    // (which would promise recovery-by-waiting the attempts don't support) and
+    // not 500 (which would blame our own code).
+    //
+    // The breakdown names how many attempts fell in each class so a mixed trail
+    // reads as what it is. When SOME hops were provider-side request rejections
+    // the body must not promise the request was fine either — it says which
+    // providers rejected it and leaves the verdict to the reader.
+    const breakdown = formatClassBreakdown(attempts);
+    const someBadRequest = attempts.some(a => a.errorClass === 'provider_bad_request');
+    const verdict = someBadRequest
+        ? 'Most of these failures are provider-side (stale models, exhausted credits or quotas), but some providers ' +
+            'also rejected the request shape — check the attempt trail for which ones before changing your request.'
+        : 'This is a provider-side failure, not a problem with your request; retry, or check provider status.';
+    return {
+        kind: 'upstream',
+        status: 502,
+        type: 'provider_error',
+        code: 'upstream_failed',
+        message: `All ${attempts.length} routed attempt(s) failed with upstream provider errors${breakdown}${budgetNote}. ` +
+            `${verdict}${trail} Last error: ${safeLastError}`,
+    };
+}
+/** " (provider_bad_request ×5, out_of_credits ×3)" — the per-class tally of a
+ *  mixed trail, most frequent first; empty when there is nothing to tally. */
+function formatClassBreakdown(attempts) {
+    if (attempts.length === 0)
+        return '';
+    const counts = new Map();
+    for (const a of attempts)
+        counts.set(a.errorClass, (counts.get(a.errorClass) ?? 0) + 1);
+    const parts = [...counts.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .map(([cls, n]) => `${cls} ×${n}`);
+    return ` (${parts.join(', ')})`;
+}
+function classifyRoutingDiagLine(line) {
+    const l = line.toLowerCase();
+    // "< estimated" first: the tpm_limit-too-small line also contains 'tpm',
+    // which would otherwise misread as a transient window.
+    if (l.includes('< estimated'))
+        return 'too_large';
+    const monthlyReasons = l.match(/key\(s\) — (.*)$/)?.[1];
+    if (monthlyReasons?.split(', ').every(reason => /^monthly-budget-cap:\d+$/.test(reason)))
+        return 'monthly_budget';
+    if (/no provider registered|no enabled\+healthy key|no usable key|decrypt-error|no-resolved-provider|custom-key-mismatch/.test(l))
+        return 'config';
+    if (/cooldown|rpm|rpd|tpm|tpd|provider-daily-cap|provider-minute-cap|provider-daily-token-cap|key-concurrency|monthly-budget-cap/.test(l))
+        return 'time_bound';
+    return 'other';
+}
+export function routingExhaustionBody(routeErr) {
+    const diag = Array.isArray(routeErr?.diagnostics) ? routeErr.diagnostics : [];
+    const message = routeErr?.message ?? 'No model available to route this request';
+    const classes = diag.map(classifyRoutingDiagLine);
+    if (diag.length === 0 || classes.every(c => c === 'config')) {
+        return {
+            kind: 'unavailable',
+            status: 503,
+            type: 'service_unavailable',
+            code: 'no_providers_configured',
+            message: diag.length === 0
+                ? `No models are enabled/configured to serve this request. Add provider API keys and enable models in the dashboard. ${message}`
+                : `No candidate model has a configured, usable provider key. Add provider API keys in the dashboard. ${message}`,
+        };
+    }
+    if (classes.every(c => c === 'too_large' || c === 'config') && classes.includes('too_large')) {
+        return {
+            kind: 'context_too_large',
+            status: 413,
+            type: 'invalid_request_error',
+            code: 'context_length_exceeded',
+            message: `The request is too large for every available candidate's context/token window. ` +
+                `Reduce the prompt/history size or enable a larger-context model. ${message}`,
+        };
+    }
+    if (classes.includes('monthly_budget') && classes.every(c => c === 'monthly_budget' || c === 'config' || c === 'too_large')) {
+        return {
+            kind: 'rate_limit', status: 429, type: 'rate_limit_error', code: 'quota_exceeded',
+            retryAtMs: Date.parse(soonestBudgetResetAt()),
+            message: `Key budget exhausted for the current period. Raise the cap or wait until the next UTC reset. ${message}`,
+        };
+    }
+    if (classes.some(c => c === 'time_bound') && classes.every(c => c !== 'other')) {
+        const retryAtMs = getSoonestCooldownExpiry() ?? undefined;
+        return {
+            kind: 'rate_limit',
+            status: 429,
+            type: 'rate_limit_error',
+            code: 'rate_limit_exceeded',
+            ...(retryAtMs != null ? { retryAtMs } : {}),
+            message,
+        };
+    }
+    // Capability filters or mixed reasons: keep the router's verdict (429) but
+    // stamp a machine-readable code so clients can tell it from a plain
+    // rate-limit exhaustion.
+    return {
+        kind: 'rate_limit',
+        status: typeof routeErr?.status === 'number' ? routeErr.status : 429,
+        type: 'rate_limit_error',
+        code: 'routing_exhausted',
+        message,
+    };
+}
+/** Log the router's per-candidate disposition for a zero-attempt exhaustion.
+ *  Lives in the loop, not the surfaces: routing gave up before any upstream was
+ *  tried, so nothing else records WHY the pool was empty, and an opaque
+ *  routing_error is indistinguishable from a genuinely dry pool (issue _1).
+ *  Every surface used to be responsible for logging this itself and only
+ *  routes/proxy.ts ever did, so /v1/messages, /v1/responses and the inbound
+ *  wires logged nothing. The loop owns the format; surfaces only name
+ *  themselves via FallbackHooks.logIdentity. */
+function logRoutingExhaustion(routeErr, identity) {
+    const disposition = Array.isArray(routeErr?.diagnostics) ? routeErr.diagnostics : [];
+    const surface = identity?.surface ?? 'unidentified surface';
+    const req = identity?.requestId ? ` req=${identity.requestId.replace(/-/g, '').slice(0, 6)}` : '';
+    const requested = identity?.requestedModel ? ` requested=${identity.requestedModel}` : '';
+    console.warn(`[FallbackLoop] ${surface} routing exhausted (no upstream tried)${req}${requested} ` +
+        `candidates=${disposition.length}` +
+        (disposition.length ? `:\n  ${disposition.join('\n  ')}` : ''));
+}
+/**
+ * The shared attempt loop. Owns iteration, the wall-clock retry budget, the
+ * routeRequest-exhaustion path, the auth/retryable/fatal classification, the
+ * per-failure bookkeeping (recordRetryableFailure / recordAuthFailure), the
+ * attempt trail, and the final exhaustion body. Everything surface-specific —
+ * request translation, stream framing, error-body shape, context handoff, group
+ * routing — lives in the hooks.
+ */
+export async function runFallbackLoop(hooks) {
+    // Durable per-attempt trace (P2 #15): every dispatched attempt — including
+    // the successful final one — is recorded with timing and outcome, then
+    // persisted as one insert batch into `request_attempts`, keyed to the
+    // terminal `requests` row of this ladder. The trace rides AsyncLocalStorage
+    // so logRequest() (called deep inside the surfaces' dispatch closures) can
+    // report back the row ids it writes without any surface changing. The flush
+    // runs after the loop returns — i.e. after the response is finished — so it
+    // never sits on the client's latency path, and it is a no-op when no
+    // `requests` row was written (a pure abort writes its own 'canceled' row,
+    // so its trace persists too).
+    const trace = newRequestTrace();
+    try {
+        await runWithRequestTrace(trace, () => runFallbackLoopAttempts(hooks, trace));
+    }
+    finally {
+        persistRequestAttempts(trace);
+    }
+}
+async function runFallbackLoopAttempts(hooks, trace) {
+    const maxRetries = hooks.maxRetries ?? FALLBACK_MAX_RETRIES;
+    const baseBudgetMs = hooks.timeBudgetMs ?? getFallbackTimeBudgetMs();
+    const startedAt = Date.now();
+    const attempts = hooks.attemptLog ?? [];
+    const keyOrdinals = new Map();
+    const keyOrdinal = (route) => {
+        const key = `${route.platform}:${route.keyId}`;
+        let ord = keyOrdinals.get(key);
+        if (ord === undefined) {
+            ord = keyOrdinals.size + 1;
+            keyOrdinals.set(key, ord);
+        }
+        return ord;
+    };
+    let lastError = null;
+    // Circuit-breaker guardrail (default off): the Nth consecutive upstream
+    // failure aborts the chain with a 503 instead of grinding the remaining
+    // candidates of a pool that is failing across the board. Auth failures count
+    // too — a wall of dead keys is exactly the "stop early" case. Within one
+    // request every recorded failure is consecutive by construction (a success
+    // ends the loop), so this is "max upstream failures per request".
+    const breaker = newBreaker(hooks.breakerLimit);
+    // Shared terminal for the client-gone stops below: these returns skip the
+    // surfaces entirely (nothing to render), so no ok/fail trace would ever come
+    // and the routing strip would keep the last attempt pinned on "processing"
+    // until MAX_AGE_MS. Only surfaces that trace start/next pass a requestId
+    // (Proxy/Responses); anthropic/inbound-chat never do, so they stay silent.
+    const traceCanceled = (platform, model, attempt) => {
+        const requestId = hooks.logIdentity?.requestId;
+        if (!requestId || !platform || !model || attempt == null)
+            return;
+        recordRouteTrace({ event: 'canceled', requestId, attempt, platform, model });
+    };
+    const stopIfBreakerTripped = (route, attempt) => {
+        if (!recordBreakerFailure(breaker))
+            return false;
+        // Tripped after the client already hung up: stop the chain, but there is
+        // no socket to render an exhaustion body to (mirrors the loop-top
+        // clientGone check).
+        if (hooks.clientGone?.()) {
+            console.log(`[FallbackLoop] breaker tripped after client disconnect — stopping without rendering (${attempts.length} failed attempt(s))`);
+            traceCanceled(route?.platform, route?.modelId, attempt);
+            return true;
+        }
+        hooks.onExhausted(exhaustedRetryError(lastError, maxRetries, { attempts, breakerFails: breaker.consecutive }), { attempts, timedOut: false });
+        return true;
+    };
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+        // Client disconnect: nobody is waiting for this chain anymore, so stop
+        // burning provider quota on retries. Nothing to render — the socket is
+        // gone — so return without calling any exhaustion hook.
+        if (attempt > 0 && hooks.clientGone?.()) {
+            console.log(`[FallbackLoop] client disconnected — stopping failover after ${attempts.length} failed attempt(s)`);
+            const prior = attempts[attempts.length - 1];
+            traceCanceled(prior?.platform, prior?.modelId, attempt - 1);
+            return;
+        }
+        let route;
+        try {
+            route = hooks.route(attempt);
+        }
+        catch (routeErr) {
+            // With no candidate to supply an endpoint-specific allowance, retain
+            // the original timeout diagnosis when the base budget is already spent.
+            if (attempt > 1 && baseBudgetMs > 0 && Date.now() - startedAt >= baseBudgetMs) {
+                hooks.onExhausted(exhaustedRetryError(lastError, maxRetries, { attempts, timedOut: true, budgetMs: baseBudgetMs }), { attempts, timedOut: true });
+                return;
+            }
+            const exhaustion = lastError
+                ? exhaustedRetryError(lastError, undefined, { attempts })
+                : routingExhaustionBody(routeErr);
+            // Zero attempts ran: log the router's per-candidate disposition, the only
+            // record of why the pool was empty. With prior attempts the trail in the
+            // exhaustion body already explains the failure.
+            if (!lastError)
+                logRoutingExhaustion(routeErr, hooks.logIdentity);
+            hooks.onRoutingExhausted(lastError, routeErr, exhaustion, { attempts, timedOut: false });
+            return;
+        }
+        let hedgeTimer;
+        try {
+            // Select the endpoint before checking the budget: a slow endpoint may
+            // still have time even after the base budget has expired. Recompute from
+            // the base for every candidate so its allowance cannot leak to a faster
+            // endpoint later in the ladder. The clock still starts at loop entry.
+            const budgetMs = attempt > 1
+                ? getEndpointTimeBudgetMs(baseBudgetMs, route.platform, route.endpointScope)
+                : baseBudgetMs;
+            // Attempt 0 and the first retry remain exempt (#751). Routing reserves a
+            // lease, so even a candidate rejected here must pass through finally.
+            if (attempt > 1 && budgetMs > 0 && Date.now() - startedAt >= budgetMs) {
+                hooks.onExhausted(exhaustedRetryError(lastError, maxRetries, { attempts, timedOut: true, budgetMs }), { attempts, timedOut: true });
+                return;
+            }
+            // Per-attempt trace record: pushed exactly once per dispatched attempt, on
+            // whichever exit the attempt takes. startOffsetMs/durationMs bracket the
+            // dispatch (for a successful stream, durationMs runs until the response
+            // finished — that IS the attempt). When the attempt ended on an error, a
+            // short REDACTED summary of it rides along — the outcome class alone loses
+            // the provider's actual words, which is exactly what the dashboard's
+            // drill-down needs to answer "why did this hop fail".
+            const attemptStartedAt = Date.now();
+            const traceAttempt = (outcome, err) => {
+                trace.records.push({
+                    ordinal: trace.records.length,
+                    platform: route.platform,
+                    modelId: route.modelId,
+                    keyOrdinal: keyOrdinal(route),
+                    keyLabel: route.keyLabel ?? null,
+                    outcome,
+                    startOffsetMs: attemptStartedAt - startedAt,
+                    durationMs: Date.now() - attemptStartedAt,
+                    errorSummary: err != null ? summarizeAttemptError(err?.message) : null,
+                });
+            };
+            // Everything from here to the end of the iteration runs inside a finally that
+            // frees the route's in-flight lease. Every exit — success, auth rotation,
+            // retryable continue, fatal, breaker trip, contract violation — passes through
+            // it, so no path can leak a lease and leave the key's concurrency budget short.
+            // Success accounting happens inside dispatch, so the persisted counters are
+            // already written by the time the provisional lease goes away.
+            // Fallback-v2 hedging: arm a timer for the remaining wall-clock budget so a
+            // STALLED attempt is aborted mid-flight (abortInFlight) instead of only
+            // refusing to start the next retry behind it. Mirrors the loop-top budget
+            // check — attempt 0 and the first retry always run (#751).
+            //
+            // The timer only covers the silent window. dispatch calls ctx.disarmHedge()
+            // as soon as the attempt proves it is alive (first byte / headers flushed),
+            // because past that point cancelling would truncate a healthy response and
+            // buy nothing: a committed stream can no longer fail over anyway. Slow is
+            // not the same as stalled, and only stalled is worth killing.
+            const disarmHedge = () => {
+                if (hedgeTimer) {
+                    clearTimeout(hedgeTimer);
+                    hedgeTimer = undefined;
+                }
+            };
+            if (attempt > 1 && budgetMs > 0 && hooks.abortInFlight) {
+                const remaining = budgetMs - (Date.now() - startedAt);
+                if (remaining > 0) {
+                    hedgeTimer = setTimeout(() => {
+                        console.log(`[FallbackLoop] retry time budget (${budgetMs}ms) expired with no first byte on ${route.platform}/${route.modelId} — aborting stalled upstream`);
+                        hooks.abortInFlight?.();
+                    }, remaining);
+                }
+            }
+            let outcome;
+            try {
+                // #590 (per-key proxy): if THIS key carries its own proxy URL, route the
+                // attempt through it (withKeyProxy → proxyFetch reads the ALS store);
+                // otherwise the global proxy / direct path applies as before. The URL
+                // arrives already decrypted on the route (services/router.ts), so an
+                // attempt costs nothing extra — no query, no decrypt.
+                outcome = await withKeyProxy(route.proxyUrl, () => hooks.dispatch(route, attempt, { disarmHedge }));
+            }
+            catch (err) {
+                // Client-caused abort: the composed fetch signal fired because OUR
+                // client hung up mid-attempt (see newClientAbortError). Not a
+                // provider-health signal — no cooldown, no penalty, no failure stats,
+                // no logFailure 'error' row — and no further attempts either: nobody is
+                // waiting, and there is no socket to render anything to. The finally
+                // below still frees the in-flight lease.
+                if (isClientAbortError(err)) {
+                    const elapsedMs = Date.now() - startedAt;
+                    console.log(`[FallbackLoop] client disconnected mid-attempt on ${route.platform}/${route.modelId} — upstream canceled, stopping without benching`);
+                    // Visibility row (#752): a pure abort used to leave NOTHING in the
+                    // requests table, so the dashboard showed no trace of the request.
+                    // Status 'canceled' is neither success nor failure — every stats/
+                    // scoring query excludes it — and this row is also the parent the
+                    // attempt-trace batch below keys to.
+                    logRequest(route.platform, route.modelId, route.keyId, 'canceled', 0, 0, elapsedMs, `client disconnected after ${(elapsedMs / 1000).toFixed(1)}s; upstream request canceled`);
+                    traceAttempt('client_abort');
+                    traceCanceled(route.platform, route.modelId, attempt);
+                    return;
+                }
+                // Time-budget hedge abort: the wall-clock retry budget expired while this
+                // attempt was still in flight, and the surface aborted the composed fetch
+                // signal (see newHedgeAbortError). The budget is spent either way, so we
+                // render timedOut exhaustion exactly like the loop-top budget check does.
+                //
+                // Whether it also counts as a provider-health signal depends on how much
+                // of the silence belongs to THIS attempt. Sharing a budget that earlier
+                // attempts already mostly spent says nothing about this route — it was
+                // simply last in line, and cancelling it is a scheduling accident. But an
+                // attempt that stayed silent for most of the operator's whole declared
+                // patience on its own produced no first byte over that entire window,
+                // and that is a stalled upstream. Left unbenched it stays at the head of the route
+                // order and re-stalls every subsequent request, burning the full budget
+                // each time and starving the healthy routes queued behind it — observed
+                // in production as one dead free-tier route 45s-ing every request for
+                // hours. Benching costs a short transient cooldown (a timeout is not a
+                // rate-limit signal, so cooldownDecisionForError keeps it light) plus the
+                // gradual model-level sink, both of which decay on their own if the route
+                // recovers.
+                if (isHedgeAbortError(err)) {
+                    const elapsedMs = Date.now() - startedAt;
+                    const attemptElapsedMs = Date.now() - attemptStartedAt;
+                    const ownedWholeBudget = budgetMs > 0
+                        && attemptElapsedMs >= budgetMs * HEDGE_BENCH_MIN_SILENT_FRACTION;
+                    if (ownedWholeBudget) {
+                        hooks.logFailure(route, err, attempt);
+                        recordRetryableFailure(route, err, hooks.state);
+                    }
+                    console.log(`[FallbackLoop] retry time budget expired mid-attempt on ${route.platform}/${route.modelId} after ${(elapsedMs / 1000).toFixed(1)}s — rendering timedOut exhaustion ${ownedWholeBudget ? `and benching the route (silent for the whole ${budgetMs}ms budget)` : 'without benching'}`);
+                    hooks.onExhausted(exhaustedRetryError(lastError, maxRetries, { attempts, timedOut: true, budgetMs }), { attempts, timedOut: true });
+                    traceAttempt('timeout', err);
+                    return;
+                }
+                hooks.logFailure(route, err, attempt);
+                if (isKeyAuthError(err)) {
+                    // KEY-fatal, not request-fatal: rotate past the bad key and revalidate
+                    // it immediately instead of 502-ing while healthy routes sit idle.
+                    recordAuthFailure(route, hooks.state);
+                    attempts.push({ platform: route.platform, modelId: route.modelId, keyOrdinal: keyOrdinal(route), errorClass: 'auth' });
+                    traceAttempt('auth', err);
+                    lastError = err;
+                    if (stopIfBreakerTripped(route, attempt))
+                        return;
+                    continue;
+                }
+                if (isRetryableError(err)) {
+                    const exempt = recordRetryableFailure(route, err, hooks.state);
+                    // An in-band provider error that arrives only after the attempt has
+                    // silently consumed most of the operator's whole budget (#1218 Gap 3:
+                    // nvidia ran 140.7s before surfacing "Service temporarily overloaded",
+                    // leaving scraps for the next hop) behaved like a stall for its entire
+                    // window — the hedge-abort bench would have fired had the abort landed
+                    // first. Give the late error the same treatment the hedge abort gets:
+                    // bench the route so the ladder's next hop keeps a usable budget,
+                    // instead of re-stalling on this route every request. Errors that
+                    // arrive EARLY (the common Groq tool_use_failed shape) stay unbenced —
+                    // a fast verdict costs the ladder nothing.
+                    const errStr = err?.message ?? '';
+                    if (budgetMs > 0
+                        && typeof errStr === 'string' && errStr.includes('in-band provider error')
+                        && Date.now() - attemptStartedAt >= budgetMs * HEDGE_BENCH_MIN_SILENT_FRACTION) {
+                        setCooldown(route.platform, route.modelId, route.keyId, TRUNCATION_BENCH_MS, 'heuristic');
+                        console.warn(`[FallbackLoop] ${route.platform}/${route.modelId}: in-band provider error after ${((Date.now() - attemptStartedAt) / 1000).toFixed(1)}s silent — benching the route ${Math.round(TRUNCATION_BENCH_MS / 1000)}s (#1218 Gap 3)`);
+                    }
+                    const errorClass = classifyAttemptError(err);
+                    attempts.push({ platform: route.platform, modelId: route.modelId, keyOrdinal: keyOrdinal(route), errorClass });
+                    traceAttempt(errorClass, err);
+                    lastError = err;
+                    // A still-exempt skipBench failure (format ignored, hidden-reasoning
+                    // truncation under the streak limit) is model behavior, not provider
+                    // health — recordRetryableFailure skipped the cooldown/penalty for it,
+                    // and it must not count toward the "pool looks unhealthy" breaker
+                    // either: three prose answers to a json_schema request say nothing
+                    // about whether candidate four is up. Once the empty-completion streak
+                    // lifts the exemption (#751), the failure counts everywhere.
+                    if (!exempt && stopIfBreakerTripped(route, attempt))
+                        return;
+                    continue;
+                }
+                traceAttempt(classifyAttemptError(err), err);
+                hooks.onFatal(route, err, attempt);
+                return;
+            }
+            // Enforce the dispatch contract: 'done'/'committed' mean the response is
+            // finished. Anything else (a stray `return` in an adapter) would silently
+            // swallow the request, so fail loudly. Deliberately OUTSIDE the try/catch:
+            // the violation message embeds route.modelId, and a model id containing a
+            // digit run like "2503" would match a retryable-error substring and make
+            // the loop re-dispatch the buggy adapter until exhaustion. Routing straight
+            // to onFatal renders an immediate 502 with no retryability classification.
+            if (outcome !== 'done' && outcome !== 'committed') {
+                const violation = new Error(`fallback-loop dispatch contract violation on ${route.platform}/${route.modelId}: ` +
+                    `expected 'done' or 'committed', got ${JSON.stringify(outcome)}`);
+                console.error('[FallbackLoop]', violation.message);
+                hooks.logFailure(route, violation, attempt);
+                traceAttempt('error', violation);
+                hooks.onFatal(route, violation, attempt);
+                return;
+            }
+            // Terminal attempt: 'done' produced the response; 'committed' means the
+            // stream had already flushed bytes when the attempt ended (mid-stream
+            // error or pre-commit disconnect) — the parent row carries the specifics.
+            traceAttempt(outcome === 'done' ? 'ok' : 'committed');
+            return;
+        }
+        finally {
+            if (hedgeTimer)
+                clearTimeout(hedgeTimer);
+            route.release?.();
+        }
+    }
+    hooks.onExhausted(exhaustedRetryError(lastError, maxRetries, { attempts }), { attempts, timedOut: false });
+}
+//# sourceMappingURL=fallback-loop.js.map