@fudrouter/fsrouter 0.6.64 → 0.6.65

package/open-sse/services/combo/responseValidation.js

@@ -0,0 +1,140 @@
+const MAX_REASON_SNIPPET = 60;
+function snippet(value) {
+  return value.length > MAX_REASON_SNIPPET ? `${value.slice(0, MAX_REASON_SNIPPET)}\u2026` : value;
+}
+function parseJsonPath(path) {
+  const tokens = [];
+  let buf = "";
+  const flush = () => {
+    if (buf) {
+      tokens.push(buf);
+      buf = "";
+    }
+  };
+  for (let i = 0; i < path.length; i++) {
+    const ch = path[i];
+    if (ch === ".") {
+      flush();
+    } else if (ch === "[") {
+      flush();
+      let inner = "";
+      i++;
+      while (i < path.length && path[i] !== "]") {
+        inner += path[i];
+        i++;
+      }
+      const trimmed = inner.trim();
+      const n = Number(trimmed);
+      tokens.push(trimmed !== "" && Number.isInteger(n) ? n : trimmed);
+    } else {
+      buf += ch;
+    }
+  }
+  flush();
+  return tokens;
+}
+function resolveJsonPath(root, path) {
+  let current = root;
+  for (const token of parseJsonPath(path)) {
+    if (current === null || current === void 0) return void 0;
+    if (typeof token === "number") {
+      if (!Array.isArray(current)) return void 0;
+      current = current[token];
+    } else {
+      if (typeof current !== "object") return void 0;
+      current = current[token];
+    }
+  }
+  return current;
+}
+function isNonEmpty(value) {
+  if (value === null || value === void 0) return false;
+  if (typeof value === "string") return value.trim().length > 0;
+  if (Array.isArray(value)) return value.length > 0;
+  if (typeof value === "object") return Object.keys(value).length > 0;
+  return Boolean(value);
+}
+function checkCondition(value, condition, expected) {
+  switch (condition) {
+    case "exists":
+      return value !== void 0 && value !== null;
+    case "nonEmpty":
+      return isNonEmpty(value);
+    case "equals":
+      return value === expected;
+    case "notEquals":
+      return value !== expected;
+  }
+}
+function extractContentText(json) {
+  if (!json || typeof json !== "object") return "";
+  const obj = json;
+  const choices = obj.choices;
+  if (Array.isArray(choices)) {
+    const parts = [];
+    for (const choice of choices) {
+      const message = choice?.message;
+      const content = message?.content;
+      if (typeof content === "string") parts.push(content);
+      else if (Array.isArray(content)) {
+        for (const part of content) {
+          const text = part?.text;
+          if (typeof text === "string") parts.push(text);
+        }
+      }
+    }
+    if (parts.length) return parts.join("");
+  }
+  const output = obj.output;
+  if (Array.isArray(output)) {
+    const parts = [];
+    for (const item of output) {
+      const content = item?.content;
+      if (Array.isArray(content)) {
+        for (const part of content) {
+          const text = part?.text;
+          if (typeof text === "string") parts.push(text);
+        }
+      }
+    }
+    if (parts.length) return parts.join("");
+  }
+  return "";
+}
+function evaluateResponseValidation(json, config) {
+  if (!config || typeof config !== "object") return { valid: true };
+  const content = extractContentText(json);
+  for (const sub of config.forbiddenSubstrings ?? []) {
+    if (typeof sub === "string" && sub.length > 0 && content.includes(sub)) {
+      return { valid: false, reason: `response contains forbidden substring "${snippet(sub)}"` };
+    }
+  }
+  for (const sub of config.requiredSubstrings ?? []) {
+    if (typeof sub === "string" && sub.length > 0 && !content.includes(sub)) {
+      return { valid: false, reason: `response missing required substring "${snippet(sub)}"` };
+    }
+  }
+  if (typeof config.minContentLength === "number" && Number.isFinite(config.minContentLength) && config.minContentLength > 0 && content.trim().length < config.minContentLength) {
+    return {
+      valid: false,
+      reason: `response content shorter than ${config.minContentLength} chars`
+    };
+  }
+  for (const predicate of config.jsonPathPredicates ?? []) {
+    if (!predicate || typeof predicate.path !== "string" || !predicate.path) continue;
+    const resolved = resolveJsonPath(json, predicate.path);
+    if (!checkCondition(resolved, predicate.condition, predicate.value)) {
+      return {
+        valid: false,
+        reason: `jsonpath check failed: "${snippet(predicate.path)}" ${predicate.condition}`
+      };
+    }
+  }
+  return { valid: true };
+}
+export {
+  evaluateResponseValidation,
+  extractContentText,
+  parseJsonPath,
+  resolveJsonPath
+};

package/open-sse/services/combo/runtimeUnits.js

@@ -0,0 +1,195 @@
+import { errorResponse } from "../../utils/error.ts";
+import { recordComboRequest } from "../comboMetrics.ts";
+import { resolveDelayMs } from "./comboPredicates.ts";
+import { validateResponseQuality, releaseQualityClone } from "./validateQuality.ts";
+function isRecord(value) {
+  return !!value && typeof value === "object" && !Array.isArray(value);
+}
+function getCombosList(allCombos) {
+  const combos = Array.isArray(allCombos) ? allCombos : allCombos?.combos || [];
+  return combos.filter(
+    (combo) => isRecord(combo) && typeof combo.name === "string"
+  );
+}
+function findComboByName(allCombos, name) {
+  return getCombosList(allCombos).find((combo) => combo.name === name) || null;
+}
+function unitDisplayName(unit) {
+  return unit.kind === "combo-ref" ? `combo:${unit.comboName}` : unit.modelStr;
+}
+function shuffleUnits(units) {
+  const result = [...units];
+  for (let index = result.length - 1; index > 0; index -= 1) {
+    const swapIndex = Math.floor(Math.random() * (index + 1));
+    [result[index], result[swapIndex]] = [result[swapIndex], result[index]];
+  }
+  return result;
+}
+function selectWeightedUnit(units) {
+  const total = units.reduce((sum, unit) => sum + Math.max(0, Number(unit.weight) || 0), 0);
+  if (total <= 0) return units[0] || null;
+  let draw = Math.random() * total;
+  for (const unit of units) {
+    draw -= Math.max(0, Number(unit.weight) || 0);
+    if (draw <= 0) return unit;
+  }
+  return units[units.length - 1] || null;
+}
+async function executeModelUnit(args) {
+  if (args.isModelAvailable) {
+    const available = await args.isModelAvailable(args.unit.modelStr, args.unit);
+    if (!available) return errorResponse(503, `Model ${args.unit.modelStr} is unavailable`);
+  }
+  return args.handleSingleModel(args.body, args.unit.modelStr, {
+    ...args.unit,
+    effectiveComboStrategy: args.effectiveComboStrategy,
+    failoverBeforeRetry: args.failoverBeforeRetry
+  });
+}
+function buildChildNestingContext(args) {
+  if (args.context.depth >= args.context.maxDepth) {
+    return errorResponse(503, `Max combo nesting depth (${args.context.maxDepth}) exceeded`);
+  }
+  if (args.context.visitedComboNames.includes(args.childComboName)) {
+    return errorResponse(503, `Circular combo reference detected: ${args.childComboName}`);
+  }
+  return {
+    ...args.context,
+    depth: args.context.depth + 1,
+    visitedComboNames: [...args.context.visitedComboNames, args.childComboName]
+  };
+}
+async function executeComboRefUnit(args) {
+  const childCombo = findComboByName(args.allCombos, args.unit.comboName);
+  if (!childCombo) return errorResponse(503, `Nested combo "${args.unit.comboName}" not found`);
+  const childNesting = buildChildNestingContext({
+    context: args.nesting,
+    childComboName: childCombo.name
+  });
+  if (childNesting instanceof Response) return childNesting;
+  return args.runCombo({
+    ...args.baseOptions,
+    body: args.body,
+    combo: childCombo,
+    nesting: childNesting
+  });
+}
+async function executeRuntimeUnit(args) {
+  if (args.unit.kind === "model") {
+    return executeModelUnit({
+      body: args.body,
+      unit: args.unit,
+      handleSingleModel: args.handleSingleModel,
+      isModelAvailable: args.isModelAvailable,
+      failoverBeforeRetry: args.failoverBeforeRetry,
+      effectiveComboStrategy: args.effectiveComboStrategy
+    });
+  }
+  return executeComboRefUnit({
+    body: args.body,
+    unit: args.unit,
+    allCombos: args.allCombos,
+    runCombo: args.runCombo,
+    baseOptions: args.baseOptions,
+    nesting: args.nesting
+  });
+}
+function orderUnitsForStrategy(strategy, units) {
+  if (strategy === "random") return shuffleUnits(units);
+  if (strategy === "weighted") {
+    const selected = selectWeightedUnit(units);
+    if (!selected) return units;
+    return [selected, ...units.filter((unit) => unit.executionKey !== selected.executionKey)];
+  }
+  return units;
+}
+async function executeRuntimeUnitCombo(args) {
+  const maxRetries = Number(args.config.maxRetries ?? 1);
+  const retryDelayMs = resolveDelayMs(args.config.retryDelayMs, 2e3);
+  const orderedUnits = orderUnitsForStrategy(args.strategy, args.units);
+  const clientRequestedStream = args.body?.stream === true;
+  const startTime = Date.now();
+  const effectiveStrategy = args.effectiveComboStrategy ?? args.strategy;
+  let lastResponse = null;
+  let fallbackCount = 0;
+  for (const unit of orderedUnits) {
+    for (let retry = 0; retry <= maxRetries; retry += 1) {
+      if (args.signal?.aborted)
+        return { response: errorResponse(499, "Client disconnected"), unit };
+      args.nesting.attemptBudget.count += 1;
+      if (args.nesting.attemptBudget.count > args.nesting.attemptBudget.limit) {
+        return { response: errorResponse(503, "Maximum combo retry limit reached"), unit };
+      }
+      if (retry > 0) {
+        await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
+      }
+      args.log.info(
+        "COMBO",
+        `Trying ${unit.kind} ${unitDisplayName(unit)}${retry > 0 ? ` (retry ${retry})` : ""}`
+      );
+      const response = await executeRuntimeUnit({
+        body: args.body,
+        unit,
+        allCombos: args.allCombos,
+        handleSingleModel: args.handleSingleModel,
+        isModelAvailable: args.isModelAvailable,
+        runCombo: args.runCombo,
+        baseOptions: args.baseOptions,
+        nesting: args.nesting,
+        failoverBeforeRetry: args.config.failoverBeforeRetry,
+        effectiveComboStrategy: effectiveStrategy
+      });
+      lastResponse = response;
+      if (response.ok) {
+        if (unit.kind === "combo-ref") {
+          recordComboRequest(args.combo.name, null, {
+            success: true,
+            latencyMs: Date.now() - startTime,
+            fallbackCount,
+            strategy: effectiveStrategy,
+            target: { executionKey: unit.executionKey, stepId: unit.stepId, label: unit.label }
+          });
+          return { response, unit };
+        }
+        let unitClone;
+        try {
+          unitClone = response.clone();
+        } catch {
+          unitClone = response;
+        }
+        const quality = await validateResponseQuality(
+          unitClone,
+          clientRequestedStream,
+          args.log,
+          args.config.responseValidation
+        );
+        releaseQualityClone(unitClone, response, quality);
+        if (quality.valid) {
+          recordComboRequest(args.combo.name, unit.modelStr, {
+            success: true,
+            latencyMs: Date.now() - startTime,
+            fallbackCount,
+            strategy: effectiveStrategy,
+            target: { executionKey: unit.executionKey, stepId: unit.stepId, label: unit.label }
+          });
+          return { response, unit };
+        }
+      }
+      if (![408, 429, 500, 502, 503, 504].includes(response.status)) break;
+    }
+    fallbackCount += 1;
+  }
+  recordComboRequest(args.combo.name, null, {
+    success: false,
+    latencyMs: Date.now() - startTime,
+    fallbackCount,
+    strategy: effectiveStrategy
+  });
+  return {
+    response: lastResponse || errorResponse(503, "All nested combo units unavailable"),
+    unit: null
+  };
+}
+export {
+  executeRuntimeUnitCombo
+};

package/open-sse/services/combo/sessionStickiness.js

@@ -0,0 +1,128 @@
+import { createHash } from "node:crypto";
+import { computeHeadroom } from "./headroomRanking.ts";
+const STICKINESS_HEADROOM_THRESHOLD = 0.15;
+const TTL_MS = 15 * 60 * 1e3;
+const MAX_ENTRIES = 500;
+let _fetcherOverride = null;
+function __setStickinessHeadroomFetcherForTests(fetcher) {
+  _fetcherOverride = fetcher;
+}
+async function resolveSaturation(connectionId, provider) {
+  if (_fetcherOverride) return _fetcherOverride(connectionId);
+  try {
+    const mod = await import('../../../dist/lib/quota/saturationSignals.js');
+    const getSaturation = mod.getSaturation;
+    const [util5h, util7d] = await Promise.all([
+      getSaturation(connectionId, provider, { unit: "percent", window: "5h" }),
+      getSaturation(connectionId, provider, { unit: "percent", window: "weekly" })
+    ]);
+    return { util5h, util7d };
+  } catch {
+    return void 0;
+  }
+}
+const stickyMap = /* @__PURE__ */ new Map();
+function deriveMessageHash(messages) {
+  if (!Array.isArray(messages) || messages.length === 0) return null;
+  const first = messages.find((m) => m?.role === "user");
+  if (!first) return null;
+  let text;
+  if (typeof first.content === "string") {
+    text = first.content;
+  } else if (Array.isArray(first.content)) {
+    text = first.content.filter((p) => p != null && typeof p === "object").map((p) => typeof p.text === "string" ? p.text : "").join("");
+  } else {
+    return null;
+  }
+  if (!text) return null;
+  return createHash("sha256").update(text).digest("hex").slice(0, 16);
+}
+function evict() {
+  const now = Date.now();
+  for (const [key, entry] of stickyMap) {
+    if (now - entry.lastUsedAt > TTL_MS) stickyMap.delete(key);
+  }
+  while (stickyMap.size > MAX_ENTRIES) {
+    let oldestKey = null;
+    let oldestTime = Infinity;
+    for (const [key, entry] of stickyMap) {
+      if (entry.lastUsedAt < oldestTime) {
+        oldestTime = entry.lastUsedAt;
+        oldestKey = key;
+      }
+    }
+    if (oldestKey === null) break;
+    stickyMap.delete(oldestKey);
+  }
+}
+function recordStickyBinding(messageHash, connectionId) {
+  const existing = stickyMap.get(messageHash);
+  if (existing) {
+    existing.connectionId = connectionId;
+    existing.lastUsedAt = Date.now();
+  } else {
+    evict();
+    stickyMap.set(messageHash, {
+      connectionId,
+      createdAt: Date.now(),
+      lastUsedAt: Date.now()
+    });
+  }
+}
+function clearStickyBinding(messageHash) {
+  stickyMap.delete(messageHash);
+}
+function clearAllStickyBindings() {
+  stickyMap.clear();
+}
+function resolveDisableSessionStickiness(config, settings) {
+  const perCombo = config?.disableSessionStickiness;
+  if (typeof perCombo === "boolean") return perCombo;
+  return settings?.disableSessionStickiness === true;
+}
+async function applySessionStickiness(orderedTargets, messages) {
+  const noOp = { targets: orderedTargets, messageHash: null, stuck: false };
+  try {
+    if (orderedTargets.length <= 1) return noOp;
+    const messageHash = deriveMessageHash(messages);
+    if (!messageHash) return noOp;
+    const existing = stickyMap.get(messageHash);
+    if (!existing) return { targets: orderedTargets, messageHash, stuck: false };
+    if (Date.now() - existing.lastUsedAt > TTL_MS) {
+      stickyMap.delete(messageHash);
+      return { targets: orderedTargets, messageHash, stuck: false };
+    }
+    const { connectionId } = existing;
+    const stickyIdx = orderedTargets.findIndex((t) => t.connectionId === connectionId);
+    if (stickyIdx === -1) {
+      clearStickyBinding(messageHash);
+      return { targets: orderedTargets, messageHash, stuck: false };
+    }
+    const stickyTarget = orderedTargets[stickyIdx];
+    const sat = await resolveSaturation(connectionId, stickyTarget.provider);
+    const headroom = computeHeadroom(sat);
+    if (headroom <= STICKINESS_HEADROOM_THRESHOLD) {
+      clearStickyBinding(messageHash);
+      return { targets: orderedTargets, messageHash, stuck: false };
+    }
+    const reordered = [
+      orderedTargets[stickyIdx],
+      ...orderedTargets.slice(0, stickyIdx),
+      ...orderedTargets.slice(stickyIdx + 1)
+    ];
+    existing.lastUsedAt = Date.now();
+    return { targets: reordered, messageHash, stuck: true };
+  } catch {
+    return noOp;
+  }
+}
+export {
+  STICKINESS_HEADROOM_THRESHOLD,
+  __setStickinessHeadroomFetcherForTests,
+  applySessionStickiness,
+  clearAllStickyBindings,
+  clearStickyBinding,
+  deriveMessageHash,
+  recordStickyBinding,
+  resolveDisableSessionStickiness
+};

package/open-sse/services/combo/shadowRouting.js

@@ -0,0 +1,131 @@
+import { secureRandomFloat } from "../../../shared/utils/secureRandom";
+import { recordComboShadowRequest } from "../comboMetrics.ts";
+import { isRecord } from "./comboData.ts";
+import { resolveNestedComboTargets } from "./comboStructure.ts";
+import { toRecordedTarget } from "./comboPredicates.ts";
+function normalizeShadowRoutingConfig(config) {
+  const raw = isRecord(config.shadowRouting) ? config.shadowRouting : {};
+  const sampleRate = Number(raw.sampleRate ?? 1);
+  const maxTargets = Number(raw.maxTargets ?? 2);
+  const timeoutMs = Number(raw.timeoutMs ?? 3e4);
+  return {
+    enabled: raw.enabled === true,
+    targets: Array.isArray(raw.targets) ? raw.targets : [],
+    sampleRate: Number.isFinite(sampleRate) ? Math.max(0, Math.min(1, sampleRate)) : 1,
+    maxTargets: Number.isFinite(maxTargets) ? Math.max(1, Math.min(10, Math.floor(maxTargets))) : 2,
+    timeoutMs: Number.isFinite(timeoutMs) ? Math.max(1e3, Math.min(12e4, Math.floor(timeoutMs))) : 3e4
+  };
+}
+function resolveShadowTargets(combo, config, allCombos) {
+  const shadowConfig = normalizeShadowRoutingConfig(config);
+  if (!shadowConfig.enabled || shadowConfig.targets.length === 0) return [];
+  if (shadowConfig.sampleRate <= 0 || secureRandomFloat() > shadowConfig.sampleRate) return [];
+  const shadowCombo = {
+    ...combo,
+    name: `${combo.name}:shadow`,
+    models: shadowConfig.targets
+  };
+  return resolveNestedComboTargets(shadowCombo, allCombos, /* @__PURE__ */ new Set([combo.name]), 0, ["shadow"]).slice(0, shadowConfig.maxTargets).map((target) => ({
+    ...target,
+    trafficType: "shadow"
+  }));
+}
+async function drainShadowResponse(response) {
+  try {
+    if (!response.body) return;
+    await response.arrayBuffer();
+  } catch {
+  }
+}
+function withTimeout(promise, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error("Shadow route timed out")), timeoutMs);
+    promise.then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      (error) => {
+        clearTimeout(timer);
+        reject(error);
+      }
+    );
+  });
+}
+function cloneRequestBodyForShadowRouting(body) {
+  if (typeof structuredClone === "function") {
+    return structuredClone(body);
+  }
+  return JSON.parse(JSON.stringify(body));
+}
+function scheduleShadowRouting(combo, config, body, targets, handleSingleModel, isModelAvailable, strategy, log) {
+  if (targets.length === 0) return;
+  const shadowConfig = normalizeShadowRoutingConfig(config);
+  let shadowBaseBody;
+  try {
+    shadowBaseBody = cloneRequestBodyForShadowRouting(body);
+  } catch (error) {
+    log.warn("COMBO", "Shadow routing skipped: failed to clone request body", {
+      error: error instanceof Error ? error.message : String(error)
+    });
+    return;
+  }
+  const run = async () => {
+    await Promise.all(
+      targets.map(async (target) => {
+        const startedAt = Date.now();
+        try {
+          const shadowBody = {
+            ...cloneRequestBodyForShadowRouting(shadowBaseBody),
+            model: target.modelStr,
+            stream: false
+          };
+          if (isModelAvailable) {
+            const available = await isModelAvailable(target.modelStr, target);
+            if (!available) {
+              recordComboShadowRequest(combo.name, target.modelStr, {
+                success: false,
+                latencyMs: Date.now() - startedAt,
+                target: toRecordedTarget(target)
+              });
+              log.info("COMBO", `Shadow target skipped (unavailable): ${target.modelStr}`);
+              return;
+            }
+          }
+          const response = await withTimeout(
+            handleSingleModel(shadowBody, target.modelStr, {
+              ...target,
+              failoverBeforeRetry: true,
+              trafficType: "shadow"
+            }),
+            shadowConfig.timeoutMs
+          );
+          await drainShadowResponse(response.clone());
+          recordComboShadowRequest(combo.name, target.modelStr, {
+            success: response.ok,
+            latencyMs: Date.now() - startedAt,
+            target: toRecordedTarget(target)
+          });
+          log.info(
+            "COMBO",
+            `Shadow target ${target.modelStr} completed with status ${response.status} (${strategy})`
+          );
+        } catch (error) {
+          recordComboShadowRequest(combo.name, target.modelStr, {
+            success: false,
+            latencyMs: Date.now() - startedAt,
+            target: toRecordedTarget(target)
+          });
+          log.warn("COMBO", `Shadow target ${target.modelStr} failed`, {
+            error: error instanceof Error ? error.message : String(error)
+          });
+        }
+      })
+    );
+  };
+  setTimeout(() => void run(), 0);
+}
+export {
+  resolveShadowTargets,
+  scheduleShadowRouting
+};

package/open-sse/services/combo/targetExhaustion.js

@@ -0,0 +1,75 @@
+import {
+  classifyErrorText,
+  hasPerModelQuota,
+  isProviderExhaustedReason
+} from "../accountFallback.ts";
+import { RateLimitReason } from "../../config/constants.ts";
+import { isProviderCircuitOpenResult } from "./comboPredicates.ts";
+const CONNECTION_LEVEL_ERROR_STATUSES = [408, 500, 502, 503, 504, 524];
+function isEmptyContentFailure(status, errorText) {
+  return status === 502 && /empty content/i.test(errorText);
+}
+function applyComboTargetExhaustion(target, opts) {
+  const {
+    result,
+    fallbackResult,
+    errorText,
+    rawModel,
+    isTokenLimitBreach,
+    allAccountsRateLimited,
+    sets,
+    log,
+    tag,
+    exhaustedLogLevel,
+    structuredError
+  } = opts;
+  const { exhaustedProviders, exhaustedConnections, transientRateLimitedProviders } = sets;
+  const provider = target.provider;
+  const providerExhausted = Boolean(provider && provider !== "unknown") && !hasPerModelQuota(provider, rawModel) && (isProviderExhaustedReason(fallbackResult) || classifyErrorText(structuredError?.code || errorText) === RateLimitReason.QUOTA_EXHAUSTED || allAccountsRateLimited);
+  if (providerExhausted) {
+    exhaustedProviders.add(provider);
+    const emit = exhaustedLogLevel === "debug" ? log.debug : log.info;
+    emit?.(
+      tag,
+      `Provider ${provider} quota exhausted \u2014 marking for skip on remaining targets (#1731)`
+    );
+  } else {
+    if (result.status === 429 && !isTokenLimitBreach && provider && provider !== "unknown") {
+      transientRateLimitedProviders.add(provider);
+    }
+    markConnectionLevelExhaustion(target, { result, errorText, sets, log, tag, rawModel });
+  }
+  return providerExhausted;
+}
+function markConnectionLevelExhaustion(target, opts) {
+  const { result, errorText, sets, log, tag, rawModel } = opts;
+  const provider = target.provider;
+  if (!provider || provider === "unknown" || !CONNECTION_LEVEL_ERROR_STATUSES.includes(result.status) || isProviderCircuitOpenResult(result, errorText) || // #5085: empty-content 502 is a healthy connection returning no body — model-level, not
+  // connection-level. Don't exhaust the provider; let the remaining legs (incl. same-provider)
+  // be tried in-request.
+  isEmptyContentFailure(result.status, errorText) || // Per-model-quota providers (gemini, github, passthrough, compatible) multiplex models
+  // behind one connection. A model-level 500 (e.g. Gemini "Internal error encountered")
+  // must NOT exhaust the connection — other models on the same connection may still succeed.
+  // Other connection-level statuses (408/502/503/504/524) indicate the connection itself is
+  // bad, so they correctly exhaust even for per-model-quota providers.
+  result.status === 500 && hasPerModelQuota(provider, rawModel)) {
+    return;
+  }
+  const connId = target.connectionId ?? void 0;
+  if (connId) {
+    sets.exhaustedConnections.add(`${provider}:${connId}`);
+    log.info(
+      tag,
+      `Provider ${provider} connection ${connId} error (${result.status}) \u2014 marking for skip on remaining targets (#1731v2)`
+    );
+  } else {
+    sets.exhaustedProviders.add(provider);
+    log.info(
+      tag,
+      `Provider ${provider} connection error (${result.status}) \u2014 marking for skip on remaining targets (#1731)`
+    );
+  }
+}
+export {
+  applyComboTargetExhaustion
+};