@fudrouter/fsrouter 0.6.62 → 0.6.63

package/open-sse/services/combo/targetSorters.js

@@ -0,0 +1,126 @@
+import { getCircuitBreaker } from "../../../shared/utils/circuitBreaker";
+import { secureRandomFloat, secureRandomInt } from "../../../shared/utils/secureRandom";
+import { getComboStepTarget, getComboStepWeight } from '../../../dist/lib/combos/steps.ts.js';
+import { getComboMetrics } from "../comboMetrics.ts";
+import { parseModel } from "../model.ts";
+function normalizeModelEntry(entry) {
+  return {
+    model: getComboStepTarget(entry) || "",
+    weight: getComboStepWeight(entry)
+  };
+}
+function selectWeightedTarget(targets) {
+  if (targets.length === 0) return null;
+  const totalWeight = targets.reduce((sum, target) => sum + (target.weight || 0), 0);
+  if (totalWeight <= 0) {
+    return targets[secureRandomInt(targets.length)];
+  }
+  let random = secureRandomFloat() * totalWeight;
+  for (const target of targets) {
+    random -= target.weight || 0;
+    if (random <= 0) return target;
+  }
+  return targets.at(-1);
+}
+function orderTargetsForWeightedFallback(targets, selectedExecutionKey, preserveExistingOrder = false) {
+  const selected = targets.find((target) => target.executionKey === selectedExecutionKey);
+  const rest = targets.filter((target) => target.executionKey !== selectedExecutionKey);
+  if (!preserveExistingOrder) {
+    rest.sort((a, b) => b.weight - a.weight);
+  }
+  return selected ? [selected, ...rest] : rest;
+}
+async function sortModelsByCost(models) {
+  try {
+    const { getPricingForModel } = await import('../../../dist/lib/localDb.js');
+    const withCost = await Promise.all(
+      models.map(async (modelStr) => {
+        const parsed = parseModel(modelStr);
+        const provider = parsed.provider || parsed.providerAlias || "unknown";
+        const model = parsed.model || modelStr;
+        try {
+          const pricing = await getPricingForModel(provider, model);
+          const cost = Number(pricing?.input);
+          return { modelStr, cost: Number.isFinite(cost) ? cost : Infinity };
+        } catch {
+          return { modelStr, cost: Infinity };
+        }
+      })
+    );
+    withCost.sort((a, b) => a.cost - b.cost);
+    return withCost.map((e) => e.modelStr);
+  } catch {
+    return models;
+  }
+}
+async function sortTargetsByCost(targets) {
+  const orderedModels = await sortModelsByCost(targets.map((target) => target.modelStr));
+  const byModel = /* @__PURE__ */ new Map();
+  for (const target of targets) {
+    const queue = byModel.get(target.modelStr) || [];
+    queue.push(target);
+    byModel.set(target.modelStr, queue);
+  }
+  return orderedModels.map((modelStr) => {
+    const queue = byModel.get(modelStr);
+    return queue?.shift() || null;
+  }).filter((target) => target !== null);
+}
+function sortModelsByUsage(models, comboName) {
+  const metrics = getComboMetrics(comboName);
+  if (!metrics?.byModel) return models;
+  const withUsage = models.map((modelStr) => ({
+    modelStr,
+    requests: metrics.byModel[modelStr]?.requests ?? 0
+  }));
+  withUsage.sort((a, b) => a.requests - b.requests);
+  return withUsage.map((e) => e.modelStr);
+}
+function sortTargetsByUsage(targets, comboName) {
+  const orderedModels = sortModelsByUsage(
+    targets.map((target) => target.modelStr),
+    comboName
+  );
+  const byModel = /* @__PURE__ */ new Map();
+  for (const target of targets) {
+    const queue = byModel.get(target.modelStr) || [];
+    queue.push(target);
+    byModel.set(target.modelStr, queue);
+  }
+  return orderedModels.map((modelStr) => {
+    const queue = byModel.get(modelStr);
+    return queue?.shift() || null;
+  }).filter((target) => target !== null);
+}
+function getP2CTargetScore(target, metrics) {
+  const breakerState = getCircuitBreaker(target.provider)?.getStatus?.()?.state;
+  if (breakerState === "OPEN") return -Infinity;
+  const modelMetric = metrics?.byModel?.[target.modelStr] || null;
+  const successRate = Number(modelMetric?.successRate);
+  const avgLatency = Number(modelMetric?.avgLatencyMs);
+  const successScore = Number.isFinite(successRate) ? successRate / 100 : 0.5;
+  const latencyScore = Number.isFinite(avgLatency) && avgLatency > 0 ? 1 / Math.log10(avgLatency + 10) : 0.25;
+  const breakerPenalty = breakerState === "HALF_OPEN" ? 0.25 : 0;
+  return successScore + latencyScore - breakerPenalty;
+}
+function orderTargetsByPowerOfTwoChoices(targets, comboName) {
+  if (targets.length <= 1) return targets;
+  const metrics = getComboMetrics(comboName);
+  const firstIndex = secureRandomInt(targets.length);
+  let secondIndex = secureRandomInt(targets.length - 1);
+  if (secondIndex >= firstIndex) secondIndex++;
+  const first = targets[firstIndex];
+  const second = targets[secondIndex];
+  const selectedIndex = getP2CTargetScore(second, metrics) > getP2CTargetScore(first, metrics) ? secondIndex : firstIndex;
+  return [targets[selectedIndex], ...targets.filter((_, index) => index !== selectedIndex)];
+}
+export {
+  normalizeModelEntry,
+  orderTargetsByPowerOfTwoChoices,
+  orderTargetsForWeightedFallback,
+  selectWeightedTarget,
+  sortModelsByCost,
+  sortModelsByUsage,
+  sortTargetsByCost,
+  sortTargetsByUsage
+};

package/open-sse/services/combo/targetTimeoutRunner.js

@@ -0,0 +1,65 @@
+import { errorResponse } from "../../utils/error.ts";
+function buildTargetTimeoutRunner(deps) {
+  const { handleSingleModel, comboTargetTimeoutMs, log } = deps;
+  return async (b, modelStr, target) => {
+    if (comboTargetTimeoutMs <= 0) {
+      return handleSingleModel(b, modelStr, target).catch(
+        (err) => errorResponse(502, err?.message ?? "Upstream model error")
+      );
+    }
+    const timeoutController = new AbortController();
+    let timeoutId;
+    let timedOut = false;
+    const timeoutPromise = new Promise((resolve) => {
+      timeoutId = setTimeout(() => {
+        timedOut = true;
+        log.warn(
+          "COMBO",
+          `Model ${modelStr} exceeded ${comboTargetTimeoutMs}ms timeout \u2014 falling back`
+        );
+        timeoutController.abort(new Error("combo-per-model-timeout"));
+        resolve(
+          new Response(JSON.stringify({ error: { message: `Model ${modelStr} timed out` } }), {
+            status: 524,
+            headers: { "Content-Type": "application/json" }
+          })
+        );
+      }, comboTargetTimeoutMs);
+    });
+    const targetWithSignal = {
+      ...target ?? {},
+      modelAbortSignal: timeoutController.signal
+    };
+    const parentHedgeSignal = target?.modelAbortSignal ?? null;
+    let onParentHedgeAbort = null;
+    if (parentHedgeSignal) {
+      if (parentHedgeSignal.aborted) {
+        timeoutController.abort(new Error("hedge-cancelled"));
+      } else {
+        onParentHedgeAbort = () => {
+          timeoutController.abort(new Error("hedge-cancelled"));
+        };
+        parentHedgeSignal.addEventListener("abort", onParentHedgeAbort, { once: true });
+      }
+    }
+    try {
+      return await Promise.race([
+        handleSingleModel(b, modelStr, targetWithSignal).catch((err) => {
+          if (timedOut) {
+            return new Response(null, { status: 599 });
+          }
+          return errorResponse(502, err?.message ?? "Upstream model error");
+        }),
+        timeoutPromise
+      ]);
+    } finally {
+      clearTimeout(timeoutId);
+      if (parentHedgeSignal && onParentHedgeAbort) {
+        parentHedgeSignal.removeEventListener("abort", onParentHedgeAbort);
+      }
+    }
+  };
+}
+export {
+  buildTargetTimeoutRunner
+};

package/open-sse/services/combo/validateQuality.js

@@ -0,0 +1,271 @@
+import {
+  createSSEDataLineNormalizer,
+  isKnownNonClaudeStreamPayload
+} from "../../utils/streamHelpers.ts";
+import { evaluateResponseValidation } from "./responseValidation.ts";
+import { getReasoningTokens } from '../../../dist/lib/usage/tokenAccounting.ts.js';
+function toRetryAfterDisplayValue(value) {
+  if (typeof value !== "number") return value;
+  if (value > 0 && value < 1e9) {
+    return new Date(Date.now() + value * 1e3);
+  }
+  return new Date(value);
+}
+function responsesApiOutputHasContent(output) {
+  return Array.isArray(output) && output.some((item) => {
+    if (!item || typeof item !== "object") return false;
+    const record = item;
+    if (record.type !== "message") return Boolean(record.type);
+    const content = record.content;
+    return Array.isArray(content) && content.some(
+      (part) => !!part && typeof part === "object" && typeof part.text === "string" && part.text.length > 0
+    );
+  });
+}
+async function validateResponseQuality(response, isStreaming, log, responseValidation) {
+  if (isStreaming) {
+    let parseAccumulatedSse = function() {
+      const lines = decodedSoFar.split(/\r?\n/);
+      decodedSoFar = lines[lines.length - 1];
+      for (const line of sseLineNormalizer.normalize(lines.slice(0, -1))) {
+        const trimmed = line.trim();
+        if (trimmed.startsWith("event:")) {
+          pendingEventType = trimmed.slice(6).trim();
+          continue;
+        }
+        if (!trimmed.startsWith("data:")) {
+          if (!trimmed) pendingEventType = "";
+          continue;
+        }
+        const data = trimmed.slice(5).trim();
+        if (!data || data === "[DONE]") continue;
+        let parsed;
+        try {
+          parsed = JSON.parse(data);
+        } catch {
+          continue;
+        }
+        const eventType = (typeof parsed.type === "string" ? parsed.type : null) || pendingEventType || "";
+        pendingEventType = "";
+        if (isKnownNonClaudeStreamPayload(parsed, eventType)) {
+          return true;
+        }
+        switch (eventType) {
+          case "message_start":
+            hasMessageStart = true;
+            break;
+          case "content_block_start":
+          case "content_block_delta":
+          case "content_block_stop":
+            hasContentBlock = true;
+            return true;
+          case "message_stop":
+            hasLifecycleEnd = true;
+            break;
+          case "message_delta": {
+            const delta = parsed.delta;
+            if (delta && typeof delta === "object" && delta.stop_reason != null) {
+              hasLifecycleEnd = true;
+            }
+            break;
+          }
+          default:
+            break;
+        }
+      }
+      return false;
+    }, buildReplayResponse = function(readerToForward) {
+      const prefix = bufferedChunks.slice();
+      let prefixIdx = 0;
+      const stream = new ReadableStream({
+        async pull(controller) {
+          if (prefixIdx < prefix.length) {
+            controller.enqueue(prefix[prefixIdx++]);
+            return;
+          }
+          try {
+            const { done, value } = await readerToForward.read();
+            if (done) {
+              controller.close();
+            } else {
+              controller.enqueue(value);
+            }
+          } catch {
+            controller.close();
+          }
+        }
+      });
+      return new Response(stream, {
+        status: response.status,
+        statusText: response.statusText,
+        headers: response.headers
+      });
+    };
+    const contentType2 = response.headers.get("content-type") || "";
+    if (!contentType2.includes("text/event-stream")) {
+      return { valid: true };
+    }
+    if (!response.body) {
+      return { valid: true };
+    }
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder("utf-8");
+    const bufferedChunks = [];
+    let decodedSoFar = "";
+    let hasMessageStart = false;
+    let hasContentBlock = false;
+    let hasLifecycleEnd = false;
+    let anyContentFound = false;
+    let sawAnyBytes = false;
+    const sseLineNormalizer = createSSEDataLineNormalizer();
+    let pendingEventType = "";
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) {
+          const tail = decoder.decode(void 0, { stream: false });
+          if (tail) decodedSoFar += tail;
+          if (decodedSoFar.trim()) decodedSoFar += "\n\n";
+          parseAccumulatedSse();
+          if (hasMessageStart && hasLifecycleEnd && !hasContentBlock) {
+            log.warn?.(
+              "COMBO",
+              "Streaming Claude response has complete lifecycle but zero content blocks (content_filter?) \u2014 marking as invalid for combo failover"
+            );
+            return { valid: false, reason: "streaming empty content block" };
+          }
+          if (!anyContentFound && !hasContentBlock && !sawAnyBytes) {
+            log.warn?.(
+              "COMBO",
+              "Streaming response ended with no recognized content \u2014 marking as invalid for combo failover"
+            );
+            return { valid: false, reason: "streaming no recognized content" };
+          }
+          const clonedResponse = buildReplayResponse(reader);
+          return { valid: true, clonedResponse };
+        }
+        bufferedChunks.push(value);
+        if (value && value.length > 0) sawAnyBytes = true;
+        decodedSoFar += decoder.decode(value, { stream: true });
+        const foundContent = parseAccumulatedSse();
+        if (foundContent) {
+          anyContentFound = true;
+          const clonedResponse = buildReplayResponse(reader);
+          return { valid: true, clonedResponse };
+        }
+      }
+    } catch (streamErr) {
+      const errMsg = streamErr instanceof Error ? streamErr.message : String(streamErr);
+      if (streamErr instanceof TypeError && (errMsg.includes("locked") || errMsg.includes("disturbed") || errMsg.includes("used already"))) {
+        return { valid: false, reason: "stream locked or disturbed" };
+      }
+      return { valid: true };
+    }
+  }
+  const contentType = response.headers.get("content-type") || "";
+  if (!contentType.includes("application/json") && !contentType.includes("text/")) {
+    return { valid: true };
+  }
+  let cloned;
+  try {
+    cloned = response.clone();
+  } catch {
+    return { valid: true };
+  }
+  let text;
+  try {
+    text = await cloned.text();
+  } catch {
+    return { valid: true };
+  }
+  if (!text || text.trim().length === 0) {
+    return { valid: false, reason: "empty response body" };
+  }
+  let json;
+  try {
+    json = JSON.parse(text);
+  } catch {
+    if (text.startsWith("data:") || text.startsWith("event:")) return { valid: true };
+    return { valid: false, reason: "response is not valid JSON" };
+  }
+  if (responseValidation) {
+    const verdict = evaluateResponseValidation(json, responseValidation);
+    if (!verdict.valid) {
+      return { valid: false, reason: verdict.reason };
+    }
+  }
+  const choices = json?.choices;
+  if (json?.object === "response") {
+    if (!responsesApiOutputHasContent(json.output))
+      return { valid: false, reason: "empty_choices" };
+    const status = typeof json.status === "string" ? json.status : "";
+    if (status && !["completed", "done"].includes(status)) {
+      return { valid: false, reason: "no_terminal" };
+    }
+    return {
+      valid: true,
+      clonedResponse: new Response(text, {
+        status: response.status,
+        statusText: response.statusText,
+        headers: response.headers
+      })
+    };
+  }
+  if (!Array.isArray(choices) || choices.length === 0) {
+    if (json?.output || json?.result || json?.data || json?.response) return { valid: true };
+    if (json?.error) {
+      const err = json.error;
+      return {
+        valid: false,
+        reason: `upstream error in 200 body: ${err?.message || JSON.stringify(json.error).substring(0, 200)}`
+      };
+    }
+    return { valid: true };
+  }
+  const firstChoice = choices[0];
+  const message = firstChoice?.message || firstChoice?.delta;
+  if (!message) {
+    return { valid: false, reason: "choice has no message object" };
+  }
+  const content = message.content;
+  const toolCalls = message.tool_calls;
+  const reasoningContent = message.reasoning_content ?? message.reasoning;
+  const hasReasoningContent = typeof reasoningContent === "string" && reasoningContent.trim().length > 0;
+  const hasContent = content !== null && content !== void 0 && content !== "" || hasReasoningContent;
+  const hasToolCalls = Array.isArray(toolCalls) && toolCalls.length > 0;
+  if (!hasContent && !hasToolCalls) {
+    return { valid: false, reason: "empty content and no tool_calls in response" };
+  }
+  const contentIsEmpty = content === null || content === void 0 || content === "";
+  if (contentIsEmpty && hasReasoningContent && !hasToolCalls) {
+    const usage = json?.usage;
+    if (usage) {
+      const completionTokens = Number(usage.completion_tokens) || 0;
+      const reasoningTokens = getReasoningTokens(usage);
+      if (completionTokens > 0 && reasoningTokens >= completionTokens * 0.9) {
+        return {
+          valid: false,
+          reason: `reasoning consumed ${reasoningTokens}/${completionTokens} tokens \u2014 no content output`
+        };
+      }
+    }
+  }
+  return {
+    valid: true,
+    clonedResponse: new Response(text, {
+      status: response.status,
+      statusText: response.statusText,
+      headers: response.headers
+    })
+  };
+}
+function releaseQualityClone(clone, original, quality) {
+  if (clone === original) return;
+  void quality.clonedResponse?.body?.cancel().catch(() => {
+  });
+}
+export {
+  releaseQualityClone,
+  toRetryAfterDisplayValue,
+  validateResponseQuality
+};