llm-relay 0.63.0 → 0.64.0

package/dist/server.js

@@ -13,7 +13,7 @@ import { emitSse, emitSseTail, syntheticMessageId } from "./emitSse.js";
 import { repair, destructiveMatcher } from "./repair.js";
 import { CredentialWalkReshaper, HttpReshaper, } from "./reshaper.js";
 import { DIALECT_REFUSED_DESTRUCTIVE_CODE } from "./tool-dialects.js";
-import { dialectRefusalSignalOf, ERROR_ORIGIN_HEADER, TOOL_DIALECT_HEADER, fetchBackend, fetchOpenAiFront, normalizeOpenAiErrorBody, parseRetryAfterMs, postHeaderBodyFailure, upstreamReportedModel, toolUseIdRewrites, toolCallIdRewrites, thoughtSignatureSentinels, SERVED_BY_HEADER, POOL_ATTEMPTS_HEADER, UNKNOWN_REFUSAL_HEADER, DEGRADED_HEADER, PAID_HEADER, QUOTA_DEMOTED_HEADER, CREDENTIAL_HEADER, CREDENTIAL_ATTEMPTS_HEADER, HARD_CAP_HEADER, errorOrigin } from "./backend.js";
+import { dialectRefusalSignalOf, ERROR_ORIGIN_HEADER, TOOL_DIALECT_HEADER, fetchBackend, fetchOpenAiFront, normalizeOpenAiErrorBody, parseRetryAfterMs, postHeaderBodyFailure, upstreamReportedModel, toolUseIdRewrites, toolCallIdRewrites, thoughtSignatureSentinels, SERVED_BY_HEADER, POOL_ATTEMPTS_HEADER, UNKNOWN_REFUSAL_HEADER, DEGRADED_HEADER, PAID_HEADER, QUOTA_DEMOTED_HEADER, LATENCY_DEMOTED_HEADER, CREDENTIAL_HEADER, CREDENTIAL_ATTEMPTS_HEADER, HARD_CAP_HEADER, errorOrigin } from "./backend.js";
 import { probeStreamForCommit } from "./stream-commit.js";
 import { ModelCatalog } from "./catalog.js";
 import { handleAdminRoutes } from "./routes/admin.js";
@@ -41,6 +41,7 @@ import { looksLikeContextLengthError, looksLikeMaxOutputError, parseStatedContex
 import { looksLikeRateLimitError, parseStatedRateLimit, recordObservedRateLimit } from "./rate-limits.js";
 import { clearFacts, cooldownUntil, factsFor, isCostBlocked, recordFact } from "./target-facts.js";
 import { createQuotaDemotionFn, quotaDemotionLabel } from "./quota-demotion.js";
+import { createLatencyDemotionFn, latencyDemotionLabel } from "./latency-demotion.js";
 import { createHardCapLedgerReader, evaluateHardCap, hardCapLabel } from "./hard-cap.js";
 import { applyResetRule, interpretRefusal, materializeScope, parseStatedResetMs, recordUnknownRefusal } from "./refusal-interpretation.js";
 import { CONTROL_AUTHORIZATION_HEADER, createControlAuthorization, resolveControlAuthorizationConfigDir, validateControlAuthorization, } from "./control-authorization.js";
@@ -212,6 +213,10 @@ export function createProxy(cfg, deps = {}) {
             ? deps.accountingReader
             : null,
     });
+    const latencyDemotion = createLatencyDemotionFn({
+        breaker,
+        settings: cfg.routing?.latency,
+    });
     const costClassOf = (attempt) => {
         const t = attempt.target;
         if (t.kind !== "openai" || !t.model)
@@ -323,6 +328,7 @@ export function createProxy(cfg, deps = {}) {
             server,
             ...(controlAuthorization ? { controlAuthorization } : {}),
             quotaDemotion,
+            latencyDemotion,
             costClassOf,
             hardCap: hardCapEvaluator,
         }).catch((e) => {
@@ -781,7 +787,7 @@ async function handle(req, res, cfg, h) {
     const rankedAttempts = rankCredentialAttempts(attempts, h.credentialLru, {
         evidenceFor: (attempt) => credentialEvidence(attempt, cfg, h.breaker, routingNow),
     });
-    const { ordered: orderedAttempts, quotaDemotedFirst } = orderDeploymentGroupsByUsability(rankedAttempts, h.breaker, routingNow, h.quotaDemotion, h.costClassOf);
+    const { ordered: orderedAttempts, quotaDemotedFirst, latencyDemotedFirst } = orderDeploymentGroupsByUsability(rankedAttempts, h.breaker, routingNow, h.quotaDemotion, h.costClassOf, h.latencyDemotion);
     let walkAttempts = orderedAttempts;
     let sticky = null;
     if ((isMessages || openAiFrontProtocol) && h.stickySessions) {
@@ -854,6 +860,7 @@ async function handle(req, res, cfg, h) {
             cfg,
             accounting,
             quotaDemotedFirst,
+            latencyDemotedFirst,
         }, h);
         return;
     }
@@ -1098,6 +1105,7 @@ async function handle(req, res, cfg, h) {
                 tried,
                 degraded: degradedLabel(addressedPool, degradedSpecs, target),
                 quotaDemoted: quotaDemotedFirst,
+                latencyDemoted: latencyDemotedFirst,
                 paid: paidLabel(cfg, h, target),
                 credentialHeaders: backendRes.status < 400
                     ? credentialTrace.headers(resolvedAttempt)
@@ -1224,7 +1232,7 @@ function cooledByQuota(attempt, breaker, quotaDemotion, now) {
     breaker.recordQuotaCooldown(targetIdentity(attempt), spent.resetsAt, now);
     return true;
 }
-function targetUsability(attempt, breaker, now, quotaDemotion, costClassOf) {
+function targetUsability(attempt, breaker, now, quotaDemotion, costClassOf, latencyDemotion) {
     const identity = targetIdentity(attempt);
     if (!breaker.isHealthy(identity, now))
         return "cooling";
@@ -1232,6 +1240,8 @@ function targetUsability(attempt, breaker, now, quotaDemotion, costClassOf) {
         return "cooling";
     if (cooledByQuota(attempt, breaker, quotaDemotion, now))
         return "cooling";
+    if (latencyDemotion?.(attempt, now))
+        return "cooling";
     if (breaker.hasCredentialFault(identity, now))
         return "credential-fault";
     return "live";
@@ -1290,13 +1300,13 @@ function coolingLiftTime(attempt, breaker, now, costClassOf) {
     }
     return null;
 }
-function orderDeploymentGroupsByUsability(attempts, breaker, now, quotaDemotion, costClassOf) {
+function orderDeploymentGroupsByUsability(attempts, breaker, now, quotaDemotion, costClassOf, latencyDemotion) {
     const preferred = groupCredentialAttempts(attempts)[0]?.attempts[0];
     const live = [];
     const faulted = [];
     const cooling = [];
     for (const group of groupCredentialAttempts(attempts)) {
-        const usability = targetUsability(group.attempts[0], breaker, now, quotaDemotion, costClassOf);
+        const usability = targetUsability(group.attempts[0], breaker, now, quotaDemotion, costClassOf, latencyDemotion);
         if (usability === "cooling")
             cooling.push(group);
         else if (usability === "credential-fault")
@@ -1316,6 +1326,16 @@ function orderDeploymentGroupsByUsability(attempts, breaker, now, quotaDemotion,
         return liftA - liftB;
     });
     const ordered = [...live, ...faulted, ...cooling].flatMap((group) => group.attempts);
+    let latencyDemotedFirst = null;
+    if (preferred !== undefined &&
+        latencyDemotion !== undefined &&
+        latencyDemotion !== null &&
+        ordered[0] !== undefined &&
+        ordered[0] !== preferred) {
+        const slow = latencyDemotion(preferred, now);
+        if (slow)
+            latencyDemotedFirst = latencyDemotionLabel(specOfTarget(preferred.target), slow);
+    }
     let quotaDemotedFirst = null;
     if (preferred !== undefined &&
         quotaDemotion !== undefined &&
@@ -1325,7 +1345,7 @@ function orderDeploymentGroupsByUsability(attempts, breaker, now, quotaDemotion,
         quotaDemotion(preferred, now) !== null) {
         quotaDemotedFirst = quotaDemotionLabel(specOfTarget(preferred.target), quotaDemotion(preferred, now));
     }
-    return { ordered, quotaDemotedFirst };
+    return { ordered, quotaDemotedFirst, latencyDemotedFirst };
 }
 class CredentialAttemptTrace {
     cfg;
@@ -1504,21 +1524,21 @@ function respondAllCapped(res, h, ctx, front, tracker, attempts) {
     res.end(JSON.stringify(body));
     h.logger.write(baseLog(ctx.started, ctx.path, ctx.hadTools, ctx.streamed, 429, "skipped", null, attempts));
 }
-function applyStickyOrdering(ordered, pinnedSpec, breaker, degraded, now, quotaDemotion, costClassOf) {
+function applyStickyOrdering(ordered, pinnedSpec, breaker, degraded, now, quotaDemotion, costClassOf, latencyDemotion) {
     const groups = groupCredentialAttempts(ordered);
     const pinnedIndex = groups.findIndex((group) => specOfTarget(group.attempts[0].target) === pinnedSpec);
     const pinnedGroup = pinnedIndex < 0 ? undefined : groups[pinnedIndex];
     const pinned = pinnedGroup?.attempts[0];
     if (!pinned || !pinnedGroup)
         return { targets: ordered, status: "bypassed: not-in-pool" };
-    const usability = targetUsability(pinned, breaker, now, quotaDemotion, costClassOf);
+    const usability = targetUsability(pinned, breaker, now, quotaDemotion, costClassOf, latencyDemotion);
     if (usability !== "live")
         return { targets: ordered, status: `bypassed: ${usability}` };
     if (degraded?.has(pinnedSpec)) {
         const hasLiveInBand = groups.some((group) => {
             const candidate = group.attempts[0];
             return !degraded.has(specOfTarget(candidate.target)) &&
-                targetUsability(candidate, breaker, now, quotaDemotion, costClassOf) === "live";
+                targetUsability(candidate, breaker, now, quotaDemotion, costClassOf, latencyDemotion) === "live";
         });
         if (hasLiveInBand)
             return { targets: ordered, status: "bypassed: degraded" };
@@ -1550,12 +1570,12 @@ export function orderByUsability(attempts, breaker = globalCircuitBreaker, now =
     const { ordered } = orderByUsabilityTracked(attempts, breaker, now);
     return ordered;
 }
-function orderByUsabilityTracked(attempts, breaker, now, quotaDemotion, costClassOf) {
+function orderByUsabilityTracked(attempts, breaker, now, quotaDemotion, costClassOf, latencyDemotion) {
     const live = [];
     const faulted = [];
     const cooling = [];
     for (const attempt of attempts) {
-        const usability = targetUsability(attempt, breaker, now, quotaDemotion, costClassOf);
+        const usability = targetUsability(attempt, breaker, now, quotaDemotion, costClassOf, latencyDemotion);
         if (usability === "cooling")
             cooling.push(attempt);
         else if (usability === "credential-fault")
@@ -1574,7 +1594,7 @@ function orderByUsabilityTracked(attempts, breaker, now, quotaDemotion, costClas
             return -1;
         return liftA - liftB;
     });
-    return { ordered: [...live, ...faulted, ...cooling], quotaDemotedFirst: null };
+    return { ordered: [...live, ...faulted, ...cooling], quotaDemotedFirst: null, latencyDemotedFirst: null };
 }
 export function classifyStatus(status) {
     if (status < 400)
@@ -2431,6 +2451,7 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
                     credentialHeaders: credentialTrace.headers(),
                     degraded: degradedLabel(ctx.addressedPool ?? null, ctx.degradedSpecs ?? null, target),
                     quotaDemoted: ctx.quotaDemotedFirst,
+                    latencyDemoted: ctx.latencyDemotedFirst,
                     paid: ctx.cfg ? paidLabel(ctx.cfg, h, target) : null,
                     sticky: ctx.sticky,
                 });
@@ -2542,6 +2563,8 @@ function responseHeadersForTarget(backendRes, ctx) {
         responseHeaders[DEGRADED_HEADER] = ctx.degraded;
     if (ctx.quotaDemoted)
         responseHeaders[QUOTA_DEMOTED_HEADER] = ctx.quotaDemoted;
+    if (ctx.latencyDemoted)
+        responseHeaders[LATENCY_DEMOTED_HEADER] = ctx.latencyDemoted;
     if (ctx.paid)
         responseHeaders[PAID_HEADER] = ctx.paid;
     if (ctx.credentialHeaders)