llm-relay 0.77.0 → 0.78.0

package/dist/candidate-runner.js

@@ -1,5 +1,5 @@
 import { specOfTarget } from "./benchmarks.js";
-import { CREDENTIAL_HEADER, CREDENTIAL_ATTEMPTS_HEADER, POOL_ATTEMPTS_HEADER, HARD_CAP_HEADER, ERROR_ORIGIN_HEADER, SERVED_BY_HEADER, TOOL_DIALECT_HEADER, UNKNOWN_REFUSAL_HEADER, DEGRADED_HEADER, PAID_HEADER, QUOTA_DEMOTED_HEADER, LATENCY_DEMOTED_HEADER, HEDGED_HEADER, dialectRefusalSignalOf, errorOrigin, postHeaderBodyFailure, toolUseIdRewrites, toolCallIdRewrites, thoughtSignatureSentinels, upstreamReportedModel, } from "./backend.js";
+import { CREDENTIAL_HEADER, CREDENTIAL_ATTEMPTS_HEADER, POOL_ATTEMPTS_HEADER, HARD_CAP_HEADER, ERROR_ORIGIN_HEADER, SERVED_BY_HEADER, TOOL_DIALECT_HEADER, UNKNOWN_REFUSAL_HEADER, DEGRADED_HEADER, PAID_HEADER, QUOTA_DEMOTED_HEADER, LATENCY_DEMOTED_HEADER, PROBATION_HEADER, HEDGED_HEADER, dialectRefusalSignalOf, errorOrigin, postHeaderBodyFailure, toolUseIdRewrites, toolCallIdRewrites, thoughtSignatureSentinels, upstreamReportedModel, } from "./backend.js";
 import { STICKY_PROVENANCE_HEADER } from "./session-pin.js";
 import { probeStreamForCommit } from "./stream-commit.js";
 import { DIALECT_REFUSED_DESTRUCTIVE_CODE } from "./tool-dialects.js";
@@ -25,15 +25,21 @@ import { DEFAULT_ANTHROPIC_VERSION } from "./config.js";
 import { buildAuthHeaders } from "./authEnv.js";
 import { CONTROL_AUTHORIZATION_HEADER } from "./control-authorization.js";
 import { STICKY_SESSION_HEADER } from "./session-pin.js";
-import { failClosed, HOP_BY_HOP } from "./stream-pipeline.js";
+import { CrawlAbortedError, failClosed, HOP_BY_HOP } from "./stream-pipeline.js";
 const INTERNAL_REQUEST_HEADERS = new Set([
     "x-codex-turn-metadata",
     "x-llm-relay-dashboard-session",
     STICKY_SESSION_HEADER,
     CONTROL_AUTHORIZATION_HEADER,
 ]);
-const INBOUND_AUTH = ["authorization", "x-api-key"];
-export const MID_STREAM_ERROR_KIND = "backend_stream_failed";
+const ALLOWED_FORWARD_HEADERS = new Set([
+    "content-type",
+    "accept",
+    "anthropic-version",
+    "anthropic-beta",
+]);
+const MID_STREAM_ERROR_KIND = "backend_stream_failed";
+const CRAWL_ERROR_KIND = "backend_stream_crawl";
 export function toolUseIdRewriteField(source) {
     const rewrites = toolUseIdRewrites(source);
     const outbound = toolCallIdRewrites(source);
@@ -52,7 +58,7 @@ export class CredentialConfigError extends Error {
 }
 export function buildForwardHeaders(inbound, attempt) {
     const { target, credential } = attempt;
-    const stripAuth = credential.state !== "not-declared" || target.credentialMode === "contained";
+    const isContained = credential.state !== "not-declared" || target.credentialMode === "contained";
     const out = {};
     for (const [k, v] of Object.entries(inbound)) {
         const key = k.toLowerCase();
@@ -60,7 +66,7 @@ export function buildForwardHeaders(inbound, attempt) {
             continue;
         if (INTERNAL_REQUEST_HEADERS.has(key))
             continue;
-        if (stripAuth && INBOUND_AUTH.includes(key))
+        if (isContained && !ALLOWED_FORWARD_HEADERS.has(key))
             continue;
         if (v === undefined)
             continue;
@@ -77,7 +83,7 @@ export function buildForwardHeaders(inbound, attempt) {
 export const DEFAULT_WALK_BUDGET_MS = 45_000;
 export const DEFAULT_STALL_TIMEOUT_MS = 90_000;
 const MAX_CAPPED_HEADER_CELLS = 5;
-export function filterResponseHeaders(hh) {
+function filterResponseHeaders(hh) {
     const out = {};
     for (const [k, v] of hh.entries()) {
         const lower = k.toLowerCase();
@@ -118,6 +124,8 @@ export function responseHeadersForTarget(backendRes, ctx) {
         responseHeaders[QUOTA_DEMOTED_HEADER] = ctx.quotaDemoted;
     if (ctx.latencyDemoted)
         responseHeaders[LATENCY_DEMOTED_HEADER] = ctx.latencyDemoted;
+    if (ctx.probation)
+        responseHeaders[PROBATION_HEADER] = ctx.probation;
     if (ctx.hedged)
         responseHeaders[HEDGED_HEADER] = ctx.hedged;
     if (ctx.paid)
@@ -131,7 +139,7 @@ export function responseHeadersForTarget(backendRes, ctx) {
     }
     return responseHeaders;
 }
-export function endMidStreamFailure(res, errorFrame, message) {
+function endMidStreamFailure(res, errorFrame, message) {
     if (res.writableEnded || res.destroyed)
         return;
     if (res.headersSent) {
@@ -143,32 +151,38 @@ export function endMidStreamFailure(res, errorFrame, message) {
         failClosed(res, 502, message);
     }
 }
-export function midStreamMessage(e) {
+function midStreamMessage(e) {
     const errStr = e && typeof e === "object" && "message" in e && typeof e.message === "string"
         ? e.message
         : String(e);
     return `llm-relay: backend stream failed mid-response: ${errStr}`;
 }
-export function handleMidStreamError(res, e, started, path, hadTools, streamed, backendStatus, target, attempt, h, errorFrameBuilder, reportedModelSource, deadlineAborted = false, committed = false) {
-    const message = midStreamMessage(e);
+function crawlAbortReason(signal) {
+    return signal?.reason instanceof CrawlAbortedError ? signal.reason : undefined;
+}
+function completeMidStreamAttempt(h, res, attempt, deadlineAborted, committed) {
+    if (!attempt)
+        return;
+    if (res.destroyed) {
+        completeAttemptCancelled(h, attempt, "client disconnected");
+        return;
+    }
+    completeAttemptFailure(h, attempt, {
+        failure: deadlineAborted ? "transport" : "protocol",
+        provenance: deadlineAborted ? "deadline" : "upstream",
+        status: deadlineAborted ? 504 : 502,
+        ...(committed ? { logStatus: "committed" } : {}),
+    });
+}
+export function handleMidStreamError(res, e, started, path, hadTools, streamed, backendStatus, target, attempt, h, errorFrameBuilder, reportedModelSource, deadlineAborted = false, committed = false, signal) {
+    const crawl = crawlAbortReason(signal);
+    const message = crawl ? crawl.message : midStreamMessage(e);
     const errorFrame = errorFrameBuilder(message);
     endMidStreamFailure(res, errorFrame, message);
-    if (attempt) {
-        if (res.destroyed) {
-            completeAttemptCancelled(h, attempt, "client disconnected");
-        }
-        else {
-            completeAttemptFailure(h, attempt, {
-                failure: deadlineAborted ? "transport" : "protocol",
-                provenance: deadlineAborted ? "deadline" : "upstream",
-                status: deadlineAborted ? 504 : 502,
-                ...(committed ? { logStatus: "committed" } : {}),
-            });
-        }
-    }
+    completeMidStreamAttempt(h, res, attempt, deadlineAborted, committed);
     h.logger.write({
         ...baseLog(started, path, hadTools, streamed, backendStatus, "skipped", target, attempt?.trace.snapshot(), reportedModelSource ? upstreamReportedModel(reportedModelSource) : undefined),
-        errorKinds: [MID_STREAM_ERROR_KIND],
+        errorKinds: [crawl ? CRAWL_ERROR_KIND : MID_STREAM_ERROR_KIND],
         ...(reportedModelSource ? toolUseIdRewriteField(reportedModelSource) : {}),
     });
 }
@@ -186,7 +200,58 @@ export function degradedLabel(pool, degraded, target) {
     const spec = specOfTarget(target);
     return degraded.has(spec) ? `${spec} (below ${pool})` : null;
 }
-export function cooledByAllowance(attempt, now, costClass) {
+export const DEFAULT_PROBATION_MIN_SAMPLES = 5;
+export function resolveProbationSettings(settings) {
+    return {
+        enabled: settings?.enabled ?? true,
+        minSamples: settings?.minSamples ?? DEFAULT_PROBATION_MIN_SAMPLES,
+    };
+}
+export function resolveProbation(deps, attempt) {
+    const { enabled, minSamples } = resolveProbationSettings(deps.settings);
+    if (!enabled)
+        return null;
+    const model = attempt.target.model;
+    if (typeof model !== "string" || model.length === 0)
+        return null;
+    let cost;
+    try {
+        cost = deps.costClassOf?.(attempt);
+    }
+    catch {
+        return null;
+    }
+    if (cost !== "free")
+        return null;
+    let samples;
+    try {
+        samples = deps.readRequestSamples(attempt.target.provider, model);
+    }
+    catch {
+        return null;
+    }
+    if (!Number.isFinite(samples) || samples >= minSamples)
+        return null;
+    return { samples: Math.max(0, Math.floor(samples)), minSamples };
+}
+export function createProbationFn(deps) {
+    return (attempt) => {
+        try {
+            return resolveProbation(deps, attempt);
+        }
+        catch {
+            return null;
+        }
+    };
+}
+export function probationLabel(spec, verdict) {
+    return `${spec} (${verdict.samples} of ${verdict.minSamples} request samples)`;
+}
+export function probationLabelForAttempt(h, attempt, now) {
+    const verdict = h.probation?.(attempt, now) ?? null;
+    return verdict ? probationLabel(specOfTarget(attempt.target), verdict) : null;
+}
+function cooledByAllowance(attempt, now, costClass) {
     try {
         const { target } = attempt;
         const until = cooldownUntil(target.provider, attempt.credentialId, target.model ?? null, {
@@ -199,7 +264,7 @@ export function cooledByAllowance(attempt, now, costClass) {
         return false;
     }
 }
-export function cooledByQuota(attempt, breaker, quotaDemotion, now) {
+function cooledByQuota(attempt, breaker, quotaDemotion, now) {
     if (!quotaDemotion)
         return false;
     const spent = quotaDemotion(attempt, now);
@@ -208,7 +273,7 @@ export function cooledByQuota(attempt, breaker, quotaDemotion, now) {
     breaker.recordQuotaCooldown(targetIdentity(attempt), spent.resetsAt, now);
     return true;
 }
-export function targetUsability(attempt, breaker, now, quotaDemotion, costClassOf, latencyDemotion) {
+export function targetUsability(attempt, breaker, now, quotaDemotion, costClassOf, latencyDemotion, probation) {
     const identity = targetIdentity(attempt);
     if (!breaker.isHealthy(identity, now))
         return "cooling";
@@ -220,6 +285,8 @@ export function targetUsability(attempt, breaker, now, quotaDemotion, costClassO
         return "credential-fault";
     if (latencyDemotion?.(attempt, now))
         return "slow";
+    if (probation?.(attempt, now))
+        return "probation";
     return "live";
 }
 export function expandCredentialAttempts(targets) {
@@ -260,7 +327,7 @@ export function credentialEvidence(attempt, cfg, breaker, now) {
         cost,
     };
 }
-export function coolingLiftTime(attempt, breaker, now, costClassOf) {
+function coolingLiftTime(attempt, breaker, now, costClassOf) {
     const identity = targetIdentity(attempt);
     const breakerState = breaker.getState(identity);
     if (breakerState && breakerState.cooldownUntil > now) {
@@ -276,15 +343,19 @@ export function coolingLiftTime(attempt, breaker, now, costClassOf) {
     }
     return null;
 }
-export function orderDeploymentGroupsByUsability(attempts, breaker, now, quotaDemotion, costClassOf, latencyDemotion) {
+export function orderDeploymentGroupsByUsability(attempts, breaker, now, quotaDemotion, costClassOf, latencyDemotion, probation) {
     const preferred = groupCredentialAttempts(attempts)[0]?.attempts[0];
+    const probationGroups = [];
     const live = [];
     const slow = [];
     const faulted = [];
     const cooling = [];
     for (const group of groupCredentialAttempts(attempts)) {
-        const usability = targetUsability(group.attempts[0], breaker, now, quotaDemotion, costClassOf, latencyDemotion);
+        const usability = targetUsability(group.attempts[0], breaker, now, quotaDemotion, costClassOf, latencyDemotion, probation);
         switch (usability) {
+            case "probation":
+                probationGroups.push(group);
+                break;
             case "live":
                 live.push(group);
                 break;
@@ -314,7 +385,7 @@ export function orderDeploymentGroupsByUsability(attempts, breaker, now, quotaDe
             return -1;
         return liftA - liftB;
     });
-    const ordered = [...live, ...slow, ...faulted, ...cooling].flatMap((group) => group.attempts);
+    const ordered = [...probationGroups, ...live, ...slow, ...faulted, ...cooling].flatMap((group) => group.attempts);
     let latencyDemotedFirst = null;
     if (preferred !== undefined &&
         latencyDemotion !== undefined &&
@@ -394,7 +465,7 @@ export class CredentialAttemptTrace {
         return headers;
     }
 }
-export function credentialAttemptLabel(outcome) {
+function credentialAttemptLabel(outcome) {
     if (outcome?.status !== undefined)
         return outcome.status;
     if (outcome?.kind === "provider-transport")
@@ -433,7 +504,18 @@ export function nextUncappedAttempt(h, walk, attemptTrace, tracker) {
         walk.recordRejected(candidate);
     }
 }
-export function beginAttemptRun(res, offer) {
+function firstByteFetch(fetchFn, onFirstByte) {
+    let notified = false;
+    return (input, init) => {
+        const p = fetchFn(input, init);
+        if (!notified) {
+            notified = true;
+            void p.then(onFirstByte, onFirstByte);
+        }
+        return p;
+    };
+}
+export function beginAttemptRun(res, offer, wantsStream) {
     const target = offer.target;
     const controller = new AbortController();
     const callerController = new AbortController();
@@ -447,18 +529,35 @@ export function beginAttemptRun(res, offer) {
         usage: createUsageAccumulator(),
         attempt: undefined,
         egressCallbackCalled: false,
+        fetchFn: fetch,
     };
+    const firstByteTimeoutMs = wantsStream ? undefined : target.firstByteTimeoutMs;
+    if (firstByteTimeoutMs !== undefined) {
+        run.firstByteTimeoutMs = firstByteTimeoutMs;
+        run.firstByteTimer = setTimeout(() => {
+            run.firstByteTimedOut = true;
+            controller.abort();
+        }, firstByteTimeoutMs);
+        run.fetchFn = firstByteFetch(fetch, () => {
+            if (run.firstByteTimer !== undefined) {
+                clearTimeout(run.firstByteTimer);
+                run.firstByteTimer = undefined;
+            }
+        });
+    }
     res.on("close", run.onResClose);
     return run;
 }
 export function releaseAttemptRun(res, run) {
     clearTimeout(run.timer);
+    if (run.firstByteTimer !== undefined)
+        clearTimeout(run.firstByteTimer);
     res.off("close", run.onResClose);
 }
-export function settleResponse(promise) {
+function settleResponse(promise) {
     return promise.then((value) => ({ ok: true, value }), (error) => ({ ok: false, error }));
 }
-export function walkWouldFailOver(response) {
+function walkWouldFailOver(response) {
     return errorOrigin(response) !== "local" && shouldTryNext(classifyStatus(response.status));
 }
 const COMMIT_PROBES = new WeakMap();
@@ -636,21 +735,21 @@ export function respondAllCapped(res, h, ctx, front, tracker, attempts) {
     res.end(JSON.stringify(body));
     h.logger.write(baseLog(ctx.started, ctx.path, ctx.hadTools, ctx.streamed, 429, "skipped", null, attempts));
 }
-export function applyStickyOrdering(ordered, pinnedSpec, breaker, degraded, now, quotaDemotion, costClassOf, latencyDemotion) {
+export function applyStickyOrdering(ordered, pinnedSpec, breaker, degraded, now, quotaDemotion, costClassOf, latencyDemotion, probation) {
     const groups = groupCredentialAttempts(ordered);
     const pinnedIndex = groups.findIndex((group) => specOfTarget(group.attempts[0].target) === pinnedSpec);
     const pinnedGroup = pinnedIndex < 0 ? undefined : groups[pinnedIndex];
     const pinned = pinnedGroup?.attempts[0];
     if (!pinned || !pinnedGroup)
         return { targets: ordered, status: "bypassed: not-in-pool" };
-    const usability = targetUsability(pinned, breaker, now, quotaDemotion, costClassOf, latencyDemotion);
+    const usability = targetUsability(pinned, breaker, now, quotaDemotion, costClassOf, latencyDemotion, probation);
     if (usability !== "live")
         return { targets: ordered, status: `bypassed: ${usability}` };
     if (degraded?.has(pinnedSpec)) {
         const hasLiveInBand = groups.some((group) => {
             const candidate = group.attempts[0];
             return !degraded.has(specOfTarget(candidate.target)) &&
-                targetUsability(candidate, breaker, now, quotaDemotion, costClassOf, latencyDemotion) === "live";
+                targetUsability(candidate, breaker, now, quotaDemotion, costClassOf, latencyDemotion, probation) === "live";
         });
         if (hasLiveInBand)
             return { targets: ordered, status: "bypassed: degraded" };
@@ -661,7 +760,7 @@ export function applyStickyOrdering(ordered, pinnedSpec, breaker, degraded, now,
         .flatMap((group) => group.attempts);
     return { targets: reordered, status: "pinned, reordered" };
 }
-export function stickyHeaderValue(sticky, target, status) {
+function stickyHeaderValue(sticky, target, status) {
     if (!sticky)
         return null;
     if (sticky.provenance)
@@ -678,18 +777,22 @@ export function recordStickySuccess(h, sticky, target, status) {
         return;
     h.stickySessions?.setPin(sticky.key, specOfTarget(target));
 }
-export function orderByUsability(attempts, breaker = globalCircuitBreaker, now = Date.now(), quotaDemotion, costClassOf, latencyDemotion) {
-    const { ordered } = orderByUsabilityTracked(attempts, breaker, now, quotaDemotion, costClassOf, latencyDemotion);
+export function orderByUsability(attempts, breaker = globalCircuitBreaker, now = Date.now(), quotaDemotion, costClassOf, latencyDemotion, probation) {
+    const { ordered } = orderByUsabilityTracked(attempts, breaker, now, quotaDemotion, costClassOf, latencyDemotion, probation);
     return ordered;
 }
-export function orderByUsabilityTracked(attempts, breaker, now, quotaDemotion, costClassOf, latencyDemotion) {
+export function orderByUsabilityTracked(attempts, breaker, now, quotaDemotion, costClassOf, latencyDemotion, probation) {
+    const probationMembers = [];
     const live = [];
     const slow = [];
     const faulted = [];
     const cooling = [];
     for (const attempt of attempts) {
-        const usability = targetUsability(attempt, breaker, now, quotaDemotion, costClassOf, latencyDemotion);
+        const usability = targetUsability(attempt, breaker, now, quotaDemotion, costClassOf, latencyDemotion, probation);
         switch (usability) {
+            case "probation":
+                probationMembers.push(attempt);
+                break;
             case "live":
                 live.push(attempt);
                 break;
@@ -719,7 +822,8 @@ export function orderByUsabilityTracked(attempts, breaker, now, quotaDemotion, c
             return -1;
         return liftA - liftB;
     });
-    return { ordered: [...live, ...slow, ...faulted, ...cooling], quotaDemotedFirst: null, latencyDemotedFirst: null };
+    const ordered = [...probationMembers, ...live, ...slow, ...faulted, ...cooling];
+    return { ordered, quotaDemotedFirst: null, latencyDemotedFirst: null };
 }
 export const STATUS_VERDICT_TABLE = Object.freeze({
     400: { outcome: "retriable", carriesEligibilityFact: true },
@@ -853,7 +957,7 @@ export class Pool429Tracker {
         return Math.min(this.minRetryAfterMs, finalRetryAfterMs ?? Infinity);
     }
 }
-export function recordCall(h, attempt, ok, completedAt) {
+function recordCall(h, attempt, ok, completedAt) {
     const { target, usage } = attempt;
     if (!target.model || !h.modelCallRecorder)
         return;
@@ -915,7 +1019,7 @@ export function beginHealthAttempt(h, resolvedAttempt, started, trace, usage, ac
         completed: false,
     };
 }
-export function abortOnClientClose(res, callerController, controller) {
+function abortOnClientClose(res, callerController, controller) {
     return () => {
         if (!res.writableEnded) {
             callerController.abort();
@@ -923,7 +1027,7 @@ export function abortOnClientClose(res, callerController, controller) {
         }
     };
 }
-export function observeContextLimit(status, target, body) {
+function observeContextLimit(status, target, body) {
     if (status !== 400 && status !== 413)
         return;
     if (target.model === undefined)
@@ -939,7 +1043,7 @@ export function observeContextLimit(status, target, body) {
     catch {
     }
 }
-export function observeMaxOutput(status, target, body) {
+function observeMaxOutput(status, target, body) {
     if (status !== 400 && status !== 413)
         return;
     if (target.model === undefined)
@@ -955,7 +1059,7 @@ export function observeMaxOutput(status, target, body) {
     catch {
     }
 }
-export function observeStatedRateLimits(attempt, observations) {
+function observeStatedRateLimits(attempt, observations) {
     if (observations.length === 0)
         return;
     const { target } = attempt;
@@ -975,7 +1079,7 @@ export function observeStatedRateLimits(attempt, observations) {
     catch {
     }
 }
-export function observeRateLimit(attempt, status, body) {
+function observeRateLimit(attempt, status, body) {
     if (status !== 429)
         return;
     if (attempt.target.model === undefined)
@@ -991,7 +1095,7 @@ export function observeRateLimit(attempt, status, body) {
     catch {
     }
 }
-export function refusalBodyCandidates(body) {
+function refusalBodyCandidates(body) {
     const candidates = [];
     const queued = [body];
     const seen = new Set();
@@ -1196,6 +1300,7 @@ export function markAttemptCommitted(attempt) {
 export function completeAttemptSuccess(h, attempt, status) {
     if (attempt.completed)
         return;
+    const costClass = h.costClassOf?.(attempt.resolvedAttempt);
     const completedAt = Date.now();
     const result = h.breaker.completeAttempt(attempt.handle, {
         terminal: "succeeded",
@@ -1213,7 +1318,7 @@ export function completeAttemptSuccess(h, attempt, status) {
     recordCall(h, attempt, true, completedAt);
     attempt.accounting?.complete(attempt.accountingAttempt, "success", null, attempt.usage, completedAt);
     try {
-        const cleared = clearFacts(attempt.target.provider, attempt.resolvedAttempt.credentialId, attempt.target.model ?? null);
+        const cleared = clearFacts(attempt.target.provider, attempt.resolvedAttempt.credentialId, attempt.target.model ?? null, { costClass });
         if (cleared.includes("credential-invalid")) {
             h.breaker.clearCredentialFaults(attempt.resolvedAttempt.credentialId);
         }
@@ -1246,10 +1351,10 @@ export function completeAttemptFailure(h, attempt, options) {
 export function completeAttemptCancelled(h, attempt, reason) {
     completeCancellation(h, attempt, reason, attempt.committed ? "client-gone-mid-response" : "client-gone-before-response");
 }
-export function completeAttemptAbandoned(h, attempt, reason) {
+function completeAttemptAbandoned(h, attempt, reason) {
     completeCancellation(h, attempt, reason, "relay-abandoned");
 }
-export function completeCancellation(h, attempt, reason, cause) {
+function completeCancellation(h, attempt, reason, cause) {
     if (attempt.completed)
         return;
     const completedAt = Date.now();