llm-relay 0.65.3 → 0.67.0

package/dist/server.js

@@ -13,7 +13,7 @@ import { emitSse, emitSseTail, syntheticMessageId } from "./emitSse.js";
 import { repair, destructiveMatcher } from "./repair.js";
 import { CredentialWalkReshaper, HttpReshaper, } from "./reshaper.js";
 import { DIALECT_REFUSED_DESTRUCTIVE_CODE } from "./tool-dialects.js";
-import { dialectRefusalSignalOf, ERROR_ORIGIN_HEADER, TOOL_DIALECT_HEADER, fetchBackend, fetchOpenAiFront, normalizeOpenAiErrorBody, parseRetryAfterMs, postHeaderBodyFailure, upstreamReportedModel, toolUseIdRewrites, toolCallIdRewrites, thoughtSignatureSentinels, SERVED_BY_HEADER, POOL_ATTEMPTS_HEADER, UNKNOWN_REFUSAL_HEADER, DEGRADED_HEADER, PAID_HEADER, QUOTA_DEMOTED_HEADER, LATENCY_DEMOTED_HEADER, CREDENTIAL_HEADER, CREDENTIAL_ATTEMPTS_HEADER, HARD_CAP_HEADER, errorOrigin } from "./backend.js";
+import { dialectRefusalSignalOf, ERROR_ORIGIN_HEADER, TOOL_DIALECT_HEADER, fetchBackend, fetchOpenAiFront, normalizeOpenAiErrorBody, parseRetryAfterMs, postHeaderBodyFailure, upstreamReportedModel, toolUseIdRewrites, toolCallIdRewrites, thoughtSignatureSentinels, SERVED_BY_HEADER, POOL_ATTEMPTS_HEADER, UNKNOWN_REFUSAL_HEADER, DEGRADED_HEADER, PAID_HEADER, QUOTA_DEMOTED_HEADER, LATENCY_DEMOTED_HEADER, HEDGED_HEADER, CREDENTIAL_HEADER, CREDENTIAL_ATTEMPTS_HEADER, HARD_CAP_HEADER, errorOrigin } from "./backend.js";
 import { probeStreamForCommit } from "./stream-commit.js";
 import { ModelCatalog } from "./catalog.js";
 import { handleAdminRoutes } from "./routes/admin.js";
@@ -42,6 +42,8 @@ import { looksLikeRateLimitError, parseStatedRateLimit, recordObservedRateLimit
 import { clearFacts, cooldownUntil, factsFor, isCostBlocked, recordFact } from "./target-facts.js";
 import { createQuotaDemotionFn, quotaDemotionLabel } from "./quota-demotion.js";
 import { createLatencyDemotionFn, latencyDemotionLabel } from "./latency-demotion.js";
+import { hedgeDelayDecision, resolveHedgeSettings } from "./hedge-trigger.js";
+import { raceWithHedge } from "./hedge-race.js";
 import { createHardCapLedgerReader, evaluateHardCap, hardCapLabel } from "./hard-cap.js";
 import { applyResetRule, interpretRefusal, materializeScope, parseStatedResetMs, recordUnknownRefusal } from "./refusal-interpretation.js";
 import { CONTROL_AUTHORIZATION_HEADER, createControlAuthorization, resolveControlAuthorizationConfigDir, validateControlAuthorization, } from "./control-authorization.js";
@@ -228,6 +230,13 @@ export function createProxy(cfg, deps = {}) {
             return undefined;
         }
     };
+    const hedgeSettings = resolveHedgeSettings(cfg.routing?.hedge);
+    const hedgeDelay = (attempt) => {
+        const t = attempt.target;
+        if (!t.model)
+            return null;
+        return hedgeDelayDecision(pingLoop.getModelPings(t.provider, t.model), costClassOf(attempt) === "free", hedgeSettings);
+    };
     const hardCapEvaluator = (attempt, now) => {
         try {
             const parsed = parseCredentialId(attempt.credentialId);
@@ -329,6 +338,8 @@ export function createProxy(cfg, deps = {}) {
             ...(controlAuthorization ? { controlAuthorization } : {}),
             quotaDemotion,
             latencyDemotion,
+            hedgeDelay,
+            hedgeMaxInFlight: hedgeSettings.enabled ? 2 : 1,
             costClassOf,
             hardCap: hardCapEvaluator,
         }).catch((e) => {
@@ -502,7 +513,7 @@ class RequestAccountingState {
         catch {
         }
     }
-    complete(attempt, outcome, failureKind, usage, endedAt = Date.now()) {
+    complete(attempt, outcome, failureKind, usage, endedAt = Date.now(), abandonedByRelay = false) {
         if (attempt === null)
             return;
         try {
@@ -511,6 +522,7 @@ class RequestAccountingState {
                 failureKind,
                 endedAt,
                 tokens: accountingTokens(usage, attempt.role === "serve" ? this.estimatedInputTokens : 0, attempt.role === "repair" || this.committed.has(attempt)),
+                abandonedByRelay,
             });
         }
         catch {
@@ -869,6 +881,7 @@ async function handle(req, res, cfg, h) {
         walkBudgetMs: cfg.walkBudgetMs ?? DEFAULT_WALK_BUDGET_MS,
         selectionNow: routingNow,
         evidenceFor: (attempt) => credentialEvidence(attempt, cfg, h.breaker, routingNow),
+        maxInFlight: h.hedgeMaxInFlight,
     });
     const credentialTrace = new CredentialAttemptTrace(cfg);
     if (openAiFrontProtocol) {
@@ -909,29 +922,41 @@ async function handle(req, res, cfg, h) {
     const pool429 = new Pool429Tracker();
     const tried = [];
     while (!res.destroyed) {
-        const resolvedAttempt = nextUncappedAttempt(h, credentialWalk, attemptTrace, pool429);
-        if (!resolvedAttempt)
+        const primaryOffer = nextUncappedAttempt(h, credentialWalk, attemptTrace, pool429);
+        if (!primaryOffer)
             break;
-        target = resolvedAttempt.target;
-        const controller = new AbortController();
-        const callerController = new AbortController();
-        const timer = setTimeout(() => controller.abort(), target.timeoutMs);
-        const onResClose = abortOnClientClose(res, callerController, controller);
-        res.on("close", onResClose);
+        const startAttempt = (run, forwardHeaders) => fetchBackend(run.resolvedAttempt, {
+            path,
+            method: req.method ?? "POST",
+            reqBuf,
+            reqJson,
+            anthropicHeaders: forwardHeaders,
+            wantsStream,
+            isDestructive: h.isDestructive,
+            usage: run.usage,
+            signal: run.controller.signal,
+            onEgress: () => {
+                run.egressCallbackCalled = true;
+                pool429.noteEgress();
+                const egressAt = Date.now();
+                run.attempt = beginHealthAttempt(h, run.resolvedAttempt, egressAt, attemptTrace, run.usage, accounting) ?? undefined;
+                if (!run.attempt)
+                    throw new Error("llm-relay: could not begin provider attempt");
+                run.attempt.accountingAttempt = accounting?.startServe(run.resolvedAttempt, egressAt) ?? null;
+                recordCredentialStarted(credentialWalk, credentialTrace, run.resolvedAttempt);
+                tried.push(specOfTarget(run.target));
+            },
+        });
+        const primaryRun = beginAttemptRun(res, primaryOffer);
+        let resolvedAttempt = primaryRun.resolvedAttempt;
+        target = primaryRun.target;
+        let timer = primaryRun.timer;
+        let onResClose = primaryRun.onResClose;
+        let controller;
+        let callerController;
+        let egressCallbackCalled;
+        let hedged;
         let attempt;
-        const usage = createUsageAccumulator();
-        let egressCallbackCalled = false;
-        const onEgress = () => {
-            egressCallbackCalled = true;
-            pool429.noteEgress();
-            const egressAt = Date.now();
-            attempt = beginHealthAttempt(h, resolvedAttempt, egressAt, attemptTrace, usage, accounting) ?? undefined;
-            if (!attempt)
-                throw new Error("llm-relay: could not begin provider attempt");
-            attempt.accountingAttempt = accounting?.startServe(resolvedAttempt, egressAt) ?? null;
-            recordCredentialStarted(credentialWalk, credentialTrace, resolvedAttempt);
-            tried.push(specOfTarget(target));
-        };
         let credentialRecorded = false;
         try {
             let forwardHeaders;
@@ -948,68 +973,87 @@ async function handle(req, res, cfg, h) {
                 throw e;
             }
             let backendRes;
-            try {
-                backendRes = await fetchBackend(resolvedAttempt, {
-                    path,
-                    method: req.method ?? "POST",
-                    reqBuf,
-                    reqJson,
-                    anthropicHeaders: forwardHeaders,
-                    wantsStream,
-                    isDestructive: h.isDestructive,
-                    usage,
-                    signal: controller.signal,
-                    onEgress,
+            {
+                const raced = await runAttemptWithHedge({ run: primaryRun, promise: startAttempt(primaryRun, forwardHeaders) }, {
+                    h,
+                    res,
+                    walk: credentialWalk,
+                    credentialTrace,
+                    attemptTrace,
+                    tracker: pool429,
+                    startRun: (offer) => {
+                        const hedgeRun = beginAttemptRun(res, offer);
+                        try {
+                            return { run: hedgeRun, promise: startAttempt(hedgeRun, buildForwardHeaders(req.headers, offer)) };
+                        }
+                        catch {
+                            releaseAttemptRun(res, hedgeRun);
+                            credentialWalk.recordRejected(offer);
+                            return undefined;
+                        }
+                    },
                 });
-                if (!attempt) {
-                    credentialWalk.recordRejected(resolvedAttempt);
-                    if (errorOrigin(backendRes) === "local") {
-                        await forwardLocalResponse(res, backendRes);
-                        h.logger.write(baseLog(started, path, hadTools, false, backendRes.status, "skipped", null, attemptTrace.snapshot()));
+                resolvedAttempt = raced.run.resolvedAttempt;
+                target = raced.run.target;
+                controller = raced.run.controller;
+                callerController = raced.run.callerController;
+                timer = raced.run.timer;
+                onResClose = raced.run.onResClose;
+                attempt = raced.run.attempt;
+                egressCallbackCalled = raced.run.egressCallbackCalled;
+                hedged = raced.hedged;
+                const settled = raced.settled;
+                if (!settled.ok) {
+                    const e = settled.error;
+                    if (!attempt) {
+                        if (credentialWalk.isPending(resolvedAttempt)) {
+                            credentialWalk.recordRejected(resolvedAttempt);
+                        }
+                        if (res.destroyed)
+                            return;
+                        const status = controller.signal.aborted ? 504 : 502;
+                        failClosed(res, status, egressCallbackCalled
+                            ? "llm-relay: could not begin provider attempt"
+                            : "llm-relay: backend preparation failed");
+                        h.logger.write(baseLog(started, path, hadTools, false, status, "skipped", null, attemptTrace.snapshot()));
+                        return;
                     }
-                    else {
-                        await backendRes.body?.cancel().catch(() => { });
-                        failClosed(res, 502, "llm-relay: backend returned before provider egress");
-                        h.logger.write(baseLog(started, path, hadTools, false, 502, "skipped", null, attemptTrace.snapshot()));
+                    const aborted = controller.signal.aborted;
+                    const status = aborted ? 504 : 502;
+                    if (res.destroyed) {
+                        completeAttemptCancelled(h, attempt, "client disconnected");
+                        recordCredentialOutcome(credentialWalk, credentialTrace, resolvedAttempt, { kind: "cancelled" });
+                        credentialRecorded = true;
+                        return;
                     }
+                    completeAttemptFailure(h, attempt, {
+                        failure: "transport",
+                        provenance: aborted ? "deadline" : "upstream",
+                        status,
+                    });
+                    recordCredentialOutcome(credentialWalk, credentialTrace, resolvedAttempt, aborted ? { kind: "timeout" } : { kind: "provider-transport" });
+                    credentialRecorded = true;
+                    const walkEnd = endWalk(h, res, "anthropic", credentialWalk, attemptTrace, pool429, status, sticky, credentialTrace, () => baseLog(started, path, hadTools, false, status, "skipped", target, attemptTrace.snapshot()), {
+                        kind: "transport",
+                        message: aborted ? "backend timed out" : `backend unreachable: ${e.message}`,
+                    });
+                    if (walkEnd)
+                        continue;
                     return;
                 }
+                backendRes = settled.value;
             }
-            catch (e) {
-                if (!attempt) {
-                    if (credentialWalk.pending === resolvedAttempt) {
-                        credentialWalk.recordRejected(resolvedAttempt);
-                    }
-                    if (res.destroyed)
-                        return;
-                    const status = controller.signal.aborted ? 504 : 502;
-                    failClosed(res, status, egressCallbackCalled
-                        ? "llm-relay: could not begin provider attempt"
-                        : "llm-relay: backend preparation failed");
-                    h.logger.write(baseLog(started, path, hadTools, false, status, "skipped", null, attemptTrace.snapshot()));
-                    return;
+            if (!attempt) {
+                credentialWalk.recordRejected(resolvedAttempt);
+                if (errorOrigin(backendRes) === "local") {
+                    await forwardLocalResponse(res, backendRes);
+                    h.logger.write(baseLog(started, path, hadTools, false, backendRes.status, "skipped", null, attemptTrace.snapshot()));
                 }
-                const aborted = controller.signal.aborted;
-                const status = aborted ? 504 : 502;
-                if (res.destroyed) {
-                    completeAttemptCancelled(h, attempt, "client disconnected");
-                    recordCredentialOutcome(credentialWalk, credentialTrace, resolvedAttempt, { kind: "cancelled" });
-                    credentialRecorded = true;
-                    return;
+                else {
+                    await backendRes.body?.cancel().catch(() => { });
+                    failClosed(res, 502, "llm-relay: backend returned before provider egress");
+                    h.logger.write(baseLog(started, path, hadTools, false, 502, "skipped", null, attemptTrace.snapshot()));
                 }
-                completeAttemptFailure(h, attempt, {
-                    failure: "transport",
-                    provenance: aborted ? "deadline" : "upstream",
-                    status,
-                });
-                recordCredentialOutcome(credentialWalk, credentialTrace, resolvedAttempt, aborted ? { kind: "timeout" } : { kind: "provider-transport" });
-                credentialRecorded = true;
-                const walkEnd = endWalk(h, res, "anthropic", credentialWalk, attemptTrace, pool429, status, sticky, credentialTrace, () => baseLog(started, path, hadTools, false, status, "skipped", target, attemptTrace.snapshot()), {
-                    kind: "transport",
-                    message: aborted ? "backend timed out" : `backend unreachable: ${e.message}`,
-                });
-                if (walkEnd)
-                    continue;
                 return;
             }
             const reportedModelSource = backendRes;
@@ -1133,6 +1177,7 @@ async function handle(req, res, cfg, h) {
                 degraded: degradedLabel(addressedPool, degradedSpecs, target),
                 quotaDemoted: quotaDemotedFirst,
                 latencyDemoted: latencyDemotedFirst,
+                hedged,
                 paid: paidLabel(cfg, h, target),
                 credentialHeaders: backendRes.status < 400
                     ? credentialTrace.headers(resolvedAttempt)
@@ -1201,7 +1246,7 @@ async function handle(req, res, cfg, h) {
                 recordCredentialOutcome(credentialWalk, credentialTrace, resolvedAttempt, res.destroyed ? { kind: "cancelled" } : { kind: "local" });
                 credentialRecorded = true;
             }
-            else if (!credentialRecorded && credentialWalk.pending === resolvedAttempt) {
+            else if (!credentialRecorded && credentialWalk.isPending(resolvedAttempt)) {
                 credentialWalk.recordRejected(resolvedAttempt);
             }
             if (attempt && !attempt.completed) {
@@ -1391,11 +1436,12 @@ class CredentialAttemptTrace {
         });
     }
     record(attempt, outcome) {
-        const pending = [...this.entries].reverse().find((entry) => entry.outcome === undefined);
-        if (!pending ||
-            pending.provider !== attempt.target.provider ||
-            pending.deployment !== specOfTarget(attempt.target) ||
-            pending.credentialId !== attempt.credentialId) {
+        const deployment = specOfTarget(attempt.target);
+        const pending = [...this.entries].reverse().find((entry) => entry.outcome === undefined &&
+            entry.provider === attempt.target.provider &&
+            entry.deployment === deployment &&
+            entry.credentialId === attempt.credentialId);
+        if (!pending) {
             throw new Error("credential attempt trace outcome does not match a started attempt");
         }
         pending.outcome = Object.freeze({ ...outcome });
@@ -1470,6 +1516,95 @@ function nextUncappedAttempt(h, walk, attemptTrace, tracker) {
         walk.recordRejected(candidate);
     }
 }
+function beginAttemptRun(res, offer) {
+    const target = offer.target;
+    const controller = new AbortController();
+    const callerController = new AbortController();
+    const run = {
+        resolvedAttempt: offer,
+        target,
+        controller,
+        callerController,
+        timer: setTimeout(() => controller.abort(), target.timeoutMs),
+        onResClose: abortOnClientClose(res, callerController, controller),
+        usage: createUsageAccumulator(),
+        attempt: undefined,
+        egressCallbackCalled: false,
+    };
+    res.on("close", run.onResClose);
+    return run;
+}
+function releaseAttemptRun(res, run) {
+    clearTimeout(run.timer);
+    res.off("close", run.onResClose);
+}
+function settleResponse(promise) {
+    return promise.then((value) => ({ ok: true, value }), (error) => ({ ok: false, error }));
+}
+function walkWouldFailOver(response) {
+    return errorOrigin(response) !== "local" && shouldTryNext(classifyStatus(response.status));
+}
+function hedgedLabel(primary, hedge, winner, decision) {
+    const side = winner === "hedge" ? "hedge won" : "primary won";
+    return `${specOfTarget(primary.target)} -> ${specOfTarget(hedge.target)} (${side} after ${decision.delayMs}ms, ${decision.basis})`;
+}
+function retireHedgeLoser(deps, loser) {
+    releaseAttemptRun(deps.res, loser);
+    try {
+        loser.controller.abort();
+    }
+    catch {
+    }
+    if (loser.attempt) {
+        completeAttemptAbandoned(deps.h, loser.attempt, "hedge loser aborted");
+        deps.credentialTrace.record(loser.resolvedAttempt, { kind: "cancelled" });
+        deps.walk.recordAbandoned(loser.resolvedAttempt);
+    }
+    else if (deps.walk.isPending(loser.resolvedAttempt)) {
+        deps.walk.recordRejected(loser.resolvedAttempt);
+    }
+}
+async function runAttemptWithHedge(primary, deps) {
+    const decision = deps.h.hedgeDelay(primary.run.resolvedAttempt);
+    if (decision === null) {
+        return { run: primary.run, settled: await settleResponse(primary.promise), hedged: null };
+    }
+    let hedge;
+    let raced;
+    try {
+        raced = await raceWithHedge({
+            primary: { promise: primary.promise, abort: () => primary.run.controller.abort() },
+            delayMs: decision.delayMs,
+            startHedge: () => {
+                if (deps.res.destroyed)
+                    return undefined;
+                const offer = nextUncappedAttempt(deps.h, deps.walk, deps.attemptTrace, deps.tracker);
+                if (!offer)
+                    return undefined;
+                if (offer === primary.run.resolvedAttempt)
+                    return undefined;
+                const started = deps.startRun(offer);
+                if (!started)
+                    return undefined;
+                hedge = started;
+                return { promise: started.promise, abort: () => started.run.controller.abort() };
+            },
+            isWin: (settled) => settled.ok && !walkWouldFailOver(settled.value),
+        });
+    }
+    catch (e) {
+        if (hedge)
+            retireHedgeLoser(deps, hedge.run);
+        throw e;
+    }
+    if (!raced.hedgeStarted || !hedge) {
+        return { run: primary.run, settled: raced.settled, hedged: null };
+    }
+    const winner = raced.winner === "hedge" ? hedge.run : primary.run;
+    const loser = raced.winner === "hedge" ? primary.run : hedge.run;
+    retireHedgeLoser(deps, loser);
+    return { run: winner, settled: raced.settled, hedged: hedgedLabel(primary.run, hedge.run, raced.winner, decision) };
+}
 function walkExitHeaders(tracker, sticky, credentialTrace, { errorOrigin, errorType, servedBy, } = {}) {
     const before = {};
     if (servedBy !== undefined)
@@ -1818,6 +1953,7 @@ function beginHealthAttempt(h, resolvedAttempt, started, trace, usage, accountin
     if (!begun.ok)
         return null;
     return {
+        committed: false,
         handle: begun.value,
         identity,
         resolvedAttempt,
@@ -2091,6 +2227,7 @@ function accountingFailureForAttempt(options) {
     return "provider_error";
 }
 function markAttemptCommitted(attempt) {
+    attempt.committed = true;
     attempt.accounting?.markCommitted(attempt.accountingAttempt, Date.now());
 }
 function completeAttemptSuccess(h, attempt, status) {
@@ -2144,6 +2281,12 @@ function completeAttemptFailure(h, attempt, options) {
     attempt.accounting?.complete(attempt.accountingAttempt, "error", accountingFailureForAttempt(options), attempt.usage, completedAt);
 }
 function completeAttemptCancelled(h, attempt, reason) {
+    completeCancellation(h, attempt, reason, attempt.committed ? "client-gone-mid-response" : "client-gone-before-response");
+}
+function completeAttemptAbandoned(h, attempt, reason) {
+    completeCancellation(h, attempt, reason, "relay-abandoned");
+}
+function completeCancellation(h, attempt, reason, cause) {
     if (attempt.completed)
         return;
     const completedAt = Date.now();
@@ -2153,6 +2296,7 @@ function completeAttemptCancelled(h, attempt, reason) {
         provenance: "client-cancellation",
         completedAt,
         elapsedMs: completedAt - attempt.started,
+        cause,
         reason,
     });
     if (!result.ok)
@@ -2160,7 +2304,7 @@ function completeAttemptCancelled(h, attempt, reason) {
     attempt.completed = true;
     attempt.terminal = "cancelled";
     attempt.trace.record(attempt.target, "cancelled", attempt.started, completedAt);
-    attempt.accounting?.complete(attempt.accountingAttempt, "cancelled", "aborted", attempt.usage, completedAt);
+    attempt.accounting?.complete(attempt.accountingAttempt, "cancelled", "aborted", attempt.usage, completedAt, cause === "relay-abandoned");
 }
 function completePostHeaderBodyFailure(h, downstream, signal, attempt, credentialWalk, credentialTrace, resolvedAttempt) {
     if (downstream.destroyed) {
@@ -2200,45 +2344,13 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
     const directTools = toolSchemaMap(ctx.reqJson);
     const pool429 = new Pool429Tracker();
     while (!res.destroyed) {
-        const resolvedAttempt = nextUncappedAttempt(h, credentialWalk, attemptTrace, pool429);
-        if (!resolvedAttempt)
+        const primaryOffer = nextUncappedAttempt(h, credentialWalk, attemptTrace, pool429);
+        if (!primaryOffer)
             break;
-        const target = resolvedAttempt.target;
-        const controller = new AbortController();
-        const callerController = new AbortController();
-        const timer = setTimeout(() => controller.abort(), target.timeoutMs);
-        const onResClose = abortOnClientClose(res, callerController, controller);
-        res.on("close", onResClose);
-        let attempt;
-        const usage = createUsageAccumulator();
-        let egressCallbackCalled = false;
-        const onEgress = () => {
-            egressCallbackCalled = true;
-            pool429.noteEgress();
-            const egressAt = Date.now();
-            attempt = beginHealthAttempt(h, resolvedAttempt, egressAt, attemptTrace, usage, ctx.accounting) ?? undefined;
-            if (!attempt)
-                throw new Error("llm-relay: could not begin provider attempt");
-            attempt.accountingAttempt = ctx.accounting?.startServe(resolvedAttempt, egressAt) ?? null;
-            recordCredentialStarted(credentialWalk, credentialTrace, resolvedAttempt);
-            tried.push(specOfTarget(target));
-        };
-        let credentialRecorded = false;
-        try {
-            let forwardHeaders;
-            try {
-                forwardHeaders = buildForwardHeaders(ctx.inboundHeaders, resolvedAttempt);
-            }
-            catch (e) {
-                credentialWalk.recordRejected(resolvedAttempt);
-                if (e instanceof CredentialConfigError) {
-                    failClosed(res, 502, `llm-relay configuration: ${e.message}`);
-                    h.logger.write(baseLog(ctx.started, ctx.path, ctx.hadTools, false, 502, "skipped", null, attemptTrace.snapshot()));
-                    return;
-                }
-                throw e;
-            }
+        const audits = new Map();
+        const startAttempt = (run, forwardHeaders) => {
             const recoveryAudit = { value: null };
+            audits.set(run, recoveryAudit);
             const processRecoveredChat = async (recovered) => {
                 const assistant = {
                     content: [
@@ -2264,7 +2376,7 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
                 };
                 if (validation.valid || ctx.cfg?.mode !== "repair")
                     return recovered;
-                const rawReshaper = h.resolveReshaper(resolvedAttempt);
+                const rawReshaper = h.resolveReshaper(run.resolvedAttempt);
                 const reshaper = rawReshaper ? withRepairAccounting(rawReshaper, ctx.accounting) : undefined;
                 if (!reshaper)
                     return recovered;
@@ -2273,8 +2385,8 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
                     reshaper,
                     maxAttempts: ctx.cfg.repair.maxAttempts,
                     isDestructive: h.isDestructive,
-                    backendModel: target.model ?? null,
-                    signal: callerController.signal,
+                    backendModel: run.target.model ?? null,
+                    signal: run.callerController.signal,
                 });
                 recoveryAudit.value.repair = decision.outcome;
                 if (decision.outcome !== "fixed" || !decision.message) {
@@ -2299,69 +2411,136 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
                 }
                 return fixed;
             };
-            let upstream;
+            return fetchOpenAiFront(run.resolvedAttempt, {
+                reqJson: ctx.reqJson,
+                wantsStream: ctx.wantsStream,
+                protocol: ctx.protocol,
+                anthropicHeaders: forwardHeaders,
+                signal: run.controller.signal,
+                isDestructive: h.isDestructive,
+                processRecoveredChat,
+                usage: run.usage,
+                onEgress: () => {
+                    run.egressCallbackCalled = true;
+                    pool429.noteEgress();
+                    const egressAt = Date.now();
+                    run.attempt = beginHealthAttempt(h, run.resolvedAttempt, egressAt, attemptTrace, run.usage, ctx.accounting) ?? undefined;
+                    if (!run.attempt)
+                        throw new Error("llm-relay: could not begin provider attempt");
+                    run.attempt.accountingAttempt = ctx.accounting?.startServe(run.resolvedAttempt, egressAt) ?? null;
+                    recordCredentialStarted(credentialWalk, credentialTrace, run.resolvedAttempt);
+                    tried.push(specOfTarget(run.target));
+                },
+            });
+        };
+        const primaryRun = beginAttemptRun(res, primaryOffer);
+        let resolvedAttempt = primaryRun.resolvedAttempt;
+        let target = primaryRun.target;
+        let timer = primaryRun.timer;
+        let onResClose = primaryRun.onResClose;
+        let controller;
+        let egressCallbackCalled;
+        let hedged;
+        let recoveryAudit;
+        let attempt;
+        let credentialRecorded = false;
+        try {
+            let forwardHeaders;
             try {
-                upstream = await fetchOpenAiFront(resolvedAttempt, {
-                    reqJson: ctx.reqJson,
-                    wantsStream: ctx.wantsStream,
-                    protocol: ctx.protocol,
-                    anthropicHeaders: forwardHeaders,
-                    signal: controller.signal,
-                    isDestructive: h.isDestructive,
-                    processRecoveredChat,
-                    usage,
-                    onEgress,
-                });
-                if (!attempt) {
-                    credentialWalk.recordRejected(resolvedAttempt);
-                    if (errorOrigin(upstream) === "local") {
-                        await forwardLocalResponse(res, upstream);
-                        h.logger.write(baseLog(ctx.started, ctx.path, ctx.hadTools, false, upstream.status, "skipped", null, attemptTrace.snapshot()));
-                    }
-                    else {
-                        await upstream.body?.cancel().catch(() => { });
-                        failClosed(res, 502, "llm-relay: backend returned before provider egress");
-                        h.logger.write(baseLog(ctx.started, ctx.path, ctx.hadTools, false, 502, "skipped", null, attemptTrace.snapshot()));
-                    }
+                forwardHeaders = buildForwardHeaders(ctx.inboundHeaders, resolvedAttempt);
+            }
+            catch (e) {
+                credentialWalk.recordRejected(resolvedAttempt);
+                if (e instanceof CredentialConfigError) {
+                    failClosed(res, 502, `llm-relay configuration: ${e.message}`);
+                    h.logger.write(baseLog(ctx.started, ctx.path, ctx.hadTools, false, 502, "skipped", null, attemptTrace.snapshot()));
                     return;
                 }
+                throw e;
             }
-            catch (e) {
-                if (!attempt) {
-                    if (credentialWalk.pending === resolvedAttempt) {
-                        credentialWalk.recordRejected(resolvedAttempt);
+            let upstream;
+            {
+                const raced = await runAttemptWithHedge({ run: primaryRun, promise: startAttempt(primaryRun, forwardHeaders) }, {
+                    h,
+                    res,
+                    walk: credentialWalk,
+                    credentialTrace,
+                    attemptTrace,
+                    tracker: pool429,
+                    startRun: (offer) => {
+                        const hedgeRun = beginAttemptRun(res, offer);
+                        try {
+                            return { run: hedgeRun, promise: startAttempt(hedgeRun, buildForwardHeaders(ctx.inboundHeaders, offer)) };
+                        }
+                        catch {
+                            releaseAttemptRun(res, hedgeRun);
+                            credentialWalk.recordRejected(offer);
+                            return undefined;
+                        }
+                    },
+                });
+                resolvedAttempt = raced.run.resolvedAttempt;
+                target = raced.run.target;
+                controller = raced.run.controller;
+                timer = raced.run.timer;
+                onResClose = raced.run.onResClose;
+                attempt = raced.run.attempt;
+                egressCallbackCalled = raced.run.egressCallbackCalled;
+                hedged = raced.hedged;
+                recoveryAudit = audits.get(raced.run) ?? { value: null };
+                const settled = raced.settled;
+                if (!settled.ok) {
+                    const e = settled.error;
+                    if (!attempt) {
+                        if (credentialWalk.isPending(resolvedAttempt)) {
+                            credentialWalk.recordRejected(resolvedAttempt);
+                        }
+                        if (res.destroyed)
+                            return;
+                        const status = controller.signal.aborted ? 504 : 502;
+                        failClosed(res, status, egressCallbackCalled
+                            ? "llm-relay: could not begin provider attempt"
+                            : "llm-relay: backend preparation failed");
+                        h.logger.write(baseLog(ctx.started, ctx.path, ctx.hadTools, false, status, "skipped", null, attemptTrace.snapshot()));
+                        return;
                     }
-                    if (res.destroyed)
+                    const aborted = controller.signal.aborted;
+                    const status = aborted ? 504 : 502;
+                    if (res.destroyed) {
+                        completeAttemptCancelled(h, attempt, "client disconnected");
+                        recordCredentialOutcome(credentialWalk, credentialTrace, resolvedAttempt, { kind: "cancelled" });
+                        credentialRecorded = true;
                         return;
-                    const status = controller.signal.aborted ? 504 : 502;
-                    failClosed(res, status, egressCallbackCalled
-                        ? "llm-relay: could not begin provider attempt"
-                        : "llm-relay: backend preparation failed");
-                    h.logger.write(baseLog(ctx.started, ctx.path, ctx.hadTools, false, status, "skipped", null, attemptTrace.snapshot()));
-                    return;
-                }
-                const aborted = controller.signal.aborted;
-                const status = aborted ? 504 : 502;
-                if (res.destroyed) {
-                    completeAttemptCancelled(h, attempt, "client disconnected");
-                    recordCredentialOutcome(credentialWalk, credentialTrace, resolvedAttempt, { kind: "cancelled" });
+                    }
+                    completeAttemptFailure(h, attempt, {
+                        failure: "transport",
+                        provenance: aborted ? "deadline" : "upstream",
+                        status,
+                    });
+                    recordCredentialOutcome(credentialWalk, credentialTrace, resolvedAttempt, aborted ? { kind: "timeout" } : { kind: "provider-transport" });
                     credentialRecorded = true;
+                    const walkEnd = endWalk(h, res, "openai", credentialWalk, attemptTrace, pool429, status, ctx.sticky, credentialTrace, () => baseLog(ctx.started, ctx.path, ctx.hadTools, false, status, "skipped", target, attemptTrace.snapshot()), {
+                        kind: "transport",
+                        message: aborted ? "backend timed out" : `backend unreachable: ${e.message}`,
+                        servedBy: tried.join(", "),
+                    });
+                    if (walkEnd)
+                        continue;
                     return;
                 }
-                completeAttemptFailure(h, attempt, {
-                    failure: "transport",
-                    provenance: aborted ? "deadline" : "upstream",
-                    status,
-                });
-                recordCredentialOutcome(credentialWalk, credentialTrace, resolvedAttempt, aborted ? { kind: "timeout" } : { kind: "provider-transport" });
-                credentialRecorded = true;
-                const walkEnd = endWalk(h, res, "openai", credentialWalk, attemptTrace, pool429, status, ctx.sticky, credentialTrace, () => baseLog(ctx.started, ctx.path, ctx.hadTools, false, status, "skipped", target, attemptTrace.snapshot()), {
-                    kind: "transport",
-                    message: aborted ? "backend timed out" : `backend unreachable: ${e.message}`,
-                    servedBy: tried.join(", "),
-                });
-                if (walkEnd)
-                    continue;
+                upstream = settled.value;
+            }
+            if (!attempt) {
+                credentialWalk.recordRejected(resolvedAttempt);
+                if (errorOrigin(upstream) === "local") {
+                    await forwardLocalResponse(res, upstream);
+                    h.logger.write(baseLog(ctx.started, ctx.path, ctx.hadTools, false, upstream.status, "skipped", null, attemptTrace.snapshot()));
+                }
+                else {
+                    await upstream.body?.cancel().catch(() => { });
+                    failClosed(res, 502, "llm-relay: backend returned before provider egress");
+                    h.logger.write(baseLog(ctx.started, ctx.path, ctx.hadTools, false, 502, "skipped", null, attemptTrace.snapshot()));
+                }
                 return;
             }
             const reportedModelSource = upstream;
@@ -2479,6 +2658,7 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
                     degraded: degradedLabel(ctx.addressedPool ?? null, ctx.degradedSpecs ?? null, target),
                     quotaDemoted: ctx.quotaDemotedFirst,
                     latencyDemoted: ctx.latencyDemotedFirst,
+                    hedged,
                     paid: ctx.cfg ? paidLabel(ctx.cfg, h, target) : null,
                     sticky: ctx.sticky,
                 });
@@ -2542,7 +2722,7 @@ async function openAiFrontPath(res, credentialWalk, credentialTrace, ctx, h) {
                 recordCredentialOutcome(credentialWalk, credentialTrace, resolvedAttempt, res.destroyed ? { kind: "cancelled" } : { kind: "local" });
                 credentialRecorded = true;
             }
-            else if (!credentialRecorded && credentialWalk.pending === resolvedAttempt) {
+            else if (!credentialRecorded && credentialWalk.isPending(resolvedAttempt)) {
                 credentialWalk.recordRejected(resolvedAttempt);
             }
             if (attempt && !attempt.completed) {
@@ -2592,6 +2772,8 @@ function responseHeadersForTarget(backendRes, ctx) {
         responseHeaders[QUOTA_DEMOTED_HEADER] = ctx.quotaDemoted;
     if (ctx.latencyDemoted)
         responseHeaders[LATENCY_DEMOTED_HEADER] = ctx.latencyDemoted;
+    if (ctx.hedged)
+        responseHeaders[HEDGED_HEADER] = ctx.hedged;
     if (ctx.paid)
         responseHeaders[PAID_HEADER] = ctx.paid;
     if (ctx.credentialHeaders)