@juspay/neurolink 10.8.4 → 10.8.6

package/dist/lib/server/routes/claudeProxyRoutes.js

@@ -28,10 +28,12 @@ import { createSSEInterceptor } from "../../proxy/sseInterceptor.js";
 import { createStreamTerminalOutcomeTracker, mergeStreamTerminalOutcome, preflightAnthropicStream, } from "../../proxy/streamOutcome.js";
 import { isPermanentRefreshFailure, needsRefresh, persistTokens, refreshToken, refreshTokenFromLatest, } from "../../proxy/tokenRefresh.js";
 import { buildProxyTranslationPlan, parseRetryAfterMs, } from "../../proxy/routingPolicy.js";
+import { DEFAULT_MAX_INFLIGHT_PER_ACCOUNT, MAX_MAX_INFLIGHT_PER_ACCOUNT, MIN_MAX_INFLIGHT_PER_ACCOUNT, } from "../../proxy/modelRouter.js";
 import { writeJsonSnapshotAtomically } from "../../proxy/snapshotPersistence.js";
 import { recordAttempt, recordAttemptError, recordFinalError, recordFinalSuccess, } from "../../proxy/usageStats.js";
+import { sanitizeForLog } from "../../utils/logSanitize.js";
 import { logger } from "../../utils/logger.js";
-import { withTimeout } from "../../utils/async/withTimeout.js";
+import { raceWithAbort, withTimeout } from "../../utils/async/withTimeout.js";
 import { ProviderHealthChecker } from "../../utils/providerHealth.js";
 // ---------------------------------------------------------------------------
 // Helpers
@@ -97,6 +99,153 @@ const transientRateLimitRetryBudgets = new Map();
  * transient cooldown. This converts a local 429/retry storm into a bounded
  * queue and avoids releasing every waiting request at the same millisecond. */
 const transientCooldownAdmissionSchedules = new Map();
+/**
+ * Central per-account admission protects an OAuth account from a concurrent
+ * request herd. A lease stays held until a JSON response completes or an SSE
+ * response reaches a terminal state, so starting a stream cannot immediately
+ * make room for another stream on the same account.
+ */
+const accountAdmissionStates = new Map();
+function getAccountAdmissionState(accountKey) {
+    let state = accountAdmissionStates.get(accountKey);
+    if (!state) {
+        state = { active: 0, waiters: [] };
+        accountAdmissionStates.set(accountKey, state);
+    }
+    return state;
+}
+function normalizeAccountAdmissionCapacity(capacity) {
+    return Number.isInteger(capacity) &&
+        capacity >= MIN_MAX_INFLIGHT_PER_ACCOUNT &&
+        capacity <= MAX_MAX_INFLIGHT_PER_ACCOUNT
+        ? capacity
+        : DEFAULT_MAX_INFLIGHT_PER_ACCOUNT;
+}
+function drainAccountAdmissionWaiters(accountKey, state) {
+    while (state.waiters.length > 0 && state.active < state.waiters[0].capacity) {
+        const waiter = state.waiters.shift();
+        if (!waiter) {
+            return;
+        }
+        state.active += 1;
+        waiter.resolve(createAccountAdmissionLease(accountKey, state));
+    }
+}
+function createAccountAdmissionLease(accountKey, state) {
+    let released = false;
+    return {
+        release: () => {
+            if (released) {
+                return;
+            }
+            released = true;
+            state.active = Math.max(0, state.active - 1);
+            drainAccountAdmissionWaiters(accountKey, state);
+            discardAccountAdmissionState(accountKey, state);
+        },
+    };
+}
+function discardAccountAdmissionState(accountKey, state) {
+    if (state.active === 0 && state.waiters.length === 0) {
+        accountAdmissionStates.delete(accountKey);
+    }
+}
+function tryAcquireAccountAdmission(accountKey, capacity) {
+    const state = getAccountAdmissionState(accountKey);
+    const normalizedCapacity = normalizeAccountAdmissionCapacity(capacity);
+    if (state.waiters.length > 0 || state.active >= normalizedCapacity) {
+        return undefined;
+    }
+    state.active += 1;
+    return createAccountAdmissionLease(accountKey, state);
+}
+function isAccountAdmissionAvailable(accountKey, capacity) {
+    const state = accountAdmissionStates.get(accountKey);
+    const normalizedCapacity = normalizeAccountAdmissionCapacity(capacity);
+    return (!state || (state.waiters.length === 0 && state.active < normalizedCapacity));
+}
+function enqueueAccountAdmission(accountKey, capacity) {
+    const state = getAccountAdmissionState(accountKey);
+    const normalizedCapacity = normalizeAccountAdmissionCapacity(capacity);
+    let queued = true;
+    let grantedLease;
+    let resolveAdmission;
+    const promise = new Promise((resolve) => {
+        resolveAdmission = resolve;
+    });
+    const waiter = {
+        capacity: normalizedCapacity,
+        resolve: (lease) => {
+            queued = false;
+            grantedLease = lease;
+            resolveAdmission(lease);
+        },
+    };
+    state.waiters.push(waiter);
+    drainAccountAdmissionWaiters(accountKey, state);
+    return {
+        accountKey,
+        promise,
+        cancel: () => {
+            if (grantedLease) {
+                const lease = grantedLease;
+                grantedLease = undefined;
+                lease.release();
+                return;
+            }
+            if (!queued) {
+                return;
+            }
+            queued = false;
+            const index = state.waiters.indexOf(waiter);
+            if (index >= 0) {
+                state.waiters.splice(index, 1);
+                drainAccountAdmissionWaiters(accountKey, state);
+            }
+            discardAccountAdmissionState(accountKey, state);
+        },
+    };
+}
+async function acquireAccountAdmission(accountKey, capacity, abortSignal, timeoutMs = MAX_TRANSIENT_QUEUE_WAIT_MS) {
+    const queued = enqueueAccountAdmission(accountKey, capacity);
+    try {
+        return await withTimeout(raceWithAbort(queued.promise, abortSignal), timeoutMs, `Account admission for ${accountKey} timed out after ${timeoutMs}ms`);
+    }
+    catch (error) {
+        queued.cancel();
+        if (abortSignal?.aborted) {
+            throw error;
+        }
+        return undefined;
+    }
+}
+async function acquireFirstAvailableAccountAdmission(accountKeys, capacity, abortSignal, timeoutMs = MAX_TRANSIENT_QUEUE_WAIT_MS) {
+    const queuedAdmissions = [...new Set(accountKeys)].map((accountKey) => enqueueAccountAdmission(accountKey, capacity));
+    let winnerKey;
+    try {
+        return await withTimeout(raceWithAbort(Promise.race(queuedAdmissions.map((queued) => queued.promise.then((lease) => {
+            if (winnerKey) {
+                lease.release();
+                return new Promise(() => undefined);
+            }
+            winnerKey = queued.accountKey;
+            return { accountKey: queued.accountKey, lease };
+        }))), abortSignal), timeoutMs, `Account admission timed out after ${timeoutMs}ms`);
+    }
+    catch (error) {
+        if (abortSignal?.aborted) {
+            throw error;
+        }
+        return undefined;
+    }
+    finally {
+        for (const queued of queuedAdmissions) {
+            if (queued.accountKey !== winnerKey) {
+                queued.cancel();
+            }
+        }
+    }
+}
 /** Track whether we've run the one-time startup prune. */
 let startupPruneDone = false;
 /** Default cooling period when retries are exhausted and upstream didn't
@@ -1124,7 +1273,7 @@ async function handleTranslatedClaudeRequest(args) {
     const plan = buildProxyTranslationPlan({
         provider: route.provider,
         model: route.model,
-    }, modelRouter?.getFallbackChain() ?? [], body.model, parsed);
+    }, modelRouter?.getFallbackChain() ?? [], body.model, parsed, modelRouter?.isAutoFallbackEnabled?.() ?? false);
     logProxyRoutingPlan(logProxyBody, "translated_request", plan);
     const attempts = plan.attempts;
     if (body.stream) {
@@ -1960,7 +2109,7 @@ async function executeClaudeFallbackWithRetry(args) {
                 throw error;
             }
             const delayMs = TRANSIENT_SAME_ACCOUNT_RETRY_DELAYS_MS[retry] ?? 250;
-            logger.always(`[proxy] retrying fallback=${args.providerLabel} after transient network error (${retry + 1}/${MAX_FALLBACK_NETWORK_RETRIES}) in ${delayMs}ms: ${describeTransportError(error)}`);
+            logger.always(`[proxy] retrying fallback=${args.providerLabel} after transient network error (${retry + 1}/${MAX_FALLBACK_NETWORK_RETRIES}) in ${delayMs}ms: ${redactProviderErrorMessage(describeTransportError(error))}`);
             await sleep(delayMs);
         }
     }
@@ -1990,7 +2139,17 @@ async function tryConfiguredClaudeFallbackChain(args) {
         }
         const availability = await ProviderHealthChecker.checkFallbackProviderAvailability(fallback.provider, fallback.model);
         if (!availability.available) {
-            logger.always(`[proxy] fallback ${fallback.provider}/${fallback.model} health-check failed (${availability.reason ?? "provider unavailable"}), attempting anyway`);
+            const reason = availability.reason ?? "provider unavailable";
+            logger.always(`[proxy] fallback ${fallback.provider}/${fallback.model} health-check failed (${reason}), skipping`);
+            recordFallbackAttempt({
+                provider: fallback.provider,
+                model: fallback.model,
+                status: "failure",
+                errorMessage: `[unavailable] ${reason}`,
+                durationMs: 0,
+            });
+            lastFallbackError = `[${fallback.provider}/${fallback.model}] unavailable: ${reason}`;
+            continue;
         }
         const fallbackStart = Date.now();
         try {
@@ -2025,9 +2184,9 @@ async function tryConfiguredClaudeFallbackChain(args) {
             return { response };
         }
         catch (fallbackErr) {
-            const errMsg = fallbackErr instanceof Error
+            const errMsg = redactProviderErrorMessage(fallbackErr instanceof Error
                 ? fallbackErr.message
-                : String(fallbackErr);
+                : String(fallbackErr));
             let errorClass = "unknown";
             if (errMsg.includes("Rate limit") ||
                 errMsg.includes("rate_limit") ||
@@ -2049,7 +2208,7 @@ async function tryConfiguredClaudeFallbackChain(args) {
             else if (errMsg.includes("Resource exhausted")) {
                 errorClass = "provider_quota";
             }
-            logger.always(`[proxy] fallback ${fallback.provider}/${fallback.model} failed [${errorClass}]: ${describeTransportError(fallbackErr)}`);
+            logger.always(`[proxy] fallback ${fallback.provider}/${fallback.model} failed [${errorClass}]: ${redactProviderErrorMessage(describeTransportError(fallbackErr))}`);
             recordFallbackAttempt({
                 provider: fallback.provider,
                 model: fallback.model,
@@ -2057,7 +2216,7 @@ async function tryConfiguredClaudeFallbackChain(args) {
                 errorMessage: `[${errorClass}] ${errMsg}`,
                 durationMs: Date.now() - fallbackStart,
             });
-            lastFallbackError = `[${fallback.provider}/${fallback.model}] ${describeTransportError(fallbackErr)}`;
+            lastFallbackError = `[${fallback.provider}/${fallback.model}] ${redactProviderErrorMessage(describeTransportError(fallbackErr))}`;
         }
     }
     return { response: null, lastErrorMessage: lastFallbackError };
@@ -2067,7 +2226,7 @@ async function tryAutoClaudeFallback(args) {
     const fallbackStart = Date.now();
     try {
         const parsed = parseClaudeRequest(body);
-        const plan = buildProxyTranslationPlan({ provider: "anthropic", model: body.model }, [], body.model, parsed);
+        const plan = buildProxyTranslationPlan({ provider: "anthropic", model: body.model }, [], body.model, parsed, true);
         logProxyRoutingPlan(logProxyBody, "auto_fallback", plan);
         const autoAttempt = plan.attempts.find((attempt) => attempt.label === "auto-provider");
         if (!autoAttempt) {
@@ -2101,7 +2260,7 @@ async function tryAutoClaudeFallback(args) {
         return { response };
     }
     catch (fallbackErr) {
-        const errorMessage = describeTransportError(fallbackErr);
+        const errorMessage = redactProviderErrorMessage(describeTransportError(fallbackErr));
         logger.always(`[proxy] fallback auto-provider failed: ${errorMessage}`);
         recordFallbackAttempt({
             provider: "auto-provider",
@@ -2219,7 +2378,7 @@ function buildClaudeAnthropicFailureResponse(args) {
     });
 }
 async function handleAnthropicSuccessfulResponse(args) {
-    const { ctx, body, account, accountState, response, tracer, requestStartTime, fetchStartMs, attemptNumber, finalBodyStr, upstreamSpan, logAttempt, logProxyBody, logFinalRequest, } = args;
+    const { ctx, body, account, accountState, response, tracer, requestStartTime, fetchStartMs, attemptNumber, finalBodyStr, upstreamSpan, logAttempt, logProxyBody, onStreamTerminal, logFinalRequest, } = args;
     accountState.consecutiveRefreshFailures = 0;
     logger.always(`[proxy] ← ${response.status} account=${account.label}`);
     const quota = parseQuotaHeaders(response.headers);
@@ -2262,6 +2421,7 @@ async function handleAnthropicSuccessfulResponse(args) {
             logAttempt,
             logProxyBody,
             logFinalRequest,
+            onStreamTerminal,
         });
     }
     return handleAnthropicJsonSuccessResponse({
@@ -2280,7 +2440,7 @@ async function handleAnthropicSuccessfulResponse(args) {
     });
 }
 async function handleAnthropicStreamingSuccessResponse(args) {
-    const { account, accountState, response, responseHeaders, tracer, requestStartTime, fetchStartMs, attemptNumber, finalBodyStr, upstreamSpan, logAttempt, logProxyBody, logFinalRequest, } = args;
+    const { account, accountState, response, responseHeaders, tracer, requestStartTime, fetchStartMs, attemptNumber, finalBodyStr, upstreamSpan, logAttempt, logProxyBody, onStreamTerminal, logFinalRequest, } = args;
     if (!response.body) {
         recordAttemptError(account.label, account.type, 502);
         logAttempt(502, "stream_error", "No response body from upstream");
@@ -2467,7 +2627,7 @@ async function handleAnthropicStreamingSuccessResponse(args) {
             return reader.cancel();
         },
     });
-    const result = attachAnthropicSuccessStreamTelemetry({
+    const { response: result, telemetryDone } = attachAnthropicSuccessStreamTelemetry({
         account,
         response,
         responseHeaders,
@@ -2481,7 +2641,8 @@ async function handleAnthropicStreamingSuccessResponse(args) {
         logProxyBody,
         logFinalRequest,
     });
-    return { response: result };
+    void telemetryDone.then(() => onStreamTerminal?.(), () => onStreamTerminal?.());
+    return { response: result, holdsAccountAdmission: true };
 }
 function getStreamFailureDetails(outcome) {
     if (outcome.kind === "upstream_error") {
@@ -2509,6 +2670,7 @@ function attachAnthropicSuccessStreamTelemetry(args) {
     const { account, response, responseHeaders, remainingStream, streamOutcome, tracer, requestStartTime, attemptNumber, finalBodyStr, upstreamSpan, logProxyBody, logFinalRequest, } = args;
     const { stream: clientCaptureStream, capture: clientCapture } = createRawStreamCapture();
     let streamSource = remainingStream;
+    let telemetryDone;
     if (tracer) {
         try {
             const { stream: interceptor, telemetry } = createSSEInterceptor({
@@ -2520,7 +2682,7 @@ function attachAnthropicSuccessStreamTelemetry(args) {
             const capturedResponse = response;
             const capturedRequestBytes = finalBodyStr.length;
             const capturedAccountLabel = account.label;
-            Promise.all([telemetry, clientCapture, streamOutcome])
+            telemetryDone = Promise.all([telemetry, clientCapture, streamOutcome])
                 .then(([data, clientBody, rawOutcome]) => {
                 const terminalOutcome = mergeStreamTerminalOutcome(rawOutcome, data.streamErrorMessage);
                 recordCommittedAnthropicStreamAttemptFailure(terminalOutcome, account);
@@ -2604,7 +2766,8 @@ function attachAnthropicSuccessStreamTelemetry(args) {
         catch {
             // Interceptor attachment failed after stream setup. Preserve delivery but
             // still settle the request from the actual stream terminal outcome.
-            streamOutcome.then((outcome) => {
+            telemetryDone = streamOutcome
+                .then((outcome) => {
                 recordCommittedAnthropicStreamAttemptFailure(outcome, account);
                 const failure = getStreamFailureDetails(outcome);
                 upstreamSpan?.end();
@@ -2617,7 +2780,8 @@ function attachAnthropicSuccessStreamTelemetry(args) {
                     tracer.end(response.status, Date.now() - requestStartTime);
                     logFinalRequest(response.status, account.label, account.type);
                 }
-            });
+            })
+                .catch(() => undefined);
         }
     }
     else {
@@ -2628,7 +2792,11 @@ function attachAnthropicSuccessStreamTelemetry(args) {
             });
             streamSource = streamSource.pipeThrough(noTracerInterceptor);
             const capturedAccountLabel = account.label;
-            Promise.all([noTracerTelemetry, clientCapture, streamOutcome])
+            telemetryDone = Promise.all([
+                noTracerTelemetry,
+                clientCapture,
+                streamOutcome,
+            ])
                 .then(([data, clientBody, rawOutcome]) => {
                 const terminalOutcome = mergeStreamTerminalOutcome(rawOutcome, data.streamErrorMessage);
                 recordCommittedAnthropicStreamAttemptFailure(terminalOutcome, account);
@@ -2694,7 +2862,8 @@ function attachAnthropicSuccessStreamTelemetry(args) {
                 .catch(() => {
                 // Non-fatal
             });
-            streamOutcome.then((outcome) => {
+            telemetryDone = streamOutcome
+                .then((outcome) => {
                 recordCommittedAnthropicStreamAttemptFailure(outcome, account);
                 const failure = getStreamFailureDetails(outcome);
                 if (failure) {
@@ -2703,7 +2872,8 @@ function attachAnthropicSuccessStreamTelemetry(args) {
                 else {
                     logFinalRequest(response.status, account.label, account.type);
                 }
-            });
+            })
+                .catch(() => undefined);
         }
     }
     const clientStream = streamSource.pipeThrough(clientCaptureStream);
@@ -2724,10 +2894,13 @@ function attachAnthropicSuccessStreamTelemetry(args) {
             clientResponseHeaders[headerName] = value;
         }
     }
-    return new Response(clientStream, {
-        status: response.status,
-        headers: clientResponseHeaders,
-    });
+    return {
+        response: new Response(clientStream, {
+            status: response.status,
+            headers: clientResponseHeaders,
+        }),
+        telemetryDone,
+    };
 }
 async function handleAnthropicJsonSuccessResponse(args) {
     const { account, response, responseHeaders, tracer, requestStartTime, fetchStartMs, attemptNumber, finalBodyStr, upstreamSpan, logAttempt, logProxyBody, logFinalRequest, } = args;
@@ -2902,7 +3075,7 @@ async function handleAnthropicSuccessfulNonStreamRetryResponse(args) {
     return retryJson;
 }
 async function handleAnthropicAuthRetry(args) {
-    const { ctx, body, account, accountState, headers, buildUpstreamBody, url, enabledAccounts, orderedAccounts, tracer, requestStartTime, allocateAttemptNumber, upstreamSpan, logAttempt, logProxyBody, logFinalRequest, lastError, authFailureMessage, sawRateLimit, sawTransientFailure, sawNetworkError, } = args;
+    const { ctx, body, account, accountState, headers, buildUpstreamBody, url, enabledAccounts, orderedAccounts, tracer, requestStartTime, allocateAttemptNumber, upstreamSpan, logAttempt, logProxyBody, onStreamTerminal, logFinalRequest, lastError, authFailureMessage, sawRateLimit, sawTransientFailure, sawNetworkError, } = args;
     recordAttemptError(account.label, account.type, 401);
     logAttempt(401, "authentication_error", "received 401 from Anthropic", {
         retryable: true,
@@ -2986,6 +3159,7 @@ async function handleAnthropicAuthRetry(args) {
                         logAttempt: retryLogAttempt,
                         logProxyBody,
                         logFinalRequest,
+                        onStreamTerminal,
                     })
                     : {
                         response: await handleAnthropicSuccessfulNonStreamRetryResponse({
@@ -3018,6 +3192,7 @@ async function handleAnthropicAuthRetry(args) {
                 }
                 return {
                     response: successResult.response,
+                    holdsAccountAdmission: successResult.holdsAccountAdmission,
                     continueLoop: false,
                     lastError: currentLastError,
                     authFailureMessage: currentAuthFailureMessage,
@@ -3399,9 +3574,10 @@ async function handleAnthropicNonOkResponse(args) {
         };
     }
     if (isTransientHttpFailure(response.status, errBody)) {
+        const upstreamOverload = isUpstreamOverload(response.status, errBody);
         recordAttemptError(account.label, account.type, response.status, response.status === 429 ? "transient" : undefined);
         currentSawTransientFailure = true;
-        logger.always(`[proxy] ← ${response.status} account=${account.label} (transient)`);
+        logger.always(`[proxy] ← ${response.status} account=${account.label} (${upstreamOverload ? "overloaded" : "transient"})`);
         currentLastError = errBody;
         logAttempt(response.status, "api_error", summarizeErrorMessage(errBody), response.status === 429
             ? {
@@ -3410,11 +3586,11 @@ async function handleAnthropicNonOkResponse(args) {
                 cooldownReason: "transient",
             }
             : undefined);
-        tracer?.setError("transient_error", summarizeErrorMessage(errBody));
-        tracer?.recordRetry(account.label, "transient");
+        tracer?.setError(upstreamOverload ? "overloaded_error" : "transient_error", summarizeErrorMessage(errBody));
+        tracer?.recordRetry(account.label, upstreamOverload ? "overloaded" : "transient");
         return {
             continueLoop: true,
-            retrySameAccount: true,
+            retrySameAccount: !upstreamOverload,
             lastError: currentLastError,
             authFailureMessage: currentAuthFailureMessage,
             sawTransientFailure: currentSawTransientFailure,
@@ -4018,6 +4194,15 @@ async function handleAnthropicRoutedClaudeRequest(args) {
             loopState.authCooldownMessage = `All ${orderedAccounts.length} Anthropic accounts are temporarily unavailable while OAuth refresh is cooling. Earliest retry at ${new Date(earliestRetryAt).toISOString()}.`;
         }
     }
+    const accountAdmissionCapacity = modelRouter?.getMaxInflightPerAccount?.() ??
+        DEFAULT_MAX_INFLIGHT_PER_ACCOUNT;
+    // When every eligible account is busy, reserve the first account that frees
+    // instead of arbitrarily waiting behind the last configured account.
+    let queuedAccountAdmission;
+    if (effectiveAccounts.length > 0 &&
+        effectiveAccounts.every((account) => !isAccountAdmissionAvailable(account.key, accountAdmissionCapacity))) {
+        queuedAccountAdmission = await acquireFirstAvailableAccountAdmission(effectiveAccounts.map((account) => account.key), accountAdmissionCapacity, ctx.abortSignal);
+    }
     accountLoop: for (const account of effectiveAccounts) {
         const accountState = getOrCreateRuntimeState(account.key);
         let transientSameAccountRetries = 0;
@@ -4066,229 +4251,264 @@ async function handleAnthropicRoutedClaudeRequest(args) {
                 preparedAttempt.fetchStartMs === undefined) {
                 continue accountLoop;
             }
-            const fetchResult = await fetchAnthropicAccountResponse({
-                url,
-                headers: preparedAttempt.headers,
-                finalBodyStr: preparedAttempt.finalBodyStr,
-                account,
-                accountState,
-                enabledAccounts,
-                orderedAccounts,
-                tracer,
-                logAttempt,
-                logProxyBody,
-                fetchStartMs: preparedAttempt.fetchStartMs,
-                attemptNumber: loopState.attemptNumber,
-                currentLastError: loopState.lastError,
-                currentSawRateLimit: loopState.sawRateLimit,
-                currentSawNetworkError: loopState.sawNetworkError,
-                upstreamSpan: preparedAttempt.upstreamSpan,
-            });
-            loopState.lastError = fetchResult.lastError;
-            loopState.sawRateLimit = fetchResult.sawRateLimit;
-            loopState.sawNetworkError = fetchResult.sawNetworkError;
-            if (fetchResult.terminalError) {
-                return finalizeAnthropicTerminalFetchError({
-                    terminalError: fetchResult.terminalError,
+            let admissionLease;
+            if (queuedAccountAdmission?.accountKey === account.key) {
+                admissionLease = queuedAccountAdmission.lease;
+                queuedAccountAdmission = undefined;
+            }
+            else {
+                admissionLease = tryAcquireAccountAdmission(account.key, accountAdmissionCapacity);
+                if (admissionLease && queuedAccountAdmission) {
+                    // A preferred account became available while the race was being
+                    // established; release the lower-priority reservation.
+                    queuedAccountAdmission.lease.release();
+                    queuedAccountAdmission = undefined;
+                }
+            }
+            if (!admissionLease) {
+                // A later account may be immediately available. Preserve the routing
+                // order but do not leave a request queued behind a busy first choice.
+                continue accountLoop;
+            }
+            let admissionTransferredToStream = false;
+            try {
+                const fetchResult = await fetchAnthropicAccountResponse({
+                    url,
+                    headers: preparedAttempt.headers,
+                    finalBodyStr: preparedAttempt.finalBodyStr,
                     account,
+                    accountState,
+                    enabledAccounts,
+                    orderedAccounts,
                     tracer,
-                    requestStartTime,
-                    attemptNumber: loopState.attemptNumber,
+                    logAttempt,
                     logProxyBody,
-                    logFinalRequest,
+                    fetchStartMs: preparedAttempt.fetchStartMs,
+                    attemptNumber: loopState.attemptNumber,
+                    currentLastError: loopState.lastError,
+                    currentSawRateLimit: loopState.sawRateLimit,
+                    currentSawNetworkError: loopState.sawNetworkError,
+                    upstreamSpan: preparedAttempt.upstreamSpan,
                 });
-            }
-            if (fetchResult.continueLoop || !fetchResult.response) {
-                // Genuine 429 (carries a cooldown plan derived from quota headers).
-                if (fetchResult.cooldownPlan) {
-                    const plan = fetchResult.cooldownPlan;
-                    // Refresh the account's quota snapshot for proactive selection.
-                    if (fetchResult.quota) {
-                        accountState.quota = fetchResult.quota;
-                        saveAccountQuota(account.label, fetchResult.quota).catch(() => {
-                            // Non-fatal: routing already has the in-memory snapshot.
-                        });
-                    }
-                    // Publish the cooldown before retrying so requests arriving behind
-                    // this one skip the throttled account instead of joining the burst.
-                    let cooldownExtended = false;
-                    if (!accountState.coolingUntil ||
-                        plan.coolingUntil > accountState.coolingUntil) {
-                        accountState.coolingUntil = plan.coolingUntil;
-                        accountState.coolingReason = plan.reason;
-                        cooldownExtended = true;
-                    }
-                    if (cooldownExtended) {
-                        await saveAccountCooldown(account.key, accountState.coolingUntil, accountState.coolingReason ?? plan.reason).catch(() => {
-                            // Non-fatal: routing already has the in-memory cooldown.
-                        });
+                loopState.lastError = fetchResult.lastError;
+                loopState.sawRateLimit = fetchResult.sawRateLimit;
+                loopState.sawNetworkError = fetchResult.sawNetworkError;
+                if (fetchResult.terminalError) {
+                    return finalizeAnthropicTerminalFetchError({
+                        terminalError: fetchResult.terminalError,
+                        account,
+                        tracer,
+                        requestStartTime,
+                        attemptNumber: loopState.attemptNumber,
+                        logProxyBody,
+                        logFinalRequest,
+                    });
+                }
+                if (fetchResult.continueLoop || !fetchResult.response) {
+                    // Genuine 429 (carries a cooldown plan derived from quota headers).
+                    if (fetchResult.cooldownPlan) {
+                        const plan = fetchResult.cooldownPlan;
+                        // Refresh the account's quota snapshot for proactive selection.
+                        if (fetchResult.quota) {
+                            accountState.quota = fetchResult.quota;
+                            saveAccountQuota(account.label, fetchResult.quota).catch(() => {
+                                // Non-fatal: routing already has the in-memory snapshot.
+                            });
+                        }
+                        // Publish the cooldown before retrying so requests arriving behind
+                        // this one skip the throttled account instead of joining the burst.
+                        let cooldownExtended = false;
+                        if (!accountState.coolingUntil ||
+                            plan.coolingUntil > accountState.coolingUntil) {
+                            accountState.coolingUntil = plan.coolingUntil;
+                            accountState.coolingReason = plan.reason;
+                            cooldownExtended = true;
+                        }
+                        if (cooldownExtended) {
+                            await saveAccountCooldown(account.key, accountState.coolingUntil, accountState.coolingReason ?? plan.reason).catch(() => {
+                                // Non-fatal: routing already has the in-memory cooldown.
+                            });
+                        }
+                        // Transient retries are budgeted across all concurrent requests for
+                        // this account/window. Exhaustion plans rotate immediately and never
+                        // claim this budget.
+                        const sharedRetrySlot = fetchResult.retrySameAccount
+                            ? claimTransientRateLimitRetry(account.key, plan.coolingUntil)
+                            : undefined;
+                        if (fetchResult.retrySameAccount &&
+                            fetchResult.retryAfterMs !== undefined &&
+                            rateLimitSameAccountRetries <
+                                MAX_RATE_LIMIT_SAME_ACCOUNT_RETRIES &&
+                            sharedRetrySlot !== undefined) {
+                            rateLimitSameAccountRetries += 1;
+                            const base = Math.min(fetchResult.retryAfterMs || 1_000, MAX_RATE_LIMIT_RETRY_DELAY_MS);
+                            // Stagger the two shared slots, then cap after jitter so the final
+                            // sleep never exceeds the configured maximum.
+                            const delayMs = Math.min(MAX_RATE_LIMIT_RETRY_DELAY_MS, jitteredDelay(base * sharedRetrySlot));
+                            logger.always(`[proxy] retrying same account=${account.label} after transient 429 (shared slot ${sharedRetrySlot}/${MAX_RATE_LIMIT_SAME_ACCOUNT_RETRIES}) in ${delayMs}ms`);
+                            await sleep(delayMs);
+                            continue;
+                        }
+                        // Exhaustion, or the shared transient retry budget being used up:
+                        // rotate while the already-published cooldown remains active.
+                        advancePrimaryIfCurrent(account.key, enabledAccounts.length, orderedAccounts[0]?.key);
+                        logger.always(`[proxy] account=${account.label} rate-limited (${plan.reason}); cooling ~${minutesUntil(plan.coolingUntil, Date.now())}m until ${new Date(plan.coolingUntil).toISOString()}, rotating`);
+                        continue accountLoop;
                     }
-                    // Transient retries are budgeted across all concurrent requests for
-                    // this account/window. Exhaustion plans rotate immediately and never
-                    // claim this budget.
-                    const sharedRetrySlot = fetchResult.retrySameAccount
-                        ? claimTransientRateLimitRetry(account.key, plan.coolingUntil)
-                        : undefined;
+                    // Transient error retry (network errors, 529 overloaded)
                     if (fetchResult.retrySameAccount &&
-                        fetchResult.retryAfterMs !== undefined &&
-                        rateLimitSameAccountRetries < MAX_RATE_LIMIT_SAME_ACCOUNT_RETRIES &&
-                        sharedRetrySlot !== undefined) {
-                        rateLimitSameAccountRetries += 1;
-                        const base = Math.min(fetchResult.retryAfterMs || 1_000, MAX_RATE_LIMIT_RETRY_DELAY_MS);
-                        // Stagger the two shared slots, then cap after jitter so the final
-                        // sleep never exceeds the configured maximum.
-                        const delayMs = Math.min(MAX_RATE_LIMIT_RETRY_DELAY_MS, jitteredDelay(base * sharedRetrySlot));
-                        logger.always(`[proxy] retrying same account=${account.label} after transient 429 (shared slot ${sharedRetrySlot}/${MAX_RATE_LIMIT_SAME_ACCOUNT_RETRIES}) in ${delayMs}ms`);
+                        transientSameAccountRetries < MAX_TRANSIENT_SAME_ACCOUNT_RETRIES) {
+                        transientSameAccountRetries += 1;
+                        const delayMs = getTransientSameAccountRetryDelayMs(transientSameAccountRetries);
+                        logger.always(`[proxy] retrying same account=${account.label} after transient network error (${transientSameAccountRetries}/${MAX_TRANSIENT_SAME_ACCOUNT_RETRIES}) in ${delayMs}ms`);
                         await sleep(delayMs);
                         continue;
                     }
-                    // Exhaustion, or the shared transient retry budget being used up:
-                    // rotate while the already-published cooldown remains active.
-                    advancePrimaryIfCurrent(account.key, enabledAccounts.length, orderedAccounts[0]?.key);
-                    logger.always(`[proxy] account=${account.label} rate-limited (${plan.reason}); cooling ~${minutesUntil(plan.coolingUntil, Date.now())}m until ${new Date(plan.coolingUntil).toISOString()}, rotating`);
+                    if (fetchResult.retrySameAccount) {
+                        logger.always(`[proxy] exhausted transient same-account retries for account=${account.label}; rotating`);
+                    }
                     continue accountLoop;
                 }
-                // Transient error retry (network errors, 529 overloaded)
-                if (fetchResult.retrySameAccount &&
-                    transientSameAccountRetries < MAX_TRANSIENT_SAME_ACCOUNT_RETRIES) {
-                    transientSameAccountRetries += 1;
-                    const delayMs = getTransientSameAccountRetryDelayMs(transientSameAccountRetries);
-                    logger.always(`[proxy] retrying same account=${account.label} after transient network error (${transientSameAccountRetries}/${MAX_TRANSIENT_SAME_ACCOUNT_RETRIES}) in ${delayMs}ms`);
-                    await sleep(delayMs);
-                    continue;
+                let upstreamSpan = fetchResult.upstreamSpan;
+                const response = fetchResult.response;
+                if (response.status === 401 &&
+                    account.type === "oauth" &&
+                    account.refreshToken) {
+                    const authRetryResult = await handleAnthropicAuthRetry({
+                        ctx,
+                        body,
+                        account,
+                        accountState,
+                        headers: preparedAttempt.headers,
+                        buildUpstreamBody: preparedAttempt.buildUpstreamBody,
+                        url,
+                        enabledAccounts,
+                        orderedAccounts,
+                        tracer,
+                        requestStartTime,
+                        allocateAttemptNumber: () => {
+                            loopState.attemptNumber += 1;
+                            return loopState.attemptNumber;
+                        },
+                        upstreamSpan,
+                        logAttempt,
+                        logProxyBody,
+                        logFinalRequest,
+                        onStreamTerminal: admissionLease.release,
+                        lastError: loopState.lastError,
+                        authFailureMessage: loopState.authFailureMessage,
+                        sawRateLimit: loopState.sawRateLimit,
+                        sawTransientFailure: loopState.sawTransientFailure,
+                        sawNetworkError: loopState.sawNetworkError,
+                    });
+                    loopState.lastError = authRetryResult.lastError;
+                    loopState.authFailureMessage = authRetryResult.authFailureMessage;
+                    loopState.sawRateLimit = authRetryResult.sawRateLimit;
+                    loopState.sawTransientFailure = authRetryResult.sawTransientFailure;
+                    loopState.sawNetworkError = authRetryResult.sawNetworkError;
+                    upstreamSpan = authRetryResult.upstreamSpan;
+                    if (authRetryResult.response !== undefined) {
+                        admissionTransferredToStream =
+                            authRetryResult.holdsAccountAdmission === true;
+                        return authRetryResult.response;
+                    }
+                    if (authRetryResult.continueLoop) {
+                        continue accountLoop;
+                    }
+                }
+                if (!response.ok) {
+                    const nonOkResult = await handleAnthropicNonOkResponse({
+                        response,
+                        account,
+                        accountState,
+                        enabledAccounts,
+                        orderedAccounts,
+                        tracer,
+                        requestStartTime,
+                        fetchStartMs: preparedAttempt.fetchStartMs,
+                        attemptNumber: loopState.attemptNumber,
+                        logAttempt,
+                        logProxyBody,
+                        logFinalRequest,
+                        lastError: loopState.lastError,
+                        authFailureMessage: loopState.authFailureMessage,
+                        sawTransientFailure: loopState.sawTransientFailure,
+                        invalidRequestFailure: loopState.invalidRequestFailure,
+                    });
+                    loopState.lastError = nonOkResult.lastError;
+                    loopState.authFailureMessage = nonOkResult.authFailureMessage;
+                    loopState.sawTransientFailure = nonOkResult.sawTransientFailure;
+                    loopState.invalidRequestFailure = nonOkResult.invalidRequestFailure;
+                    if (nonOkResult.response !== undefined) {
+                        return nonOkResult.response;
+                    }
+                    if (nonOkResult.continueLoop) {
+                        if (nonOkResult.retrySameAccount &&
+                            transientSameAccountRetries < MAX_TRANSIENT_SAME_ACCOUNT_RETRIES) {
+                            transientSameAccountRetries += 1;
+                            const delayMs = getTransientSameAccountRetryDelayMs(transientSameAccountRetries);
+                            logger.always(`[proxy] retrying same account=${account.label} after transient upstream ${response.status} (${transientSameAccountRetries}/${MAX_TRANSIENT_SAME_ACCOUNT_RETRIES}) in ${delayMs}ms`);
+                            await sleep(delayMs);
+                            continue;
+                        }
+                        if (nonOkResult.retrySameAccount) {
+                            logger.always(`[proxy] exhausted transient same-account retries for account=${account.label}; rotating`);
+                        }
+                        continue accountLoop;
+                    }
+                    break accountLoop;
                 }
-                if (fetchResult.retrySameAccount) {
-                    logger.always(`[proxy] exhausted transient same-account retries for account=${account.label}; rotating`);
+                // Clear cooling on success — but only if the stored cooldown has already
+                // expired, so an older in-flight success can't wipe an active exhaustion
+                // cooldown just set by a concurrent 429. The success handler re-applies a
+                // cooldown via maybeCoolFromQuota if the fresh quota headers report the
+                // window flipped to "rejected" on this very request.
+                if (accountState.coolingUntil &&
+                    Date.now() >= accountState.coolingUntil) {
+                    const expiredCooldown = accountState.coolingUntil;
+                    accountState.coolingUntil = undefined;
+                    accountState.coolingReason = undefined;
+                    clearAccountCooldown(account.key, expiredCooldown).catch(() => {
+                        // Best-effort cleanup; expired entries are ignored during seeding.
+                    });
                 }
-                continue accountLoop;
-            }
-            let upstreamSpan = fetchResult.upstreamSpan;
-            const response = fetchResult.response;
-            if (response.status === 401 &&
-                account.type === "oauth" &&
-                account.refreshToken) {
-                const authRetryResult = await handleAnthropicAuthRetry({
+                const successResult = await handleAnthropicSuccessfulResponse({
                     ctx,
                     body,
                     account,
                     accountState,
-                    headers: preparedAttempt.headers,
-                    buildUpstreamBody: preparedAttempt.buildUpstreamBody,
-                    url,
-                    enabledAccounts,
-                    orderedAccounts,
-                    tracer,
-                    requestStartTime,
-                    allocateAttemptNumber: () => {
-                        loopState.attemptNumber += 1;
-                        return loopState.attemptNumber;
-                    },
-                    upstreamSpan,
-                    logAttempt,
-                    logProxyBody,
-                    logFinalRequest,
-                    lastError: loopState.lastError,
-                    authFailureMessage: loopState.authFailureMessage,
-                    sawRateLimit: loopState.sawRateLimit,
-                    sawTransientFailure: loopState.sawTransientFailure,
-                    sawNetworkError: loopState.sawNetworkError,
-                });
-                loopState.lastError = authRetryResult.lastError;
-                loopState.authFailureMessage = authRetryResult.authFailureMessage;
-                loopState.sawRateLimit = authRetryResult.sawRateLimit;
-                loopState.sawTransientFailure = authRetryResult.sawTransientFailure;
-                loopState.sawNetworkError = authRetryResult.sawNetworkError;
-                upstreamSpan = authRetryResult.upstreamSpan;
-                if (authRetryResult.response !== undefined) {
-                    return authRetryResult.response;
-                }
-                if (authRetryResult.continueLoop) {
-                    continue accountLoop;
-                }
-            }
-            if (!response.ok) {
-                const nonOkResult = await handleAnthropicNonOkResponse({
                     response,
-                    account,
-                    accountState,
-                    enabledAccounts,
-                    orderedAccounts,
                     tracer,
                     requestStartTime,
                     fetchStartMs: preparedAttempt.fetchStartMs,
                     attemptNumber: loopState.attemptNumber,
+                    finalBodyStr: preparedAttempt.finalBodyStr,
+                    upstreamSpan,
                     logAttempt,
                     logProxyBody,
                     logFinalRequest,
-                    lastError: loopState.lastError,
-                    authFailureMessage: loopState.authFailureMessage,
-                    sawTransientFailure: loopState.sawTransientFailure,
-                    invalidRequestFailure: loopState.invalidRequestFailure,
+                    onStreamTerminal: admissionLease.release,
                 });
-                loopState.lastError = nonOkResult.lastError;
-                loopState.authFailureMessage = nonOkResult.authFailureMessage;
-                loopState.sawTransientFailure = nonOkResult.sawTransientFailure;
-                loopState.invalidRequestFailure = nonOkResult.invalidRequestFailure;
-                if (nonOkResult.response !== undefined) {
-                    return nonOkResult.response;
-                }
-                if (nonOkResult.continueLoop) {
-                    if (nonOkResult.retrySameAccount &&
-                        transientSameAccountRetries < MAX_TRANSIENT_SAME_ACCOUNT_RETRIES) {
-                        transientSameAccountRetries += 1;
-                        const delayMs = getTransientSameAccountRetryDelayMs(transientSameAccountRetries);
-                        logger.always(`[proxy] retrying same account=${account.label} after transient upstream ${response.status} (${transientSameAccountRetries}/${MAX_TRANSIENT_SAME_ACCOUNT_RETRIES}) in ${delayMs}ms`);
-                        await sleep(delayMs);
-                        continue;
-                    }
-                    if (nonOkResult.retrySameAccount) {
-                        logger.always(`[proxy] exhausted transient same-account retries for account=${account.label}; rotating`);
+                if ("retryNextAccount" in successResult) {
+                    if (successResult.failure) {
+                        loopState.lastError = successResult.failure.message;
+                        loopState.sawRateLimit ||= successResult.failure.rateLimit;
+                        loopState.sawTransientFailure ||= !successResult.failure.rateLimit;
                     }
                     continue accountLoop;
                 }
-                break accountLoop;
-            }
-            // Clear cooling on success — but only if the stored cooldown has already
-            // expired, so an older in-flight success can't wipe an active exhaustion
-            // cooldown just set by a concurrent 429. The success handler re-applies a
-            // cooldown via maybeCoolFromQuota if the fresh quota headers report the
-            // window flipped to "rejected" on this very request.
-            if (accountState.coolingUntil &&
-                Date.now() >= accountState.coolingUntil) {
-                const expiredCooldown = accountState.coolingUntil;
-                accountState.coolingUntil = undefined;
-                accountState.coolingReason = undefined;
-                clearAccountCooldown(account.key, expiredCooldown).catch(() => {
-                    // Best-effort cleanup; expired entries are ignored during seeding.
-                });
+                admissionTransferredToStream =
+                    successResult.holdsAccountAdmission === true;
+                return successResult.response;
             }
-            const successResult = await handleAnthropicSuccessfulResponse({
-                ctx,
-                body,
-                account,
-                accountState,
-                response,
-                tracer,
-                requestStartTime,
-                fetchStartMs: preparedAttempt.fetchStartMs,
-                attemptNumber: loopState.attemptNumber,
-                finalBodyStr: preparedAttempt.finalBodyStr,
-                upstreamSpan,
-                logAttempt,
-                logProxyBody,
-                logFinalRequest,
-            });
-            if ("retryNextAccount" in successResult) {
-                if (successResult.failure) {
-                    loopState.lastError = successResult.failure.message;
-                    loopState.sawRateLimit ||= successResult.failure.rateLimit;
-                    loopState.sawTransientFailure ||= !successResult.failure.rateLimit;
+            finally {
+                if (!admissionTransferredToStream) {
+                    admissionLease.release();
                 }
-                continue accountLoop;
             }
-            return successResult.response;
         }
     }
+    queuedAccountAdmission?.lease.release();
     if (loopState.attemptNumber === 0) {
         acctSelectionSpan?.end();
     }
@@ -4311,9 +4531,9 @@ async function handleAnthropicRoutedClaudeRequest(args) {
             return configuredFallbackResult.response;
         }
         fallbackFailureMessage = configuredFallbackResult.lastErrorMessage;
-        // Try auto-provider fallback when the configured chain didn't produce a
-        // response (either no chain configured, or all entries failed/deduped).
-        if (!loopState.sawRateLimit) {
+        // A translation-layer-selected provider is only permitted by an explicit
+        // routing setting. Empty fallback chains otherwise stay within OAuth.
+        if (!loopState.sawRateLimit && modelRouter?.isAutoFallbackEnabled?.()) {
             const autoFallbackResult = await tryAutoClaudeFallback({
                 ctx,
                 body,
@@ -4872,6 +5092,19 @@ export function isTransientHttpFailure(status, errBody) {
         normalized.includes("cloudflare") ||
         normalized.includes("internal server error"));
 }
+/**
+ * An upstream overload is already a capacity signal. Retrying it on the same
+ * OAuth account only consumes its remaining concurrency and delays rotation.
+ */
+export function isUpstreamOverload(status, errBody) {
+    return (status === 529 ||
+        parseClaudeErrorBody(errBody).errorType === "overloaded_error");
+}
+/** Remove provider credentials before they enter persistent proxy diagnostics. */
+const MAX_PROVIDER_ERROR_MESSAGE_LENGTH = 500;
+export function redactProviderErrorMessage(message) {
+    return sanitizeForLog(message, MAX_PROVIDER_ERROR_MESSAGE_LENGTH);
+}
 // ---------------------------------------------------------------------------
 // Test hooks (not part of the public SDK API). Only consumed by the proxy
 // continuous-test-suite to drive the in-process resolver/reset logic without
@@ -4922,6 +5155,7 @@ export const __testHooks = {
         accountRuntimeState.clear();
         transientRateLimitRetryBudgets.clear();
         transientCooldownAdmissionSchedules.clear();
+        accountAdmissionStates.clear();
         primaryAccountIndex = 0;
         lastKnownAccountCount = 0;
     },
@@ -4929,12 +5163,24 @@ export const __testHooks = {
     isAntiAbuseConstruction429,
     fetchAnthropicAccountResponse,
     finalizeAnthropicTerminalFetchError,
+    handleAnthropicNonOkResponse,
     handleAnthropicAuthRetry,
     handleAnthropicStreamingSuccessResponse,
     claimTransientRateLimitRetry,
     claimTransientCooldownAdmission,
     waitForTransientAccountAvailability,
+    acquireAccountAdmission,
+    acquireFirstAvailableAccountAdmission,
+    tryAcquireAccountAdmission,
+    getAccountAdmissionSnapshot: (accountKey) => {
+        const state = accountAdmissionStates.get(accountKey);
+        return state
+            ? { active: state.active, waiting: state.waiters.length }
+            : { active: 0, waiting: 0 };
+    },
     describeTransportError,
+    redactProviderErrorMessage,
+    isUpstreamOverload,
     shouldAttemptClaudeFallback,
     executeClaudeFallbackWithRetry,
     buildClaudeAnthropicFailureResponse,