@juspay/neurolink 9.86.3 → 9.86.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/auth/tokenStore.d.ts +6 -0
- package/dist/auth/tokenStore.js +36 -4
- package/dist/browser/neurolink.min.js +417 -417
- package/dist/cli/commands/auth.js +3 -0
- package/dist/cli/commands/proxy.js +310 -107
- package/dist/core/modules/GenerationHandler.js +19 -1
- package/dist/lib/auth/tokenStore.d.ts +6 -0
- package/dist/lib/auth/tokenStore.js +36 -4
- package/dist/lib/core/modules/GenerationHandler.js +19 -1
- package/dist/lib/neurolink.js +9 -1
- package/dist/lib/proxy/accountCooldown.d.ts +5 -0
- package/dist/lib/proxy/accountCooldown.js +93 -0
- package/dist/lib/proxy/accountQuota.d.ts +3 -4
- package/dist/lib/proxy/accountQuota.js +75 -44
- package/dist/lib/proxy/accountSelection.d.ts +8 -0
- package/dist/lib/proxy/accountSelection.js +26 -0
- package/dist/lib/proxy/proxyConfig.js +24 -0
- package/dist/lib/proxy/proxyPaths.js +2 -0
- package/dist/lib/proxy/requestLogger.js +2 -0
- package/dist/lib/proxy/snapshotPersistence.js +1 -1
- package/dist/lib/proxy/sseInterceptor.js +16 -0
- package/dist/lib/proxy/streamOutcome.d.ts +3 -0
- package/dist/lib/proxy/streamOutcome.js +27 -0
- package/dist/lib/proxy/tokenRefresh.d.ts +8 -0
- package/dist/lib/proxy/tokenRefresh.js +92 -15
- package/dist/lib/proxy/updateState.js +17 -4
- package/dist/lib/server/routes/claudeProxyRoutes.d.ts +18 -3
- package/dist/lib/server/routes/claudeProxyRoutes.js +624 -223
- package/dist/lib/types/auth.d.ts +1 -1
- package/dist/lib/types/cli.d.ts +2 -0
- package/dist/lib/types/proxy.d.ts +49 -8
- package/dist/lib/types/subscription.d.ts +5 -0
- package/dist/neurolink.js +9 -1
- package/dist/proxy/accountCooldown.d.ts +5 -0
- package/dist/proxy/accountCooldown.js +92 -0
- package/dist/proxy/accountQuota.d.ts +3 -4
- package/dist/proxy/accountQuota.js +75 -44
- package/dist/proxy/accountSelection.d.ts +8 -0
- package/dist/proxy/accountSelection.js +25 -0
- package/dist/proxy/proxyConfig.js +24 -0
- package/dist/proxy/proxyPaths.js +2 -0
- package/dist/proxy/requestLogger.js +2 -0
- package/dist/proxy/snapshotPersistence.js +1 -1
- package/dist/proxy/sseInterceptor.js +16 -0
- package/dist/proxy/streamOutcome.d.ts +3 -0
- package/dist/proxy/streamOutcome.js +26 -0
- package/dist/proxy/tokenRefresh.d.ts +8 -0
- package/dist/proxy/tokenRefresh.js +92 -15
- package/dist/proxy/updateState.js +17 -4
- package/dist/server/routes/claudeProxyRoutes.d.ts +18 -3
- package/dist/server/routes/claudeProxyRoutes.js +624 -223
- package/dist/types/auth.d.ts +1 -1
- package/dist/types/cli.d.ts +2 -0
- package/dist/types/proxy.d.ts +49 -8
- package/dist/types/subscription.d.ts +5 -0
- package/package.json +1 -1
|
@@ -13,7 +13,9 @@ import { access, readFile } from "node:fs/promises";
|
|
|
13
13
|
import { homedir } from "node:os";
|
|
14
14
|
import { join } from "node:path";
|
|
15
15
|
import { buildStableClaudeCodeBillingHeader, CLAUDE_CLI_USER_AGENT, CLAUDE_CODE_OAUTH_BETAS, getOrCreateClaudeCodeIdentity, parseClaudeCodeUserId, } from "../../auth/anthropicOAuth.js";
|
|
16
|
-
import {
|
|
16
|
+
import { clearAccountCooldown, loadAccountCooldowns, saveAccountCooldown, } from "../../proxy/accountCooldown.js";
|
|
17
|
+
import { anthropicAccountKeysEqual, ENV_ANTHROPIC_ACCOUNT_KEY, isAccountAllowed, LEGACY_ANTHROPIC_ACCOUNT_KEY, shouldLoadFallbackCredential, } from "../../proxy/accountSelection.js";
|
|
18
|
+
import { getUnifiedRateLimitStatus, loadAccountQuotas, parseQuotaHeaders, saveAccountQuota, } from "../../proxy/accountQuota.js";
|
|
17
19
|
import { buildClaudeError, ClaudeStreamSerializer, generateToolUseId, parseClaudeRequest, serializeClaudeResponse, } from "../../proxy/claudeFormat.js";
|
|
18
20
|
import { buildAnthropicModelsListResponse, buildTranslationOptions, extractText, extractToolArgs, extractUsageFromStreamResult, handleTranslatedJsonRequest, handleTranslatedStreamRequest, hasTranslatedOutput, } from "../../proxy/proxyTranslationEngine.js";
|
|
19
21
|
import { tracers } from "../../telemetry/tracers.js";
|
|
@@ -23,7 +25,8 @@ import { createRawStreamCapture } from "../../proxy/rawStreamCapture.js";
|
|
|
23
25
|
import { relocateClientSystemIntoMessages } from "../../proxy/systemRelocation.js";
|
|
24
26
|
import { logBodyCapture, logRequest, logRequestAttempt, logStreamError, } from "../../proxy/requestLogger.js";
|
|
25
27
|
import { createSSEInterceptor } from "../../proxy/sseInterceptor.js";
|
|
26
|
-
import {
|
|
28
|
+
import { createStreamTerminalOutcomeTracker, mergeStreamTerminalOutcome, } from "../../proxy/streamOutcome.js";
|
|
29
|
+
import { isPermanentRefreshFailure, needsRefresh, persistTokens, refreshToken, } from "../../proxy/tokenRefresh.js";
|
|
27
30
|
import { buildProxyTranslationPlan, parseRetryAfterMs, } from "../../proxy/routingPolicy.js";
|
|
28
31
|
import { writeJsonSnapshotAtomically } from "../../proxy/snapshotPersistence.js";
|
|
29
32
|
import { recordAttempt, recordAttemptError, recordFinalError, recordFinalSuccess, } from "../../proxy/usageStats.js";
|
|
@@ -54,8 +57,8 @@ let lastKnownAccountCount = 0;
|
|
|
54
57
|
* When undefined, home semantics fall back to enabledAccounts[0] (insertion
|
|
55
58
|
* order) — preserves pre-existing behavior. */
|
|
56
59
|
let configuredPrimaryAccountKey;
|
|
60
|
+
let configuredAccountAllowlist;
|
|
57
61
|
const MAX_AUTH_RETRIES = 5;
|
|
58
|
-
const MAX_CONSECUTIVE_REFRESH_FAILURES = 15;
|
|
59
62
|
const MAX_TRANSIENT_SAME_ACCOUNT_RETRIES = 2;
|
|
60
63
|
const TRANSIENT_SAME_ACCOUNT_RETRY_DELAYS_MS = [250, 1_000];
|
|
61
64
|
/** Maximum upstream 429 attempts per account before rotating — for a TRANSIENT
|
|
@@ -78,6 +81,11 @@ const MIN_COOLDOWN_MS = 5_000;
|
|
|
78
81
|
* we should return to the account soon rather than parking it for the full
|
|
79
82
|
* reset window. */
|
|
80
83
|
const TRANSIENT_MAX_COOLDOWN_MS = 15 * 60 * 1000; // 15 minutes
|
|
84
|
+
/** Fallback for authoritative rejected states that do not provide a usable
|
|
85
|
+
* reset or retry-after. These are not transient burst limits. */
|
|
86
|
+
const DEFAULT_HARD_COOLDOWN_MS = 15 * 60 * 1000; // 15 minutes
|
|
87
|
+
const AUTH_REFRESH_BASE_COOLDOWN_MS = 30_000;
|
|
88
|
+
const AUTH_REFRESH_MAX_COOLDOWN_MS = 5 * 60 * 1000;
|
|
81
89
|
/** Timeout for upstream requests to Anthropic. Must be generous enough
|
|
82
90
|
* to cover the full lifecycle of streaming responses, including extended
|
|
83
91
|
* thinking from Opus models (which can exceed 5 minutes for large contexts). */
|
|
@@ -109,10 +117,11 @@ function advancePrimaryIfCurrent(accountKey, enabledCount, primaryAccountKey) {
|
|
|
109
117
|
* removed). The resolution is per-request because enabledAccounts membership
|
|
110
118
|
* can shift between requests. */
|
|
111
119
|
function resolveHomeIndex(enabledAccounts) {
|
|
112
|
-
|
|
120
|
+
const primaryAccountKey = configuredPrimaryAccountKey;
|
|
121
|
+
if (!primaryAccountKey) {
|
|
113
122
|
return 0;
|
|
114
123
|
}
|
|
115
|
-
const idx = enabledAccounts.findIndex((a) => a.key
|
|
124
|
+
const idx = enabledAccounts.findIndex((a) => anthropicAccountKeysEqual(a.key, primaryAccountKey));
|
|
116
125
|
return idx >= 0 ? idx : 0;
|
|
117
126
|
}
|
|
118
127
|
/** If the configured home primary is no longer cooling, reset
|
|
@@ -135,7 +144,12 @@ function maybeResetPrimaryToHome(enabledAccounts) {
|
|
|
135
144
|
// Home account is no longer cooling — reset to it
|
|
136
145
|
primaryAccountIndex = homeIndex;
|
|
137
146
|
if (homeState?.coolingUntil) {
|
|
147
|
+
const expiredCooldown = homeState.coolingUntil;
|
|
138
148
|
homeState.coolingUntil = undefined;
|
|
149
|
+
homeState.coolingReason = undefined;
|
|
150
|
+
clearAccountCooldown(homeAccount.key, expiredCooldown).catch(() => {
|
|
151
|
+
// Best-effort cleanup; an expired entry is ignored on the next boot.
|
|
152
|
+
});
|
|
139
153
|
logger.always(`[proxy] home primary account=${homeAccount.label} cooling expired, resetting primaryAccountIndex to ${homeIndex}`);
|
|
140
154
|
}
|
|
141
155
|
}
|
|
@@ -179,7 +193,7 @@ function clampCooldownUntil(untilMs, now) {
|
|
|
179
193
|
* burst / acceleration limit) is transient: honor retry-after as a floor,
|
|
180
194
|
* allow a couple of jittered same-account retries, then a short cooldown.
|
|
181
195
|
*/
|
|
182
|
-
function planCooldownFor429(quota, retryAfterMs, now) {
|
|
196
|
+
function planCooldownFor429(quota, retryAfterMs, now, unifiedStatus = quota?.unifiedStatus) {
|
|
183
197
|
// Weekly exhaustion takes precedence — it's the longest, hardest ceiling.
|
|
184
198
|
if (quota && quota.weeklyStatus === "rejected") {
|
|
185
199
|
const reset = resetEpochToMs(quota.weeklyResetAt, now) ??
|
|
@@ -199,6 +213,17 @@ function planCooldownFor429(quota, retryAfterMs, now) {
|
|
|
199
213
|
rotateImmediately: true,
|
|
200
214
|
};
|
|
201
215
|
}
|
|
216
|
+
// Anthropic may reject the authoritative top-level unified limit while both
|
|
217
|
+
// 5h and 7d sub-window statuses still say "allowed". Treating this as a
|
|
218
|
+
// transient burst retries a known-exhausted account and delays failover.
|
|
219
|
+
if (unifiedStatus?.trim().toLowerCase() === "rejected") {
|
|
220
|
+
const reset = retryAfterMs > 0 ? now + retryAfterMs : now + DEFAULT_HARD_COOLDOWN_MS;
|
|
221
|
+
return {
|
|
222
|
+
reason: "unified",
|
|
223
|
+
coolingUntil: clampCooldownUntil(reset, now),
|
|
224
|
+
rotateImmediately: true,
|
|
225
|
+
};
|
|
226
|
+
}
|
|
202
227
|
// Transient burst: cool only for retry-after, floored at MIN_COOLDOWN_MS so a
|
|
203
228
|
// tiny retry-after can't make the account eligible almost immediately, and
|
|
204
229
|
// capped so it recovers quickly.
|
|
@@ -232,15 +257,21 @@ function maybeCoolFromQuota(state, quota, now) {
|
|
|
232
257
|
until = resetEpochToMs(quota.sessionResetAt, now);
|
|
233
258
|
reason = "session";
|
|
234
259
|
}
|
|
260
|
+
else if (quota.unifiedStatus === "rejected") {
|
|
261
|
+
until = now + DEFAULT_HARD_COOLDOWN_MS;
|
|
262
|
+
reason = "unified";
|
|
263
|
+
}
|
|
235
264
|
if (until === undefined) {
|
|
236
|
-
return;
|
|
265
|
+
return false;
|
|
237
266
|
}
|
|
238
267
|
const clamped = clampCooldownUntil(until, now);
|
|
239
268
|
if (!state.coolingUntil || clamped > state.coolingUntil) {
|
|
240
269
|
state.coolingUntil = clamped;
|
|
241
270
|
state.coolingReason = reason;
|
|
242
271
|
logger.always(`[proxy] proactively cooling account (${reason}) ~${minutesUntil(clamped, now)}m from success-response quota (status rejected)`);
|
|
272
|
+
return true;
|
|
243
273
|
}
|
|
274
|
+
return false;
|
|
244
275
|
}
|
|
245
276
|
/**
|
|
246
277
|
* Seed each account's runtime quota from the persisted snapshots in
|
|
@@ -254,11 +285,22 @@ function maybeCoolFromQuota(state, quota, now) {
|
|
|
254
285
|
*/
|
|
255
286
|
async function seedRuntimeQuotasFromDisk(accounts) {
|
|
256
287
|
try {
|
|
257
|
-
const
|
|
288
|
+
const [persistedQuotas, persistedCooldowns] = await Promise.all([
|
|
289
|
+
loadAccountQuotas(),
|
|
290
|
+
loadAccountCooldowns(),
|
|
291
|
+
]);
|
|
292
|
+
const now = Date.now();
|
|
258
293
|
for (const account of accounts) {
|
|
259
294
|
const state = getOrCreateRuntimeState(account.key);
|
|
260
|
-
if (!state.quota &&
|
|
261
|
-
state.quota =
|
|
295
|
+
if (!state.quota && persistedQuotas[account.label]) {
|
|
296
|
+
state.quota = persistedQuotas[account.label];
|
|
297
|
+
}
|
|
298
|
+
const persistedCooldown = persistedCooldowns[account.key];
|
|
299
|
+
if (persistedCooldown?.coolingUntil > now &&
|
|
300
|
+
(!state.coolingUntil ||
|
|
301
|
+
persistedCooldown.coolingUntil > state.coolingUntil)) {
|
|
302
|
+
state.coolingUntil = persistedCooldown.coolingUntil;
|
|
303
|
+
state.coolingReason = persistedCooldown.reason;
|
|
262
304
|
}
|
|
263
305
|
}
|
|
264
306
|
}
|
|
@@ -695,47 +737,63 @@ async function tryLoadLegacyAccount(creds, legacyCredPath) {
|
|
|
695
737
|
let legacyToken = creds.oauth.accessToken;
|
|
696
738
|
let legacyRefresh = creds.oauth.refreshToken;
|
|
697
739
|
let legacyExpiry = creds.oauth.expiresAt;
|
|
740
|
+
const legacyLabel = creds.email?.trim() || "legacy-default";
|
|
698
741
|
const legacyExpired = legacyExpiry ? legacyExpiry < Date.now() : false;
|
|
742
|
+
const legacyAccount = () => ({
|
|
743
|
+
key: LEGACY_ANTHROPIC_ACCOUNT_KEY,
|
|
744
|
+
label: legacyLabel,
|
|
745
|
+
token: legacyToken,
|
|
746
|
+
refreshToken: legacyRefresh,
|
|
747
|
+
expiresAt: legacyExpiry,
|
|
748
|
+
type: "oauth",
|
|
749
|
+
persistTarget: { credPath: legacyCredPath },
|
|
750
|
+
});
|
|
699
751
|
if (!legacyExpired) {
|
|
700
|
-
return
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
752
|
+
return legacyAccount();
|
|
753
|
+
}
|
|
754
|
+
const state = getOrCreateRuntimeState(LEGACY_ANTHROPIC_ACCOUNT_KEY);
|
|
755
|
+
if (state.permanentlyDisabled) {
|
|
756
|
+
return undefined;
|
|
757
|
+
}
|
|
758
|
+
const persistedCooldown = (await loadAccountCooldowns())[LEGACY_ANTHROPIC_ACCOUNT_KEY];
|
|
759
|
+
if (persistedCooldown?.coolingUntil > Date.now() &&
|
|
760
|
+
(!state.coolingUntil || persistedCooldown.coolingUntil > state.coolingUntil)) {
|
|
761
|
+
state.coolingUntil = persistedCooldown.coolingUntil;
|
|
762
|
+
state.coolingReason = persistedCooldown.reason;
|
|
763
|
+
}
|
|
764
|
+
if (state.coolingReason === "auth" &&
|
|
765
|
+
state.coolingUntil &&
|
|
766
|
+
state.coolingUntil > Date.now()) {
|
|
767
|
+
return legacyAccount();
|
|
709
768
|
}
|
|
710
769
|
if (!legacyRefresh) {
|
|
711
770
|
logger.always("[proxy] skipping legacy account (expired, no refresh token)");
|
|
771
|
+
await disableAccountUntilReauth(legacyAccount(), state, "missing_refresh_token");
|
|
712
772
|
return undefined;
|
|
713
773
|
}
|
|
714
774
|
const tmp = {
|
|
715
775
|
token: legacyToken,
|
|
716
776
|
refreshToken: legacyRefresh,
|
|
717
777
|
expiresAt: legacyExpiry,
|
|
718
|
-
label:
|
|
778
|
+
label: legacyLabel,
|
|
719
779
|
};
|
|
720
780
|
const ok = await refreshToken(tmp);
|
|
721
781
|
if (!ok.success) {
|
|
722
|
-
|
|
723
|
-
|
|
782
|
+
if (isPermanentRefreshFailure(ok)) {
|
|
783
|
+
await disableAccountUntilReauth(legacyAccount(), state, "refresh_invalid");
|
|
784
|
+
return undefined;
|
|
785
|
+
}
|
|
786
|
+
const coolingUntil = await coolAccountAfterTransientRefreshFailure(legacyAccount(), state);
|
|
787
|
+
logger.always(`[proxy] legacy refresh temporarily unavailable (${ok.status ?? "network"}); cooling until ${new Date(coolingUntil).toISOString()}`);
|
|
788
|
+
return legacyAccount();
|
|
724
789
|
}
|
|
725
790
|
legacyToken = tmp.token;
|
|
726
791
|
legacyRefresh = tmp.refreshToken;
|
|
727
792
|
legacyExpiry = tmp.expiresAt;
|
|
728
793
|
await persistTokens(legacyCredPath, tmp);
|
|
794
|
+
await clearAuthCooldownAfterRefresh(legacyAccount(), state);
|
|
729
795
|
logger.always("[proxy] refreshed legacy account at startup");
|
|
730
|
-
return
|
|
731
|
-
key: "anthropic:legacy-default",
|
|
732
|
-
label: "default",
|
|
733
|
-
token: legacyToken,
|
|
734
|
-
refreshToken: legacyRefresh,
|
|
735
|
-
expiresAt: legacyExpiry,
|
|
736
|
-
type: "oauth",
|
|
737
|
-
persistTarget: { credPath: legacyCredPath },
|
|
738
|
-
};
|
|
796
|
+
return legacyAccount();
|
|
739
797
|
}
|
|
740
798
|
async function handleTranslatedClaudeRequest(args) {
|
|
741
799
|
const { ctx, body, route, modelRouter, tracer, requestStartTime, logProxyBody, } = args;
|
|
@@ -925,6 +983,43 @@ async function handleClaudePassthroughRequest(args) {
|
|
|
925
983
|
logProxyBody,
|
|
926
984
|
});
|
|
927
985
|
}
|
|
986
|
+
function trackUpstreamReadableStream(source) {
|
|
987
|
+
const reader = source.getReader();
|
|
988
|
+
const tracker = createStreamTerminalOutcomeTracker();
|
|
989
|
+
let closed = false;
|
|
990
|
+
const stream = new ReadableStream({
|
|
991
|
+
async pull(controller) {
|
|
992
|
+
if (closed) {
|
|
993
|
+
return;
|
|
994
|
+
}
|
|
995
|
+
try {
|
|
996
|
+
const { done, value } = await reader.read();
|
|
997
|
+
if (closed) {
|
|
998
|
+
return;
|
|
999
|
+
}
|
|
1000
|
+
if (done) {
|
|
1001
|
+
closed = true;
|
|
1002
|
+
tracker.complete();
|
|
1003
|
+
controller.close();
|
|
1004
|
+
return;
|
|
1005
|
+
}
|
|
1006
|
+
controller.enqueue(value);
|
|
1007
|
+
}
|
|
1008
|
+
catch (error) {
|
|
1009
|
+
closed = true;
|
|
1010
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1011
|
+
tracker.fail(message);
|
|
1012
|
+
controller.error(error);
|
|
1013
|
+
}
|
|
1014
|
+
},
|
|
1015
|
+
cancel(reason) {
|
|
1016
|
+
closed = true;
|
|
1017
|
+
tracker.cancel();
|
|
1018
|
+
return reader.cancel(reason);
|
|
1019
|
+
},
|
|
1020
|
+
});
|
|
1021
|
+
return { stream, outcome: tracker.outcome };
|
|
1022
|
+
}
|
|
928
1023
|
async function handleClaudePassthroughStreamResponse(args) {
|
|
929
1024
|
const { ctx, body, bodyStr, response, tracer, requestStartTime, toolCount, upstreamSpan, upstreamResponseHeaders, logProxyBody, } = args;
|
|
930
1025
|
const responseHeaders = { ...upstreamResponseHeaders };
|
|
@@ -933,7 +1028,8 @@ async function handleClaudePassthroughStreamResponse(args) {
|
|
|
933
1028
|
if (!responseBody) {
|
|
934
1029
|
throw new Error("Expected passthrough stream response body");
|
|
935
1030
|
}
|
|
936
|
-
|
|
1031
|
+
const trackedStream = trackUpstreamReadableStream(responseBody);
|
|
1032
|
+
let streamSource = trackedStream.stream;
|
|
937
1033
|
if (tracer) {
|
|
938
1034
|
try {
|
|
939
1035
|
const { stream: interceptor, telemetry } = createSSEInterceptor({
|
|
@@ -944,8 +1040,10 @@ async function handleClaudePassthroughStreamResponse(args) {
|
|
|
944
1040
|
const capturedUpstreamSpan = upstreamSpan;
|
|
945
1041
|
const capturedResponse = response;
|
|
946
1042
|
const capturedRequestBytes = bodyStr.length;
|
|
947
|
-
Promise.all([telemetry, clientCapture])
|
|
948
|
-
.then(([data, clientBody]) => {
|
|
1043
|
+
Promise.all([telemetry, clientCapture, trackedStream.outcome])
|
|
1044
|
+
.then(([data, clientBody, rawOutcome]) => {
|
|
1045
|
+
const terminalOutcome = mergeStreamTerminalOutcome(rawOutcome, data.streamErrorMessage);
|
|
1046
|
+
const failure = getStreamFailureDetails(terminalOutcome);
|
|
949
1047
|
capturedTracer.setUsage({
|
|
950
1048
|
inputTokens: data.usage.inputTokens,
|
|
951
1049
|
outputTokens: data.usage.outputTokens,
|
|
@@ -975,7 +1073,10 @@ async function handleClaudePassthroughStreamResponse(args) {
|
|
|
975
1073
|
capturedTracer.recordMetrics();
|
|
976
1074
|
capturedTracer.recordBodySizes(capturedRequestBytes, data.totalBytesReceived);
|
|
977
1075
|
capturedUpstreamSpan?.end();
|
|
978
|
-
|
|
1076
|
+
if (failure) {
|
|
1077
|
+
capturedTracer.setError(failure.errorType, failure.message);
|
|
1078
|
+
}
|
|
1079
|
+
capturedTracer.end(failure?.status ?? response.status, Date.now() - requestStartTime);
|
|
979
1080
|
const traceCtx = capturedTracer.getTraceContext();
|
|
980
1081
|
logRequest({
|
|
981
1082
|
timestamp: new Date().toISOString(),
|
|
@@ -987,7 +1088,7 @@ async function handleClaudePassthroughStreamResponse(args) {
|
|
|
987
1088
|
toolCount,
|
|
988
1089
|
account: "passthrough",
|
|
989
1090
|
accountType: "passthrough",
|
|
990
|
-
responseStatus:
|
|
1091
|
+
responseStatus: failure?.status ?? response.status,
|
|
991
1092
|
responseTimeMs: Date.now() - requestStartTime,
|
|
992
1093
|
inputTokens: data.usage.inputTokens,
|
|
993
1094
|
outputTokens: data.usage.outputTokens,
|
|
@@ -995,6 +1096,12 @@ async function handleClaudePassthroughStreamResponse(args) {
|
|
|
995
1096
|
cacheReadTokens: data.usage.cacheReadInputTokens,
|
|
996
1097
|
traceId: traceCtx.traceId,
|
|
997
1098
|
spanId: traceCtx.spanId,
|
|
1099
|
+
...(failure
|
|
1100
|
+
? {
|
|
1101
|
+
errorType: failure.errorType,
|
|
1102
|
+
errorMessage: failure.message,
|
|
1103
|
+
}
|
|
1104
|
+
: {}),
|
|
998
1105
|
});
|
|
999
1106
|
logProxyBody({
|
|
1000
1107
|
phase: "upstream_response",
|
|
@@ -1046,40 +1153,140 @@ async function handleClaudePassthroughStreamResponse(args) {
|
|
|
1046
1153
|
});
|
|
1047
1154
|
}
|
|
1048
1155
|
catch {
|
|
1049
|
-
|
|
1156
|
+
trackedStream.outcome.then((outcome) => {
|
|
1157
|
+
const failure = getStreamFailureDetails(outcome);
|
|
1158
|
+
upstreamSpan?.end();
|
|
1159
|
+
if (failure) {
|
|
1160
|
+
tracer.setError(failure.errorType, failure.message);
|
|
1161
|
+
}
|
|
1162
|
+
tracer.end(failure?.status ?? response.status, Date.now() - requestStartTime);
|
|
1163
|
+
const traceCtx = tracer.getTraceContext();
|
|
1164
|
+
logRequest({
|
|
1165
|
+
timestamp: new Date().toISOString(),
|
|
1166
|
+
requestId: ctx.requestId,
|
|
1167
|
+
method: ctx.method,
|
|
1168
|
+
path: ctx.path,
|
|
1169
|
+
model: body.model,
|
|
1170
|
+
stream: true,
|
|
1171
|
+
toolCount,
|
|
1172
|
+
account: "passthrough",
|
|
1173
|
+
accountType: "passthrough",
|
|
1174
|
+
responseStatus: failure?.status ?? response.status,
|
|
1175
|
+
responseTimeMs: Date.now() - requestStartTime,
|
|
1176
|
+
traceId: traceCtx.traceId,
|
|
1177
|
+
spanId: traceCtx.spanId,
|
|
1178
|
+
...(failure
|
|
1179
|
+
? {
|
|
1180
|
+
errorType: failure.errorType,
|
|
1181
|
+
errorMessage: failure.message,
|
|
1182
|
+
}
|
|
1183
|
+
: {}),
|
|
1184
|
+
});
|
|
1185
|
+
});
|
|
1050
1186
|
}
|
|
1051
1187
|
}
|
|
1052
1188
|
else {
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
phase: "upstream_response",
|
|
1057
|
-
headers: responseHeaders,
|
|
1058
|
-
body: clientBody.text,
|
|
1059
|
-
bodySize: clientBody.totalBytes,
|
|
1060
|
-
contentType: responseHeaders["content-type"] ?? "text/event-stream",
|
|
1061
|
-
account: "passthrough",
|
|
1062
|
-
accountType: "passthrough",
|
|
1063
|
-
attempt: 1,
|
|
1064
|
-
responseStatus: 200,
|
|
1065
|
-
durationMs: Date.now() - requestStartTime,
|
|
1189
|
+
try {
|
|
1190
|
+
const { stream: interceptor, telemetry } = createSSEInterceptor({
|
|
1191
|
+
captureRawText: true,
|
|
1066
1192
|
});
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1193
|
+
streamSource = streamSource.pipeThrough(interceptor);
|
|
1194
|
+
Promise.all([telemetry, clientCapture, trackedStream.outcome])
|
|
1195
|
+
.then(([data, clientBody, rawOutcome]) => {
|
|
1196
|
+
const terminalOutcome = mergeStreamTerminalOutcome(rawOutcome, data.streamErrorMessage);
|
|
1197
|
+
const failure = getStreamFailureDetails(terminalOutcome);
|
|
1198
|
+
logRequest({
|
|
1199
|
+
timestamp: new Date().toISOString(),
|
|
1200
|
+
requestId: ctx.requestId,
|
|
1201
|
+
method: ctx.method,
|
|
1202
|
+
path: ctx.path,
|
|
1203
|
+
model: body.model,
|
|
1204
|
+
stream: true,
|
|
1205
|
+
toolCount,
|
|
1206
|
+
account: "passthrough",
|
|
1207
|
+
accountType: "passthrough",
|
|
1208
|
+
responseStatus: failure?.status ?? response.status,
|
|
1209
|
+
responseTimeMs: Date.now() - requestStartTime,
|
|
1210
|
+
inputTokens: data.usage.inputTokens,
|
|
1211
|
+
outputTokens: data.usage.outputTokens,
|
|
1212
|
+
cacheCreationTokens: data.usage.cacheCreationInputTokens,
|
|
1213
|
+
cacheReadTokens: data.usage.cacheReadInputTokens,
|
|
1214
|
+
...(failure
|
|
1215
|
+
? {
|
|
1216
|
+
errorType: failure.errorType,
|
|
1217
|
+
errorMessage: failure.message,
|
|
1218
|
+
}
|
|
1219
|
+
: {}),
|
|
1220
|
+
});
|
|
1221
|
+
logProxyBody({
|
|
1222
|
+
phase: "upstream_response",
|
|
1223
|
+
headers: responseHeaders,
|
|
1224
|
+
body: data.rawText ?? "",
|
|
1225
|
+
bodySize: data.totalBytesReceived,
|
|
1226
|
+
contentType: responseHeaders["content-type"] ?? "text/event-stream",
|
|
1227
|
+
account: "passthrough",
|
|
1228
|
+
accountType: "passthrough",
|
|
1229
|
+
attempt: 1,
|
|
1230
|
+
responseStatus: response.status,
|
|
1231
|
+
durationMs: Date.now() - requestStartTime,
|
|
1232
|
+
});
|
|
1233
|
+
logProxyBody({
|
|
1234
|
+
phase: "client_response",
|
|
1235
|
+
headers: responseHeaders,
|
|
1236
|
+
body: clientBody.text,
|
|
1237
|
+
bodySize: clientBody.totalBytes,
|
|
1238
|
+
contentType: responseHeaders["content-type"] ?? "text/event-stream",
|
|
1239
|
+
account: "passthrough",
|
|
1240
|
+
accountType: "passthrough",
|
|
1241
|
+
attempt: 1,
|
|
1242
|
+
responseStatus: response.status,
|
|
1243
|
+
durationMs: Date.now() - requestStartTime,
|
|
1244
|
+
});
|
|
1245
|
+
})
|
|
1246
|
+
.catch((error) => {
|
|
1247
|
+
logRequest({
|
|
1248
|
+
timestamp: new Date().toISOString(),
|
|
1249
|
+
requestId: ctx.requestId,
|
|
1250
|
+
method: ctx.method,
|
|
1251
|
+
path: ctx.path,
|
|
1252
|
+
model: body.model,
|
|
1253
|
+
stream: true,
|
|
1254
|
+
toolCount,
|
|
1255
|
+
account: "passthrough",
|
|
1256
|
+
accountType: "passthrough",
|
|
1257
|
+
responseStatus: 500,
|
|
1258
|
+
responseTimeMs: Date.now() - requestStartTime,
|
|
1259
|
+
errorType: "stream_telemetry_error",
|
|
1260
|
+
errorMessage: error instanceof Error ? error.message : String(error),
|
|
1261
|
+
});
|
|
1078
1262
|
});
|
|
1079
|
-
}
|
|
1080
|
-
|
|
1081
|
-
//
|
|
1082
|
-
|
|
1263
|
+
}
|
|
1264
|
+
catch {
|
|
1265
|
+
// Streaming capture is best-effort; the tracked source still propagates
|
|
1266
|
+
// the transport failure to the client.
|
|
1267
|
+
trackedStream.outcome.then((outcome) => {
|
|
1268
|
+
const failure = getStreamFailureDetails(outcome);
|
|
1269
|
+
logRequest({
|
|
1270
|
+
timestamp: new Date().toISOString(),
|
|
1271
|
+
requestId: ctx.requestId,
|
|
1272
|
+
method: ctx.method,
|
|
1273
|
+
path: ctx.path,
|
|
1274
|
+
model: body.model,
|
|
1275
|
+
stream: true,
|
|
1276
|
+
toolCount,
|
|
1277
|
+
account: "passthrough",
|
|
1278
|
+
accountType: "passthrough",
|
|
1279
|
+
responseStatus: failure?.status ?? response.status,
|
|
1280
|
+
responseTimeMs: Date.now() - requestStartTime,
|
|
1281
|
+
...(failure
|
|
1282
|
+
? {
|
|
1283
|
+
errorType: failure.errorType,
|
|
1284
|
+
errorMessage: failure.message,
|
|
1285
|
+
}
|
|
1286
|
+
: {}),
|
|
1287
|
+
});
|
|
1288
|
+
});
|
|
1289
|
+
}
|
|
1083
1290
|
}
|
|
1084
1291
|
const clientStream = streamSource.pipeThrough(clientCaptureStream);
|
|
1085
1292
|
return new Response(clientStream, {
|
|
@@ -1198,23 +1405,27 @@ async function loadClaudeProxyAccounts(args) {
|
|
|
1198
1405
|
const accounts = [];
|
|
1199
1406
|
const legacyCredPath = `${os.homedir()}/.neurolink/anthropic-credentials.json`;
|
|
1200
1407
|
const { tokenStore } = await import("../../auth/tokenStore.js");
|
|
1408
|
+
const persistedCooldowns = await loadAccountCooldowns();
|
|
1201
1409
|
if (!startupPruneDone) {
|
|
1202
1410
|
await tokenStore.pruneExpired();
|
|
1203
1411
|
startupPruneDone = true;
|
|
1204
1412
|
}
|
|
1205
1413
|
const compoundKeys = await tokenStore.listByPrefix("anthropic:");
|
|
1206
1414
|
for (const key of compoundKeys) {
|
|
1415
|
+
if (!isAccountAllowed(key, configuredAccountAllowlist)) {
|
|
1416
|
+
logger.debug(`[proxy] skipping account=${key} (not in account allowlist)`);
|
|
1417
|
+
continue;
|
|
1418
|
+
}
|
|
1207
1419
|
if (await tokenStore.isDisabled(key)) {
|
|
1208
1420
|
const existingState = getOrCreateRuntimeState(key);
|
|
1209
|
-
const
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
if (tokenChanged) {
|
|
1421
|
+
const disabledReason = await tokenStore.getDisabledReason(key);
|
|
1422
|
+
// Older releases permanently disabled accounts after any refresh error,
|
|
1423
|
+
// including timeouts, 429s and 5xx responses. Re-evaluate those legacy
|
|
1424
|
+
// entries once under the terminal/transient classifier below.
|
|
1425
|
+
const legacyTransientDisable = disabledReason === "refresh_failed";
|
|
1426
|
+
if (legacyTransientDisable) {
|
|
1216
1427
|
await tokenStore.markEnabled(key);
|
|
1217
|
-
logger.always(`[proxy] account=${key.split(":")[1] ?? key} re-enabled
|
|
1428
|
+
logger.always(`[proxy] account=${key.split(":")[1] ?? key} re-enabled for legacy refresh failure recheck`);
|
|
1218
1429
|
existingState.permanentlyDisabled = false;
|
|
1219
1430
|
existingState.consecutiveRefreshFailures = 0;
|
|
1220
1431
|
}
|
|
@@ -1231,16 +1442,41 @@ async function loadClaudeProxyAccounts(args) {
|
|
|
1231
1442
|
let accessToken = tokens.accessToken;
|
|
1232
1443
|
let refreshTok = tokens.refreshToken;
|
|
1233
1444
|
let expiresAt = tokens.expiresAt;
|
|
1445
|
+
const label = key.split(":")[1] ?? key;
|
|
1446
|
+
const accountType = tokens.tokenType === "Bearer" ? "oauth" : "api_key";
|
|
1447
|
+
const existingState = getOrCreateRuntimeState(key);
|
|
1448
|
+
const persistedCooldown = persistedCooldowns[key];
|
|
1449
|
+
if (persistedCooldown?.coolingUntil > Date.now() &&
|
|
1450
|
+
(!existingState.coolingUntil ||
|
|
1451
|
+
persistedCooldown.coolingUntil > existingState.coolingUntil)) {
|
|
1452
|
+
existingState.coolingUntil = persistedCooldown.coolingUntil;
|
|
1453
|
+
existingState.coolingReason = persistedCooldown.reason;
|
|
1454
|
+
}
|
|
1455
|
+
const addAccount = () => {
|
|
1456
|
+
accounts.push({
|
|
1457
|
+
key,
|
|
1458
|
+
label,
|
|
1459
|
+
token: accessToken,
|
|
1460
|
+
refreshToken: refreshTok,
|
|
1461
|
+
expiresAt,
|
|
1462
|
+
type: accountType,
|
|
1463
|
+
persistTarget: { providerKey: key },
|
|
1464
|
+
});
|
|
1465
|
+
};
|
|
1234
1466
|
const isExpired = expiresAt ? expiresAt < Date.now() : false;
|
|
1235
1467
|
if (isExpired) {
|
|
1236
|
-
const label = key.split(":")[1] ?? key;
|
|
1237
|
-
const existingState = getOrCreateRuntimeState(key);
|
|
1238
1468
|
if (existingState.permanentlyDisabled) {
|
|
1239
1469
|
continue;
|
|
1240
1470
|
}
|
|
1471
|
+
if (existingState.coolingReason === "auth" &&
|
|
1472
|
+
existingState.coolingUntil &&
|
|
1473
|
+
existingState.coolingUntil > Date.now()) {
|
|
1474
|
+
addAccount();
|
|
1475
|
+
continue;
|
|
1476
|
+
}
|
|
1241
1477
|
if (!refreshTok) {
|
|
1242
1478
|
logger.always(`[proxy] skipping account=${label} (expired, no refresh token)`);
|
|
1243
|
-
await disableAccountUntilReauth({ key, label, token: accessToken, type: "oauth" }, existingState);
|
|
1479
|
+
await disableAccountUntilReauth({ key, label, token: accessToken, type: "oauth" }, existingState, "missing_refresh_token");
|
|
1244
1480
|
continue;
|
|
1245
1481
|
}
|
|
1246
1482
|
const tempAccount = {
|
|
@@ -1251,8 +1487,21 @@ async function loadClaudeProxyAccounts(args) {
|
|
|
1251
1487
|
};
|
|
1252
1488
|
const refreshed = await refreshToken(tempAccount);
|
|
1253
1489
|
if (!refreshed.success) {
|
|
1254
|
-
|
|
1255
|
-
|
|
1490
|
+
const account = {
|
|
1491
|
+
key,
|
|
1492
|
+
label,
|
|
1493
|
+
token: accessToken,
|
|
1494
|
+
type: "oauth",
|
|
1495
|
+
};
|
|
1496
|
+
if (isPermanentRefreshFailure(refreshed)) {
|
|
1497
|
+
logger.always(`[proxy] skipping account=${label} (refresh token rejected: ${refreshed.error?.slice(0, 200) ?? "unknown"})`);
|
|
1498
|
+
await disableAccountUntilReauth(account, existingState, "refresh_invalid");
|
|
1499
|
+
}
|
|
1500
|
+
else {
|
|
1501
|
+
const coolingUntil = await coolAccountAfterTransientRefreshFailure(account, existingState);
|
|
1502
|
+
logger.always(`[proxy] account=${label} refresh temporarily unavailable (${refreshed.status ?? "network"}); cooling until ${new Date(coolingUntil).toISOString()} and rotating`);
|
|
1503
|
+
addAccount();
|
|
1504
|
+
}
|
|
1256
1505
|
continue;
|
|
1257
1506
|
}
|
|
1258
1507
|
accessToken = tempAccount.token;
|
|
@@ -1265,19 +1514,12 @@ async function loadClaudeProxyAccounts(args) {
|
|
|
1265
1514
|
tokenType: "Bearer",
|
|
1266
1515
|
});
|
|
1267
1516
|
logger.always(`[proxy] refreshed expired account=${key.split(":")[1] ?? key} at startup`);
|
|
1517
|
+
await clearAuthCooldownAfterRefresh({ key }, existingState);
|
|
1268
1518
|
}
|
|
1269
|
-
|
|
1270
|
-
accounts.push({
|
|
1271
|
-
key,
|
|
1272
|
-
label: key.split(":")[1] ?? key,
|
|
1273
|
-
token: accessToken,
|
|
1274
|
-
refreshToken: refreshTok,
|
|
1275
|
-
expiresAt,
|
|
1276
|
-
type: accountType,
|
|
1277
|
-
persistTarget: { providerKey: key },
|
|
1278
|
-
});
|
|
1519
|
+
addAccount();
|
|
1279
1520
|
}
|
|
1280
|
-
if (accounts.length === 0
|
|
1521
|
+
if (accounts.length === 0 &&
|
|
1522
|
+
shouldLoadFallbackCredential(compoundKeys.length, LEGACY_ANTHROPIC_ACCOUNT_KEY, configuredAccountAllowlist)) {
|
|
1281
1523
|
try {
|
|
1282
1524
|
const creds = JSON.parse(fs.readFileSync(legacyCredPath, "utf8"));
|
|
1283
1525
|
const legacyAccount = await tryLoadLegacyAccount(creds, legacyCredPath);
|
|
@@ -1289,19 +1531,26 @@ async function loadClaudeProxyAccounts(args) {
|
|
|
1289
1531
|
// file absent or invalid
|
|
1290
1532
|
}
|
|
1291
1533
|
}
|
|
1292
|
-
if (process.env.ANTHROPIC_API_KEY &&
|
|
1534
|
+
if (process.env.ANTHROPIC_API_KEY &&
|
|
1535
|
+
accounts.length === 0 &&
|
|
1536
|
+
shouldLoadFallbackCredential(compoundKeys.length, ENV_ANTHROPIC_ACCOUNT_KEY, configuredAccountAllowlist)) {
|
|
1293
1537
|
accounts.push({
|
|
1294
|
-
key:
|
|
1538
|
+
key: ENV_ANTHROPIC_ACCOUNT_KEY,
|
|
1295
1539
|
label: "env",
|
|
1296
1540
|
token: process.env.ANTHROPIC_API_KEY,
|
|
1297
1541
|
type: "api_key",
|
|
1298
1542
|
});
|
|
1299
1543
|
}
|
|
1300
1544
|
if (accounts.length === 0) {
|
|
1301
|
-
|
|
1545
|
+
const noCredentialsMessage = configuredAccountAllowlist
|
|
1546
|
+
? "No allowed Anthropic credentials are currently available"
|
|
1547
|
+
: compoundKeys.length > 0
|
|
1548
|
+
? "Configured Anthropic accounts are disabled or unavailable"
|
|
1549
|
+
: "No Anthropic credentials found";
|
|
1550
|
+
tracer?.setError("authentication_error", noCredentialsMessage);
|
|
1302
1551
|
tracer?.end(401, Date.now() - requestStartTime);
|
|
1303
1552
|
return {
|
|
1304
|
-
response: buildLoggedClaudeError(401,
|
|
1553
|
+
response: buildLoggedClaudeError(401, noCredentialsMessage),
|
|
1305
1554
|
};
|
|
1306
1555
|
}
|
|
1307
1556
|
for (const account of accounts) {
|
|
@@ -1310,8 +1559,10 @@ async function loadClaudeProxyAccounts(args) {
|
|
|
1310
1559
|
state.lastRefreshToken !== account.refreshToken;
|
|
1311
1560
|
if (tokenChanged) {
|
|
1312
1561
|
if (state.permanentlyDisabled) {
|
|
1313
|
-
logger.always(`[proxy] account=${account.label} credentials
|
|
1562
|
+
logger.always(`[proxy] account=${account.label} eligible credentials reloaded; clearing stale runtime auth-disable state`);
|
|
1314
1563
|
}
|
|
1564
|
+
// Eligibility was already resolved while loading accounts. This only
|
|
1565
|
+
// clears stale in-process auth state after an explicit credential reload.
|
|
1315
1566
|
state.consecutiveRefreshFailures = 0;
|
|
1316
1567
|
state.permanentlyDisabled = false;
|
|
1317
1568
|
}
|
|
@@ -1608,12 +1859,17 @@ async function tryAutoClaudeFallback(args) {
|
|
|
1608
1859
|
}
|
|
1609
1860
|
}
|
|
1610
1861
|
function buildClaudeAnthropicFailureResponse(args) {
|
|
1611
|
-
const { tracer, requestStartTime, authFailureMessage, invalidRequestFailure, sawNetworkError, sawTransientFailure, sawRateLimit, lastError, orderedAccounts, buildLoggedClaudeError, logProxyBody, logFinalRequest, } = args;
|
|
1862
|
+
const { tracer, requestStartTime, authFailureMessage, authCooldownMessage, invalidRequestFailure, sawNetworkError, sawTransientFailure, sawRateLimit, lastError, orderedAccounts, buildLoggedClaudeError, logProxyBody, logFinalRequest, } = args;
|
|
1612
1863
|
if (authFailureMessage && !sawRateLimit) {
|
|
1613
1864
|
tracer?.setError("authentication_error", authFailureMessage);
|
|
1614
1865
|
tracer?.end(401, Date.now() - requestStartTime);
|
|
1615
1866
|
return buildLoggedClaudeError(401, authFailureMessage);
|
|
1616
1867
|
}
|
|
1868
|
+
if (authCooldownMessage && !sawRateLimit) {
|
|
1869
|
+
tracer?.setError("token_refresh_unavailable", authCooldownMessage);
|
|
1870
|
+
tracer?.end(503, Date.now() - requestStartTime);
|
|
1871
|
+
return buildLoggedClaudeError(503, authCooldownMessage, "token_refresh_unavailable");
|
|
1872
|
+
}
|
|
1617
1873
|
if (invalidRequestFailure) {
|
|
1618
1874
|
tracer?.setError("invalid_request_error", summarizeErrorMessage(invalidRequestFailure.body));
|
|
1619
1875
|
tracer?.end(invalidRequestFailure.status, Date.now() - requestStartTime);
|
|
@@ -1654,10 +1910,21 @@ function buildClaudeAnthropicFailureResponse(args) {
|
|
|
1654
1910
|
tracer?.end(502, Date.now() - requestStartTime);
|
|
1655
1911
|
return buildLoggedClaudeError(502, msg);
|
|
1656
1912
|
}
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1913
|
+
const now = Date.now();
|
|
1914
|
+
const activeRateLimitCooldowns = orderedAccounts
|
|
1915
|
+
.map((account) => getOrCreateRuntimeState(account.key))
|
|
1916
|
+
.filter((state) => state.coolingUntil &&
|
|
1917
|
+
state.coolingUntil > now &&
|
|
1918
|
+
state.coolingReason !== "auth");
|
|
1919
|
+
const earliestRetryAt = Math.min(...activeRateLimitCooldowns.map((state) => state.coolingUntil ?? Number.POSITIVE_INFINITY));
|
|
1920
|
+
const allAccountsCooling = orderedAccounts.length > 0 &&
|
|
1921
|
+
activeRateLimitCooldowns.length === orderedAccounts.length;
|
|
1922
|
+
const retryAfterSec = allAccountsCooling
|
|
1923
|
+
? Math.max(1, Math.ceil((earliestRetryAt - now) / 1000))
|
|
1924
|
+
: 1;
|
|
1925
|
+
const errorMessage = allAccountsCooling
|
|
1926
|
+
? `All ${orderedAccounts.length} Anthropic accounts are cooling after upstream rate limits. Earliest retry at ${new Date(earliestRetryAt).toISOString()}.`
|
|
1927
|
+
: `All ${orderedAccounts.length} accounts rate-limited after per-account retries.`;
|
|
1661
1928
|
logger.always(`[proxy] all accounts rate-limited, retry in ${retryAfterSec}s`);
|
|
1662
1929
|
const errorBody = buildClaudeError(429, errorMessage, "overloaded_error");
|
|
1663
1930
|
tracer?.setError("rate_limit_error", errorMessage);
|
|
@@ -1695,7 +1962,14 @@ async function handleAnthropicSuccessfulResponse(args) {
|
|
|
1695
1962
|
// account whose window resets soonest (max-utilization) and proactively
|
|
1696
1963
|
// skip any whose window is already rejected — without eating a 429 first.
|
|
1697
1964
|
accountState.quota = quota;
|
|
1698
|
-
maybeCoolFromQuota(accountState, quota, Date.now())
|
|
1965
|
+
if (maybeCoolFromQuota(accountState, quota, Date.now())) {
|
|
1966
|
+
const { coolingUntil, coolingReason } = accountState;
|
|
1967
|
+
if (coolingUntil !== undefined && coolingReason !== undefined) {
|
|
1968
|
+
saveAccountCooldown(account.key, coolingUntil, coolingReason).catch(() => {
|
|
1969
|
+
// Non-fatal: cooldown is already active in memory.
|
|
1970
|
+
});
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1699
1973
|
saveAccountQuota(account.label, quota).catch(() => {
|
|
1700
1974
|
// Non-fatal: quota persistence is best-effort
|
|
1701
1975
|
});
|
|
@@ -1762,14 +2036,40 @@ async function handleAnthropicStreamingSuccessResponse(args) {
|
|
|
1762
2036
|
return { response: clientError };
|
|
1763
2037
|
}
|
|
1764
2038
|
const reader = response.body.getReader();
|
|
1765
|
-
|
|
2039
|
+
let firstChunk;
|
|
2040
|
+
try {
|
|
2041
|
+
firstChunk = await reader.read();
|
|
2042
|
+
}
|
|
2043
|
+
catch (error) {
|
|
2044
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2045
|
+
logger.always(`[proxy] stream failed before first chunk account=${account.label}: ${message}; trying next account`);
|
|
2046
|
+
recordAttemptError(account.label, account.type, 502);
|
|
2047
|
+
tracer?.recordRetry(account.label, "stream_before_first_chunk");
|
|
2048
|
+
upstreamSpan?.end();
|
|
2049
|
+
logProxyBody({
|
|
2050
|
+
phase: "upstream_response",
|
|
2051
|
+
headers: responseHeaders,
|
|
2052
|
+
body: message,
|
|
2053
|
+
bodySize: Buffer.byteLength(message, "utf8"),
|
|
2054
|
+
contentType: responseHeaders["content-type"] ?? "text/event-stream",
|
|
2055
|
+
account: account.label,
|
|
2056
|
+
accountType: account.type,
|
|
2057
|
+
attempt: attemptNumber,
|
|
2058
|
+
responseStatus: 502,
|
|
2059
|
+
durationMs: Date.now() - fetchStartMs,
|
|
2060
|
+
});
|
|
2061
|
+
return { retryNextAccount: true };
|
|
2062
|
+
}
|
|
1766
2063
|
if (firstChunk.done || !firstChunk.value || firstChunk.value.length === 0) {
|
|
1767
|
-
reader.cancel()
|
|
2064
|
+
await reader.cancel().catch(() => {
|
|
2065
|
+
// Best-effort release before rotating to another account.
|
|
2066
|
+
});
|
|
1768
2067
|
logger.always(`[proxy] ← empty stream from account=${account.label}, trying next`);
|
|
1769
2068
|
tracer?.recordRetry(account.label, "empty_stream");
|
|
1770
2069
|
upstreamSpan?.end();
|
|
1771
2070
|
return { retryNextAccount: true };
|
|
1772
2071
|
}
|
|
2072
|
+
const streamOutcomeTracker = createStreamTerminalOutcomeTracker();
|
|
1773
2073
|
let mainStreamClosed = false;
|
|
1774
2074
|
const remainingStream = new ReadableStream({
|
|
1775
2075
|
start(controller) {
|
|
@@ -1786,6 +2086,7 @@ async function handleAnthropicStreamingSuccessResponse(args) {
|
|
|
1786
2086
|
}
|
|
1787
2087
|
if (done) {
|
|
1788
2088
|
mainStreamClosed = true;
|
|
2089
|
+
streamOutcomeTracker.complete();
|
|
1789
2090
|
controller.close();
|
|
1790
2091
|
return;
|
|
1791
2092
|
}
|
|
@@ -1794,6 +2095,7 @@ async function handleAnthropicStreamingSuccessResponse(args) {
|
|
|
1794
2095
|
catch (streamErr) {
|
|
1795
2096
|
const errMsg = streamErr instanceof Error ? streamErr.message : String(streamErr);
|
|
1796
2097
|
logger.always(`[proxy] mid-stream error account=${account.label}: ${errMsg}`);
|
|
2098
|
+
streamOutcomeTracker.fail(errMsg);
|
|
1797
2099
|
logStreamError({
|
|
1798
2100
|
timestamp: new Date().toISOString(),
|
|
1799
2101
|
requestId: ctx.requestId,
|
|
@@ -1812,7 +2114,8 @@ async function handleAnthropicStreamingSuccessResponse(args) {
|
|
|
1812
2114
|
},
|
|
1813
2115
|
cancel() {
|
|
1814
2116
|
mainStreamClosed = true;
|
|
1815
|
-
|
|
2117
|
+
streamOutcomeTracker.cancel();
|
|
2118
|
+
return reader.cancel();
|
|
1816
2119
|
},
|
|
1817
2120
|
});
|
|
1818
2121
|
const result = attachAnthropicSuccessStreamTelemetry({
|
|
@@ -1820,6 +2123,7 @@ async function handleAnthropicStreamingSuccessResponse(args) {
|
|
|
1820
2123
|
response,
|
|
1821
2124
|
responseHeaders,
|
|
1822
2125
|
remainingStream,
|
|
2126
|
+
streamOutcome: streamOutcomeTracker.outcome,
|
|
1823
2127
|
tracer,
|
|
1824
2128
|
requestStartTime,
|
|
1825
2129
|
attemptNumber,
|
|
@@ -1830,8 +2134,25 @@ async function handleAnthropicStreamingSuccessResponse(args) {
|
|
|
1830
2134
|
});
|
|
1831
2135
|
return { response: result };
|
|
1832
2136
|
}
|
|
2137
|
+
function getStreamFailureDetails(outcome) {
|
|
2138
|
+
if (outcome.kind === "upstream_error") {
|
|
2139
|
+
return {
|
|
2140
|
+
status: 502,
|
|
2141
|
+
errorType: "stream_error",
|
|
2142
|
+
message: outcome.message,
|
|
2143
|
+
};
|
|
2144
|
+
}
|
|
2145
|
+
if (outcome.kind === "client_cancelled") {
|
|
2146
|
+
return {
|
|
2147
|
+
status: 499,
|
|
2148
|
+
errorType: "client_cancelled",
|
|
2149
|
+
message: "Client cancelled the streaming response",
|
|
2150
|
+
};
|
|
2151
|
+
}
|
|
2152
|
+
return undefined;
|
|
2153
|
+
}
|
|
1833
2154
|
function attachAnthropicSuccessStreamTelemetry(args) {
|
|
1834
|
-
const { account, response, responseHeaders, remainingStream, tracer, requestStartTime, attemptNumber, finalBodyStr, upstreamSpan, logProxyBody, logFinalRequest, } = args;
|
|
2155
|
+
const { account, response, responseHeaders, remainingStream, streamOutcome, tracer, requestStartTime, attemptNumber, finalBodyStr, upstreamSpan, logProxyBody, logFinalRequest, } = args;
|
|
1835
2156
|
const { stream: clientCaptureStream, capture: clientCapture } = createRawStreamCapture();
|
|
1836
2157
|
let streamSource = remainingStream;
|
|
1837
2158
|
if (tracer) {
|
|
@@ -1845,8 +2166,9 @@ function attachAnthropicSuccessStreamTelemetry(args) {
|
|
|
1845
2166
|
const capturedResponse = response;
|
|
1846
2167
|
const capturedRequestBytes = finalBodyStr.length;
|
|
1847
2168
|
const capturedAccountLabel = account.label;
|
|
1848
|
-
Promise.all([telemetry, clientCapture])
|
|
1849
|
-
.then(([data, clientBody]) => {
|
|
2169
|
+
Promise.all([telemetry, clientCapture, streamOutcome])
|
|
2170
|
+
.then(([data, clientBody, rawOutcome]) => {
|
|
2171
|
+
const terminalOutcome = mergeStreamTerminalOutcome(rawOutcome, data.streamErrorMessage);
|
|
1850
2172
|
capturedTracer.setUsage({
|
|
1851
2173
|
inputTokens: data.usage.inputTokens,
|
|
1852
2174
|
outputTokens: data.usage.outputTokens,
|
|
@@ -1876,14 +2198,24 @@ function attachAnthropicSuccessStreamTelemetry(args) {
|
|
|
1876
2198
|
capturedTracer.recordMetrics();
|
|
1877
2199
|
capturedTracer.recordBodySizes(capturedRequestBytes, data.totalBytesReceived);
|
|
1878
2200
|
capturedUpstreamSpan?.end();
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
logFinalRequest(200, capturedAccountLabel, account.type, undefined, undefined, {
|
|
2201
|
+
const failure = getStreamFailureDetails(terminalOutcome);
|
|
2202
|
+
const usage = {
|
|
1882
2203
|
inputTokens: data.usage.inputTokens,
|
|
1883
2204
|
outputTokens: data.usage.outputTokens,
|
|
1884
2205
|
cacheCreationTokens: data.usage.cacheCreationInputTokens,
|
|
1885
2206
|
cacheReadTokens: data.usage.cacheReadInputTokens,
|
|
1886
|
-
}
|
|
2207
|
+
};
|
|
2208
|
+
if (failure) {
|
|
2209
|
+
capturedTracer.setError(failure.errorType, failure.message);
|
|
2210
|
+
capturedTracer.end(failure.status, Date.now() - requestStartTime);
|
|
2211
|
+
recordFinalError(failure.status, capturedAccountLabel, account.type);
|
|
2212
|
+
logFinalRequest(failure.status, capturedAccountLabel, account.type, failure.errorType, failure.message, usage);
|
|
2213
|
+
}
|
|
2214
|
+
else {
|
|
2215
|
+
capturedTracer.end(200, Date.now() - requestStartTime);
|
|
2216
|
+
recordFinalSuccess(capturedAccountLabel, account.type);
|
|
2217
|
+
logFinalRequest(200, capturedAccountLabel, account.type, undefined, undefined, usage);
|
|
2218
|
+
}
|
|
1887
2219
|
logProxyBody({
|
|
1888
2220
|
phase: "upstream_response",
|
|
1889
2221
|
headers: responseHeaders,
|
|
@@ -1918,7 +2250,23 @@ function attachAnthropicSuccessStreamTelemetry(args) {
|
|
|
1918
2250
|
});
|
|
1919
2251
|
}
|
|
1920
2252
|
catch {
|
|
1921
|
-
// Interceptor attachment failed after stream setup
|
|
2253
|
+
// Interceptor attachment failed after stream setup. Preserve delivery but
|
|
2254
|
+
// still settle the request from the actual stream terminal outcome.
|
|
2255
|
+
streamOutcome.then((outcome) => {
|
|
2256
|
+
const failure = getStreamFailureDetails(outcome);
|
|
2257
|
+
upstreamSpan?.end();
|
|
2258
|
+
if (failure) {
|
|
2259
|
+
tracer.setError(failure.errorType, failure.message);
|
|
2260
|
+
tracer.end(failure.status, Date.now() - requestStartTime);
|
|
2261
|
+
recordFinalError(failure.status, account.label, account.type);
|
|
2262
|
+
logFinalRequest(failure.status, account.label, account.type, failure.errorType, failure.message);
|
|
2263
|
+
}
|
|
2264
|
+
else {
|
|
2265
|
+
tracer.end(response.status, Date.now() - requestStartTime);
|
|
2266
|
+
recordFinalSuccess(account.label, account.type);
|
|
2267
|
+
logFinalRequest(response.status, account.label, account.type);
|
|
2268
|
+
}
|
|
2269
|
+
});
|
|
1922
2270
|
}
|
|
1923
2271
|
}
|
|
1924
2272
|
else {
|
|
@@ -1929,15 +2277,24 @@ function attachAnthropicSuccessStreamTelemetry(args) {
|
|
|
1929
2277
|
});
|
|
1930
2278
|
streamSource = streamSource.pipeThrough(noTracerInterceptor);
|
|
1931
2279
|
const capturedAccountLabel = account.label;
|
|
1932
|
-
Promise.all([noTracerTelemetry, clientCapture])
|
|
1933
|
-
.then(([data, clientBody]) => {
|
|
1934
|
-
|
|
1935
|
-
|
|
2280
|
+
Promise.all([noTracerTelemetry, clientCapture, streamOutcome])
|
|
2281
|
+
.then(([data, clientBody, rawOutcome]) => {
|
|
2282
|
+
const terminalOutcome = mergeStreamTerminalOutcome(rawOutcome, data.streamErrorMessage);
|
|
2283
|
+
const failure = getStreamFailureDetails(terminalOutcome);
|
|
2284
|
+
const usage = {
|
|
1936
2285
|
inputTokens: data.usage.inputTokens,
|
|
1937
2286
|
outputTokens: data.usage.outputTokens,
|
|
1938
2287
|
cacheCreationTokens: data.usage.cacheCreationInputTokens,
|
|
1939
2288
|
cacheReadTokens: data.usage.cacheReadInputTokens,
|
|
1940
|
-
}
|
|
2289
|
+
};
|
|
2290
|
+
if (failure) {
|
|
2291
|
+
recordFinalError(failure.status, capturedAccountLabel, account.type);
|
|
2292
|
+
logFinalRequest(failure.status, capturedAccountLabel, account.type, failure.errorType, failure.message, usage);
|
|
2293
|
+
}
|
|
2294
|
+
else {
|
|
2295
|
+
recordFinalSuccess(capturedAccountLabel, account.type);
|
|
2296
|
+
logFinalRequest(200, capturedAccountLabel, account.type, undefined, undefined, usage);
|
|
2297
|
+
}
|
|
1941
2298
|
logProxyBody({
|
|
1942
2299
|
phase: "upstream_response",
|
|
1943
2300
|
headers: responseHeaders,
|
|
@@ -1963,9 +2320,10 @@ function attachAnthropicSuccessStreamTelemetry(args) {
|
|
|
1963
2320
|
durationMs: Date.now() - requestStartTime,
|
|
1964
2321
|
});
|
|
1965
2322
|
})
|
|
1966
|
-
.catch(() => {
|
|
1967
|
-
|
|
1968
|
-
|
|
2323
|
+
.catch((error) => {
|
|
2324
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2325
|
+
recordFinalError(500, account.label, account.type);
|
|
2326
|
+
logFinalRequest(500, account.label, account.type, "stream_telemetry_error", message);
|
|
1969
2327
|
});
|
|
1970
2328
|
}
|
|
1971
2329
|
catch {
|
|
@@ -1987,8 +2345,17 @@ function attachAnthropicSuccessStreamTelemetry(args) {
|
|
|
1987
2345
|
.catch(() => {
|
|
1988
2346
|
// Non-fatal
|
|
1989
2347
|
});
|
|
1990
|
-
|
|
1991
|
-
|
|
2348
|
+
streamOutcome.then((outcome) => {
|
|
2349
|
+
const failure = getStreamFailureDetails(outcome);
|
|
2350
|
+
if (failure) {
|
|
2351
|
+
recordFinalError(failure.status, account.label, account.type);
|
|
2352
|
+
logFinalRequest(failure.status, account.label, account.type, failure.errorType, failure.message);
|
|
2353
|
+
}
|
|
2354
|
+
else {
|
|
2355
|
+
recordFinalSuccess(account.label, account.type);
|
|
2356
|
+
logFinalRequest(response.status, account.label, account.type);
|
|
2357
|
+
}
|
|
2358
|
+
});
|
|
1992
2359
|
}
|
|
1993
2360
|
}
|
|
1994
2361
|
const clientStream = streamSource.pipeThrough(clientCaptureStream);
|
|
@@ -2109,7 +2476,14 @@ async function handleAnthropicSuccessfulRetryResponse(args) {
|
|
|
2109
2476
|
// stash quota for proactive selection and proactively cool if this
|
|
2110
2477
|
// response reveals the window flipped to "rejected".
|
|
2111
2478
|
accountState.quota = retryQuota;
|
|
2112
|
-
maybeCoolFromQuota(accountState, retryQuota, Date.now())
|
|
2479
|
+
if (maybeCoolFromQuota(accountState, retryQuota, Date.now())) {
|
|
2480
|
+
const { coolingUntil, coolingReason } = accountState;
|
|
2481
|
+
if (coolingUntil !== undefined && coolingReason !== undefined) {
|
|
2482
|
+
saveAccountCooldown(account.key, coolingUntil, coolingReason).catch(() => {
|
|
2483
|
+
// Non-fatal: cooldown is already active in memory.
|
|
2484
|
+
});
|
|
2485
|
+
}
|
|
2486
|
+
}
|
|
2113
2487
|
saveAccountQuota(account.label, retryQuota).catch((error) => {
|
|
2114
2488
|
logger.debug("[proxy] Failed to persist account quota after auth retry", {
|
|
2115
2489
|
account: account.label,
|
|
@@ -2119,6 +2493,7 @@ async function handleAnthropicSuccessfulRetryResponse(args) {
|
|
|
2119
2493
|
}
|
|
2120
2494
|
if (body.stream && retryResp.body) {
|
|
2121
2495
|
const retryReader = retryResp.body.getReader();
|
|
2496
|
+
const streamOutcomeTracker = createStreamTerminalOutcomeTracker();
|
|
2122
2497
|
let retryStreamClosed = false;
|
|
2123
2498
|
const retryStream = new ReadableStream({
|
|
2124
2499
|
async pull(controller) {
|
|
@@ -2132,6 +2507,7 @@ async function handleAnthropicSuccessfulRetryResponse(args) {
|
|
|
2132
2507
|
}
|
|
2133
2508
|
if (done) {
|
|
2134
2509
|
retryStreamClosed = true;
|
|
2510
|
+
streamOutcomeTracker.complete();
|
|
2135
2511
|
controller.close();
|
|
2136
2512
|
return;
|
|
2137
2513
|
}
|
|
@@ -2140,6 +2516,7 @@ async function handleAnthropicSuccessfulRetryResponse(args) {
|
|
|
2140
2516
|
catch (streamErr) {
|
|
2141
2517
|
const errMsg = streamErr instanceof Error ? streamErr.message : String(streamErr);
|
|
2142
2518
|
logger.always(`[proxy] mid-stream error (auth-retry) account=${account.label}: ${errMsg}`);
|
|
2519
|
+
streamOutcomeTracker.fail(errMsg);
|
|
2143
2520
|
logStreamError({
|
|
2144
2521
|
timestamp: new Date().toISOString(),
|
|
2145
2522
|
requestId: ctx.requestId,
|
|
@@ -2158,74 +2535,23 @@ async function handleAnthropicSuccessfulRetryResponse(args) {
|
|
|
2158
2535
|
},
|
|
2159
2536
|
cancel() {
|
|
2160
2537
|
retryStreamClosed = true;
|
|
2161
|
-
|
|
2538
|
+
streamOutcomeTracker.cancel();
|
|
2539
|
+
return retryReader.cancel();
|
|
2162
2540
|
},
|
|
2163
2541
|
});
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
inputTokens: data.usage.inputTokens,
|
|
2178
|
-
outputTokens: data.usage.outputTokens,
|
|
2179
|
-
cacheCreationTokens: data.usage.cacheCreationInputTokens,
|
|
2180
|
-
cacheReadTokens: data.usage.cacheReadInputTokens,
|
|
2181
|
-
});
|
|
2182
|
-
capturedTracer.logStreamEvents(data.events);
|
|
2183
|
-
capturedTracer.setResponseInfo(responseInfoFromStream(data));
|
|
2184
|
-
capturedTracer.logUpstreamResponseHeaders(Object.fromEntries([...capturedRetryResp.headers.entries()]));
|
|
2185
|
-
capturedTracer.recordMetrics();
|
|
2186
|
-
capturedTracer.recordBodySizes(capturedRetryRequestBytes, data.totalBytesReceived);
|
|
2187
|
-
capturedUpstreamSpan?.end();
|
|
2188
|
-
capturedTracer.end(200, Date.now() - requestStartTime);
|
|
2189
|
-
recordFinalSuccess(capturedAccountLabel, account.type);
|
|
2190
|
-
logFinalRequest(200, capturedAccountLabel, account.type, undefined, undefined, {
|
|
2191
|
-
inputTokens: data.usage.inputTokens,
|
|
2192
|
-
outputTokens: data.usage.outputTokens,
|
|
2193
|
-
cacheCreationTokens: data.usage.cacheCreationInputTokens,
|
|
2194
|
-
cacheReadTokens: data.usage.cacheReadInputTokens,
|
|
2195
|
-
});
|
|
2196
|
-
})
|
|
2197
|
-
.catch((error) => {
|
|
2198
|
-
capturedTracer.setError("stream_error", error instanceof Error ? error.message : String(error));
|
|
2199
|
-
capturedUpstreamSpan?.end();
|
|
2200
|
-
capturedTracer.end(500, Date.now() - requestStartTime);
|
|
2201
|
-
recordFinalError(500, capturedAccountLabel, account.type);
|
|
2202
|
-
logFinalRequest(500, capturedAccountLabel, account.type, "stream_error", error instanceof Error ? error.message : String(error));
|
|
2203
|
-
});
|
|
2204
|
-
}
|
|
2205
|
-
catch {
|
|
2206
|
-
retryClientStream = retryStream;
|
|
2207
|
-
}
|
|
2208
|
-
}
|
|
2209
|
-
const responseHeaders = {
|
|
2210
|
-
"content-type": "text/event-stream",
|
|
2211
|
-
"cache-control": "no-cache",
|
|
2212
|
-
connection: "keep-alive",
|
|
2213
|
-
};
|
|
2214
|
-
for (const headerName of [
|
|
2215
|
-
"retry-after",
|
|
2216
|
-
"anthropic-ratelimit-requests-remaining",
|
|
2217
|
-
"anthropic-ratelimit-requests-limit",
|
|
2218
|
-
"anthropic-ratelimit-tokens-remaining",
|
|
2219
|
-
"anthropic-ratelimit-tokens-limit",
|
|
2220
|
-
]) {
|
|
2221
|
-
const value = retryResp.headers.get(headerName);
|
|
2222
|
-
if (value) {
|
|
2223
|
-
responseHeaders[headerName] = value;
|
|
2224
|
-
}
|
|
2225
|
-
}
|
|
2226
|
-
return new Response(retryClientStream, {
|
|
2227
|
-
status: retryResp.status,
|
|
2228
|
-
headers: responseHeaders,
|
|
2542
|
+
return attachAnthropicSuccessStreamTelemetry({
|
|
2543
|
+
account,
|
|
2544
|
+
response: retryResp,
|
|
2545
|
+
responseHeaders: Object.fromEntries([...retryResp.headers.entries()]),
|
|
2546
|
+
remainingStream: retryStream,
|
|
2547
|
+
streamOutcome: streamOutcomeTracker.outcome,
|
|
2548
|
+
tracer,
|
|
2549
|
+
requestStartTime,
|
|
2550
|
+
attemptNumber,
|
|
2551
|
+
finalBodyStr,
|
|
2552
|
+
upstreamSpan,
|
|
2553
|
+
logProxyBody,
|
|
2554
|
+
logFinalRequest,
|
|
2229
2555
|
});
|
|
2230
2556
|
}
|
|
2231
2557
|
const retryRespHeaders = Object.fromEntries([...retryResp.headers.entries()]);
|
|
@@ -2303,24 +2629,24 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
2303
2629
|
logger.always(`[proxy] ← 401 account=${account.label} refreshing (attempt ${authRetry + 1}/${MAX_AUTH_RETRIES})`);
|
|
2304
2630
|
const refreshSucceeded = await refreshToken(account);
|
|
2305
2631
|
if (!refreshSucceeded.success) {
|
|
2306
|
-
accountState.consecutiveRefreshFailures += 1;
|
|
2307
2632
|
authRetryError = `refresh failed for account=${account.label} attempt ${authRetry + 1}/${MAX_AUTH_RETRIES}: ${refreshSucceeded.error?.slice(0, 200) ?? "unknown"}`;
|
|
2308
2633
|
currentLastError = authRetryError;
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
MAX_CONSECUTIVE_REFRESH_FAILURES) {
|
|
2312
|
-
await disableAccountUntilReauth(account, accountState);
|
|
2634
|
+
if (isPermanentRefreshFailure(refreshSucceeded)) {
|
|
2635
|
+
await disableAccountUntilReauth(account, accountState, "refresh_invalid");
|
|
2313
2636
|
currentAuthFailureMessage = formatReauthMessage(account.label);
|
|
2314
|
-
|
|
2637
|
+
logger.always(`[proxy] account=${account.label} refresh token rejected; disabled until re-authentication`);
|
|
2315
2638
|
}
|
|
2316
|
-
|
|
2317
|
-
await
|
|
2639
|
+
else {
|
|
2640
|
+
const coolingUntil = await coolAccountAfterTransientRefreshFailure(account, accountState);
|
|
2641
|
+
currentSawTransientFailure = true;
|
|
2642
|
+
logger.always(`[proxy] account=${account.label} refresh temporarily unavailable (${refreshSucceeded.status ?? "network"}); cooling until ${new Date(coolingUntil).toISOString()} and rotating`);
|
|
2318
2643
|
}
|
|
2319
|
-
|
|
2644
|
+
break;
|
|
2320
2645
|
}
|
|
2321
2646
|
if (account.persistTarget) {
|
|
2322
2647
|
await persistTokens(account.persistTarget, account);
|
|
2323
2648
|
}
|
|
2649
|
+
await clearAuthCooldownAfterRefresh(account, accountState);
|
|
2324
2650
|
headers.authorization = `Bearer ${account.token}`;
|
|
2325
2651
|
try {
|
|
2326
2652
|
const retryResp = await fetch("https://api.anthropic.com/v1/messages?beta=true", {
|
|
@@ -2407,12 +2733,20 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
2407
2733
|
if (retryQuota429) {
|
|
2408
2734
|
accountState.quota = retryQuota429;
|
|
2409
2735
|
}
|
|
2410
|
-
const retryPlan = planCooldownFor429(retryQuota429, parseRetryAfterMs(retryRespHeaders["retry-after"] ?? null), nowRetry);
|
|
2736
|
+
const retryPlan = planCooldownFor429(retryQuota429, parseRetryAfterMs(retryRespHeaders["retry-after"] ?? null), nowRetry, getUnifiedRateLimitStatus(retryRespHeaders));
|
|
2411
2737
|
if (!accountState.coolingUntil ||
|
|
2412
2738
|
retryPlan.coolingUntil > accountState.coolingUntil) {
|
|
2413
2739
|
accountState.coolingUntil = retryPlan.coolingUntil;
|
|
2414
2740
|
accountState.coolingReason = retryPlan.reason;
|
|
2415
2741
|
}
|
|
2742
|
+
if (retryQuota429) {
|
|
2743
|
+
saveAccountQuota(account.label, retryQuota429).catch(() => {
|
|
2744
|
+
// Non-fatal: routing already has the in-memory snapshot.
|
|
2745
|
+
});
|
|
2746
|
+
}
|
|
2747
|
+
await saveAccountCooldown(account.key, accountState.coolingUntil ?? retryPlan.coolingUntil, accountState.coolingReason ?? retryPlan.reason).catch(() => {
|
|
2748
|
+
// Non-fatal: routing already has the in-memory cooldown.
|
|
2749
|
+
});
|
|
2416
2750
|
advancePrimaryIfCurrent(account.key, enabledAccounts.length, orderedAccounts[0]?.key);
|
|
2417
2751
|
break;
|
|
2418
2752
|
}
|
|
@@ -2529,7 +2863,7 @@ function buildAnthropicTerminalErrorResponse(args) {
|
|
|
2529
2863
|
}
|
|
2530
2864
|
}
|
|
2531
2865
|
async function handleAnthropicNonOkResponse(args) {
|
|
2532
|
-
const { response, account, accountState, enabledAccounts, orderedAccounts, tracer, requestStartTime, fetchStartMs, attemptNumber, logAttempt, logProxyBody, logFinalRequest, lastError, authFailureMessage, sawTransientFailure, invalidRequestFailure,
|
|
2866
|
+
const { response, account, accountState, enabledAccounts, orderedAccounts, tracer, requestStartTime, fetchStartMs, attemptNumber, logAttempt, logProxyBody, logFinalRequest, lastError, authFailureMessage, sawTransientFailure, invalidRequestFailure, } = args;
|
|
2533
2867
|
let currentLastError = lastError;
|
|
2534
2868
|
let currentAuthFailureMessage = authFailureMessage;
|
|
2535
2869
|
let currentSawTransientFailure = sawTransientFailure;
|
|
@@ -2578,10 +2912,7 @@ async function handleAnthropicNonOkResponse(args) {
|
|
|
2578
2912
|
account.type === "oauth" &&
|
|
2579
2913
|
!account.refreshToken) {
|
|
2580
2914
|
recordAttemptError(account.label, account.type, response.status);
|
|
2581
|
-
accountState
|
|
2582
|
-
if (accountState.consecutiveRefreshFailures >= maxConsecutiveRefreshFailures) {
|
|
2583
|
-
await disableAccountUntilReauth(account, accountState);
|
|
2584
|
-
}
|
|
2915
|
+
await disableAccountUntilReauth(account, accountState, "missing_refresh_token");
|
|
2585
2916
|
currentAuthFailureMessage = formatReauthMessage(account.label);
|
|
2586
2917
|
logger.always(`[proxy] ← ${response.status} account=${account.label} (auth failure, no refresh token)`);
|
|
2587
2918
|
currentLastError = errBody;
|
|
@@ -2847,23 +3178,25 @@ async function prepareAnthropicAccountAttempt(args) {
|
|
|
2847
3178
|
if (account.persistTarget) {
|
|
2848
3179
|
await persistTokens(account.persistTarget, account);
|
|
2849
3180
|
}
|
|
2850
|
-
|
|
3181
|
+
await clearAuthCooldownAfterRefresh(account, accountState);
|
|
2851
3182
|
}
|
|
2852
3183
|
else {
|
|
2853
|
-
accountState.consecutiveRefreshFailures += 1;
|
|
2854
3184
|
lastError = `token refresh failed for account=${account.label}: ${refreshed.error?.slice(0, 200) ?? "unknown"}`;
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
MAX_CONSECUTIVE_REFRESH_FAILURES) {
|
|
2858
|
-
await disableAccountUntilReauth(account, accountState);
|
|
3185
|
+
if (isPermanentRefreshFailure(refreshed)) {
|
|
3186
|
+
await disableAccountUntilReauth(account, accountState, "refresh_invalid");
|
|
2859
3187
|
authFailureMessage = formatReauthMessage(account.label);
|
|
2860
|
-
logAttempt(401, "authentication_error", String(lastError));
|
|
2861
|
-
return {
|
|
2862
|
-
continueLoop: true,
|
|
2863
|
-
lastError,
|
|
2864
|
-
authFailureMessage,
|
|
2865
|
-
};
|
|
2866
3188
|
}
|
|
3189
|
+
else {
|
|
3190
|
+
await coolAccountAfterTransientRefreshFailure(account, accountState);
|
|
3191
|
+
}
|
|
3192
|
+
logAttempt(isPermanentRefreshFailure(refreshed) ? 401 : 503, isPermanentRefreshFailure(refreshed)
|
|
3193
|
+
? "authentication_error"
|
|
3194
|
+
: "token_refresh_unavailable", String(lastError));
|
|
3195
|
+
return {
|
|
3196
|
+
continueLoop: true,
|
|
3197
|
+
lastError,
|
|
3198
|
+
authFailureMessage,
|
|
3199
|
+
};
|
|
2867
3200
|
}
|
|
2868
3201
|
}
|
|
2869
3202
|
const isOAuth = account.type === "oauth";
|
|
@@ -3103,10 +3436,12 @@ async function fetchAnthropicAccountResponse(args) {
|
|
|
3103
3436
|
// ACTUAL reset, not a 60s hardcap.
|
|
3104
3437
|
const now = Date.now();
|
|
3105
3438
|
const quota = parseQuotaHeaders(errRespHeaders);
|
|
3106
|
-
const
|
|
3439
|
+
const unifiedStatus = getUnifiedRateLimitStatus(errRespHeaders);
|
|
3440
|
+
const cooldownPlan = planCooldownFor429(quota, retryAfterMs, now, unifiedStatus);
|
|
3107
3441
|
logger.always(`[proxy] ← 429 account=${account.label} reason=${cooldownPlan.reason} ` +
|
|
3108
3442
|
`retry-after=${retryAfterMs}ms 5h-status=${errRespHeaders["anthropic-ratelimit-unified-5h-status"] ?? "unknown"} ` +
|
|
3109
3443
|
`7d-status=${errRespHeaders["anthropic-ratelimit-unified-7d-status"] ?? "unknown"} ` +
|
|
3444
|
+
`unified-status=${unifiedStatus ?? "unknown"} ` +
|
|
3110
3445
|
`→ ${cooldownPlan.rotateImmediately ? `rotate now, cool ${minutesUntil(cooldownPlan.coolingUntil, now)}m` : "retry same account (transient)"}`);
|
|
3111
3446
|
logAttempt(429, "rate_limit_error", String(lastError));
|
|
3112
3447
|
tracer?.setError("rate_limit_error", String(lastError).slice(0, 500));
|
|
@@ -3155,6 +3490,7 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
3155
3490
|
sawTransientFailure: false,
|
|
3156
3491
|
invalidRequestFailure: null,
|
|
3157
3492
|
authFailureMessage: null,
|
|
3493
|
+
authCooldownMessage: null,
|
|
3158
3494
|
attemptNumber: 0,
|
|
3159
3495
|
};
|
|
3160
3496
|
const acctSelectionSpan = tracer?.startAccountSelection();
|
|
@@ -3167,10 +3503,24 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
3167
3503
|
if (!usingQuotaOrder) {
|
|
3168
3504
|
maybeResetPrimaryToHome(enabledAccounts);
|
|
3169
3505
|
}
|
|
3170
|
-
//
|
|
3171
|
-
//
|
|
3506
|
+
// Never re-hammer accounts with a known active cooldown. When every account
|
|
3507
|
+
// is cooling, report the earliest persisted retry time without an upstream
|
|
3508
|
+
// call; restarting the proxy must not erase or bypass this quarantine.
|
|
3172
3509
|
const nonCoolingAccounts = orderedAccounts.filter((a) => !isAccountCooling(a.key));
|
|
3173
|
-
const effectiveAccounts = nonCoolingAccounts
|
|
3510
|
+
const effectiveAccounts = nonCoolingAccounts;
|
|
3511
|
+
if (effectiveAccounts.length === 0 && orderedAccounts.length > 0) {
|
|
3512
|
+
const coolingStates = orderedAccounts.map((account) => getOrCreateRuntimeState(account.key));
|
|
3513
|
+
const hasRateLimitCooldown = coolingStates.some((state) => state.coolingReason !== "auth");
|
|
3514
|
+
loopState.sawRateLimit = hasRateLimitCooldown;
|
|
3515
|
+
loopState.sawTransientFailure = !hasRateLimitCooldown;
|
|
3516
|
+
loopState.lastError = hasRateLimitCooldown
|
|
3517
|
+
? "All Anthropic accounts have active rate-limit cooldowns"
|
|
3518
|
+
: "All Anthropic accounts have active authentication cooldowns";
|
|
3519
|
+
if (!hasRateLimitCooldown) {
|
|
3520
|
+
const earliestRetryAt = Math.min(...coolingStates.map((state) => state.coolingUntil ?? Number.POSITIVE_INFINITY));
|
|
3521
|
+
loopState.authCooldownMessage = `All ${orderedAccounts.length} Anthropic accounts are temporarily unavailable while OAuth refresh is cooling. Earliest retry at ${new Date(earliestRetryAt).toISOString()}.`;
|
|
3522
|
+
}
|
|
3523
|
+
}
|
|
3174
3524
|
accountLoop: for (const account of effectiveAccounts) {
|
|
3175
3525
|
const accountState = getOrCreateRuntimeState(account.key);
|
|
3176
3526
|
let transientSameAccountRetries = 0;
|
|
@@ -3247,6 +3597,9 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
3247
3597
|
// Refresh the account's quota snapshot for proactive selection.
|
|
3248
3598
|
if (fetchResult.quota) {
|
|
3249
3599
|
accountState.quota = fetchResult.quota;
|
|
3600
|
+
saveAccountQuota(account.label, fetchResult.quota).catch(() => {
|
|
3601
|
+
// Non-fatal: routing already has the in-memory snapshot.
|
|
3602
|
+
});
|
|
3250
3603
|
}
|
|
3251
3604
|
// Transient burst (window still allowed): a couple of jittered
|
|
3252
3605
|
// same-account retries before giving up. Exhaustion plans set
|
|
@@ -3271,6 +3624,9 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
3271
3624
|
accountState.coolingUntil = plan.coolingUntil;
|
|
3272
3625
|
accountState.coolingReason = plan.reason;
|
|
3273
3626
|
}
|
|
3627
|
+
await saveAccountCooldown(account.key, accountState.coolingUntil ?? plan.coolingUntil, accountState.coolingReason ?? plan.reason).catch(() => {
|
|
3628
|
+
// Non-fatal: routing already has the in-memory cooldown.
|
|
3629
|
+
});
|
|
3274
3630
|
advancePrimaryIfCurrent(account.key, enabledAccounts.length, orderedAccounts[0]?.key);
|
|
3275
3631
|
logger.always(`[proxy] account=${account.label} rate-limited (${plan.reason}); cooling ~${minutesUntil(plan.coolingUntil, Date.now())}m until ${new Date(plan.coolingUntil).toISOString()}, rotating`);
|
|
3276
3632
|
continue accountLoop;
|
|
@@ -3350,7 +3706,6 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
3350
3706
|
authFailureMessage: loopState.authFailureMessage,
|
|
3351
3707
|
sawTransientFailure: loopState.sawTransientFailure,
|
|
3352
3708
|
invalidRequestFailure: loopState.invalidRequestFailure,
|
|
3353
|
-
maxConsecutiveRefreshFailures: MAX_CONSECUTIVE_REFRESH_FAILURES,
|
|
3354
3709
|
});
|
|
3355
3710
|
loopState.lastError = nonOkResult.lastError;
|
|
3356
3711
|
loopState.authFailureMessage = nonOkResult.authFailureMessage;
|
|
@@ -3382,8 +3737,12 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
3382
3737
|
// window flipped to "rejected" on this very request.
|
|
3383
3738
|
if (accountState.coolingUntil &&
|
|
3384
3739
|
Date.now() >= accountState.coolingUntil) {
|
|
3740
|
+
const expiredCooldown = accountState.coolingUntil;
|
|
3385
3741
|
accountState.coolingUntil = undefined;
|
|
3386
3742
|
accountState.coolingReason = undefined;
|
|
3743
|
+
clearAccountCooldown(account.key, expiredCooldown).catch(() => {
|
|
3744
|
+
// Best-effort cleanup; expired entries are ignored during seeding.
|
|
3745
|
+
});
|
|
3387
3746
|
}
|
|
3388
3747
|
const successResult = await handleAnthropicSuccessfulResponse({
|
|
3389
3748
|
ctx,
|
|
@@ -3441,6 +3800,7 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
3441
3800
|
tracer,
|
|
3442
3801
|
requestStartTime,
|
|
3443
3802
|
authFailureMessage: loopState.authFailureMessage,
|
|
3803
|
+
authCooldownMessage: loopState.authCooldownMessage,
|
|
3444
3804
|
invalidRequestFailure: loopState.invalidRequestFailure,
|
|
3445
3805
|
sawNetworkError: loopState.sawNetworkError,
|
|
3446
3806
|
sawTransientFailure: loopState.sawTransientFailure,
|
|
@@ -3465,8 +3825,11 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
3465
3825
|
* @param basePath - Base path prefix (default: "" since Claude API uses /v1/...).
|
|
3466
3826
|
* @returns RouteGroup with Claude-compatible endpoints.
|
|
3467
3827
|
*/
|
|
3468
|
-
export function createClaudeProxyRoutes(modelRouter, basePath = "", accountStrategy = "fill-first", passthroughMode = false, primaryAccountKey) {
|
|
3828
|
+
export function createClaudeProxyRoutes(modelRouter, basePath = "", accountStrategy = "fill-first", passthroughMode = false, primaryAccountKey, accountAllowlist) {
|
|
3469
3829
|
configuredPrimaryAccountKey = primaryAccountKey;
|
|
3830
|
+
configuredAccountAllowlist = accountAllowlist
|
|
3831
|
+
? new Set(accountAllowlist)
|
|
3832
|
+
: undefined;
|
|
3470
3833
|
return {
|
|
3471
3834
|
prefix: `${basePath}/v1`,
|
|
3472
3835
|
routes: [
|
|
@@ -3630,18 +3993,48 @@ function getOrCreateRuntimeState(accountKey) {
|
|
|
3630
3993
|
accountRuntimeState.set(accountKey, initial);
|
|
3631
3994
|
return initial;
|
|
3632
3995
|
}
|
|
3633
|
-
async function disableAccountUntilReauth(account, state) {
|
|
3996
|
+
async function disableAccountUntilReauth(account, state, reason) {
|
|
3634
3997
|
state.permanentlyDisabled = true;
|
|
3635
3998
|
// Decision 7 (usage): Persist disabled state to disk so it survives restarts
|
|
3636
3999
|
try {
|
|
3637
4000
|
const { tokenStore } = await import("../../auth/tokenStore.js");
|
|
3638
|
-
await tokenStore.markDisabled(account.key,
|
|
4001
|
+
await tokenStore.markDisabled(account.key, reason);
|
|
3639
4002
|
}
|
|
3640
4003
|
catch (e) {
|
|
3641
4004
|
logger.debug(`[proxy] failed to persist disabled state for ${account.label}: ${e instanceof Error ? e.message : String(e)}`);
|
|
3642
4005
|
}
|
|
3643
4006
|
logger.always(`[proxy] account=${account.label} disabled until re-authentication. Run: neurolink auth login anthropic --method oauth`);
|
|
3644
4007
|
}
|
|
4008
|
+
async function coolAccountAfterTransientRefreshFailure(account, state) {
|
|
4009
|
+
state.consecutiveRefreshFailures += 1;
|
|
4010
|
+
const exponent = Math.min(state.consecutiveRefreshFailures - 1, 4);
|
|
4011
|
+
const delayMs = Math.min(AUTH_REFRESH_MAX_COOLDOWN_MS, AUTH_REFRESH_BASE_COOLDOWN_MS * 2 ** exponent);
|
|
4012
|
+
const coolingUntil = Date.now() + delayMs;
|
|
4013
|
+
if (!state.coolingUntil || coolingUntil > state.coolingUntil) {
|
|
4014
|
+
state.coolingUntil = coolingUntil;
|
|
4015
|
+
state.coolingReason = "auth";
|
|
4016
|
+
}
|
|
4017
|
+
const effectiveCoolingUntil = state.coolingUntil ?? coolingUntil;
|
|
4018
|
+
const effectiveReason = state.coolingReason ?? "auth";
|
|
4019
|
+
state.coolingUntil = effectiveCoolingUntil;
|
|
4020
|
+
state.coolingReason = effectiveReason;
|
|
4021
|
+
await saveAccountCooldown(account.key, effectiveCoolingUntil, effectiveReason).catch(() => {
|
|
4022
|
+
// Non-fatal: the in-memory cooldown still prevents immediate re-hammering.
|
|
4023
|
+
});
|
|
4024
|
+
return effectiveCoolingUntil;
|
|
4025
|
+
}
|
|
4026
|
+
async function clearAuthCooldownAfterRefresh(account, state) {
|
|
4027
|
+
state.consecutiveRefreshFailures = 0;
|
|
4028
|
+
if (state.coolingReason !== "auth" || !state.coolingUntil) {
|
|
4029
|
+
return;
|
|
4030
|
+
}
|
|
4031
|
+
const previousCoolingUntil = state.coolingUntil;
|
|
4032
|
+
state.coolingUntil = undefined;
|
|
4033
|
+
state.coolingReason = undefined;
|
|
4034
|
+
await clearAccountCooldown(account.key, previousCoolingUntil).catch(() => {
|
|
4035
|
+
// Best-effort cleanup; an expired auth cooldown is harmless on next boot.
|
|
4036
|
+
});
|
|
4037
|
+
}
|
|
3645
4038
|
function formatReauthMessage(labels) {
|
|
3646
4039
|
const value = Array.isArray(labels) ? labels.join(", ") : labels;
|
|
3647
4040
|
return `Account(s) require re-authentication: ${value}. Run: neurolink auth login anthropic --method oauth`;
|
|
@@ -3814,6 +4207,9 @@ export const __testHooks = {
|
|
|
3814
4207
|
resolveHomeIndex,
|
|
3815
4208
|
maybeResetPrimaryToHome,
|
|
3816
4209
|
planCooldownFor429,
|
|
4210
|
+
isPermanentRefreshFailure,
|
|
4211
|
+
getStreamFailureDetails,
|
|
4212
|
+
trackUpstreamReadableStream,
|
|
3817
4213
|
orderAccountsByQuota,
|
|
3818
4214
|
resetEpochToMs,
|
|
3819
4215
|
seedRuntimeQuotasFromDisk,
|
|
@@ -3825,6 +4221,10 @@ export const __testHooks = {
|
|
|
3825
4221
|
configuredPrimaryAccountKey = key;
|
|
3826
4222
|
},
|
|
3827
4223
|
getConfiguredPrimaryAccountKey: () => configuredPrimaryAccountKey,
|
|
4224
|
+
setConfiguredAccountAllowlist: (allowlist) => {
|
|
4225
|
+
configuredAccountAllowlist = allowlist ? new Set(allowlist) : undefined;
|
|
4226
|
+
},
|
|
4227
|
+
getConfiguredAccountAllowlist: () => configuredAccountAllowlist ? [...configuredAccountAllowlist] : undefined,
|
|
3828
4228
|
setPrimaryAccountIndex: (index) => {
|
|
3829
4229
|
primaryAccountIndex = index;
|
|
3830
4230
|
},
|
|
@@ -3839,6 +4239,7 @@ export const __testHooks = {
|
|
|
3839
4239
|
primaryAccountIndex = 0;
|
|
3840
4240
|
lastKnownAccountCount = 0;
|
|
3841
4241
|
configuredPrimaryAccountKey = undefined;
|
|
4242
|
+
configuredAccountAllowlist = undefined;
|
|
3842
4243
|
},
|
|
3843
4244
|
};
|
|
3844
4245
|
//# sourceMappingURL=claudeProxyRoutes.js.map
|