@juspay/neurolink 10.9.0 → 10.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/browser/neurolink.min.js +379 -379
- package/dist/cli/commands/proxy.js +80 -20
- package/dist/lib/providers/anthropic/client.d.ts +22 -7
- package/dist/lib/providers/anthropic/client.js +83 -58
- package/dist/lib/providers/anthropic/rateLimitCapture.d.ts +82 -0
- package/dist/lib/providers/anthropic/rateLimitCapture.js +375 -0
- package/dist/lib/proxy/proxyLifecycle.d.ts +5 -0
- package/dist/lib/proxy/proxyLifecycle.js +61 -11
- package/dist/lib/proxy/quotaHeaders.d.ts +73 -0
- package/dist/lib/proxy/quotaHeaders.js +189 -0
- package/dist/lib/proxy/usageStats.d.ts +2 -0
- package/dist/lib/proxy/usageStats.js +4 -0
- package/dist/lib/server/routes/claudeProxyRoutes.js +132 -17
- package/dist/lib/types/analytics.d.ts +8 -0
- package/dist/lib/types/generate.d.ts +12 -0
- package/dist/lib/types/proxy.d.ts +50 -0
- package/dist/lib/types/subscription.d.ts +77 -0
- package/dist/providers/anthropic/client.d.ts +22 -7
- package/dist/providers/anthropic/client.js +83 -58
- package/dist/providers/anthropic/rateLimitCapture.d.ts +82 -0
- package/dist/providers/anthropic/rateLimitCapture.js +374 -0
- package/dist/proxy/proxyLifecycle.d.ts +5 -0
- package/dist/proxy/proxyLifecycle.js +61 -11
- package/dist/proxy/quotaHeaders.d.ts +73 -0
- package/dist/proxy/quotaHeaders.js +188 -0
- package/dist/proxy/usageStats.d.ts +2 -0
- package/dist/proxy/usageStats.js +4 -0
- package/dist/server/routes/claudeProxyRoutes.js +132 -17
- package/dist/types/analytics.d.ts +8 -0
- package/dist/types/generate.d.ts +12 -0
- package/dist/types/proxy.d.ts +50 -0
- package/dist/types/subscription.d.ts +77 -0
- package/package.json +3 -1
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Proxy response quota headers.
|
|
3
|
+
*
|
|
4
|
+
* The proxy already knows, per request, exactly how much subscription capacity
|
|
5
|
+
* the serving account has left — it parses Anthropic's `anthropic-ratelimit-*`
|
|
6
|
+
* headers to route on them (see `accountQuota.ts`). Historically none of that
|
|
7
|
+
* reached the client: only the SSE path forwarded a small legacy allowlist, and
|
|
8
|
+
* every JSON/error path dropped headers entirely.
|
|
9
|
+
*
|
|
10
|
+
* This module turns that state into response headers in two layers:
|
|
11
|
+
*
|
|
12
|
+
* 1. **Verbatim passthrough** of Anthropic's own `anthropic-ratelimit-*` and
|
|
13
|
+
* `retry-after` headers. This is the load-bearing part: a proxied response
|
|
14
|
+
* then looks byte-identical to a direct one, so a consumer needs exactly one
|
|
15
|
+
* parser for both.
|
|
16
|
+
* 2. **`x-neurolink-*`** for what only the proxy can know — which account
|
|
17
|
+
* served the request, pool headroom, whether the numbers are live or stale,
|
|
18
|
+
* and the derived "how much is left" percentages.
|
|
19
|
+
*
|
|
20
|
+
* Pure CPU, no I/O — safe on the hot path and directly unit-testable.
|
|
21
|
+
*
|
|
22
|
+
* @module proxy/quotaHeaders
|
|
23
|
+
*/
|
|
24
|
+
/** Upstream headers forwarded to the client untouched. */
|
|
25
|
+
const UPSTREAM_PREFIX = "anthropic-ratelimit-";
|
|
26
|
+
const UPSTREAM_EXTRA_HEADERS = ["retry-after"];
|
|
27
|
+
/**
|
|
28
|
+
* A value beyond ~year 2100 expressed in seconds is already milliseconds —
|
|
29
|
+
* same heuristic the cooldown planner uses, kept in sync deliberately so
|
|
30
|
+
* "resets in" and the actual cooldown never disagree.
|
|
31
|
+
*/
|
|
32
|
+
const EPOCH_SECONDS_CEILING = 4_102_444_800;
|
|
33
|
+
/**
|
|
34
|
+
* Strip anything that cannot legally appear in a header value.
|
|
35
|
+
*
|
|
36
|
+
* Account labels originate from user email addresses, so this is not
|
|
37
|
+
* theoretical: a CR/LF in a label would let a crafted account name inject
|
|
38
|
+
* additional response headers. Non-ASCII is dropped rather than encoded —
|
|
39
|
+
* these are diagnostic values, not content.
|
|
40
|
+
*/
|
|
41
|
+
function sanitizeHeaderValue(value) {
|
|
42
|
+
return value.replace(/[^\x20-\x7E]/g, "").trim();
|
|
43
|
+
}
|
|
44
|
+
/** Epoch seconds (or ms) → seconds from now, floored at 0. Undefined when the
|
|
45
|
+
* timestamp is absent, zero, or already in the past. */
|
|
46
|
+
function secondsUntil(resetEpoch, now) {
|
|
47
|
+
if (!resetEpoch || resetEpoch <= 0) {
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
const ms = resetEpoch > EPOCH_SECONDS_CEILING ? resetEpoch : resetEpoch * 1000;
|
|
51
|
+
if (ms <= now) {
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
return Math.round((ms - now) / 1000);
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Convert a 0.0-1.0 utilization fraction into a whole-percent "left" figure.
|
|
58
|
+
*
|
|
59
|
+
* Anthropic publishes utilization (used), never remaining, for subscription
|
|
60
|
+
* windows — there is no absolute message or token count to report, so the
|
|
61
|
+
* honest derived form is a percentage. Clamped because a utilization above 1.0
|
|
62
|
+
* (overage) would otherwise produce a negative "left".
|
|
63
|
+
*/
|
|
64
|
+
export function utilizationToLeftPct(used) {
|
|
65
|
+
if (!Number.isFinite(used)) {
|
|
66
|
+
return 0;
|
|
67
|
+
}
|
|
68
|
+
return Math.max(0, Math.min(100, Math.round((1 - used) * 100)));
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Copy Anthropic's rate-limit headers verbatim from an upstream response.
|
|
72
|
+
*
|
|
73
|
+
* Covers both header families: the unified subscription windows
|
|
74
|
+
* (`unified-5h-*`, `unified-7d-*`) and the legacy per-tier counters
|
|
75
|
+
* (`requests-remaining`, `tokens-remaining`, ...) that API-key accounts get.
|
|
76
|
+
* Which family is present depends on the serving account type, which is why
|
|
77
|
+
* `x-neurolink-account-type` accompanies them.
|
|
78
|
+
*/
|
|
79
|
+
export function pickUpstreamRateLimitHeaders(headers) {
|
|
80
|
+
const picked = {};
|
|
81
|
+
const consider = (key, value) => {
|
|
82
|
+
const lower = key.toLowerCase();
|
|
83
|
+
if (lower.startsWith(UPSTREAM_PREFIX) ||
|
|
84
|
+
UPSTREAM_EXTRA_HEADERS.includes(lower)) {
|
|
85
|
+
const clean = sanitizeHeaderValue(value);
|
|
86
|
+
if (clean) {
|
|
87
|
+
picked[lower] = clean;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
if (typeof headers.forEach === "function") {
|
|
92
|
+
headers.forEach((value, key) => consider(key, value));
|
|
93
|
+
return picked;
|
|
94
|
+
}
|
|
95
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
96
|
+
if (typeof value === "string") {
|
|
97
|
+
consider(key, value);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return picked;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Build the `x-neurolink-*` half of the contract from proxy-side state.
|
|
104
|
+
*
|
|
105
|
+
* Always emits `x-neurolink-quota-source` — even when it is "none". A consumer
|
|
106
|
+
* that sees quota numbers with no provenance cannot tell a fresh reading from a
|
|
107
|
+
* snapshot carried over from a previous request, and would happily log stale
|
|
108
|
+
* capacity as current.
|
|
109
|
+
*/
|
|
110
|
+
export function buildQuotaResponseHeaders(context, now = Date.now()) {
|
|
111
|
+
const headers = {
|
|
112
|
+
"x-neurolink-quota-source": context.source,
|
|
113
|
+
};
|
|
114
|
+
const set = (name, value) => {
|
|
115
|
+
if (value === undefined || value === null) {
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
const clean = sanitizeHeaderValue(String(value));
|
|
119
|
+
if (clean) {
|
|
120
|
+
headers[name] = clean;
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
set("x-neurolink-account", context.accountLabel);
|
|
124
|
+
set("x-neurolink-account-type", context.accountType);
|
|
125
|
+
set("x-neurolink-served-by", context.servedBy);
|
|
126
|
+
set("x-neurolink-attempt", context.attempt);
|
|
127
|
+
set("x-neurolink-account-cooling-until", context.coolingUntil);
|
|
128
|
+
set("x-neurolink-account-cooling-reason", context.coolingReason);
|
|
129
|
+
if (context.pool) {
|
|
130
|
+
set("x-neurolink-pool-available", context.pool.available);
|
|
131
|
+
set("x-neurolink-pool-cooling", context.pool.cooling);
|
|
132
|
+
set("x-neurolink-pool-best-session-left", context.pool.bestSessionLeftPct);
|
|
133
|
+
}
|
|
134
|
+
const quota = context.quota;
|
|
135
|
+
if (quota && context.source !== "none") {
|
|
136
|
+
set("x-neurolink-quota-session-left-pct", utilizationToLeftPct(quota.sessionUsed));
|
|
137
|
+
set("x-neurolink-quota-session-status", quota.sessionStatus);
|
|
138
|
+
set("x-neurolink-quota-session-resets-in", secondsUntil(quota.sessionResetAt, now));
|
|
139
|
+
set("x-neurolink-quota-weekly-left-pct", utilizationToLeftPct(quota.weeklyUsed));
|
|
140
|
+
set("x-neurolink-quota-weekly-status", quota.weeklyStatus);
|
|
141
|
+
set("x-neurolink-quota-weekly-resets-in", secondsUntil(quota.weeklyResetAt, now));
|
|
142
|
+
set("x-neurolink-quota-unified-status", quota.unifiedStatus);
|
|
143
|
+
set("x-neurolink-quota-overage-status", quota.overageStatus);
|
|
144
|
+
set("x-neurolink-quota-fallback-pct", Math.round((quota.fallbackPercentage ?? 0) * 100));
|
|
145
|
+
set("x-neurolink-quota-updated-at", quota.lastUpdated);
|
|
146
|
+
}
|
|
147
|
+
return headers;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Full response header set: upstream verbatim + proxy-derived.
|
|
151
|
+
*
|
|
152
|
+
* Upstream headers are applied first so a `x-neurolink-*` key can never be
|
|
153
|
+
* shadowed by an upstream one (they share no names today, but the ordering
|
|
154
|
+
* makes the precedence explicit rather than incidental).
|
|
155
|
+
*/
|
|
156
|
+
export function buildProxyLimitHeaders(args) {
|
|
157
|
+
return {
|
|
158
|
+
...(args.upstreamHeaders
|
|
159
|
+
? pickUpstreamRateLimitHeaders(args.upstreamHeaders)
|
|
160
|
+
: {}),
|
|
161
|
+
...buildQuotaResponseHeaders(args.context, args.now),
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
/** Compute pool headroom from the runtime account states backing a request. */
|
|
165
|
+
export function summarizePoolHeadroom(entries, now = Date.now()) {
|
|
166
|
+
let available = 0;
|
|
167
|
+
let cooling = 0;
|
|
168
|
+
let bestSessionLeftPct;
|
|
169
|
+
for (const entry of entries) {
|
|
170
|
+
const isCooling = !!entry.coolingUntil && entry.coolingUntil > now;
|
|
171
|
+
if (isCooling) {
|
|
172
|
+
cooling += 1;
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
available += 1;
|
|
176
|
+
if (entry.quota) {
|
|
177
|
+
const left = utilizationToLeftPct(entry.quota.sessionUsed);
|
|
178
|
+
if (bestSessionLeftPct === undefined || left > bestSessionLeftPct) {
|
|
179
|
+
bestSessionLeftPct = left;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return {
|
|
184
|
+
available,
|
|
185
|
+
cooling,
|
|
186
|
+
...(bestSessionLeftPct !== undefined ? { bestSessionLeftPct } : {}),
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
//# sourceMappingURL=quotaHeaders.js.map
|
|
@@ -77,6 +77,8 @@ export declare function recordFinalSuccess(accountLabel?: string, accountType?:
|
|
|
77
77
|
export declare function recordAttemptError(accountLabel: string, accountType: string, status: number, rateLimitKind?: "transient" | "quota"): void;
|
|
78
78
|
export declare function recordFinalError(status: number, accountLabel?: string, accountType?: string, details?: ProxyTerminalErrorDetails): void;
|
|
79
79
|
export declare function getStats(): ProxyStats;
|
|
80
|
+
/** Return the process-local coherent snapshot without filesystem reconciliation. */
|
|
81
|
+
export declare function getUsageSnapshot(): ProxyUsageStatsSnapshot;
|
|
80
82
|
export declare function getReconciledStats(): Promise<ProxyStats>;
|
|
81
83
|
export declare function getReconciledUsageSnapshot(): Promise<ProxyUsageStatsSnapshot>;
|
|
82
84
|
export declare function getAccountStats(label: string): AccountStats | undefined;
|
|
@@ -1121,6 +1121,10 @@ export function recordFinalError(status, accountLabel, accountType, details) {
|
|
|
1121
1121
|
export function getStats() {
|
|
1122
1122
|
return defaultStore.getStats();
|
|
1123
1123
|
}
|
|
1124
|
+
/** Return the process-local coherent snapshot without filesystem reconciliation. */
|
|
1125
|
+
export function getUsageSnapshot() {
|
|
1126
|
+
return defaultStore.getUsageSnapshot();
|
|
1127
|
+
}
|
|
1124
1128
|
export async function getReconciledStats() {
|
|
1125
1129
|
return defaultStore.reconcile();
|
|
1126
1130
|
}
|
|
@@ -17,6 +17,7 @@ import { buildStableClaudeCodeBillingHeader, CLAUDE_CLI_USER_AGENT, CLAUDE_CODE_
|
|
|
17
17
|
import { clearAccountCooldown, loadAccountCooldowns, saveAccountCooldown, } from "../../proxy/accountCooldown.js";
|
|
18
18
|
import { anthropicAccountKeysEqual, ENV_ANTHROPIC_ACCOUNT_KEY, isAccountAllowed, LEGACY_ANTHROPIC_ACCOUNT_KEY, shouldLoadFallbackCredential, } from "../../proxy/accountSelection.js";
|
|
19
19
|
import { getUnifiedRateLimitStatus, isQuotaOverageAvailable, loadAccountQuotas, parseQuotaHeaders, saveAccountQuota, } from "../../proxy/accountQuota.js";
|
|
20
|
+
import { buildProxyLimitHeaders, summarizePoolHeadroom, } from "../../proxy/quotaHeaders.js";
|
|
20
21
|
import { buildClaudeError, ClaudeStreamSerializer, generateToolUseId, parseClaudeRequest, serializeClaudeResponse, } from "../../proxy/claudeFormat.js";
|
|
21
22
|
import { buildAnthropicModelsListResponse, buildTranslationOptions, extractText, extractToolArgs, extractUsageFromStreamResult, handleTranslatedJsonRequest, handleTranslatedStreamRequest, hasTranslatedOutput, } from "../../proxy/proxyTranslationEngine.js";
|
|
22
23
|
import { tracers } from "../../telemetry/tracers.js";
|
|
@@ -426,6 +427,56 @@ function resetEpochToMs(resetEpoch, now) {
|
|
|
426
427
|
const ms = resetEpoch > 4_102_444_800 ? resetEpoch : resetEpoch * 1000;
|
|
427
428
|
return ms > now ? ms : undefined;
|
|
428
429
|
}
|
|
430
|
+
/**
|
|
431
|
+
* Publish limit/quota headers for this response onto the request context.
|
|
432
|
+
*
|
|
433
|
+
* Every response path funnels through here so the header contract is defined
|
|
434
|
+
* once. The proxy runtime copies `ctx.responseHeaders` onto JSON and error
|
|
435
|
+
* responses, and merges them into streaming Responses for keys those don't
|
|
436
|
+
* already set — so a single call covers both shapes.
|
|
437
|
+
*/
|
|
438
|
+
function publishLimitHeaders(ctx, args) {
|
|
439
|
+
try {
|
|
440
|
+
const pool = args.poolAccounts
|
|
441
|
+
? summarizePoolHeadroom(args.poolAccounts.map((account) => {
|
|
442
|
+
const state = accountRuntimeState.get(account.key);
|
|
443
|
+
return {
|
|
444
|
+
...(state?.coolingUntil !== undefined
|
|
445
|
+
? { coolingUntil: state.coolingUntil }
|
|
446
|
+
: {}),
|
|
447
|
+
...(state?.quota ? { quota: state.quota } : {}),
|
|
448
|
+
};
|
|
449
|
+
}))
|
|
450
|
+
: undefined;
|
|
451
|
+
const headers = buildProxyLimitHeaders({
|
|
452
|
+
...(args.upstreamHeaders
|
|
453
|
+
? { upstreamHeaders: args.upstreamHeaders }
|
|
454
|
+
: {}),
|
|
455
|
+
context: {
|
|
456
|
+
quota: args.quota ?? null,
|
|
457
|
+
source: args.source,
|
|
458
|
+
...(args.account ? { accountLabel: args.account.label } : {}),
|
|
459
|
+
...((args.accountType ?? args.account?.type)
|
|
460
|
+
? { accountType: args.accountType ?? args.account?.type }
|
|
461
|
+
: {}),
|
|
462
|
+
...(args.servedBy ? { servedBy: args.servedBy } : {}),
|
|
463
|
+
...(args.attempt !== undefined ? { attempt: args.attempt } : {}),
|
|
464
|
+
...(args.accountState?.coolingUntil !== undefined
|
|
465
|
+
? { coolingUntil: args.accountState.coolingUntil }
|
|
466
|
+
: {}),
|
|
467
|
+
...(args.accountState?.coolingReason
|
|
468
|
+
? { coolingReason: args.accountState.coolingReason }
|
|
469
|
+
: {}),
|
|
470
|
+
...(pool ? { pool } : {}),
|
|
471
|
+
},
|
|
472
|
+
});
|
|
473
|
+
ctx.responseHeaders = { ...(ctx.responseHeaders ?? {}), ...headers };
|
|
474
|
+
}
|
|
475
|
+
catch (error) {
|
|
476
|
+
// Diagnostics must never break a response that is otherwise fine.
|
|
477
|
+
logger.debug(`[proxy] failed to publish limit headers: ${error instanceof Error ? error.message : String(error)}`);
|
|
478
|
+
}
|
|
479
|
+
}
|
|
429
480
|
/** Clamp a cooldown target epoch-ms into [now+MIN, now+MAX]. */
|
|
430
481
|
function clampCooldownUntil(untilMs, now) {
|
|
431
482
|
return Math.min(Math.max(untilMs, now + MIN_COOLDOWN_MS), now + MAX_COOLDOWN_MS);
|
|
@@ -1466,6 +1517,20 @@ async function handleClaudePassthroughRequest(args) {
|
|
|
1466
1517
|
upstreamResponseHeaders[key] = value;
|
|
1467
1518
|
});
|
|
1468
1519
|
tracer?.logUpstreamResponseHeaders(upstreamResponseHeaders);
|
|
1520
|
+
// Passthrough uses the caller's own credentials, so there is no account pool
|
|
1521
|
+
// to report — but the upstream quota headers are still the caller's real
|
|
1522
|
+
// limits. Published before the ok/non-ok split so a 429 carries them too.
|
|
1523
|
+
{
|
|
1524
|
+
const passthroughQuota = parseQuotaHeaders(response.headers);
|
|
1525
|
+
publishLimitHeaders(ctx, {
|
|
1526
|
+
upstreamHeaders: response.headers,
|
|
1527
|
+
quota: passthroughQuota,
|
|
1528
|
+
source: passthroughQuota ? "live" : "none",
|
|
1529
|
+
accountType: "passthrough",
|
|
1530
|
+
servedBy: "anthropic",
|
|
1531
|
+
attempt: 1,
|
|
1532
|
+
});
|
|
1533
|
+
}
|
|
1469
1534
|
if (!response.ok) {
|
|
1470
1535
|
const errorText = await response.text();
|
|
1471
1536
|
recordAttemptError("passthrough", "passthrough", response.status, response.status === 429 ? "quota" : undefined);
|
|
@@ -1572,7 +1637,7 @@ function trackUpstreamReadableStream(source) {
|
|
|
1572
1637
|
return { stream, outcome: tracker.outcome };
|
|
1573
1638
|
}
|
|
1574
1639
|
async function handleClaudePassthroughStreamResponse(args) {
|
|
1575
|
-
const { bodyStr, response, tracer, requestStartTime, upstreamSpan, upstreamResponseHeaders, logProxyBody, logFinalRequest, } = args;
|
|
1640
|
+
const { ctx, bodyStr, response, tracer, requestStartTime, upstreamSpan, upstreamResponseHeaders, logProxyBody, logFinalRequest, } = args;
|
|
1576
1641
|
const responseHeaders = { ...upstreamResponseHeaders };
|
|
1577
1642
|
const { stream: clientCaptureStream, capture: clientCapture } = createRawStreamCapture();
|
|
1578
1643
|
const responseBody = response.body;
|
|
@@ -1754,7 +1819,13 @@ async function handleClaudePassthroughStreamResponse(args) {
|
|
|
1754
1819
|
const clientStream = streamSource.pipeThrough(clientCaptureStream);
|
|
1755
1820
|
return new Response(clientStream, {
|
|
1756
1821
|
status: response.status,
|
|
1757
|
-
headers
|
|
1822
|
+
// Upstream headers first, then the proxy's own — passthrough already
|
|
1823
|
+
// forwarded everything Anthropic sent, so this only adds the x-neurolink-*
|
|
1824
|
+
// fields the client cannot derive on its own.
|
|
1825
|
+
headers: {
|
|
1826
|
+
...responseHeaders,
|
|
1827
|
+
...(ctx.responseHeaders ?? {}),
|
|
1828
|
+
},
|
|
1758
1829
|
});
|
|
1759
1830
|
}
|
|
1760
1831
|
async function handleClaudePassthroughJsonResponse(args) {
|
|
@@ -2244,6 +2315,14 @@ async function tryConfiguredClaudeFallbackChain(args) {
|
|
|
2244
2315
|
attemptCount: fallbackPlan.attempts.slice(1).length,
|
|
2245
2316
|
reason: "fallback_success",
|
|
2246
2317
|
});
|
|
2318
|
+
// A different provider produced this response — say so, and report no
|
|
2319
|
+
// quota. Emitting the last Anthropic snapshot here would attribute one
|
|
2320
|
+
// provider's capacity to another's output.
|
|
2321
|
+
publishLimitHeaders(ctx, {
|
|
2322
|
+
quota: null,
|
|
2323
|
+
source: "none",
|
|
2324
|
+
servedBy: fallback.provider,
|
|
2325
|
+
});
|
|
2247
2326
|
return { response };
|
|
2248
2327
|
}
|
|
2249
2328
|
catch (fallbackErr) {
|
|
@@ -2320,6 +2399,13 @@ async function tryAutoClaudeFallback(args) {
|
|
|
2320
2399
|
attemptCount: 1,
|
|
2321
2400
|
reason: "fallback_success",
|
|
2322
2401
|
});
|
|
2402
|
+
// See the configured-chain path: never attribute Anthropic quota to a
|
|
2403
|
+
// response another provider produced.
|
|
2404
|
+
publishLimitHeaders(ctx, {
|
|
2405
|
+
quota: null,
|
|
2406
|
+
source: "none",
|
|
2407
|
+
servedBy: "auto-provider",
|
|
2408
|
+
});
|
|
2323
2409
|
return { response };
|
|
2324
2410
|
}
|
|
2325
2411
|
catch (fallbackErr) {
|
|
@@ -2441,7 +2527,7 @@ function buildClaudeAnthropicFailureResponse(args) {
|
|
|
2441
2527
|
});
|
|
2442
2528
|
}
|
|
2443
2529
|
async function handleAnthropicSuccessfulResponse(args) {
|
|
2444
|
-
const { ctx, body, account, accountState, response, tracer, requestStartTime, fetchStartMs, attemptNumber, finalBodyStr, upstreamSpan, logAttempt, logProxyBody, onStreamTerminal, logFinalRequest, } = args;
|
|
2530
|
+
const { ctx, body, account, accountState, response, tracer, requestStartTime, fetchStartMs, attemptNumber, finalBodyStr, upstreamSpan, logAttempt, logProxyBody, onStreamTerminal, poolAccounts, logFinalRequest, } = args;
|
|
2445
2531
|
accountState.consecutiveRefreshFailures = 0;
|
|
2446
2532
|
logger.always(`[proxy] ← ${response.status} account=${account.label}`);
|
|
2447
2533
|
const quota = parseQuotaHeaders(response.headers);
|
|
@@ -2470,6 +2556,20 @@ async function handleAnthropicSuccessfulResponse(args) {
|
|
|
2470
2556
|
responseHeaders[key] = value;
|
|
2471
2557
|
});
|
|
2472
2558
|
tracer?.logUpstreamResponseHeaders(responseHeaders);
|
|
2559
|
+
// Surface limits to the client. `quota` is non-null only when this upstream
|
|
2560
|
+
// response actually carried the unified headers; otherwise fall back to the
|
|
2561
|
+
// account's last snapshot and label it as such, so a consumer never mistakes
|
|
2562
|
+
// a carried-over reading for a fresh one.
|
|
2563
|
+
publishLimitHeaders(ctx, {
|
|
2564
|
+
upstreamHeaders: response.headers,
|
|
2565
|
+
quota: quota ?? accountState.quota ?? null,
|
|
2566
|
+
source: quota ? "live" : accountState.quota ? "snapshot" : "none",
|
|
2567
|
+
account,
|
|
2568
|
+
accountState,
|
|
2569
|
+
servedBy: "anthropic",
|
|
2570
|
+
attempt: attemptNumber,
|
|
2571
|
+
...(poolAccounts ? { poolAccounts } : {}),
|
|
2572
|
+
});
|
|
2473
2573
|
if (body.stream) {
|
|
2474
2574
|
return handleAnthropicStreamingSuccessResponse({
|
|
2475
2575
|
ctx,
|
|
@@ -2506,7 +2606,7 @@ async function handleAnthropicSuccessfulResponse(args) {
|
|
|
2506
2606
|
});
|
|
2507
2607
|
}
|
|
2508
2608
|
async function handleAnthropicStreamingSuccessResponse(args) {
|
|
2509
|
-
const { account, accountState, response, responseHeaders, tracer, requestStartTime, fetchStartMs, attemptNumber, finalBodyStr, upstreamSpan, logAttempt, logProxyBody, onStreamTerminal, logFinalRequest, } = args;
|
|
2609
|
+
const { ctx, account, accountState, response, responseHeaders, tracer, requestStartTime, fetchStartMs, attemptNumber, finalBodyStr, upstreamSpan, logAttempt, logProxyBody, onStreamTerminal, logFinalRequest, } = args;
|
|
2510
2610
|
if (!response.body) {
|
|
2511
2611
|
recordAttemptError(account.label, account.type, 502);
|
|
2512
2612
|
logAttempt(502, "stream_error", "No response body from upstream");
|
|
@@ -2703,6 +2803,7 @@ async function handleAnthropicStreamingSuccessResponse(args) {
|
|
|
2703
2803
|
},
|
|
2704
2804
|
});
|
|
2705
2805
|
const { response: result, telemetryDone } = attachAnthropicSuccessStreamTelemetry({
|
|
2806
|
+
ctx,
|
|
2706
2807
|
account,
|
|
2707
2808
|
response,
|
|
2708
2809
|
responseHeaders,
|
|
@@ -2742,7 +2843,7 @@ function recordCommittedAnthropicStreamAttemptFailure(outcome, account) {
|
|
|
2742
2843
|
}
|
|
2743
2844
|
}
|
|
2744
2845
|
function attachAnthropicSuccessStreamTelemetry(args) {
|
|
2745
|
-
const { account, response, responseHeaders, remainingStream, streamOutcome, tracer, requestStartTime, attemptNumber, finalBodyStr, upstreamSpan, logProxyBody, logFinalRequest, } = args;
|
|
2846
|
+
const { ctx, account, response, responseHeaders, remainingStream, streamOutcome, tracer, requestStartTime, attemptNumber, finalBodyStr, upstreamSpan, logProxyBody, logFinalRequest, } = args;
|
|
2746
2847
|
const { stream: clientCaptureStream, capture: clientCapture } = createRawStreamCapture();
|
|
2747
2848
|
let streamSource = remainingStream;
|
|
2748
2849
|
let telemetryDone;
|
|
@@ -2952,23 +3053,18 @@ function attachAnthropicSuccessStreamTelemetry(args) {
|
|
|
2952
3053
|
}
|
|
2953
3054
|
}
|
|
2954
3055
|
const clientStream = streamSource.pipeThrough(clientCaptureStream);
|
|
3056
|
+
// Limit headers published on the context are applied here rather than left
|
|
3057
|
+
// to the runtime wrapper: this Response goes straight to the client on every
|
|
3058
|
+
// mount (proxy runtime and the generic server adapters alike), so the
|
|
3059
|
+
// streaming path has to carry them itself. The previous five-name allowlist
|
|
3060
|
+
// forwarded only the legacy counters and dropped the unified subscription
|
|
3061
|
+
// windows entirely.
|
|
2955
3062
|
const clientResponseHeaders = {
|
|
3063
|
+
...(ctx.responseHeaders ?? {}),
|
|
2956
3064
|
"content-type": "text/event-stream",
|
|
2957
3065
|
"cache-control": "no-cache",
|
|
2958
3066
|
connection: "keep-alive",
|
|
2959
3067
|
};
|
|
2960
|
-
for (const headerName of [
|
|
2961
|
-
"retry-after",
|
|
2962
|
-
"anthropic-ratelimit-requests-remaining",
|
|
2963
|
-
"anthropic-ratelimit-requests-limit",
|
|
2964
|
-
"anthropic-ratelimit-tokens-remaining",
|
|
2965
|
-
"anthropic-ratelimit-tokens-limit",
|
|
2966
|
-
]) {
|
|
2967
|
-
const value = response.headers.get(headerName);
|
|
2968
|
-
if (value) {
|
|
2969
|
-
clientResponseHeaders[headerName] = value;
|
|
2970
|
-
}
|
|
2971
|
-
}
|
|
2972
3068
|
return {
|
|
2973
3069
|
response: new Response(clientStream, {
|
|
2974
3070
|
status: response.status,
|
|
@@ -4612,6 +4708,7 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
4612
4708
|
logProxyBody,
|
|
4613
4709
|
logFinalRequest,
|
|
4614
4710
|
onStreamTerminal: admissionLease.release,
|
|
4711
|
+
poolAccounts: enabledAccounts,
|
|
4615
4712
|
});
|
|
4616
4713
|
if ("retryNextAccount" in successResult) {
|
|
4617
4714
|
if (successResult.failure) {
|
|
@@ -4674,6 +4771,24 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
4674
4771
|
}
|
|
4675
4772
|
loopState.fallbackFailureMessage = fallbackFailureMessage;
|
|
4676
4773
|
}
|
|
4774
|
+
// Terminal failure — usually "every account is rate-limited". This is the
|
|
4775
|
+
// response a caller most needs limit data on, and historically the one that
|
|
4776
|
+
// carried none. No single account served it, so report the account we would
|
|
4777
|
+
// have tried first plus pool headroom, explicitly marked as a snapshot.
|
|
4778
|
+
{
|
|
4779
|
+
const primary = orderedAccounts[0];
|
|
4780
|
+
const primaryState = primary
|
|
4781
|
+
? accountRuntimeState.get(primary.key)
|
|
4782
|
+
: undefined;
|
|
4783
|
+
publishLimitHeaders(ctx, {
|
|
4784
|
+
quota: primaryState?.quota ?? null,
|
|
4785
|
+
source: primaryState?.quota ? "snapshot" : "none",
|
|
4786
|
+
...(primary ? { account: primary } : {}),
|
|
4787
|
+
...(primaryState ? { accountState: primaryState } : {}),
|
|
4788
|
+
attempt: loopState.attemptNumber,
|
|
4789
|
+
poolAccounts: enabledAccounts,
|
|
4790
|
+
});
|
|
4791
|
+
}
|
|
4677
4792
|
return buildClaudeAnthropicFailureResponse({
|
|
4678
4793
|
tracer,
|
|
4679
4794
|
requestStartTime,
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
* Comprehensive usage tracking, performance metrics, and cost analysis types
|
|
4
4
|
*/
|
|
5
5
|
import type { JsonValue, UnknownRecord } from "./common.js";
|
|
6
|
+
import type { ClaudeLimitSnapshot } from "./subscription.js";
|
|
6
7
|
/**
|
|
7
8
|
* Token usage information (consolidated from multiple sources)
|
|
8
9
|
*/
|
|
@@ -45,6 +46,13 @@ export type AnalyticsData = {
|
|
|
45
46
|
elapsedMs?: number;
|
|
46
47
|
/** Verbatim provider finish/stop reason for the terminal model call. */
|
|
47
48
|
rawFinishReason?: string;
|
|
49
|
+
/**
|
|
50
|
+
* Account limit state observed on this request — subscription window
|
|
51
|
+
* headroom, reset times, and (via the NeuroLink Claude proxy) which account
|
|
52
|
+
* served it and how much the pool has left. Present for Anthropic traffic
|
|
53
|
+
* whose response carried rate-limit headers.
|
|
54
|
+
*/
|
|
55
|
+
limits?: ClaudeLimitSnapshot;
|
|
48
56
|
};
|
|
49
57
|
/**
|
|
50
58
|
* Stream Analytics Data - Enhanced for performance tracking
|
|
@@ -3,6 +3,7 @@ import type { RAGConfig } from "./rag.js";
|
|
|
3
3
|
import type { KnowledgeGroundingMetadata, KnowledgeRequestScope } from "./knowledge.js";
|
|
4
4
|
import type { SkillsCallOptions } from "./skills.js";
|
|
5
5
|
import type { AnalyticsData, TokenUsage } from "./analytics.js";
|
|
6
|
+
import type { ClaudeLimitSnapshot } from "./subscription.js";
|
|
6
7
|
import type { JsonValue } from "./common.js";
|
|
7
8
|
import type { Content, ImageWithAltText } from "./content.js";
|
|
8
9
|
import type { ChatMessage, ConversationMemoryConfig } from "./conversation.js";
|
|
@@ -963,6 +964,17 @@ export type GenerateResult = {
|
|
|
963
964
|
message: string;
|
|
964
965
|
}>;
|
|
965
966
|
};
|
|
967
|
+
/**
|
|
968
|
+
* Account limit state for this request, parsed from Anthropic's
|
|
969
|
+
* `anthropic-ratelimit-*` response headers (plus the NeuroLink Claude
|
|
970
|
+
* proxy's `x-neurolink-*` additions when routed through it).
|
|
971
|
+
*
|
|
972
|
+
* Subscription windows report utilization, so headroom is a percentage
|
|
973
|
+
* (`sessionLeftPct`) rather than an absolute count — Anthropic publishes no
|
|
974
|
+
* remaining message or token figure for them. API-key accounts do carry
|
|
975
|
+
* absolute `requestsRemaining` / `tokensRemaining`.
|
|
976
|
+
*/
|
|
977
|
+
limits?: ClaudeLimitSnapshot;
|
|
966
978
|
};
|
|
967
979
|
/**
|
|
968
980
|
* Unified options for both generation and streaming
|
|
@@ -944,6 +944,49 @@ export type AccountQuota = {
|
|
|
944
944
|
/** Epoch ms when we last captured this data */
|
|
945
945
|
lastUpdated: number;
|
|
946
946
|
};
|
|
947
|
+
/**
|
|
948
|
+
* Provenance of the quota numbers attached to a single proxy response.
|
|
949
|
+
* - "live" : parsed from THIS upstream response's headers.
|
|
950
|
+
* - "snapshot" : the last known snapshot for the serving account; the upstream
|
|
951
|
+
* response carried no quota headers.
|
|
952
|
+
* - "none" : no Anthropic account served this request (fallback provider,
|
|
953
|
+
* or a failure before any account was reached).
|
|
954
|
+
*
|
|
955
|
+
* Consumers must not treat "snapshot" as current. Without this distinction a
|
|
956
|
+
* stale reading is indistinguishable from a fresh one.
|
|
957
|
+
*/
|
|
958
|
+
export type ProxyQuotaSource = "live" | "snapshot" | "none";
|
|
959
|
+
/** Aggregate account-pool headroom at the moment a response was produced. */
|
|
960
|
+
export type ProxyPoolHeadroom = {
|
|
961
|
+
/** Accounts eligible to serve a request right now (not cooling/disabled). */
|
|
962
|
+
available: number;
|
|
963
|
+
/** Accounts currently in a cooldown window. */
|
|
964
|
+
cooling: number;
|
|
965
|
+
/** Best session headroom across available accounts, as a percentage 0-100.
|
|
966
|
+
* Undefined when no available account has a quota snapshot. */
|
|
967
|
+
bestSessionLeftPct?: number;
|
|
968
|
+
};
|
|
969
|
+
/**
|
|
970
|
+
* Everything the proxy knows about limits and routing for one response.
|
|
971
|
+
* Consumed by `buildQuotaResponseHeaders` — kept as data (not headers) so the
|
|
972
|
+
* assembly stays pure and testable.
|
|
973
|
+
*/
|
|
974
|
+
export type ProxyQuotaHeaderContext = {
|
|
975
|
+
quota: AccountQuota | null;
|
|
976
|
+
source: ProxyQuotaSource;
|
|
977
|
+
/** Account label that served the request; omitted for fallback/no-account. */
|
|
978
|
+
accountLabel?: string;
|
|
979
|
+
accountType?: string;
|
|
980
|
+
/** Which upstream actually produced the response ("anthropic" or a fallback
|
|
981
|
+
* provider name). Lets a consumer avoid attributing quota to the wrong one. */
|
|
982
|
+
servedBy?: string;
|
|
983
|
+
/** 1-based attempt index within the routing loop. */
|
|
984
|
+
attempt?: number;
|
|
985
|
+
/** Epoch ms until which the serving account is cooling, when applicable. */
|
|
986
|
+
coolingUntil?: number;
|
|
987
|
+
coolingReason?: AccountCoolingReason;
|
|
988
|
+
pool?: ProxyPoolHeadroom;
|
|
989
|
+
};
|
|
947
990
|
/** Why an account is currently cooling. Drives cooldown duration and logging.
|
|
948
991
|
* - "weekly" : 7d unified limit rejected — cool until the weekly reset.
|
|
949
992
|
* - "session" : 5h unified limit rejected — cool until the session reset.
|
|
@@ -1246,6 +1289,8 @@ export type ProxyLifecycleLoggerSnapshot = {
|
|
|
1246
1289
|
invalidDrops: number;
|
|
1247
1290
|
writeDrops: number;
|
|
1248
1291
|
writeFailures: number;
|
|
1292
|
+
/** Events requeued after a transient lifecycle metadata write failure. */
|
|
1293
|
+
writeRetries: number;
|
|
1249
1294
|
pending: number;
|
|
1250
1295
|
inFlight: number;
|
|
1251
1296
|
flushing: boolean;
|
|
@@ -1257,12 +1302,15 @@ export type ProxyLifecycleLoggerOptions = {
|
|
|
1257
1302
|
queueCapacity?: number;
|
|
1258
1303
|
batchSize?: number;
|
|
1259
1304
|
flushIntervalMs?: number;
|
|
1305
|
+
/** Bounded retries for a metadata batch that cannot be appended immediately. */
|
|
1306
|
+
maxWriteRetries?: number;
|
|
1260
1307
|
};
|
|
1261
1308
|
/** Serialized lifecycle line awaiting a bounded batch write. */
|
|
1262
1309
|
export type QueuedProxyLifecycleEvent = {
|
|
1263
1310
|
logDir: string;
|
|
1264
1311
|
date: string;
|
|
1265
1312
|
record: Record<string, unknown>;
|
|
1313
|
+
writeRetries: number;
|
|
1266
1314
|
};
|
|
1267
1315
|
/** Percentile summary used by offline proxy log analysis. */
|
|
1268
1316
|
export type ProxyLatencySummary = {
|
|
@@ -2167,6 +2215,8 @@ export type StatusStats = {
|
|
|
2167
2215
|
terminalErrorDetailsComparable?: boolean;
|
|
2168
2216
|
terminalErrorDetailsMissing?: number;
|
|
2169
2217
|
terminalErrorDetailsExcess?: number;
|
|
2218
|
+
/** Whether this status response reconciled shared state or used local memory. */
|
|
2219
|
+
snapshotSource?: "reconciled" | "memory";
|
|
2170
2220
|
accounts?: {
|
|
2171
2221
|
label: string;
|
|
2172
2222
|
type: string;
|