@juspay/neurolink 10.9.0 → 10.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/browser/neurolink.min.js +379 -379
- package/dist/cli/commands/proxy.js +80 -20
- package/dist/lib/providers/anthropic/client.d.ts +22 -7
- package/dist/lib/providers/anthropic/client.js +83 -58
- package/dist/lib/providers/anthropic/rateLimitCapture.d.ts +82 -0
- package/dist/lib/providers/anthropic/rateLimitCapture.js +375 -0
- package/dist/lib/proxy/proxyLifecycle.d.ts +5 -0
- package/dist/lib/proxy/proxyLifecycle.js +61 -11
- package/dist/lib/proxy/quotaHeaders.d.ts +73 -0
- package/dist/lib/proxy/quotaHeaders.js +189 -0
- package/dist/lib/proxy/usageStats.d.ts +2 -0
- package/dist/lib/proxy/usageStats.js +4 -0
- package/dist/lib/server/routes/claudeProxyRoutes.js +132 -17
- package/dist/lib/types/analytics.d.ts +8 -0
- package/dist/lib/types/generate.d.ts +12 -0
- package/dist/lib/types/proxy.d.ts +50 -0
- package/dist/lib/types/subscription.d.ts +77 -0
- package/dist/providers/anthropic/client.d.ts +22 -7
- package/dist/providers/anthropic/client.js +83 -58
- package/dist/providers/anthropic/rateLimitCapture.d.ts +82 -0
- package/dist/providers/anthropic/rateLimitCapture.js +374 -0
- package/dist/proxy/proxyLifecycle.d.ts +5 -0
- package/dist/proxy/proxyLifecycle.js +61 -11
- package/dist/proxy/quotaHeaders.d.ts +73 -0
- package/dist/proxy/quotaHeaders.js +188 -0
- package/dist/proxy/usageStats.d.ts +2 -0
- package/dist/proxy/usageStats.js +4 -0
- package/dist/server/routes/claudeProxyRoutes.js +132 -17
- package/dist/types/analytics.d.ts +8 -0
- package/dist/types/generate.d.ts +12 -0
- package/dist/types/proxy.d.ts +50 -0
- package/dist/types/subscription.d.ts +77 -0
- package/package.json +3 -1
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anthropic rate-limit / quota header capture.
|
|
3
|
+
*
|
|
4
|
+
* Anthropic returns limit state on the response headers of every request:
|
|
5
|
+
* `anthropic-ratelimit-unified-*` for subscription (OAuth) accounts,
|
|
6
|
+
* `anthropic-ratelimit-{requests,tokens}-*` for API-key accounts. The NeuroLink
|
|
7
|
+
* Claude proxy forwards those verbatim and adds `x-neurolink-*` for what only
|
|
8
|
+
* it knows (which account served the request, pool headroom, whether the
|
|
9
|
+
* numbers are live or a carried-over snapshot).
|
|
10
|
+
*
|
|
11
|
+
* None of it used to reach the SDK: `doGenerate` returned a hardcoded empty
|
|
12
|
+
* header bag and the streaming loop never looked. The capture point here is the
|
|
13
|
+
* `fetch` the Anthropic SDK is constructed with — it is invoked exactly once
|
|
14
|
+
* per HTTP request on BOTH the streaming and non-streaming paths, so a single
|
|
15
|
+
* wrapper covers everything without touching the SSE loop or switching the
|
|
16
|
+
* non-streaming call to `.withResponse()`.
|
|
17
|
+
*
|
|
18
|
+
* Scoping is per-request via AsyncLocalStorage rather than a field on the
|
|
19
|
+
* provider: a provider instance is shared across concurrent calls, so an
|
|
20
|
+
* instance field would race and attribute one request's limits to another.
|
|
21
|
+
*
|
|
22
|
+
* @module providers/anthropic/rateLimitCapture
|
|
23
|
+
*/
|
|
24
|
+
import { AsyncLocalStorage } from "async_hooks";
|
|
25
|
+
import { trace } from "@opentelemetry/api";
|
|
26
|
+
import { logger } from "../../utils/logger.js";
|
|
27
|
+
/** Below this much session headroom (percent), log at WARN instead of INFO. */
|
|
28
|
+
const LOW_HEADROOM_WARN_PCT = 15;
|
|
29
|
+
const limitCaptureStorage = new AsyncLocalStorage();
|
|
30
|
+
function headerOf(headers, name) {
|
|
31
|
+
const value = headers.get(name);
|
|
32
|
+
return value === null || value === "" ? undefined : value;
|
|
33
|
+
}
|
|
34
|
+
function numberOf(headers, name) {
|
|
35
|
+
const raw = headerOf(headers, name);
|
|
36
|
+
if (raw === undefined) {
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
const parsed = Number(raw);
|
|
40
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
41
|
+
}
|
|
42
|
+
function intOf(headers, name) {
|
|
43
|
+
const raw = headerOf(headers, name);
|
|
44
|
+
if (raw === undefined) {
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
const parsed = parseInt(raw, 10);
|
|
48
|
+
return Number.isNaN(parsed) ? undefined : parsed;
|
|
49
|
+
}
|
|
50
|
+
/** 0.0-1.0 utilization → whole-percent remaining, clamped to [0, 100]. */
|
|
51
|
+
function leftPctFrom(utilization) {
|
|
52
|
+
if (utilization === undefined) {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
return Math.max(0, Math.min(100, Math.round((1 - utilization) * 100)));
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Parse both Anthropic rate-limit header families into a single shape.
|
|
59
|
+
*
|
|
60
|
+
* Which family is present depends on the account type, so every field is
|
|
61
|
+
* optional and absence is normal rather than an error.
|
|
62
|
+
*/
|
|
63
|
+
export function parseAnthropicLimitHeaders(headers) {
|
|
64
|
+
const sessionUtilization = numberOf(headers, "anthropic-ratelimit-unified-5h-utilization");
|
|
65
|
+
const weeklyUtilization = numberOf(headers, "anthropic-ratelimit-unified-7d-utilization");
|
|
66
|
+
const info = {};
|
|
67
|
+
// Legacy per-tier counters — these ARE absolute remaining counts.
|
|
68
|
+
const requestsLimit = intOf(headers, "anthropic-ratelimit-requests-limit");
|
|
69
|
+
const requestsRemaining = intOf(headers, "anthropic-ratelimit-requests-remaining");
|
|
70
|
+
const requestsReset = headerOf(headers, "anthropic-ratelimit-requests-reset");
|
|
71
|
+
const tokensLimit = intOf(headers, "anthropic-ratelimit-tokens-limit");
|
|
72
|
+
const tokensRemaining = intOf(headers, "anthropic-ratelimit-tokens-remaining");
|
|
73
|
+
const tokensReset = headerOf(headers, "anthropic-ratelimit-tokens-reset");
|
|
74
|
+
const retryAfter = intOf(headers, "retry-after");
|
|
75
|
+
if (requestsLimit !== undefined) {
|
|
76
|
+
info.requestsLimit = requestsLimit;
|
|
77
|
+
}
|
|
78
|
+
if (requestsRemaining !== undefined) {
|
|
79
|
+
info.requestsRemaining = requestsRemaining;
|
|
80
|
+
}
|
|
81
|
+
if (requestsReset !== undefined) {
|
|
82
|
+
info.requestsReset = requestsReset;
|
|
83
|
+
}
|
|
84
|
+
if (tokensLimit !== undefined) {
|
|
85
|
+
info.tokensLimit = tokensLimit;
|
|
86
|
+
}
|
|
87
|
+
if (tokensRemaining !== undefined) {
|
|
88
|
+
info.tokensRemaining = tokensRemaining;
|
|
89
|
+
}
|
|
90
|
+
if (tokensReset !== undefined) {
|
|
91
|
+
info.tokensReset = tokensReset;
|
|
92
|
+
}
|
|
93
|
+
if (retryAfter !== undefined) {
|
|
94
|
+
info.retryAfter = retryAfter;
|
|
95
|
+
}
|
|
96
|
+
// Unified subscription windows — utilization only, no absolute remaining.
|
|
97
|
+
if (sessionUtilization !== undefined) {
|
|
98
|
+
info.sessionUtilization = sessionUtilization;
|
|
99
|
+
const left = leftPctFrom(sessionUtilization);
|
|
100
|
+
if (left !== undefined) {
|
|
101
|
+
info.sessionLeftPct = left;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
const sessionStatus = headerOf(headers, "anthropic-ratelimit-unified-5h-status");
|
|
105
|
+
if (sessionStatus !== undefined) {
|
|
106
|
+
info.sessionStatus = sessionStatus;
|
|
107
|
+
}
|
|
108
|
+
const sessionResetAt = intOf(headers, "anthropic-ratelimit-unified-5h-reset");
|
|
109
|
+
if (sessionResetAt !== undefined) {
|
|
110
|
+
info.sessionResetAt = sessionResetAt;
|
|
111
|
+
}
|
|
112
|
+
if (weeklyUtilization !== undefined) {
|
|
113
|
+
info.weeklyUtilization = weeklyUtilization;
|
|
114
|
+
const left = leftPctFrom(weeklyUtilization);
|
|
115
|
+
if (left !== undefined) {
|
|
116
|
+
info.weeklyLeftPct = left;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const weeklyStatus = headerOf(headers, "anthropic-ratelimit-unified-7d-status");
|
|
120
|
+
if (weeklyStatus !== undefined) {
|
|
121
|
+
info.weeklyStatus = weeklyStatus;
|
|
122
|
+
}
|
|
123
|
+
const weeklyResetAt = intOf(headers, "anthropic-ratelimit-unified-7d-reset");
|
|
124
|
+
if (weeklyResetAt !== undefined) {
|
|
125
|
+
info.weeklyResetAt = weeklyResetAt;
|
|
126
|
+
}
|
|
127
|
+
const unifiedStatus = headerOf(headers, "anthropic-ratelimit-unified-status");
|
|
128
|
+
if (unifiedStatus !== undefined) {
|
|
129
|
+
info.unifiedStatus = unifiedStatus;
|
|
130
|
+
}
|
|
131
|
+
const overageStatus = headerOf(headers, "anthropic-ratelimit-unified-overage-status");
|
|
132
|
+
if (overageStatus !== undefined) {
|
|
133
|
+
info.overageStatus = overageStatus;
|
|
134
|
+
}
|
|
135
|
+
return info;
|
|
136
|
+
}
|
|
137
|
+
/** True when a parsed info object carries no usable signal at all. */
|
|
138
|
+
function isEmptyRateLimitInfo(info) {
|
|
139
|
+
return Object.keys(info).length === 0;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Build a snapshot from a response, or undefined when the response carries
|
|
143
|
+
* neither Anthropic rate-limit headers nor NeuroLink proxy metadata.
|
|
144
|
+
*/
|
|
145
|
+
export function buildLimitSnapshot(headers, status, now = Date.now()) {
|
|
146
|
+
const rateLimit = parseAnthropicLimitHeaders(headers);
|
|
147
|
+
const quotaSource = headerOf(headers, "x-neurolink-quota-source");
|
|
148
|
+
const account = headerOf(headers, "x-neurolink-account");
|
|
149
|
+
const accountType = headerOf(headers, "x-neurolink-account-type");
|
|
150
|
+
const servedBy = headerOf(headers, "x-neurolink-served-by");
|
|
151
|
+
const poolAvailable = intOf(headers, "x-neurolink-pool-available");
|
|
152
|
+
const poolCooling = intOf(headers, "x-neurolink-pool-cooling");
|
|
153
|
+
const poolBest = intOf(headers, "x-neurolink-pool-best-session-left");
|
|
154
|
+
const coolingUntil = intOf(headers, "x-neurolink-account-cooling-until");
|
|
155
|
+
const coolingReason = headerOf(headers, "x-neurolink-account-cooling-reason");
|
|
156
|
+
const requestId = headerOf(headers, "x-request-id");
|
|
157
|
+
const hasProxyMetadata = quotaSource !== undefined ||
|
|
158
|
+
account !== undefined ||
|
|
159
|
+
servedBy !== undefined;
|
|
160
|
+
if (isEmptyRateLimitInfo(rateLimit) && !hasProxyMetadata) {
|
|
161
|
+
return undefined;
|
|
162
|
+
}
|
|
163
|
+
const pool = poolAvailable !== undefined ||
|
|
164
|
+
poolCooling !== undefined ||
|
|
165
|
+
poolBest !== undefined
|
|
166
|
+
? {
|
|
167
|
+
...(poolAvailable !== undefined ? { available: poolAvailable } : {}),
|
|
168
|
+
...(poolCooling !== undefined ? { cooling: poolCooling } : {}),
|
|
169
|
+
...(poolBest !== undefined ? { bestSessionLeftPct: poolBest } : {}),
|
|
170
|
+
}
|
|
171
|
+
: undefined;
|
|
172
|
+
return {
|
|
173
|
+
rateLimit,
|
|
174
|
+
...(quotaSource === "live" ||
|
|
175
|
+
quotaSource === "snapshot" ||
|
|
176
|
+
quotaSource === "none"
|
|
177
|
+
? { quotaSource }
|
|
178
|
+
: {}),
|
|
179
|
+
...(account !== undefined ? { account } : {}),
|
|
180
|
+
...(accountType !== undefined ? { accountType } : {}),
|
|
181
|
+
...(servedBy !== undefined ? { servedBy } : {}),
|
|
182
|
+
...(coolingUntil !== undefined
|
|
183
|
+
? { accountCoolingUntil: coolingUntil }
|
|
184
|
+
: {}),
|
|
185
|
+
...(coolingReason !== undefined
|
|
186
|
+
? { accountCoolingReason: coolingReason }
|
|
187
|
+
: {}),
|
|
188
|
+
...(pool ? { pool } : {}),
|
|
189
|
+
...(requestId !== undefined ? { requestId } : {}),
|
|
190
|
+
status,
|
|
191
|
+
capturedAt: now,
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Wrap a fetch so every response's limit headers are captured into the
|
|
196
|
+
* enclosing `withLimitCapture` scope. A no-op outside such a scope.
|
|
197
|
+
*
|
|
198
|
+
* Capture never alters the response and never throws — a parsing failure must
|
|
199
|
+
* not be able to break a request that the provider would otherwise complete.
|
|
200
|
+
*/
|
|
201
|
+
export function wrapFetchWithLimitCapture(inner) {
|
|
202
|
+
return async (input, init) => {
|
|
203
|
+
const response = await inner(input, init);
|
|
204
|
+
const slot = limitCaptureStorage.getStore();
|
|
205
|
+
if (!slot) {
|
|
206
|
+
return response;
|
|
207
|
+
}
|
|
208
|
+
try {
|
|
209
|
+
const raw = {};
|
|
210
|
+
response.headers.forEach((value, key) => {
|
|
211
|
+
raw[key] = value;
|
|
212
|
+
});
|
|
213
|
+
slot.headers = raw;
|
|
214
|
+
const snapshot = buildLimitSnapshot(response.headers, response.status);
|
|
215
|
+
if (snapshot) {
|
|
216
|
+
// Last write wins: a retried or multi-step call reports the most
|
|
217
|
+
// recent upstream state, which is the one a caller acts on.
|
|
218
|
+
slot.snapshot = snapshot;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
// Diagnostics only — never disturb the response.
|
|
223
|
+
}
|
|
224
|
+
return response;
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* Run `body` in a capture scope and return its result alongside whatever limit
|
|
229
|
+
* snapshot the underlying HTTP request(s) produced.
|
|
230
|
+
*/
|
|
231
|
+
export async function withLimitCapture(body) {
|
|
232
|
+
const slot = {};
|
|
233
|
+
const result = await limitCaptureStorage.run(slot, body);
|
|
234
|
+
return {
|
|
235
|
+
result,
|
|
236
|
+
...(slot.snapshot ? { snapshot: slot.snapshot } : {}),
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
/**
|
|
240
|
+
* Current scope's snapshot, if any. Lets a long-running loop (the streaming
|
|
241
|
+
* path) read limits mid-flight without unwinding the scope.
|
|
242
|
+
*/
|
|
243
|
+
export function getCapturedLimitSnapshot() {
|
|
244
|
+
return limitCaptureStorage.getStore()?.snapshot;
|
|
245
|
+
}
|
|
246
|
+
/** Raw headers of the most recent captured response in this scope. */
|
|
247
|
+
export function getCapturedResponseHeaders() {
|
|
248
|
+
return limitCaptureStorage.getStore()?.headers;
|
|
249
|
+
}
|
|
250
|
+
/** Enter a capture scope without wrapping a single call — for streaming, where
|
|
251
|
+
* the scope must outlive the function that opened it. */
|
|
252
|
+
export function runInLimitCaptureScope(body) {
|
|
253
|
+
return limitCaptureStorage.run({}, body);
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Attach limit state to the currently active OTel span.
|
|
257
|
+
*
|
|
258
|
+
* Uses the active span rather than threading one down from the generation
|
|
259
|
+
* layer: that layer is provider-agnostic and should not learn about Anthropic
|
|
260
|
+
* quota headers just to record them. The active span during a turn is the one
|
|
261
|
+
* already carrying `gen_ai.usage.*` and `neurolink.cost`, so "what did this
|
|
262
|
+
* cost" and "how much is left" answer from the same trace.
|
|
263
|
+
*/
|
|
264
|
+
export function setLimitSpanAttributes(snapshot) {
|
|
265
|
+
const span = trace.getActiveSpan();
|
|
266
|
+
if (!span) {
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
const { rateLimit } = snapshot;
|
|
270
|
+
const attrs = [
|
|
271
|
+
["neurolink.claude.quota.session_left_pct", rateLimit.sessionLeftPct],
|
|
272
|
+
["neurolink.claude.quota.weekly_left_pct", rateLimit.weeklyLeftPct],
|
|
273
|
+
["neurolink.claude.quota.requests_remaining", rateLimit.requestsRemaining],
|
|
274
|
+
["neurolink.claude.quota.tokens_remaining", rateLimit.tokensRemaining],
|
|
275
|
+
["neurolink.claude.quota.source", snapshot.quotaSource],
|
|
276
|
+
["neurolink.claude.account", snapshot.account],
|
|
277
|
+
["neurolink.claude.served_by", snapshot.servedBy],
|
|
278
|
+
["neurolink.claude.pool.available", snapshot.pool?.available],
|
|
279
|
+
[
|
|
280
|
+
"neurolink.claude.pool.best_session_left_pct",
|
|
281
|
+
snapshot.pool?.bestSessionLeftPct,
|
|
282
|
+
],
|
|
283
|
+
];
|
|
284
|
+
for (const [key, value] of attrs) {
|
|
285
|
+
if (value !== undefined) {
|
|
286
|
+
span.setAttribute(key, value);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
/** Seconds-from-now for an epoch-seconds reset, or undefined if absent/past. */
|
|
291
|
+
function resetsInSeconds(resetAt, now) {
|
|
292
|
+
if (!resetAt || resetAt <= 0) {
|
|
293
|
+
return undefined;
|
|
294
|
+
}
|
|
295
|
+
// Tolerate a value already expressed in ms (year 2100 in seconds).
|
|
296
|
+
const ms = resetAt > 4_102_444_800 ? resetAt : resetAt * 1000;
|
|
297
|
+
return ms > now ? Math.round((ms - now) / 1000) : undefined;
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* Emit one structured line per request describing remaining capacity.
|
|
301
|
+
*
|
|
302
|
+
* Leads with headroom ("how much is left") because that is the figure an
|
|
303
|
+
* operator acts on; the raw utilization stays available on the snapshot for
|
|
304
|
+
* anything computing against it. Escalates to WARN when the session window is
|
|
305
|
+
* nearly spent or the provider has already flagged the account as
|
|
306
|
+
* throttled/rejected.
|
|
307
|
+
*/
|
|
308
|
+
export function logClaudeLimitSnapshot(snapshot, model, now = Date.now()) {
|
|
309
|
+
const { rateLimit } = snapshot;
|
|
310
|
+
// A fallback provider served this — there is no Anthropic capacity to report.
|
|
311
|
+
if (snapshot.quotaSource === "none" && snapshot.servedBy) {
|
|
312
|
+
logger.debug("[Anthropic] request served without account quota", {
|
|
313
|
+
servedBy: snapshot.servedBy,
|
|
314
|
+
...(model ? { model } : {}),
|
|
315
|
+
});
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
const details = {
|
|
319
|
+
...(model ? { model } : {}),
|
|
320
|
+
...(snapshot.account ? { account: snapshot.account } : {}),
|
|
321
|
+
...(snapshot.accountType ? { accountType: snapshot.accountType } : {}),
|
|
322
|
+
...(snapshot.servedBy ? { servedBy: snapshot.servedBy } : {}),
|
|
323
|
+
...(snapshot.quotaSource ? { quotaSource: snapshot.quotaSource } : {}),
|
|
324
|
+
};
|
|
325
|
+
if (rateLimit.sessionLeftPct !== undefined) {
|
|
326
|
+
details.sessionLeftPct = rateLimit.sessionLeftPct;
|
|
327
|
+
const resets = resetsInSeconds(rateLimit.sessionResetAt, now);
|
|
328
|
+
if (resets !== undefined) {
|
|
329
|
+
details.sessionResetsInSec = resets;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
if (rateLimit.weeklyLeftPct !== undefined) {
|
|
333
|
+
details.weeklyLeftPct = rateLimit.weeklyLeftPct;
|
|
334
|
+
const resets = resetsInSeconds(rateLimit.weeklyResetAt, now);
|
|
335
|
+
if (resets !== undefined) {
|
|
336
|
+
details.weeklyResetsInSec = resets;
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
// API-key accounts report absolute remaining rather than a percentage.
|
|
340
|
+
if (rateLimit.requestsRemaining !== undefined) {
|
|
341
|
+
details.requestsRemaining = rateLimit.requestsRemaining;
|
|
342
|
+
}
|
|
343
|
+
if (rateLimit.tokensRemaining !== undefined) {
|
|
344
|
+
details.tokensRemaining = rateLimit.tokensRemaining;
|
|
345
|
+
}
|
|
346
|
+
if (rateLimit.retryAfter !== undefined) {
|
|
347
|
+
details.retryAfterSec = rateLimit.retryAfter;
|
|
348
|
+
}
|
|
349
|
+
if (snapshot.pool) {
|
|
350
|
+
details.pool = snapshot.pool;
|
|
351
|
+
}
|
|
352
|
+
if (Object.keys(details).length === 0) {
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
const status = (rateLimit.unifiedStatus ??
|
|
356
|
+
rateLimit.sessionStatus ??
|
|
357
|
+
"").toLowerCase();
|
|
358
|
+
const lowHeadroom = rateLimit.sessionLeftPct !== undefined &&
|
|
359
|
+
rateLimit.sessionLeftPct <= LOW_HEADROOM_WARN_PCT;
|
|
360
|
+
const flagged = status === "throttled" || status === "rejected";
|
|
361
|
+
if (lowHeadroom || flagged) {
|
|
362
|
+
// `always`, not `warn`: this logger suppresses everything below `error`
|
|
363
|
+
// unless debug mode is on, and "you are about to run out of capacity" is
|
|
364
|
+
// precisely the thing an operator must see during a normal run. The
|
|
365
|
+
// routine per-request line below stays debug-gated so this stays rare
|
|
366
|
+
// enough to mean something.
|
|
367
|
+
logger.always(`[Anthropic] account limits running low — ${JSON.stringify({
|
|
368
|
+
...details,
|
|
369
|
+
...(status ? { status } : {}),
|
|
370
|
+
})}`);
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
logger.info("[Anthropic] account limits", details);
|
|
374
|
+
}
|
|
375
|
+
//# sourceMappingURL=rateLimitCapture.js.map
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { appendFile } from "node:fs/promises";
|
|
1
2
|
import type { ProxyLifecycleEventInput, ProxyLifecycleLoggerOptions, ProxyLifecycleLoggerSnapshot } from "../types/index.js";
|
|
2
3
|
export declare function hashProxyLifecycleSessionId(sessionId: string | undefined): string | undefined;
|
|
3
4
|
export declare function configureProxyLifecycleLogger(options: ProxyLifecycleLoggerOptions): void;
|
|
@@ -6,3 +7,7 @@ export declare function logProxyLifecycleEvent(input: ProxyLifecycleEventInput):
|
|
|
6
7
|
export declare function flushProxyLifecycleEvents(): Promise<void>;
|
|
7
8
|
export declare function getProxyLifecycleLoggerSnapshot(): ProxyLifecycleLoggerSnapshot;
|
|
8
9
|
export declare function resetProxyLifecycleLoggerForTests(): void;
|
|
10
|
+
/** Isolated failure injection for lifecycle durability tests. */
|
|
11
|
+
export declare const __proxyLifecycleTestHooks: {
|
|
12
|
+
setAppendFileForTests(append: typeof appendFile): void;
|
|
13
|
+
};
|
|
@@ -9,6 +9,8 @@ const SCHEMA_VERSION = 1;
|
|
|
9
9
|
const DEFAULT_QUEUE_CAPACITY = 10_000;
|
|
10
10
|
const DEFAULT_BATCH_SIZE = 256;
|
|
11
11
|
const DEFAULT_FLUSH_INTERVAL_MS = 25;
|
|
12
|
+
const DEFAULT_MAX_WRITE_RETRIES = 3;
|
|
13
|
+
const MAX_WRITE_RETRY_DELAY_MS = 1_000;
|
|
12
14
|
const LIFECYCLE_APPEND_TIMEOUT_MS = 2_000;
|
|
13
15
|
const MAX_SHORT_FIELD_LENGTH = 256;
|
|
14
16
|
const SESSION_KEY_FILE = ".proxy-lifecycle-session-key";
|
|
@@ -17,6 +19,7 @@ let lifecycleLogDir;
|
|
|
17
19
|
let queueCapacity = DEFAULT_QUEUE_CAPACITY;
|
|
18
20
|
let batchSize = DEFAULT_BATCH_SIZE;
|
|
19
21
|
let flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
|
|
22
|
+
let maxWriteRetries = DEFAULT_MAX_WRITE_RETRIES;
|
|
20
23
|
let processInstanceId = randomUUID();
|
|
21
24
|
let sessionHashKey = randomBytes(32);
|
|
22
25
|
let nextSequence = 1;
|
|
@@ -28,10 +31,13 @@ let queueDrops = 0;
|
|
|
28
31
|
let invalidDrops = 0;
|
|
29
32
|
let writeDrops = 0;
|
|
30
33
|
let writeFailures = 0;
|
|
34
|
+
let writeRetries = 0;
|
|
31
35
|
let inFlight = 0;
|
|
32
36
|
let queue = [];
|
|
33
37
|
let flushTimer;
|
|
34
38
|
let flushInFlight;
|
|
39
|
+
let nextFlushDelayMs;
|
|
40
|
+
let appendLifecycleFile = appendFile;
|
|
35
41
|
function positiveInteger(value, fallback) {
|
|
36
42
|
return Number.isInteger(value) && (value ?? 0) > 0
|
|
37
43
|
? value
|
|
@@ -123,14 +129,14 @@ function clearScheduledFlush() {
|
|
|
123
129
|
flushTimer = undefined;
|
|
124
130
|
}
|
|
125
131
|
}
|
|
126
|
-
function scheduleFlush() {
|
|
132
|
+
function scheduleFlush(delayMs = flushIntervalMs) {
|
|
127
133
|
if (flushTimer || flushInFlight || queue.length === 0) {
|
|
128
134
|
return;
|
|
129
135
|
}
|
|
130
136
|
flushTimer = setTimeout(() => {
|
|
131
137
|
flushTimer = undefined;
|
|
132
138
|
void startFlush();
|
|
133
|
-
},
|
|
139
|
+
}, delayMs);
|
|
134
140
|
flushTimer.unref?.();
|
|
135
141
|
}
|
|
136
142
|
async function flushBatch() {
|
|
@@ -143,27 +149,53 @@ async function flushBatch() {
|
|
|
143
149
|
const byPath = new Map();
|
|
144
150
|
for (const item of batch) {
|
|
145
151
|
const path = join(item.logDir, `proxy-lifecycle-${item.date}.jsonl`);
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
byPath.set(path,
|
|
152
|
+
const items = byPath.get(path) ?? [];
|
|
153
|
+
items.push(item);
|
|
154
|
+
byPath.set(path, items);
|
|
149
155
|
}
|
|
150
|
-
|
|
156
|
+
const retries = [];
|
|
157
|
+
let retryDelayMs = 0;
|
|
158
|
+
for (const [path, items] of byPath) {
|
|
159
|
+
const lines = items.map((item) => `${JSON.stringify(item.record)}\n`);
|
|
151
160
|
try {
|
|
152
161
|
// This best-effort telemetry sink intentionally avoids fsync so request
|
|
153
162
|
// throughput is not coupled to storage latency. Loss is surfaced by
|
|
154
163
|
// writeDrops/writeFailures rather than delaying proxy responses.
|
|
155
|
-
await withTimeout(
|
|
164
|
+
await withTimeout(appendLifecycleFile(path, lines.join(""), { mode: 0o600 }), LIFECYCLE_APPEND_TIMEOUT_MS, "Timed out writing proxy lifecycle metadata");
|
|
156
165
|
written += lines.length;
|
|
157
166
|
}
|
|
158
167
|
catch (error) {
|
|
159
|
-
dropped += lines.length;
|
|
160
|
-
writeDrops += lines.length;
|
|
161
168
|
writeFailures += 1;
|
|
169
|
+
const retryable = items.filter((item) => item.writeRetries < maxWriteRetries);
|
|
170
|
+
const exhausted = items.length - retryable.length;
|
|
171
|
+
if (retryable.length > 0) {
|
|
172
|
+
const nextRetries = retryable.map((item) => ({
|
|
173
|
+
...item,
|
|
174
|
+
writeRetries: item.writeRetries + 1,
|
|
175
|
+
}));
|
|
176
|
+
retries.push(...nextRetries);
|
|
177
|
+
writeRetries += nextRetries.length;
|
|
178
|
+
retryDelayMs = Math.max(retryDelayMs, Math.min(MAX_WRITE_RETRY_DELAY_MS, flushIntervalMs *
|
|
179
|
+
2 ** Math.max(...nextRetries.map((item) => item.writeRetries))));
|
|
180
|
+
}
|
|
181
|
+
if (exhausted > 0) {
|
|
182
|
+
dropped += exhausted;
|
|
183
|
+
writeDrops += exhausted;
|
|
184
|
+
}
|
|
162
185
|
logger.warn("[proxy] lifecycle metadata write failed", {
|
|
186
|
+
path,
|
|
187
|
+
retrying: retryable.length,
|
|
188
|
+
dropped: exhausted,
|
|
163
189
|
error: error instanceof Error ? error.message : String(error),
|
|
164
190
|
});
|
|
165
191
|
}
|
|
166
192
|
}
|
|
193
|
+
if (retries.length > 0) {
|
|
194
|
+
// Keep retried records ahead of newly admitted records. This preserves
|
|
195
|
+
// per-file sequence order while continuing to keep request paths async.
|
|
196
|
+
queue.unshift(...retries);
|
|
197
|
+
nextFlushDelayMs = Math.max(nextFlushDelayMs ?? 0, retryDelayMs || flushIntervalMs);
|
|
198
|
+
}
|
|
167
199
|
}
|
|
168
200
|
finally {
|
|
169
201
|
inFlight = Math.max(0, inFlight - batch.length);
|
|
@@ -179,21 +211,27 @@ function startFlush() {
|
|
|
179
211
|
if (flushInFlight === currentFlush) {
|
|
180
212
|
flushInFlight = undefined;
|
|
181
213
|
}
|
|
182
|
-
|
|
214
|
+
const delayMs = nextFlushDelayMs;
|
|
215
|
+
nextFlushDelayMs = undefined;
|
|
216
|
+
scheduleFlush(delayMs);
|
|
183
217
|
}, () => {
|
|
184
218
|
if (flushInFlight === currentFlush) {
|
|
185
219
|
flushInFlight = undefined;
|
|
186
220
|
}
|
|
187
|
-
|
|
221
|
+
const delayMs = nextFlushDelayMs;
|
|
222
|
+
nextFlushDelayMs = undefined;
|
|
223
|
+
scheduleFlush(delayMs);
|
|
188
224
|
});
|
|
189
225
|
return currentFlush;
|
|
190
226
|
}
|
|
191
227
|
export function configureProxyLifecycleLogger(options) {
|
|
192
228
|
clearScheduledFlush();
|
|
229
|
+
nextFlushDelayMs = undefined;
|
|
193
230
|
loggerEnabled = false;
|
|
194
231
|
lifecycleLogDir = undefined;
|
|
195
232
|
queueCapacity = positiveInteger(options.queueCapacity, DEFAULT_QUEUE_CAPACITY);
|
|
196
233
|
batchSize = positiveInteger(options.batchSize, DEFAULT_BATCH_SIZE);
|
|
234
|
+
maxWriteRetries = positiveInteger(options.maxWriteRetries, DEFAULT_MAX_WRITE_RETRIES);
|
|
197
235
|
flushIntervalMs = positiveInteger(options.flushIntervalMs, DEFAULT_FLUSH_INTERVAL_MS);
|
|
198
236
|
if (options.enabled && options.logDir) {
|
|
199
237
|
try {
|
|
@@ -268,6 +306,7 @@ export function logProxyLifecycleEvent(input) {
|
|
|
268
306
|
logDir: lifecycleLogDir,
|
|
269
307
|
date: String(record.timestamp).slice(0, 10),
|
|
270
308
|
record,
|
|
309
|
+
writeRetries: 0,
|
|
271
310
|
});
|
|
272
311
|
enqueued += 1;
|
|
273
312
|
scheduleFlush();
|
|
@@ -303,6 +342,7 @@ export function getProxyLifecycleLoggerSnapshot() {
|
|
|
303
342
|
invalidDrops,
|
|
304
343
|
writeDrops,
|
|
305
344
|
writeFailures,
|
|
345
|
+
writeRetries,
|
|
306
346
|
pending: queue.length,
|
|
307
347
|
inFlight,
|
|
308
348
|
flushing: flushInFlight !== undefined,
|
|
@@ -315,6 +355,7 @@ export function resetProxyLifecycleLoggerForTests() {
|
|
|
315
355
|
queueCapacity = DEFAULT_QUEUE_CAPACITY;
|
|
316
356
|
batchSize = DEFAULT_BATCH_SIZE;
|
|
317
357
|
flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
|
|
358
|
+
maxWriteRetries = DEFAULT_MAX_WRITE_RETRIES;
|
|
318
359
|
processInstanceId = randomUUID();
|
|
319
360
|
sessionHashKey = randomBytes(32);
|
|
320
361
|
nextSequence = 1;
|
|
@@ -326,8 +367,17 @@ export function resetProxyLifecycleLoggerForTests() {
|
|
|
326
367
|
invalidDrops = 0;
|
|
327
368
|
writeDrops = 0;
|
|
328
369
|
writeFailures = 0;
|
|
370
|
+
writeRetries = 0;
|
|
329
371
|
inFlight = 0;
|
|
330
372
|
queue = [];
|
|
331
373
|
flushInFlight = undefined;
|
|
374
|
+
nextFlushDelayMs = undefined;
|
|
375
|
+
appendLifecycleFile = appendFile;
|
|
332
376
|
}
|
|
377
|
+
/** Isolated failure injection for lifecycle durability tests. */
|
|
378
|
+
export const __proxyLifecycleTestHooks = {
|
|
379
|
+
setAppendFileForTests(append) {
|
|
380
|
+
appendLifecycleFile = append;
|
|
381
|
+
},
|
|
382
|
+
};
|
|
333
383
|
//# sourceMappingURL=proxyLifecycle.js.map
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Proxy response quota headers.
|
|
3
|
+
*
|
|
4
|
+
* The proxy already knows, per request, exactly how much subscription capacity
|
|
5
|
+
* the serving account has left — it parses Anthropic's `anthropic-ratelimit-*`
|
|
6
|
+
* headers to route on them (see `accountQuota.ts`). Historically none of that
|
|
7
|
+
* reached the client: only the SSE path forwarded a small legacy allowlist, and
|
|
8
|
+
* every JSON/error path dropped headers entirely.
|
|
9
|
+
*
|
|
10
|
+
* This module turns that state into response headers in two layers:
|
|
11
|
+
*
|
|
12
|
+
* 1. **Verbatim passthrough** of Anthropic's own `anthropic-ratelimit-*` and
|
|
13
|
+
* `retry-after` headers. This is the load-bearing part: a proxied response
|
|
14
|
+
* then looks byte-identical to a direct one, so a consumer needs exactly one
|
|
15
|
+
* parser for both.
|
|
16
|
+
* 2. **`x-neurolink-*`** for what only the proxy can know — which account
|
|
17
|
+
* served the request, pool headroom, whether the numbers are live or stale,
|
|
18
|
+
* and the derived "how much is left" percentages.
|
|
19
|
+
*
|
|
20
|
+
* Pure CPU, no I/O — safe on the hot path and directly unit-testable.
|
|
21
|
+
*
|
|
22
|
+
* @module proxy/quotaHeaders
|
|
23
|
+
*/
|
|
24
|
+
import type { AccountQuota, ProxyQuotaHeaderContext } from "../types/index.js";
|
|
25
|
+
/**
|
|
26
|
+
* Convert a 0.0-1.0 utilization fraction into a whole-percent "left" figure.
|
|
27
|
+
*
|
|
28
|
+
* Anthropic publishes utilization (used), never remaining, for subscription
|
|
29
|
+
* windows — there is no absolute message or token count to report, so the
|
|
30
|
+
* honest derived form is a percentage. Clamped because a utilization above 1.0
|
|
31
|
+
* (overage) would otherwise produce a negative "left".
|
|
32
|
+
*/
|
|
33
|
+
export declare function utilizationToLeftPct(used: number): number;
|
|
34
|
+
/**
|
|
35
|
+
* Copy Anthropic's rate-limit headers verbatim from an upstream response.
|
|
36
|
+
*
|
|
37
|
+
* Covers both header families: the unified subscription windows
|
|
38
|
+
* (`unified-5h-*`, `unified-7d-*`) and the legacy per-tier counters
|
|
39
|
+
* (`requests-remaining`, `tokens-remaining`, ...) that API-key accounts get.
|
|
40
|
+
* Which family is present depends on the serving account type, which is why
|
|
41
|
+
* `x-neurolink-account-type` accompanies them.
|
|
42
|
+
*/
|
|
43
|
+
export declare function pickUpstreamRateLimitHeaders(headers: Headers | Record<string, string>): Record<string, string>;
|
|
44
|
+
/**
|
|
45
|
+
* Build the `x-neurolink-*` half of the contract from proxy-side state.
|
|
46
|
+
*
|
|
47
|
+
* Always emits `x-neurolink-quota-source` — even when it is "none". A consumer
|
|
48
|
+
* that sees quota numbers with no provenance cannot tell a fresh reading from a
|
|
49
|
+
* snapshot carried over from a previous request, and would happily log stale
|
|
50
|
+
* capacity as current.
|
|
51
|
+
*/
|
|
52
|
+
export declare function buildQuotaResponseHeaders(context: ProxyQuotaHeaderContext, now?: number): Record<string, string>;
|
|
53
|
+
/**
|
|
54
|
+
* Full response header set: upstream verbatim + proxy-derived.
|
|
55
|
+
*
|
|
56
|
+
* Upstream headers are applied first so a `x-neurolink-*` key can never be
|
|
57
|
+
* shadowed by an upstream one (they share no names today, but the ordering
|
|
58
|
+
* makes the precedence explicit rather than incidental).
|
|
59
|
+
*/
|
|
60
|
+
export declare function buildProxyLimitHeaders(args: {
|
|
61
|
+
upstreamHeaders?: Headers | Record<string, string>;
|
|
62
|
+
context: ProxyQuotaHeaderContext;
|
|
63
|
+
now?: number;
|
|
64
|
+
}): Record<string, string>;
|
|
65
|
+
/** Compute pool headroom from the runtime account states backing a request. */
|
|
66
|
+
export declare function summarizePoolHeadroom(entries: ReadonlyArray<{
|
|
67
|
+
coolingUntil?: number;
|
|
68
|
+
quota?: AccountQuota;
|
|
69
|
+
}>, now?: number): {
|
|
70
|
+
available: number;
|
|
71
|
+
cooling: number;
|
|
72
|
+
bestSessionLeftPct?: number;
|
|
73
|
+
};
|