@juspay/neurolink 10.4.3 → 10.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/auth/tokenStore.d.ts +7 -0
- package/dist/auth/tokenStore.js +54 -0
- package/dist/browser/neurolink.min.js +374 -374
- package/dist/cli/commands/proxy.d.ts +1 -0
- package/dist/cli/commands/proxy.js +183 -24
- package/dist/lib/auth/tokenStore.d.ts +7 -0
- package/dist/lib/auth/tokenStore.js +54 -0
- package/dist/lib/proxy/proxyTranslationEngine.d.ts +1 -0
- package/dist/lib/proxy/proxyTranslationEngine.js +56 -12
- package/dist/lib/proxy/rawStreamCapture.js +47 -1
- package/dist/lib/proxy/sseInterceptor.js +47 -1
- package/dist/lib/proxy/tokenRefresh.d.ts +6 -0
- package/dist/lib/proxy/tokenRefresh.js +70 -15
- package/dist/lib/proxy/usageStats.d.ts +25 -3
- package/dist/lib/proxy/usageStats.js +546 -55
- package/dist/lib/server/routes/claudeProxyRoutes.d.ts +2 -0
- package/dist/lib/server/routes/claudeProxyRoutes.js +198 -222
- package/dist/lib/types/proxy.d.ts +60 -1
- package/dist/proxy/proxyTranslationEngine.d.ts +1 -0
- package/dist/proxy/proxyTranslationEngine.js +56 -12
- package/dist/proxy/rawStreamCapture.js +47 -1
- package/dist/proxy/sseInterceptor.js +47 -1
- package/dist/proxy/tokenRefresh.d.ts +6 -0
- package/dist/proxy/tokenRefresh.js +70 -15
- package/dist/proxy/usageStats.d.ts +25 -3
- package/dist/proxy/usageStats.js +546 -55
- package/dist/server/routes/claudeProxyRoutes.d.ts +2 -0
- package/dist/server/routes/claudeProxyRoutes.js +198 -222
- package/dist/types/proxy.d.ts +60 -1
- package/package.json +1 -1
|
@@ -671,6 +671,29 @@ export type AnthropicUpstreamFetchResult = {
|
|
|
671
671
|
sawNetworkError: boolean;
|
|
672
672
|
upstreamSpan?: Span;
|
|
673
673
|
};
|
|
674
|
+
export type ProxyTerminalErrorCategory = "authentication" | "client_cancelled" | "fallback_exhausted" | "invalid_request" | "proxy_error" | "rate_limit" | "stream_error" | "unclassified" | "upstream_error" | "other";
|
|
675
|
+
/** Compact, redacted terminal failure retained independently of body logs. */
|
|
676
|
+
export type ProxyTerminalErrorSummary = {
|
|
677
|
+
id: string;
|
|
678
|
+
at: number;
|
|
679
|
+
status: number;
|
|
680
|
+
category: ProxyTerminalErrorCategory;
|
|
681
|
+
requestId?: string;
|
|
682
|
+
account?: string;
|
|
683
|
+
accountType?: string;
|
|
684
|
+
errorType?: string;
|
|
685
|
+
errorCode?: string;
|
|
686
|
+
terminalOutcome?: string;
|
|
687
|
+
message?: string;
|
|
688
|
+
};
|
|
689
|
+
/** Optional terminal context supplied when a final request error is recorded. */
|
|
690
|
+
export type ProxyTerminalErrorDetails = {
|
|
691
|
+
requestId?: string;
|
|
692
|
+
errorType?: string;
|
|
693
|
+
errorCode?: string;
|
|
694
|
+
terminalOutcome?: string;
|
|
695
|
+
message?: string;
|
|
696
|
+
};
|
|
674
697
|
export type AccountStats = {
|
|
675
698
|
label: string;
|
|
676
699
|
type: string;
|
|
@@ -700,6 +723,19 @@ export type ProxyStats = {
|
|
|
700
723
|
totalQuotaRateLimits: number;
|
|
701
724
|
accounts: Record<string, AccountStats>;
|
|
702
725
|
};
|
|
726
|
+
/** Bounded terminal-error state stored separately from counters and body logs. */
|
|
727
|
+
export type ProxyTerminalErrorJournal = {
|
|
728
|
+
startedAt: number;
|
|
729
|
+
totalErrors: number;
|
|
730
|
+
counts: Record<ProxyTerminalErrorCategory, number>;
|
|
731
|
+
recent: ProxyTerminalErrorSummary[];
|
|
732
|
+
};
|
|
733
|
+
export type ProxyUsageStatsSnapshot = {
|
|
734
|
+
stats: ProxyStats;
|
|
735
|
+
statsVersion: number;
|
|
736
|
+
terminalErrors: ProxyTerminalErrorJournal;
|
|
737
|
+
terminalErrorsVersion: number;
|
|
738
|
+
};
|
|
703
739
|
/** Durability and reconciliation state for the proxy usage counters. */
|
|
704
740
|
export type ProxyStatsPersistenceStatus = {
|
|
705
741
|
enabled: boolean;
|
|
@@ -712,6 +748,14 @@ export type ProxyStatsPersistenceStatus = {
|
|
|
712
748
|
lastReconciledAt: number | null;
|
|
713
749
|
lastRecoveryAt: number | null;
|
|
714
750
|
lastError: string | null;
|
|
751
|
+
terminalErrorsFilePath?: string | null;
|
|
752
|
+
terminalErrorsRevision?: number;
|
|
753
|
+
terminalErrorsPending?: number;
|
|
754
|
+
terminalErrorsInFlight?: number;
|
|
755
|
+
terminalErrorsUnpersisted?: number;
|
|
756
|
+
terminalErrorsLastFlushedAt?: number | null;
|
|
757
|
+
terminalErrorsLastRecoveryAt?: number | null;
|
|
758
|
+
terminalErrorsLastError?: string | null;
|
|
715
759
|
};
|
|
716
760
|
/** Versioned on-disk snapshot shared by overlapping proxy workers. */
|
|
717
761
|
export type PersistedProxyStatsSnapshot = {
|
|
@@ -720,6 +764,13 @@ export type PersistedProxyStatsSnapshot = {
|
|
|
720
764
|
updatedAt: number;
|
|
721
765
|
stats: ProxyStats;
|
|
722
766
|
};
|
|
767
|
+
/** Versioned terminal-error snapshot isolated from rolling-worker counter writes. */
|
|
768
|
+
export type PersistedProxyTerminalErrorSnapshot = {
|
|
769
|
+
schemaVersion: 1;
|
|
770
|
+
revision: number;
|
|
771
|
+
updatedAt: number;
|
|
772
|
+
journal: ProxyTerminalErrorJournal;
|
|
773
|
+
};
|
|
723
774
|
/** Ownership metadata for the proxy statistics cross-process lock. */
|
|
724
775
|
export type ProxyStatsLockOwner = {
|
|
725
776
|
token: string;
|
|
@@ -729,6 +780,7 @@ export type ProxyStatsLockOwner = {
|
|
|
729
780
|
/** Construction options for an isolated proxy statistics store. */
|
|
730
781
|
export type ProxyUsageStatsStoreOptions = {
|
|
731
782
|
filePath?: string;
|
|
783
|
+
terminalErrorsFilePath?: string;
|
|
732
784
|
flushIntervalMs?: number;
|
|
733
785
|
lockTimeoutMs?: number;
|
|
734
786
|
staleLockMs?: number;
|
|
@@ -1951,6 +2003,11 @@ export type StatusStats = {
|
|
|
1951
2003
|
totalRateLimits: number;
|
|
1952
2004
|
totalTransientRateLimits?: number;
|
|
1953
2005
|
totalQuotaRateLimits?: number;
|
|
2006
|
+
terminalErrors?: ProxyTerminalErrorJournal;
|
|
2007
|
+
lastTerminalError?: ProxyTerminalErrorSummary | null;
|
|
2008
|
+
terminalErrorDetailsComparable?: boolean;
|
|
2009
|
+
terminalErrorDetailsMissing?: number;
|
|
2010
|
+
terminalErrorDetailsExcess?: number;
|
|
1954
2011
|
accounts?: {
|
|
1955
2012
|
label: string;
|
|
1956
2013
|
type: string;
|
|
@@ -1963,7 +2020,9 @@ export type StatusStats = {
|
|
|
1963
2020
|
transientRateLimits?: number;
|
|
1964
2021
|
quotaRateLimits?: number;
|
|
1965
2022
|
cooling: boolean;
|
|
1966
|
-
|
|
2023
|
+
allowed?: boolean;
|
|
2024
|
+
expired?: boolean;
|
|
2025
|
+
status?: "active" | "cooling" | "disabled" | "expired" | "excluded" | "removed" | "internal" | "unattributed";
|
|
1967
2026
|
}[];
|
|
1968
2027
|
persistence?: ProxyStatsPersistenceStatus;
|
|
1969
2028
|
};
|
|
@@ -77,6 +77,7 @@ export declare function handleTranslatedJsonRequest(args: {
|
|
|
77
77
|
attempts: ProxyTranslationAttempt[];
|
|
78
78
|
tracer?: ProxyTracer;
|
|
79
79
|
requestStartTime: number;
|
|
80
|
+
terminalFailureStatus?: number;
|
|
80
81
|
}): Promise<unknown>;
|
|
81
82
|
/**
|
|
82
83
|
* Build the /v1/models response in OpenAI list format.
|
|
@@ -247,6 +247,7 @@ export async function handleTranslatedStreamRequest(args) {
|
|
|
247
247
|
let keepAliveTimer;
|
|
248
248
|
let cancelled = false;
|
|
249
249
|
let succeeded = false;
|
|
250
|
+
let streamInterruptedAfterOutput = false;
|
|
250
251
|
let translatedModel;
|
|
251
252
|
let finalStreamError = "No translation providers succeeded";
|
|
252
253
|
let upstreamIterator;
|
|
@@ -267,6 +268,9 @@ export async function handleTranslatedStreamRequest(args) {
|
|
|
267
268
|
}, KEEPALIVE_INTERVAL_MS);
|
|
268
269
|
try {
|
|
269
270
|
for (let attemptIndex = 0; attemptIndex < attempts.length; attemptIndex++) {
|
|
271
|
+
if (cancelled) {
|
|
272
|
+
break;
|
|
273
|
+
}
|
|
270
274
|
const attempt = attempts[attemptIndex];
|
|
271
275
|
lastAttemptLabel = attempt.label ?? "translation";
|
|
272
276
|
logger.always(`[proxy:${format}] attempt ${attemptIndex + 1}/${attempts.length}: ${attempt.label}`);
|
|
@@ -295,10 +299,14 @@ export async function handleTranslatedStreamRequest(args) {
|
|
|
295
299
|
}
|
|
296
300
|
}
|
|
297
301
|
}
|
|
302
|
+
if (cancelled) {
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
298
305
|
const toolCalls = streamResult.toolCalls ?? [];
|
|
299
306
|
if (!hasTranslatedOutput(collectedText, toolCalls)) {
|
|
300
307
|
finalStreamError = `Translated provider ${attempt.label} returned no content or tool calls`;
|
|
301
308
|
logger.debug(`${tag} translation attempt ${attempt.label} returned no content or tool calls`);
|
|
309
|
+
recordAttemptError(lastAttemptLabel, "translation", 502);
|
|
302
310
|
continue;
|
|
303
311
|
}
|
|
304
312
|
if (!cancelled && toolCalls.length) {
|
|
@@ -331,7 +339,6 @@ export async function handleTranslatedStreamRequest(args) {
|
|
|
331
339
|
tracer?.recordMetrics();
|
|
332
340
|
translatedModel = streamResult.model;
|
|
333
341
|
succeeded = true;
|
|
334
|
-
recordFinalSuccess(lastAttemptLabel, "translation");
|
|
335
342
|
return;
|
|
336
343
|
}
|
|
337
344
|
catch (streamErr) {
|
|
@@ -343,6 +350,8 @@ export async function handleTranslatedStreamRequest(args) {
|
|
|
343
350
|
? streamErr.message
|
|
344
351
|
: String(streamErr);
|
|
345
352
|
if (collectedText.trim().length > 0) {
|
|
353
|
+
streamInterruptedAfterOutput = true;
|
|
354
|
+
recordAttemptError(lastAttemptLabel, "translation", 502);
|
|
346
355
|
logger.always(`${tag} mid-stream error: ${finalStreamError}`);
|
|
347
356
|
for (const frame of serializer.emitError(`Upstream stream interrupted: ${finalStreamError}`)) {
|
|
348
357
|
controller.enqueue(encoder.encode(frame));
|
|
@@ -354,7 +363,6 @@ export async function handleTranslatedStreamRequest(args) {
|
|
|
354
363
|
}
|
|
355
364
|
}
|
|
356
365
|
// All attempts exhausted
|
|
357
|
-
recordFinalError(500, lastAttemptLabel, "translation");
|
|
358
366
|
if (!cancelled) {
|
|
359
367
|
logger.always(`${tag} all translation attempts failed: ${finalStreamError}`);
|
|
360
368
|
for (const frame of serializer.emitError(finalStreamError)) {
|
|
@@ -372,12 +380,38 @@ export async function handleTranslatedStreamRequest(args) {
|
|
|
372
380
|
if (tracer && translatedModel && translatedModel !== requestModel) {
|
|
373
381
|
tracer.setModelSubstitution(requestModel, translatedModel);
|
|
374
382
|
}
|
|
375
|
-
|
|
376
|
-
|
|
383
|
+
const terminalStatus = cancelled
|
|
384
|
+
? 499
|
|
385
|
+
: succeeded
|
|
386
|
+
? 200
|
|
387
|
+
: streamInterruptedAfterOutput
|
|
388
|
+
? 502
|
|
389
|
+
: 500;
|
|
390
|
+
const terminalErrorType = cancelled
|
|
391
|
+
? "client_cancelled"
|
|
392
|
+
: streamInterruptedAfterOutput
|
|
393
|
+
? "stream_error"
|
|
394
|
+
: succeeded
|
|
395
|
+
? undefined
|
|
396
|
+
: "generation_error";
|
|
397
|
+
const terminalErrorMessage = cancelled
|
|
398
|
+
? "Client cancelled the streaming response"
|
|
399
|
+
: succeeded
|
|
400
|
+
? undefined
|
|
401
|
+
: finalStreamError;
|
|
402
|
+
if (terminalErrorType && terminalErrorMessage) {
|
|
403
|
+
tracer?.setError(terminalErrorType, terminalErrorMessage.slice(0, 500));
|
|
404
|
+
recordFinalError(terminalStatus, lastAttemptLabel, "translation", {
|
|
405
|
+
requestId: ctx.requestId,
|
|
406
|
+
errorType: terminalErrorType,
|
|
407
|
+
terminalOutcome: terminalErrorType,
|
|
408
|
+
message: terminalErrorMessage,
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
else {
|
|
412
|
+
recordFinalSuccess(lastAttemptLabel, "translation");
|
|
377
413
|
}
|
|
378
|
-
|
|
379
|
-
// responseStatus below (success path is 200, exhausted-attempts path is 500).
|
|
380
|
-
tracer?.end(succeeded ? 200 : 500, Date.now() - requestStartTime);
|
|
414
|
+
tracer?.end(terminalStatus, Date.now() - requestStartTime);
|
|
381
415
|
const traceCtx = tracer?.getTraceContext();
|
|
382
416
|
logRequest({
|
|
383
417
|
timestamp: new Date().toISOString(),
|
|
@@ -389,8 +423,12 @@ export async function handleTranslatedStreamRequest(args) {
|
|
|
389
423
|
toolCount: Object.keys(parsed.tools).length,
|
|
390
424
|
account: "translation",
|
|
391
425
|
accountType: "translation",
|
|
392
|
-
responseStatus:
|
|
426
|
+
responseStatus: terminalStatus,
|
|
393
427
|
responseTimeMs: Date.now() - requestStartTime,
|
|
428
|
+
...(terminalErrorType ? { errorType: terminalErrorType } : {}),
|
|
429
|
+
...(terminalErrorMessage
|
|
430
|
+
? { errorMessage: terminalErrorMessage.slice(0, 500) }
|
|
431
|
+
: {}),
|
|
394
432
|
...(traceCtx?.traceId ? { traceId: traceCtx.traceId } : {}),
|
|
395
433
|
...(traceCtx?.spanId ? { spanId: traceCtx.spanId } : {}),
|
|
396
434
|
});
|
|
@@ -424,7 +462,7 @@ export async function handleTranslatedStreamRequest(args) {
|
|
|
424
462
|
* Handles a translated non-streaming request for either Claude or OpenAI format.
|
|
425
463
|
*/
|
|
426
464
|
export async function handleTranslatedJsonRequest(args) {
|
|
427
|
-
const { ctx, format, requestModel, parsed, attempts, tracer, requestStartTime, } = args;
|
|
465
|
+
const { ctx, format, requestModel, parsed, attempts, tracer, requestStartTime, terminalFailureStatus = 500, } = args;
|
|
428
466
|
const tag = logTag(format);
|
|
429
467
|
let lastAttemptError = "No translation providers succeeded";
|
|
430
468
|
let lastAttemptLabel = "translation";
|
|
@@ -448,6 +486,7 @@ export async function handleTranslatedJsonRequest(args) {
|
|
|
448
486
|
if (!hasTranslatedOutput(collectedText, streamResult.toolCalls)) {
|
|
449
487
|
lastAttemptError = `Translated provider ${attempt.label} returned no content or tool calls`;
|
|
450
488
|
logger.debug(`${tag} translation attempt ${attempt.label} returned no content or tool calls`);
|
|
489
|
+
recordAttemptError(lastAttemptLabel, "translation", 502);
|
|
451
490
|
continue;
|
|
452
491
|
}
|
|
453
492
|
const internal = {
|
|
@@ -505,9 +544,14 @@ export async function handleTranslatedJsonRequest(args) {
|
|
|
505
544
|
recordAttemptError(lastAttemptLabel, "translation", 500);
|
|
506
545
|
}
|
|
507
546
|
}
|
|
508
|
-
recordFinalError(
|
|
547
|
+
recordFinalError(terminalFailureStatus, lastAttemptLabel, "translation", {
|
|
548
|
+
requestId: ctx.requestId,
|
|
549
|
+
errorType: "generation_error",
|
|
550
|
+
terminalOutcome: "handler_error",
|
|
551
|
+
message: lastAttemptError,
|
|
552
|
+
});
|
|
509
553
|
tracer?.setError("generation_error", lastAttemptError.slice(0, 500));
|
|
510
|
-
tracer?.end(
|
|
554
|
+
tracer?.end(terminalFailureStatus, Date.now() - requestStartTime);
|
|
511
555
|
const traceCtx = tracer?.getTraceContext();
|
|
512
556
|
logRequest({
|
|
513
557
|
timestamp: new Date().toISOString(),
|
|
@@ -519,7 +563,7 @@ export async function handleTranslatedJsonRequest(args) {
|
|
|
519
563
|
toolCount: Object.keys(parsed.tools).length,
|
|
520
564
|
account: "translation",
|
|
521
565
|
accountType: "translation",
|
|
522
|
-
responseStatus:
|
|
566
|
+
responseStatus: terminalFailureStatus,
|
|
523
567
|
responseTimeMs: Date.now() - requestStartTime,
|
|
524
568
|
errorType: "generation_error",
|
|
525
569
|
errorMessage: lastAttemptError.slice(0, 500),
|
|
@@ -60,7 +60,11 @@ export function createRawStreamCapture() {
|
|
|
60
60
|
},
|
|
61
61
|
});
|
|
62
62
|
const innerWriter = transform.writable.getWriter();
|
|
63
|
+
let writableController;
|
|
63
64
|
const writable = new WritableStream({
|
|
65
|
+
start(controller) {
|
|
66
|
+
writableController = controller;
|
|
67
|
+
},
|
|
64
68
|
write(chunk) {
|
|
65
69
|
return innerWriter.write(chunk);
|
|
66
70
|
},
|
|
@@ -72,9 +76,51 @@ export function createRawStreamCapture() {
|
|
|
72
76
|
return innerWriter.abort(reason);
|
|
73
77
|
},
|
|
74
78
|
});
|
|
79
|
+
// A downstream reader cancellation errors the TransformStream's writable
|
|
80
|
+
// side, but this wrapper otherwise hides that state from the upstream pipe.
|
|
81
|
+
// Mirror it onto the wrapper so cancellation reaches the source reader.
|
|
82
|
+
void innerWriter.closed.catch((reason) => {
|
|
83
|
+
settle();
|
|
84
|
+
try {
|
|
85
|
+
writableController.error(reason);
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
// The wrapper may already be closing or aborted.
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
const innerReader = transform.readable.getReader();
|
|
92
|
+
const readable = new ReadableStream({
|
|
93
|
+
async pull(controller) {
|
|
94
|
+
try {
|
|
95
|
+
const { done, value } = await innerReader.read();
|
|
96
|
+
if (done) {
|
|
97
|
+
controller.close();
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
controller.enqueue(value);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
controller.error(error);
|
|
105
|
+
}
|
|
106
|
+
},
|
|
107
|
+
async cancel(reason) {
|
|
108
|
+
settle();
|
|
109
|
+
try {
|
|
110
|
+
writableController.error(reason);
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
// The wrapper may already be closing or aborted.
|
|
114
|
+
}
|
|
115
|
+
await Promise.allSettled([
|
|
116
|
+
innerReader.cancel(reason),
|
|
117
|
+
innerWriter.abort(reason),
|
|
118
|
+
]);
|
|
119
|
+
},
|
|
120
|
+
});
|
|
75
121
|
return {
|
|
76
122
|
stream: {
|
|
77
|
-
readable
|
|
123
|
+
readable,
|
|
78
124
|
writable,
|
|
79
125
|
},
|
|
80
126
|
capture,
|
|
@@ -429,7 +429,11 @@ export function createSSEInterceptor(options = {}) {
|
|
|
429
429
|
// Wrap the writable side so we can intercept abort() — which does NOT
|
|
430
430
|
// trigger the TransformStream's flush() or cancel() callbacks.
|
|
431
431
|
const innerWriter = transform.writable.getWriter();
|
|
432
|
+
let writableController;
|
|
432
433
|
const writable = new WritableStream({
|
|
434
|
+
start(controller) {
|
|
435
|
+
writableController = controller;
|
|
436
|
+
},
|
|
433
437
|
write(chunk) {
|
|
434
438
|
return innerWriter.write(chunk);
|
|
435
439
|
},
|
|
@@ -441,8 +445,50 @@ export function createSSEInterceptor(options = {}) {
|
|
|
441
445
|
return innerWriter.abort(reason);
|
|
442
446
|
},
|
|
443
447
|
});
|
|
448
|
+
// Preserve downstream cancellation across the wrapped writable. Without
|
|
449
|
+
// this bridge, client disconnects leave the upstream source and telemetry
|
|
450
|
+
// promise open indefinitely after the readable side is cancelled.
|
|
451
|
+
void innerWriter.closed.catch((reason) => {
|
|
452
|
+
settle();
|
|
453
|
+
try {
|
|
454
|
+
writableController.error(reason);
|
|
455
|
+
}
|
|
456
|
+
catch {
|
|
457
|
+
// The wrapper may already be closing or aborted.
|
|
458
|
+
}
|
|
459
|
+
});
|
|
460
|
+
const innerReader = transform.readable.getReader();
|
|
461
|
+
const readable = new ReadableStream({
|
|
462
|
+
async pull(controller) {
|
|
463
|
+
try {
|
|
464
|
+
const { done, value } = await innerReader.read();
|
|
465
|
+
if (done) {
|
|
466
|
+
controller.close();
|
|
467
|
+
}
|
|
468
|
+
else {
|
|
469
|
+
controller.enqueue(value);
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
catch (error) {
|
|
473
|
+
controller.error(error);
|
|
474
|
+
}
|
|
475
|
+
},
|
|
476
|
+
async cancel(reason) {
|
|
477
|
+
settle();
|
|
478
|
+
try {
|
|
479
|
+
writableController.error(reason);
|
|
480
|
+
}
|
|
481
|
+
catch {
|
|
482
|
+
// The wrapper may already be closing or aborted.
|
|
483
|
+
}
|
|
484
|
+
await Promise.allSettled([
|
|
485
|
+
innerReader.cancel(reason),
|
|
486
|
+
innerWriter.abort(reason),
|
|
487
|
+
]);
|
|
488
|
+
},
|
|
489
|
+
});
|
|
444
490
|
const stream = {
|
|
445
|
-
readable
|
|
491
|
+
readable,
|
|
446
492
|
writable,
|
|
447
493
|
};
|
|
448
494
|
return { stream, telemetry: telemetryPromise };
|
|
@@ -8,5 +8,11 @@ export declare function isPermanentRefreshFailure(result: RefreshResult): boolea
|
|
|
8
8
|
* loaded either the old token or the newly rotated token around persistence.
|
|
9
9
|
*/
|
|
10
10
|
export declare function refreshToken(account: RefreshableAccount): Promise<RefreshResult>;
|
|
11
|
+
/**
|
|
12
|
+
* Refreshes against the latest persisted credential generation. Queued
|
|
13
|
+
* requests can otherwise reject a rotated refresh token and disable a newer
|
|
14
|
+
* login that was saved while they were waiting.
|
|
15
|
+
*/
|
|
16
|
+
export declare function refreshTokenFromLatest(account: RefreshableAccount, target?: TokenPersistTarget): Promise<RefreshResult>;
|
|
11
17
|
export declare function clearRefreshStateForTests(): void;
|
|
12
18
|
export declare function persistTokens(target: TokenPersistTarget, account: RefreshableAccount): Promise<void>;
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { logger } from "../utils/logger.js";
|
|
2
2
|
import { tokenStore } from "../auth/tokenStore.js";
|
|
3
|
+
import { ANTHROPIC_TOKEN_URL, ANTHROPIC_TOKEN_URL_FALLBACK, CLAUDE_CLI_USER_AGENT, CLAUDE_CODE_CLIENT_ID, } from "../auth/anthropicOAuth.js";
|
|
3
4
|
import { writeJsonSnapshotAtomically } from "./snapshotPersistence.js";
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const CLIENT_ID = "9d1c250a-e61b-44d9-88ed-5944d1962f5e";
|
|
5
|
+
import { withTimeout } from "../utils/async/withTimeout.js";
|
|
6
|
+
import { redactUrlForError } from "../utils/logSanitize.js";
|
|
7
7
|
const BUFFER_MS = 5 * 60 * 1000;
|
|
8
8
|
const SUCCESS_CACHE_MS = 60_000;
|
|
9
|
-
|
|
9
|
+
// Bound the best-effort persisted-credential reconciliation read so a hung or
|
|
10
|
+
// slow token-store decrypt can never stall (or abort) a live token refresh.
|
|
11
|
+
const PEEK_TOKENS_TIMEOUT_MS = 2_000;
|
|
10
12
|
const refreshesInFlight = new Map();
|
|
11
13
|
export function needsRefresh(account) {
|
|
12
14
|
return !!(account.expiresAt &&
|
|
@@ -23,28 +25,33 @@ async function performTokenRefresh(account) {
|
|
|
23
25
|
if (!account.refreshToken) {
|
|
24
26
|
return { success: false, error: "No refresh token available" };
|
|
25
27
|
}
|
|
26
|
-
|
|
28
|
+
// OAuth 2.0 token endpoints require an `application/x-www-form-urlencoded`
|
|
29
|
+
// request body (RFC 6749 §6); a JSON body is nonstandard for this grant and
|
|
30
|
+
// is rejected by RFC-compliant endpoints. Mirror the interactive auth flow
|
|
31
|
+
// in `anthropicOAuth.ts::_refreshAccessToken`.
|
|
32
|
+
const requestBody = new URLSearchParams({
|
|
27
33
|
grant_type: "refresh_token",
|
|
28
34
|
refresh_token: account.refreshToken,
|
|
29
|
-
client_id:
|
|
35
|
+
client_id: CLAUDE_CODE_CLIENT_ID,
|
|
30
36
|
}).toString();
|
|
31
37
|
const headers = {
|
|
32
38
|
"Content-Type": "application/x-www-form-urlencoded",
|
|
33
|
-
"
|
|
39
|
+
Accept: "application/json",
|
|
40
|
+
"User-Agent": CLAUDE_CLI_USER_AGENT,
|
|
34
41
|
};
|
|
35
|
-
const urls = [
|
|
42
|
+
const urls = [ANTHROPIC_TOKEN_URL, ANTHROPIC_TOKEN_URL_FALLBACK];
|
|
36
43
|
let terminalFailure;
|
|
37
44
|
for (const url of urls) {
|
|
38
45
|
try {
|
|
39
46
|
const resp = await fetch(url, {
|
|
40
47
|
method: "POST",
|
|
41
48
|
headers,
|
|
42
|
-
body:
|
|
49
|
+
body: requestBody,
|
|
43
50
|
signal: AbortSignal.timeout(10_000),
|
|
44
51
|
});
|
|
45
52
|
if (!resp.ok) {
|
|
46
53
|
const errorBody = await resp.text();
|
|
47
|
-
logger.warn(`[token-refresh] failed for ${account.label} at ${url}`, {
|
|
54
|
+
logger.warn(`[token-refresh] failed for ${account.label} at ${redactUrlForError(url)}`, {
|
|
48
55
|
status: resp.status,
|
|
49
56
|
error: errorBody.slice(0, 500),
|
|
50
57
|
});
|
|
@@ -57,7 +64,7 @@ async function performTokenRefresh(account) {
|
|
|
57
64
|
terminalFailure = failure;
|
|
58
65
|
}
|
|
59
66
|
// If primary URL returned a non-ok status, try fallback.
|
|
60
|
-
if (url ===
|
|
67
|
+
if (url === ANTHROPIC_TOKEN_URL) {
|
|
61
68
|
continue;
|
|
62
69
|
}
|
|
63
70
|
return terminalFailure ?? failure;
|
|
@@ -75,11 +82,11 @@ async function performTokenRefresh(account) {
|
|
|
75
82
|
return { success: true };
|
|
76
83
|
}
|
|
77
84
|
catch (e) {
|
|
78
|
-
logger.warn(`[token-refresh] exception for ${account.label} at ${url}`, {
|
|
85
|
+
logger.warn(`[token-refresh] exception for ${account.label} at ${redactUrlForError(url)}`, {
|
|
79
86
|
error: String(e),
|
|
80
87
|
});
|
|
81
88
|
// If primary URL threw, try fallback
|
|
82
|
-
if (url ===
|
|
89
|
+
if (url === ANTHROPIC_TOKEN_URL) {
|
|
83
90
|
continue;
|
|
84
91
|
}
|
|
85
92
|
return terminalFailure ?? { success: false, error: String(e) };
|
|
@@ -151,6 +158,54 @@ export async function refreshToken(account) {
|
|
|
151
158
|
}
|
|
152
159
|
return shared.result;
|
|
153
160
|
}
|
|
161
|
+
async function reloadPersistedTokenStoreAccount(account, target) {
|
|
162
|
+
if (!target || typeof target === "string" || !("providerKey" in target)) {
|
|
163
|
+
return false;
|
|
164
|
+
}
|
|
165
|
+
// Best-effort, bounded reconciliation: this only adopts a newer persisted
|
|
166
|
+
// credential generation if one exists. A decryption/IO failure — or a hung
|
|
167
|
+
// read — must NOT abort the caller, which can still refresh with the
|
|
168
|
+
// already-loaded token. Swallow the failure and treat it as "no newer
|
|
169
|
+
// persisted generation found" so refresh proceeds instead of the account
|
|
170
|
+
// being spuriously disabled.
|
|
171
|
+
let stored;
|
|
172
|
+
try {
|
|
173
|
+
stored = await withTimeout(tokenStore.peekTokens(target.providerKey), PEEK_TOKENS_TIMEOUT_MS, "[token-refresh] peekTokens reconciliation timed out");
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
logger.debug("[token-refresh] skipping persisted-state reconciliation (peek failed)", { error: err instanceof Error ? err.message : String(err) });
|
|
177
|
+
return false;
|
|
178
|
+
}
|
|
179
|
+
if (!stored ||
|
|
180
|
+
stored.expiresAt <= (account.expiresAt ?? 0) ||
|
|
181
|
+
(stored.accessToken === account.token &&
|
|
182
|
+
stored.refreshToken === account.refreshToken &&
|
|
183
|
+
stored.expiresAt === account.expiresAt)) {
|
|
184
|
+
return false;
|
|
185
|
+
}
|
|
186
|
+
account.token = stored.accessToken;
|
|
187
|
+
account.refreshToken = stored.refreshToken;
|
|
188
|
+
account.expiresAt = stored.expiresAt;
|
|
189
|
+
return true;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Refreshes against the latest persisted credential generation. Queued
|
|
193
|
+
* requests can otherwise reject a rotated refresh token and disable a newer
|
|
194
|
+
* login that was saved while they were waiting.
|
|
195
|
+
*/
|
|
196
|
+
export async function refreshTokenFromLatest(account, target) {
|
|
197
|
+
const reloadedBeforeRefresh = await reloadPersistedTokenStoreAccount(account, target);
|
|
198
|
+
if (reloadedBeforeRefresh && !needsRefresh(account)) {
|
|
199
|
+
return { success: true };
|
|
200
|
+
}
|
|
201
|
+
const result = await refreshToken(account);
|
|
202
|
+
if (result.success) {
|
|
203
|
+
return result;
|
|
204
|
+
}
|
|
205
|
+
return (await reloadPersistedTokenStoreAccount(account, target))
|
|
206
|
+
? { success: true }
|
|
207
|
+
: result;
|
|
208
|
+
}
|
|
154
209
|
export function clearRefreshStateForTests() {
|
|
155
210
|
refreshesInFlight.clear();
|
|
156
211
|
}
|
|
@@ -183,7 +238,7 @@ async function persistLegacyCredentials(credPath, account) {
|
|
|
183
238
|
}
|
|
184
239
|
async function persistTokenStoreAccount(providerKey, account) {
|
|
185
240
|
try {
|
|
186
|
-
const existing = await tokenStore.
|
|
241
|
+
const existing = await withTimeout(tokenStore.peekTokens(providerKey), PEEK_TOKENS_TIMEOUT_MS, "[token-refresh] peekTokens for persistence timed out");
|
|
187
242
|
const merged = {
|
|
188
243
|
accessToken: account.token,
|
|
189
244
|
refreshToken: account.refreshToken ?? existing?.refreshToken,
|
|
@@ -191,7 +246,7 @@ async function persistTokenStoreAccount(providerKey, account) {
|
|
|
191
246
|
tokenType: existing?.tokenType ?? "Bearer",
|
|
192
247
|
...(existing?.scope ? { scope: existing.scope } : {}),
|
|
193
248
|
};
|
|
194
|
-
await tokenStore.saveTokens(providerKey, merged);
|
|
249
|
+
await withTimeout(tokenStore.saveTokens(providerKey, merged), PEEK_TOKENS_TIMEOUT_MS, "[token-refresh] saveTokens timed out");
|
|
195
250
|
}
|
|
196
251
|
catch (err) {
|
|
197
252
|
logger.warn("[token-refresh] Failed to persist TokenStore credentials", {
|
|
@@ -5,34 +5,49 @@
|
|
|
5
5
|
* shared snapshot asynchronously, under a cross-process lock, so overlapping
|
|
6
6
|
* rolling workers cannot overwrite each other's counters.
|
|
7
7
|
*/
|
|
8
|
-
import type { AccountStats, ProxyStats, ProxyStatsPersistenceStatus, ProxyUsageStatsStoreOptions } from "../types/index.js";
|
|
8
|
+
import type { AccountStats, ProxyTerminalErrorDetails, ProxyTerminalErrorJournal, ProxyStats, ProxyStatsPersistenceStatus, ProxyUsageStatsSnapshot, ProxyUsageStatsStoreOptions } from "../types/index.js";
|
|
9
9
|
export declare class ProxyUsageStatsStore {
|
|
10
10
|
private readonly now;
|
|
11
11
|
private readonly flushIntervalMs;
|
|
12
12
|
private readonly lockTimeoutMs;
|
|
13
13
|
private readonly staleLockMs;
|
|
14
14
|
private filePath?;
|
|
15
|
+
private terminalErrorsFilePath?;
|
|
15
16
|
private stats;
|
|
16
17
|
private pending;
|
|
18
|
+
private terminalErrors;
|
|
19
|
+
private pendingTerminalErrors;
|
|
17
20
|
private pendingMutations;
|
|
18
21
|
private inFlightMutations;
|
|
22
|
+
private pendingTerminalErrorMutations;
|
|
23
|
+
private inFlightTerminalErrorMutations;
|
|
19
24
|
private revision;
|
|
25
|
+
private terminalErrorsRevision;
|
|
20
26
|
private flushTimer;
|
|
21
27
|
private readonly flushMutex;
|
|
22
28
|
private lastFlushedAt?;
|
|
23
29
|
private lastReconciledAt?;
|
|
24
30
|
private lastRecoveryAt?;
|
|
25
31
|
private lastError?;
|
|
32
|
+
private terminalErrorsLastFlushedAt?;
|
|
33
|
+
private terminalErrorsLastRecoveryAt?;
|
|
34
|
+
private terminalErrorsLastError?;
|
|
35
|
+
private snapshotVersion;
|
|
26
36
|
constructor(options?: ProxyUsageStatsStoreOptions);
|
|
27
37
|
initialize(filePath?: string): Promise<void>;
|
|
28
38
|
recordAttempt(accountLabel: string, accountType: string): void;
|
|
29
39
|
recordFinalSuccess(accountLabel?: string, accountType?: string): void;
|
|
30
40
|
recordAttemptError(accountLabel: string, accountType: string, status: number, rateLimitKind?: "transient" | "quota"): void;
|
|
31
|
-
recordFinalError(
|
|
41
|
+
recordFinalError(status: number, accountLabel?: string, accountType?: string, details?: ProxyTerminalErrorDetails): void;
|
|
32
42
|
getStats(): ProxyStats;
|
|
33
43
|
getAccountStats(label: string): AccountStats | undefined;
|
|
44
|
+
getTerminalErrors(): ProxyTerminalErrorJournal;
|
|
45
|
+
getUsageSnapshot(): ProxyUsageStatsSnapshot;
|
|
34
46
|
getPersistenceStatus(): ProxyStatsPersistenceStatus;
|
|
35
47
|
flush(): Promise<void>;
|
|
48
|
+
private persistStatsDelta;
|
|
49
|
+
private persistTerminalErrorDelta;
|
|
50
|
+
reconcileUsageSnapshot(): Promise<ProxyUsageStatsSnapshot>;
|
|
36
51
|
reconcile(): Promise<ProxyStats>;
|
|
37
52
|
resetMemory(): void;
|
|
38
53
|
resetForTests(): Promise<void>;
|
|
@@ -41,13 +56,18 @@ export declare class ProxyUsageStatsStore {
|
|
|
41
56
|
private applyFinalSuccess;
|
|
42
57
|
private applyAttemptError;
|
|
43
58
|
private applyFinalError;
|
|
59
|
+
private applyTerminalError;
|
|
44
60
|
private ensureAccount;
|
|
45
61
|
private scheduleFlush;
|
|
46
62
|
private cancelFlushTimer;
|
|
47
63
|
private readSnapshot;
|
|
64
|
+
private readTerminalErrorSnapshot;
|
|
48
65
|
private applySnapshot;
|
|
66
|
+
private applyTerminalErrorSnapshot;
|
|
49
67
|
private recoverCorruptSnapshot;
|
|
68
|
+
private recoverCorruptTerminalErrorSnapshot;
|
|
50
69
|
private quarantineCorruptSnapshot;
|
|
70
|
+
private quarantineCorruptTerminalErrorSnapshot;
|
|
51
71
|
private pruneCorruptSnapshots;
|
|
52
72
|
private describeError;
|
|
53
73
|
}
|
|
@@ -55,10 +75,12 @@ export declare function initUsageStats(filePath: string): Promise<void>;
|
|
|
55
75
|
export declare function recordAttempt(accountLabel: string, accountType: string): void;
|
|
56
76
|
export declare function recordFinalSuccess(accountLabel?: string, accountType?: string): void;
|
|
57
77
|
export declare function recordAttemptError(accountLabel: string, accountType: string, status: number, rateLimitKind?: "transient" | "quota"): void;
|
|
58
|
-
export declare function recordFinalError(
|
|
78
|
+
export declare function recordFinalError(status: number, accountLabel?: string, accountType?: string, details?: ProxyTerminalErrorDetails): void;
|
|
59
79
|
export declare function getStats(): ProxyStats;
|
|
60
80
|
export declare function getReconciledStats(): Promise<ProxyStats>;
|
|
81
|
+
export declare function getReconciledUsageSnapshot(): Promise<ProxyUsageStatsSnapshot>;
|
|
61
82
|
export declare function getAccountStats(label: string): AccountStats | undefined;
|
|
83
|
+
export declare function getTerminalErrors(): ProxyTerminalErrorJournal;
|
|
62
84
|
export declare function getUsageStatsPersistenceStatus(): ProxyStatsPersistenceStatus;
|
|
63
85
|
export declare function flushUsageStats(): Promise<void>;
|
|
64
86
|
/** Reset process-local counters while intentionally preserving durable state. */
|