@juspay/neurolink 10.6.4 → 10.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/analytics/pricing.d.ts +4 -1
- package/dist/analytics/pricing.js +6 -2
- package/dist/browser/neurolink.min.js +406 -406
- package/dist/cli/commands/proxyAnalyze.js +12 -0
- package/dist/core/analytics.js +8 -2
- package/dist/core/baseProvider.js +9 -7
- package/dist/core/modules/GenerationHandler.d.ts +8 -0
- package/dist/core/modules/GenerationHandler.js +28 -38
- package/dist/core/modules/TelemetryHandler.d.ts +3 -9
- package/dist/core/modules/TelemetryHandler.js +17 -12
- package/dist/lib/analytics/pricing.d.ts +4 -1
- package/dist/lib/analytics/pricing.js +6 -2
- package/dist/lib/core/analytics.js +8 -2
- package/dist/lib/core/baseProvider.js +9 -7
- package/dist/lib/core/modules/GenerationHandler.d.ts +8 -0
- package/dist/lib/core/modules/GenerationHandler.js +28 -38
- package/dist/lib/core/modules/TelemetryHandler.d.ts +3 -9
- package/dist/lib/core/modules/TelemetryHandler.js +17 -12
- package/dist/lib/neurolink.js +18 -1
- package/dist/lib/providers/amazonBedrock.js +68 -6
- package/dist/lib/providers/anthropic.js +50 -16
- package/dist/lib/providers/googleAiStudio.js +29 -4
- package/dist/lib/providers/googleNativeGemini3.js +10 -0
- package/dist/lib/providers/googleVertex.js +150 -25
- package/dist/lib/providers/litellm.js +13 -2
- package/dist/lib/providers/openAI.js +13 -2
- package/dist/lib/providers/openaiChatCompletionsBase.js +45 -10
- package/dist/lib/providers/openaiChatCompletionsClient.d.ts +2 -14
- package/dist/lib/providers/openaiChatCompletionsClient.js +20 -1
- package/dist/lib/providers/sagemaker/language-model.js +5 -3
- package/dist/lib/providers/sagemaker/parsers.js +18 -9
- package/dist/lib/providers/sagemaker/streaming.d.ts +9 -0
- package/dist/lib/providers/sagemaker/streaming.js +64 -6
- package/dist/lib/proxy/accountCooldown.js +2 -7
- package/dist/lib/proxy/proxyAnalysis.js +197 -0
- package/dist/lib/proxy/requestLogger.js +11 -0
- package/dist/lib/proxy/routingEvidence.d.ts +6 -0
- package/dist/lib/proxy/routingEvidence.js +33 -0
- package/dist/lib/server/routes/claudeProxyRoutes.d.ts +2 -1
- package/dist/lib/server/routes/claudeProxyRoutes.js +257 -87
- package/dist/lib/types/common.d.ts +3 -0
- package/dist/lib/types/openaiCompatible.d.ts +8 -7
- package/dist/lib/types/providers.d.ts +6 -0
- package/dist/lib/types/proxy.d.ts +97 -2
- package/dist/lib/utils/pricing.js +15 -3
- package/dist/lib/utils/tokenUtils.js +38 -3
- package/dist/neurolink.js +18 -1
- package/dist/providers/amazonBedrock.js +68 -6
- package/dist/providers/anthropic.js +50 -16
- package/dist/providers/googleAiStudio.js +29 -4
- package/dist/providers/googleNativeGemini3.js +10 -0
- package/dist/providers/googleVertex.js +150 -25
- package/dist/providers/litellm.js +13 -2
- package/dist/providers/openAI.js +13 -2
- package/dist/providers/openaiChatCompletionsBase.js +45 -10
- package/dist/providers/openaiChatCompletionsClient.d.ts +2 -14
- package/dist/providers/openaiChatCompletionsClient.js +20 -1
- package/dist/providers/sagemaker/language-model.js +5 -3
- package/dist/providers/sagemaker/parsers.js +18 -9
- package/dist/providers/sagemaker/streaming.d.ts +9 -0
- package/dist/providers/sagemaker/streaming.js +64 -6
- package/dist/proxy/accountCooldown.js +2 -7
- package/dist/proxy/proxyAnalysis.js +197 -0
- package/dist/proxy/requestLogger.js +11 -0
- package/dist/proxy/routingEvidence.d.ts +6 -0
- package/dist/proxy/routingEvidence.js +32 -0
- package/dist/server/routes/claudeProxyRoutes.d.ts +2 -1
- package/dist/server/routes/claudeProxyRoutes.js +257 -87
- package/dist/types/common.d.ts +3 -0
- package/dist/types/openaiCompatible.d.ts +8 -7
- package/dist/types/providers.d.ts +6 -0
- package/dist/types/proxy.d.ts +97 -2
- package/dist/utils/pricing.js +15 -3
- package/dist/utils/tokenUtils.js +38 -3
- package/package.json +2 -1
|
@@ -433,29 +433,101 @@ function accountSortMetrics(accountKey, now, sessionSoftLimit, sessionResetToler
|
|
|
433
433
|
const sessionReset = resetEpochToMs(q?.sessionResetAt, now);
|
|
434
434
|
const sessionTicking = sessionReset !== undefined;
|
|
435
435
|
const weeklyTicking = weeklyReset !== undefined;
|
|
436
|
-
const sessionUsed = sessionTicking ? (q
|
|
437
|
-
const weeklyUsed = weeklyTicking ? (q
|
|
438
|
-
const sessionStatus =
|
|
439
|
-
?
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
436
|
+
const sessionUsed = q ? (sessionTicking ? (q.sessionUsed ?? 0) : 0) : null;
|
|
437
|
+
const weeklyUsed = q ? (weeklyTicking ? (q.weeklyUsed ?? null) : 0) : null;
|
|
438
|
+
const sessionStatus = q
|
|
439
|
+
? sessionTicking
|
|
440
|
+
? (q.sessionStatus ?? "unknown")
|
|
441
|
+
: "allowed"
|
|
442
|
+
: null;
|
|
443
|
+
const weeklyStatus = q
|
|
444
|
+
? weeklyTicking
|
|
445
|
+
? (q.weeklyStatus ?? "unknown")
|
|
446
|
+
: "allowed"
|
|
447
|
+
: null;
|
|
444
448
|
const saturated = sessionStatus === "throttled" ||
|
|
445
|
-
(sessionTicking && sessionUsed >= sessionSoftLimit);
|
|
449
|
+
(sessionTicking && (sessionUsed ?? 0) >= sessionSoftLimit);
|
|
450
|
+
const quotaLastUpdated = q && Number.isFinite(q.lastUpdated) ? q.lastUpdated : null;
|
|
446
451
|
return {
|
|
447
452
|
usable: !coolingActive &&
|
|
448
453
|
weeklyStatus !== "rejected" &&
|
|
449
454
|
sessionStatus !== "rejected",
|
|
450
455
|
saturated,
|
|
451
456
|
hasQuota: !!q,
|
|
457
|
+
quotaLastUpdated,
|
|
458
|
+
quotaAgeMs: quotaLastUpdated === null ? null : Math.max(0, now - quotaLastUpdated),
|
|
459
|
+
coolingActive,
|
|
460
|
+
coolingReason: st?.coolingReason ?? null,
|
|
452
461
|
coolingUntil: st?.coolingUntil ?? 0,
|
|
462
|
+
unifiedStatus: q?.unifiedStatus ?? null,
|
|
463
|
+
overageStatus: q?.overageStatus ?? null,
|
|
464
|
+
sessionStatus,
|
|
465
|
+
sessionUsed,
|
|
453
466
|
sessionResetBucket: sessionTicking
|
|
454
467
|
? Math.floor(sessionReset / sessionResetToleranceMs)
|
|
455
468
|
: Number.POSITIVE_INFINITY,
|
|
456
469
|
sessionReset: sessionReset ?? Number.POSITIVE_INFINITY,
|
|
470
|
+
weeklyStatus,
|
|
457
471
|
weeklyReset: weeklyReset ?? Number.POSITIVE_INFINITY,
|
|
458
472
|
weeklyUsed,
|
|
473
|
+
weeklyUsedForSort: weeklyUsed ?? -1,
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
function compareAccountRoutingFactors(a, b, metricsByKey, primaryKey) {
|
|
477
|
+
const ma = metricsByKey.get(a.key);
|
|
478
|
+
const mb = metricsByKey.get(b.key);
|
|
479
|
+
if (!ma || !mb) {
|
|
480
|
+
return [0, "insertion_order"];
|
|
481
|
+
}
|
|
482
|
+
if (ma.usable !== mb.usable) {
|
|
483
|
+
return [ma.usable ? -1 : 1, "availability"];
|
|
484
|
+
}
|
|
485
|
+
if (!ma.usable && !mb.usable) {
|
|
486
|
+
const au = ma.coolingUntil || Number.POSITIVE_INFINITY;
|
|
487
|
+
const bu = mb.coolingUntil || Number.POSITIVE_INFINITY;
|
|
488
|
+
return [
|
|
489
|
+
au === bu ? 0 : au - bu,
|
|
490
|
+
au === bu ? "insertion_order" : "cooldown_recovery",
|
|
491
|
+
];
|
|
492
|
+
}
|
|
493
|
+
if (ma.hasQuota !== mb.hasQuota) {
|
|
494
|
+
return [ma.hasQuota ? 1 : -1, "quota_probe"];
|
|
495
|
+
}
|
|
496
|
+
if (ma.saturated !== mb.saturated) {
|
|
497
|
+
return [ma.saturated ? 1 : -1, "session_headroom"];
|
|
498
|
+
}
|
|
499
|
+
if (ma.saturated && mb.saturated) {
|
|
500
|
+
if (ma.sessionResetBucket !== mb.sessionResetBucket) {
|
|
501
|
+
return [ma.sessionResetBucket - mb.sessionResetBucket, "session_reset"];
|
|
502
|
+
}
|
|
503
|
+
if (ma.weeklyReset !== mb.weeklyReset) {
|
|
504
|
+
return [ma.weeklyReset - mb.weeklyReset, "weekly_reset"];
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
else {
|
|
508
|
+
if (ma.weeklyReset !== mb.weeklyReset) {
|
|
509
|
+
return [ma.weeklyReset - mb.weeklyReset, "weekly_reset"];
|
|
510
|
+
}
|
|
511
|
+
if (ma.sessionResetBucket !== mb.sessionResetBucket) {
|
|
512
|
+
return [ma.sessionResetBucket - mb.sessionResetBucket, "session_reset"];
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
if (ma.weeklyUsedForSort !== mb.weeklyUsedForSort) {
|
|
516
|
+
return [mb.weeklyUsedForSort - ma.weeklyUsedForSort, "weekly_utilization"];
|
|
517
|
+
}
|
|
518
|
+
if (primaryKey && (a.key === primaryKey) !== (b.key === primaryKey)) {
|
|
519
|
+
return [a.key === primaryKey ? -1 : 1, "configured_primary"];
|
|
520
|
+
}
|
|
521
|
+
return [0, "insertion_order"];
|
|
522
|
+
}
|
|
523
|
+
function orderAccountsByQuotaWithMetrics(accounts, now, primaryKey, sessionSoftLimit, sessionResetToleranceMs) {
|
|
524
|
+
const metricsByKey = new Map(accounts.map((account) => [
|
|
525
|
+
account.key,
|
|
526
|
+
accountSortMetrics(account.key, now, sessionSoftLimit, sessionResetToleranceMs),
|
|
527
|
+
]));
|
|
528
|
+
return {
|
|
529
|
+
orderedAccounts: [...accounts].sort((a, b) => compareAccountRoutingFactors(a, b, metricsByKey, primaryKey)[0]),
|
|
530
|
+
metricsByKey,
|
|
459
531
|
};
|
|
460
532
|
}
|
|
461
533
|
/**
|
|
@@ -484,48 +556,148 @@ function accountSortMetrics(accountKey, now, sessionSoftLimit, sessionResetToler
|
|
|
484
556
|
* last resort.
|
|
485
557
|
*/
|
|
486
558
|
function orderAccountsByQuota(accounts, now, primaryKey, sessionSoftLimit = getSessionSoftLimit(), sessionResetToleranceMs = getSessionResetToleranceMs()) {
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
return au - bu;
|
|
498
|
-
}
|
|
499
|
-
if (ma.hasQuota !== mb.hasQuota) {
|
|
500
|
-
return ma.hasQuota ? 1 : -1;
|
|
559
|
+
return orderAccountsByQuotaWithMetrics(accounts, now, primaryKey, sessionSoftLimit, sessionResetToleranceMs).orderedAccounts;
|
|
560
|
+
}
|
|
561
|
+
function buildRoutingDecision(args) {
|
|
562
|
+
const { accounts, orderedAccounts, metricsByKey, evaluatedAt, strategy, primaryKey, quotaRoutingEnabled, quotaOrdered, sessionSoftLimit, sessionResetToleranceMs, rotationOffset, } = args;
|
|
563
|
+
const sourceIndexes = new Map(accounts.map((account, index) => [account.key, index]));
|
|
564
|
+
const configuredPrimaryMatched = !!primaryKey && accounts.some((account) => account.key === primaryKey);
|
|
565
|
+
const candidates = orderedAccounts.map((account, rank) => {
|
|
566
|
+
const metrics = metricsByKey.get(account.key);
|
|
567
|
+
if (!metrics) {
|
|
568
|
+
throw new Error(`Missing precomputed routing metrics for account ${account.label}`);
|
|
501
569
|
}
|
|
502
|
-
|
|
503
|
-
|
|
570
|
+
return {
|
|
571
|
+
account: account.label,
|
|
572
|
+
accountType: account.type,
|
|
573
|
+
sourceIndex: sourceIndexes.get(account.key) ?? rank,
|
|
574
|
+
rank,
|
|
575
|
+
configuredPrimary: !!primaryKey && account.key === primaryKey,
|
|
576
|
+
usable: metrics.usable,
|
|
577
|
+
saturated: metrics.saturated,
|
|
578
|
+
quotaObserved: metrics.hasQuota,
|
|
579
|
+
quotaLastUpdated: metrics.quotaLastUpdated,
|
|
580
|
+
quotaAgeMs: metrics.quotaAgeMs,
|
|
581
|
+
coolingActive: metrics.coolingActive,
|
|
582
|
+
coolingReason: metrics.coolingReason,
|
|
583
|
+
coolingUntil: metrics.coolingUntil > 0 && Number.isFinite(metrics.coolingUntil)
|
|
584
|
+
? metrics.coolingUntil
|
|
585
|
+
: null,
|
|
586
|
+
unifiedStatus: metrics.unifiedStatus,
|
|
587
|
+
overageStatus: metrics.overageStatus,
|
|
588
|
+
sessionStatus: metrics.sessionStatus,
|
|
589
|
+
sessionUsed: metrics.sessionUsed,
|
|
590
|
+
sessionResetAt: Number.isFinite(metrics.sessionReset)
|
|
591
|
+
? metrics.sessionReset
|
|
592
|
+
: null,
|
|
593
|
+
sessionResetBucket: Number.isFinite(metrics.sessionResetBucket)
|
|
594
|
+
? metrics.sessionResetBucket
|
|
595
|
+
: null,
|
|
596
|
+
weeklyStatus: metrics.weeklyStatus,
|
|
597
|
+
weeklyUsed: metrics.weeklyUsed,
|
|
598
|
+
weeklyResetAt: Number.isFinite(metrics.weeklyReset)
|
|
599
|
+
? metrics.weeklyReset
|
|
600
|
+
: null,
|
|
601
|
+
};
|
|
602
|
+
});
|
|
603
|
+
const initialAccount = orderedAccounts[0];
|
|
604
|
+
let mode;
|
|
605
|
+
let selectionReason;
|
|
606
|
+
if (orderedAccounts.length === 1) {
|
|
607
|
+
mode = "single_account";
|
|
608
|
+
selectionReason = "single_account";
|
|
609
|
+
}
|
|
610
|
+
else if (quotaOrdered) {
|
|
611
|
+
mode = "quota";
|
|
612
|
+
selectionReason = compareAccountRoutingFactors(orderedAccounts[0], orderedAccounts[1], metricsByKey, primaryKey)[1];
|
|
613
|
+
}
|
|
614
|
+
else if (strategy === "round-robin") {
|
|
615
|
+
mode = "round_robin";
|
|
616
|
+
selectionReason = "round_robin";
|
|
617
|
+
}
|
|
618
|
+
else {
|
|
619
|
+
mode = "primary";
|
|
620
|
+
selectionReason =
|
|
621
|
+
configuredPrimaryMatched && initialAccount?.key === primaryKey
|
|
622
|
+
? "configured_primary"
|
|
623
|
+
: "insertion_order";
|
|
624
|
+
}
|
|
625
|
+
return {
|
|
626
|
+
schemaVersion: 1,
|
|
627
|
+
evaluatedAt: new Date(evaluatedAt).toISOString(),
|
|
628
|
+
strategy,
|
|
629
|
+
mode,
|
|
630
|
+
selectionReason,
|
|
631
|
+
quotaRoutingEnabled,
|
|
632
|
+
quotaInputsUsed: quotaOrdered,
|
|
633
|
+
sessionSoftLimit,
|
|
634
|
+
sessionResetToleranceMs,
|
|
635
|
+
configuredPrimaryAccount: primaryKey ?? null,
|
|
636
|
+
configuredPrimaryMatched,
|
|
637
|
+
rotationOffset,
|
|
638
|
+
initialAccount: initialAccount?.label ?? "",
|
|
639
|
+
candidates,
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
function selectClaudeProxyAccountOrder(args) {
|
|
643
|
+
const { enabledAccounts, accountStrategy, primaryAccountKey, quotaRoutingEnabled, sessionSoftLimit, sessionResetToleranceMs, setRoutingDecision, } = args;
|
|
644
|
+
let orderedAccounts = [...enabledAccounts];
|
|
645
|
+
const evaluatedAt = Date.now();
|
|
646
|
+
let metricsByKey;
|
|
647
|
+
let rotationOffset = 0;
|
|
648
|
+
const quotaOrdered = accountStrategy === "fill-first" &&
|
|
649
|
+
orderedAccounts.length > 1 &&
|
|
650
|
+
quotaRoutingEnabled;
|
|
651
|
+
if (!quotaOrdered && accountStrategy === "fill-first") {
|
|
652
|
+
// A hot-reloaded primary change must apply to this request.
|
|
653
|
+
maybeResetPrimaryToHome(enabledAccounts, primaryAccountKey);
|
|
654
|
+
}
|
|
655
|
+
if (quotaOrdered) {
|
|
656
|
+
const quotaOrder = orderAccountsByQuotaWithMetrics(enabledAccounts, evaluatedAt, primaryAccountKey, sessionSoftLimit, sessionResetToleranceMs);
|
|
657
|
+
orderedAccounts = quotaOrder.orderedAccounts;
|
|
658
|
+
metricsByKey = quotaOrder.metricsByKey;
|
|
659
|
+
if (logger.shouldLog("debug")) {
|
|
660
|
+
logger.debug(`[proxy] quota-ordered fill sequence: ${orderedAccounts
|
|
661
|
+
.map((account) => account.label)
|
|
662
|
+
.join(" → ")}`);
|
|
504
663
|
}
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
}
|
|
664
|
+
}
|
|
665
|
+
else {
|
|
666
|
+
if (accountStrategy === "round-robin" &&
|
|
667
|
+
orderedAccounts.length !== lastKnownAccountCount) {
|
|
668
|
+
primaryAccountIndex = resolveHomeIndex(orderedAccounts, primaryAccountKey);
|
|
669
|
+
lastKnownAccountCount = orderedAccounts.length;
|
|
512
670
|
}
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
671
|
+
if (orderedAccounts.length > 1) {
|
|
672
|
+
rotationOffset = primaryAccountIndex % orderedAccounts.length;
|
|
673
|
+
if (accountStrategy === "round-robin") {
|
|
674
|
+
primaryAccountIndex =
|
|
675
|
+
(primaryAccountIndex + 1) % orderedAccounts.length;
|
|
516
676
|
}
|
|
517
|
-
if (
|
|
518
|
-
|
|
677
|
+
if (rotationOffset > 0) {
|
|
678
|
+
const head = orderedAccounts.splice(0, rotationOffset);
|
|
679
|
+
orderedAccounts.push(...head);
|
|
519
680
|
}
|
|
520
681
|
}
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
682
|
+
metricsByKey = new Map(enabledAccounts.map((account) => [
|
|
683
|
+
account.key,
|
|
684
|
+
accountSortMetrics(account.key, evaluatedAt, sessionSoftLimit, sessionResetToleranceMs),
|
|
685
|
+
]));
|
|
686
|
+
}
|
|
687
|
+
setRoutingDecision(buildRoutingDecision({
|
|
688
|
+
accounts: enabledAccounts,
|
|
689
|
+
orderedAccounts,
|
|
690
|
+
metricsByKey,
|
|
691
|
+
evaluatedAt,
|
|
692
|
+
strategy: accountStrategy,
|
|
693
|
+
primaryKey: primaryAccountKey,
|
|
694
|
+
quotaRoutingEnabled,
|
|
695
|
+
quotaOrdered,
|
|
696
|
+
sessionSoftLimit,
|
|
697
|
+
sessionResetToleranceMs,
|
|
698
|
+
rotationOffset,
|
|
699
|
+
}));
|
|
700
|
+
return orderedAccounts;
|
|
529
701
|
}
|
|
530
702
|
// ---------------------------------------------------------------------------
|
|
531
703
|
// OAuth polyfill helpers (extracted to reduce block nesting)
|
|
@@ -1447,7 +1619,7 @@ async function handleClaudePassthroughJsonResponse(args) {
|
|
|
1447
1619
|
return responseJson;
|
|
1448
1620
|
}
|
|
1449
1621
|
async function loadClaudeProxyAccounts(args) {
|
|
1450
|
-
const { ctx, body, tracer, requestStartTime, accountStrategy, primaryAccountKey, accountAllowlist, quotaRoutingEnabled = isQuotaRoutingEnabled(), sessionSoftLimit = getSessionSoftLimit(), sessionResetToleranceMs = getSessionResetToleranceMs(), buildLoggedClaudeError, } = args;
|
|
1622
|
+
const { ctx, body, tracer, requestStartTime, accountStrategy, primaryAccountKey, accountAllowlist, quotaRoutingEnabled = isQuotaRoutingEnabled(), sessionSoftLimit = getSessionSoftLimit(), sessionResetToleranceMs = getSessionResetToleranceMs(), buildLoggedClaudeError, setRoutingDecision, } = args;
|
|
1451
1623
|
const fs = await import("fs");
|
|
1452
1624
|
const os = await import("os");
|
|
1453
1625
|
const accounts = [];
|
|
@@ -1616,45 +1788,15 @@ async function loadClaudeProxyAccounts(args) {
|
|
|
1616
1788
|
tracer?.end(401, Date.now() - requestStartTime);
|
|
1617
1789
|
return { response: buildLoggedClaudeError(401, reauthMsg) };
|
|
1618
1790
|
}
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
}
|
|
1629
|
-
if (quotaOrdered) {
|
|
1630
|
-
// Fill-first with a smart fill order: spend the account whose weekly window
|
|
1631
|
-
// expires soonest while temporarily demoting sessions without headroom.
|
|
1632
|
-
// Supersedes the static home/primary index.
|
|
1633
|
-
orderedAccounts = orderAccountsByQuota(enabledAccounts, Date.now(), primaryAccountKey, sessionSoftLimit, sessionResetToleranceMs);
|
|
1634
|
-
if (logger.shouldLog("debug")) {
|
|
1635
|
-
logger.debug(`[proxy] quota-ordered fill sequence: ${orderedAccounts
|
|
1636
|
-
.map((a) => a.label)
|
|
1637
|
-
.join(" → ")}`);
|
|
1638
|
-
}
|
|
1639
|
-
}
|
|
1640
|
-
else {
|
|
1641
|
-
if (accountStrategy === "round-robin" &&
|
|
1642
|
-
orderedAccounts.length !== lastKnownAccountCount) {
|
|
1643
|
-
primaryAccountIndex = resolveHomeIndex(orderedAccounts, primaryAccountKey);
|
|
1644
|
-
lastKnownAccountCount = orderedAccounts.length;
|
|
1645
|
-
}
|
|
1646
|
-
if (orderedAccounts.length > 1) {
|
|
1647
|
-
const idx = primaryAccountIndex % orderedAccounts.length;
|
|
1648
|
-
if (accountStrategy === "round-robin") {
|
|
1649
|
-
primaryAccountIndex =
|
|
1650
|
-
(primaryAccountIndex + 1) % orderedAccounts.length;
|
|
1651
|
-
}
|
|
1652
|
-
if (idx > 0) {
|
|
1653
|
-
const head = orderedAccounts.splice(0, idx);
|
|
1654
|
-
orderedAccounts.push(...head);
|
|
1655
|
-
}
|
|
1656
|
-
}
|
|
1657
|
-
}
|
|
1791
|
+
const orderedAccounts = selectClaudeProxyAccountOrder({
|
|
1792
|
+
enabledAccounts,
|
|
1793
|
+
accountStrategy,
|
|
1794
|
+
primaryAccountKey,
|
|
1795
|
+
quotaRoutingEnabled,
|
|
1796
|
+
sessionSoftLimit,
|
|
1797
|
+
sessionResetToleranceMs,
|
|
1798
|
+
setRoutingDecision,
|
|
1799
|
+
});
|
|
1658
1800
|
const normalizedAnthropicBody = normalizeClaudeRequestForAnthropic(body);
|
|
1659
1801
|
const bodyStr = JSON.stringify(normalizedAnthropicBody);
|
|
1660
1802
|
const requestStart = Date.now();
|
|
@@ -3345,6 +3487,14 @@ function createClaudeRequestRuntimeContext(args) {
|
|
|
3345
3487
|
: {}),
|
|
3346
3488
|
});
|
|
3347
3489
|
};
|
|
3490
|
+
let routingDecision;
|
|
3491
|
+
const setRoutingDecision = (decision) => {
|
|
3492
|
+
if (routingDecision) {
|
|
3493
|
+
logger.debug(`[claude-proxy] ignored duplicate routing decision for request ${ctx.requestId}`);
|
|
3494
|
+
return;
|
|
3495
|
+
}
|
|
3496
|
+
routingDecision = decision;
|
|
3497
|
+
};
|
|
3348
3498
|
let finalRequestLogged = false;
|
|
3349
3499
|
const logFinalRequest = (status, accountLabel, accountType, errorType, errorMessage, extra) => {
|
|
3350
3500
|
if (finalRequestLogged) {
|
|
@@ -3404,6 +3554,7 @@ function createClaudeRequestRuntimeContext(args) {
|
|
|
3404
3554
|
...(traceCtx
|
|
3405
3555
|
? { traceId: traceCtx.traceId, spanId: traceCtx.spanId }
|
|
3406
3556
|
: {}),
|
|
3557
|
+
...(routingDecision ? { routingDecision } : {}),
|
|
3407
3558
|
});
|
|
3408
3559
|
};
|
|
3409
3560
|
const buildLoggedClaudeError = (status, message, errorType, extra) => {
|
|
@@ -3435,6 +3586,7 @@ function createClaudeRequestRuntimeContext(args) {
|
|
|
3435
3586
|
logProxyBody,
|
|
3436
3587
|
logFinalRequest,
|
|
3437
3588
|
buildLoggedClaudeError,
|
|
3589
|
+
setRoutingDecision,
|
|
3438
3590
|
};
|
|
3439
3591
|
}
|
|
3440
3592
|
function createAnthropicAttemptLogger(args) {
|
|
@@ -3803,7 +3955,7 @@ function shouldAttemptClaudeFallback(loopState) {
|
|
|
3803
3955
|
return loopState.invalidRequestFailure === null;
|
|
3804
3956
|
}
|
|
3805
3957
|
async function handleAnthropicRoutedClaudeRequest(args) {
|
|
3806
|
-
const { ctx, body, modelRouter, tracer, requestStartTime, accountStrategy, primaryAccountKey, accountAllowlist, quotaRoutingEnabled = isQuotaRoutingEnabled(), sessionSoftLimit = getSessionSoftLimit(), sessionResetToleranceMs = getSessionResetToleranceMs(), buildLoggedClaudeError, logProxyBody, logFinalRequest, } = args;
|
|
3958
|
+
const { ctx, body, modelRouter, tracer, requestStartTime, accountStrategy, primaryAccountKey, accountAllowlist, quotaRoutingEnabled = isQuotaRoutingEnabled(), sessionSoftLimit = getSessionSoftLimit(), sessionResetToleranceMs = getSessionResetToleranceMs(), buildLoggedClaudeError, logProxyBody, logFinalRequest, setRoutingDecision, } = args;
|
|
3807
3959
|
const parsedRequest = parseClaudeRequest(body);
|
|
3808
3960
|
const loadedAccounts = await loadClaudeProxyAccounts({
|
|
3809
3961
|
ctx,
|
|
@@ -3817,6 +3969,7 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
3817
3969
|
sessionSoftLimit,
|
|
3818
3970
|
sessionResetToleranceMs,
|
|
3819
3971
|
buildLoggedClaudeError,
|
|
3972
|
+
setRoutingDecision,
|
|
3820
3973
|
});
|
|
3821
3974
|
if ("response" in loadedAccounts) {
|
|
3822
3975
|
return loadedAccounts.response;
|
|
@@ -4293,7 +4446,7 @@ export function createClaudeProxyRoutes(modelRouter, basePath = "", accountStrat
|
|
|
4293
4446
|
};
|
|
4294
4447
|
const clientRequestBody = JSON.stringify(body);
|
|
4295
4448
|
// 3. Create request runtime context (tracer, loggers, error builder)
|
|
4296
|
-
const { tracer, requestStartTime, logProxyBody, logFinalRequest, buildLoggedClaudeError, } = createClaudeRequestRuntimeContext({
|
|
4449
|
+
const { tracer, requestStartTime, logProxyBody, logFinalRequest, buildLoggedClaudeError, setRoutingDecision, } = createClaudeRequestRuntimeContext({
|
|
4297
4450
|
ctx,
|
|
4298
4451
|
body,
|
|
4299
4452
|
clientRequestBody,
|
|
@@ -4333,6 +4486,7 @@ export function createClaudeProxyRoutes(modelRouter, basePath = "", accountStrat
|
|
|
4333
4486
|
buildLoggedClaudeError,
|
|
4334
4487
|
logProxyBody,
|
|
4335
4488
|
logFinalRequest,
|
|
4489
|
+
setRoutingDecision,
|
|
4336
4490
|
});
|
|
4337
4491
|
}
|
|
4338
4492
|
else {
|
|
@@ -4726,6 +4880,22 @@ export const __testHooks = {
|
|
|
4726
4880
|
getStreamFailureDetails,
|
|
4727
4881
|
trackUpstreamReadableStream,
|
|
4728
4882
|
orderAccountsByQuota,
|
|
4883
|
+
buildQuotaRoutingDecision: (accounts, now, primaryKey, sessionSoftLimit = getSessionSoftLimit(), sessionResetToleranceMs = getSessionResetToleranceMs()) => {
|
|
4884
|
+
const order = orderAccountsByQuotaWithMetrics(accounts, now, primaryKey, sessionSoftLimit, sessionResetToleranceMs);
|
|
4885
|
+
return buildRoutingDecision({
|
|
4886
|
+
accounts,
|
|
4887
|
+
orderedAccounts: order.orderedAccounts,
|
|
4888
|
+
metricsByKey: order.metricsByKey,
|
|
4889
|
+
evaluatedAt: now,
|
|
4890
|
+
strategy: "fill-first",
|
|
4891
|
+
primaryKey,
|
|
4892
|
+
quotaRoutingEnabled: true,
|
|
4893
|
+
quotaOrdered: accounts.length > 1,
|
|
4894
|
+
sessionSoftLimit,
|
|
4895
|
+
sessionResetToleranceMs,
|
|
4896
|
+
rotationOffset: 0,
|
|
4897
|
+
});
|
|
4898
|
+
},
|
|
4729
4899
|
resetEpochToMs,
|
|
4730
4900
|
seedRuntimeQuotasFromDisk,
|
|
4731
4901
|
reconcileEligibleAccountRuntimeState,
|
package/dist/types/common.d.ts
CHANGED
|
@@ -225,6 +225,7 @@ export type RawUsageObject = {
|
|
|
225
225
|
cacheReadTokens?: number;
|
|
226
226
|
cachedInputTokens?: number;
|
|
227
227
|
inputTokenDetails?: {
|
|
228
|
+
noCacheTokens?: number;
|
|
228
229
|
cacheReadTokens?: number;
|
|
229
230
|
cacheWriteTokens?: number;
|
|
230
231
|
};
|
|
@@ -248,6 +249,8 @@ export type DeferredUsage = {
|
|
|
248
249
|
totalTokens: number;
|
|
249
250
|
cacheReadTokens?: number;
|
|
250
251
|
cacheCreationTokens?: number;
|
|
252
|
+
/** Reasoning/thinking tokens — a SUBSET already included in completionTokens. */
|
|
253
|
+
reasoningTokens?: number;
|
|
251
254
|
};
|
|
252
255
|
/**
|
|
253
256
|
* Options for token extraction from raw usage objects.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { NeuroLinkEvents, TypedEventEmitter } from "./common.js";
|
|
1
|
+
import type { DeferredUsage, NeuroLinkEvents, TypedEventEmitter } from "./common.js";
|
|
2
2
|
import type { JSONSchema7 } from "./middleware.js";
|
|
3
3
|
import type { LanguageModelV3, LanguageModelV3CallOptions } from "./middleware.js";
|
|
4
4
|
import type { StreamOptions } from "./stream.js";
|
|
@@ -271,12 +271,13 @@ export type OpenAICompatBuildBodyArgs = {
|
|
|
271
271
|
* the deferred analytics promises.
|
|
272
272
|
*/
|
|
273
273
|
export type OpenAICompatStreamLifecycleListeners = {
|
|
274
|
-
/**
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
274
|
+
/**
|
|
275
|
+
* Fired once the deferred usage promise resolves with the final aggregated
|
|
276
|
+
* token counts. promptTokens is the UNCACHED remainder; cacheReadTokens
|
|
277
|
+
* carries the cached portion (non-overlapping convention), and
|
|
278
|
+
* reasoningTokens is a subset of completionTokens.
|
|
279
|
+
*/
|
|
280
|
+
onUsage?: (usage: DeferredUsage) => void;
|
|
280
281
|
/**
|
|
281
282
|
* Fired once the deferred finish promise resolves. `reason` is "stop",
|
|
282
283
|
* "length", "tool-calls", "content-filter", or "error". When the loop
|
|
@@ -1623,6 +1623,12 @@ export type CollectedChunkResult = {
|
|
|
1623
1623
|
cacheReadTokens?: number;
|
|
1624
1624
|
/** Cache creation tokens (symmetry; Gemini does not emit this). */
|
|
1625
1625
|
cacheCreationTokens?: number;
|
|
1626
|
+
/**
|
|
1627
|
+
* Gemini thinking tokens (usageMetadata.thoughtsTokenCount). Billed at the
|
|
1628
|
+
* output rate but NOT included in candidatesTokenCount — Gemini reports
|
|
1629
|
+
* totalTokenCount = prompt + candidates + thoughts.
|
|
1630
|
+
*/
|
|
1631
|
+
reasoningTokens?: number;
|
|
1626
1632
|
};
|
|
1627
1633
|
/** Push-based text channel for incremental streaming. */
|
|
1628
1634
|
export type TextChannel = {
|
package/dist/types/proxy.d.ts
CHANGED
|
@@ -18,6 +18,7 @@ import type { Hono } from "hono";
|
|
|
18
18
|
import type { Ora } from "ora";
|
|
19
19
|
import type { MCPToolRegistry } from "../mcp/toolRegistry.js";
|
|
20
20
|
import type { ProxyTracer } from "../proxy/proxyTracer.js";
|
|
21
|
+
import type { ACCOUNT_COOLING_REASONS, PROXY_ACCOUNT_TYPES, PROXY_ACCOUNT_ROUTING_MODES, PROXY_ACCOUNT_ROUTING_REASONS, PROXY_ACCOUNT_ROUTING_STRATEGIES } from "../proxy/routingEvidence.js";
|
|
21
22
|
import type { FallbackEntry, ModelMapping, ProxyRoutingConfig, CloakingConfig } from "./subscription.js";
|
|
22
23
|
import type { RouteDeprecation } from "./server.js";
|
|
23
24
|
/**
|
|
@@ -423,6 +424,70 @@ export type LoadProxyConfigOptions = {
|
|
|
423
424
|
/** Custom environment object (defaults to process.env) */
|
|
424
425
|
env?: Record<string, string | undefined>;
|
|
425
426
|
};
|
|
427
|
+
export type ProxyAccountRoutingStrategy = (typeof PROXY_ACCOUNT_ROUTING_STRATEGIES)[number];
|
|
428
|
+
export type ProxyAccountRoutingMode = (typeof PROXY_ACCOUNT_ROUTING_MODES)[number];
|
|
429
|
+
export type ProxyAccountRoutingReason = (typeof PROXY_ACCOUNT_ROUTING_REASONS)[number];
|
|
430
|
+
export type ProxyAccountType = (typeof PROXY_ACCOUNT_TYPES)[number];
|
|
431
|
+
export type ProxyAccountRoutingCandidate = {
|
|
432
|
+
account: string;
|
|
433
|
+
accountType: ProxyAccountType;
|
|
434
|
+
sourceIndex: number;
|
|
435
|
+
rank: number;
|
|
436
|
+
configuredPrimary: boolean;
|
|
437
|
+
usable: boolean;
|
|
438
|
+
saturated: boolean;
|
|
439
|
+
quotaObserved: boolean;
|
|
440
|
+
quotaLastUpdated: number | null;
|
|
441
|
+
quotaAgeMs: number | null;
|
|
442
|
+
coolingActive: boolean;
|
|
443
|
+
coolingReason: AccountCoolingReason | null;
|
|
444
|
+
coolingUntil: number | null;
|
|
445
|
+
unifiedStatus: string | null;
|
|
446
|
+
overageStatus: string | null;
|
|
447
|
+
sessionStatus: string | null;
|
|
448
|
+
sessionUsed: number | null;
|
|
449
|
+
sessionResetAt: number | null;
|
|
450
|
+
sessionResetBucket: number | null;
|
|
451
|
+
weeklyStatus: string | null;
|
|
452
|
+
weeklyUsed: number | null;
|
|
453
|
+
weeklyResetAt: number | null;
|
|
454
|
+
};
|
|
455
|
+
export type ProxyAccountRoutingDecision = {
|
|
456
|
+
schemaVersion: 1;
|
|
457
|
+
evaluatedAt: string;
|
|
458
|
+
strategy: ProxyAccountRoutingStrategy;
|
|
459
|
+
mode: ProxyAccountRoutingMode;
|
|
460
|
+
selectionReason: ProxyAccountRoutingReason;
|
|
461
|
+
quotaRoutingEnabled: boolean;
|
|
462
|
+
quotaInputsUsed: boolean;
|
|
463
|
+
sessionSoftLimit: number;
|
|
464
|
+
sessionResetToleranceMs: number;
|
|
465
|
+
configuredPrimaryAccount: string | null;
|
|
466
|
+
configuredPrimaryMatched: boolean;
|
|
467
|
+
rotationOffset: number;
|
|
468
|
+
initialAccount: string;
|
|
469
|
+
candidates: ProxyAccountRoutingCandidate[];
|
|
470
|
+
};
|
|
471
|
+
export type ProxyAccountSortMetrics = {
|
|
472
|
+
usable: boolean;
|
|
473
|
+
saturated: boolean;
|
|
474
|
+
hasQuota: boolean;
|
|
475
|
+
quotaLastUpdated: number | null;
|
|
476
|
+
quotaAgeMs: number | null;
|
|
477
|
+
coolingActive: boolean;
|
|
478
|
+
coolingReason: AccountCoolingReason | null;
|
|
479
|
+
coolingUntil: number;
|
|
480
|
+
unifiedStatus: string | null;
|
|
481
|
+
overageStatus: string | null;
|
|
482
|
+
sessionStatus: string | null;
|
|
483
|
+
sessionUsed: number | null;
|
|
484
|
+
sessionResetBucket: number;
|
|
485
|
+
sessionReset: number;
|
|
486
|
+
weeklyStatus: string | null;
|
|
487
|
+
weeklyReset: number;
|
|
488
|
+
weeklyUsed: number | null;
|
|
489
|
+
weeklyUsedForSort: number;
|
|
490
|
+
};
|
|
426
491
|
export type RequestLogEntry = {
|
|
427
492
|
timestamp: string;
|
|
428
493
|
requestId: string;
|
|
@@ -447,6 +512,8 @@ export type RequestLogEntry = {
|
|
|
447
512
|
traceId?: string;
|
|
448
513
|
/** OTel span ID for correlation with distributed traces */
|
|
449
514
|
spanId?: string;
|
|
515
|
+
/** Exact secret-free inputs and result of initial account selection. */
|
|
516
|
+
routingDecision?: ProxyAccountRoutingDecision;
|
|
450
517
|
};
|
|
451
518
|
export type RequestAttemptLogEntry = {
|
|
452
519
|
timestamp: string;
|
|
@@ -515,6 +582,9 @@ export type ClaudeRequestRuntimeContext = {
|
|
|
515
582
|
logFinalRequest: ClaudeFinalRequestLogger;
|
|
516
583
|
buildLoggedClaudeError: ClaudeLoggedErrorBuilder;
|
|
517
584
|
};
|
|
585
|
+
export type RoutedClaudeRequestRuntimeContext = ClaudeRequestRuntimeContext & {
|
|
586
|
+
setRoutingDecision: (decision: ProxyAccountRoutingDecision) => void;
|
|
587
|
+
};
|
|
518
588
|
export type AnthropicAttemptLogger = (status: number, errorType?: string, errorMessage?: string, extra?: {
|
|
519
589
|
inputTokens?: number;
|
|
520
590
|
outputTokens?: number;
|
|
@@ -838,7 +908,7 @@ export type AccountQuota = {
|
|
|
838
908
|
* - "unified" : top-level unified limit rejected — cool for retry-after.
|
|
839
909
|
* - "transient" : short per-minute/burst 429 — cool for retry-after only.
|
|
840
910
|
* - "auth" : transient refresh failure with bounded backoff. */
|
|
841
|
-
export type AccountCoolingReason =
|
|
911
|
+
export type AccountCoolingReason = (typeof ACCOUNT_COOLING_REASONS)[number];
|
|
842
912
|
/** Restart-safe cooldown snapshot for one account. */
|
|
843
913
|
export type PersistedAccountCooldown = {
|
|
844
914
|
coolingUntil: number;
|
|
@@ -872,7 +942,7 @@ export type ProxyPassthroughAccount = {
|
|
|
872
942
|
token: string;
|
|
873
943
|
refreshToken?: string;
|
|
874
944
|
expiresAt?: number;
|
|
875
|
-
type:
|
|
945
|
+
type: ProxyAccountType;
|
|
876
946
|
persistTarget?: TokenPersistTarget;
|
|
877
947
|
};
|
|
878
948
|
/** Dependencies for creating Claude proxy routes. */
|
|
@@ -1190,6 +1260,7 @@ export type ProxyAnalysisReport = {
|
|
|
1190
1260
|
attempts: boolean;
|
|
1191
1261
|
attemptLatency: boolean;
|
|
1192
1262
|
cacheUsage: boolean;
|
|
1263
|
+
routingDecisions: boolean;
|
|
1193
1264
|
};
|
|
1194
1265
|
dataQuality: {
|
|
1195
1266
|
linesRead: number;
|
|
@@ -1211,6 +1282,11 @@ export type ProxyAnalysisReport = {
|
|
|
1211
1282
|
writeFailures: number;
|
|
1212
1283
|
truncatedCaptures: number;
|
|
1213
1284
|
};
|
|
1285
|
+
routingDecisions: {
|
|
1286
|
+
valid: number;
|
|
1287
|
+
invalid: number;
|
|
1288
|
+
absent: number;
|
|
1289
|
+
};
|
|
1214
1290
|
};
|
|
1215
1291
|
lifecycle: {
|
|
1216
1292
|
accepted: number;
|
|
@@ -1259,6 +1335,14 @@ export type ProxyAnalysisReport = {
|
|
|
1259
1335
|
inputTokens: number;
|
|
1260
1336
|
requestHitRate: number | null;
|
|
1261
1337
|
};
|
|
1338
|
+
routing: {
|
|
1339
|
+
modes: Record<string, number>;
|
|
1340
|
+
selectionReasons: Record<string, number>;
|
|
1341
|
+
initialAccounts: Record<string, number>;
|
|
1342
|
+
finalAccountChanges: number;
|
|
1343
|
+
finalOutsideCandidateSet: number;
|
|
1344
|
+
records: ProxyAnalysisRoutingRecord[];
|
|
1345
|
+
};
|
|
1262
1346
|
accounts: ProxyAnalysisAccount[];
|
|
1263
1347
|
};
|
|
1264
1348
|
/** Inputs for the offline proxy JSONL analyzer. */
|
|
@@ -1276,6 +1360,7 @@ export type ProxyAnalysisAttemptRecord = {
|
|
|
1276
1360
|
};
|
|
1277
1361
|
/** Final request fields retained while joining offline proxy log records. */
|
|
1278
1362
|
export type ProxyAnalysisFinalRequestRecord = {
|
|
1363
|
+
timestamp: string;
|
|
1279
1364
|
status: number;
|
|
1280
1365
|
durationMs: number | null;
|
|
1281
1366
|
account: string;
|
|
@@ -1285,6 +1370,16 @@ export type ProxyAnalysisFinalRequestRecord = {
|
|
|
1285
1370
|
cacheCreationTokens: number | null;
|
|
1286
1371
|
errorType: string | null;
|
|
1287
1372
|
errorCode: string | null;
|
|
1373
|
+
routingDecision: ProxyAccountRoutingDecision | null;
|
|
1374
|
+
};
|
|
1375
|
+
/** Validated account-routing evidence joined to a final request log. */
|
|
1376
|
+
export type ProxyAnalysisRoutingRecord = {
|
|
1377
|
+
requestId: string;
|
|
1378
|
+
timestamp: string;
|
|
1379
|
+
responseStatus: number;
|
|
1380
|
+
finalAccount: string;
|
|
1381
|
+
finalAccountType: string;
|
|
1382
|
+
decision: ProxyAccountRoutingDecision;
|
|
1288
1383
|
};
|
|
1289
1384
|
/** Request metadata retained by the HTTP adapter for terminal error logging. */
|
|
1290
1385
|
export type RuntimeRequestMetadata = {
|