@juspay/neurolink 10.9.0 → 10.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/browser/neurolink.min.js +379 -379
- package/dist/cli/commands/proxy.js +80 -20
- package/dist/lib/providers/anthropic/client.d.ts +22 -7
- package/dist/lib/providers/anthropic/client.js +83 -58
- package/dist/lib/providers/anthropic/rateLimitCapture.d.ts +82 -0
- package/dist/lib/providers/anthropic/rateLimitCapture.js +375 -0
- package/dist/lib/proxy/proxyLifecycle.d.ts +5 -0
- package/dist/lib/proxy/proxyLifecycle.js +61 -11
- package/dist/lib/proxy/quotaHeaders.d.ts +73 -0
- package/dist/lib/proxy/quotaHeaders.js +189 -0
- package/dist/lib/proxy/usageStats.d.ts +2 -0
- package/dist/lib/proxy/usageStats.js +4 -0
- package/dist/lib/server/routes/claudeProxyRoutes.js +132 -17
- package/dist/lib/types/analytics.d.ts +8 -0
- package/dist/lib/types/generate.d.ts +12 -0
- package/dist/lib/types/proxy.d.ts +50 -0
- package/dist/lib/types/subscription.d.ts +77 -0
- package/dist/providers/anthropic/client.d.ts +22 -7
- package/dist/providers/anthropic/client.js +83 -58
- package/dist/providers/anthropic/rateLimitCapture.d.ts +82 -0
- package/dist/providers/anthropic/rateLimitCapture.js +374 -0
- package/dist/proxy/proxyLifecycle.d.ts +5 -0
- package/dist/proxy/proxyLifecycle.js +61 -11
- package/dist/proxy/quotaHeaders.d.ts +73 -0
- package/dist/proxy/quotaHeaders.js +188 -0
- package/dist/proxy/usageStats.d.ts +2 -0
- package/dist/proxy/usageStats.js +4 -0
- package/dist/server/routes/claudeProxyRoutes.js +132 -17
- package/dist/types/analytics.d.ts +8 -0
- package/dist/types/generate.d.ts +12 -0
- package/dist/types/proxy.d.ts +50 -0
- package/dist/types/subscription.d.ts +77 -0
- package/package.json +3 -1
|
@@ -45,6 +45,8 @@ const PROXY_TELEMETRY_SCRIPT_PATH = fileURLToPath(new URL("../../../scripts/obse
|
|
|
45
45
|
const PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS = 5_000;
|
|
46
46
|
const LEGACY_STATUS_ACCOUNT_CACHE_TTL_MS = 5_000;
|
|
47
47
|
const PROXY_STATUS_TOKEN_READ_TIMEOUT_MS = 2_000;
|
|
48
|
+
const PROXY_STATUS_RECONCILE_TIMEOUT_MS = 750;
|
|
49
|
+
const PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS = 750;
|
|
48
50
|
let legacyStatusAccountCache;
|
|
49
51
|
// Allowed drift between a pid's OS-reported start time and the persisted
|
|
50
52
|
// ProxySupervisorState.startTime before processLooksLikeProxySupervisor
|
|
@@ -1495,9 +1497,34 @@ export async function createProxyStartApp(params) {
|
|
|
1495
1497
|
toolRegistry: params.neurolink.getToolRegistry(),
|
|
1496
1498
|
timestamp: Date.now(),
|
|
1497
1499
|
metadata: {},
|
|
1500
|
+
// Route handlers publish limit/quota headers here. Only the streaming
|
|
1501
|
+
// paths build their own Response (and set headers directly); every
|
|
1502
|
+
// JSON and error path returns a plain object, so without this the
|
|
1503
|
+
// headers had nowhere to go. Applied to each c.json() return below.
|
|
1504
|
+
responseHeaders: {},
|
|
1505
|
+
};
|
|
1506
|
+
/** Copy handler-published headers onto the outgoing response. */
|
|
1507
|
+
const applyResponseHeaders = () => {
|
|
1508
|
+
for (const [key, value] of Object.entries(ctx.responseHeaders)) {
|
|
1509
|
+
c.header(key, value);
|
|
1510
|
+
}
|
|
1498
1511
|
};
|
|
1499
1512
|
const result = await route.handler(ctx);
|
|
1500
1513
|
if (result instanceof Response) {
|
|
1514
|
+
// Streaming responses own their headers; merge in anything the
|
|
1515
|
+
// handler published on the context that the Response lacks. A Response
|
|
1516
|
+
// obtained from fetch() carries immutable headers, so this is
|
|
1517
|
+
// best-effort — the body must still reach the client either way.
|
|
1518
|
+
try {
|
|
1519
|
+
for (const [key, value] of Object.entries(ctx.responseHeaders)) {
|
|
1520
|
+
if (!result.headers.has(key)) {
|
|
1521
|
+
result.headers.set(key, value);
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
catch {
|
|
1526
|
+
// Immutable header guard — skip enrichment, keep the response.
|
|
1527
|
+
}
|
|
1501
1528
|
return result;
|
|
1502
1529
|
}
|
|
1503
1530
|
if (result &&
|
|
@@ -1542,6 +1569,7 @@ export async function createProxyStartApp(params) {
|
|
|
1542
1569
|
});
|
|
1543
1570
|
return new Response(responseStream, {
|
|
1544
1571
|
headers: {
|
|
1572
|
+
...ctx.responseHeaders,
|
|
1545
1573
|
"Content-Type": "text/event-stream",
|
|
1546
1574
|
"Cache-Control": "no-cache",
|
|
1547
1575
|
Connection: "keep-alive",
|
|
@@ -1554,6 +1582,7 @@ export async function createProxyStartApp(params) {
|
|
|
1554
1582
|
const httpResult = result;
|
|
1555
1583
|
const status = httpResult.httpStatus ?? 200;
|
|
1556
1584
|
delete httpResult.httpStatus;
|
|
1585
|
+
applyResponseHeaders();
|
|
1557
1586
|
return c.json(result, status);
|
|
1558
1587
|
}
|
|
1559
1588
|
if (result &&
|
|
@@ -1562,8 +1591,10 @@ export async function createProxyStartApp(params) {
|
|
|
1562
1591
|
result.type === "error") {
|
|
1563
1592
|
const errorResult = result;
|
|
1564
1593
|
const status = mapClaudeErrorTypeToStatus(errorResult.error?.type);
|
|
1594
|
+
applyResponseHeaders();
|
|
1565
1595
|
return c.json(result, status);
|
|
1566
1596
|
}
|
|
1597
|
+
applyResponseHeaders();
|
|
1567
1598
|
return c.json(result ?? {});
|
|
1568
1599
|
});
|
|
1569
1600
|
}
|
|
@@ -1592,9 +1623,20 @@ export async function createProxyStartApp(params) {
|
|
|
1592
1623
|
const activeAccountAllowlist = runtimeConfig
|
|
1593
1624
|
? runtimeConfig.accountAllowlist
|
|
1594
1625
|
: params.accountAllowlist;
|
|
1595
|
-
const { getReconciledUsageSnapshot, getUsageStatsPersistenceStatus } = await import("../../lib/proxy/usageStats.js");
|
|
1626
|
+
const { getReconciledUsageSnapshot, getUsageSnapshot, getUsageStatsPersistenceStatus, } = await import("../../lib/proxy/usageStats.js");
|
|
1596
1627
|
const { loadAccountCooldowns } = await import("../../lib/proxy/accountCooldown.js");
|
|
1597
|
-
|
|
1628
|
+
let usageSnapshot = getUsageSnapshot();
|
|
1629
|
+
let snapshotSource = "memory";
|
|
1630
|
+
try {
|
|
1631
|
+
usageSnapshot = await withTimeout(getReconciledUsageSnapshot(), PROXY_STATUS_RECONCILE_TIMEOUT_MS, "[proxy] /status usage reconciliation timed out");
|
|
1632
|
+
snapshotSource = "reconciled";
|
|
1633
|
+
}
|
|
1634
|
+
catch (error) {
|
|
1635
|
+
// Status must not become an outage amplifier when a cross-process lock or
|
|
1636
|
+
// a slow filesystem stalls reconciliation. The process-local snapshot is
|
|
1637
|
+
// coherent and its source is explicit to callers.
|
|
1638
|
+
logger.debug(`[proxy] /status using memory stats snapshot: ${error instanceof Error ? error.message : String(error)}`);
|
|
1639
|
+
}
|
|
1598
1640
|
const { stats, terminalErrors } = usageSnapshot;
|
|
1599
1641
|
const terminalErrorDetailsComparable = usageSnapshot.statsVersion === usageSnapshot.terminalErrorsVersion;
|
|
1600
1642
|
const lastTerminalError = terminalErrors.recent.at(-1) ?? null;
|
|
@@ -1608,40 +1650,52 @@ export async function createProxyStartApp(params) {
|
|
|
1608
1650
|
const supervisorState = loadProxySupervisorState();
|
|
1609
1651
|
const rollingSupervisorRunning = isRollingHandoffCapable(supervisorState);
|
|
1610
1652
|
const updateState = loadUpdateState();
|
|
1611
|
-
const cooldowns = await loadAccountCooldowns()
|
|
1653
|
+
const cooldowns = await withTimeout(loadAccountCooldowns(), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status cooldown inspection timed out").catch((error) => {
|
|
1654
|
+
logger.debug(`[proxy] /status using empty cooldown snapshot: ${error instanceof Error ? error.message : String(error)}`);
|
|
1655
|
+
return {};
|
|
1656
|
+
});
|
|
1612
1657
|
const storedAccountKeys = new Set();
|
|
1613
1658
|
const storedAccountExpirations = new Map();
|
|
1614
1659
|
const disabledAccountKeys = new Set();
|
|
1615
1660
|
let accountInventoryLoaded = false;
|
|
1616
1661
|
try {
|
|
1617
1662
|
const { tokenStore } = await import("../../lib/auth/tokenStore.js");
|
|
1618
|
-
const storedKeys = await tokenStore.listByPrefix("anthropic:");
|
|
1663
|
+
const storedKeys = await withTimeout(tokenStore.listByPrefix("anthropic:"), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status account enumeration timed out");
|
|
1619
1664
|
for (const key of storedKeys) {
|
|
1620
|
-
|
|
1621
|
-
storedAccountKeys.add(normalizedKey);
|
|
1665
|
+
storedAccountKeys.add(normalizeAnthropicAccountKey(key));
|
|
1622
1666
|
}
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1667
|
+
// Once account names are known, preserve them even when optional token
|
|
1668
|
+
// metadata is slow. That keeps the status table useful and avoids
|
|
1669
|
+
// incorrectly presenting known accounts as removed.
|
|
1670
|
+
accountInventoryLoaded = true;
|
|
1671
|
+
const inventory = await withTimeout((async () => {
|
|
1672
|
+
const tokenExpirations = await Promise.all(storedKeys.map(async (key) => {
|
|
1673
|
+
try {
|
|
1674
|
+
const tokens = await withTimeout(tokenStore.peekTokens(key), PROXY_STATUS_TOKEN_READ_TIMEOUT_MS, "[proxy] /status token inspection timed out");
|
|
1675
|
+
return tokens ? [key, tokens.expiresAt] : undefined;
|
|
1629
1676
|
}
|
|
1677
|
+
catch (error) {
|
|
1678
|
+
logger.debug(`[proxy] /status: failed to inspect token metadata for ${normalizeAnthropicAccountKey(key)}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1679
|
+
return undefined;
|
|
1680
|
+
}
|
|
1681
|
+
}));
|
|
1682
|
+
const disabledKeys = await tokenStore.listDisabled();
|
|
1683
|
+
return { tokenExpirations, disabledKeys };
|
|
1684
|
+
})(), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status account metadata timed out");
|
|
1685
|
+
for (const expiration of inventory.tokenExpirations) {
|
|
1686
|
+
if (expiration) {
|
|
1687
|
+
storedAccountExpirations.set(normalizeAnthropicAccountKey(expiration[0]), expiration[1]);
|
|
1630
1688
|
}
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
}
|
|
1634
|
-
}));
|
|
1635
|
-
for (const key of await tokenStore.listDisabled()) {
|
|
1689
|
+
}
|
|
1690
|
+
for (const key of inventory.disabledKeys) {
|
|
1636
1691
|
disabledAccountKeys.add(normalizeAnthropicAccountKey(key));
|
|
1637
1692
|
}
|
|
1638
|
-
accountInventoryLoaded = true;
|
|
1639
1693
|
}
|
|
1640
1694
|
catch (err) {
|
|
1641
1695
|
logger.debug(`[proxy] /status: failed to resolve account cooldown labels: ${err instanceof Error ? err.message : String(err)}`);
|
|
1642
1696
|
}
|
|
1643
1697
|
const legacyAccountLabel = accountInventoryLoaded
|
|
1644
|
-
? await resolveLegacyStatusAccountLabel(storedAccountKeys.size)
|
|
1698
|
+
? await withTimeout(resolveLegacyStatusAccountLabel(storedAccountKeys.size), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status legacy account inspection timed out").catch(() => null)
|
|
1645
1699
|
: null;
|
|
1646
1700
|
const now = Date.now();
|
|
1647
1701
|
const health = buildProxyHealthResponse(readiness, {
|
|
@@ -1649,7 +1703,12 @@ export async function createProxyStartApp(params) {
|
|
|
1649
1703
|
passthrough: activePassthrough,
|
|
1650
1704
|
version: PROXY_VERSION,
|
|
1651
1705
|
});
|
|
1652
|
-
const primaryAccount = await resolveStatusPrimaryAccount(activeProxyConfig)
|
|
1706
|
+
const primaryAccount = await withTimeout(resolveStatusPrimaryAccount(activeProxyConfig), PROXY_STATUS_ACCOUNT_INVENTORY_TIMEOUT_MS, "[proxy] /status primary account inspection timed out").catch(() => ({
|
|
1707
|
+
configured: activeProxyConfig?.routing?.primaryAccount?.trim() || null,
|
|
1708
|
+
key: null,
|
|
1709
|
+
label: null,
|
|
1710
|
+
source: "fallback",
|
|
1711
|
+
}));
|
|
1653
1712
|
const activeUpdaterPid = supervisorState?.updaterPid ?? runtimeState?.updaterPid;
|
|
1654
1713
|
const accountRows = Object.values(stats.accounts).map((account) => {
|
|
1655
1714
|
const normalizedKey = normalizeAnthropicAccountKey(account.label);
|
|
@@ -1813,6 +1872,7 @@ export async function createProxyStartApp(params) {
|
|
|
1813
1872
|
terminalErrorDetailsComparable,
|
|
1814
1873
|
terminalErrorDetailsMissing,
|
|
1815
1874
|
terminalErrorDetailsExcess,
|
|
1875
|
+
snapshotSource,
|
|
1816
1876
|
accounts: accountRows,
|
|
1817
1877
|
primaryAccount,
|
|
1818
1878
|
persistence: getUsageStatsPersistenceStatus(),
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type AIProviderName } from "../../constants/enums.js";
|
|
2
2
|
import { BaseProvider } from "../../core/baseProvider.js";
|
|
3
|
-
import type { AnthropicProviderConfig, StreamOptions, StreamResult, ValidationSchema, EnhancedGenerateResult, TextGenerationOptions, AnthropicAuthMethod, AnthropicResponseMetadata, ClaudeSubscriptionTier, ClaudeUsageInfo } from "../../types/index.js";
|
|
3
|
+
import type { AnthropicProviderConfig, StreamOptions, StreamResult, ValidationSchema, EnhancedGenerateResult, TextGenerationOptions, AnthropicAuthMethod, AnthropicRateLimitInfo, AnthropicResponseMetadata, ClaudeSubscriptionTier, ClaudeUsageInfo } from "../../types/index.js";
|
|
4
4
|
import type { LanguageModel } from "../../types/index.js";
|
|
5
5
|
/**
|
|
6
6
|
* Anthropic Provider v2 - BaseProvider Implementation
|
|
@@ -103,12 +103,18 @@ export declare class AnthropicProvider extends BaseProvider {
|
|
|
103
103
|
*/
|
|
104
104
|
getLastResponseMetadata(): AnthropicResponseMetadata | null;
|
|
105
105
|
/**
|
|
106
|
-
* Update response metadata from
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
106
|
+
* Update response metadata from a captured limit snapshot.
|
|
107
|
+
*
|
|
108
|
+
* Takes already-parsed rate-limit info rather than raw headers: parsing now
|
|
109
|
+
* lives in `rateLimitCapture`, which is the only layer that sees the raw
|
|
110
|
+
* response and understands both header families (unified subscription
|
|
111
|
+
* windows and legacy per-tier counters).
|
|
112
|
+
*
|
|
113
|
+
* @param rateLimit - Parsed rate-limit figures
|
|
114
|
+
* @param requestId - Optional Anthropic request ID
|
|
115
|
+
* @param usageUpdate - Optional token counts to fold into usage tracking
|
|
110
116
|
*/
|
|
111
|
-
protected updateResponseMetadata(
|
|
117
|
+
protected updateResponseMetadata(rateLimit: AnthropicRateLimitInfo, requestId?: string, usageUpdate?: {
|
|
112
118
|
inputTokens?: number;
|
|
113
119
|
outputTokens?: number;
|
|
114
120
|
}): void;
|
|
@@ -127,7 +133,16 @@ export declare class AnthropicProvider extends BaseProvider {
|
|
|
127
133
|
* BaseProvider so that expired tokens are renewed automatically.
|
|
128
134
|
*/
|
|
129
135
|
generate(optionsOrPrompt: TextGenerationOptions | string, analysisSchema?: ValidationSchema): Promise<EnhancedGenerateResult | null>;
|
|
130
|
-
|
|
136
|
+
/**
|
|
137
|
+
* Fold a captured snapshot into the provider's usage bookkeeping and log it.
|
|
138
|
+
*
|
|
139
|
+
* `updateResponseMetadata` had no callers before this — the metadata it
|
|
140
|
+
* maintains, and the public `getLastResponseMetadata()` / `getUsageInfo()`
|
|
141
|
+
* that read it, were never populated by anything.
|
|
142
|
+
*/
|
|
143
|
+
private recordLimitSnapshot;
|
|
144
|
+
protected executeStream(options: StreamOptions, analysisSchema?: ValidationSchema): Promise<StreamResult>;
|
|
145
|
+
private executeStreamInCaptureScope;
|
|
131
146
|
isAvailable(): Promise<boolean>;
|
|
132
147
|
getModel(): LanguageModel;
|
|
133
148
|
}
|
|
@@ -11,6 +11,7 @@ import { streamAnalyticsCollector } from "../../core/streamAnalytics.js";
|
|
|
11
11
|
import { getModelCapabilities, getRecommendedModelForTier, isModelAvailableForTier, } from "../../models/anthropicModels.js";
|
|
12
12
|
import { createOAuthFetch } from "../../proxy/oauthFetch.js";
|
|
13
13
|
import { createProxyFetch } from "../../proxy/proxyFetch.js";
|
|
14
|
+
import { getCapturedLimitSnapshot, getCapturedResponseHeaders, logClaudeLimitSnapshot, runInLimitCaptureScope, setLimitSpanAttributes, withLimitCapture, wrapFetchWithLimitCapture, } from "./rateLimitCapture.js";
|
|
14
15
|
import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } from "../../types/index.js";
|
|
15
16
|
import { logger } from "../../utils/logger.js";
|
|
16
17
|
import { redactUrlCredentials } from "../../utils/logSanitize.js";
|
|
@@ -166,35 +167,10 @@ const detectAuthMethod = (oauthToken) => {
|
|
|
166
167
|
});
|
|
167
168
|
return method;
|
|
168
169
|
};
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
*/
|
|
174
|
-
const parseRateLimitHeaders = (headers) => {
|
|
175
|
-
const getHeader = (name) => {
|
|
176
|
-
if (headers instanceof Headers) {
|
|
177
|
-
return headers.get(name);
|
|
178
|
-
}
|
|
179
|
-
return headers[name] || headers[name.toLowerCase()] || null;
|
|
180
|
-
};
|
|
181
|
-
const parseNumber = (value) => {
|
|
182
|
-
if (!value) {
|
|
183
|
-
return undefined;
|
|
184
|
-
}
|
|
185
|
-
const num = parseInt(value, 10);
|
|
186
|
-
return isNaN(num) ? undefined : num;
|
|
187
|
-
};
|
|
188
|
-
return {
|
|
189
|
-
requestsLimit: parseNumber(getHeader("anthropic-ratelimit-requests-limit")),
|
|
190
|
-
requestsRemaining: parseNumber(getHeader("anthropic-ratelimit-requests-remaining")),
|
|
191
|
-
requestsReset: getHeader("anthropic-ratelimit-requests-reset") || undefined,
|
|
192
|
-
tokensLimit: parseNumber(getHeader("anthropic-ratelimit-tokens-limit")),
|
|
193
|
-
tokensRemaining: parseNumber(getHeader("anthropic-ratelimit-tokens-remaining")),
|
|
194
|
-
tokensReset: getHeader("anthropic-ratelimit-tokens-reset") || undefined,
|
|
195
|
-
retryAfter: parseNumber(getHeader("retry-after")),
|
|
196
|
-
};
|
|
197
|
-
};
|
|
170
|
+
// Rate-limit header parsing lives in `rateLimitCapture.ts`, which sees the raw
|
|
171
|
+
// fetch Response and understands both header families (unified subscription
|
|
172
|
+
// windows and the legacy per-tier counters). The module-private copy that used
|
|
173
|
+
// to sit here parsed only the legacy family and had no callers.
|
|
198
174
|
// ───────────────────────────────────────────────────────────────────────────
|
|
199
175
|
// Native Messages-API conversion helpers (NeuroLink/V3 shapes → Anthropic)
|
|
200
176
|
// ───────────────────────────────────────────────────────────────────────────
|
|
@@ -590,7 +566,10 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
590
566
|
client = new Anthropic({
|
|
591
567
|
apiKey: "oauth-authenticated", // Placeholder, actual auth is in fetch wrapper
|
|
592
568
|
// Note: No headers passed - fetch wrapper sets oauth-2025-04-20 beta header
|
|
593
|
-
fetch
|
|
569
|
+
// Limit capture wraps the OAuth fetch so subscription quota headers
|
|
570
|
+
// (anthropic-ratelimit-unified-*) are recorded on every request —
|
|
571
|
+
// streaming and non-streaming alike.
|
|
572
|
+
fetch: wrapFetchWithLimitCapture(oauthFetch),
|
|
594
573
|
timeout: ANTHROPIC_CLIENT_TIMEOUT_MS,
|
|
595
574
|
// The SDK's built-in retry honors Retry-After hints without any
|
|
596
575
|
// upper bound (a 429 with retry-after: 8549 sleeps 2.4h per retry,
|
|
@@ -642,7 +621,10 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
642
621
|
apiKey: apiKeyToUse,
|
|
643
622
|
defaultHeaders: headers,
|
|
644
623
|
...(normalizedBaseURL && { baseURL: normalizedBaseURL }),
|
|
645
|
-
|
|
624
|
+
// Same capture as the OAuth branch: works for direct API-key traffic
|
|
625
|
+
// (legacy requests/tokens counters) and for the NeuroLink Claude proxy
|
|
626
|
+
// (verbatim unified quota plus x-neurolink-* account/pool state).
|
|
627
|
+
fetch: wrapFetchWithLimitCapture(createProxyFetch()),
|
|
646
628
|
timeout: ANTHROPIC_CLIENT_TIMEOUT_MS,
|
|
647
629
|
// See the OAuth-path client above: unbounded Retry-After sleeps in
|
|
648
630
|
// the SDK's retry loop must never stall fallback orchestration.
|
|
@@ -971,24 +953,22 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
971
953
|
return this.lastResponseMetadata;
|
|
972
954
|
}
|
|
973
955
|
/**
|
|
974
|
-
* Update response metadata from
|
|
975
|
-
*
|
|
976
|
-
*
|
|
977
|
-
*
|
|
956
|
+
* Update response metadata from a captured limit snapshot.
|
|
957
|
+
*
|
|
958
|
+
* Takes already-parsed rate-limit info rather than raw headers: parsing now
|
|
959
|
+
* lives in `rateLimitCapture`, which is the only layer that sees the raw
|
|
960
|
+
* response and understands both header families (unified subscription
|
|
961
|
+
* windows and legacy per-tier counters).
|
|
962
|
+
*
|
|
963
|
+
* @param rateLimit - Parsed rate-limit figures
|
|
964
|
+
* @param requestId - Optional Anthropic request ID
|
|
965
|
+
* @param usageUpdate - Optional token counts to fold into usage tracking
|
|
978
966
|
*/
|
|
979
|
-
updateResponseMetadata(
|
|
967
|
+
updateResponseMetadata(rateLimit, requestId, usageUpdate) {
|
|
980
968
|
this.lastResponseMetadata = {
|
|
981
|
-
rateLimit
|
|
982
|
-
requestId
|
|
983
|
-
(headers instanceof Headers
|
|
984
|
-
? headers.get("x-request-id") || undefined
|
|
985
|
-
: headers["x-request-id"]),
|
|
986
|
-
serverTiming: headers instanceof Headers
|
|
987
|
-
? headers.get("server-timing") || undefined
|
|
988
|
-
: headers["server-timing"],
|
|
969
|
+
rateLimit,
|
|
970
|
+
...(requestId ? { requestId } : {}),
|
|
989
971
|
};
|
|
990
|
-
// Update usage tracking
|
|
991
|
-
const rateLimit = this.lastResponseMetadata.rateLimit;
|
|
992
972
|
if (this.usageInfo) {
|
|
993
973
|
this.usageInfo.requestCount++;
|
|
994
974
|
this.usageInfo.messagesUsed++;
|
|
@@ -1254,7 +1234,10 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1254
1234
|
response: {
|
|
1255
1235
|
id: response.id,
|
|
1256
1236
|
modelId: response.model,
|
|
1257
|
-
headers
|
|
1237
|
+
// Real response headers, captured by the fetch wrapper. This used
|
|
1238
|
+
// to be a hardcoded `{}`, which silently discarded every
|
|
1239
|
+
// rate-limit and quota header Anthropic returns.
|
|
1240
|
+
headers: getCapturedResponseHeaders() ?? {},
|
|
1258
1241
|
body: response,
|
|
1259
1242
|
},
|
|
1260
1243
|
};
|
|
@@ -1305,9 +1288,41 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1305
1288
|
*/
|
|
1306
1289
|
async generate(optionsOrPrompt, analysisSchema) {
|
|
1307
1290
|
await this.refreshAuthIfNeeded();
|
|
1308
|
-
|
|
1291
|
+
// Open a per-request capture scope around the whole turn. Scoping here
|
|
1292
|
+
// rather than on the instance is what makes it concurrency-safe: several
|
|
1293
|
+
// generate() calls can be in flight on one provider instance, and an
|
|
1294
|
+
// instance field would attribute one call's limits to another.
|
|
1295
|
+
const { result, snapshot } = await withLimitCapture(() => super.generate(optionsOrPrompt, analysisSchema));
|
|
1296
|
+
if (result && snapshot) {
|
|
1297
|
+
this.recordLimitSnapshot(snapshot);
|
|
1298
|
+
result.limits = snapshot;
|
|
1299
|
+
if (result.analytics) {
|
|
1300
|
+
result.analytics.limits = snapshot;
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
return result;
|
|
1304
|
+
}
|
|
1305
|
+
/**
|
|
1306
|
+
* Fold a captured snapshot into the provider's usage bookkeeping and log it.
|
|
1307
|
+
*
|
|
1308
|
+
* `updateResponseMetadata` had no callers before this — the metadata it
|
|
1309
|
+
* maintains, and the public `getLastResponseMetadata()` / `getUsageInfo()`
|
|
1310
|
+
* that read it, were never populated by anything.
|
|
1311
|
+
*/
|
|
1312
|
+
recordLimitSnapshot(snapshot) {
|
|
1313
|
+
this.updateResponseMetadata(snapshot.rateLimit, snapshot.requestId);
|
|
1314
|
+
setLimitSpanAttributes(snapshot);
|
|
1315
|
+
logClaudeLimitSnapshot(snapshot, this.modelName);
|
|
1316
|
+
}
|
|
1317
|
+
async executeStream(options, analysisSchema) {
|
|
1318
|
+
// The capture scope must outlive this call: the SSE loop keeps running in
|
|
1319
|
+
// the background after executeStream returns, and its per-step HTTP
|
|
1320
|
+
// requests are what carry the limit headers. AsyncLocalStorage.run
|
|
1321
|
+
// propagates into every continuation started inside, so the whole stream
|
|
1322
|
+
// lifetime shares one slot.
|
|
1323
|
+
return runInLimitCaptureScope(() => this.executeStreamInCaptureScope(options, analysisSchema));
|
|
1309
1324
|
}
|
|
1310
|
-
async
|
|
1325
|
+
async executeStreamInCaptureScope(options, _analysisSchema) {
|
|
1311
1326
|
// Refresh OAuth token if needed before making any API request.
|
|
1312
1327
|
await this.refreshAuthIfNeeded();
|
|
1313
1328
|
this.validateStreamOptions(options);
|
|
@@ -1758,15 +1773,25 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1758
1773
|
// stream consumers and session cost tracking saw no usage at all.
|
|
1759
1774
|
// Chained off finishPromise so requestDuration reflects the DRAINED
|
|
1760
1775
|
// stream, not the milliseconds it took to construct this result object.
|
|
1761
|
-
analytics: finishPromise.then(() =>
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1776
|
+
analytics: finishPromise.then(async () => {
|
|
1777
|
+
const analytics = await streamAnalyticsCollector.createAnalytics(this.providerName, modelId, {
|
|
1778
|
+
textStream: (async function* () { })(),
|
|
1779
|
+
usage: usagePromise,
|
|
1780
|
+
finishReason: finishPromise,
|
|
1781
|
+
}, Date.now() - streamStartTime, {
|
|
1782
|
+
requestId: options.requestId ??
|
|
1783
|
+
`${this.providerName}-stream-${Date.now()}`,
|
|
1784
|
+
streamingMode: true,
|
|
1785
|
+
});
|
|
1786
|
+
// Still inside the capture scope opened by executeStream, so this sees
|
|
1787
|
+
// the limits reported by the last upstream step of the stream.
|
|
1788
|
+
const snapshot = getCapturedLimitSnapshot();
|
|
1789
|
+
if (snapshot && analytics) {
|
|
1790
|
+
this.recordLimitSnapshot(snapshot);
|
|
1791
|
+
analytics.limits = snapshot;
|
|
1792
|
+
}
|
|
1793
|
+
return analytics;
|
|
1794
|
+
}),
|
|
1770
1795
|
};
|
|
1771
1796
|
}
|
|
1772
1797
|
async isAvailable() {
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anthropic rate-limit / quota header capture.
|
|
3
|
+
*
|
|
4
|
+
* Anthropic returns limit state on the response headers of every request:
|
|
5
|
+
* `anthropic-ratelimit-unified-*` for subscription (OAuth) accounts,
|
|
6
|
+
* `anthropic-ratelimit-{requests,tokens}-*` for API-key accounts. The NeuroLink
|
|
7
|
+
* Claude proxy forwards those verbatim and adds `x-neurolink-*` for what only
|
|
8
|
+
* it knows (which account served the request, pool headroom, whether the
|
|
9
|
+
* numbers are live or a carried-over snapshot).
|
|
10
|
+
*
|
|
11
|
+
* None of it used to reach the SDK: `doGenerate` returned a hardcoded empty
|
|
12
|
+
* header bag and the streaming loop never looked. The capture point here is the
|
|
13
|
+
* `fetch` the Anthropic SDK is constructed with — it is invoked exactly once
|
|
14
|
+
* per HTTP request on BOTH the streaming and non-streaming paths, so a single
|
|
15
|
+
* wrapper covers everything without touching the SSE loop or switching the
|
|
16
|
+
* non-streaming call to `.withResponse()`.
|
|
17
|
+
*
|
|
18
|
+
* Scoping is per-request via AsyncLocalStorage rather than a field on the
|
|
19
|
+
* provider: a provider instance is shared across concurrent calls, so an
|
|
20
|
+
* instance field would race and attribute one request's limits to another.
|
|
21
|
+
*
|
|
22
|
+
* @module providers/anthropic/rateLimitCapture
|
|
23
|
+
*/
|
|
24
|
+
import type { AnthropicRateLimitInfo, ClaudeLimitSnapshot } from "../../types/index.js";
|
|
25
|
+
/**
|
|
26
|
+
* Parse both Anthropic rate-limit header families into a single shape.
|
|
27
|
+
*
|
|
28
|
+
* Which family is present depends on the account type, so every field is
|
|
29
|
+
* optional and absence is normal rather than an error.
|
|
30
|
+
*/
|
|
31
|
+
export declare function parseAnthropicLimitHeaders(headers: Headers): AnthropicRateLimitInfo;
|
|
32
|
+
/**
|
|
33
|
+
* Build a snapshot from a response, or undefined when the response carries
|
|
34
|
+
* neither Anthropic rate-limit headers nor NeuroLink proxy metadata.
|
|
35
|
+
*/
|
|
36
|
+
export declare function buildLimitSnapshot(headers: Headers, status: number, now?: number): ClaudeLimitSnapshot | undefined;
|
|
37
|
+
/**
|
|
38
|
+
* Wrap a fetch so every response's limit headers are captured into the
|
|
39
|
+
* enclosing `withLimitCapture` scope. A no-op outside such a scope.
|
|
40
|
+
*
|
|
41
|
+
* Capture never alters the response and never throws — a parsing failure must
|
|
42
|
+
* not be able to break a request that the provider would otherwise complete.
|
|
43
|
+
*/
|
|
44
|
+
export declare function wrapFetchWithLimitCapture(inner: typeof fetch): typeof fetch;
|
|
45
|
+
/**
|
|
46
|
+
* Run `body` in a capture scope and return its result alongside whatever limit
|
|
47
|
+
* snapshot the underlying HTTP request(s) produced.
|
|
48
|
+
*/
|
|
49
|
+
export declare function withLimitCapture<T>(body: () => Promise<T>): Promise<{
|
|
50
|
+
result: T;
|
|
51
|
+
snapshot?: ClaudeLimitSnapshot;
|
|
52
|
+
}>;
|
|
53
|
+
/**
|
|
54
|
+
* Current scope's snapshot, if any. Lets a long-running loop (the streaming
|
|
55
|
+
* path) read limits mid-flight without unwinding the scope.
|
|
56
|
+
*/
|
|
57
|
+
export declare function getCapturedLimitSnapshot(): ClaudeLimitSnapshot | undefined;
|
|
58
|
+
/** Raw headers of the most recent captured response in this scope. */
|
|
59
|
+
export declare function getCapturedResponseHeaders(): Record<string, string> | undefined;
|
|
60
|
+
/** Enter a capture scope without wrapping a single call — for streaming, where
|
|
61
|
+
* the scope must outlive the function that opened it. */
|
|
62
|
+
export declare function runInLimitCaptureScope<T>(body: () => T): T;
|
|
63
|
+
/**
|
|
64
|
+
* Attach limit state to the currently active OTel span.
|
|
65
|
+
*
|
|
66
|
+
* Uses the active span rather than threading one down from the generation
|
|
67
|
+
* layer: that layer is provider-agnostic and should not learn about Anthropic
|
|
68
|
+
* quota headers just to record them. The active span during a turn is the one
|
|
69
|
+
* already carrying `gen_ai.usage.*` and `neurolink.cost`, so "what did this
|
|
70
|
+
* cost" and "how much is left" answer from the same trace.
|
|
71
|
+
*/
|
|
72
|
+
export declare function setLimitSpanAttributes(snapshot: ClaudeLimitSnapshot): void;
|
|
73
|
+
/**
|
|
74
|
+
* Emit one structured line per request describing remaining capacity.
|
|
75
|
+
*
|
|
76
|
+
* Leads with headroom ("how much is left") because that is the figure an
|
|
77
|
+
* operator acts on; the raw utilization stays available on the snapshot for
|
|
78
|
+
* anything computing against it. Escalates to WARN when the session window is
|
|
79
|
+
* nearly spent or the provider has already flagged the account as
|
|
80
|
+
* throttled/rejected.
|
|
81
|
+
*/
|
|
82
|
+
export declare function logClaudeLimitSnapshot(snapshot: ClaudeLimitSnapshot, model?: string, now?: number): void;
|