@juspay/neurolink 10.6.4 → 10.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/analytics/pricing.d.ts +4 -1
- package/dist/analytics/pricing.js +6 -2
- package/dist/browser/neurolink.min.js +406 -406
- package/dist/cli/commands/proxyAnalyze.js +12 -0
- package/dist/core/analytics.js +8 -2
- package/dist/core/baseProvider.js +9 -7
- package/dist/core/modules/GenerationHandler.d.ts +8 -0
- package/dist/core/modules/GenerationHandler.js +28 -38
- package/dist/core/modules/TelemetryHandler.d.ts +3 -9
- package/dist/core/modules/TelemetryHandler.js +17 -12
- package/dist/lib/analytics/pricing.d.ts +4 -1
- package/dist/lib/analytics/pricing.js +6 -2
- package/dist/lib/core/analytics.js +8 -2
- package/dist/lib/core/baseProvider.js +9 -7
- package/dist/lib/core/modules/GenerationHandler.d.ts +8 -0
- package/dist/lib/core/modules/GenerationHandler.js +28 -38
- package/dist/lib/core/modules/TelemetryHandler.d.ts +3 -9
- package/dist/lib/core/modules/TelemetryHandler.js +17 -12
- package/dist/lib/neurolink.js +18 -1
- package/dist/lib/providers/amazonBedrock.js +68 -6
- package/dist/lib/providers/anthropic.js +50 -16
- package/dist/lib/providers/googleAiStudio.js +29 -4
- package/dist/lib/providers/googleNativeGemini3.js +10 -0
- package/dist/lib/providers/googleVertex.js +150 -25
- package/dist/lib/providers/litellm.js +13 -2
- package/dist/lib/providers/openAI.js +13 -2
- package/dist/lib/providers/openaiChatCompletionsBase.js +45 -10
- package/dist/lib/providers/openaiChatCompletionsClient.d.ts +2 -14
- package/dist/lib/providers/openaiChatCompletionsClient.js +20 -1
- package/dist/lib/providers/sagemaker/language-model.js +5 -3
- package/dist/lib/providers/sagemaker/parsers.js +18 -9
- package/dist/lib/providers/sagemaker/streaming.d.ts +9 -0
- package/dist/lib/providers/sagemaker/streaming.js +64 -6
- package/dist/lib/proxy/accountCooldown.js +2 -7
- package/dist/lib/proxy/proxyAnalysis.js +197 -0
- package/dist/lib/proxy/requestLogger.js +11 -0
- package/dist/lib/proxy/routingEvidence.d.ts +6 -0
- package/dist/lib/proxy/routingEvidence.js +33 -0
- package/dist/lib/server/routes/claudeProxyRoutes.d.ts +2 -1
- package/dist/lib/server/routes/claudeProxyRoutes.js +257 -87
- package/dist/lib/types/common.d.ts +3 -0
- package/dist/lib/types/openaiCompatible.d.ts +8 -7
- package/dist/lib/types/providers.d.ts +6 -0
- package/dist/lib/types/proxy.d.ts +97 -2
- package/dist/lib/utils/pricing.js +15 -3
- package/dist/lib/utils/tokenUtils.js +38 -3
- package/dist/neurolink.js +18 -1
- package/dist/providers/amazonBedrock.js +68 -6
- package/dist/providers/anthropic.js +50 -16
- package/dist/providers/googleAiStudio.js +29 -4
- package/dist/providers/googleNativeGemini3.js +10 -0
- package/dist/providers/googleVertex.js +150 -25
- package/dist/providers/litellm.js +13 -2
- package/dist/providers/openAI.js +13 -2
- package/dist/providers/openaiChatCompletionsBase.js +45 -10
- package/dist/providers/openaiChatCompletionsClient.d.ts +2 -14
- package/dist/providers/openaiChatCompletionsClient.js +20 -1
- package/dist/providers/sagemaker/language-model.js +5 -3
- package/dist/providers/sagemaker/parsers.js +18 -9
- package/dist/providers/sagemaker/streaming.d.ts +9 -0
- package/dist/providers/sagemaker/streaming.js +64 -6
- package/dist/proxy/accountCooldown.js +2 -7
- package/dist/proxy/proxyAnalysis.js +197 -0
- package/dist/proxy/requestLogger.js +11 -0
- package/dist/proxy/routingEvidence.d.ts +6 -0
- package/dist/proxy/routingEvidence.js +32 -0
- package/dist/server/routes/claudeProxyRoutes.d.ts +2 -1
- package/dist/server/routes/claudeProxyRoutes.js +257 -87
- package/dist/types/common.d.ts +3 -0
- package/dist/types/openaiCompatible.d.ts +8 -7
- package/dist/types/providers.d.ts +6 -0
- package/dist/types/proxy.d.ts +97 -2
- package/dist/utils/pricing.js +15 -3
- package/dist/utils/tokenUtils.js +38 -3
- package/package.json +2 -1
|
@@ -653,15 +653,27 @@ function findRates(provider, model) {
|
|
|
653
653
|
if (!providerPricing) {
|
|
654
654
|
return undefined;
|
|
655
655
|
}
|
|
656
|
+
// Bedrock model IDs carry region/vendor prefixes ("us.anthropic.claude-…",
|
|
657
|
+
// "global.anthropic.claude-…", "anthropic.claude-…") or arrive as full
|
|
658
|
+
// inference-profile ARNs ("arn:aws:bedrock:…:inference-profile/us.anthropic.
|
|
659
|
+
// claude-…" — the form this repo's own setup guides recommend). None of
|
|
660
|
+
// those match the bare pricing keys ("claude-…"), so every Bedrock call
|
|
661
|
+
// priced to $0. Strip everything up to and including the vendor prefix; the
|
|
662
|
+
// trailing "-v1:0" version suffix is absorbed by the longest-prefix match
|
|
663
|
+
// below. Non-Anthropic Bedrock models (meta./amazon./mistral.) don't match
|
|
664
|
+
// and fall through unchanged, exactly as before.
|
|
665
|
+
const modelKey = stripped === "bedrock" || stripped === "amazonbedrock"
|
|
666
|
+
? model.replace(/^.*\banthropic\./, "")
|
|
667
|
+
: model;
|
|
656
668
|
// Exact match
|
|
657
|
-
if (providerPricing[
|
|
658
|
-
return providerPricing[
|
|
669
|
+
if (providerPricing[modelKey]) {
|
|
670
|
+
return providerPricing[modelKey];
|
|
659
671
|
}
|
|
660
672
|
// Longest-prefix match (skip the synthetic "_default" sentinel below)
|
|
661
673
|
const sortedKeys = Object.keys(providerPricing)
|
|
662
674
|
.filter((k) => k !== "_default")
|
|
663
675
|
.sort((a, b) => b.length - a.length);
|
|
664
|
-
const key = sortedKeys.find((k) =>
|
|
676
|
+
const key = sortedKeys.find((k) => modelKey.startsWith(k));
|
|
665
677
|
if (key) {
|
|
666
678
|
return providerPricing[key];
|
|
667
679
|
}
|
|
@@ -44,10 +44,13 @@ export function extractOutputTokens(usage) {
|
|
|
44
44
|
* Falls back to input + output if total is not provided
|
|
45
45
|
*/
|
|
46
46
|
export function extractTotalTokens(usage, input, output) {
|
|
47
|
-
|
|
47
|
+
// A literal 0 alongside non-zero components means the provider omitted the
|
|
48
|
+
// figure (several gateways/parsers emit total: 0) — fall through to the
|
|
49
|
+
// computed sum instead of letting the zero win.
|
|
50
|
+
if (typeof usage.total === "number" && usage.total > 0) {
|
|
48
51
|
return usage.total;
|
|
49
52
|
}
|
|
50
|
-
if (typeof usage.totalTokens === "number") {
|
|
53
|
+
if (typeof usage.totalTokens === "number" && usage.totalTokens > 0) {
|
|
51
54
|
return usage.totalTokens;
|
|
52
55
|
}
|
|
53
56
|
return input + output;
|
|
@@ -185,7 +188,6 @@ export function extractTokenUsage(result, options = {}) {
|
|
|
185
188
|
// Extract base token counts
|
|
186
189
|
let input = extractInputTokens(usage);
|
|
187
190
|
const output = extractOutputTokens(usage);
|
|
188
|
-
const total = extractTotalTokens(usage, input, output);
|
|
189
191
|
// Extract optional token fields
|
|
190
192
|
const reasoning = extractReasoningTokens(usage);
|
|
191
193
|
const cacheCreationTokens = extractCacheCreationTokens(usage);
|
|
@@ -204,6 +206,39 @@ export function extractTokenUsage(result, options = {}) {
|
|
|
204
206
|
input = Math.max(0, input - overlappingCached);
|
|
205
207
|
}
|
|
206
208
|
}
|
|
209
|
+
// ai@6-shape rebase: when the cache values came ONLY from the ai@6
|
|
210
|
+
// normalized shape (cachedInputTokens / inputTokenDetails.*), the flat
|
|
211
|
+
// input is cache-INCLUSIVE — asLanguageModelUsage reports
|
|
212
|
+
// inputTokens.total = noCache + cacheRead + cacheWrite — so `input` must
|
|
213
|
+
// be rebased onto the uncached portion or calculateCost bills the cached
|
|
214
|
+
// tokens twice (full input rate + cacheRead/cacheCreation rate). The
|
|
215
|
+
// native fields excluded below (cacheReadInputTokens / cacheReadTokens /
|
|
216
|
+
// cacheCreationInputTokens / cacheCreationTokens) follow the
|
|
217
|
+
// non-overlapping convention where input is already the uncached
|
|
218
|
+
// remainder, and the overlapping snake_case shape was already rebased
|
|
219
|
+
// above — neither must be rebased again.
|
|
220
|
+
if ((cacheReadTokens !== undefined || cacheCreationTokens !== undefined) &&
|
|
221
|
+
usage.cacheReadInputTokens === undefined &&
|
|
222
|
+
usage.cacheReadTokens === undefined &&
|
|
223
|
+
usage.cacheCreationInputTokens === undefined &&
|
|
224
|
+
usage.cacheCreationTokens === undefined &&
|
|
225
|
+
usage.prompt_tokens_details?.cached_tokens === undefined &&
|
|
226
|
+
(usage.cachedInputTokens !== undefined ||
|
|
227
|
+
usage.inputTokenDetails !== undefined)) {
|
|
228
|
+
const noCacheTokens = usage.inputTokenDetails?.noCacheTokens;
|
|
229
|
+
if (typeof noCacheTokens === "number") {
|
|
230
|
+
input = noCacheTokens;
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
input = Math.max(0, input -
|
|
234
|
+
(cacheReadTokens ?? 0) -
|
|
235
|
+
(usage.inputTokenDetails?.cacheWriteTokens ?? 0));
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
// Total: prefer the provider-reported figure; the fallback conserves every
|
|
239
|
+
// billed component — cache tokens are additive on top of the (post-rebase)
|
|
240
|
+
// uncached input. Reasoning is already inside `output`, so it is NOT added.
|
|
241
|
+
const total = extractTotalTokens(usage, input + (cacheReadTokens ?? 0) + (cacheCreationTokens ?? 0), output);
|
|
207
242
|
// Calculate cache savings if enabled
|
|
208
243
|
const cacheSavingsPercent = calculateCacheSavings
|
|
209
244
|
? calculateCacheSavingsPercent(cacheReadTokens, input)
|
package/dist/neurolink.js
CHANGED
|
@@ -2746,7 +2746,10 @@ Current user's request: ${currentInput}`;
|
|
|
2746
2746
|
if (!usage || model === "unknown") {
|
|
2747
2747
|
return undefined;
|
|
2748
2748
|
}
|
|
2749
|
-
const totalCost = calculateAdvancedCost(model, usage.input || 0, usage.output || 0, provider === "unknown" ? undefined : provider
|
|
2749
|
+
const totalCost = calculateAdvancedCost(model, usage.input || 0, usage.output || 0, provider === "unknown" ? undefined : provider, {
|
|
2750
|
+
cacheReadTokens: usage.cacheReadTokens,
|
|
2751
|
+
cacheCreationTokens: usage.cacheCreationTokens,
|
|
2752
|
+
});
|
|
2750
2753
|
return totalCost > 0 ? totalCost : undefined;
|
|
2751
2754
|
}
|
|
2752
2755
|
/**
|
|
@@ -4134,6 +4137,20 @@ Current user's request: ${currentInput}`;
|
|
|
4134
4137
|
input: textResult.usage.input || 0,
|
|
4135
4138
|
output: textResult.usage.output || 0,
|
|
4136
4139
|
total: textResult.usage.total || 0,
|
|
4140
|
+
// Optional tiers must be forwarded or they silently vanish at
|
|
4141
|
+
// this DTO boundary (cache-aware cost + savings need them).
|
|
4142
|
+
...(textResult.usage.cacheReadTokens !== undefined && {
|
|
4143
|
+
cacheReadTokens: textResult.usage.cacheReadTokens,
|
|
4144
|
+
}),
|
|
4145
|
+
...(textResult.usage.cacheCreationTokens !== undefined && {
|
|
4146
|
+
cacheCreationTokens: textResult.usage.cacheCreationTokens,
|
|
4147
|
+
}),
|
|
4148
|
+
...(textResult.usage.reasoning !== undefined && {
|
|
4149
|
+
reasoning: textResult.usage.reasoning,
|
|
4150
|
+
}),
|
|
4151
|
+
...(textResult.usage.cacheSavingsPercent !== undefined && {
|
|
4152
|
+
cacheSavingsPercent: textResult.usage.cacheSavingsPercent,
|
|
4153
|
+
}),
|
|
4137
4154
|
}
|
|
4138
4155
|
: undefined,
|
|
4139
4156
|
responseTime: textResult.responseTime,
|
|
@@ -253,6 +253,8 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
253
253
|
let iteration = 0;
|
|
254
254
|
let totalInputTokens = 0;
|
|
255
255
|
let totalOutputTokens = 0;
|
|
256
|
+
let totalCacheReadTokens = 0;
|
|
257
|
+
let totalCacheWriteTokens = 0;
|
|
256
258
|
let lastFinishReason;
|
|
257
259
|
while (iteration < maxIterations) {
|
|
258
260
|
iteration++;
|
|
@@ -263,8 +265,12 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
263
265
|
logger.debug(`[AmazonBedrockProvider] Received Bedrock response`, JSON.stringify(response, null, 2));
|
|
264
266
|
// Accumulate real token counts and capture the stop reason so
|
|
265
267
|
// Pipeline B (Langfuse) gets correct usage and finishReason.
|
|
268
|
+
// Converse follows the Anthropic additive convention: inputTokens is
|
|
269
|
+
// the UNCACHED remainder; cache reads/writes are reported separately.
|
|
266
270
|
totalInputTokens += response.usage?.inputTokens ?? 0;
|
|
267
271
|
totalOutputTokens += response.usage?.outputTokens ?? 0;
|
|
272
|
+
totalCacheReadTokens += response.usage?.cacheReadInputTokens ?? 0;
|
|
273
|
+
totalCacheWriteTokens += response.usage?.cacheWriteInputTokens ?? 0;
|
|
268
274
|
if (response.stopReason) {
|
|
269
275
|
lastFinishReason = response.stopReason;
|
|
270
276
|
}
|
|
@@ -281,7 +287,18 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
281
287
|
usage: {
|
|
282
288
|
input: totalInputTokens,
|
|
283
289
|
output: totalOutputTokens,
|
|
284
|
-
|
|
290
|
+
// Cache reads/writes are billed tokens reported separately
|
|
291
|
+
// from inputTokens — the total must include them.
|
|
292
|
+
total: totalInputTokens +
|
|
293
|
+
totalCacheReadTokens +
|
|
294
|
+
totalCacheWriteTokens +
|
|
295
|
+
totalOutputTokens,
|
|
296
|
+
...(totalCacheReadTokens > 0 && {
|
|
297
|
+
cacheReadTokens: totalCacheReadTokens,
|
|
298
|
+
}),
|
|
299
|
+
...(totalCacheWriteTokens > 0 && {
|
|
300
|
+
cacheCreationTokens: totalCacheWriteTokens,
|
|
301
|
+
}),
|
|
285
302
|
},
|
|
286
303
|
finishReason: lastFinishReason,
|
|
287
304
|
};
|
|
@@ -387,13 +404,26 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
387
404
|
const totalDuration = Date.now() - startTime;
|
|
388
405
|
logger.info(`[AmazonBedrockProvider] Total callBedrock duration: ${totalDuration}ms`);
|
|
389
406
|
generateSpan.setAttribute("gen_ai.response.stop_reason", response.stopReason ?? "");
|
|
390
|
-
|
|
407
|
+
const spanCacheRead = response.usage?.cacheReadInputTokens ?? 0;
|
|
408
|
+
const spanCacheWrite = response.usage?.cacheWriteInputTokens ?? 0;
|
|
409
|
+
// Converse's inputTokens is only the UNCACHED remainder — the span
|
|
410
|
+
// attribute reports the FULL prompt (uncached + cache read/write)
|
|
411
|
+
// so telemetry matches the cache-inclusive pricing inputs below.
|
|
412
|
+
generateSpan.setAttribute("gen_ai.usage.input_tokens", (response.usage?.inputTokens ?? 0) + spanCacheRead + spanCacheWrite);
|
|
413
|
+
generateSpan.setAttribute("gen_ai.usage.cache_read_input_tokens", spanCacheRead);
|
|
414
|
+
generateSpan.setAttribute("gen_ai.usage.cache_creation_input_tokens", spanCacheWrite);
|
|
391
415
|
generateSpan.setAttribute("gen_ai.usage.output_tokens", response.usage?.outputTokens ?? 0);
|
|
392
416
|
const cost = calculateCost(this.providerName, this.modelName, {
|
|
393
417
|
input: response.usage?.inputTokens ?? 0,
|
|
394
418
|
output: response.usage?.outputTokens ?? 0,
|
|
395
419
|
total: (response.usage?.inputTokens ?? 0) +
|
|
420
|
+
spanCacheRead +
|
|
421
|
+
spanCacheWrite +
|
|
396
422
|
(response.usage?.outputTokens ?? 0),
|
|
423
|
+
...(spanCacheRead > 0 && { cacheReadTokens: spanCacheRead }),
|
|
424
|
+
...(spanCacheWrite > 0 && {
|
|
425
|
+
cacheCreationTokens: spanCacheWrite,
|
|
426
|
+
}),
|
|
397
427
|
});
|
|
398
428
|
if (cost && cost > 0) {
|
|
399
429
|
generateSpan.setAttribute("neurolink.cost", cost);
|
|
@@ -996,7 +1026,9 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
996
1026
|
};
|
|
997
1027
|
return {
|
|
998
1028
|
stream: asyncIterable,
|
|
999
|
-
|
|
1029
|
+
// The generate() fallback already computed real usage — a
|
|
1030
|
+
// hardcoded zero object here threw it away.
|
|
1031
|
+
usage: generateResult.usage,
|
|
1000
1032
|
model: this.modelName || this.getDefaultModel(),
|
|
1001
1033
|
provider: this.getProviderName(),
|
|
1002
1034
|
metadata: {
|
|
@@ -1025,6 +1057,8 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
1025
1057
|
// so Pipeline B (Langfuse) gets real token counts from Bedrock streams.
|
|
1026
1058
|
let streamTotalInputTokens = 0;
|
|
1027
1059
|
let streamTotalOutputTokens = 0;
|
|
1060
|
+
let streamTotalCacheReadTokens = 0;
|
|
1061
|
+
let streamTotalCacheWriteTokens = 0;
|
|
1028
1062
|
let streamLastStopReason;
|
|
1029
1063
|
// The REAL issue: ReadableStream errors don't bubble up to the caller
|
|
1030
1064
|
// So we need to make the first streaming call synchronously to test permissions
|
|
@@ -1130,6 +1164,12 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
1130
1164
|
chunk.metadata.usage.inputTokens ?? 0;
|
|
1131
1165
|
streamTotalOutputTokens +=
|
|
1132
1166
|
chunk.metadata.usage.outputTokens ?? 0;
|
|
1167
|
+
// inputTokens excludes cache reads/writes (Converse follows
|
|
1168
|
+
// the Anthropic additive convention) — track them too.
|
|
1169
|
+
streamTotalCacheReadTokens +=
|
|
1170
|
+
chunk.metadata.usage.cacheReadInputTokens ?? 0;
|
|
1171
|
+
streamTotalCacheWriteTokens +=
|
|
1172
|
+
chunk.metadata.usage.cacheWriteInputTokens ?? 0;
|
|
1133
1173
|
// Stream is effectively complete after metadata chunk
|
|
1134
1174
|
break;
|
|
1135
1175
|
}
|
|
@@ -1177,6 +1217,8 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
1177
1217
|
if (usage) {
|
|
1178
1218
|
streamTotalInputTokens += usage.input;
|
|
1179
1219
|
streamTotalOutputTokens += usage.output;
|
|
1220
|
+
streamTotalCacheReadTokens += usage.cacheReadTokens ?? 0;
|
|
1221
|
+
streamTotalCacheWriteTokens += usage.cacheCreationTokens ?? 0;
|
|
1180
1222
|
}
|
|
1181
1223
|
if (stopReason) {
|
|
1182
1224
|
streamLastStopReason = stopReason;
|
|
@@ -1248,7 +1290,18 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
1248
1290
|
const aggregatedUsage = {
|
|
1249
1291
|
input: streamTotalInputTokens,
|
|
1250
1292
|
output: streamTotalOutputTokens,
|
|
1251
|
-
|
|
1293
|
+
// Cache reads/writes are billed tokens reported separately
|
|
1294
|
+
// from inputTokens — the total must include them.
|
|
1295
|
+
total: streamTotalInputTokens +
|
|
1296
|
+
streamTotalCacheReadTokens +
|
|
1297
|
+
streamTotalCacheWriteTokens +
|
|
1298
|
+
streamTotalOutputTokens,
|
|
1299
|
+
...(streamTotalCacheReadTokens > 0 && {
|
|
1300
|
+
cacheReadTokens: streamTotalCacheReadTokens,
|
|
1301
|
+
}),
|
|
1302
|
+
...(streamTotalCacheWriteTokens > 0 && {
|
|
1303
|
+
cacheCreationTokens: streamTotalCacheWriteTokens,
|
|
1304
|
+
}),
|
|
1252
1305
|
};
|
|
1253
1306
|
// Resolve analytics with accumulated token counts from Bedrock
|
|
1254
1307
|
// metadata chunks so Pipeline A also reports real usage.
|
|
@@ -1276,7 +1329,9 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
1276
1329
|
};
|
|
1277
1330
|
return {
|
|
1278
1331
|
stream: wrappedStreamIterable,
|
|
1279
|
-
usage
|
|
1332
|
+
// No usage key here on purpose: the real aggregate resolves through
|
|
1333
|
+
// `analytics` after the stream drains. A literal zero object is
|
|
1334
|
+
// truthy and would block every downstream usage fallback.
|
|
1280
1335
|
model: this.modelName || this.getDefaultModel(),
|
|
1281
1336
|
provider: this.getProviderName(),
|
|
1282
1337
|
analytics: analyticsPromise,
|
|
@@ -1468,10 +1523,17 @@ export class AmazonBedrockProvider extends BaseProvider {
|
|
|
1468
1523
|
if (chunk.metadata?.usage) {
|
|
1469
1524
|
const input = chunk.metadata.usage.inputTokens ?? 0;
|
|
1470
1525
|
const output = chunk.metadata.usage.outputTokens ?? 0;
|
|
1526
|
+
const cacheRead = chunk.metadata.usage.cacheReadInputTokens ?? 0;
|
|
1527
|
+
const cacheWrite = chunk.metadata.usage.cacheWriteInputTokens ?? 0;
|
|
1471
1528
|
streamUsage = {
|
|
1472
1529
|
input,
|
|
1473
1530
|
output,
|
|
1474
|
-
|
|
1531
|
+
// Computed rather than trusting totalTokens: inputTokens excludes
|
|
1532
|
+
// cache reads/writes (additive convention), and the total must
|
|
1533
|
+
// count every billed component.
|
|
1534
|
+
total: input + cacheRead + cacheWrite + output,
|
|
1535
|
+
...(cacheRead > 0 && { cacheReadTokens: cacheRead }),
|
|
1536
|
+
...(cacheWrite > 0 && { cacheCreationTokens: cacheWrite }),
|
|
1475
1537
|
};
|
|
1476
1538
|
// Stream is effectively complete after metadata chunk
|
|
1477
1539
|
break;
|
|
@@ -7,6 +7,7 @@ import { ANTHROPIC_TOKEN_URL, CLAUDE_CLI_USER_AGENT, CLAUDE_CODE_CLIENT_ID, CLAU
|
|
|
7
7
|
import { AnthropicModels, TOKEN_EXPIRY_BUFFER_MS, } from "../constants/enums.js";
|
|
8
8
|
import { BaseProvider } from "../core/baseProvider.js";
|
|
9
9
|
import { DEFAULT_MAX_STEPS } from "../core/constants.js";
|
|
10
|
+
import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
|
|
10
11
|
import { getModelCapabilities, getRecommendedModelForTier, isModelAvailableForTier, } from "../models/anthropicModels.js";
|
|
11
12
|
import { createOAuthFetch } from "../proxy/oauthFetch.js";
|
|
12
13
|
import { createProxyFetch } from "../proxy/proxyFetch.js";
|
|
@@ -1408,13 +1409,24 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1408
1409
|
let capturedProviderError;
|
|
1409
1410
|
const client = this.client;
|
|
1410
1411
|
const toolsUsed = [];
|
|
1412
|
+
const streamStartTime = Date.now();
|
|
1413
|
+
// Hoisted out of runLoop so the error path can resolve the usage
|
|
1414
|
+
// accumulated by steps that completed BEFORE the failure — those steps
|
|
1415
|
+
// were billed and must not be reported as zero.
|
|
1416
|
+
let totalInput = 0;
|
|
1417
|
+
let totalOutput = 0;
|
|
1418
|
+
let totalCacheRead = 0;
|
|
1419
|
+
let totalCacheWrite = 0;
|
|
1420
|
+
let lastStop = null;
|
|
1421
|
+
const buildDeferredUsage = () => ({
|
|
1422
|
+
promptTokens: totalInput,
|
|
1423
|
+
completionTokens: totalOutput,
|
|
1424
|
+
totalTokens: totalInput + totalCacheRead + totalCacheWrite + totalOutput,
|
|
1425
|
+
...(totalCacheRead > 0 ? { cacheReadTokens: totalCacheRead } : {}),
|
|
1426
|
+
...(totalCacheWrite > 0 ? { cacheCreationTokens: totalCacheWrite } : {}),
|
|
1427
|
+
});
|
|
1411
1428
|
const runLoop = async () => {
|
|
1412
1429
|
const conversation = payload.messages.slice();
|
|
1413
|
-
let totalInput = 0;
|
|
1414
|
-
let totalOutput = 0;
|
|
1415
|
-
let totalCacheRead = 0;
|
|
1416
|
-
let totalCacheWrite = 0;
|
|
1417
|
-
let lastStop = null;
|
|
1418
1430
|
for (let step = 0; step < maxSteps; step++) {
|
|
1419
1431
|
// Mid-turn discovery sync: search_tools (tools.discovery) hydrates
|
|
1420
1432
|
// new tools into toolsRecord between steps; Claude only calls tools
|
|
@@ -1471,10 +1483,20 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1471
1483
|
const thinkingAcc = new Map();
|
|
1472
1484
|
const toolAcc = new Map();
|
|
1473
1485
|
let stopReason = null;
|
|
1486
|
+
// message_start carries a small output placeholder and message_delta
|
|
1487
|
+
// reports the CUMULATIVE output for the message — latest wins within
|
|
1488
|
+
// the step (adding both double-counted the placeholder every step).
|
|
1489
|
+
// Write-through: each event folds only the DELTA over this step's
|
|
1490
|
+
// previous value into totalOutput, so the total is correct at every
|
|
1491
|
+
// point mid-drain — a step killed mid-stream (abort/timeout) still
|
|
1492
|
+
// counts the billed output it already reported.
|
|
1493
|
+
let stepOutputTokens = 0;
|
|
1474
1494
|
for await (const event of events) {
|
|
1475
1495
|
if (event.type === "message_start") {
|
|
1476
1496
|
totalInput += event.message.usage.input_tokens ?? 0;
|
|
1477
|
-
|
|
1497
|
+
const startOutputTokens = event.message.usage.output_tokens ?? 0;
|
|
1498
|
+
totalOutput += startOutputTokens - stepOutputTokens;
|
|
1499
|
+
stepOutputTokens = startOutputTokens;
|
|
1478
1500
|
// Anthropic reports cache reads/writes SEPARATELY from
|
|
1479
1501
|
// input_tokens on the same message_start event — without these
|
|
1480
1502
|
// the streaming path silently drops all cache accounting.
|
|
@@ -1526,7 +1548,9 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1526
1548
|
}
|
|
1527
1549
|
else if (event.type === "message_delta") {
|
|
1528
1550
|
stopReason = event.delta.stop_reason ?? stopReason;
|
|
1529
|
-
|
|
1551
|
+
const cumulativeOutputTokens = event.usage?.output_tokens ?? stepOutputTokens;
|
|
1552
|
+
totalOutput += cumulativeOutputTokens - stepOutputTokens;
|
|
1553
|
+
stepOutputTokens = cumulativeOutputTokens;
|
|
1530
1554
|
}
|
|
1531
1555
|
}
|
|
1532
1556
|
lastStop = stopReason;
|
|
@@ -1654,15 +1678,7 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1654
1678
|
});
|
|
1655
1679
|
conversation.push({ role: "user", content: resultBlocks });
|
|
1656
1680
|
}
|
|
1657
|
-
resolveUsage(
|
|
1658
|
-
promptTokens: totalInput,
|
|
1659
|
-
completionTokens: totalOutput,
|
|
1660
|
-
totalTokens: totalInput + totalCacheRead + totalCacheWrite + totalOutput,
|
|
1661
|
-
...(totalCacheRead > 0 ? { cacheReadTokens: totalCacheRead } : {}),
|
|
1662
|
-
...(totalCacheWrite > 0
|
|
1663
|
-
? { cacheCreationTokens: totalCacheWrite }
|
|
1664
|
-
: {}),
|
|
1665
|
-
});
|
|
1681
|
+
resolveUsage(buildDeferredUsage());
|
|
1666
1682
|
resolveFinish(lastStop ?? "stop");
|
|
1667
1683
|
};
|
|
1668
1684
|
const loopPromise = runLoop()
|
|
@@ -1673,6 +1689,9 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1673
1689
|
logger.error("Anthropic: Stream error", {
|
|
1674
1690
|
error: error instanceof Error ? error.message : String(error),
|
|
1675
1691
|
});
|
|
1692
|
+
// Report whatever the completed steps accumulated — they were billed
|
|
1693
|
+
// — and unblock any consumer awaiting the usage promise.
|
|
1694
|
+
resolveUsage(buildDeferredUsage());
|
|
1676
1695
|
resolveFinish("error");
|
|
1677
1696
|
throw this.formatProviderError(error);
|
|
1678
1697
|
})
|
|
@@ -1734,6 +1753,21 @@ export class AnthropicProvider extends BaseProvider {
|
|
|
1734
1753
|
model: this.modelName,
|
|
1735
1754
|
toolCalls: [],
|
|
1736
1755
|
toolResults: [],
|
|
1756
|
+
// Wire the deferred usage/finish promises into the analytics collector
|
|
1757
|
+
// (mirrors openaiChatCompletionsBase). Without this the loop computed a
|
|
1758
|
+
// fully correct aggregate that was consumed only by the OTel span —
|
|
1759
|
+
// stream consumers and session cost tracking saw no usage at all.
|
|
1760
|
+
// Chained off finishPromise so requestDuration reflects the DRAINED
|
|
1761
|
+
// stream, not the milliseconds it took to construct this result object.
|
|
1762
|
+
analytics: finishPromise.then(() => streamAnalyticsCollector.createAnalytics(this.providerName, modelId, {
|
|
1763
|
+
textStream: (async function* () { })(),
|
|
1764
|
+
usage: usagePromise,
|
|
1765
|
+
finishReason: finishPromise,
|
|
1766
|
+
}, Date.now() - streamStartTime, {
|
|
1767
|
+
requestId: options.requestId ??
|
|
1768
|
+
`${this.providerName}-stream-${Date.now()}`,
|
|
1769
|
+
streamingMode: true,
|
|
1770
|
+
})),
|
|
1737
1771
|
};
|
|
1738
1772
|
}
|
|
1739
1773
|
async isAvailable() {
|
|
@@ -591,6 +591,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
591
591
|
let totalInputTokens = 0;
|
|
592
592
|
let totalOutputTokens = 0;
|
|
593
593
|
let totalCacheReadTokens = 0;
|
|
594
|
+
let totalReasoningTokens = 0;
|
|
594
595
|
let step = 0;
|
|
595
596
|
let completedWithFinalAnswer = false;
|
|
596
597
|
const failedTools = new Map();
|
|
@@ -627,6 +628,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
627
628
|
totalInputTokens += chunkResult.inputTokens;
|
|
628
629
|
totalOutputTokens += chunkResult.outputTokens;
|
|
629
630
|
totalCacheReadTokens += chunkResult.cacheReadTokens ?? 0;
|
|
631
|
+
totalReasoningTokens += chunkResult.reasoningTokens ?? 0;
|
|
630
632
|
const stepText = extractTextFromParts(chunkResult.rawResponseParts);
|
|
631
633
|
// If no function calls, this was the final step — channel
|
|
632
634
|
// already received all text parts incrementally.
|
|
@@ -723,13 +725,21 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
723
725
|
model: modelName,
|
|
724
726
|
tokenUsage: {
|
|
725
727
|
input: adjustedInputTokens,
|
|
726
|
-
output
|
|
728
|
+
// Thinking tokens are billed at the output rate but Gemini
|
|
729
|
+
// does NOT include them in candidatesTokenCount, so they
|
|
730
|
+
// are folded into `output` — what calculateCost bills at
|
|
731
|
+
// the output rate — with `reasoning` as the subset.
|
|
732
|
+
output: totalOutputTokens + totalReasoningTokens,
|
|
727
733
|
total: adjustedInputTokens +
|
|
728
734
|
totalCacheReadTokens +
|
|
729
|
-
totalOutputTokens
|
|
735
|
+
totalOutputTokens +
|
|
736
|
+
totalReasoningTokens,
|
|
730
737
|
...(totalCacheReadTokens > 0
|
|
731
738
|
? { cacheReadTokens: totalCacheReadTokens }
|
|
732
739
|
: {}),
|
|
740
|
+
...(totalReasoningTokens > 0
|
|
741
|
+
? { reasoning: totalReasoningTokens }
|
|
742
|
+
: {}),
|
|
733
743
|
},
|
|
734
744
|
requestDuration: responseTime,
|
|
735
745
|
timestamp: new Date().toISOString(),
|
|
@@ -862,6 +872,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
862
872
|
let totalInputTokens = 0;
|
|
863
873
|
let totalOutputTokens = 0;
|
|
864
874
|
let totalCacheReadTokens = 0;
|
|
875
|
+
let totalReasoningTokens = 0;
|
|
865
876
|
const allToolCalls = [];
|
|
866
877
|
const toolExecutions = [];
|
|
867
878
|
let step = 0;
|
|
@@ -892,6 +903,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
892
903
|
totalInputTokens += chunkResult.inputTokens;
|
|
893
904
|
totalOutputTokens += chunkResult.outputTokens;
|
|
894
905
|
totalCacheReadTokens += chunkResult.cacheReadTokens ?? 0;
|
|
906
|
+
totalReasoningTokens += chunkResult.reasoningTokens ?? 0;
|
|
895
907
|
const stepText = extractTextFromParts(chunkResult.rawResponseParts);
|
|
896
908
|
// If no function calls, we're done
|
|
897
909
|
if (chunkResult.stepFunctionCalls.length === 0) {
|
|
@@ -975,12 +987,25 @@ export class GoogleAIStudioProvider extends BaseProvider {
|
|
|
975
987
|
model: modelName,
|
|
976
988
|
usage: {
|
|
977
989
|
input: adjustedInputTokens,
|
|
978
|
-
output
|
|
979
|
-
|
|
990
|
+
// Thinking tokens are billed at the output rate but Gemini
|
|
991
|
+
// does NOT include them in candidatesTokenCount, so they are
|
|
992
|
+
// folded into `output` — what calculateCost bills at the
|
|
993
|
+
// output rate — with `reasoning` as the subset.
|
|
994
|
+
output: totalOutputTokens + totalReasoningTokens,
|
|
995
|
+
total: adjustedInputTokens +
|
|
996
|
+
totalCacheReadTokens +
|
|
997
|
+
totalOutputTokens +
|
|
998
|
+
totalReasoningTokens,
|
|
980
999
|
...(totalCacheReadTokens > 0
|
|
981
1000
|
? { cacheReadTokens: totalCacheReadTokens }
|
|
982
1001
|
: {}),
|
|
1002
|
+
...(totalReasoningTokens > 0
|
|
1003
|
+
? { reasoning: totalReasoningTokens }
|
|
1004
|
+
: {}),
|
|
983
1005
|
},
|
|
1006
|
+
...(totalReasoningTokens > 0 && {
|
|
1007
|
+
reasoningTokens: totalReasoningTokens,
|
|
1008
|
+
}),
|
|
984
1009
|
responseTime,
|
|
985
1010
|
toolsUsed: allToolCalls.map((tc) => tc.toolName),
|
|
986
1011
|
toolExecutions: resolveToolExecutionRecords(options, toolExecutions),
|
|
@@ -601,6 +601,7 @@ export async function collectStreamChunks(stream) {
|
|
|
601
601
|
let inputTokens = 0;
|
|
602
602
|
let outputTokens = 0;
|
|
603
603
|
let cacheReadTokens = 0;
|
|
604
|
+
let reasoningTokens = 0;
|
|
604
605
|
for await (const chunk of stream) {
|
|
605
606
|
// Extract raw parts from candidates FIRST
|
|
606
607
|
// This avoids using chunk.text which triggers SDK warning when
|
|
@@ -623,6 +624,9 @@ export async function collectStreamChunks(stream) {
|
|
|
623
624
|
// cachedContentTokenCount is OVERLAPPING (a subset already inside
|
|
624
625
|
// promptTokenCount). Surface it so the call site subtracts once.
|
|
625
626
|
cacheReadTokens = Math.max(cacheReadTokens, usage.cachedContentTokenCount || 0);
|
|
627
|
+
// thoughtsTokenCount (thinking tokens, billed at the output rate) is
|
|
628
|
+
// NOT part of candidatesTokenCount — surface it separately.
|
|
629
|
+
reasoningTokens = Math.max(reasoningTokens, usage.thoughtsTokenCount || 0);
|
|
626
630
|
}
|
|
627
631
|
}
|
|
628
632
|
return {
|
|
@@ -631,6 +635,7 @@ export async function collectStreamChunks(stream) {
|
|
|
631
635
|
inputTokens,
|
|
632
636
|
outputTokens,
|
|
633
637
|
cacheReadTokens,
|
|
638
|
+
reasoningTokens,
|
|
634
639
|
};
|
|
635
640
|
}
|
|
636
641
|
/**
|
|
@@ -724,6 +729,7 @@ export async function collectStreamChunksIncremental(stream, channel) {
|
|
|
724
729
|
let inputTokens = 0;
|
|
725
730
|
let outputTokens = 0;
|
|
726
731
|
let cacheReadTokens = 0;
|
|
732
|
+
let reasoningTokens = 0;
|
|
727
733
|
for await (const chunk of stream) {
|
|
728
734
|
const chunkRecord = chunk;
|
|
729
735
|
const candidates = chunkRecord.candidates;
|
|
@@ -748,6 +754,9 @@ export async function collectStreamChunksIncremental(stream, channel) {
|
|
|
748
754
|
// cachedContentTokenCount is OVERLAPPING (a subset already inside
|
|
749
755
|
// promptTokenCount). Surface it so the call site subtracts once.
|
|
750
756
|
cacheReadTokens = Math.max(cacheReadTokens, usage.cachedContentTokenCount || 0);
|
|
757
|
+
// thoughtsTokenCount (thinking tokens, billed at the output rate) is
|
|
758
|
+
// NOT part of candidatesTokenCount — surface it separately.
|
|
759
|
+
reasoningTokens = Math.max(reasoningTokens, usage.thoughtsTokenCount || 0);
|
|
751
760
|
}
|
|
752
761
|
}
|
|
753
762
|
return {
|
|
@@ -756,6 +765,7 @@ export async function collectStreamChunksIncremental(stream, channel) {
|
|
|
756
765
|
inputTokens,
|
|
757
766
|
outputTokens,
|
|
758
767
|
cacheReadTokens,
|
|
768
|
+
reasoningTokens,
|
|
759
769
|
};
|
|
760
770
|
}
|
|
761
771
|
/**
|