@librechat/agents 3.3.3 → 3.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/dist/cjs/agents/AgentContext.cjs +2 -1
  2. package/dist/cjs/agents/AgentContext.cjs.map +1 -1
  3. package/dist/cjs/graphs/Graph.cjs +342 -109
  4. package/dist/cjs/graphs/Graph.cjs.map +1 -1
  5. package/dist/cjs/graphs/MultiAgentGraph.cjs +3 -1
  6. package/dist/cjs/graphs/MultiAgentGraph.cjs.map +1 -1
  7. package/dist/cjs/llm/anthropic/utils/message_inputs.cjs +79 -6
  8. package/dist/cjs/llm/anthropic/utils/message_inputs.cjs.map +1 -1
  9. package/dist/cjs/llm/bedrock/utils/message_inputs.cjs +37 -10
  10. package/dist/cjs/llm/bedrock/utils/message_inputs.cjs.map +1 -1
  11. package/dist/cjs/llm/google/utils/common.cjs +19 -7
  12. package/dist/cjs/llm/google/utils/common.cjs.map +1 -1
  13. package/dist/cjs/llm/invoke.cjs +64 -3
  14. package/dist/cjs/llm/invoke.cjs.map +1 -1
  15. package/dist/cjs/llm/openai/index.cjs +41 -5
  16. package/dist/cjs/llm/openai/index.cjs.map +1 -1
  17. package/dist/cjs/llm/openai/utils/index.cjs +25 -4
  18. package/dist/cjs/llm/openai/utils/index.cjs.map +1 -1
  19. package/dist/cjs/llm/openrouter/index.cjs +4 -1
  20. package/dist/cjs/llm/openrouter/index.cjs.map +1 -1
  21. package/dist/cjs/main.cjs +19 -2
  22. package/dist/cjs/messages/cache.cjs +54 -33
  23. package/dist/cjs/messages/cache.cjs.map +1 -1
  24. package/dist/cjs/messages/contextPruning.cjs +17 -43
  25. package/dist/cjs/messages/contextPruning.cjs.map +1 -1
  26. package/dist/cjs/messages/core.cjs +315 -23
  27. package/dist/cjs/messages/core.cjs.map +1 -1
  28. package/dist/cjs/messages/format.cjs +218 -60
  29. package/dist/cjs/messages/format.cjs.map +1 -1
  30. package/dist/cjs/messages/index.cjs +1 -1
  31. package/dist/cjs/messages/prune.cjs +503 -127
  32. package/dist/cjs/messages/prune.cjs.map +1 -1
  33. package/dist/cjs/stream.cjs +7 -3
  34. package/dist/cjs/stream.cjs.map +1 -1
  35. package/dist/cjs/summarization/node.cjs +28 -14
  36. package/dist/cjs/summarization/node.cjs.map +1 -1
  37. package/dist/cjs/tools/ToolNode.cjs +54 -41
  38. package/dist/cjs/tools/ToolNode.cjs.map +1 -1
  39. package/dist/cjs/tools/toolOutputReferences.cjs +6 -0
  40. package/dist/cjs/tools/toolOutputReferences.cjs.map +1 -1
  41. package/dist/cjs/utils/index.cjs +1 -1
  42. package/dist/cjs/utils/llm.cjs +1 -1
  43. package/dist/cjs/utils/llm.cjs.map +1 -1
  44. package/dist/cjs/utils/tokens.cjs +307 -20
  45. package/dist/cjs/utils/tokens.cjs.map +1 -1
  46. package/dist/cjs/utils/toolContent.cjs +1514 -0
  47. package/dist/cjs/utils/toolContent.cjs.map +1 -0
  48. package/dist/esm/agents/AgentContext.mjs +2 -1
  49. package/dist/esm/agents/AgentContext.mjs.map +1 -1
  50. package/dist/esm/graphs/Graph.mjs +349 -116
  51. package/dist/esm/graphs/Graph.mjs.map +1 -1
  52. package/dist/esm/graphs/MultiAgentGraph.mjs +3 -1
  53. package/dist/esm/graphs/MultiAgentGraph.mjs.map +1 -1
  54. package/dist/esm/llm/anthropic/utils/message_inputs.mjs +79 -6
  55. package/dist/esm/llm/anthropic/utils/message_inputs.mjs.map +1 -1
  56. package/dist/esm/llm/bedrock/utils/message_inputs.mjs +37 -10
  57. package/dist/esm/llm/bedrock/utils/message_inputs.mjs.map +1 -1
  58. package/dist/esm/llm/google/utils/common.mjs +19 -7
  59. package/dist/esm/llm/google/utils/common.mjs.map +1 -1
  60. package/dist/esm/llm/invoke.mjs +65 -5
  61. package/dist/esm/llm/invoke.mjs.map +1 -1
  62. package/dist/esm/llm/openai/index.mjs +41 -5
  63. package/dist/esm/llm/openai/index.mjs.map +1 -1
  64. package/dist/esm/llm/openai/utils/index.mjs +25 -4
  65. package/dist/esm/llm/openai/utils/index.mjs.map +1 -1
  66. package/dist/esm/llm/openrouter/index.mjs +4 -1
  67. package/dist/esm/llm/openrouter/index.mjs.map +1 -1
  68. package/dist/esm/main.mjs +7 -7
  69. package/dist/esm/messages/cache.mjs +55 -34
  70. package/dist/esm/messages/cache.mjs.map +1 -1
  71. package/dist/esm/messages/contextPruning.mjs +17 -43
  72. package/dist/esm/messages/contextPruning.mjs.map +1 -1
  73. package/dist/esm/messages/core.mjs +305 -24
  74. package/dist/esm/messages/core.mjs.map +1 -1
  75. package/dist/esm/messages/format.mjs +218 -61
  76. package/dist/esm/messages/format.mjs.map +1 -1
  77. package/dist/esm/messages/index.mjs +1 -1
  78. package/dist/esm/messages/prune.mjs +500 -128
  79. package/dist/esm/messages/prune.mjs.map +1 -1
  80. package/dist/esm/stream.mjs +7 -3
  81. package/dist/esm/stream.mjs.map +1 -1
  82. package/dist/esm/summarization/node.mjs +28 -14
  83. package/dist/esm/summarization/node.mjs.map +1 -1
  84. package/dist/esm/tools/ToolNode.mjs +54 -41
  85. package/dist/esm/tools/ToolNode.mjs.map +1 -1
  86. package/dist/esm/tools/toolOutputReferences.mjs +6 -0
  87. package/dist/esm/tools/toolOutputReferences.mjs.map +1 -1
  88. package/dist/esm/utils/index.mjs +1 -1
  89. package/dist/esm/utils/llm.mjs +1 -1
  90. package/dist/esm/utils/llm.mjs.map +1 -1
  91. package/dist/esm/utils/tokens.mjs +307 -21
  92. package/dist/esm/utils/tokens.mjs.map +1 -1
  93. package/dist/esm/utils/toolContent.mjs +1503 -0
  94. package/dist/esm/utils/toolContent.mjs.map +1 -0
  95. package/dist/types/graphs/Graph.d.ts +1 -1
  96. package/dist/types/llm/invoke.d.ts +27 -1
  97. package/dist/types/llm/openai/index.d.ts +4 -0
  98. package/dist/types/llm/openai/utils/index.d.ts +2 -0
  99. package/dist/types/llm/openrouter/index.d.ts +5 -0
  100. package/dist/types/messages/contextPruning.d.ts +2 -1
  101. package/dist/types/messages/core.d.ts +36 -0
  102. package/dist/types/messages/format.d.ts +6 -0
  103. package/dist/types/messages/prune.d.ts +16 -12
  104. package/dist/types/types/stream.d.ts +1 -1
  105. package/dist/types/utils/tokens.d.ts +7 -0
  106. package/dist/types/utils/toolContent.d.ts +107 -0
  107. package/package.json +1 -1
  108. package/src/__tests__/stream.eagerEventExecution.test.ts +74 -0
  109. package/src/agents/AgentContext.ts +1 -0
  110. package/src/graphs/Graph.ts +698 -200
  111. package/src/graphs/MultiAgentGraph.ts +6 -1
  112. package/src/graphs/__tests__/Graph.contextOverflow.test.ts +1128 -4
  113. package/src/llm/anthropic/utils/message_inputs.ts +158 -12
  114. package/src/llm/anthropic/utils/tool-id-normalization.test.ts +109 -0
  115. package/src/llm/bedrock/utils/message_inputs.test.ts +49 -1
  116. package/src/llm/bedrock/utils/message_inputs.ts +86 -13
  117. package/src/llm/google/utils/common.test.ts +75 -9
  118. package/src/llm/google/utils/common.ts +34 -3
  119. package/src/llm/invoke.test.ts +488 -0
  120. package/src/llm/invoke.ts +196 -2
  121. package/src/llm/openai/index.ts +106 -6
  122. package/src/llm/openai/utils/index.ts +109 -53
  123. package/src/llm/openai/utils/messages.test.ts +330 -1
  124. package/src/llm/openrouter/index.ts +12 -2
  125. package/src/llm/openrouter/reasoning.test.ts +314 -0
  126. package/src/messages/__tests__/observationMasking.test.ts +93 -2
  127. package/src/messages/cache.tail.test.ts +193 -0
  128. package/src/messages/cache.test.ts +113 -0
  129. package/src/messages/cache.ts +92 -60
  130. package/src/messages/contextPruning.test.ts +184 -0
  131. package/src/messages/contextPruning.ts +49 -42
  132. package/src/messages/core.ts +653 -40
  133. package/src/messages/ensureThinkingBlock.test.ts +49 -1
  134. package/src/messages/foldToollessToolBlocks.test.ts +163 -5
  135. package/src/messages/format.ts +363 -91
  136. package/src/messages/formatAgentMessages.test.ts +604 -14
  137. package/src/messages/formatAgentMessages.tools.test.ts +103 -0
  138. package/src/messages/prune.ts +996 -183
  139. package/src/specs/prune.test.ts +1083 -6
  140. package/src/specs/summarization-unit.test.ts +105 -0
  141. package/src/specs/tokens.test.ts +609 -32
  142. package/src/stream.ts +23 -15
  143. package/src/summarization/__tests__/node.test.ts +77 -0
  144. package/src/summarization/node.ts +46 -18
  145. package/src/tools/ToolNode.ts +147 -68
  146. package/src/tools/__tests__/ToolNode.onResultCompletion.test.ts +148 -1
  147. package/src/tools/__tests__/ToolNode.outputReferences.test.ts +112 -3
  148. package/src/tools/__tests__/annotateMessagesForLLM.test.ts +32 -0
  149. package/src/tools/__tests__/directToolHooks.test.ts +255 -7
  150. package/src/tools/toolOutputReferences.ts +6 -0
  151. package/src/types/stream.ts +1 -1
  152. package/src/utils/llm.test.ts +18 -0
  153. package/src/utils/llm.ts +4 -1
  154. package/src/utils/tokens.ts +586 -31
  155. package/src/utils/toolContent.test.ts +1432 -0
  156. package/src/utils/toolContent.ts +2707 -0
@@ -1,7 +1,14 @@
1
- import { HumanMessage } from '@langchain/core/messages';
1
+ import {
2
+ AIMessage,
3
+ ChatMessage,
4
+ HumanMessage,
5
+ ToolMessage,
6
+ } from '@langchain/core/messages';
2
7
  import {
3
8
  encodingForModel,
4
9
  createTokenCounter,
10
+ getTokenCountForMessage,
11
+ hasUnsafeStructuredSerialization,
5
12
  TokenEncoderManager,
6
13
  estimateImageBlockTokens,
7
14
  estimateDocumentBlockTokens,
@@ -82,15 +89,507 @@ describe('createTokenCounter with different encodings', () => {
82
89
  });
83
90
  });
84
91
 
92
+ describe('getTokenCountForMessage', () => {
93
+ const messageWithToolArgs = (args: unknown): AIMessage =>
94
+ ({
95
+ content: '',
96
+ tool_calls: [{ id: 'adversarial-call', name: 'stress', args }],
97
+ getType: () => 'ai',
98
+ }) as unknown as AIMessage;
99
+
100
+ test('bounds direct message strings before tokenization and charges omitted characters', () => {
101
+ const callbackLengths: number[] = [];
102
+ const content = 'x'.repeat(300_000);
103
+
104
+ const count = getTokenCountForMessage(new HumanMessage(content), (text) => {
105
+ callbackLengths.push(text.length);
106
+ return text.length;
107
+ });
108
+
109
+ expect(Math.max(...callbackLengths)).toBe(200_000);
110
+ expect(count).toBe(600_003);
111
+ });
112
+
113
+ test('bounds direct string tool args before tokenization and charges omitted characters', () => {
114
+ const callbackLengths: number[] = [];
115
+ const args = 'x'.repeat(300_000);
116
+
117
+ const count = getTokenCountForMessage(messageWithToolArgs(args), (text) => {
118
+ callbackLengths.push(text.length);
119
+ return text.length;
120
+ });
121
+
122
+ expect(Math.max(...callbackLengths)).toBe(200_000);
123
+ expect(count).toBeGreaterThan(args.length);
124
+ });
125
+
126
+ test('counts legacy function_call arguments when parsed tool_calls are absent', () => {
127
+ const message = new AIMessage({
128
+ content: '',
129
+ additional_kwargs: {
130
+ function_call: {
131
+ name: 'legacy_lookup',
132
+ arguments: `{"query":"${'x'.repeat(5_000)}"}`,
133
+ },
134
+ },
135
+ });
136
+
137
+ const count = getTokenCountForMessage(message, (text) => text.length);
138
+
139
+ expect(message.tool_calls).toHaveLength(0);
140
+ expect(count).toBeGreaterThan(5_000);
141
+ });
142
+
143
+ test('keeps nested dense strings on the bounded structured path', () => {
144
+ const callbackLengths: number[] = [];
145
+ const payload = 'x'.repeat(4 * 1024 * 1024);
146
+
147
+ const count = getTokenCountForMessage(
148
+ messageWithToolArgs({ payload }),
149
+ (text) => {
150
+ callbackLengths.push(text.length);
151
+ return text.length;
152
+ }
153
+ );
154
+
155
+ expect(Math.max(...callbackLengths)).toBeLessThanOrEqual(200_000);
156
+ expect(count).toBeGreaterThan(payload.length);
157
+ });
158
+
159
+ test('fails closed when prototype traps throw', () => {
160
+ let prototypeCalls = 0;
161
+ const args = new Proxy(
162
+ { safe: true },
163
+ {
164
+ getPrototypeOf() {
165
+ prototypeCalls++;
166
+ throw new Error('prototype denied');
167
+ },
168
+ }
169
+ );
170
+
171
+ expect(hasUnsafeStructuredSerialization(args)).toBe(true);
172
+ expect(
173
+ getTokenCountForMessage(messageWithToolArgs(args), (text) => text.length)
174
+ ).toBe(Number.MAX_SAFE_INTEGER);
175
+ expect(prototypeCalls).toBeLessThanOrEqual(2);
176
+ });
177
+
178
+ test('fails closed without recursing through self-referential prototype proxies', () => {
179
+ let prototypeCalls = 0;
180
+ const args: Record<string, unknown> = new Proxy<Record<string, unknown>>(
181
+ {},
182
+ {
183
+ getPrototypeOf(): object | null {
184
+ prototypeCalls++;
185
+ return args;
186
+ },
187
+ }
188
+ );
189
+
190
+ expect(hasUnsafeStructuredSerialization(args)).toBe(true);
191
+ expect(
192
+ getTokenCountForMessage(messageWithToolArgs(args), (text) => text.length)
193
+ ).toBe(Number.MAX_SAFE_INTEGER);
194
+ expect(prototypeCalls).toBeLessThanOrEqual(2);
195
+ });
196
+
197
+ test('fails closed when descriptor and own-key proxy traps throw', () => {
198
+ const descriptorProxy = new Proxy(
199
+ {},
200
+ {
201
+ getOwnPropertyDescriptor() {
202
+ throw new Error('descriptor denied');
203
+ },
204
+ }
205
+ );
206
+ const ownKeysProxy = new Proxy(
207
+ {},
208
+ {
209
+ ownKeys() {
210
+ throw new Error('keys denied');
211
+ },
212
+ }
213
+ );
214
+
215
+ for (const args of [descriptorProxy, ownKeysProxy]) {
216
+ expect(hasUnsafeStructuredSerialization(args)).toBe(true);
217
+ expect(
218
+ getTokenCountForMessage(
219
+ messageWithToolArgs(args),
220
+ (text) => text.length
221
+ )
222
+ ).toBe(Number.MAX_SAFE_INTEGER);
223
+ }
224
+ });
225
+
226
+ test('fails closed on get traps and revoked proxies without invoking them', () => {
227
+ let getCalls = 0;
228
+ const getProxy = new Proxy(
229
+ {},
230
+ {
231
+ get() {
232
+ getCalls++;
233
+ return { payload: 'x'.repeat(1_000_000) };
234
+ },
235
+ }
236
+ );
237
+ const revoked = Proxy.revocable({}, {});
238
+ revoked.revoke();
239
+ const inheritedProxy = Object.create(getProxy);
240
+
241
+ for (const args of [
242
+ getProxy,
243
+ { nested: getProxy },
244
+ inheritedProxy,
245
+ revoked.proxy,
246
+ ]) {
247
+ expect(hasUnsafeStructuredSerialization(args)).toBe(true);
248
+ expect(
249
+ getTokenCountForMessage(
250
+ messageWithToolArgs(args),
251
+ (text) => text.length
252
+ )
253
+ ).toBe(Number.MAX_SAFE_INTEGER);
254
+ }
255
+ expect(getCalls).toBe(0);
256
+ });
257
+
258
+ test('fails closed on proxied content blocks without invoking their traps', () => {
259
+ let getCalls = 0;
260
+ const contentBlock = new Proxy(
261
+ { type: 'text', text: 'safe' },
262
+ {
263
+ get() {
264
+ getCalls++;
265
+ throw new Error('content read denied');
266
+ },
267
+ }
268
+ );
269
+ const message = {
270
+ content: [contentBlock],
271
+ getType: () => 'tool',
272
+ } as unknown as ToolMessage;
273
+
274
+ expect(getTokenCountForMessage(message, (text) => text.length)).toBe(
275
+ Number.MAX_SAFE_INTEGER
276
+ );
277
+ expect(getCalls).toBe(0);
278
+ });
279
+
280
+ test('detects inherited accessors without invoking them', () => {
281
+ let accessorCalls = 0;
282
+ const prototype = {};
283
+ Object.defineProperty(prototype, 'danger', {
284
+ enumerable: true,
285
+ get() {
286
+ accessorCalls++;
287
+ return 'x'.repeat(1_000_000);
288
+ },
289
+ });
290
+ const args = Object.assign(Object.create(prototype), { safe: true });
291
+
292
+ expect(hasUnsafeStructuredSerialization(args)).toBe(true);
293
+ expect(
294
+ getTokenCountForMessage(messageWithToolArgs(args), (text) => text.length)
295
+ ).toBe(Number.MAX_SAFE_INTEGER);
296
+ expect(accessorCalls).toBe(0);
297
+ });
298
+
299
+ test('counts tool_calls-only names and arguments', () => {
300
+ const message = new AIMessage({
301
+ content: '',
302
+ tool_calls: [
303
+ {
304
+ id: 'tool-call-only',
305
+ name: 'evaluate_script',
306
+ args: { code: 'x'.repeat(5_000) },
307
+ },
308
+ ],
309
+ });
310
+
311
+ const count = getTokenCountForMessage(message, (text) => text.length);
312
+
313
+ expect(count).toBeGreaterThan(5_000);
314
+ });
315
+
316
+ test('counts adversarial structured tool-call args without invoking toJSON', () => {
317
+ let toJSONCalls = 0;
318
+ const message = new AIMessage({
319
+ content: '',
320
+ tool_calls: [
321
+ {
322
+ id: 'adversarial-tool-call',
323
+ name: 'evaluate_script',
324
+ args: {
325
+ query: 'safe',
326
+ toJSON() {
327
+ toJSONCalls++;
328
+ return { code: 'y'.repeat(2_000_000) };
329
+ },
330
+ },
331
+ },
332
+ ],
333
+ });
334
+
335
+ const count = getTokenCountForMessage(message, (text) => text.length);
336
+
337
+ expect(count).toBeGreaterThanOrEqual(Number.MAX_SAFE_INTEGER);
338
+ expect(toJSONCalls).toBe(0);
339
+ });
340
+
341
+ test('retains conservative pressure when structured args exceed the preview', () => {
342
+ const message = new AIMessage({
343
+ content: '',
344
+ tool_calls: [
345
+ {
346
+ id: 'large-structured-tool-call',
347
+ name: 'evaluate_script',
348
+ args: { code: 'x'.repeat(300_000) },
349
+ },
350
+ ],
351
+ });
352
+
353
+ const count = getTokenCountForMessage(message, (text) => text.length);
354
+
355
+ expect(count).toBeGreaterThan(300_000);
356
+ });
357
+
358
+ test('counts opaque structured tool-result blocks as serialized content', () => {
359
+ const message = new ToolMessage({
360
+ content: [
361
+ {
362
+ type: 'json',
363
+ rows: Array.from({ length: 20 }, (_, index) => ({
364
+ id: index,
365
+ value: `row-${index}-${'x'.repeat(100)}`,
366
+ })),
367
+ },
368
+ ],
369
+ tool_call_id: 'select-query',
370
+ });
371
+
372
+ const count = getTokenCountForMessage(message, (text) => text.length);
373
+
374
+ expect(count).toBeGreaterThan(2_000);
375
+ });
376
+
377
+ test('counts spoofed media types as serialized structured output', () => {
378
+ const message = new ToolMessage({
379
+ content: [
380
+ {
381
+ type: 'image_url',
382
+ rows: [{ value: 'x'.repeat(5_000) }],
383
+ },
384
+ ],
385
+ tool_call_id: 'spoofed-image',
386
+ });
387
+
388
+ const count = getTokenCountForMessage(message, (text) => text.length);
389
+
390
+ expect(count).toBeGreaterThan(5_000);
391
+ });
392
+
393
+ test('prices Google image media blocks with the image estimator', () => {
394
+ const message = new ToolMessage({
395
+ content: [
396
+ {
397
+ type: 'media',
398
+ mimeType: 'image/png',
399
+ data: pngDataUri(1024, 768),
400
+ },
401
+ ],
402
+ tool_call_id: 'google-image-media',
403
+ });
404
+
405
+ const count = getTokenCountForMessage(
406
+ message,
407
+ (text) => text.length,
408
+ 'o200k_base'
409
+ );
410
+
411
+ // 3 message tokens + ceil((85 + 4 * 170) * 1.05)
412
+ expect(count).toBe(807);
413
+ });
414
+
415
+ test('prices computer screenshots as images instead of base64 text', () => {
416
+ const screenshot = pngDataUri(1024, 768);
417
+ const structured = new ToolMessage({
418
+ content: [{ type: 'computer_screenshot', image_url: screenshot }],
419
+ tool_call_id: 'computer-structured',
420
+ additional_kwargs: { type: 'computer_call_output' },
421
+ });
422
+ const string = new ToolMessage({
423
+ content: screenshot,
424
+ tool_call_id: 'computer-string',
425
+ additional_kwargs: { type: 'computer_call_output' },
426
+ });
427
+
428
+ expect(
429
+ getTokenCountForMessage(structured, (text) => text.length, 'o200k_base')
430
+ ).toBe(807);
431
+ expect(
432
+ getTokenCountForMessage(string, (text) => text.length, 'o200k_base')
433
+ ).toBe(807);
434
+ const file = new ToolMessage({
435
+ content: [
436
+ { type: 'input_image', file_id: 'file-screenshot123', detail: 'low' },
437
+ ],
438
+ tool_call_id: 'computer-file',
439
+ additional_kwargs: { type: 'computer_call_output' },
440
+ });
441
+ expect(
442
+ getTokenCountForMessage(file, (text) => text.length, 'o200k_base')
443
+ ).toBe(93);
444
+ const highDetailFile = new ToolMessage({
445
+ content: [
446
+ { type: 'input_image', file_id: 'file-screenshot456', detail: 'high' },
447
+ ],
448
+ tool_call_id: 'computer-file-high',
449
+ additional_kwargs: { type: 'computer_call_output' },
450
+ });
451
+ expect(
452
+ getTokenCountForMessage(
453
+ highDetailFile,
454
+ (text) => text.length,
455
+ 'o200k_base'
456
+ )
457
+ ).toBe(1079);
458
+ });
459
+
460
+ test('does not image-price malformed marked computer output', () => {
461
+ const content = 'not a screenshot URL';
462
+ const message = new ToolMessage({
463
+ content,
464
+ tool_call_id: 'computer-invalid',
465
+ additional_kwargs: { type: 'computer_call_output' },
466
+ });
467
+
468
+ expect(
469
+ getTokenCountForMessage(message, (text) => text.length, 'o200k_base')
470
+ ).toBe(3 + content.length);
471
+ });
472
+
473
+ test('counts legacy calls on generic assistant messages', () => {
474
+ const argumentsValue = `{"query":"${'x'.repeat(5_000)}"}`;
475
+ const message = new ChatMessage({
476
+ role: 'assistant',
477
+ content: '',
478
+ additional_kwargs: {
479
+ function_call: {
480
+ name: 'legacy_lookup',
481
+ arguments: argumentsValue,
482
+ },
483
+ },
484
+ });
485
+
486
+ expect(
487
+ getTokenCountForMessage(message, (text) => text.length, 'o200k_base')
488
+ ).toBeGreaterThan(argumentsValue.length);
489
+ });
490
+
491
+ test('prices Google PDF media blocks per estimated page', () => {
492
+ const message = new ToolMessage({
493
+ content: [
494
+ {
495
+ type: 'media',
496
+ mime_type: 'application/pdf',
497
+ data: 'A'.repeat(150_000),
498
+ },
499
+ ],
500
+ tool_call_id: 'google-pdf-media',
501
+ });
502
+
503
+ const count = getTokenCountForMessage(
504
+ message,
505
+ (text) => text.length,
506
+ 'o200k_base'
507
+ );
508
+
509
+ // 3 message tokens + ceil((2 pages * 1500) * 1.05)
510
+ expect(count).toBe(3153);
511
+ });
512
+
513
+ test('continues pricing Google audio and video media as timed content', () => {
514
+ const audio = new ToolMessage({
515
+ content: [
516
+ {
517
+ type: 'media',
518
+ mime_type: ' Audio/MP3 ; codecs=mp3 ',
519
+ data: 'A'.repeat(320_000),
520
+ },
521
+ ],
522
+ tool_call_id: 'google-audio-media',
523
+ });
524
+ const video = new ToolMessage({
525
+ content: [
526
+ {
527
+ type: 'media',
528
+ mimeType: 'video/mp4',
529
+ data: 'A'.repeat(1_000_000),
530
+ },
531
+ ],
532
+ tool_call_id: 'google-video-media',
533
+ });
534
+
535
+ expect(
536
+ getTokenCountForMessage(audio, (text) => text.length, 'o200k_base')
537
+ ).toBe(507);
538
+ expect(
539
+ getTokenCountForMessage(video, (text) => text.length, 'o200k_base')
540
+ ).toBe(948);
541
+ });
542
+
543
+ test.each([
544
+ ['mimeType', 'text/plain'],
545
+ ['mime_type', 'application/json'],
546
+ ])(
547
+ 'conservatively prices unknown Google media MIME via %s',
548
+ (mimeKey, mimeType) => {
549
+ let toJSONCalls = 0;
550
+ const data = 'A'.repeat(100_000);
551
+ const message = new ToolMessage({
552
+ content: [
553
+ {
554
+ type: 'media',
555
+ [mimeKey]: mimeType,
556
+ data,
557
+ toJSON() {
558
+ toJSONCalls++;
559
+ return { expanded: 'x'.repeat(1_000_000) };
560
+ },
561
+ },
562
+ ],
563
+ tool_call_id: 'google-unknown-media',
564
+ });
565
+
566
+ const count = getTokenCountForMessage(
567
+ message,
568
+ (text) => text.length,
569
+ 'o200k_base'
570
+ );
571
+
572
+ expect(count).toBeGreaterThan(data.length);
573
+ expect(toJSONCalls).toBe(0);
574
+ }
575
+ );
576
+ });
577
+
85
578
  describe('estimateImageBlockTokens', () => {
86
579
  test('claude: tokens = ceil(w*h/750), floored at 1024', () => {
87
- const block = { type: 'image_url', image_url: { url: pngDataUri(1024, 768) } };
580
+ const block = {
581
+ type: 'image_url',
582
+ image_url: { url: pngDataUri(1024, 768) },
583
+ };
88
584
  // 1024*768/750 = 1048.58 -> ceil 1049 (> 1024 floor)
89
585
  expect(estimateImageBlockTokens(block, 'claude')).toBe(1049);
90
586
  });
91
587
 
92
588
  test('openai: tokens = 85 + tiles*170 (512px tiles)', () => {
93
- const block = { type: 'image_url', image_url: { url: pngDataUri(1024, 768) } };
589
+ const block = {
590
+ type: 'image_url',
591
+ image_url: { url: pngDataUri(1024, 768) },
592
+ };
94
593
  // ceil(1024/512)*ceil(768/512) = 2*2 = 4 tiles -> 85 + 680 = 765
95
594
  expect(estimateImageBlockTokens(block, 'o200k_base')).toBe(765);
96
595
  });
@@ -110,7 +609,9 @@ describe('estimateDocumentBlockTokens', () => {
110
609
 
111
610
  test('text document is tokenized directly via getTokenCount', () => {
112
611
  const block = { type: 'file', source_type: 'text', text: 'hello world' };
113
- expect(estimateDocumentBlockTokens(block, 'o200k_base', countChars)).toBe(11);
612
+ expect(estimateDocumentBlockTokens(block, 'o200k_base', countChars)).toBe(
613
+ 11
614
+ );
114
615
  });
115
616
 
116
617
  test('base64 PDF is priced per estimated page', () => {
@@ -122,12 +623,16 @@ describe('estimateDocumentBlockTokens', () => {
122
623
  };
123
624
  // ceil(150000/75000) = 2 pages
124
625
  expect(estimateDocumentBlockTokens(block, 'claude', countChars)).toBe(4000); // 2 * 2000
125
- expect(estimateDocumentBlockTokens(block, 'o200k_base', countChars)).toBe(3000); // 2 * 1500
626
+ expect(estimateDocumentBlockTokens(block, 'o200k_base', countChars)).toBe(
627
+ 3000
628
+ ); // 2 * 1500
126
629
  });
127
630
 
128
631
  test('url-referenced document uses the conservative fallback', () => {
129
632
  const block = { type: 'file', source_type: 'url', url: 'https://x/y.pdf' };
130
- expect(estimateDocumentBlockTokens(block, 'o200k_base', countChars)).toBe(2000);
633
+ expect(estimateDocumentBlockTokens(block, 'o200k_base', countChars)).toBe(
634
+ 2000
635
+ );
131
636
  });
132
637
  });
133
638
 
@@ -137,49 +642,79 @@ describe('estimateTimedMediaBlockTokens', () => {
137
642
  test('Google video (type=media, video/*): duration from size at ~300 tok/s', () => {
138
643
  // 1,000,000 b64 -> 750,000 bytes / 250,000 Bps = 3s * 300 = 900
139
644
  expect(
140
- estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'video/mp4', data: B64(1_000_000) }),
645
+ estimateTimedMediaBlockTokens({
646
+ type: 'media',
647
+ mimeType: 'video/mp4',
648
+ data: B64(1_000_000),
649
+ })
141
650
  ).toBe(900);
142
651
  });
143
652
 
144
653
  test('Google audio (type=media, audio/*): 32 tok/s at ~16KB/s', () => {
145
654
  // 320,000 b64 -> 240,000 bytes / 16,000 Bps = 15s * 32 = 480
146
655
  expect(
147
- estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'audio/mp3', data: B64(320_000) }),
656
+ estimateTimedMediaBlockTokens({
657
+ type: 'media',
658
+ mimeType: 'audio/mp3',
659
+ data: B64(320_000),
660
+ })
148
661
  ).toBe(480);
149
662
  });
150
663
 
151
664
  test('OpenRouter input_audio: estimates from base64 data', () => {
152
665
  expect(
153
- estimateTimedMediaBlockTokens({ type: 'input_audio', input_audio: { data: B64(320_000) } }),
666
+ estimateTimedMediaBlockTokens({
667
+ type: 'input_audio',
668
+ input_audio: { data: B64(320_000) },
669
+ })
154
670
  ).toBe(480);
155
671
  });
156
672
 
157
673
  test('OpenRouter video_url with a data: URL estimates from size', () => {
158
674
  const url = `data:video/mp4;base64,${B64(1_000_000)}`;
159
- expect(estimateTimedMediaBlockTokens({ type: 'video_url', video_url: { url } })).toBe(900);
675
+ expect(
676
+ estimateTimedMediaBlockTokens({ type: 'video_url', video_url: { url } })
677
+ ).toBe(900);
160
678
  });
161
679
 
162
680
  test('bare remote URL falls back to the flat ~30s estimate', () => {
163
681
  expect(
164
- estimateTimedMediaBlockTokens({ type: 'video_url', video_url: { url: 'https://x/v.mp4' } }),
682
+ estimateTimedMediaBlockTokens({
683
+ type: 'video_url',
684
+ video_url: { url: 'https://x/v.mp4' },
685
+ })
165
686
  ).toBe(9000);
166
- expect(estimateTimedMediaBlockTokens({ type: 'input_audio', input_audio: {} })).toBe(960);
687
+ expect(
688
+ estimateTimedMediaBlockTokens({ type: 'input_audio', input_audio: {} })
689
+ ).toBe(960);
167
690
  });
168
691
 
169
692
  test('clamps to at least one second of tokens for tiny payloads', () => {
170
693
  expect(
171
- estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'audio/wav', data: 'AAAA' }),
694
+ estimateTimedMediaBlockTokens({
695
+ type: 'media',
696
+ mimeType: 'audio/wav',
697
+ data: 'AAAA',
698
+ })
172
699
  ).toBe(32);
173
700
  });
174
701
 
175
702
  test('standard type=video / type=audio blocks (Bedrock converter shape)', () => {
176
703
  // 750,000 bytes video / 250,000 = 3s * 300 = 900
177
704
  expect(
178
- estimateTimedMediaBlockTokens({ type: 'video', mimeType: 'video/mp4', data: B64(1_000_000) }),
705
+ estimateTimedMediaBlockTokens({
706
+ type: 'video',
707
+ mimeType: 'video/mp4',
708
+ data: B64(1_000_000),
709
+ })
179
710
  ).toBe(900);
180
711
  // 240,000 bytes audio / 16,000 = 15s * 32 = 480
181
712
  expect(
182
- estimateTimedMediaBlockTokens({ type: 'audio', mimeType: 'audio/mpeg', data: B64(320_000) }),
713
+ estimateTimedMediaBlockTokens({
714
+ type: 'audio',
715
+ mimeType: 'audio/mpeg',
716
+ data: B64(320_000),
717
+ })
183
718
  ).toBe(480);
184
719
  });
185
720
 
@@ -190,7 +725,7 @@ describe('estimateTimedMediaBlockTokens', () => {
190
725
  type: 'audio',
191
726
  mimeType: 'audio/wav',
192
727
  data: new Uint8Array(240_000),
193
- }),
728
+ })
194
729
  ).toBe(480);
195
730
  // base64 data url on a bare video block
196
731
  expect(
@@ -198,37 +733,63 @@ describe('estimateTimedMediaBlockTokens', () => {
198
733
  type: 'video',
199
734
  mimeType: 'video/mp4',
200
735
  url: `data:video/mp4;base64,${B64(1_000_000)}`,
201
- }),
736
+ })
202
737
  ).toBe(900);
203
738
  });
204
739
 
205
740
  test('non-video/audio media (image/document MIME) is NOT priced as video', () => {
206
741
  expect(
207
- estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'image/png', fileUri: 's3://x' }),
742
+ estimateTimedMediaBlockTokens({
743
+ type: 'media',
744
+ mimeType: 'image/png',
745
+ fileUri: 's3://x',
746
+ })
208
747
  ).toBe(0);
209
748
  expect(
210
- estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'image/png', data: B64(400_000) }),
749
+ estimateTimedMediaBlockTokens({
750
+ type: 'media',
751
+ mimeType: 'image/png',
752
+ data: B64(400_000),
753
+ })
211
754
  ).toBe(0);
212
755
  expect(
213
- estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'application/pdf', data: B64(400) }),
756
+ estimateTimedMediaBlockTokens({
757
+ type: 'media',
758
+ mimeType: 'application/pdf',
759
+ data: B64(400),
760
+ })
214
761
  ).toBe(0);
215
762
  });
216
763
 
217
764
  test('fileId / bare URL with no size falls back to the ~30s estimate', () => {
218
765
  expect(
219
- estimateTimedMediaBlockTokens({ type: 'video', mimeType: 'video/mp4', fileId: 's3://v' }),
766
+ estimateTimedMediaBlockTokens({
767
+ type: 'video',
768
+ mimeType: 'video/mp4',
769
+ fileId: 's3://v',
770
+ })
220
771
  ).toBe(9000);
221
772
  expect(
222
- estimateTimedMediaBlockTokens({ type: 'audio', mimeType: 'audio/mp3', fileId: 's3://a' }),
773
+ estimateTimedMediaBlockTokens({
774
+ type: 'audio',
775
+ mimeType: 'audio/mp3',
776
+ fileId: 's3://a',
777
+ })
223
778
  ).toBe(960);
224
779
  });
225
780
 
226
781
  test('reads native Bedrock nested source.bytes (video/audio)', () => {
227
782
  expect(
228
- estimateTimedMediaBlockTokens({ type: 'video', video: { source: { bytes: new Uint8Array(750_000) } } }),
783
+ estimateTimedMediaBlockTokens({
784
+ type: 'video',
785
+ video: { source: { bytes: new Uint8Array(750_000) } },
786
+ })
229
787
  ).toBe(900); // 750,000 / 250,000 = 3s * 300
230
788
  expect(
231
- estimateTimedMediaBlockTokens({ type: 'audio', audio: { source: { bytes: new Uint8Array(240_000) } } }),
789
+ estimateTimedMediaBlockTokens({
790
+ type: 'audio',
791
+ audio: { source: { bytes: new Uint8Array(240_000) } },
792
+ })
232
793
  ).toBe(480); // 240,000 / 16,000 = 15s * 32
233
794
  });
234
795
 
@@ -238,41 +799,57 @@ describe('estimateTimedMediaBlockTokens', () => {
238
799
  estimateTimedMediaBlockTokens({
239
800
  type: 'video',
240
801
  video: { url: `data:video/mp4;base64,${B64(1_000_000)}` },
241
- }),
802
+ })
242
803
  ).toBe(900);
243
804
  // nested base64 data (240,000 -> 15s * 32)
244
805
  expect(
245
- estimateTimedMediaBlockTokens({ type: 'audio', audio: { data: B64(320_000) } }),
806
+ estimateTimedMediaBlockTokens({
807
+ type: 'audio',
808
+ audio: { data: B64(320_000) },
809
+ })
246
810
  ).toBe(480);
247
811
  // nested REMOTE url has no size -> ~30s fallback
248
812
  expect(
249
- estimateTimedMediaBlockTokens({ type: 'video', video: { url: 'https://x/v.mp4' } }),
813
+ estimateTimedMediaBlockTokens({
814
+ type: 'video',
815
+ video: { url: 'https://x/v.mp4' },
816
+ })
250
817
  ).toBe(9000);
251
818
  });
252
819
 
253
820
  test('non-data URI schemes (gs://, s3://) are treated as remote, not base64', () => {
254
821
  // gs:// audio must hit the ~30s remote fallback, not clamp to 32
255
822
  expect(
256
- estimateTimedMediaBlockTokens({ type: 'audio', mimeType: 'audio/mp3', url: 'gs://bucket/a.mp3' }),
823
+ estimateTimedMediaBlockTokens({
824
+ type: 'audio',
825
+ mimeType: 'audio/mp3',
826
+ url: 'gs://bucket/a.mp3',
827
+ })
257
828
  ).toBe(960);
258
829
  expect(
259
- estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'video/mp4', data: 's3://bucket/v.mp4' }),
830
+ estimateTimedMediaBlockTokens({
831
+ type: 'media',
832
+ mimeType: 'video/mp4',
833
+ data: 's3://bucket/v.mp4',
834
+ })
260
835
  ).toBe(9000);
261
836
  });
262
837
 
263
838
  test('classifies Google MIME-as-type blocks (type is the mime string)', () => {
264
839
  // { type: 'audio/wav', data } -> 240,000 bytes / 16,000 = 15s * 32 = 480
265
840
  expect(
266
- estimateTimedMediaBlockTokens({ type: 'audio/wav', data: B64(320_000) }),
841
+ estimateTimedMediaBlockTokens({ type: 'audio/wav', data: B64(320_000) })
267
842
  ).toBe(480);
268
843
  // { type: 'video/mp4', data } -> 750,000 / 250,000 = 3s * 300 = 900
269
844
  expect(
270
- estimateTimedMediaBlockTokens({ type: 'video/mp4', data: B64(1_000_000) }),
845
+ estimateTimedMediaBlockTokens({ type: 'video/mp4', data: B64(1_000_000) })
271
846
  ).toBe(900);
272
847
  });
273
848
 
274
849
  test('returns 0 for non-timed-media blocks', () => {
275
850
  expect(estimateTimedMediaBlockTokens({ type: 'text' })).toBe(0);
276
- expect(estimateTimedMediaBlockTokens({ type: 'image/png', data: B64(400) })).toBe(0);
851
+ expect(
852
+ estimateTimedMediaBlockTokens({ type: 'image/png', data: B64(400) })
853
+ ).toBe(0);
277
854
  });
278
855
  });