@librechat/agents 3.3.3 → 3.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/agents/AgentContext.cjs +2 -1
- package/dist/cjs/agents/AgentContext.cjs.map +1 -1
- package/dist/cjs/common/constants.cjs +21 -0
- package/dist/cjs/common/constants.cjs.map +1 -1
- package/dist/cjs/events.cjs +10 -1
- package/dist/cjs/events.cjs.map +1 -1
- package/dist/cjs/graphs/Graph.cjs +793 -111
- package/dist/cjs/graphs/Graph.cjs.map +1 -1
- package/dist/cjs/graphs/MultiAgentGraph.cjs +3 -1
- package/dist/cjs/graphs/MultiAgentGraph.cjs.map +1 -1
- package/dist/cjs/hooks/HookRegistry.cjs +25 -0
- package/dist/cjs/hooks/HookRegistry.cjs.map +1 -1
- package/dist/cjs/hooks/index.cjs +12 -0
- package/dist/cjs/hooks/index.cjs.map +1 -1
- package/dist/cjs/hooks/types.cjs +1 -0
- package/dist/cjs/hooks/types.cjs.map +1 -1
- package/dist/cjs/langfuse.cjs +8 -0
- package/dist/cjs/langfuse.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/utils/message_inputs.cjs +79 -6
- package/dist/cjs/llm/anthropic/utils/message_inputs.cjs.map +1 -1
- package/dist/cjs/llm/bedrock/utils/message_inputs.cjs +38 -13
- package/dist/cjs/llm/bedrock/utils/message_inputs.cjs.map +1 -1
- package/dist/cjs/llm/google/utils/common.cjs +19 -7
- package/dist/cjs/llm/google/utils/common.cjs.map +1 -1
- package/dist/cjs/llm/invoke.cjs +331 -5
- package/dist/cjs/llm/invoke.cjs.map +1 -1
- package/dist/cjs/llm/openai/index.cjs +41 -5
- package/dist/cjs/llm/openai/index.cjs.map +1 -1
- package/dist/cjs/llm/openai/utils/index.cjs +25 -4
- package/dist/cjs/llm/openai/utils/index.cjs.map +1 -1
- package/dist/cjs/llm/openrouter/index.cjs +4 -1
- package/dist/cjs/llm/openrouter/index.cjs.map +1 -1
- package/dist/cjs/llm/preempt.cjs +132 -0
- package/dist/cjs/llm/preempt.cjs.map +1 -0
- package/dist/cjs/main.cjs +36 -5
- package/dist/cjs/messages/alternation.cjs +92 -0
- package/dist/cjs/messages/alternation.cjs.map +1 -0
- package/dist/cjs/messages/cache.cjs +54 -33
- package/dist/cjs/messages/cache.cjs.map +1 -1
- package/dist/cjs/messages/contextPruning.cjs +17 -43
- package/dist/cjs/messages/contextPruning.cjs.map +1 -1
- package/dist/cjs/messages/core.cjs +315 -23
- package/dist/cjs/messages/core.cjs.map +1 -1
- package/dist/cjs/messages/format.cjs +290 -60
- package/dist/cjs/messages/format.cjs.map +1 -1
- package/dist/cjs/messages/handoffCue.cjs +63 -0
- package/dist/cjs/messages/handoffCue.cjs.map +1 -0
- package/dist/cjs/messages/index.cjs +4 -1
- package/dist/cjs/messages/injected.cjs +60 -0
- package/dist/cjs/messages/injected.cjs.map +1 -0
- package/dist/cjs/messages/prune.cjs +503 -127
- package/dist/cjs/messages/prune.cjs.map +1 -1
- package/dist/cjs/run.cjs +80 -7
- package/dist/cjs/run.cjs.map +1 -1
- package/dist/cjs/session/handlers.cjs +18 -9
- package/dist/cjs/session/handlers.cjs.map +1 -1
- package/dist/cjs/stream.cjs +35 -10
- package/dist/cjs/stream.cjs.map +1 -1
- package/dist/cjs/summarization/node.cjs +28 -14
- package/dist/cjs/summarization/node.cjs.map +1 -1
- package/dist/cjs/tools/ToolNode.cjs +56 -63
- package/dist/cjs/tools/ToolNode.cjs.map +1 -1
- package/dist/cjs/tools/search/keenable-scraper.cjs +90 -0
- package/dist/cjs/tools/search/keenable-scraper.cjs.map +1 -0
- package/dist/cjs/tools/search/tool.cjs +9 -1
- package/dist/cjs/tools/search/tool.cjs.map +1 -1
- package/dist/cjs/tools/toolOutputReferences.cjs +6 -0
- package/dist/cjs/tools/toolOutputReferences.cjs.map +1 -1
- package/dist/cjs/utils/index.cjs +1 -1
- package/dist/cjs/utils/llm.cjs +1 -1
- package/dist/cjs/utils/llm.cjs.map +1 -1
- package/dist/cjs/utils/tokens.cjs +307 -20
- package/dist/cjs/utils/tokens.cjs.map +1 -1
- package/dist/cjs/utils/toolContent.cjs +1514 -0
- package/dist/cjs/utils/toolContent.cjs.map +1 -0
- package/dist/esm/agents/AgentContext.mjs +2 -1
- package/dist/esm/agents/AgentContext.mjs.map +1 -1
- package/dist/esm/common/constants.mjs +19 -1
- package/dist/esm/common/constants.mjs.map +1 -1
- package/dist/esm/events.mjs +10 -1
- package/dist/esm/events.mjs.map +1 -1
- package/dist/esm/graphs/Graph.mjs +798 -116
- package/dist/esm/graphs/Graph.mjs.map +1 -1
- package/dist/esm/graphs/MultiAgentGraph.mjs +3 -1
- package/dist/esm/graphs/MultiAgentGraph.mjs.map +1 -1
- package/dist/esm/hooks/HookRegistry.mjs +25 -0
- package/dist/esm/hooks/HookRegistry.mjs.map +1 -1
- package/dist/esm/hooks/index.mjs +12 -1
- package/dist/esm/hooks/index.mjs.map +1 -1
- package/dist/esm/hooks/types.mjs +1 -0
- package/dist/esm/hooks/types.mjs.map +1 -1
- package/dist/esm/langfuse.mjs +9 -1
- package/dist/esm/langfuse.mjs.map +1 -1
- package/dist/esm/llm/anthropic/utils/message_inputs.mjs +79 -6
- package/dist/esm/llm/anthropic/utils/message_inputs.mjs.map +1 -1
- package/dist/esm/llm/bedrock/utils/message_inputs.mjs +38 -13
- package/dist/esm/llm/bedrock/utils/message_inputs.mjs.map +1 -1
- package/dist/esm/llm/google/utils/common.mjs +19 -7
- package/dist/esm/llm/google/utils/common.mjs.map +1 -1
- package/dist/esm/llm/invoke.mjs +332 -8
- package/dist/esm/llm/invoke.mjs.map +1 -1
- package/dist/esm/llm/openai/index.mjs +41 -5
- package/dist/esm/llm/openai/index.mjs.map +1 -1
- package/dist/esm/llm/openai/utils/index.mjs +25 -4
- package/dist/esm/llm/openai/utils/index.mjs.map +1 -1
- package/dist/esm/llm/openrouter/index.mjs +4 -1
- package/dist/esm/llm/openrouter/index.mjs.map +1 -1
- package/dist/esm/llm/preempt.mjs +131 -0
- package/dist/esm/llm/preempt.mjs.map +1 -0
- package/dist/esm/main.mjs +14 -11
- package/dist/esm/messages/alternation.mjs +91 -0
- package/dist/esm/messages/alternation.mjs.map +1 -0
- package/dist/esm/messages/cache.mjs +55 -34
- package/dist/esm/messages/cache.mjs.map +1 -1
- package/dist/esm/messages/contextPruning.mjs +17 -43
- package/dist/esm/messages/contextPruning.mjs.map +1 -1
- package/dist/esm/messages/core.mjs +305 -24
- package/dist/esm/messages/core.mjs.map +1 -1
- package/dist/esm/messages/format.mjs +290 -61
- package/dist/esm/messages/format.mjs.map +1 -1
- package/dist/esm/messages/handoffCue.mjs +61 -0
- package/dist/esm/messages/handoffCue.mjs.map +1 -0
- package/dist/esm/messages/index.mjs +4 -1
- package/dist/esm/messages/injected.mjs +60 -0
- package/dist/esm/messages/injected.mjs.map +1 -0
- package/dist/esm/messages/prune.mjs +500 -128
- package/dist/esm/messages/prune.mjs.map +1 -1
- package/dist/esm/run.mjs +80 -7
- package/dist/esm/run.mjs.map +1 -1
- package/dist/esm/session/handlers.mjs +19 -10
- package/dist/esm/session/handlers.mjs.map +1 -1
- package/dist/esm/stream.mjs +34 -11
- package/dist/esm/stream.mjs.map +1 -1
- package/dist/esm/summarization/node.mjs +28 -14
- package/dist/esm/summarization/node.mjs.map +1 -1
- package/dist/esm/tools/ToolNode.mjs +56 -63
- package/dist/esm/tools/ToolNode.mjs.map +1 -1
- package/dist/esm/tools/search/keenable-scraper.mjs +88 -0
- package/dist/esm/tools/search/keenable-scraper.mjs.map +1 -0
- package/dist/esm/tools/search/tool.mjs +9 -1
- package/dist/esm/tools/search/tool.mjs.map +1 -1
- package/dist/esm/tools/toolOutputReferences.mjs +6 -0
- package/dist/esm/tools/toolOutputReferences.mjs.map +1 -1
- package/dist/esm/utils/index.mjs +1 -1
- package/dist/esm/utils/llm.mjs +1 -1
- package/dist/esm/utils/llm.mjs.map +1 -1
- package/dist/esm/utils/tokens.mjs +307 -21
- package/dist/esm/utils/tokens.mjs.map +1 -1
- package/dist/esm/utils/toolContent.mjs +1503 -0
- package/dist/esm/utils/toolContent.mjs.map +1 -0
- package/dist/types/common/constants.d.ts +18 -0
- package/dist/types/graphs/Graph.d.ts +139 -2
- package/dist/types/hooks/HookRegistry.d.ts +15 -0
- package/dist/types/hooks/index.d.ts +12 -1
- package/dist/types/hooks/types.d.ts +45 -6
- package/dist/types/index.d.ts +1 -1
- package/dist/types/llm/invoke.d.ts +34 -1
- package/dist/types/llm/openai/index.d.ts +4 -0
- package/dist/types/llm/openai/utils/index.d.ts +2 -0
- package/dist/types/llm/openrouter/index.d.ts +5 -0
- package/dist/types/llm/preempt.d.ts +40 -0
- package/dist/types/messages/alternation.d.ts +23 -0
- package/dist/types/messages/contextPruning.d.ts +2 -1
- package/dist/types/messages/core.d.ts +36 -0
- package/dist/types/messages/format.d.ts +6 -0
- package/dist/types/messages/handoffCue.d.ts +40 -0
- package/dist/types/messages/index.d.ts +3 -0
- package/dist/types/messages/injected.d.ts +3 -0
- package/dist/types/messages/prune.d.ts +16 -12
- package/dist/types/run.d.ts +7 -0
- package/dist/types/stream.d.ts +14 -0
- package/dist/types/tools/ToolNode.d.ts +0 -7
- package/dist/types/tools/search/keenable-scraper.d.ts +15 -0
- package/dist/types/tools/search/types.d.ts +31 -2
- package/dist/types/types/graph.d.ts +8 -1
- package/dist/types/types/run.d.ts +65 -0
- package/dist/types/types/stream.d.ts +1 -26
- package/dist/types/utils/tokens.d.ts +7 -0
- package/dist/types/utils/toolContent.d.ts +107 -0
- package/package.json +1 -1
- package/src/__tests__/stream.eagerEventExecution.test.ts +74 -0
- package/src/agents/AgentContext.ts +1 -0
- package/src/{splitStream.test.ts → aggregator.test.ts} +59 -666
- package/src/common/constants.ts +21 -0
- package/src/events.ts +15 -1
- package/src/graphs/Graph.ts +1261 -198
- package/src/graphs/MultiAgentGraph.ts +6 -1
- package/src/graphs/__tests__/Graph.contextOverflow.test.ts +1139 -4
- package/src/graphs/__tests__/Graph.preemptSignal.test.ts +126 -0
- package/src/hooks/HookRegistry.ts +40 -0
- package/src/hooks/__tests__/preemptBoundary.test.ts +152 -0
- package/src/hooks/index.ts +16 -2
- package/src/hooks/types.ts +47 -3
- package/src/index.ts +1 -1
- package/src/langfuse.ts +26 -1
- package/src/llm/anthropic/utils/message_inputs.ts +158 -12
- package/src/llm/anthropic/utils/tool-id-normalization.test.ts +109 -0
- package/src/llm/bedrock/utils/message_inputs.test.ts +131 -1
- package/src/llm/bedrock/utils/message_inputs.ts +100 -24
- package/src/llm/bedrock/utils/toolResultCachePoint.test.ts +15 -0
- package/src/llm/google/utils/common.test.ts +75 -9
- package/src/llm/google/utils/common.ts +34 -3
- package/src/llm/invoke.alternation.test.ts +87 -0
- package/src/llm/invoke.handoffCue.test.ts +112 -0
- package/src/llm/invoke.test.ts +488 -0
- package/src/llm/invoke.ts +611 -8
- package/src/llm/openai/index.ts +106 -6
- package/src/llm/openai/utils/index.ts +109 -53
- package/src/llm/openai/utils/messages.test.ts +330 -1
- package/src/llm/openrouter/index.ts +12 -2
- package/src/llm/openrouter/reasoning.test.ts +314 -0
- package/src/llm/preempt.test.ts +323 -0
- package/src/llm/preempt.ts +178 -0
- package/src/messages/__tests__/observationMasking.test.ts +93 -2
- package/src/messages/alternation.test.ts +212 -0
- package/src/messages/alternation.ts +112 -0
- package/src/messages/cache.tail.test.ts +193 -0
- package/src/messages/cache.test.ts +113 -0
- package/src/messages/cache.ts +92 -60
- package/src/messages/contextPruning.test.ts +184 -0
- package/src/messages/contextPruning.ts +49 -42
- package/src/messages/core.ts +653 -40
- package/src/messages/ensureThinkingBlock.test.ts +49 -1
- package/src/messages/foldToollessToolBlocks.test.ts +163 -5
- package/src/messages/format.ts +454 -91
- package/src/messages/formatAgentMessages.steer.test.ts +267 -0
- package/src/messages/formatAgentMessages.test.ts +604 -14
- package/src/messages/formatAgentMessages.tools.test.ts +103 -0
- package/src/messages/handoffCue.test.ts +96 -0
- package/src/messages/handoffCue.ts +78 -0
- package/src/messages/index.ts +3 -0
- package/src/messages/injected.test.ts +90 -0
- package/src/messages/injected.ts +74 -0
- package/src/messages/prune.ts +996 -183
- package/src/run.ts +91 -6
- package/src/scripts/preempt-probe.ts +330 -0
- package/src/scripts/preempt-scenarios.ts +388 -0
- package/src/session/handlers.ts +32 -12
- package/src/specs/handoffCue.test.ts +165 -0
- package/src/specs/langfuse-callbacks.test.ts +352 -2
- package/src/specs/preemptSeal.test.ts +309 -0
- package/src/specs/prune.test.ts +1083 -6
- package/src/specs/summarization-unit.test.ts +105 -0
- package/src/specs/tokens.test.ts +609 -32
- package/src/stream.dispatch.test.ts +63 -0
- package/src/stream.ts +59 -23
- package/src/summarization/__tests__/node.test.ts +77 -0
- package/src/summarization/node.ts +46 -18
- package/src/tools/ToolNode.ts +150 -99
- package/src/tools/__tests__/ToolNode.onResultCompletion.test.ts +148 -1
- package/src/tools/__tests__/ToolNode.outputReferences.test.ts +112 -3
- package/src/tools/__tests__/annotateMessagesForLLM.test.ts +32 -0
- package/src/tools/__tests__/directToolHooks.test.ts +255 -7
- package/src/tools/search/keenable-scraper.test.ts +153 -0
- package/src/tools/search/keenable-scraper.ts +137 -0
- package/src/tools/search/tool.ts +13 -2
- package/src/tools/search/types.ts +50 -3
- package/src/tools/toolOutputReferences.ts +6 -0
- package/src/types/graph.ts +12 -1
- package/src/types/run.ts +67 -0
- package/src/types/stream.ts +1 -42
- package/src/utils/llm.test.ts +18 -0
- package/src/utils/llm.ts +4 -1
- package/src/utils/tokens.ts +586 -31
- package/src/utils/toolContent.test.ts +1432 -0
- package/src/utils/toolContent.ts +2707 -0
- package/dist/cjs/splitStream.cjs +0 -151
- package/dist/cjs/splitStream.cjs.map +0 -1
- package/dist/esm/splitStream.mjs +0 -150
- package/dist/esm/splitStream.mjs.map +0 -1
- package/dist/types/mockStream.d.ts +0 -32
- package/dist/types/splitStream.d.ts +0 -37
- package/src/mockStream.ts +0 -99
- package/src/splitStream.ts +0 -234
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
require("../common/enum.cjs");
|
|
2
|
+
const require_toolContent = require("./toolContent.cjs");
|
|
2
3
|
let ai_tokenizer = require("ai-tokenizer");
|
|
4
|
+
let node_util_types = require("node:util/types");
|
|
3
5
|
//#region src/utils/tokens.ts
|
|
4
6
|
/** Anthropic minimum image token cost. */
|
|
5
7
|
const ANTHROPIC_IMAGE_MIN_TOKENS = 1024;
|
|
@@ -51,6 +53,15 @@ const TIMED_MEDIA_TYPES = new Set([
|
|
|
51
53
|
"input_audio"
|
|
52
54
|
]);
|
|
53
55
|
/**
|
|
56
|
+
* Structured content is tokenized exactly up to this size. Larger values are
|
|
57
|
+
* traversed by the bounded serializer and priced conservatively for the omitted
|
|
58
|
+
* suffix, avoiding a full JSON string allocation in the context guard.
|
|
59
|
+
*/
|
|
60
|
+
const MAX_STRUCTURED_TOKENIZATION_CHARS = 2e5;
|
|
61
|
+
/** One UTF-16 character can encode to several tokenizer pieces. */
|
|
62
|
+
const MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR = 4;
|
|
63
|
+
const MAX_STRUCTURED_SAFETY_INSPECTION_WORK = 1e4;
|
|
64
|
+
/**
|
|
54
65
|
* Extracts image dimensions from the first bytes of a base64-encoded
|
|
55
66
|
* PNG, JPEG, GIF, or WebP without decoding the full image.
|
|
56
67
|
* Returns null if the format is unrecognized or data is too short.
|
|
@@ -102,10 +113,11 @@ function estimateOpenAIImageTokens(width, height, detail = "high") {
|
|
|
102
113
|
*/
|
|
103
114
|
function estimateImageBlockTokens(block, encoding) {
|
|
104
115
|
let base64Data;
|
|
105
|
-
if (block.type === "image_url" || block.type === "image_url") {
|
|
116
|
+
if (block.type === "image_url" || block.type === "image_url" || block.type === "computer_screenshot" || block.type === "input_image") {
|
|
106
117
|
const imageUrl = block.image_url;
|
|
107
118
|
const url = typeof imageUrl === "string" ? imageUrl : imageUrl?.url;
|
|
108
119
|
if (typeof url === "string" && url.startsWith("data:")) base64Data = url;
|
|
120
|
+
else if (typeof block.file_id === "string" && block.file_id !== "") return block.detail === "low" ? OPENAI_IMAGE_LOW_TOKENS : ANTHROPIC_IMAGE_MIN_TOKENS;
|
|
109
121
|
else return ANTHROPIC_IMAGE_MIN_TOKENS;
|
|
110
122
|
} else if (block.type === "image") {
|
|
111
123
|
const source = block.source;
|
|
@@ -211,19 +223,87 @@ function mediaBlockByteLength(block) {
|
|
|
211
223
|
* NOT be priced as video. Also handles the Google shape where `type` IS the
|
|
212
224
|
* MIME string (e.g. `{ type: 'audio/wav', data }`). */
|
|
213
225
|
function timedMediaKind(block) {
|
|
214
|
-
const type = typeof block.type === "string" ? block.type : "";
|
|
226
|
+
const type = typeof block.type === "string" ? normalizeMediaMimeType(block.type) : "";
|
|
215
227
|
if (type === "input_audio" || type === "audio") return "audio";
|
|
216
228
|
if (type === "video_url" || type === "video") return "video";
|
|
217
|
-
const
|
|
229
|
+
const mediaMime = getMediaMimeType(block);
|
|
230
|
+
const mime = type === "media" && mediaMime != null ? mediaMime : type;
|
|
218
231
|
if (mime.startsWith("audio/")) return "audio";
|
|
219
232
|
if (mime.startsWith("video/")) return "video";
|
|
220
233
|
return null;
|
|
221
234
|
}
|
|
235
|
+
function getMediaMimeType(block) {
|
|
236
|
+
if (typeof block.mimeType === "string") return normalizeMediaMimeType(block.mimeType);
|
|
237
|
+
if (typeof block.mime_type === "string") return normalizeMediaMimeType(block.mime_type);
|
|
238
|
+
}
|
|
239
|
+
function normalizeMediaMimeType(mimeType) {
|
|
240
|
+
return mimeType.split(";")[0].trim().toLowerCase();
|
|
241
|
+
}
|
|
222
242
|
/** Whether a content-block `type` can carry timed media — the fixed set plus the
|
|
223
243
|
* Google MIME-as-type shape (`audio/*` / `video/*`). Gates callers before they
|
|
224
244
|
* hand the block to {@link estimateTimedMediaBlockTokens}. */
|
|
225
245
|
function isTimedMediaType(type) {
|
|
226
|
-
|
|
246
|
+
const normalizedType = normalizeMediaMimeType(type);
|
|
247
|
+
return TIMED_MEDIA_TYPES.has(normalizedType) || normalizedType.startsWith("audio/") || normalizedType.startsWith("video/");
|
|
248
|
+
}
|
|
249
|
+
/** Conservatively prices unknown inline media without materializing JSON. */
|
|
250
|
+
function estimateUnknownMediaBlockTokens(data) {
|
|
251
|
+
if (typeof data === "string") {
|
|
252
|
+
if (base64ByteLength(data) <= 0) return URL_DOCUMENT_FALLBACK_TOKENS;
|
|
253
|
+
return Math.max(URL_DOCUMENT_FALLBACK_TOKENS, data.length);
|
|
254
|
+
}
|
|
255
|
+
if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) return Math.max(URL_DOCUMENT_FALLBACK_TOKENS, Math.ceil(data.byteLength * 4 / 3));
|
|
256
|
+
return URL_DOCUMENT_FALLBACK_TOKENS;
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Estimates Google `type: "media"` blocks whose MIME describes static content.
|
|
260
|
+
* Timed audio/video continues through {@link estimateTimedMediaBlockTokens}.
|
|
261
|
+
*/
|
|
262
|
+
function estimateStaticMediaBlockTokens(block, encoding, getTokenCount) {
|
|
263
|
+
const mediaMime = getMediaMimeType(block);
|
|
264
|
+
if (block.type !== "media" || mediaMime == null) return;
|
|
265
|
+
const mimeType = mediaMime;
|
|
266
|
+
const nestedMedia = block.media != null && typeof block.media === "object" ? block.media : void 0;
|
|
267
|
+
const data = block.data ?? block.bytes ?? nestedMedia?.data ?? nestedMedia?.bytes;
|
|
268
|
+
const reference = block.url ?? block.fileUri ?? block.fileId ?? nestedMedia?.url ?? nestedMedia?.fileUri ?? nestedMedia?.fileId;
|
|
269
|
+
if (mimeType.startsWith("image/")) {
|
|
270
|
+
let encodedImage;
|
|
271
|
+
if (typeof data === "string") encodedImage = data;
|
|
272
|
+
else if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) {
|
|
273
|
+
const view = data instanceof ArrayBuffer ? new Uint8Array(data) : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
274
|
+
encodedImage = Buffer.from(view.subarray(0, 80)).toString("base64");
|
|
275
|
+
}
|
|
276
|
+
if (encodedImage != null && encodedImage.length > 0) return estimateImageBlockTokens({
|
|
277
|
+
type: "image",
|
|
278
|
+
source: {
|
|
279
|
+
type: "base64",
|
|
280
|
+
data: encodedImage
|
|
281
|
+
}
|
|
282
|
+
}, encoding);
|
|
283
|
+
return estimateImageBlockTokens({
|
|
284
|
+
type: "image_url",
|
|
285
|
+
image_url: { url: typeof reference === "string" ? reference : "" }
|
|
286
|
+
}, encoding);
|
|
287
|
+
}
|
|
288
|
+
if (mimeType === "application/pdf") {
|
|
289
|
+
if (typeof data === "string" && data.length > 0) {
|
|
290
|
+
const comma = data.startsWith("data:") ? data.indexOf(",") : -1;
|
|
291
|
+
return estimateDocumentBlockTokens({
|
|
292
|
+
type: "file",
|
|
293
|
+
source_type: "base64",
|
|
294
|
+
mime_type: mimeType,
|
|
295
|
+
data: comma >= 0 ? data.slice(comma + 1) : data
|
|
296
|
+
}, encoding, getTokenCount);
|
|
297
|
+
}
|
|
298
|
+
if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) {
|
|
299
|
+
const estimatedBase64Chars = Math.ceil(data.byteLength * 4 / 3);
|
|
300
|
+
const pdfTokensPerPage = encoding === "claude" ? ANTHROPIC_PDF_TOKENS_PER_PAGE : OPENAI_PDF_TOKENS_PER_PAGE;
|
|
301
|
+
return Math.max(1, Math.ceil(estimatedBase64Chars / BASE64_BYTES_PER_PDF_PAGE)) * pdfTokensPerPage;
|
|
302
|
+
}
|
|
303
|
+
return URL_DOCUMENT_FALLBACK_TOKENS;
|
|
304
|
+
}
|
|
305
|
+
if (mimeType.startsWith("audio/") || mimeType.startsWith("video/")) return;
|
|
306
|
+
return estimateUnknownMediaBlockTokens(data);
|
|
227
307
|
}
|
|
228
308
|
/**
|
|
229
309
|
* Estimates token cost for a timed-media block (video/audio). Handles Google
|
|
@@ -262,45 +342,251 @@ function encodingForModel(model) {
|
|
|
262
342
|
if (model.toLowerCase().includes("claude")) return "claude";
|
|
263
343
|
return "o200k_base";
|
|
264
344
|
}
|
|
345
|
+
function consumeStructuredSafetyWork(inspection) {
|
|
346
|
+
if (inspection.remaining <= 0) return false;
|
|
347
|
+
inspection.remaining--;
|
|
348
|
+
return true;
|
|
349
|
+
}
|
|
350
|
+
function hasUnsafeToJSON(value, inspection) {
|
|
351
|
+
let current = value;
|
|
352
|
+
const seen = /* @__PURE__ */ new Set();
|
|
353
|
+
try {
|
|
354
|
+
while (current != null) {
|
|
355
|
+
if ((0, node_util_types.isProxy)(current)) return true;
|
|
356
|
+
if (seen.has(current) || !consumeStructuredSafetyWork(inspection)) return true;
|
|
357
|
+
seen.add(current);
|
|
358
|
+
const descriptor = Object.getOwnPropertyDescriptor(current, "toJSON");
|
|
359
|
+
if (descriptor != null) return "value" in descriptor && typeof descriptor.value === "function" || !("value" in descriptor);
|
|
360
|
+
current = Object.getPrototypeOf(current);
|
|
361
|
+
}
|
|
362
|
+
} catch {
|
|
363
|
+
return true;
|
|
364
|
+
}
|
|
365
|
+
return false;
|
|
366
|
+
}
|
|
367
|
+
const arrayBufferByteLengthGetter = Object.getOwnPropertyDescriptor(ArrayBuffer.prototype, "byteLength")?.get;
|
|
368
|
+
/**
|
|
369
|
+
* Tests native binary values without `instanceof`, whose prototype walk can
|
|
370
|
+
* recurse or throw for adversarial proxies.
|
|
371
|
+
*/
|
|
372
|
+
function isNativeBinaryValue(value) {
|
|
373
|
+
try {
|
|
374
|
+
if (ArrayBuffer.isView(value)) return true;
|
|
375
|
+
} catch {
|
|
376
|
+
return true;
|
|
377
|
+
}
|
|
378
|
+
if (arrayBufferByteLengthGetter == null) return false;
|
|
379
|
+
try {
|
|
380
|
+
arrayBufferByteLengthGetter.call(value);
|
|
381
|
+
return true;
|
|
382
|
+
} catch {
|
|
383
|
+
return false;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* Detects an inherited accessor without reading it. Any proxy/prototype trap or
|
|
388
|
+
* cyclic prototype chain is unsafe rather than observable to the caller.
|
|
389
|
+
*/
|
|
390
|
+
function hasUnsafeInheritedProperty(value, key, inspection) {
|
|
391
|
+
const seen = new Set([value]);
|
|
392
|
+
let current;
|
|
393
|
+
try {
|
|
394
|
+
current = Object.getPrototypeOf(value);
|
|
395
|
+
} catch {
|
|
396
|
+
return true;
|
|
397
|
+
}
|
|
398
|
+
while (current != null) {
|
|
399
|
+
if ((0, node_util_types.isProxy)(current)) return true;
|
|
400
|
+
if (seen.has(current) || !consumeStructuredSafetyWork(inspection)) return true;
|
|
401
|
+
seen.add(current);
|
|
402
|
+
let descriptor;
|
|
403
|
+
try {
|
|
404
|
+
descriptor = Object.getOwnPropertyDescriptor(current, key);
|
|
405
|
+
} catch {
|
|
406
|
+
return true;
|
|
407
|
+
}
|
|
408
|
+
if (descriptor != null) return !("value" in descriptor);
|
|
409
|
+
try {
|
|
410
|
+
current = Object.getPrototypeOf(current);
|
|
411
|
+
} catch {
|
|
412
|
+
return true;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
return false;
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Detects values whose native JSON representation can differ after local
|
|
419
|
+
* measurement. Accessor properties and `toJSON` hooks are deliberately not
|
|
420
|
+
* invoked; values too large to inspect within bounded work are treated as
|
|
421
|
+
* unsafe so the caller can normalize or reject them conservatively.
|
|
422
|
+
*/
|
|
423
|
+
function hasUnsafeStructuredSerialization(value) {
|
|
424
|
+
if (typeof value === "bigint") return true;
|
|
425
|
+
if (value == null || typeof value !== "object") return false;
|
|
426
|
+
if ((0, node_util_types.isProxy)(value)) return true;
|
|
427
|
+
if (isNativeBinaryValue(value)) return true;
|
|
428
|
+
const stack = [value];
|
|
429
|
+
const seen = /* @__PURE__ */ new Set();
|
|
430
|
+
const inspection = { remaining: MAX_STRUCTURED_SAFETY_INSPECTION_WORK };
|
|
431
|
+
while (stack.length > 0) {
|
|
432
|
+
const current = stack.pop();
|
|
433
|
+
if ((0, node_util_types.isProxy)(current)) return true;
|
|
434
|
+
if (seen.has(current)) return true;
|
|
435
|
+
seen.add(current);
|
|
436
|
+
if (!consumeStructuredSafetyWork(inspection)) return true;
|
|
437
|
+
try {
|
|
438
|
+
if (hasUnsafeToJSON(current, inspection)) return true;
|
|
439
|
+
for (const key in current) {
|
|
440
|
+
if (!consumeStructuredSafetyWork(inspection)) return true;
|
|
441
|
+
const descriptor = Object.getOwnPropertyDescriptor(current, key);
|
|
442
|
+
if (descriptor == null) {
|
|
443
|
+
if (hasUnsafeInheritedProperty(current, key, inspection)) return true;
|
|
444
|
+
continue;
|
|
445
|
+
}
|
|
446
|
+
if (descriptor.enumerable !== true) continue;
|
|
447
|
+
if (!("value" in descriptor)) return true;
|
|
448
|
+
if ("value" in descriptor) {
|
|
449
|
+
if (typeof descriptor.value === "bigint") return true;
|
|
450
|
+
if (descriptor.value != null && typeof descriptor.value === "object" && isNativeBinaryValue(descriptor.value)) return true;
|
|
451
|
+
if (descriptor.value != null && typeof descriptor.value === "object") stack.push(descriptor.value);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
} catch {
|
|
455
|
+
return true;
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
return false;
|
|
459
|
+
}
|
|
460
|
+
function getBoundedTextTokenCount(value, getTokenCount) {
|
|
461
|
+
const preview = value.length > MAX_STRUCTURED_TOKENIZATION_CHARS ? value.slice(0, MAX_STRUCTURED_TOKENIZATION_CHARS) : value;
|
|
462
|
+
const previewTokens = getTokenCount(preview);
|
|
463
|
+
const omittedChars = value.length - preview.length;
|
|
464
|
+
if (omittedChars <= 0) return previewTokens;
|
|
465
|
+
return Math.min(Number.MAX_SAFE_INTEGER, previewTokens + omittedChars * MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR);
|
|
466
|
+
}
|
|
467
|
+
function getBoundedStructuredTokenCount(value, getTokenCount) {
|
|
468
|
+
if (hasUnsafeStructuredSerialization(value)) return Number.MAX_SAFE_INTEGER;
|
|
469
|
+
const serialized = require_toolContent.serializeStructuredValueBounded(value, MAX_STRUCTURED_TOKENIZATION_CHARS);
|
|
470
|
+
const previewTokens = getTokenCount(serialized.content);
|
|
471
|
+
if (!serialized.truncated) return previewTokens;
|
|
472
|
+
if (!Number.isSafeInteger(serialized.originalChars)) return Number.MAX_SAFE_INTEGER;
|
|
473
|
+
const omittedChars = Math.max(0, serialized.originalChars - serialized.content.length);
|
|
474
|
+
return Math.min(Number.MAX_SAFE_INTEGER, previewTokens + omittedChars * MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR);
|
|
475
|
+
}
|
|
265
476
|
function getTokenCountForMessage(message, getTokenCount, encoding = "o200k_base") {
|
|
266
477
|
const tokensPerMessage = 3;
|
|
478
|
+
const countText = (text) => getBoundedTextTokenCount(text, getTokenCount);
|
|
479
|
+
if ((0, node_util_types.isProxy)(message)) return Number.MAX_SAFE_INTEGER;
|
|
480
|
+
const representedToolCallIds = /* @__PURE__ */ new Set();
|
|
267
481
|
const processValue = (value) => {
|
|
482
|
+
if (value != null && typeof value === "object" && (0, node_util_types.isProxy)(value)) {
|
|
483
|
+
numTokens = Number.MAX_SAFE_INTEGER;
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
268
486
|
if (Array.isArray(value)) for (const raw of value) {
|
|
487
|
+
if (typeof raw === "string" || typeof raw === "number" || typeof raw === "boolean") {
|
|
488
|
+
processValue(raw);
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
269
491
|
const item = raw;
|
|
270
|
-
if (item == null || typeof item
|
|
492
|
+
if (item == null || typeof item !== "object") continue;
|
|
493
|
+
if ((0, node_util_types.isProxy)(item)) {
|
|
494
|
+
numTokens = Number.MAX_SAFE_INTEGER;
|
|
495
|
+
return;
|
|
496
|
+
}
|
|
497
|
+
if (typeof item.type !== "string") {
|
|
498
|
+
numTokens += getBoundedStructuredTokenCount(item, countText);
|
|
499
|
+
continue;
|
|
500
|
+
}
|
|
501
|
+
if (item.type === "tool_call" || item.type === "tool_use") {
|
|
502
|
+
const inlineId = typeof item.id === "string" ? item.id : item.tool_call?.id;
|
|
503
|
+
if (typeof inlineId === "string" && inlineId.length > 0) representedToolCallIds.add(inlineId);
|
|
504
|
+
}
|
|
271
505
|
if (item.type === "error") continue;
|
|
272
|
-
if (item.type === "image_url" || item.type === "image_url" || item.type === "image") {
|
|
506
|
+
if (require_toolContent.isAtomicToolContentBlock(item) && (item.type === "image_url" || item.type === "image_url" || item.type === "image" || item.type === "computer_screenshot" || item.type === "input_image")) {
|
|
273
507
|
numTokens += Math.ceil(estimateImageBlockTokens(item, encoding) * IMAGE_TOKEN_SAFETY_MARGIN);
|
|
274
508
|
continue;
|
|
275
509
|
}
|
|
276
|
-
if (item.type === "document" || item.type === "file" || item.type === "image_file") {
|
|
277
|
-
numTokens += Math.ceil(estimateDocumentBlockTokens(item, encoding,
|
|
510
|
+
if (require_toolContent.isAtomicToolContentBlock(item) && (item.type === "document" || item.type === "file" || item.type === "image_file")) {
|
|
511
|
+
numTokens += Math.ceil(estimateDocumentBlockTokens(item, encoding, countText) * IMAGE_TOKEN_SAFETY_MARGIN);
|
|
278
512
|
continue;
|
|
279
513
|
}
|
|
280
|
-
if (
|
|
281
|
-
|
|
282
|
-
|
|
514
|
+
if (require_toolContent.isAtomicToolContentBlock(item) && item.type === "media") {
|
|
515
|
+
const staticMediaTokens = estimateStaticMediaBlockTokens(item, encoding, countText);
|
|
516
|
+
if (staticMediaTokens != null) {
|
|
517
|
+
numTokens += Math.ceil(staticMediaTokens * IMAGE_TOKEN_SAFETY_MARGIN);
|
|
518
|
+
continue;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
if (require_toolContent.isAtomicToolContentBlock(item) && isTimedMediaType(item.type)) {
|
|
522
|
+
const timedMediaTokens = estimateTimedMediaBlockTokens(item);
|
|
523
|
+
if (timedMediaTokens > 0) {
|
|
524
|
+
numTokens += Math.ceil(timedMediaTokens * IMAGE_TOKEN_SAFETY_MARGIN);
|
|
525
|
+
continue;
|
|
526
|
+
}
|
|
283
527
|
}
|
|
284
528
|
if (item.type === "tool_call" && item.tool_call != null) {
|
|
285
529
|
const toolName = item.tool_call.name;
|
|
286
|
-
if (typeof toolName === "string" && toolName.length > 0) numTokens +=
|
|
530
|
+
if (typeof toolName === "string" && toolName.length > 0) numTokens += countText(toolName);
|
|
287
531
|
const args = item.tool_call.args;
|
|
288
|
-
if (typeof args === "string"
|
|
532
|
+
if (args != null) numTokens += typeof args === "string" ? countText(args) : getBoundedStructuredTokenCount(args, countText);
|
|
289
533
|
const output = item.tool_call.output;
|
|
290
|
-
if (
|
|
534
|
+
if (output != null) processValue(output);
|
|
291
535
|
continue;
|
|
292
536
|
}
|
|
293
537
|
const nestedValue = item[item.type];
|
|
294
|
-
if (nestedValue == null)
|
|
538
|
+
if (nestedValue == null) {
|
|
539
|
+
numTokens += getBoundedStructuredTokenCount(item, countText);
|
|
540
|
+
continue;
|
|
541
|
+
}
|
|
295
542
|
processValue(nestedValue);
|
|
296
543
|
}
|
|
297
|
-
else if (typeof value === "string") numTokens +=
|
|
298
|
-
else if (typeof value === "number") numTokens +=
|
|
299
|
-
else if (typeof value === "boolean") numTokens +=
|
|
544
|
+
else if (typeof value === "string") numTokens += countText(value);
|
|
545
|
+
else if (typeof value === "number") numTokens += countText(value.toString());
|
|
546
|
+
else if (typeof value === "boolean") numTokens += countText(value.toString());
|
|
547
|
+
else if (value != null && typeof value === "object") numTokens += getBoundedStructuredTokenCount(value, countText);
|
|
300
548
|
};
|
|
549
|
+
const rawAdditionalKwargs = message.additional_kwargs;
|
|
550
|
+
const additionalKwargs = rawAdditionalKwargs != null && typeof rawAdditionalKwargs === "object" ? rawAdditionalKwargs : void 0;
|
|
551
|
+
if (additionalKwargs != null && (0, node_util_types.isProxy)(additionalKwargs)) return Number.MAX_SAFE_INTEGER;
|
|
552
|
+
let additionalType;
|
|
553
|
+
try {
|
|
554
|
+
additionalType = additionalKwargs != null ? Object.getOwnPropertyDescriptor(additionalKwargs, "type") : void 0;
|
|
555
|
+
} catch {
|
|
556
|
+
return Number.MAX_SAFE_INTEGER;
|
|
557
|
+
}
|
|
558
|
+
if (additionalType != null && !("value" in additionalType)) return Number.MAX_SAFE_INTEGER;
|
|
301
559
|
let numTokens = tokensPerMessage;
|
|
302
|
-
|
|
303
|
-
|
|
560
|
+
const messageType = message.getType();
|
|
561
|
+
if (messageType === "tool" && additionalType?.value === "computer_call_output" && require_toolContent.isComputerCallOutputContent(message.content) && typeof message.content === "string") numTokens += Math.ceil(estimateImageBlockTokens({
|
|
562
|
+
type: "computer_screenshot",
|
|
563
|
+
image_url: message.content
|
|
564
|
+
}, encoding) * IMAGE_TOKEN_SAFETY_MARGIN);
|
|
565
|
+
else processValue(message.content);
|
|
566
|
+
if (numTokens >= Number.MAX_SAFE_INTEGER) return Number.MAX_SAFE_INTEGER;
|
|
567
|
+
const messageRole = message.role;
|
|
568
|
+
if (messageType === "ai" || messageRole === "assistant") {
|
|
569
|
+
const toolCalls = message.tool_calls ?? [];
|
|
570
|
+
if ((0, node_util_types.isProxy)(toolCalls)) return Number.MAX_SAFE_INTEGER;
|
|
571
|
+
for (const toolCall of toolCalls) {
|
|
572
|
+
if ((0, node_util_types.isProxy)(toolCall)) return Number.MAX_SAFE_INTEGER;
|
|
573
|
+
if (typeof toolCall.id === "string" && representedToolCallIds.has(toolCall.id)) continue;
|
|
574
|
+
if (typeof toolCall.name === "string" && toolCall.name.length > 0) numTokens += countText(toolCall.name);
|
|
575
|
+
const args = toolCall.args;
|
|
576
|
+
if (args != null) numTokens += typeof args === "string" ? countText(args) : getBoundedStructuredTokenCount(args, countText);
|
|
577
|
+
}
|
|
578
|
+
let legacyFunctionCall;
|
|
579
|
+
try {
|
|
580
|
+
legacyFunctionCall = additionalKwargs != null ? Object.getOwnPropertyDescriptor(additionalKwargs, "function_call") : void 0;
|
|
581
|
+
} catch {
|
|
582
|
+
return Number.MAX_SAFE_INTEGER;
|
|
583
|
+
}
|
|
584
|
+
if (legacyFunctionCall != null) {
|
|
585
|
+
if (!("value" in legacyFunctionCall)) return Number.MAX_SAFE_INTEGER;
|
|
586
|
+
if (legacyFunctionCall.value != null) numTokens += getBoundedStructuredTokenCount(legacyFunctionCall.value, countText);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
return Math.min(Number.MAX_SAFE_INTEGER, numTokens);
|
|
304
590
|
}
|
|
305
591
|
/**
|
|
306
592
|
* Largest-remainder apportionment: scales each count by `multiplier` and
|
|
@@ -373,5 +659,6 @@ exports.estimateOpenAIImageTokens = estimateOpenAIImageTokens;
|
|
|
373
659
|
exports.estimateTimedMediaBlockTokens = estimateTimedMediaBlockTokens;
|
|
374
660
|
exports.extractImageDimensions = extractImageDimensions;
|
|
375
661
|
exports.getTokenCountForMessage = getTokenCountForMessage;
|
|
662
|
+
exports.hasUnsafeStructuredSerialization = hasUnsafeStructuredSerialization;
|
|
376
663
|
|
|
377
664
|
//# sourceMappingURL=tokens.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokens.cjs","names":["Tokenizer"],"sources":["../../../src/utils/tokens.ts"],"sourcesContent":["import { Tokenizer } from 'ai-tokenizer';\nimport type { BaseMessage } from '@langchain/core/messages';\nimport { ContentTypes } from '@/common/enum';\n\nexport type EncodingName = 'o200k_base' | 'claude';\n\n/** Anthropic minimum image token cost. */\nconst ANTHROPIC_IMAGE_MIN_TOKENS = 1024;\n/** Anthropic divisor: tokens = width × height / 750. */\nconst ANTHROPIC_IMAGE_DIVISOR = 750;\n/** OpenAI low-detail fixed cost. */\nconst OPENAI_IMAGE_LOW_TOKENS = 85;\n/** OpenAI high-detail tile size. */\nconst OPENAI_IMAGE_TILE_SIZE = 512;\n/** OpenAI high-detail tokens per tile. */\nconst OPENAI_IMAGE_TOKENS_PER_TILE = 170;\n/** Google Gemini fixed per-image cost. */\nconst _GEMINI_IMAGE_TOKENS = 258;\n/** Safety margin for image and document token estimates (5% overestimate). */\nexport const IMAGE_TOKEN_SAFETY_MARGIN = 1.05;\n\n/**\n * Anthropic PDF: each page costs image tokens + text tokens.\n * Typical range is 1500-3000 tokens/page. Using 2000 as midpoint.\n */\nconst ANTHROPIC_PDF_TOKENS_PER_PAGE = 2000;\n/** OpenAI PDF: each page rendered as high-detail image. ~1500 tokens typical. */\nconst OPENAI_PDF_TOKENS_PER_PAGE = 1500;\n/** Gemini PDF: fixed 258 tokens per page. */\nconst _GEMINI_PDF_TOKENS_PER_PAGE = 258;\n/** Approximate base64 bytes per PDF page for page count estimation. */\nconst BASE64_BYTES_PER_PDF_PAGE = 75_000;\n/** Fallback token cost for URL-referenced documents without local data. */\nconst URL_DOCUMENT_FALLBACK_TOKENS = 2000;\n\n/**\n * Timed media (video/audio) is priced by DURATION, not size, and the content\n * carries no duration — so duration is estimated from encoded size at\n * representative bitrates and priced at Gemini's rates (the dominant provider\n * for native video/audio). Deliberately rough; superseded by real provider\n * usage after the first turn.\n */\n/** Gemini video ≈ 258 tokens/frame (1 fps) + 32 tokens/s audio ≈ 300 tokens/s. */\nconst VIDEO_TOKENS_PER_SECOND = 300;\n/** Gemini audio: 32 tokens/s (single channel). */\nconst AUDIO_TOKENS_PER_SECOND = 32;\n/** Representative encoded byte rates for duration-from-size estimation. */\nconst VIDEO_BYTES_PER_SECOND = 250_000; // ~2 Mbps\nconst AUDIO_BYTES_PER_SECOND = 16_000; // ~128 kbps\n/** Flat fallback when only a URL is present (no size to estimate from) — ~30s. */\nconst VIDEO_URL_FALLBACK_TOKENS = 9000;\nconst AUDIO_URL_FALLBACK_TOKENS = 960;\n/** Content block types that can carry timed media (video/audio). `media` is\n * generic (Google) so it is classified by MIME; the rest are unambiguous. */\nconst TIMED_MEDIA_TYPES = new Set([\n 'media',\n 'video',\n 'audio',\n 'video_url',\n 'input_audio',\n]);\n\n/**\n * Extracts image dimensions from the first bytes of a base64-encoded\n * PNG, JPEG, GIF, or WebP without decoding the full image.\n * Returns null if the format is unrecognized or data is too short.\n */\nexport function extractImageDimensions(\n base64Data: string\n): { width: number; height: number } | null {\n const raw = base64Data.startsWith('data:')\n ? base64Data.slice(base64Data.indexOf(',') + 1)\n : base64Data;\n\n if (raw.length < 32) {\n return null;\n }\n\n const bytes = new Uint8Array(Buffer.from(raw.slice(0, 80), 'base64'));\n\n if (bytes[0] === 0x89 && bytes[1] === 0x50) {\n // PNG: width at bytes 16-19, height at 20-23 (big-endian)\n const width =\n (bytes[16] << 24) | (bytes[17] << 16) | (bytes[18] << 8) | bytes[19];\n const height =\n (bytes[20] << 24) | (bytes[21] << 16) | (bytes[22] << 8) | bytes[23];\n return { width, height };\n }\n\n if (bytes[0] === 0xff && bytes[1] === 0xd8) {\n // JPEG: scan for SOF0 (0xFFC0) or SOF2 (0xFFC2) marker\n for (let i = 2; i < bytes.length - 9; i++) {\n if (\n bytes[i] === 0xff &&\n (bytes[i + 1] === 0xc0 || bytes[i + 1] === 0xc2)\n ) {\n const height = (bytes[i + 5] << 8) | bytes[i + 6];\n const width = (bytes[i + 7] << 8) | bytes[i + 8];\n return { width, height };\n }\n }\n return null;\n }\n\n if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46) {\n // GIF: width at bytes 6-7, height at 8-9 (little-endian)\n const width = bytes[6] | (bytes[7] << 8);\n const height = bytes[8] | (bytes[9] << 8);\n return { width, height };\n }\n\n if (\n bytes[0] === 0x52 &&\n bytes[1] === 0x49 &&\n bytes[2] === 0x46 &&\n bytes[3] === 0x46 &&\n bytes[8] === 0x57 &&\n bytes[9] === 0x45 &&\n bytes[10] === 0x42 &&\n bytes[11] === 0x50\n ) {\n // WebP VP8: width at bytes 26-27, height at 28-29\n if (bytes.length > 29) {\n const width = (bytes[26] | (bytes[27] << 8)) & 0x3fff;\n const height = (bytes[28] | (bytes[29] << 8)) & 0x3fff;\n return { width, height };\n }\n return null;\n }\n\n return null;\n}\n\n/** Estimates image token cost for Anthropic/Bedrock (Claude). */\nexport function estimateAnthropicImageTokens(\n width: number,\n height: number\n): number {\n return Math.max(\n ANTHROPIC_IMAGE_MIN_TOKENS,\n Math.ceil((width * height) / ANTHROPIC_IMAGE_DIVISOR)\n );\n}\n\n/** Estimates image token cost for OpenAI (high detail). */\nexport function estimateOpenAIImageTokens(\n width: number,\n height: number,\n detail: string = 'high'\n): number {\n if (detail === 'low') {\n return OPENAI_IMAGE_LOW_TOKENS;\n }\n const tiles =\n Math.ceil(width / OPENAI_IMAGE_TILE_SIZE) *\n Math.ceil(height / OPENAI_IMAGE_TILE_SIZE);\n return OPENAI_IMAGE_LOW_TOKENS + tiles * OPENAI_IMAGE_TOKENS_PER_TILE;\n}\n\n/**\n * Estimates token cost for an image content block.\n * Extracts dimensions from base64 header when available.\n * Falls back to Anthropic minimum (1024) when dimensions can't be determined.\n */\nexport function estimateImageBlockTokens(\n block: Record<string, unknown>,\n encoding: EncodingName\n): number {\n let base64Data: string | undefined;\n\n if (block.type === ContentTypes.IMAGE_URL || block.type === 'image_url') {\n const imageUrl = block.image_url as string | { url?: string } | undefined;\n const url = typeof imageUrl === 'string' ? imageUrl : imageUrl?.url;\n if (typeof url === 'string' && url.startsWith('data:')) {\n base64Data = url;\n } else {\n return ANTHROPIC_IMAGE_MIN_TOKENS;\n }\n } else if (block.type === 'image') {\n const source = block.source as { type?: string; data?: string } | undefined;\n if (source?.type === 'base64' && typeof source.data === 'string') {\n base64Data = source.data;\n } else {\n return ANTHROPIC_IMAGE_MIN_TOKENS;\n }\n } else {\n return ANTHROPIC_IMAGE_MIN_TOKENS;\n }\n\n const dims = extractImageDimensions(base64Data);\n if (dims == null) {\n return ANTHROPIC_IMAGE_MIN_TOKENS;\n }\n\n if (encoding === 'claude') {\n return estimateAnthropicImageTokens(dims.width, dims.height);\n }\n return estimateOpenAIImageTokens(dims.width, dims.height);\n}\n\n/**\n * Estimates token cost for a document/file content block.\n * Handles both LangChain standard format (`type: 'file'` with `source_type`)\n * and Anthropic format (`type: 'document'` with `source`).\n *\n * - Plain text: tokenized directly via `getTokenCount`.\n * - Base64 PDF: page count estimated from base64 length × per-page cost.\n * - URL reference: conservative flat estimate.\n */\nexport function estimateDocumentBlockTokens(\n block: Record<string, unknown>,\n encoding: EncodingName,\n getTokenCount: (text: string) => number\n): number {\n const pdfTokensPerPage =\n encoding === 'claude'\n ? ANTHROPIC_PDF_TOKENS_PER_PAGE\n : OPENAI_PDF_TOKENS_PER_PAGE;\n\n // LangChain standard format: type='file', source_type, data/text/url, mime_type\n const sourceType = block.source_type as string | undefined;\n if (typeof sourceType === 'string') {\n const mimeType = ((block.mime_type as string | undefined) ?? '').split(\n ';'\n )[0];\n\n if (sourceType === 'text' && typeof block.text === 'string') {\n return getTokenCount(block.text as string);\n }\n\n if (sourceType === 'base64' && typeof block.data === 'string') {\n if (mimeType === 'application/pdf' || mimeType === '') {\n const pageEstimate = Math.max(\n 1,\n Math.ceil((block.data as string).length / BASE64_BYTES_PER_PDF_PAGE)\n );\n return pageEstimate * pdfTokensPerPage;\n }\n // Image inside a file block — delegate to image estimation\n if (mimeType.startsWith('image/')) {\n return estimateImageBlockTokens(\n {\n ...block,\n type: 'image',\n source: { type: 'base64', data: block.data },\n },\n encoding\n );\n }\n return getTokenCount(block.data as string);\n }\n\n if (sourceType === 'url') {\n return URL_DOCUMENT_FALLBACK_TOKENS;\n }\n\n return URL_DOCUMENT_FALLBACK_TOKENS;\n }\n\n // Anthropic format: type='document', source: { type, data, media_type }\n const source = block.source as\n | {\n type?: string;\n data?: string;\n media_type?: string;\n content?: unknown[];\n }\n | undefined;\n\n if (source == null) {\n return URL_DOCUMENT_FALLBACK_TOKENS;\n }\n\n if (source.type === 'text' && typeof source.data === 'string') {\n return getTokenCount(source.data);\n }\n\n if (source.type === 'base64' && typeof source.data === 'string') {\n const mediaType = (source.media_type ?? '').split(';')[0];\n if (mediaType === 'application/pdf' || mediaType === '') {\n const pageEstimate = Math.max(\n 1,\n Math.ceil(source.data.length / BASE64_BYTES_PER_PDF_PAGE)\n );\n return pageEstimate * pdfTokensPerPage;\n }\n if (mediaType.startsWith('image/')) {\n return estimateImageBlockTokens(\n { type: 'image', source: { type: 'base64', data: source.data } },\n encoding\n );\n }\n return getTokenCount(source.data);\n }\n\n if (source.type === 'url') {\n return URL_DOCUMENT_FALLBACK_TOKENS;\n }\n\n // content-type source (wraps other blocks like images)\n if (source.type === 'content' && Array.isArray(source.content)) {\n let total = 0;\n for (const inner of source.content) {\n if (inner != null && typeof inner === 'object' && 'type' in inner) {\n const innerBlock = inner as Record<string, unknown>;\n if (innerBlock.type === 'image') {\n total += estimateImageBlockTokens(innerBlock, encoding);\n }\n }\n }\n return total > 0 ? total : URL_DOCUMENT_FALLBACK_TOKENS;\n }\n\n return URL_DOCUMENT_FALLBACK_TOKENS;\n}\n\n/** Decoded byte length of base64 or a `data:` URL; 0 for any remote URI (http,\n * gs, s3, file, …) or empty. Base64 never contains `:`, so any scheme prefix\n * marks a remote reference with no local size. */\nfunction base64ByteLength(value: string | undefined): number {\n if (typeof value !== 'string' || value.length === 0) {\n return 0;\n }\n if (value.startsWith('data:')) {\n const comma = value.indexOf(',');\n return comma < 0 ? 0 : Math.floor(((value.length - comma - 1) * 3) / 4);\n }\n if (/^[a-z][a-z0-9+.-]*:/i.test(value)) {\n return 0;\n }\n return Math.floor((value.length * 3) / 4);\n}\n\nfunction timedMediaTokens(\n bytes: number,\n bytesPerSecond: number,\n tokensPerSecond: number,\n urlFallback: number\n): number {\n if (bytes <= 0) {\n return urlFallback;\n }\n return Math.max(\n tokensPerSecond,\n Math.ceil((bytes / bytesPerSecond) * tokensPerSecond)\n );\n}\n\n/** Decoded byte length of a media block payload — top-level `data` (base64\n * string or `Uint8Array`) or base64 `url`, or the nested `video|audio.{data,\n * url, source.bytes}` shapes (`formatMessage` media arrays / native Bedrock);\n * 0 for a bare remote URL, `fileId`/`fileUri`, S3 location, or empty. */\nfunction mediaBlockByteLength(block: Record<string, unknown>): number {\n const data = block.data;\n if (typeof data === 'string') {\n return base64ByteLength(data);\n }\n if (data instanceof Uint8Array) {\n return data.length;\n }\n if (typeof block.url === 'string') {\n return base64ByteLength(block.url);\n }\n const nested = (block.video ?? block.audio) as\n | { data?: unknown; url?: unknown; source?: { bytes?: unknown } }\n | undefined;\n if (nested != null) {\n if (typeof nested.data === 'string') {\n return base64ByteLength(nested.data);\n }\n if (typeof nested.url === 'string') {\n return base64ByteLength(nested.url);\n }\n if (nested.source?.bytes instanceof Uint8Array) {\n return nested.source.bytes.length;\n }\n }\n return 0;\n}\n\n/** Classifies a block as timed media, or null when it is not video/audio — e.g.\n * a generic Google `media` block carrying an image/document MIME, which must\n * NOT be priced as video. Also handles the Google shape where `type` IS the\n * MIME string (e.g. `{ type: 'audio/wav', data }`). */\nfunction timedMediaKind(\n block: Record<string, unknown>\n): 'video' | 'audio' | null {\n const type = typeof block.type === 'string' ? block.type : '';\n if (type === 'input_audio' || type === 'audio') {\n return 'audio';\n }\n if (type === 'video_url' || type === 'video') {\n return 'video';\n }\n const mime =\n type === 'media' && typeof block.mimeType === 'string' ? block.mimeType : type;\n if (mime.startsWith('audio/')) {\n return 'audio';\n }\n if (mime.startsWith('video/')) {\n return 'video';\n }\n return null;\n}\n\n/** Whether a content-block `type` can carry timed media — the fixed set plus the\n * Google MIME-as-type shape (`audio/*` / `video/*`). Gates callers before they\n * hand the block to {@link estimateTimedMediaBlockTokens}. */\nfunction isTimedMediaType(type: string): boolean {\n return (\n TIMED_MEDIA_TYPES.has(type) ||\n type.startsWith('audio/') ||\n type.startsWith('video/')\n );\n}\n\n/**\n * Estimates token cost for a timed-media block (video/audio). Handles Google\n * `{ type: 'media', mimeType, data|url|fileUri }`, OpenRouter\n * `{ type: 'video_url' }` / `{ type: 'input_audio' }`, and standard\n * `{ type: 'video' }` / `{ type: 'audio' }` blocks (payload as `data` base64 or\n * `Uint8Array`, base64 `url`, or `fileId`). Duration is inferred from encoded\n * size (providers price by duration, which the block does not carry) at Gemini's\n * rates; a payload with no size (bare URL / file id) falls back to a ~30s\n * estimate. Returns 0 for non-timed media (e.g. an image/document `media` block)\n * so it is never mispriced as video.\n */\nexport function estimateTimedMediaBlockTokens(\n block: Record<string, unknown>\n): number {\n const kind = timedMediaKind(block);\n if (kind == null) {\n return 0;\n }\n let bytes: number;\n if (block.type === 'input_audio') {\n const audio = block.input_audio as { data?: string } | undefined;\n bytes = base64ByteLength(audio?.data);\n } else if (block.type === 'video_url') {\n const video = block.video_url as string | { url?: string } | undefined;\n bytes = base64ByteLength(typeof video === 'string' ? video : video?.url);\n } else {\n bytes = mediaBlockByteLength(block);\n }\n if (kind === 'audio') {\n return timedMediaTokens(\n bytes,\n AUDIO_BYTES_PER_SECOND,\n AUDIO_TOKENS_PER_SECOND,\n AUDIO_URL_FALLBACK_TOKENS\n );\n }\n return timedMediaTokens(\n bytes,\n VIDEO_BYTES_PER_SECOND,\n VIDEO_TOKENS_PER_SECOND,\n VIDEO_URL_FALLBACK_TOKENS\n );\n}\n\nconst tokenizers: Partial<Record<EncodingName, Tokenizer>> = {};\n\nasync function getTokenizer(\n encoding: EncodingName = 'o200k_base'\n): Promise<Tokenizer> {\n const cached = tokenizers[encoding];\n if (cached) {\n return cached;\n }\n const data =\n encoding === 'claude'\n ? await import('ai-tokenizer/encoding/claude')\n : await import('ai-tokenizer/encoding/o200k_base');\n const instance = new Tokenizer(data);\n tokenizers[encoding] = instance;\n return instance;\n}\n\nexport function encodingForModel(model: string): EncodingName {\n if (model.toLowerCase().includes('claude')) {\n return 'claude';\n }\n return 'o200k_base';\n}\n\nexport function getTokenCountForMessage(\n message: BaseMessage,\n getTokenCount: (text: string) => number,\n encoding: EncodingName = 'o200k_base'\n): number {\n const tokensPerMessage = 3;\n\n type ContentBlock = Record<string, unknown> & {\n type?: string;\n tool_call?: { name?: string; args?: string; output?: string };\n };\n\n const processValue = (value: unknown): void => {\n if (Array.isArray(value)) {\n for (const raw of value) {\n const item = raw as ContentBlock | null | undefined;\n if (item == null || typeof item.type !== 'string') {\n continue;\n }\n if (item.type === ContentTypes.ERROR) {\n continue;\n }\n\n if (\n item.type === ContentTypes.IMAGE_URL ||\n item.type === 'image_url' ||\n item.type === 'image'\n ) {\n numTokens += Math.ceil(\n estimateImageBlockTokens(item, encoding) * IMAGE_TOKEN_SAFETY_MARGIN\n );\n continue;\n }\n\n if (\n item.type === 'document' ||\n item.type === 'file' ||\n item.type === ContentTypes.IMAGE_FILE\n ) {\n numTokens += Math.ceil(\n estimateDocumentBlockTokens(item, encoding, getTokenCount) *\n IMAGE_TOKEN_SAFETY_MARGIN\n );\n continue;\n }\n\n if (isTimedMediaType(item.type)) {\n numTokens += Math.ceil(\n estimateTimedMediaBlockTokens(item) * IMAGE_TOKEN_SAFETY_MARGIN\n );\n continue;\n }\n\n if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {\n const toolName = item.tool_call.name;\n if (typeof toolName === 'string' && toolName.length > 0) {\n numTokens += getTokenCount(toolName);\n }\n const args = item.tool_call.args;\n if (typeof args === 'string' && args.length > 0) {\n numTokens += getTokenCount(args);\n }\n const output = item.tool_call.output;\n if (typeof output === 'string' && output.length > 0) {\n numTokens += getTokenCount(output);\n }\n continue;\n }\n\n const nestedValue = item[item.type];\n if (nestedValue == null) {\n continue;\n }\n\n processValue(nestedValue);\n }\n } else if (typeof value === 'string') {\n numTokens += getTokenCount(value);\n } else if (typeof value === 'number') {\n numTokens += getTokenCount(value.toString());\n } else if (typeof value === 'boolean') {\n numTokens += getTokenCount(value.toString());\n }\n };\n\n let numTokens = tokensPerMessage;\n processValue(message.content);\n return numTokens;\n}\n\n/**\n * Largest-remainder apportionment: scales each count by `multiplier` and\n * distributes the rounding remainder so the results sum exactly to\n * `targetTotal`. Keeps per-item breakdowns reconciled with an aggregate\n * computed as a single rounded product of the summed raw counts.\n */\nexport function apportionTokenCounts(\n rawCounts: Record<string, number>,\n multiplier: number,\n targetTotal: number\n): Record<string, number> {\n const result: Record<string, number> = Object.create(null);\n const remainders: Array<{ name: string; remainder: number }> = [];\n let floorSum = 0;\n for (const [name, rawCount] of Object.entries(rawCounts)) {\n const scaled = rawCount * multiplier;\n const floored = Math.floor(scaled);\n result[name] = floored;\n floorSum += floored;\n remainders.push({ name, remainder: scaled - floored });\n }\n let leftover = targetTotal - floorSum;\n if (leftover <= 0 || remainders.length === 0) {\n return result;\n }\n remainders.sort((a, b) => b.remainder - a.remainder);\n for (let i = 0; leftover > 0; i = (i + 1) % remainders.length) {\n result[remainders[i].name] += 1;\n leftover--;\n }\n return result;\n}\n\n/**\n * Anthropic's API consistently reports ~10% more tokens than the local\n * claude tokenizer due to internal message framing and content encoding.\n * Verified empirically across content types via the count_tokens endpoint.\n */\nconst CLAUDE_TOKEN_CORRECTION = 1.1;\n\n/**\n * Creates a token counter function using the specified encoding.\n * Lazily loads the encoding data on first use via dynamic import.\n */\nexport const createTokenCounter = async (\n encoding: EncodingName = 'o200k_base'\n): Promise<(message: BaseMessage) => number> => {\n const tok = await getTokenizer(encoding);\n const countTokens = (text: string): number => tok.count(text);\n const isClaude = encoding === 'claude';\n return (message: BaseMessage): number => {\n const count = getTokenCountForMessage(message, countTokens, encoding);\n return isClaude ? Math.ceil(count * CLAUDE_TOKEN_CORRECTION) : count;\n };\n};\n\n/** Utility to manage the token encoder lifecycle explicitly. */\nexport const TokenEncoderManager = {\n async initialize(): Promise<void> {\n // No-op: ai-tokenizer is synchronously initialized from bundled data.\n },\n\n reset(): void {\n for (const key of Object.keys(tokenizers)) {\n delete tokenizers[key as EncodingName];\n }\n },\n\n isInitialized(): boolean {\n return Object.keys(tokenizers).length > 0;\n },\n};\n"],"mappings":";;;;AAOA,MAAM,6BAA6B;;AAEnC,MAAM,0BAA0B;;AAEhC,MAAM,0BAA0B;;AAEhC,MAAM,yBAAyB;;AAE/B,MAAM,+BAA+B;;AAIrC,MAAa,4BAA4B;;;;;AAMzC,MAAM,gCAAgC;;AAEtC,MAAM,6BAA6B;;AAInC,MAAM,4BAA4B;;AAElC,MAAM,+BAA+B;;;;;;;;;AAUrC,MAAM,0BAA0B;;AAEhC,MAAM,0BAA0B;;AAEhC,MAAM,yBAAyB;AAC/B,MAAM,yBAAyB;;AAE/B,MAAM,4BAA4B;AAClC,MAAM,4BAA4B;;;AAGlC,MAAM,oBAAoB,IAAI,IAAI;CAChC;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;AAOD,SAAgB,uBACd,YAC0C;CAC1C,MAAM,MAAM,WAAW,WAAW,OAAO,IACrC,WAAW,MAAM,WAAW,QAAQ,GAAG,IAAI,CAAC,IAC5C;CAEJ,IAAI,IAAI,SAAS,IACf,OAAO;CAGT,MAAM,QAAQ,IAAI,WAAW,OAAO,KAAK,IAAI,MAAM,GAAG,EAAE,GAAG,QAAQ,CAAC;CAEpE,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,IAMpC,OAAO;EAAE,OAHN,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;EAGnD,QADb,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;CAC5C;CAGzB,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,KAAM;EAE1C,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,GAAG,KACpC,IACE,MAAM,OAAO,QACZ,MAAM,IAAI,OAAO,OAAQ,MAAM,IAAI,OAAO,MAC3C;GACA,MAAM,SAAU,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;GAE/C,OAAO;IAAE,OADM,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;IAC9B;GAAO;EACzB;EAEF,OAAO;CACT;CAEA,IAAI,MAAM,OAAO,MAAQ,MAAM,OAAO,MAAQ,MAAM,OAAO,IAIzD,OAAO;EAAE,OAFK,MAAM,KAAM,MAAM,MAAM;EAEtB,QADD,MAAM,KAAM,MAAM,MAAM;CAChB;CAGzB,IACE,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,QAAQ,MACd,MAAM,QAAQ,IACd;EAEA,IAAI,MAAM,SAAS,IAGjB,OAAO;GAAE,QAFM,MAAM,MAAO,MAAM,OAAO,KAAM;GAE/B,SADA,MAAM,MAAO,MAAM,OAAO,KAAM;EACzB;EAEzB,OAAO;CACT;CAEA,OAAO;AACT;;AAGA,SAAgB,6BACd,OACA,QACQ;CACR,OAAO,KAAK,IACV,4BACA,KAAK,KAAM,QAAQ,SAAU,uBAAuB,CACtD;AACF;;AAGA,SAAgB,0BACd,OACA,QACA,SAAiB,QACT;CACR,IAAI,WAAW,OACb,OAAO;CAKT,OAAO,0BAFL,KAAK,KAAK,QAAQ,sBAAsB,IACxC,KAAK,KAAK,SAAS,sBAAsB,IACF;AAC3C;;;;;;AAOA,SAAgB,yBACd,OACA,UACQ;CACR,IAAI;CAEJ,IAAI,MAAM,SAAA,eAAmC,MAAM,SAAS,aAAa;EACvE,MAAM,WAAW,MAAM;EACvB,MAAM,MAAM,OAAO,aAAa,WAAW,WAAW,UAAU;EAChE,IAAI,OAAO,QAAQ,YAAY,IAAI,WAAW,OAAO,GACnD,aAAa;OAEb,OAAO;CAEX,OAAO,IAAI,MAAM,SAAS,SAAS;EACjC,MAAM,SAAS,MAAM;EACrB,IAAI,QAAQ,SAAS,YAAY,OAAO,OAAO,SAAS,UACtD,aAAa,OAAO;OAEpB,OAAO;CAEX,OACE,OAAO;CAGT,MAAM,OAAO,uBAAuB,UAAU;CAC9C,IAAI,QAAQ,MACV,OAAO;CAGT,IAAI,aAAa,UACf,OAAO,6BAA6B,KAAK,OAAO,KAAK,MAAM;CAE7D,OAAO,0BAA0B,KAAK,OAAO,KAAK,MAAM;AAC1D;;;;;;;;;;AAWA,SAAgB,4BACd,OACA,UACA,eACQ;CACR,MAAM,mBACJ,aAAa,WACT,gCACA;CAGN,MAAM,aAAa,MAAM;CACzB,IAAI,OAAO,eAAe,UAAU;EAClC,MAAM,YAAa,MAAM,aAAoC,GAAA,CAAI,MAC/D,GACF,CAAC,CAAC;EAEF,IAAI,eAAe,UAAU,OAAO,MAAM,SAAS,UACjD,OAAO,cAAc,MAAM,IAAc;EAG3C,IAAI,eAAe,YAAY,OAAO,MAAM,SAAS,UAAU;GAC7D,IAAI,aAAa,qBAAqB,aAAa,IAKjD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAM,MAAM,KAAgB,SAAS,yBAAyB,CAEnD,IAAI;GAGxB,IAAI,SAAS,WAAW,QAAQ,GAC9B,OAAO,yBACL;IACE,GAAG;IACH,MAAM;IACN,QAAQ;KAAE,MAAM;KAAU,MAAM,MAAM;IAAK;GAC7C,GACA,QACF;GAEF,OAAO,cAAc,MAAM,IAAc;EAC3C;EAEA,IAAI,eAAe,OACjB,OAAO;EAGT,OAAO;CACT;CAGA,MAAM,SAAS,MAAM;CASrB,IAAI,UAAU,MACZ,OAAO;CAGT,IAAI,OAAO,SAAS,UAAU,OAAO,OAAO,SAAS,UACnD,OAAO,cAAc,OAAO,IAAI;CAGlC,IAAI,OAAO,SAAS,YAAY,OAAO,OAAO,SAAS,UAAU;EAC/D,MAAM,aAAa,OAAO,cAAc,GAAA,CAAI,MAAM,GAAG,CAAC,CAAC;EACvD,IAAI,cAAc,qBAAqB,cAAc,IAKnD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAK,OAAO,KAAK,SAAS,yBAAyB,CAExC,IAAI;EAExB,IAAI,UAAU,WAAW,QAAQ,GAC/B,OAAO,yBACL;GAAE,MAAM;GAAS,QAAQ;IAAE,MAAM;IAAU,MAAM,OAAO;GAAK;EAAE,GAC/D,QACF;EAEF,OAAO,cAAc,OAAO,IAAI;CAClC;CAEA,IAAI,OAAO,SAAS,OAClB,OAAO;CAIT,IAAI,OAAO,SAAS,aAAa,MAAM,QAAQ,OAAO,OAAO,GAAG;EAC9D,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,OAAO,SACzB,IAAI,SAAS,QAAQ,OAAO,UAAU,YAAY,UAAU,OAAO;GACjE,MAAM,aAAa;GACnB,IAAI,WAAW,SAAS,SACtB,SAAS,yBAAyB,YAAY,QAAQ;EAE1D;EAEF,OAAO,QAAQ,IAAI,QAAQ;CAC7B;CAEA,OAAO;AACT;;;;AAKA,SAAS,iBAAiB,OAAmC;CAC3D,IAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAChD,OAAO;CAET,IAAI,MAAM,WAAW,OAAO,GAAG;EAC7B,MAAM,QAAQ,MAAM,QAAQ,GAAG;EAC/B,OAAO,QAAQ,IAAI,IAAI,KAAK,OAAQ,MAAM,SAAS,QAAQ,KAAK,IAAK,CAAC;CACxE;CACA,IAAI,uBAAuB,KAAK,KAAK,GACnC,OAAO;CAET,OAAO,KAAK,MAAO,MAAM,SAAS,IAAK,CAAC;AAC1C;AAEA,SAAS,iBACP,OACA,gBACA,iBACA,aACQ;CACR,IAAI,SAAS,GACX,OAAO;CAET,OAAO,KAAK,IACV,iBACA,KAAK,KAAM,QAAQ,iBAAkB,eAAe,CACtD;AACF;;;;;AAMA,SAAS,qBAAqB,OAAwC;CACpE,MAAM,OAAO,MAAM;CACnB,IAAI,OAAO,SAAS,UAClB,OAAO,iBAAiB,IAAI;CAE9B,IAAI,gBAAgB,YAClB,OAAO,KAAK;CAEd,IAAI,OAAO,MAAM,QAAQ,UACvB,OAAO,iBAAiB,MAAM,GAAG;CAEnC,MAAM,SAAU,MAAM,SAAS,MAAM;CAGrC,IAAI,UAAU,MAAM;EAClB,IAAI,OAAO,OAAO,SAAS,UACzB,OAAO,iBAAiB,OAAO,IAAI;EAErC,IAAI,OAAO,OAAO,QAAQ,UACxB,OAAO,iBAAiB,OAAO,GAAG;EAEpC,IAAI,OAAO,QAAQ,iBAAiB,YAClC,OAAO,OAAO,OAAO,MAAM;CAE/B;CACA,OAAO;AACT;;;;;AAMA,SAAS,eACP,OAC0B;CAC1B,MAAM,OAAO,OAAO,MAAM,SAAS,WAAW,MAAM,OAAO;CAC3D,IAAI,SAAS,iBAAiB,SAAS,SACrC,OAAO;CAET,IAAI,SAAS,eAAe,SAAS,SACnC,OAAO;CAET,MAAM,OACJ,SAAS,WAAW,OAAO,MAAM,aAAa,WAAW,MAAM,WAAW;CAC5E,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,OAAO;AACT;;;;AAKA,SAAS,iBAAiB,MAAuB;CAC/C,OACE,kBAAkB,IAAI,IAAI,KAC1B,KAAK,WAAW,QAAQ,KACxB,KAAK,WAAW,QAAQ;AAE5B;;;;;;;;;;;;AAaA,SAAgB,8BACd,OACQ;CACR,MAAM,OAAO,eAAe,KAAK;CACjC,IAAI,QAAQ,MACV,OAAO;CAET,IAAI;CACJ,IAAI,MAAM,SAAS,eAAe;EAChC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,IAAI;CACtC,OAAO,IAAI,MAAM,SAAS,aAAa;EACrC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,UAAU,WAAW,QAAQ,OAAO,GAAG;CACzE,OACE,QAAQ,qBAAqB,KAAK;CAEpC,IAAI,SAAS,SACX,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;CAEF,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;AACF;AAEA,MAAM,aAAuD,CAAC;AAE9D,eAAe,aACb,WAAyB,cACL;CACpB,MAAM,SAAS,WAAW;CAC1B,IAAI,QACF,OAAO;CAMT,MAAM,WAAW,IAAIA,aAAAA,UAHnB,aAAa,WACT,MAAM,OAAO,kCACb,MAAM,OAAO,mCACgB;CACnC,WAAW,YAAY;CACvB,OAAO;AACT;AAEA,SAAgB,iBAAiB,OAA6B;CAC5D,IAAI,MAAM,YAAY,CAAC,CAAC,SAAS,QAAQ,GACvC,OAAO;CAET,OAAO;AACT;AAEA,SAAgB,wBACd,SACA,eACA,WAAyB,cACjB;CACR,MAAM,mBAAmB;CAOzB,MAAM,gBAAgB,UAAyB;EAC7C,IAAI,MAAM,QAAQ,KAAK,GACrB,KAAK,MAAM,OAAO,OAAO;GACvB,MAAM,OAAO;GACb,IAAI,QAAQ,QAAQ,OAAO,KAAK,SAAS,UACvC;GAEF,IAAI,KAAK,SAAA,SACP;GAGF,IACE,KAAK,SAAA,eACL,KAAK,SAAS,eACd,KAAK,SAAS,SACd;IACA,aAAa,KAAK,KAChB,yBAAyB,MAAM,QAAQ,IAAI,yBAC7C;IACA;GACF;GAEA,IACE,KAAK,SAAS,cACd,KAAK,SAAS,UACd,KAAK,SAAA,cACL;IACA,aAAa,KAAK,KAChB,4BAA4B,MAAM,UAAU,aAAa,IACvD,yBACJ;IACA;GACF;GAEA,IAAI,iBAAiB,KAAK,IAAI,GAAG;IAC/B,aAAa,KAAK,KAChB,8BAA8B,IAAI,IAAI,yBACxC;IACA;GACF;GAEA,IAAI,KAAK,SAAA,eAAmC,KAAK,aAAa,MAAM;IAClE,MAAM,WAAW,KAAK,UAAU;IAChC,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,aAAa,cAAc,QAAQ;IAErC,MAAM,OAAO,KAAK,UAAU;IAC5B,IAAI,OAAO,SAAS,YAAY,KAAK,SAAS,GAC5C,aAAa,cAAc,IAAI;IAEjC,MAAM,SAAS,KAAK,UAAU;IAC9B,IAAI,OAAO,WAAW,YAAY,OAAO,SAAS,GAChD,aAAa,cAAc,MAAM;IAEnC;GACF;GAEA,MAAM,cAAc,KAAK,KAAK;GAC9B,IAAI,eAAe,MACjB;GAGF,aAAa,WAAW;EAC1B;OACK,IAAI,OAAO,UAAU,UAC1B,aAAa,cAAc,KAAK;OAC3B,IAAI,OAAO,UAAU,UAC1B,aAAa,cAAc,MAAM,SAAS,CAAC;OACtC,IAAI,OAAO,UAAU,WAC1B,aAAa,cAAc,MAAM,SAAS,CAAC;CAE/C;CAEA,IAAI,YAAY;CAChB,aAAa,QAAQ,OAAO;CAC5B,OAAO;AACT;;;;;;;AAQA,SAAgB,qBACd,WACA,YACA,aACwB;CACxB,MAAM,SAAiC,OAAO,OAAO,IAAI;CACzD,MAAM,aAAyD,CAAC;CAChE,IAAI,WAAW;CACf,KAAK,MAAM,CAAC,MAAM,aAAa,OAAO,QAAQ,SAAS,GAAG;EACxD,MAAM,SAAS,WAAW;EAC1B,MAAM,UAAU,KAAK,MAAM,MAAM;EACjC,OAAO,QAAQ;EACf,YAAY;EACZ,WAAW,KAAK;GAAE;GAAM,WAAW,SAAS;EAAQ,CAAC;CACvD;CACA,IAAI,WAAW,cAAc;CAC7B,IAAI,YAAY,KAAK,WAAW,WAAW,GACzC,OAAO;CAET,WAAW,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CACnD,KAAK,IAAI,IAAI,GAAG,WAAW,GAAG,KAAK,IAAI,KAAK,WAAW,QAAQ;EAC7D,OAAO,WAAW,EAAE,CAAC,SAAS;EAC9B;CACF;CACA,OAAO;AACT;;;;;;AAOA,MAAM,0BAA0B;;;;;AAMhC,MAAa,qBAAqB,OAChC,WAAyB,iBACqB;CAC9C,MAAM,MAAM,MAAM,aAAa,QAAQ;CACvC,MAAM,eAAe,SAAyB,IAAI,MAAM,IAAI;CAC5D,MAAM,WAAW,aAAa;CAC9B,QAAQ,YAAiC;EACvC,MAAM,QAAQ,wBAAwB,SAAS,aAAa,QAAQ;EACpE,OAAO,WAAW,KAAK,KAAK,QAAQ,uBAAuB,IAAI;CACjE;AACF;;AAGA,MAAa,sBAAsB;CACjC,MAAM,aAA4B,CAElC;CAEA,QAAc;EACZ,KAAK,MAAM,OAAO,OAAO,KAAK,UAAU,GACtC,OAAO,WAAW;CAEtB;CAEA,gBAAyB;EACvB,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,SAAS;CAC1C;AACF"}
|
|
1
|
+
{"version":3,"file":"tokens.cjs","names":["Tokenizer","serializeStructuredValueBounded","isAtomicToolContentBlock","isComputerCallOutputContent"],"sources":["../../../src/utils/tokens.ts"],"sourcesContent":["import { Tokenizer } from 'ai-tokenizer';\nimport { isProxy } from 'node:util/types';\nimport type { AIMessage, BaseMessage } from '@langchain/core/messages';\nimport {\n isComputerCallOutputContent,\n isAtomicToolContentBlock,\n serializeStructuredValueBounded,\n} from './toolContent';\nimport { ContentTypes } from '@/common/enum';\n\nexport type EncodingName = 'o200k_base' | 'claude';\n\n/** Anthropic minimum image token cost. */\nconst ANTHROPIC_IMAGE_MIN_TOKENS = 1024;\n/** Anthropic divisor: tokens = width × height / 750. */\nconst ANTHROPIC_IMAGE_DIVISOR = 750;\n/** OpenAI low-detail fixed cost. */\nconst OPENAI_IMAGE_LOW_TOKENS = 85;\n/** OpenAI high-detail tile size. */\nconst OPENAI_IMAGE_TILE_SIZE = 512;\n/** OpenAI high-detail tokens per tile. */\nconst OPENAI_IMAGE_TOKENS_PER_TILE = 170;\n/** Google Gemini fixed per-image cost. */\nconst _GEMINI_IMAGE_TOKENS = 258;\n/** Safety margin for image and document token estimates (5% overestimate). */\nexport const IMAGE_TOKEN_SAFETY_MARGIN = 1.05;\n\n/**\n * Anthropic PDF: each page costs image tokens + text tokens.\n * Typical range is 1500-3000 tokens/page. Using 2000 as midpoint.\n */\nconst ANTHROPIC_PDF_TOKENS_PER_PAGE = 2000;\n/** OpenAI PDF: each page rendered as high-detail image. ~1500 tokens typical. */\nconst OPENAI_PDF_TOKENS_PER_PAGE = 1500;\n/** Gemini PDF: fixed 258 tokens per page. */\nconst _GEMINI_PDF_TOKENS_PER_PAGE = 258;\n/** Approximate base64 bytes per PDF page for page count estimation. */\nconst BASE64_BYTES_PER_PDF_PAGE = 75_000;\n/** Fallback token cost for URL-referenced documents without local data. */\nconst URL_DOCUMENT_FALLBACK_TOKENS = 2000;\n\n/**\n * Timed media (video/audio) is priced by DURATION, not size, and the content\n * carries no duration — so duration is estimated from encoded size at\n * representative bitrates and priced at Gemini's rates (the dominant provider\n * for native video/audio). Deliberately rough; superseded by real provider\n * usage after the first turn.\n */\n/** Gemini video ≈ 258 tokens/frame (1 fps) + 32 tokens/s audio ≈ 300 tokens/s. */\nconst VIDEO_TOKENS_PER_SECOND = 300;\n/** Gemini audio: 32 tokens/s (single channel). */\nconst AUDIO_TOKENS_PER_SECOND = 32;\n/** Representative encoded byte rates for duration-from-size estimation. */\nconst VIDEO_BYTES_PER_SECOND = 250_000; // ~2 Mbps\nconst AUDIO_BYTES_PER_SECOND = 16_000; // ~128 kbps\n/** Flat fallback when only a URL is present (no size to estimate from) — ~30s. */\nconst VIDEO_URL_FALLBACK_TOKENS = 9000;\nconst AUDIO_URL_FALLBACK_TOKENS = 960;\n/** Content block types that can carry timed media (video/audio). `media` is\n * generic (Google) so it is classified by MIME; the rest are unambiguous. */\nconst TIMED_MEDIA_TYPES = new Set([\n 'media',\n 'video',\n 'audio',\n 'video_url',\n 'input_audio',\n]);\n/**\n * Structured content is tokenized exactly up to this size. Larger values are\n * traversed by the bounded serializer and priced conservatively for the omitted\n * suffix, avoiding a full JSON string allocation in the context guard.\n */\nconst MAX_STRUCTURED_TOKENIZATION_CHARS = 200_000;\n/** One UTF-16 character can encode to several tokenizer pieces. */\nconst MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR = 4;\nconst MAX_STRUCTURED_SAFETY_INSPECTION_WORK = 10_000;\n\n/**\n * Extracts image dimensions from the first bytes of a base64-encoded\n * PNG, JPEG, GIF, or WebP without decoding the full image.\n * Returns null if the format is unrecognized or data is too short.\n */\nexport function extractImageDimensions(\n base64Data: string\n): { width: number; height: number } | null {\n const raw = base64Data.startsWith('data:')\n ? base64Data.slice(base64Data.indexOf(',') + 1)\n : base64Data;\n\n if (raw.length < 32) {\n return null;\n }\n\n const bytes = new Uint8Array(Buffer.from(raw.slice(0, 80), 'base64'));\n\n if (bytes[0] === 0x89 && bytes[1] === 0x50) {\n // PNG: width at bytes 16-19, height at 20-23 (big-endian)\n const width =\n (bytes[16] << 24) | (bytes[17] << 16) | (bytes[18] << 8) | bytes[19];\n const height =\n (bytes[20] << 24) | (bytes[21] << 16) | (bytes[22] << 8) | bytes[23];\n return { width, height };\n }\n\n if (bytes[0] === 0xff && bytes[1] === 0xd8) {\n // JPEG: scan for SOF0 (0xFFC0) or SOF2 (0xFFC2) marker\n for (let i = 2; i < bytes.length - 9; i++) {\n if (\n bytes[i] === 0xff &&\n (bytes[i + 1] === 0xc0 || bytes[i + 1] === 0xc2)\n ) {\n const height = (bytes[i + 5] << 8) | bytes[i + 6];\n const width = (bytes[i + 7] << 8) | bytes[i + 8];\n return { width, height };\n }\n }\n return null;\n }\n\n if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46) {\n // GIF: width at bytes 6-7, height at 8-9 (little-endian)\n const width = bytes[6] | (bytes[7] << 8);\n const height = bytes[8] | (bytes[9] << 8);\n return { width, height };\n }\n\n if (\n bytes[0] === 0x52 &&\n bytes[1] === 0x49 &&\n bytes[2] === 0x46 &&\n bytes[3] === 0x46 &&\n bytes[8] === 0x57 &&\n bytes[9] === 0x45 &&\n bytes[10] === 0x42 &&\n bytes[11] === 0x50\n ) {\n // WebP VP8: width at bytes 26-27, height at 28-29\n if (bytes.length > 29) {\n const width = (bytes[26] | (bytes[27] << 8)) & 0x3fff;\n const height = (bytes[28] | (bytes[29] << 8)) & 0x3fff;\n return { width, height };\n }\n return null;\n }\n\n return null;\n}\n\n/** Estimates image token cost for Anthropic/Bedrock (Claude). */\nexport function estimateAnthropicImageTokens(\n width: number,\n height: number\n): number {\n return Math.max(\n ANTHROPIC_IMAGE_MIN_TOKENS,\n Math.ceil((width * height) / ANTHROPIC_IMAGE_DIVISOR)\n );\n}\n\n/** Estimates image token cost for OpenAI (high detail). */\nexport function estimateOpenAIImageTokens(\n width: number,\n height: number,\n detail: string = 'high'\n): number {\n if (detail === 'low') {\n return OPENAI_IMAGE_LOW_TOKENS;\n }\n const tiles =\n Math.ceil(width / OPENAI_IMAGE_TILE_SIZE) *\n Math.ceil(height / OPENAI_IMAGE_TILE_SIZE);\n return OPENAI_IMAGE_LOW_TOKENS + tiles * OPENAI_IMAGE_TOKENS_PER_TILE;\n}\n\n/**\n * Estimates token cost for an image content block.\n * Extracts dimensions from base64 header when available.\n * Falls back to Anthropic minimum (1024) when dimensions can't be determined.\n */\nexport function estimateImageBlockTokens(\n block: Record<string, unknown>,\n encoding: EncodingName\n): number {\n let base64Data: string | undefined;\n\n if (\n block.type === ContentTypes.IMAGE_URL ||\n block.type === 'image_url' ||\n block.type === 'computer_screenshot' ||\n block.type === 'input_image'\n ) {\n const imageUrl = block.image_url as string | { url?: string } | undefined;\n const url = typeof imageUrl === 'string' ? imageUrl : imageUrl?.url;\n if (typeof url === 'string' && url.startsWith('data:')) {\n base64Data = url;\n } else if (typeof block.file_id === 'string' && block.file_id !== '') {\n return block.detail === 'low'\n ? OPENAI_IMAGE_LOW_TOKENS\n : ANTHROPIC_IMAGE_MIN_TOKENS;\n } else {\n return ANTHROPIC_IMAGE_MIN_TOKENS;\n }\n } else if (block.type === 'image') {\n const source = block.source as { type?: string; data?: string } | undefined;\n if (source?.type === 'base64' && typeof source.data === 'string') {\n base64Data = source.data;\n } else {\n return ANTHROPIC_IMAGE_MIN_TOKENS;\n }\n } else {\n return ANTHROPIC_IMAGE_MIN_TOKENS;\n }\n\n const dims = extractImageDimensions(base64Data);\n if (dims == null) {\n return ANTHROPIC_IMAGE_MIN_TOKENS;\n }\n\n if (encoding === 'claude') {\n return estimateAnthropicImageTokens(dims.width, dims.height);\n }\n return estimateOpenAIImageTokens(dims.width, dims.height);\n}\n\n/**\n * Estimates token cost for a document/file content block.\n * Handles both LangChain standard format (`type: 'file'` with `source_type`)\n * and Anthropic format (`type: 'document'` with `source`).\n *\n * - Plain text: tokenized directly via `getTokenCount`.\n * - Base64 PDF: page count estimated from base64 length × per-page cost.\n * - URL reference: conservative flat estimate.\n */\nexport function estimateDocumentBlockTokens(\n block: Record<string, unknown>,\n encoding: EncodingName,\n getTokenCount: (text: string) => number\n): number {\n const pdfTokensPerPage =\n encoding === 'claude'\n ? ANTHROPIC_PDF_TOKENS_PER_PAGE\n : OPENAI_PDF_TOKENS_PER_PAGE;\n\n // LangChain standard format: type='file', source_type, data/text/url, mime_type\n const sourceType = block.source_type as string | undefined;\n if (typeof sourceType === 'string') {\n const mimeType = ((block.mime_type as string | undefined) ?? '').split(\n ';'\n )[0];\n\n if (sourceType === 'text' && typeof block.text === 'string') {\n return getTokenCount(block.text as string);\n }\n\n if (sourceType === 'base64' && typeof block.data === 'string') {\n if (mimeType === 'application/pdf' || mimeType === '') {\n const pageEstimate = Math.max(\n 1,\n Math.ceil((block.data as string).length / BASE64_BYTES_PER_PDF_PAGE)\n );\n return pageEstimate * pdfTokensPerPage;\n }\n // Image inside a file block — delegate to image estimation\n if (mimeType.startsWith('image/')) {\n return estimateImageBlockTokens(\n {\n ...block,\n type: 'image',\n source: { type: 'base64', data: block.data },\n },\n encoding\n );\n }\n return getTokenCount(block.data as string);\n }\n\n if (sourceType === 'url') {\n return URL_DOCUMENT_FALLBACK_TOKENS;\n }\n\n return URL_DOCUMENT_FALLBACK_TOKENS;\n }\n\n // Anthropic format: type='document', source: { type, data, media_type }\n const source = block.source as\n | {\n type?: string;\n data?: string;\n media_type?: string;\n content?: unknown[];\n }\n | undefined;\n\n if (source == null) {\n return URL_DOCUMENT_FALLBACK_TOKENS;\n }\n\n if (source.type === 'text' && typeof source.data === 'string') {\n return getTokenCount(source.data);\n }\n\n if (source.type === 'base64' && typeof source.data === 'string') {\n const mediaType = (source.media_type ?? '').split(';')[0];\n if (mediaType === 'application/pdf' || mediaType === '') {\n const pageEstimate = Math.max(\n 1,\n Math.ceil(source.data.length / BASE64_BYTES_PER_PDF_PAGE)\n );\n return pageEstimate * pdfTokensPerPage;\n }\n if (mediaType.startsWith('image/')) {\n return estimateImageBlockTokens(\n { type: 'image', source: { type: 'base64', data: source.data } },\n encoding\n );\n }\n return getTokenCount(source.data);\n }\n\n if (source.type === 'url') {\n return URL_DOCUMENT_FALLBACK_TOKENS;\n }\n\n // content-type source (wraps other blocks like images)\n if (source.type === 'content' && Array.isArray(source.content)) {\n let total = 0;\n for (const inner of source.content) {\n if (inner != null && typeof inner === 'object' && 'type' in inner) {\n const innerBlock = inner as Record<string, unknown>;\n if (innerBlock.type === 'image') {\n total += estimateImageBlockTokens(innerBlock, encoding);\n }\n }\n }\n return total > 0 ? total : URL_DOCUMENT_FALLBACK_TOKENS;\n }\n\n return URL_DOCUMENT_FALLBACK_TOKENS;\n}\n\n/** Decoded byte length of base64 or a `data:` URL; 0 for any remote URI (http,\n * gs, s3, file, …) or empty. Base64 never contains `:`, so any scheme prefix\n * marks a remote reference with no local size. */\nfunction base64ByteLength(value: string | undefined): number {\n if (typeof value !== 'string' || value.length === 0) {\n return 0;\n }\n if (value.startsWith('data:')) {\n const comma = value.indexOf(',');\n return comma < 0 ? 0 : Math.floor(((value.length - comma - 1) * 3) / 4);\n }\n if (/^[a-z][a-z0-9+.-]*:/i.test(value)) {\n return 0;\n }\n return Math.floor((value.length * 3) / 4);\n}\n\nfunction timedMediaTokens(\n bytes: number,\n bytesPerSecond: number,\n tokensPerSecond: number,\n urlFallback: number\n): number {\n if (bytes <= 0) {\n return urlFallback;\n }\n return Math.max(\n tokensPerSecond,\n Math.ceil((bytes / bytesPerSecond) * tokensPerSecond)\n );\n}\n\n/** Decoded byte length of a media block payload — top-level `data` (base64\n * string or `Uint8Array`) or base64 `url`, or the nested `video|audio.{data,\n * url, source.bytes}` shapes (`formatMessage` media arrays / native Bedrock);\n * 0 for a bare remote URL, `fileId`/`fileUri`, S3 location, or empty. */\nfunction mediaBlockByteLength(block: Record<string, unknown>): number {\n const data = block.data;\n if (typeof data === 'string') {\n return base64ByteLength(data);\n }\n if (data instanceof Uint8Array) {\n return data.length;\n }\n if (typeof block.url === 'string') {\n return base64ByteLength(block.url);\n }\n const nested = (block.video ?? block.audio) as\n | { data?: unknown; url?: unknown; source?: { bytes?: unknown } }\n | undefined;\n if (nested != null) {\n if (typeof nested.data === 'string') {\n return base64ByteLength(nested.data);\n }\n if (typeof nested.url === 'string') {\n return base64ByteLength(nested.url);\n }\n if (nested.source?.bytes instanceof Uint8Array) {\n return nested.source.bytes.length;\n }\n }\n return 0;\n}\n\n/** Classifies a block as timed media, or null when it is not video/audio — e.g.\n * a generic Google `media` block carrying an image/document MIME, which must\n * NOT be priced as video. Also handles the Google shape where `type` IS the\n * MIME string (e.g. `{ type: 'audio/wav', data }`). */\nfunction timedMediaKind(\n block: Record<string, unknown>\n): 'video' | 'audio' | null {\n const type =\n typeof block.type === 'string' ? normalizeMediaMimeType(block.type) : '';\n if (type === 'input_audio' || type === 'audio') {\n return 'audio';\n }\n if (type === 'video_url' || type === 'video') {\n return 'video';\n }\n const mediaMime = getMediaMimeType(block);\n const mime = type === 'media' && mediaMime != null ? mediaMime : type;\n if (mime.startsWith('audio/')) {\n return 'audio';\n }\n if (mime.startsWith('video/')) {\n return 'video';\n }\n return null;\n}\n\nfunction getMediaMimeType(block: Record<string, unknown>): string | undefined {\n if (typeof block.mimeType === 'string') {\n return normalizeMediaMimeType(block.mimeType);\n }\n if (typeof block.mime_type === 'string') {\n return normalizeMediaMimeType(block.mime_type);\n }\n return undefined;\n}\n\nfunction normalizeMediaMimeType(mimeType: string): string {\n return mimeType.split(';')[0].trim().toLowerCase();\n}\n\n/** Whether a content-block `type` can carry timed media — the fixed set plus the\n * Google MIME-as-type shape (`audio/*` / `video/*`). Gates callers before they\n * hand the block to {@link estimateTimedMediaBlockTokens}. */\nfunction isTimedMediaType(type: string): boolean {\n const normalizedType = normalizeMediaMimeType(type);\n return (\n TIMED_MEDIA_TYPES.has(normalizedType) ||\n normalizedType.startsWith('audio/') ||\n normalizedType.startsWith('video/')\n );\n}\n\n/** Conservatively prices unknown inline media without materializing JSON. */\nfunction estimateUnknownMediaBlockTokens(data: unknown): number {\n if (typeof data === 'string') {\n if (base64ByteLength(data) <= 0) {\n return URL_DOCUMENT_FALLBACK_TOKENS;\n }\n return Math.max(URL_DOCUMENT_FALLBACK_TOKENS, data.length);\n }\n if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) {\n return Math.max(\n URL_DOCUMENT_FALLBACK_TOKENS,\n Math.ceil((data.byteLength * 4) / 3)\n );\n }\n return URL_DOCUMENT_FALLBACK_TOKENS;\n}\n\n/**\n * Estimates Google `type: \"media\"` blocks whose MIME describes static content.\n * Timed audio/video continues through {@link estimateTimedMediaBlockTokens}.\n */\nfunction estimateStaticMediaBlockTokens(\n block: Record<string, unknown>,\n encoding: EncodingName,\n getTokenCount: (text: string) => number\n): number | undefined {\n const mediaMime = getMediaMimeType(block);\n if (block.type !== 'media' || mediaMime == null) {\n return undefined;\n }\n\n const mimeType = mediaMime;\n const nestedMedia =\n block.media != null && typeof block.media === 'object'\n ? (block.media as Record<string, unknown>)\n : undefined;\n const data =\n block.data ?? block.bytes ?? nestedMedia?.data ?? nestedMedia?.bytes;\n const reference =\n block.url ??\n block.fileUri ??\n block.fileId ??\n nestedMedia?.url ??\n nestedMedia?.fileUri ??\n nestedMedia?.fileId;\n\n if (mimeType.startsWith('image/')) {\n let encodedImage: string | undefined;\n if (typeof data === 'string') {\n encodedImage = data;\n } else if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) {\n const view =\n data instanceof ArrayBuffer\n ? new Uint8Array(data)\n : new Uint8Array(data.buffer, data.byteOffset, data.byteLength);\n encodedImage = Buffer.from(view.subarray(0, 80)).toString('base64');\n }\n\n if (encodedImage != null && encodedImage.length > 0) {\n return estimateImageBlockTokens(\n {\n type: 'image',\n source: { type: 'base64', data: encodedImage },\n },\n encoding\n );\n }\n return estimateImageBlockTokens(\n {\n type: 'image_url',\n image_url: { url: typeof reference === 'string' ? reference : '' },\n },\n encoding\n );\n }\n\n if (mimeType === 'application/pdf') {\n if (typeof data === 'string' && data.length > 0) {\n const comma = data.startsWith('data:') ? data.indexOf(',') : -1;\n const base64Data = comma >= 0 ? data.slice(comma + 1) : data;\n return estimateDocumentBlockTokens(\n {\n type: 'file',\n source_type: 'base64',\n mime_type: mimeType,\n data: base64Data,\n },\n encoding,\n getTokenCount\n );\n }\n if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) {\n const estimatedBase64Chars = Math.ceil((data.byteLength * 4) / 3);\n const pdfTokensPerPage =\n encoding === 'claude'\n ? ANTHROPIC_PDF_TOKENS_PER_PAGE\n : OPENAI_PDF_TOKENS_PER_PAGE;\n return (\n Math.max(\n 1,\n Math.ceil(estimatedBase64Chars / BASE64_BYTES_PER_PDF_PAGE)\n ) * pdfTokensPerPage\n );\n }\n return URL_DOCUMENT_FALLBACK_TOKENS;\n }\n\n if (mimeType.startsWith('audio/') || mimeType.startsWith('video/')) {\n return undefined;\n }\n return estimateUnknownMediaBlockTokens(data);\n}\n\n/**\n * Estimates token cost for a timed-media block (video/audio). Handles Google\n * `{ type: 'media', mimeType, data|url|fileUri }`, OpenRouter\n * `{ type: 'video_url' }` / `{ type: 'input_audio' }`, and standard\n * `{ type: 'video' }` / `{ type: 'audio' }` blocks (payload as `data` base64 or\n * `Uint8Array`, base64 `url`, or `fileId`). Duration is inferred from encoded\n * size (providers price by duration, which the block does not carry) at Gemini's\n * rates; a payload with no size (bare URL / file id) falls back to a ~30s\n * estimate. Returns 0 for non-timed media (e.g. an image/document `media` block)\n * so it is never mispriced as video.\n */\nexport function estimateTimedMediaBlockTokens(\n block: Record<string, unknown>\n): number {\n const kind = timedMediaKind(block);\n if (kind == null) {\n return 0;\n }\n let bytes: number;\n if (block.type === 'input_audio') {\n const audio = block.input_audio as { data?: string } | undefined;\n bytes = base64ByteLength(audio?.data);\n } else if (block.type === 'video_url') {\n const video = block.video_url as string | { url?: string } | undefined;\n bytes = base64ByteLength(typeof video === 'string' ? video : video?.url);\n } else {\n bytes = mediaBlockByteLength(block);\n }\n if (kind === 'audio') {\n return timedMediaTokens(\n bytes,\n AUDIO_BYTES_PER_SECOND,\n AUDIO_TOKENS_PER_SECOND,\n AUDIO_URL_FALLBACK_TOKENS\n );\n }\n return timedMediaTokens(\n bytes,\n VIDEO_BYTES_PER_SECOND,\n VIDEO_TOKENS_PER_SECOND,\n VIDEO_URL_FALLBACK_TOKENS\n );\n}\n\nconst tokenizers: Partial<Record<EncodingName, Tokenizer>> = {};\n\nasync function getTokenizer(\n encoding: EncodingName = 'o200k_base'\n): Promise<Tokenizer> {\n const cached = tokenizers[encoding];\n if (cached) {\n return cached;\n }\n const data =\n encoding === 'claude'\n ? await import('ai-tokenizer/encoding/claude')\n : await import('ai-tokenizer/encoding/o200k_base');\n const instance = new Tokenizer(data);\n tokenizers[encoding] = instance;\n return instance;\n}\n\nexport function encodingForModel(model: string): EncodingName {\n if (model.toLowerCase().includes('claude')) {\n return 'claude';\n }\n return 'o200k_base';\n}\n\ntype StructuredSafetyInspection = {\n remaining: number;\n};\n\nfunction consumeStructuredSafetyWork(\n inspection: StructuredSafetyInspection\n): boolean {\n if (inspection.remaining <= 0) {\n return false;\n }\n inspection.remaining--;\n return true;\n}\n\nfunction hasUnsafeToJSON(\n value: object,\n inspection: StructuredSafetyInspection\n): boolean {\n let current: object | null = value;\n const seen = new Set<object>();\n try {\n while (current != null) {\n if (isProxy(current)) {\n return true;\n }\n if (seen.has(current) || !consumeStructuredSafetyWork(inspection)) {\n return true;\n }\n seen.add(current);\n const descriptor = Object.getOwnPropertyDescriptor(current, 'toJSON');\n if (descriptor != null) {\n return (\n ('value' in descriptor && typeof descriptor.value === 'function') ||\n !('value' in descriptor)\n );\n }\n current = Object.getPrototypeOf(current) as object | null;\n }\n } catch {\n return true;\n }\n return false;\n}\n\nconst arrayBufferByteLengthGetter = Object.getOwnPropertyDescriptor(\n ArrayBuffer.prototype,\n 'byteLength'\n)?.get;\n\n/**\n * Tests native binary values without `instanceof`, whose prototype walk can\n * recurse or throw for adversarial proxies.\n */\nfunction isNativeBinaryValue(value: object): boolean {\n try {\n if (ArrayBuffer.isView(value)) {\n return true;\n }\n } catch {\n return true;\n }\n if (arrayBufferByteLengthGetter == null) {\n return false;\n }\n try {\n arrayBufferByteLengthGetter.call(value);\n return true;\n } catch {\n return false;\n }\n}\n\n/**\n * Detects an inherited accessor without reading it. Any proxy/prototype trap or\n * cyclic prototype chain is unsafe rather than observable to the caller.\n */\nfunction hasUnsafeInheritedProperty(\n value: object,\n key: string,\n inspection: StructuredSafetyInspection\n): boolean {\n const seen = new Set<object>([value]);\n let current: object | null;\n try {\n current = Object.getPrototypeOf(value) as object | null;\n } catch {\n return true;\n }\n\n while (current != null) {\n if (isProxy(current)) {\n return true;\n }\n if (seen.has(current) || !consumeStructuredSafetyWork(inspection)) {\n return true;\n }\n seen.add(current);\n let descriptor: PropertyDescriptor | undefined;\n try {\n descriptor = Object.getOwnPropertyDescriptor(current, key);\n } catch {\n return true;\n }\n if (descriptor != null) {\n return !('value' in descriptor);\n }\n try {\n current = Object.getPrototypeOf(current) as object | null;\n } catch {\n return true;\n }\n }\n return false;\n}\n\n/**\n * Detects values whose native JSON representation can differ after local\n * measurement. Accessor properties and `toJSON` hooks are deliberately not\n * invoked; values too large to inspect within bounded work are treated as\n * unsafe so the caller can normalize or reject them conservatively.\n */\nexport function hasUnsafeStructuredSerialization(value: unknown): boolean {\n if (typeof value === 'bigint') {\n return true;\n }\n if (value == null || typeof value !== 'object') {\n return false;\n }\n if (isProxy(value)) {\n return true;\n }\n if (isNativeBinaryValue(value)) {\n return true;\n }\n\n const stack: object[] = [value];\n const seen = new Set<object>();\n const inspection: StructuredSafetyInspection = {\n remaining: MAX_STRUCTURED_SAFETY_INSPECTION_WORK,\n };\n while (stack.length > 0) {\n const current = stack.pop() as object;\n if (isProxy(current)) {\n return true;\n }\n if (seen.has(current)) {\n return true;\n }\n seen.add(current);\n if (!consumeStructuredSafetyWork(inspection)) {\n return true;\n }\n\n try {\n if (hasUnsafeToJSON(current, inspection)) {\n return true;\n }\n for (const key in current) {\n if (!consumeStructuredSafetyWork(inspection)) {\n return true;\n }\n const descriptor = Object.getOwnPropertyDescriptor(current, key);\n if (descriptor == null) {\n if (hasUnsafeInheritedProperty(current, key, inspection)) {\n return true;\n }\n continue;\n }\n if (descriptor.enumerable !== true) {\n continue;\n }\n if (!('value' in descriptor)) {\n return true;\n }\n if ('value' in descriptor) {\n if (typeof descriptor.value === 'bigint') {\n return true;\n }\n if (\n descriptor.value != null &&\n typeof descriptor.value === 'object' &&\n isNativeBinaryValue(descriptor.value)\n ) {\n return true;\n }\n if (\n descriptor.value != null &&\n typeof descriptor.value === 'object'\n ) {\n stack.push(descriptor.value);\n }\n }\n }\n } catch {\n return true;\n }\n }\n return false;\n}\n\nfunction getBoundedTextTokenCount(\n value: string,\n getTokenCount: (text: string) => number\n): number {\n const preview =\n value.length > MAX_STRUCTURED_TOKENIZATION_CHARS\n ? value.slice(0, MAX_STRUCTURED_TOKENIZATION_CHARS)\n : value;\n const previewTokens = getTokenCount(preview);\n const omittedChars = value.length - preview.length;\n if (omittedChars <= 0) {\n return previewTokens;\n }\n return Math.min(\n Number.MAX_SAFE_INTEGER,\n previewTokens + omittedChars * MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR\n );\n}\n\nfunction getBoundedStructuredTokenCount(\n value: unknown,\n getTokenCount: (text: string) => number\n): number {\n if (hasUnsafeStructuredSerialization(value)) {\n return Number.MAX_SAFE_INTEGER;\n }\n const serialized = serializeStructuredValueBounded(\n value,\n MAX_STRUCTURED_TOKENIZATION_CHARS\n );\n const previewTokens = getTokenCount(serialized.content);\n if (!serialized.truncated) {\n return previewTokens;\n }\n if (!Number.isSafeInteger(serialized.originalChars)) {\n return Number.MAX_SAFE_INTEGER;\n }\n\n const omittedChars = Math.max(\n 0,\n serialized.originalChars - serialized.content.length\n );\n return Math.min(\n Number.MAX_SAFE_INTEGER,\n previewTokens + omittedChars * MAX_TOKENS_PER_OMITTED_STRUCTURED_CHAR\n );\n}\n\nexport function getTokenCountForMessage(\n message: BaseMessage,\n getTokenCount: (text: string) => number,\n encoding: EncodingName = 'o200k_base'\n): number {\n const tokensPerMessage = 3;\n const countText = (text: string): number =>\n getBoundedTextTokenCount(text, getTokenCount);\n if (isProxy(message)) {\n return Number.MAX_SAFE_INTEGER;\n }\n\n type ContentBlock = Record<string, unknown> & {\n type?: string;\n tool_call?: {\n id?: string;\n name?: string;\n args?: unknown;\n output?: unknown;\n };\n };\n const representedToolCallIds = new Set<string>();\n\n const processValue = (value: unknown): void => {\n if (value != null && typeof value === 'object' && isProxy(value)) {\n numTokens = Number.MAX_SAFE_INTEGER;\n return;\n }\n if (Array.isArray(value)) {\n for (const raw of value) {\n if (\n typeof raw === 'string' ||\n typeof raw === 'number' ||\n typeof raw === 'boolean'\n ) {\n processValue(raw);\n continue;\n }\n const item = raw as ContentBlock | null | undefined;\n if (item == null || typeof item !== 'object') {\n continue;\n }\n if (isProxy(item)) {\n numTokens = Number.MAX_SAFE_INTEGER;\n return;\n }\n if (typeof item.type !== 'string') {\n numTokens += getBoundedStructuredTokenCount(item, countText);\n continue;\n }\n if (item.type === ContentTypes.TOOL_CALL || item.type === 'tool_use') {\n const inlineId =\n typeof item.id === 'string' ? item.id : item.tool_call?.id;\n if (typeof inlineId === 'string' && inlineId.length > 0) {\n representedToolCallIds.add(inlineId);\n }\n }\n if (item.type === ContentTypes.ERROR) {\n continue;\n }\n\n if (\n isAtomicToolContentBlock(item) &&\n (item.type === ContentTypes.IMAGE_URL ||\n item.type === 'image_url' ||\n item.type === 'image' ||\n item.type === 'computer_screenshot' ||\n item.type === 'input_image')\n ) {\n numTokens += Math.ceil(\n estimateImageBlockTokens(item, encoding) * IMAGE_TOKEN_SAFETY_MARGIN\n );\n continue;\n }\n\n if (\n isAtomicToolContentBlock(item) &&\n (item.type === 'document' ||\n item.type === 'file' ||\n item.type === ContentTypes.IMAGE_FILE)\n ) {\n numTokens += Math.ceil(\n estimateDocumentBlockTokens(item, encoding, countText) *\n IMAGE_TOKEN_SAFETY_MARGIN\n );\n continue;\n }\n\n if (isAtomicToolContentBlock(item) && item.type === 'media') {\n const staticMediaTokens = estimateStaticMediaBlockTokens(\n item,\n encoding,\n countText\n );\n if (staticMediaTokens != null) {\n numTokens += Math.ceil(\n staticMediaTokens * IMAGE_TOKEN_SAFETY_MARGIN\n );\n continue;\n }\n }\n\n if (isAtomicToolContentBlock(item) && isTimedMediaType(item.type)) {\n const timedMediaTokens = estimateTimedMediaBlockTokens(item);\n if (timedMediaTokens > 0) {\n numTokens += Math.ceil(\n timedMediaTokens * IMAGE_TOKEN_SAFETY_MARGIN\n );\n continue;\n }\n }\n\n if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {\n const toolName = item.tool_call.name;\n if (typeof toolName === 'string' && toolName.length > 0) {\n numTokens += countText(toolName);\n }\n const args = item.tool_call.args;\n if (args != null) {\n numTokens +=\n typeof args === 'string'\n ? countText(args)\n : getBoundedStructuredTokenCount(args, countText);\n }\n const output = item.tool_call.output;\n if (output != null) {\n processValue(output);\n }\n continue;\n }\n\n const nestedValue = item[item.type];\n if (nestedValue == null) {\n numTokens += getBoundedStructuredTokenCount(item, countText);\n continue;\n }\n\n processValue(nestedValue);\n }\n } else if (typeof value === 'string') {\n numTokens += countText(value);\n } else if (typeof value === 'number') {\n numTokens += countText(value.toString());\n } else if (typeof value === 'boolean') {\n numTokens += countText(value.toString());\n } else if (value != null && typeof value === 'object') {\n numTokens += getBoundedStructuredTokenCount(value, countText);\n }\n };\n\n const rawAdditionalKwargs = message.additional_kwargs as unknown;\n const additionalKwargs =\n rawAdditionalKwargs != null && typeof rawAdditionalKwargs === 'object'\n ? rawAdditionalKwargs\n : undefined;\n if (additionalKwargs != null && isProxy(additionalKwargs)) {\n return Number.MAX_SAFE_INTEGER;\n }\n let additionalType: PropertyDescriptor | undefined;\n try {\n additionalType =\n additionalKwargs != null\n ? Object.getOwnPropertyDescriptor(additionalKwargs, 'type')\n : undefined;\n } catch {\n return Number.MAX_SAFE_INTEGER;\n }\n if (additionalType != null && !('value' in additionalType)) {\n return Number.MAX_SAFE_INTEGER;\n }\n\n let numTokens = tokensPerMessage;\n const messageType = message.getType();\n const isComputerCallOutput =\n messageType === 'tool' &&\n additionalType?.value === 'computer_call_output' &&\n isComputerCallOutputContent(message.content);\n if (isComputerCallOutput && typeof message.content === 'string') {\n numTokens += Math.ceil(\n estimateImageBlockTokens(\n {\n type: 'computer_screenshot',\n image_url: message.content,\n },\n encoding\n ) * IMAGE_TOKEN_SAFETY_MARGIN\n );\n } else {\n processValue(message.content);\n }\n if (numTokens >= Number.MAX_SAFE_INTEGER) {\n return Number.MAX_SAFE_INTEGER;\n }\n const messageRole = (message as BaseMessage & { role?: unknown }).role;\n if (messageType === 'ai' || messageRole === 'assistant') {\n const toolCalls = (message as AIMessage).tool_calls ?? [];\n if (isProxy(toolCalls)) {\n return Number.MAX_SAFE_INTEGER;\n }\n for (const toolCall of toolCalls) {\n if (isProxy(toolCall)) {\n return Number.MAX_SAFE_INTEGER;\n }\n if (\n typeof toolCall.id === 'string' &&\n representedToolCallIds.has(toolCall.id)\n ) {\n continue;\n }\n if (typeof toolCall.name === 'string' && toolCall.name.length > 0) {\n numTokens += countText(toolCall.name);\n }\n const args: unknown = toolCall.args;\n if (args != null) {\n numTokens +=\n typeof args === 'string'\n ? countText(args)\n : getBoundedStructuredTokenCount(args, countText);\n }\n }\n let legacyFunctionCall: PropertyDescriptor | undefined;\n try {\n legacyFunctionCall =\n additionalKwargs != null\n ? Object.getOwnPropertyDescriptor(additionalKwargs, 'function_call')\n : undefined;\n } catch {\n return Number.MAX_SAFE_INTEGER;\n }\n if (legacyFunctionCall != null) {\n if (!('value' in legacyFunctionCall)) {\n return Number.MAX_SAFE_INTEGER;\n }\n if (legacyFunctionCall.value != null) {\n numTokens += getBoundedStructuredTokenCount(\n legacyFunctionCall.value,\n countText\n );\n }\n }\n }\n return Math.min(Number.MAX_SAFE_INTEGER, numTokens);\n}\n\n/**\n * Largest-remainder apportionment: scales each count by `multiplier` and\n * distributes the rounding remainder so the results sum exactly to\n * `targetTotal`. Keeps per-item breakdowns reconciled with an aggregate\n * computed as a single rounded product of the summed raw counts.\n */\nexport function apportionTokenCounts(\n rawCounts: Record<string, number>,\n multiplier: number,\n targetTotal: number\n): Record<string, number> {\n const result: Record<string, number> = Object.create(null);\n const remainders: Array<{ name: string; remainder: number }> = [];\n let floorSum = 0;\n for (const [name, rawCount] of Object.entries(rawCounts)) {\n const scaled = rawCount * multiplier;\n const floored = Math.floor(scaled);\n result[name] = floored;\n floorSum += floored;\n remainders.push({ name, remainder: scaled - floored });\n }\n let leftover = targetTotal - floorSum;\n if (leftover <= 0 || remainders.length === 0) {\n return result;\n }\n remainders.sort((a, b) => b.remainder - a.remainder);\n for (let i = 0; leftover > 0; i = (i + 1) % remainders.length) {\n result[remainders[i].name] += 1;\n leftover--;\n }\n return result;\n}\n\n/**\n * Anthropic's API consistently reports ~10% more tokens than the local\n * claude tokenizer due to internal message framing and content encoding.\n * Verified empirically across content types via the count_tokens endpoint.\n */\nconst CLAUDE_TOKEN_CORRECTION = 1.1;\n\n/**\n * Creates a token counter function using the specified encoding.\n * Lazily loads the encoding data on first use via dynamic import.\n */\nexport const createTokenCounter = async (\n encoding: EncodingName = 'o200k_base'\n): Promise<(message: BaseMessage) => number> => {\n const tok = await getTokenizer(encoding);\n const countTokens = (text: string): number => tok.count(text);\n const isClaude = encoding === 'claude';\n return (message: BaseMessage): number => {\n const count = getTokenCountForMessage(message, countTokens, encoding);\n return isClaude ? Math.ceil(count * CLAUDE_TOKEN_CORRECTION) : count;\n };\n};\n\n/** Utility to manage the token encoder lifecycle explicitly. */\nexport const TokenEncoderManager = {\n async initialize(): Promise<void> {\n // No-op: ai-tokenizer is synchronously initialized from bundled data.\n },\n\n reset(): void {\n for (const key of Object.keys(tokenizers)) {\n delete tokenizers[key as EncodingName];\n }\n },\n\n isInitialized(): boolean {\n return Object.keys(tokenizers).length > 0;\n },\n};\n"],"mappings":";;;;;;AAaA,MAAM,6BAA6B;;AAEnC,MAAM,0BAA0B;;AAEhC,MAAM,0BAA0B;;AAEhC,MAAM,yBAAyB;;AAE/B,MAAM,+BAA+B;;AAIrC,MAAa,4BAA4B;;;;;AAMzC,MAAM,gCAAgC;;AAEtC,MAAM,6BAA6B;;AAInC,MAAM,4BAA4B;;AAElC,MAAM,+BAA+B;;;;;;;;;AAUrC,MAAM,0BAA0B;;AAEhC,MAAM,0BAA0B;;AAEhC,MAAM,yBAAyB;AAC/B,MAAM,yBAAyB;;AAE/B,MAAM,4BAA4B;AAClC,MAAM,4BAA4B;;;AAGlC,MAAM,oBAAoB,IAAI,IAAI;CAChC;CACA;CACA;CACA;CACA;AACF,CAAC;;;;;;AAMD,MAAM,oCAAoC;;AAE1C,MAAM,yCAAyC;AAC/C,MAAM,wCAAwC;;;;;;AAO9C,SAAgB,uBACd,YAC0C;CAC1C,MAAM,MAAM,WAAW,WAAW,OAAO,IACrC,WAAW,MAAM,WAAW,QAAQ,GAAG,IAAI,CAAC,IAC5C;CAEJ,IAAI,IAAI,SAAS,IACf,OAAO;CAGT,MAAM,QAAQ,IAAI,WAAW,OAAO,KAAK,IAAI,MAAM,GAAG,EAAE,GAAG,QAAQ,CAAC;CAEpE,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,IAMpC,OAAO;EAAE,OAHN,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;EAGnD,QADb,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;CAC5C;CAGzB,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,KAAM;EAE1C,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,GAAG,KACpC,IACE,MAAM,OAAO,QACZ,MAAM,IAAI,OAAO,OAAQ,MAAM,IAAI,OAAO,MAC3C;GACA,MAAM,SAAU,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;GAE/C,OAAO;IAAE,OADM,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;IAC9B;GAAO;EACzB;EAEF,OAAO;CACT;CAEA,IAAI,MAAM,OAAO,MAAQ,MAAM,OAAO,MAAQ,MAAM,OAAO,IAIzD,OAAO;EAAE,OAFK,MAAM,KAAM,MAAM,MAAM;EAEtB,QADD,MAAM,KAAM,MAAM,MAAM;CAChB;CAGzB,IACE,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,QAAQ,MACd,MAAM,QAAQ,IACd;EAEA,IAAI,MAAM,SAAS,IAGjB,OAAO;GAAE,QAFM,MAAM,MAAO,MAAM,OAAO,KAAM;GAE/B,SADA,MAAM,MAAO,MAAM,OAAO,KAAM;EACzB;EAEzB,OAAO;CACT;CAEA,OAAO;AACT;;AAGA,SAAgB,6BACd,OACA,QACQ;CACR,OAAO,KAAK,IACV,4BACA,KAAK,KAAM,QAAQ,SAAU,uBAAuB,CACtD;AACF;;AAGA,SAAgB,0BACd,OACA,QACA,SAAiB,QACT;CACR,IAAI,WAAW,OACb,OAAO;CAKT,OAAO,0BAFL,KAAK,KAAK,QAAQ,sBAAsB,IACxC,KAAK,KAAK,SAAS,sBAAsB,IACF;AAC3C;;;;;;AAOA,SAAgB,yBACd,OACA,UACQ;CACR,IAAI;CAEJ,IACE,MAAM,SAAA,eACN,MAAM,SAAS,eACf,MAAM,SAAS,yBACf,MAAM,SAAS,eACf;EACA,MAAM,WAAW,MAAM;EACvB,MAAM,MAAM,OAAO,aAAa,WAAW,WAAW,UAAU;EAChE,IAAI,OAAO,QAAQ,YAAY,IAAI,WAAW,OAAO,GACnD,aAAa;OACR,IAAI,OAAO,MAAM,YAAY,YAAY,MAAM,YAAY,IAChE,OAAO,MAAM,WAAW,QACpB,0BACA;OAEJ,OAAO;CAEX,OAAO,IAAI,MAAM,SAAS,SAAS;EACjC,MAAM,SAAS,MAAM;EACrB,IAAI,QAAQ,SAAS,YAAY,OAAO,OAAO,SAAS,UACtD,aAAa,OAAO;OAEpB,OAAO;CAEX,OACE,OAAO;CAGT,MAAM,OAAO,uBAAuB,UAAU;CAC9C,IAAI,QAAQ,MACV,OAAO;CAGT,IAAI,aAAa,UACf,OAAO,6BAA6B,KAAK,OAAO,KAAK,MAAM;CAE7D,OAAO,0BAA0B,KAAK,OAAO,KAAK,MAAM;AAC1D;;;;;;;;;;AAWA,SAAgB,4BACd,OACA,UACA,eACQ;CACR,MAAM,mBACJ,aAAa,WACT,gCACA;CAGN,MAAM,aAAa,MAAM;CACzB,IAAI,OAAO,eAAe,UAAU;EAClC,MAAM,YAAa,MAAM,aAAoC,GAAA,CAAI,MAC/D,GACF,CAAC,CAAC;EAEF,IAAI,eAAe,UAAU,OAAO,MAAM,SAAS,UACjD,OAAO,cAAc,MAAM,IAAc;EAG3C,IAAI,eAAe,YAAY,OAAO,MAAM,SAAS,UAAU;GAC7D,IAAI,aAAa,qBAAqB,aAAa,IAKjD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAM,MAAM,KAAgB,SAAS,yBAAyB,CAEnD,IAAI;GAGxB,IAAI,SAAS,WAAW,QAAQ,GAC9B,OAAO,yBACL;IACE,GAAG;IACH,MAAM;IACN,QAAQ;KAAE,MAAM;KAAU,MAAM,MAAM;IAAK;GAC7C,GACA,QACF;GAEF,OAAO,cAAc,MAAM,IAAc;EAC3C;EAEA,IAAI,eAAe,OACjB,OAAO;EAGT,OAAO;CACT;CAGA,MAAM,SAAS,MAAM;CASrB,IAAI,UAAU,MACZ,OAAO;CAGT,IAAI,OAAO,SAAS,UAAU,OAAO,OAAO,SAAS,UACnD,OAAO,cAAc,OAAO,IAAI;CAGlC,IAAI,OAAO,SAAS,YAAY,OAAO,OAAO,SAAS,UAAU;EAC/D,MAAM,aAAa,OAAO,cAAc,GAAA,CAAI,MAAM,GAAG,CAAC,CAAC;EACvD,IAAI,cAAc,qBAAqB,cAAc,IAKnD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAK,OAAO,KAAK,SAAS,yBAAyB,CAExC,IAAI;EAExB,IAAI,UAAU,WAAW,QAAQ,GAC/B,OAAO,yBACL;GAAE,MAAM;GAAS,QAAQ;IAAE,MAAM;IAAU,MAAM,OAAO;GAAK;EAAE,GAC/D,QACF;EAEF,OAAO,cAAc,OAAO,IAAI;CAClC;CAEA,IAAI,OAAO,SAAS,OAClB,OAAO;CAIT,IAAI,OAAO,SAAS,aAAa,MAAM,QAAQ,OAAO,OAAO,GAAG;EAC9D,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,OAAO,SACzB,IAAI,SAAS,QAAQ,OAAO,UAAU,YAAY,UAAU,OAAO;GACjE,MAAM,aAAa;GACnB,IAAI,WAAW,SAAS,SACtB,SAAS,yBAAyB,YAAY,QAAQ;EAE1D;EAEF,OAAO,QAAQ,IAAI,QAAQ;CAC7B;CAEA,OAAO;AACT;;;;AAKA,SAAS,iBAAiB,OAAmC;CAC3D,IAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAChD,OAAO;CAET,IAAI,MAAM,WAAW,OAAO,GAAG;EAC7B,MAAM,QAAQ,MAAM,QAAQ,GAAG;EAC/B,OAAO,QAAQ,IAAI,IAAI,KAAK,OAAQ,MAAM,SAAS,QAAQ,KAAK,IAAK,CAAC;CACxE;CACA,IAAI,uBAAuB,KAAK,KAAK,GACnC,OAAO;CAET,OAAO,KAAK,MAAO,MAAM,SAAS,IAAK,CAAC;AAC1C;AAEA,SAAS,iBACP,OACA,gBACA,iBACA,aACQ;CACR,IAAI,SAAS,GACX,OAAO;CAET,OAAO,KAAK,IACV,iBACA,KAAK,KAAM,QAAQ,iBAAkB,eAAe,CACtD;AACF;;;;;AAMA,SAAS,qBAAqB,OAAwC;CACpE,MAAM,OAAO,MAAM;CACnB,IAAI,OAAO,SAAS,UAClB,OAAO,iBAAiB,IAAI;CAE9B,IAAI,gBAAgB,YAClB,OAAO,KAAK;CAEd,IAAI,OAAO,MAAM,QAAQ,UACvB,OAAO,iBAAiB,MAAM,GAAG;CAEnC,MAAM,SAAU,MAAM,SAAS,MAAM;CAGrC,IAAI,UAAU,MAAM;EAClB,IAAI,OAAO,OAAO,SAAS,UACzB,OAAO,iBAAiB,OAAO,IAAI;EAErC,IAAI,OAAO,OAAO,QAAQ,UACxB,OAAO,iBAAiB,OAAO,GAAG;EAEpC,IAAI,OAAO,QAAQ,iBAAiB,YAClC,OAAO,OAAO,OAAO,MAAM;CAE/B;CACA,OAAO;AACT;;;;;AAMA,SAAS,eACP,OAC0B;CAC1B,MAAM,OACJ,OAAO,MAAM,SAAS,WAAW,uBAAuB,MAAM,IAAI,IAAI;CACxE,IAAI,SAAS,iBAAiB,SAAS,SACrC,OAAO;CAET,IAAI,SAAS,eAAe,SAAS,SACnC,OAAO;CAET,MAAM,YAAY,iBAAiB,KAAK;CACxC,MAAM,OAAO,SAAS,WAAW,aAAa,OAAO,YAAY;CACjE,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,IAAI,KAAK,WAAW,QAAQ,GAC1B,OAAO;CAET,OAAO;AACT;AAEA,SAAS,iBAAiB,OAAoD;CAC5E,IAAI,OAAO,MAAM,aAAa,UAC5B,OAAO,uBAAuB,MAAM,QAAQ;CAE9C,IAAI,OAAO,MAAM,cAAc,UAC7B,OAAO,uBAAuB,MAAM,SAAS;AAGjD;AAEA,SAAS,uBAAuB,UAA0B;CACxD,OAAO,SAAS,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,YAAY;AACnD;;;;AAKA,SAAS,iBAAiB,MAAuB;CAC/C,MAAM,iBAAiB,uBAAuB,IAAI;CAClD,OACE,kBAAkB,IAAI,cAAc,KACpC,eAAe,WAAW,QAAQ,KAClC,eAAe,WAAW,QAAQ;AAEtC;;AAGA,SAAS,gCAAgC,MAAuB;CAC9D,IAAI,OAAO,SAAS,UAAU;EAC5B,IAAI,iBAAiB,IAAI,KAAK,GAC5B,OAAO;EAET,OAAO,KAAK,IAAI,8BAA8B,KAAK,MAAM;CAC3D;CACA,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GACxD,OAAO,KAAK,IACV,8BACA,KAAK,KAAM,KAAK,aAAa,IAAK,CAAC,CACrC;CAEF,OAAO;AACT;;;;;AAMA,SAAS,+BACP,OACA,UACA,eACoB;CACpB,MAAM,YAAY,iBAAiB,KAAK;CACxC,IAAI,MAAM,SAAS,WAAW,aAAa,MACzC;CAGF,MAAM,WAAW;CACjB,MAAM,cACJ,MAAM,SAAS,QAAQ,OAAO,MAAM,UAAU,WACzC,MAAM,QACP,KAAA;CACN,MAAM,OACJ,MAAM,QAAQ,MAAM,SAAS,aAAa,QAAQ,aAAa;CACjE,MAAM,YACJ,MAAM,OACN,MAAM,WACN,MAAM,UACN,aAAa,OACb,aAAa,WACb,aAAa;CAEf,IAAI,SAAS,WAAW,QAAQ,GAAG;EACjC,IAAI;EACJ,IAAI,OAAO,SAAS,UAClB,eAAe;OACV,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GAAG;GAClE,MAAM,OACJ,gBAAgB,cACZ,IAAI,WAAW,IAAI,IACnB,IAAI,WAAW,KAAK,QAAQ,KAAK,YAAY,KAAK,UAAU;GAClE,eAAe,OAAO,KAAK,KAAK,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,QAAQ;EACpE;EAEA,IAAI,gBAAgB,QAAQ,aAAa,SAAS,GAChD,OAAO,yBACL;GACE,MAAM;GACN,QAAQ;IAAE,MAAM;IAAU,MAAM;GAAa;EAC/C,GACA,QACF;EAEF,OAAO,yBACL;GACE,MAAM;GACN,WAAW,EAAE,KAAK,OAAO,cAAc,WAAW,YAAY,GAAG;EACnE,GACA,QACF;CACF;CAEA,IAAI,aAAa,mBAAmB;EAClC,IAAI,OAAO,SAAS,YAAY,KAAK,SAAS,GAAG;GAC/C,MAAM,QAAQ,KAAK,WAAW,OAAO,IAAI,KAAK,QAAQ,GAAG,IAAI;GAE7D,OAAO,4BACL;IACE,MAAM;IACN,aAAa;IACb,WAAW;IACX,MANe,SAAS,IAAI,KAAK,MAAM,QAAQ,CAAC,IAAI;GAOtD,GACA,UACA,aACF;EACF;EACA,IAAI,gBAAgB,eAAe,YAAY,OAAO,IAAI,GAAG;GAC3D,MAAM,uBAAuB,KAAK,KAAM,KAAK,aAAa,IAAK,CAAC;GAChE,MAAM,mBACJ,aAAa,WACT,gCACA;GACN,OACE,KAAK,IACH,GACA,KAAK,KAAK,uBAAuB,yBAAyB,CAC5D,IAAI;EAER;EACA,OAAO;CACT;CAEA,IAAI,SAAS,WAAW,QAAQ,KAAK,SAAS,WAAW,QAAQ,GAC/D;CAEF,OAAO,gCAAgC,IAAI;AAC7C;;;;;;;;;;;;AAaA,SAAgB,8BACd,OACQ;CACR,MAAM,OAAO,eAAe,KAAK;CACjC,IAAI,QAAQ,MACV,OAAO;CAET,IAAI;CACJ,IAAI,MAAM,SAAS,eAAe;EAChC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,IAAI;CACtC,OAAO,IAAI,MAAM,SAAS,aAAa;EACrC,MAAM,QAAQ,MAAM;EACpB,QAAQ,iBAAiB,OAAO,UAAU,WAAW,QAAQ,OAAO,GAAG;CACzE,OACE,QAAQ,qBAAqB,KAAK;CAEpC,IAAI,SAAS,SACX,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;CAEF,OAAO,iBACL,OACA,wBACA,yBACA,yBACF;AACF;AAEA,MAAM,aAAuD,CAAC;AAE9D,eAAe,aACb,WAAyB,cACL;CACpB,MAAM,SAAS,WAAW;CAC1B,IAAI,QACF,OAAO;CAMT,MAAM,WAAW,IAAIA,aAAAA,UAHnB,aAAa,WACT,MAAM,OAAO,kCACb,MAAM,OAAO,mCACgB;CACnC,WAAW,YAAY;CACvB,OAAO;AACT;AAEA,SAAgB,iBAAiB,OAA6B;CAC5D,IAAI,MAAM,YAAY,CAAC,CAAC,SAAS,QAAQ,GACvC,OAAO;CAET,OAAO;AACT;AAMA,SAAS,4BACP,YACS;CACT,IAAI,WAAW,aAAa,GAC1B,OAAO;CAET,WAAW;CACX,OAAO;AACT;AAEA,SAAS,gBACP,OACA,YACS;CACT,IAAI,UAAyB;CAC7B,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI;EACF,OAAO,WAAW,MAAM;GACtB,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,OAAO;GAET,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,4BAA4B,UAAU,GAC9D,OAAO;GAET,KAAK,IAAI,OAAO;GAChB,MAAM,aAAa,OAAO,yBAAyB,SAAS,QAAQ;GACpE,IAAI,cAAc,MAChB,OACG,WAAW,cAAc,OAAO,WAAW,UAAU,cACtD,EAAE,WAAW;GAGjB,UAAU,OAAO,eAAe,OAAO;EACzC;CACF,QAAQ;EACN,OAAO;CACT;CACA,OAAO;AACT;AAEA,MAAM,8BAA8B,OAAO,yBACzC,YAAY,WACZ,YACF,CAAC,EAAE;;;;;AAMH,SAAS,oBAAoB,OAAwB;CACnD,IAAI;EACF,IAAI,YAAY,OAAO,KAAK,GAC1B,OAAO;CAEX,QAAQ;EACN,OAAO;CACT;CACA,IAAI,+BAA+B,MACjC,OAAO;CAET,IAAI;EACF,4BAA4B,KAAK,KAAK;EACtC,OAAO;CACT,QAAQ;EACN,OAAO;CACT;AACF;;;;;AAMA,SAAS,2BACP,OACA,KACA,YACS;CACT,MAAM,OAAO,IAAI,IAAY,CAAC,KAAK,CAAC;CACpC,IAAI;CACJ,IAAI;EACF,UAAU,OAAO,eAAe,KAAK;CACvC,QAAQ;EACN,OAAO;CACT;CAEA,OAAO,WAAW,MAAM;EACtB,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,OAAO;EAET,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,4BAA4B,UAAU,GAC9D,OAAO;EAET,KAAK,IAAI,OAAO;EAChB,IAAI;EACJ,IAAI;GACF,aAAa,OAAO,yBAAyB,SAAS,GAAG;EAC3D,QAAQ;GACN,OAAO;EACT;EACA,IAAI,cAAc,MAChB,OAAO,EAAE,WAAW;EAEtB,IAAI;GACF,UAAU,OAAO,eAAe,OAAO;EACzC,QAAQ;GACN,OAAO;EACT;CACF;CACA,OAAO;AACT;;;;;;;AAQA,SAAgB,iCAAiC,OAAyB;CACxE,IAAI,OAAO,UAAU,UACnB,OAAO;CAET,IAAI,SAAS,QAAQ,OAAO,UAAU,UACpC,OAAO;CAET,KAAA,GAAA,gBAAA,QAAA,CAAY,KAAK,GACf,OAAO;CAET,IAAI,oBAAoB,KAAK,GAC3B,OAAO;CAGT,MAAM,QAAkB,CAAC,KAAK;CAC9B,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,aAAyC,EAC7C,WAAW,sCACb;CACA,OAAO,MAAM,SAAS,GAAG;EACvB,MAAM,UAAU,MAAM,IAAI;EAC1B,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,OAAO;EAET,IAAI,KAAK,IAAI,OAAO,GAClB,OAAO;EAET,KAAK,IAAI,OAAO;EAChB,IAAI,CAAC,4BAA4B,UAAU,GACzC,OAAO;EAGT,IAAI;GACF,IAAI,gBAAgB,SAAS,UAAU,GACrC,OAAO;GAET,KAAK,MAAM,OAAO,SAAS;IACzB,IAAI,CAAC,4BAA4B,UAAU,GACzC,OAAO;IAET,MAAM,aAAa,OAAO,yBAAyB,SAAS,GAAG;IAC/D,IAAI,cAAc,MAAM;KACtB,IAAI,2BAA2B,SAAS,KAAK,UAAU,GACrD,OAAO;KAET;IACF;IACA,IAAI,WAAW,eAAe,MAC5B;IAEF,IAAI,EAAE,WAAW,aACf,OAAO;IAET,IAAI,WAAW,YAAY;KACzB,IAAI,OAAO,WAAW,UAAU,UAC9B,OAAO;KAET,IACE,WAAW,SAAS,QACpB,OAAO,WAAW,UAAU,YAC5B,oBAAoB,WAAW,KAAK,GAEpC,OAAO;KAET,IACE,WAAW,SAAS,QACpB,OAAO,WAAW,UAAU,UAE5B,MAAM,KAAK,WAAW,KAAK;IAE/B;GACF;EACF,QAAQ;GACN,OAAO;EACT;CACF;CACA,OAAO;AACT;AAEA,SAAS,yBACP,OACA,eACQ;CACR,MAAM,UACJ,MAAM,SAAS,oCACX,MAAM,MAAM,GAAG,iCAAiC,IAChD;CACN,MAAM,gBAAgB,cAAc,OAAO;CAC3C,MAAM,eAAe,MAAM,SAAS,QAAQ;CAC5C,IAAI,gBAAgB,GAClB,OAAO;CAET,OAAO,KAAK,IACV,OAAO,kBACP,gBAAgB,eAAe,sCACjC;AACF;AAEA,SAAS,+BACP,OACA,eACQ;CACR,IAAI,iCAAiC,KAAK,GACxC,OAAO,OAAO;CAEhB,MAAM,aAAaC,oBAAAA,gCACjB,OACA,iCACF;CACA,MAAM,gBAAgB,cAAc,WAAW,OAAO;CACtD,IAAI,CAAC,WAAW,WACd,OAAO;CAET,IAAI,CAAC,OAAO,cAAc,WAAW,aAAa,GAChD,OAAO,OAAO;CAGhB,MAAM,eAAe,KAAK,IACxB,GACA,WAAW,gBAAgB,WAAW,QAAQ,MAChD;CACA,OAAO,KAAK,IACV,OAAO,kBACP,gBAAgB,eAAe,sCACjC;AACF;AAEA,SAAgB,wBACd,SACA,eACA,WAAyB,cACjB;CACR,MAAM,mBAAmB;CACzB,MAAM,aAAa,SACjB,yBAAyB,MAAM,aAAa;CAC9C,KAAA,GAAA,gBAAA,QAAA,CAAY,OAAO,GACjB,OAAO,OAAO;CAYhB,MAAM,yCAAyB,IAAI,IAAY;CAE/C,MAAM,gBAAgB,UAAyB;EAC7C,IAAI,SAAS,QAAQ,OAAO,UAAU,aAAA,GAAA,gBAAA,QAAA,CAAoB,KAAK,GAAG;GAChE,YAAY,OAAO;GACnB;EACF;EACA,IAAI,MAAM,QAAQ,KAAK,GACrB,KAAK,MAAM,OAAO,OAAO;GACvB,IACE,OAAO,QAAQ,YACf,OAAO,QAAQ,YACf,OAAO,QAAQ,WACf;IACA,aAAa,GAAG;IAChB;GACF;GACA,MAAM,OAAO;GACb,IAAI,QAAQ,QAAQ,OAAO,SAAS,UAClC;GAEF,KAAA,GAAA,gBAAA,QAAA,CAAY,IAAI,GAAG;IACjB,YAAY,OAAO;IACnB;GACF;GACA,IAAI,OAAO,KAAK,SAAS,UAAU;IACjC,aAAa,+BAA+B,MAAM,SAAS;IAC3D;GACF;GACA,IAAI,KAAK,SAAA,eAAmC,KAAK,SAAS,YAAY;IACpE,MAAM,WACJ,OAAO,KAAK,OAAO,WAAW,KAAK,KAAK,KAAK,WAAW;IAC1D,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,uBAAuB,IAAI,QAAQ;GAEvC;GACA,IAAI,KAAK,SAAA,SACP;GAGF,IACEC,oBAAAA,yBAAyB,IAAI,MAC5B,KAAK,SAAA,eACJ,KAAK,SAAS,eACd,KAAK,SAAS,WACd,KAAK,SAAS,yBACd,KAAK,SAAS,gBAChB;IACA,aAAa,KAAK,KAChB,yBAAyB,MAAM,QAAQ,IAAI,yBAC7C;IACA;GACF;GAEA,IACEA,oBAAAA,yBAAyB,IAAI,MAC5B,KAAK,SAAS,cACb,KAAK,SAAS,UACd,KAAK,SAAA,eACP;IACA,aAAa,KAAK,KAChB,4BAA4B,MAAM,UAAU,SAAS,IACnD,yBACJ;IACA;GACF;GAEA,IAAIA,oBAAAA,yBAAyB,IAAI,KAAK,KAAK,SAAS,SAAS;IAC3D,MAAM,oBAAoB,+BACxB,MACA,UACA,SACF;IACA,IAAI,qBAAqB,MAAM;KAC7B,aAAa,KAAK,KAChB,oBAAoB,yBACtB;KACA;IACF;GACF;GAEA,IAAIA,oBAAAA,yBAAyB,IAAI,KAAK,iBAAiB,KAAK,IAAI,GAAG;IACjE,MAAM,mBAAmB,8BAA8B,IAAI;IAC3D,IAAI,mBAAmB,GAAG;KACxB,aAAa,KAAK,KAChB,mBAAmB,yBACrB;KACA;IACF;GACF;GAEA,IAAI,KAAK,SAAA,eAAmC,KAAK,aAAa,MAAM;IAClE,MAAM,WAAW,KAAK,UAAU;IAChC,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,aAAa,UAAU,QAAQ;IAEjC,MAAM,OAAO,KAAK,UAAU;IAC5B,IAAI,QAAQ,MACV,aACE,OAAO,SAAS,WACZ,UAAU,IAAI,IACd,+BAA+B,MAAM,SAAS;IAEtD,MAAM,SAAS,KAAK,UAAU;IAC9B,IAAI,UAAU,MACZ,aAAa,MAAM;IAErB;GACF;GAEA,MAAM,cAAc,KAAK,KAAK;GAC9B,IAAI,eAAe,MAAM;IACvB,aAAa,+BAA+B,MAAM,SAAS;IAC3D;GACF;GAEA,aAAa,WAAW;EAC1B;OACK,IAAI,OAAO,UAAU,UAC1B,aAAa,UAAU,KAAK;OACvB,IAAI,OAAO,UAAU,UAC1B,aAAa,UAAU,MAAM,SAAS,CAAC;OAClC,IAAI,OAAO,UAAU,WAC1B,aAAa,UAAU,MAAM,SAAS,CAAC;OAClC,IAAI,SAAS,QAAQ,OAAO,UAAU,UAC3C,aAAa,+BAA+B,OAAO,SAAS;CAEhE;CAEA,MAAM,sBAAsB,QAAQ;CACpC,MAAM,mBACJ,uBAAuB,QAAQ,OAAO,wBAAwB,WAC1D,sBACA,KAAA;CACN,IAAI,oBAAoB,SAAA,GAAA,gBAAA,QAAA,CAAgB,gBAAgB,GACtD,OAAO,OAAO;CAEhB,IAAI;CACJ,IAAI;EACF,iBACE,oBAAoB,OAChB,OAAO,yBAAyB,kBAAkB,MAAM,IACxD,KAAA;CACR,QAAQ;EACN,OAAO,OAAO;CAChB;CACA,IAAI,kBAAkB,QAAQ,EAAE,WAAW,iBACzC,OAAO,OAAO;CAGhB,IAAI,YAAY;CAChB,MAAM,cAAc,QAAQ,QAAQ;CAKpC,IAHE,gBAAgB,UAChB,gBAAgB,UAAU,0BAC1BC,oBAAAA,4BAA4B,QAAQ,OAAO,KACjB,OAAO,QAAQ,YAAY,UACrD,aAAa,KAAK,KAChB,yBACE;EACE,MAAM;EACN,WAAW,QAAQ;CACrB,GACA,QACF,IAAI,yBACN;MAEA,aAAa,QAAQ,OAAO;CAE9B,IAAI,aAAa,OAAO,kBACtB,OAAO,OAAO;CAEhB,MAAM,cAAe,QAA6C;CAClE,IAAI,gBAAgB,QAAQ,gBAAgB,aAAa;EACvD,MAAM,YAAa,QAAsB,cAAc,CAAC;EACxD,KAAA,GAAA,gBAAA,QAAA,CAAY,SAAS,GACnB,OAAO,OAAO;EAEhB,KAAK,MAAM,YAAY,WAAW;GAChC,KAAA,GAAA,gBAAA,QAAA,CAAY,QAAQ,GAClB,OAAO,OAAO;GAEhB,IACE,OAAO,SAAS,OAAO,YACvB,uBAAuB,IAAI,SAAS,EAAE,GAEtC;GAEF,IAAI,OAAO,SAAS,SAAS,YAAY,SAAS,KAAK,SAAS,GAC9D,aAAa,UAAU,SAAS,IAAI;GAEtC,MAAM,OAAgB,SAAS;GAC/B,IAAI,QAAQ,MACV,aACE,OAAO,SAAS,WACZ,UAAU,IAAI,IACd,+BAA+B,MAAM,SAAS;EAExD;EACA,IAAI;EACJ,IAAI;GACF,qBACE,oBAAoB,OAChB,OAAO,yBAAyB,kBAAkB,eAAe,IACjE,KAAA;EACR,QAAQ;GACN,OAAO,OAAO;EAChB;EACA,IAAI,sBAAsB,MAAM;GAC9B,IAAI,EAAE,WAAW,qBACf,OAAO,OAAO;GAEhB,IAAI,mBAAmB,SAAS,MAC9B,aAAa,+BACX,mBAAmB,OACnB,SACF;EAEJ;CACF;CACA,OAAO,KAAK,IAAI,OAAO,kBAAkB,SAAS;AACpD;;;;;;;AAQA,SAAgB,qBACd,WACA,YACA,aACwB;CACxB,MAAM,SAAiC,OAAO,OAAO,IAAI;CACzD,MAAM,aAAyD,CAAC;CAChE,IAAI,WAAW;CACf,KAAK,MAAM,CAAC,MAAM,aAAa,OAAO,QAAQ,SAAS,GAAG;EACxD,MAAM,SAAS,WAAW;EAC1B,MAAM,UAAU,KAAK,MAAM,MAAM;EACjC,OAAO,QAAQ;EACf,YAAY;EACZ,WAAW,KAAK;GAAE;GAAM,WAAW,SAAS;EAAQ,CAAC;CACvD;CACA,IAAI,WAAW,cAAc;CAC7B,IAAI,YAAY,KAAK,WAAW,WAAW,GACzC,OAAO;CAET,WAAW,MAAM,GAAG,MAAM,EAAE,YAAY,EAAE,SAAS;CACnD,KAAK,IAAI,IAAI,GAAG,WAAW,GAAG,KAAK,IAAI,KAAK,WAAW,QAAQ;EAC7D,OAAO,WAAW,EAAE,CAAC,SAAS;EAC9B;CACF;CACA,OAAO;AACT;;;;;;AAOA,MAAM,0BAA0B;;;;;AAMhC,MAAa,qBAAqB,OAChC,WAAyB,iBACqB;CAC9C,MAAM,MAAM,MAAM,aAAa,QAAQ;CACvC,MAAM,eAAe,SAAyB,IAAI,MAAM,IAAI;CAC5D,MAAM,WAAW,aAAa;CAC9B,QAAQ,YAAiC;EACvC,MAAM,QAAQ,wBAAwB,SAAS,aAAa,QAAQ;EACpE,OAAO,WAAW,KAAK,KAAK,QAAQ,uBAAuB,IAAI;CACjE;AACF;;AAGA,MAAa,sBAAsB;CACjC,MAAM,aAA4B,CAElC;CAEA,QAAc;EACZ,KAAK,MAAM,OAAO,OAAO,KAAK,UAAU,GACtC,OAAO,WAAW;CAEtB;CAEA,gBAAyB;EACvB,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,SAAS;CAC1C;AACF"}
|