@librechat/agents 3.2.32 → 3.2.33

package/dist/esm/utils/tokens.mjs

@@ -1,6 +1,6 @@
-import { Tokenizer } from 'ai-tokenizer';
-import { ContentTypes } from '../common/enum.mjs';
-
+import "../common/enum.mjs";
+import { Tokenizer } from "ai-tokenizer";
+//#region src/utils/tokens.ts
 /** Anthropic minimum image token cost. */
 const ANTHROPIC_IMAGE_MIN_TOKENS = 1024;
 /** Anthropic divisor: tokens = width × height / 750. */
@@ -14,312 +14,217 @@ const OPENAI_IMAGE_TOKENS_PER_TILE = 170;
 /** Safety margin for image and document token estimates (5% overestimate). */
 const IMAGE_TOKEN_SAFETY_MARGIN = 1.05;
 /**
- * Anthropic PDF: each page costs image tokens + text tokens.
- * Typical range is 1500-3000 tokens/page. Using 2000 as midpoint.
- */
-const ANTHROPIC_PDF_TOKENS_PER_PAGE = 2000;
+* Anthropic PDF: each page costs image tokens + text tokens.
+* Typical range is 1500-3000 tokens/page. Using 2000 as midpoint.
+*/
+const ANTHROPIC_PDF_TOKENS_PER_PAGE = 2e3;
 /** OpenAI PDF: each page rendered as high-detail image. ~1500 tokens typical. */
 const OPENAI_PDF_TOKENS_PER_PAGE = 1500;
 /** Approximate base64 bytes per PDF page for page count estimation. */
-const BASE64_BYTES_PER_PDF_PAGE = 75_000;
+const BASE64_BYTES_PER_PDF_PAGE = 75e3;
 /** Fallback token cost for URL-referenced documents without local data. */
-const URL_DOCUMENT_FALLBACK_TOKENS = 2000;
+const URL_DOCUMENT_FALLBACK_TOKENS = 2e3;
 /**
- * Extracts image dimensions from the first bytes of a base64-encoded
- * PNG, JPEG, GIF, or WebP without decoding the full image.
- * Returns null if the format is unrecognized or data is too short.
- */
+* Extracts image dimensions from the first bytes of a base64-encoded
+* PNG, JPEG, GIF, or WebP without decoding the full image.
+* Returns null if the format is unrecognized or data is too short.
+*/
 function extractImageDimensions(base64Data) {
-    const raw = base64Data.startsWith('data:')
-        ? base64Data.slice(base64Data.indexOf(',') + 1)
-        : base64Data;
-    if (raw.length < 32) {
-        return null;
-    }
-    const bytes = new Uint8Array(Buffer.from(raw.slice(0, 80), 'base64'));
-    if (bytes[0] === 0x89 && bytes[1] === 0x50) {
-        // PNG: width at bytes 16-19, height at 20-23 (big-endian)
-        const width = (bytes[16] << 24) | (bytes[17] << 16) | (bytes[18] << 8) | bytes[19];
-        const height = (bytes[20] << 24) | (bytes[21] << 16) | (bytes[22] << 8) | bytes[23];
-        return { width, height };
-    }
-    if (bytes[0] === 0xff && bytes[1] === 0xd8) {
-        // JPEG: scan for SOF0 (0xFFC0) or SOF2 (0xFFC2) marker
-        for (let i = 2; i < bytes.length - 9; i++) {
-            if (bytes[i] === 0xff &&
-                (bytes[i + 1] === 0xc0 || bytes[i + 1] === 0xc2)) {
-                const height = (bytes[i + 5] << 8) | bytes[i + 6];
-                const width = (bytes[i + 7] << 8) | bytes[i + 8];
-                return { width, height };
-            }
-        }
-        return null;
-    }
-    if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46) {
-        // GIF: width at bytes 6-7, height at 8-9 (little-endian)
-        const width = bytes[6] | (bytes[7] << 8);
-        const height = bytes[8] | (bytes[9] << 8);
-        return { width, height };
-    }
-    if (bytes[0] === 0x52 &&
-        bytes[1] === 0x49 &&
-        bytes[2] === 0x46 &&
-        bytes[3] === 0x46 &&
-        bytes[8] === 0x57 &&
-        bytes[9] === 0x45 &&
-        bytes[10] === 0x42 &&
-        bytes[11] === 0x50) {
-        // WebP VP8: width at bytes 26-27, height at 28-29
-        if (bytes.length > 29) {
-            const width = (bytes[26] | (bytes[27] << 8)) & 0x3fff;
-            const height = (bytes[28] | (bytes[29] << 8)) & 0x3fff;
-            return { width, height };
-        }
-        return null;
-    }
-    return null;
+	const raw = base64Data.startsWith("data:") ? base64Data.slice(base64Data.indexOf(",") + 1) : base64Data;
+	if (raw.length < 32) return null;
+	const bytes = new Uint8Array(Buffer.from(raw.slice(0, 80), "base64"));
+	if (bytes[0] === 137 && bytes[1] === 80) return {
+		width: bytes[16] << 24 | bytes[17] << 16 | bytes[18] << 8 | bytes[19],
+		height: bytes[20] << 24 | bytes[21] << 16 | bytes[22] << 8 | bytes[23]
+	};
+	if (bytes[0] === 255 && bytes[1] === 216) {
+		for (let i = 2; i < bytes.length - 9; i++) if (bytes[i] === 255 && (bytes[i + 1] === 192 || bytes[i + 1] === 194)) {
+			const height = bytes[i + 5] << 8 | bytes[i + 6];
+			return {
+				width: bytes[i + 7] << 8 | bytes[i + 8],
+				height
+			};
+		}
+		return null;
+	}
+	if (bytes[0] === 71 && bytes[1] === 73 && bytes[2] === 70) return {
+		width: bytes[6] | bytes[7] << 8,
+		height: bytes[8] | bytes[9] << 8
+	};
+	if (bytes[0] === 82 && bytes[1] === 73 && bytes[2] === 70 && bytes[3] === 70 && bytes[8] === 87 && bytes[9] === 69 && bytes[10] === 66 && bytes[11] === 80) {
+		if (bytes.length > 29) return {
+			width: (bytes[26] | bytes[27] << 8) & 16383,
+			height: (bytes[28] | bytes[29] << 8) & 16383
+		};
+		return null;
+	}
+	return null;
 }
 /** Estimates image token cost for Anthropic/Bedrock (Claude). */
 function estimateAnthropicImageTokens(width, height) {
-    return Math.max(ANTHROPIC_IMAGE_MIN_TOKENS, Math.ceil((width * height) / ANTHROPIC_IMAGE_DIVISOR));
+	return Math.max(ANTHROPIC_IMAGE_MIN_TOKENS, Math.ceil(width * height / ANTHROPIC_IMAGE_DIVISOR));
 }
 /** Estimates image token cost for OpenAI (high detail). */
-function estimateOpenAIImageTokens(width, height, detail = 'high') {
-    if (detail === 'low') {
-        return OPENAI_IMAGE_LOW_TOKENS;
-    }
-    const tiles = Math.ceil(width / OPENAI_IMAGE_TILE_SIZE) *
-        Math.ceil(height / OPENAI_IMAGE_TILE_SIZE);
-    return OPENAI_IMAGE_LOW_TOKENS + tiles * OPENAI_IMAGE_TOKENS_PER_TILE;
+function estimateOpenAIImageTokens(width, height, detail = "high") {
+	if (detail === "low") return OPENAI_IMAGE_LOW_TOKENS;
+	return OPENAI_IMAGE_LOW_TOKENS + Math.ceil(width / OPENAI_IMAGE_TILE_SIZE) * Math.ceil(height / OPENAI_IMAGE_TILE_SIZE) * OPENAI_IMAGE_TOKENS_PER_TILE;
 }
 /**
- * Estimates token cost for an image content block.
- * Extracts dimensions from base64 header when available.
- * Falls back to Anthropic minimum (1024) when dimensions can't be determined.
- */
+* Estimates token cost for an image content block.
+* Extracts dimensions from base64 header when available.
+* Falls back to Anthropic minimum (1024) when dimensions can't be determined.
+*/
 function estimateImageBlockTokens(block, encoding) {
-    let base64Data;
-    if (block.type === ContentTypes.IMAGE_URL || block.type === 'image_url') {
-        const imageUrl = block.image_url;
-        const url = typeof imageUrl === 'string' ? imageUrl : imageUrl?.url;
-        if (typeof url === 'string' && url.startsWith('data:')) {
-            base64Data = url;
-        }
-        else {
-            return ANTHROPIC_IMAGE_MIN_TOKENS;
-        }
-    }
-    else if (block.type === 'image') {
-        const source = block.source;
-        if (source?.type === 'base64' && typeof source.data === 'string') {
-            base64Data = source.data;
-        }
-        else {
-            return ANTHROPIC_IMAGE_MIN_TOKENS;
-        }
-    }
-    else {
-        return ANTHROPIC_IMAGE_MIN_TOKENS;
-    }
-    const dims = extractImageDimensions(base64Data);
-    if (dims == null) {
-        return ANTHROPIC_IMAGE_MIN_TOKENS;
-    }
-    if (encoding === 'claude') {
-        return estimateAnthropicImageTokens(dims.width, dims.height);
-    }
-    return estimateOpenAIImageTokens(dims.width, dims.height);
+	let base64Data;
+	if (block.type === "image_url" || block.type === "image_url") {
+		const imageUrl = block.image_url;
+		const url = typeof imageUrl === "string" ? imageUrl : imageUrl?.url;
+		if (typeof url === "string" && url.startsWith("data:")) base64Data = url;
+		else return ANTHROPIC_IMAGE_MIN_TOKENS;
+	} else if (block.type === "image") {
+		const source = block.source;
+		if (source?.type === "base64" && typeof source.data === "string") base64Data = source.data;
+		else return ANTHROPIC_IMAGE_MIN_TOKENS;
+	} else return ANTHROPIC_IMAGE_MIN_TOKENS;
+	const dims = extractImageDimensions(base64Data);
+	if (dims == null) return ANTHROPIC_IMAGE_MIN_TOKENS;
+	if (encoding === "claude") return estimateAnthropicImageTokens(dims.width, dims.height);
+	return estimateOpenAIImageTokens(dims.width, dims.height);
 }
 /**
- * Estimates token cost for a document/file content block.
- * Handles both LangChain standard format (`type: 'file'` with `source_type`)
- * and Anthropic format (`type: 'document'` with `source`).
- *
- * - Plain text: tokenized directly via `getTokenCount`.
- * - Base64 PDF: page count estimated from base64 length × per-page cost.
- * - URL reference: conservative flat estimate.
- */
+* Estimates token cost for a document/file content block.
+* Handles both LangChain standard format (`type: 'file'` with `source_type`)
+* and Anthropic format (`type: 'document'` with `source`).
+*
+* - Plain text: tokenized directly via `getTokenCount`.
+* - Base64 PDF: page count estimated from base64 length × per-page cost.
+* - URL reference: conservative flat estimate.
+*/
 function estimateDocumentBlockTokens(block, encoding, getTokenCount) {
-    const pdfTokensPerPage = encoding === 'claude'
-        ? ANTHROPIC_PDF_TOKENS_PER_PAGE
-        : OPENAI_PDF_TOKENS_PER_PAGE;
-    // LangChain standard format: type='file', source_type, data/text/url, mime_type
-    const sourceType = block.source_type;
-    if (typeof sourceType === 'string') {
-        const mimeType = (block.mime_type ?? '').split(';')[0];
-        if (sourceType === 'text' && typeof block.text === 'string') {
-            return getTokenCount(block.text);
-        }
-        if (sourceType === 'base64' && typeof block.data === 'string') {
-            if (mimeType === 'application/pdf' || mimeType === '') {
-                const pageEstimate = Math.max(1, Math.ceil(block.data.length / BASE64_BYTES_PER_PDF_PAGE));
-                return pageEstimate * pdfTokensPerPage;
-            }
-            // Image inside a file block — delegate to image estimation
-            if (mimeType.startsWith('image/')) {
-                return estimateImageBlockTokens({
-                    ...block,
-                    type: 'image',
-                    source: { type: 'base64', data: block.data },
-                }, encoding);
-            }
-            return getTokenCount(block.data);
-        }
-        if (sourceType === 'url') {
-            return URL_DOCUMENT_FALLBACK_TOKENS;
-        }
-        return URL_DOCUMENT_FALLBACK_TOKENS;
-    }
-    // Anthropic format: type='document', source: { type, data, media_type }
-    const source = block.source;
-    if (source == null) {
-        return URL_DOCUMENT_FALLBACK_TOKENS;
-    }
-    if (source.type === 'text' && typeof source.data === 'string') {
-        return getTokenCount(source.data);
-    }
-    if (source.type === 'base64' && typeof source.data === 'string') {
-        const mediaType = (source.media_type ?? '').split(';')[0];
-        if (mediaType === 'application/pdf' || mediaType === '') {
-            const pageEstimate = Math.max(1, Math.ceil(source.data.length / BASE64_BYTES_PER_PDF_PAGE));
-            return pageEstimate * pdfTokensPerPage;
-        }
-        if (mediaType.startsWith('image/')) {
-            return estimateImageBlockTokens({ type: 'image', source: { type: 'base64', data: source.data } }, encoding);
-        }
-        return getTokenCount(source.data);
-    }
-    if (source.type === 'url') {
-        return URL_DOCUMENT_FALLBACK_TOKENS;
-    }
-    // content-type source (wraps other blocks like images)
-    if (source.type === 'content' && Array.isArray(source.content)) {
-        let total = 0;
-        for (const inner of source.content) {
-            if (inner != null && typeof inner === 'object' && 'type' in inner) {
-                const innerBlock = inner;
-                if (innerBlock.type === 'image') {
-                    total += estimateImageBlockTokens(innerBlock, encoding);
-                }
-            }
-        }
-        return total > 0 ? total : URL_DOCUMENT_FALLBACK_TOKENS;
-    }
-    return URL_DOCUMENT_FALLBACK_TOKENS;
+	const pdfTokensPerPage = encoding === "claude" ? ANTHROPIC_PDF_TOKENS_PER_PAGE : OPENAI_PDF_TOKENS_PER_PAGE;
+	const sourceType = block.source_type;
+	if (typeof sourceType === "string") {
+		const mimeType = (block.mime_type ?? "").split(";")[0];
+		if (sourceType === "text" && typeof block.text === "string") return getTokenCount(block.text);
+		if (sourceType === "base64" && typeof block.data === "string") {
+			if (mimeType === "application/pdf" || mimeType === "") return Math.max(1, Math.ceil(block.data.length / BASE64_BYTES_PER_PDF_PAGE)) * pdfTokensPerPage;
+			if (mimeType.startsWith("image/")) return estimateImageBlockTokens({
+				...block,
+				type: "image",
+				source: {
+					type: "base64",
+					data: block.data
+				}
+			}, encoding);
+			return getTokenCount(block.data);
+		}
+		if (sourceType === "url") return URL_DOCUMENT_FALLBACK_TOKENS;
+		return URL_DOCUMENT_FALLBACK_TOKENS;
+	}
+	const source = block.source;
+	if (source == null) return URL_DOCUMENT_FALLBACK_TOKENS;
+	if (source.type === "text" && typeof source.data === "string") return getTokenCount(source.data);
+	if (source.type === "base64" && typeof source.data === "string") {
+		const mediaType = (source.media_type ?? "").split(";")[0];
+		if (mediaType === "application/pdf" || mediaType === "") return Math.max(1, Math.ceil(source.data.length / BASE64_BYTES_PER_PDF_PAGE)) * pdfTokensPerPage;
+		if (mediaType.startsWith("image/")) return estimateImageBlockTokens({
+			type: "image",
+			source: {
+				type: "base64",
+				data: source.data
+			}
+		}, encoding);
+		return getTokenCount(source.data);
+	}
+	if (source.type === "url") return URL_DOCUMENT_FALLBACK_TOKENS;
+	if (source.type === "content" && Array.isArray(source.content)) {
+		let total = 0;
+		for (const inner of source.content) if (inner != null && typeof inner === "object" && "type" in inner) {
+			const innerBlock = inner;
+			if (innerBlock.type === "image") total += estimateImageBlockTokens(innerBlock, encoding);
+		}
+		return total > 0 ? total : URL_DOCUMENT_FALLBACK_TOKENS;
+	}
+	return URL_DOCUMENT_FALLBACK_TOKENS;
 }
 const tokenizers = {};
-async function getTokenizer(encoding = 'o200k_base') {
-    const cached = tokenizers[encoding];
-    if (cached) {
-        return cached;
-    }
-    const data = encoding === 'claude'
-        ? await import('ai-tokenizer/encoding/claude')
-        : await import('ai-tokenizer/encoding/o200k_base');
-    const instance = new Tokenizer(data);
-    tokenizers[encoding] = instance;
-    return instance;
+async function getTokenizer(encoding = "o200k_base") {
+	const cached = tokenizers[encoding];
+	if (cached) return cached;
+	const instance = new Tokenizer(encoding === "claude" ? await import("ai-tokenizer/encoding/claude") : await import("ai-tokenizer/encoding/o200k_base"));
+	tokenizers[encoding] = instance;
+	return instance;
 }
 function encodingForModel(model) {
-    if (model.toLowerCase().includes('claude')) {
-        return 'claude';
-    }
-    return 'o200k_base';
+	if (model.toLowerCase().includes("claude")) return "claude";
+	return "o200k_base";
 }
-function getTokenCountForMessage(message, getTokenCount, encoding = 'o200k_base') {
-    const tokensPerMessage = 3;
-    const processValue = (value) => {
-        if (Array.isArray(value)) {
-            for (const raw of value) {
-                const item = raw;
-                if (item == null || typeof item.type !== 'string') {
-                    continue;
-                }
-                if (item.type === ContentTypes.ERROR) {
-                    continue;
-                }
-                if (item.type === ContentTypes.IMAGE_URL ||
-                    item.type === 'image_url' ||
-                    item.type === 'image') {
-                    numTokens += Math.ceil(estimateImageBlockTokens(item, encoding) * IMAGE_TOKEN_SAFETY_MARGIN);
-                    continue;
-                }
-                if (item.type === 'document' ||
-                    item.type === 'file' ||
-                    item.type === ContentTypes.IMAGE_FILE) {
-                    numTokens += Math.ceil(estimateDocumentBlockTokens(item, encoding, getTokenCount) *
-                        IMAGE_TOKEN_SAFETY_MARGIN);
-                    continue;
-                }
-                if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {
-                    const toolName = item.tool_call.name;
-                    if (typeof toolName === 'string' && toolName.length > 0) {
-                        numTokens += getTokenCount(toolName);
-                    }
-                    const args = item.tool_call.args;
-                    if (typeof args === 'string' && args.length > 0) {
-                        numTokens += getTokenCount(args);
-                    }
-                    const output = item.tool_call.output;
-                    if (typeof output === 'string' && output.length > 0) {
-                        numTokens += getTokenCount(output);
-                    }
-                    continue;
-                }
-                const nestedValue = item[item.type];
-                if (nestedValue == null) {
-                    continue;
-                }
-                processValue(nestedValue);
-            }
-        }
-        else if (typeof value === 'string') {
-            numTokens += getTokenCount(value);
-        }
-        else if (typeof value === 'number') {
-            numTokens += getTokenCount(value.toString());
-        }
-        else if (typeof value === 'boolean') {
-            numTokens += getTokenCount(value.toString());
-        }
-    };
-    let numTokens = tokensPerMessage;
-    processValue(message.content);
-    return numTokens;
+function getTokenCountForMessage(message, getTokenCount, encoding = "o200k_base") {
+	const tokensPerMessage = 3;
+	const processValue = (value) => {
+		if (Array.isArray(value)) for (const raw of value) {
+			const item = raw;
+			if (item == null || typeof item.type !== "string") continue;
+			if (item.type === "error") continue;
+			if (item.type === "image_url" || item.type === "image_url" || item.type === "image") {
+				numTokens += Math.ceil(estimateImageBlockTokens(item, encoding) * IMAGE_TOKEN_SAFETY_MARGIN);
+				continue;
+			}
+			if (item.type === "document" || item.type === "file" || item.type === "image_file") {
+				numTokens += Math.ceil(estimateDocumentBlockTokens(item, encoding, getTokenCount) * IMAGE_TOKEN_SAFETY_MARGIN);
+				continue;
+			}
+			if (item.type === "tool_call" && item.tool_call != null) {
+				const toolName = item.tool_call.name;
+				if (typeof toolName === "string" && toolName.length > 0) numTokens += getTokenCount(toolName);
+				const args = item.tool_call.args;
+				if (typeof args === "string" && args.length > 0) numTokens += getTokenCount(args);
+				const output = item.tool_call.output;
+				if (typeof output === "string" && output.length > 0) numTokens += getTokenCount(output);
+				continue;
+			}
+			const nestedValue = item[item.type];
+			if (nestedValue == null) continue;
+			processValue(nestedValue);
+		}
+		else if (typeof value === "string") numTokens += getTokenCount(value);
+		else if (typeof value === "number") numTokens += getTokenCount(value.toString());
+		else if (typeof value === "boolean") numTokens += getTokenCount(value.toString());
+	};
+	let numTokens = tokensPerMessage;
+	processValue(message.content);
+	return numTokens;
 }
 /**
- * Anthropic's API consistently reports ~10% more tokens than the local
- * claude tokenizer due to internal message framing and content encoding.
- * Verified empirically across content types via the count_tokens endpoint.
- */
+* Anthropic's API consistently reports ~10% more tokens than the local
+* claude tokenizer due to internal message framing and content encoding.
+* Verified empirically across content types via the count_tokens endpoint.
+*/
 const CLAUDE_TOKEN_CORRECTION = 1.1;
 /**
- * Creates a token counter function using the specified encoding.
- * Lazily loads the encoding data on first use via dynamic import.
- */
-const createTokenCounter = async (encoding = 'o200k_base') => {
-    const tok = await getTokenizer(encoding);
-    const countTokens = (text) => tok.count(text);
-    const isClaude = encoding === 'claude';
-    return (message) => {
-        const count = getTokenCountForMessage(message, countTokens, encoding);
-        return isClaude ? Math.ceil(count * CLAUDE_TOKEN_CORRECTION) : count;
-    };
+* Creates a token counter function using the specified encoding.
+* Lazily loads the encoding data on first use via dynamic import.
+*/
+const createTokenCounter = async (encoding = "o200k_base") => {
+	const tok = await getTokenizer(encoding);
+	const countTokens = (text) => tok.count(text);
+	const isClaude = encoding === "claude";
+	return (message) => {
+		const count = getTokenCountForMessage(message, countTokens, encoding);
+		return isClaude ? Math.ceil(count * CLAUDE_TOKEN_CORRECTION) : count;
+	};
 };
 /** Utility to manage the token encoder lifecycle explicitly. */
 const TokenEncoderManager = {
-    async initialize() {
-        // No-op: ai-tokenizer is synchronously initialized from bundled data.
-    },
-    reset() {
-        for (const key of Object.keys(tokenizers)) {
-            delete tokenizers[key];
-        }
-    },
-    isInitialized() {
-        return Object.keys(tokenizers).length > 0;
-    },
+	async initialize() {},
+	reset() {
+		for (const key of Object.keys(tokenizers)) delete tokenizers[key];
+	},
+	isInitialized() {
+		return Object.keys(tokenizers).length > 0;
+	}
 };
-
+//#endregion
 export { TokenEncoderManager, createTokenCounter, encodingForModel, estimateAnthropicImageTokens, estimateOpenAIImageTokens, extractImageDimensions, getTokenCountForMessage };
-//# sourceMappingURL=tokens.mjs.map
+
+//# sourceMappingURL=tokens.mjs.map

package/dist/esm/utils/tokens.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"tokens.mjs","sources":["../../../src/utils/tokens.ts"],"sourcesContent":["import { Tokenizer } from 'ai-tokenizer';\nimport type { BaseMessage } from '@langchain/core/messages';\nimport { ContentTypes } from '@/common/enum';\n\nexport type EncodingName = 'o200k_base' | 'claude';\n\n/** Anthropic minimum image token cost. */\nconst ANTHROPIC_IMAGE_MIN_TOKENS = 1024;\n/** Anthropic divisor: tokens = width × height / 750. */\nconst ANTHROPIC_IMAGE_DIVISOR = 750;\n/** OpenAI low-detail fixed cost. */\nconst OPENAI_IMAGE_LOW_TOKENS = 85;\n/** OpenAI high-detail tile size. */\nconst OPENAI_IMAGE_TILE_SIZE = 512;\n/** OpenAI high-detail tokens per tile. */\nconst OPENAI_IMAGE_TOKENS_PER_TILE = 170;\n/** Google Gemini fixed per-image cost. */\nconst _GEMINI_IMAGE_TOKENS = 258;\n/** Safety margin for image and document token estimates (5% overestimate). */\nconst IMAGE_TOKEN_SAFETY_MARGIN = 1.05;\n\n/**\n * Anthropic PDF: each page costs image tokens + text tokens.\n * Typical range is 1500-3000 tokens/page. Using 2000 as midpoint.\n */\nconst ANTHROPIC_PDF_TOKENS_PER_PAGE = 2000;\n/** OpenAI PDF: each page rendered as high-detail image. ~1500 tokens typical. */\nconst OPENAI_PDF_TOKENS_PER_PAGE = 1500;\n/** Gemini PDF: fixed 258 tokens per page. */\nconst _GEMINI_PDF_TOKENS_PER_PAGE = 258;\n/** Approximate base64 bytes per PDF page for page count estimation. */\nconst BASE64_BYTES_PER_PDF_PAGE = 75_000;\n/** Fallback token cost for URL-referenced documents without local data. */\nconst URL_DOCUMENT_FALLBACK_TOKENS = 2000;\n\n/**\n * Extracts image dimensions from the first bytes of a base64-encoded\n * PNG, JPEG, GIF, or WebP without decoding the full image.\n * Returns null if the format is unrecognized or data is too short.\n */\nexport function extractImageDimensions(\n  base64Data: string\n): { width: number; height: number } | null {\n  const raw = base64Data.startsWith('data:')\n    ? base64Data.slice(base64Data.indexOf(',') + 1)\n    : base64Data;\n\n  if (raw.length < 32) {\n    return null;\n  }\n\n  const bytes = new Uint8Array(Buffer.from(raw.slice(0, 80), 'base64'));\n\n  if (bytes[0] === 0x89 && bytes[1] === 0x50) {\n    // PNG: width at bytes 16-19, height at 20-23 (big-endian)\n    const width =\n      (bytes[16] << 24) | (bytes[17] << 16) | (bytes[18] << 8) | bytes[19];\n    const height =\n      (bytes[20] << 24) | (bytes[21] << 16) | (bytes[22] << 8) | bytes[23];\n    return { width, height };\n  }\n\n  if (bytes[0] === 0xff && bytes[1] === 0xd8) {\n    // JPEG: scan for SOF0 (0xFFC0) or SOF2 (0xFFC2) marker\n    for (let i = 2; i < bytes.length - 9; i++) {\n      if (\n        bytes[i] === 0xff &&\n        (bytes[i + 1] === 0xc0 || bytes[i + 1] === 0xc2)\n      ) {\n        const height = (bytes[i + 5] << 8) | bytes[i + 6];\n        const width = (bytes[i + 7] << 8) | bytes[i + 8];\n        return { width, height };\n      }\n    }\n    return null;\n  }\n\n  if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46) {\n    // GIF: width at bytes 6-7, height at 8-9 (little-endian)\n    const width = bytes[6] | (bytes[7] << 8);\n    const height = bytes[8] | (bytes[9] << 8);\n    return { width, height };\n  }\n\n  if (\n    bytes[0] === 0x52 &&\n    bytes[1] === 0x49 &&\n    bytes[2] === 0x46 &&\n    bytes[3] === 0x46 &&\n    bytes[8] === 0x57 &&\n    bytes[9] === 0x45 &&\n    bytes[10] === 0x42 &&\n    bytes[11] === 0x50\n  ) {\n    // WebP VP8: width at bytes 26-27, height at 28-29\n    if (bytes.length > 29) {\n      const width = (bytes[26] | (bytes[27] << 8)) & 0x3fff;\n      const height = (bytes[28] | (bytes[29] << 8)) & 0x3fff;\n      return { width, height };\n    }\n    return null;\n  }\n\n  return null;\n}\n\n/** Estimates image token cost for Anthropic/Bedrock (Claude). */\nexport function estimateAnthropicImageTokens(\n  width: number,\n  height: number\n): number {\n  return Math.max(\n    ANTHROPIC_IMAGE_MIN_TOKENS,\n    Math.ceil((width * height) / ANTHROPIC_IMAGE_DIVISOR)\n  );\n}\n\n/** Estimates image token cost for OpenAI (high detail). */\nexport function estimateOpenAIImageTokens(\n  width: number,\n  height: number,\n  detail: string = 'high'\n): number {\n  if (detail === 'low') {\n    return OPENAI_IMAGE_LOW_TOKENS;\n  }\n  const tiles =\n    Math.ceil(width / OPENAI_IMAGE_TILE_SIZE) *\n    Math.ceil(height / OPENAI_IMAGE_TILE_SIZE);\n  return OPENAI_IMAGE_LOW_TOKENS + tiles * OPENAI_IMAGE_TOKENS_PER_TILE;\n}\n\n/**\n * Estimates token cost for an image content block.\n * Extracts dimensions from base64 header when available.\n * Falls back to Anthropic minimum (1024) when dimensions can't be determined.\n */\nfunction estimateImageBlockTokens(\n  block: Record<string, unknown>,\n  encoding: EncodingName\n): number {\n  let base64Data: string | undefined;\n\n  if (block.type === ContentTypes.IMAGE_URL || block.type === 'image_url') {\n    const imageUrl = block.image_url as string | { url?: string } | undefined;\n    const url = typeof imageUrl === 'string' ? imageUrl : imageUrl?.url;\n    if (typeof url === 'string' && url.startsWith('data:')) {\n      base64Data = url;\n    } else {\n      return ANTHROPIC_IMAGE_MIN_TOKENS;\n    }\n  } else if (block.type === 'image') {\n    const source = block.source as { type?: string; data?: string } | undefined;\n    if (source?.type === 'base64' && typeof source.data === 'string') {\n      base64Data = source.data;\n    } else {\n      return ANTHROPIC_IMAGE_MIN_TOKENS;\n    }\n  } else {\n    return ANTHROPIC_IMAGE_MIN_TOKENS;\n  }\n\n  const dims = extractImageDimensions(base64Data);\n  if (dims == null) {\n    return ANTHROPIC_IMAGE_MIN_TOKENS;\n  }\n\n  if (encoding === 'claude') {\n    return estimateAnthropicImageTokens(dims.width, dims.height);\n  }\n  return estimateOpenAIImageTokens(dims.width, dims.height);\n}\n\n/**\n * Estimates token cost for a document/file content block.\n * Handles both LangChain standard format (`type: 'file'` with `source_type`)\n * and Anthropic format (`type: 'document'` with `source`).\n *\n * - Plain text: tokenized directly via `getTokenCount`.\n * - Base64 PDF: page count estimated from base64 length × per-page cost.\n * - URL reference: conservative flat estimate.\n */\nfunction estimateDocumentBlockTokens(\n  block: Record<string, unknown>,\n  encoding: EncodingName,\n  getTokenCount: (text: string) => number\n): number {\n  const pdfTokensPerPage =\n    encoding === 'claude'\n      ? ANTHROPIC_PDF_TOKENS_PER_PAGE\n      : OPENAI_PDF_TOKENS_PER_PAGE;\n\n  // LangChain standard format: type='file', source_type, data/text/url, mime_type\n  const sourceType = block.source_type as string | undefined;\n  if (typeof sourceType === 'string') {\n    const mimeType = ((block.mime_type as string | undefined) ?? '').split(\n      ';'\n    )[0];\n\n    if (sourceType === 'text' && typeof block.text === 'string') {\n      return getTokenCount(block.text as string);\n    }\n\n    if (sourceType === 'base64' && typeof block.data === 'string') {\n      if (mimeType === 'application/pdf' || mimeType === '') {\n        const pageEstimate = Math.max(\n          1,\n          Math.ceil((block.data as string).length / BASE64_BYTES_PER_PDF_PAGE)\n        );\n        return pageEstimate * pdfTokensPerPage;\n      }\n      // Image inside a file block — delegate to image estimation\n      if (mimeType.startsWith('image/')) {\n        return estimateImageBlockTokens(\n          {\n            ...block,\n            type: 'image',\n            source: { type: 'base64', data: block.data },\n          },\n          encoding\n        );\n      }\n      return getTokenCount(block.data as string);\n    }\n\n    if (sourceType === 'url') {\n      return URL_DOCUMENT_FALLBACK_TOKENS;\n    }\n\n    return URL_DOCUMENT_FALLBACK_TOKENS;\n  }\n\n  // Anthropic format: type='document', source: { type, data, media_type }\n  const source = block.source as\n    | {\n        type?: string;\n        data?: string;\n        media_type?: string;\n        content?: unknown[];\n      }\n    | undefined;\n\n  if (source == null) {\n    return URL_DOCUMENT_FALLBACK_TOKENS;\n  }\n\n  if (source.type === 'text' && typeof source.data === 'string') {\n    return getTokenCount(source.data);\n  }\n\n  if (source.type === 'base64' && typeof source.data === 'string') {\n    const mediaType = (source.media_type ?? '').split(';')[0];\n    if (mediaType === 'application/pdf' || mediaType === '') {\n      const pageEstimate = Math.max(\n        1,\n        Math.ceil(source.data.length / BASE64_BYTES_PER_PDF_PAGE)\n      );\n      return pageEstimate * pdfTokensPerPage;\n    }\n    if (mediaType.startsWith('image/')) {\n      return estimateImageBlockTokens(\n        { type: 'image', source: { type: 'base64', data: source.data } },\n        encoding\n      );\n    }\n    return getTokenCount(source.data);\n  }\n\n  if (source.type === 'url') {\n    return URL_DOCUMENT_FALLBACK_TOKENS;\n  }\n\n  // content-type source (wraps other blocks like images)\n  if (source.type === 'content' && Array.isArray(source.content)) {\n    let total = 0;\n    for (const inner of source.content) {\n      if (inner != null && typeof inner === 'object' && 'type' in inner) {\n        const innerBlock = inner as Record<string, unknown>;\n        if (innerBlock.type === 'image') {\n          total += estimateImageBlockTokens(innerBlock, encoding);\n        }\n      }\n    }\n    return total > 0 ? total : URL_DOCUMENT_FALLBACK_TOKENS;\n  }\n\n  return URL_DOCUMENT_FALLBACK_TOKENS;\n}\n\nconst tokenizers: Partial<Record<EncodingName, Tokenizer>> = {};\n\nasync function getTokenizer(\n  encoding: EncodingName = 'o200k_base'\n): Promise<Tokenizer> {\n  const cached = tokenizers[encoding];\n  if (cached) {\n    return cached;\n  }\n  const data =\n    encoding === 'claude'\n      ? await import('ai-tokenizer/encoding/claude')\n      : await import('ai-tokenizer/encoding/o200k_base');\n  const instance = new Tokenizer(data);\n  tokenizers[encoding] = instance;\n  return instance;\n}\n\nexport function encodingForModel(model: string): EncodingName {\n  if (model.toLowerCase().includes('claude')) {\n    return 'claude';\n  }\n  return 'o200k_base';\n}\n\nexport function getTokenCountForMessage(\n  message: BaseMessage,\n  getTokenCount: (text: string) => number,\n  encoding: EncodingName = 'o200k_base'\n): number {\n  const tokensPerMessage = 3;\n\n  type ContentBlock = Record<string, unknown> & {\n    type?: string;\n    tool_call?: { name?: string; args?: string; output?: string };\n  };\n\n  const processValue = (value: unknown): void => {\n    if (Array.isArray(value)) {\n      for (const raw of value) {\n        const item = raw as ContentBlock | null | undefined;\n        if (item == null || typeof item.type !== 'string') {\n          continue;\n        }\n        if (item.type === ContentTypes.ERROR) {\n          continue;\n        }\n\n        if (\n          item.type === ContentTypes.IMAGE_URL ||\n          item.type === 'image_url' ||\n          item.type === 'image'\n        ) {\n          numTokens += Math.ceil(\n            estimateImageBlockTokens(item, encoding) * IMAGE_TOKEN_SAFETY_MARGIN\n          );\n          continue;\n        }\n\n        if (\n          item.type === 'document' ||\n          item.type === 'file' ||\n          item.type === ContentTypes.IMAGE_FILE\n        ) {\n          numTokens += Math.ceil(\n            estimateDocumentBlockTokens(item, encoding, getTokenCount) *\n              IMAGE_TOKEN_SAFETY_MARGIN\n          );\n          continue;\n        }\n\n        if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {\n          const toolName = item.tool_call.name;\n          if (typeof toolName === 'string' && toolName.length > 0) {\n            numTokens += getTokenCount(toolName);\n          }\n          const args = item.tool_call.args;\n          if (typeof args === 'string' && args.length > 0) {\n            numTokens += getTokenCount(args);\n          }\n          const output = item.tool_call.output;\n          if (typeof output === 'string' && output.length > 0) {\n            numTokens += getTokenCount(output);\n          }\n          continue;\n        }\n\n        const nestedValue = item[item.type];\n        if (nestedValue == null) {\n          continue;\n        }\n\n        processValue(nestedValue);\n      }\n    } else if (typeof value === 'string') {\n      numTokens += getTokenCount(value);\n    } else if (typeof value === 'number') {\n      numTokens += getTokenCount(value.toString());\n    } else if (typeof value === 'boolean') {\n      numTokens += getTokenCount(value.toString());\n    }\n  };\n\n  let numTokens = tokensPerMessage;\n  processValue(message.content);\n  return numTokens;\n}\n\n/**\n * Anthropic's API consistently reports ~10% more tokens than the local\n * claude tokenizer due to internal message framing and content encoding.\n * Verified empirically across content types via the count_tokens endpoint.\n */\nconst CLAUDE_TOKEN_CORRECTION = 1.1;\n\n/**\n * Creates a token counter function using the specified encoding.\n * Lazily loads the encoding data on first use via dynamic import.\n */\nexport const createTokenCounter = async (\n  encoding: EncodingName = 'o200k_base'\n): Promise<(message: BaseMessage) => number> => {\n  const tok = await getTokenizer(encoding);\n  const countTokens = (text: string): number => tok.count(text);\n  const isClaude = encoding === 'claude';\n  return (message: BaseMessage): number => {\n    const count = getTokenCountForMessage(message, countTokens, encoding);\n    return isClaude ? Math.ceil(count * CLAUDE_TOKEN_CORRECTION) : count;\n  };\n};\n\n/** Utility to manage the token encoder lifecycle explicitly. */\nexport const TokenEncoderManager = {\n  async initialize(): Promise<void> {\n    // No-op: ai-tokenizer is synchronously initialized from bundled data.\n  },\n\n  reset(): void {\n    for (const key of Object.keys(tokenizers)) {\n      delete tokenizers[key as EncodingName];\n    }\n  },\n\n  isInitialized(): boolean {\n    return Object.keys(tokenizers).length > 0;\n  },\n};\n"],"names":[],"mappings":";;;AAMA;AACA,MAAM,0BAA0B,GAAG,IAAI;AACvC;AACA,MAAM,uBAAuB,GAAG,GAAG;AACnC;AACA,MAAM,uBAAuB,GAAG,EAAE;AAClC;AACA,MAAM,sBAAsB,GAAG,GAAG;AAClC;AACA,MAAM,4BAA4B,GAAG,GAAG;AAGxC;AACA,MAAM,yBAAyB,GAAG,IAAI;AAEtC;;;AAGG;AACH,MAAM,6BAA6B,GAAG,IAAI;AAC1C;AACA,MAAM,0BAA0B,GAAG,IAAI;AAGvC;AACA,MAAM,yBAAyB,GAAG,MAAM;AACxC;AACA,MAAM,4BAA4B,GAAG,IAAI;AAEzC;;;;AAIG;AACG,SAAU,sBAAsB,CACpC,UAAkB,EAAA;AAElB,IAAA,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC,OAAO;AACvC,UAAE,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;UAC5C,UAAU;AAEd,IAAA,IAAI,GAAG,CAAC,MAAM,GAAG,EAAE,EAAE;AACnB,QAAA,OAAO,IAAI;IACb;IAEA,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;AAErE,IAAA,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE;;AAE1C,QAAA,MAAM,KAAK,GACT,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,CAAC;AACtE,QAAA,MAAM,MAAM,GACV,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,CAAC;AACtE,QAAA,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE;IAC1B;AAEA,IAAA,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE;;AAE1C,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;AACzC,YAAA,IACE,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;AACjB,iBAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,EAChD;AACA,gBAAA,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;AACjD,gBAAA,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;AAChD,gBAAA,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE;YAC1B;QACF;AACA,QAAA,OAAO,IAAI;IACb;IAEA,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE;;AAE/D,QAAA,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACxC,QAAA,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACzC,QAAA,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE;IAC1B;AAEA,IAAA,IACE,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;AACjB,QAAA,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;AACjB,QAAA,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;AACjB,QAAA,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;AACjB,QAAA,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;AACjB,QAAA,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;AACjB,QAAA,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI;AAClB,QAAA,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,EAClB;;AAEA,QAAA,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE,EAAE;AACrB,YAAA,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM;AACrD,YAAA,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM;AACtD,YAAA,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE;QAC1B;AACA,QAAA,OAAO,IAAI;IACb;AAEA,IAAA,OAAO,IAAI;AACb;AAEA;AACM,SAAU,4BAA4B,CAC1C,KAAa,EACb,MAAc,EAAA;AAEd,IAAA,OAAO,IAAI,CAAC,GAAG,CACb,0BAA0B,EAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,MAAM,IAAI,uBAAuB,CAAC,CACtD;AACH;AAEA;AACM,SAAU,yBAAyB,CACvC,KAAa,EACb,MAAc,EACd,SAAiB,MAAM,EAAA;AAEvB,IAAA,IAAI,MAAM,KAAK,KAAK,EAAE;AACpB,QAAA,OAAO,uBAAuB;IAChC;IACA,MAAM,KAAK,GACT,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,sBAAsB,CAAC;AACzC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,sBAAsB,CAAC;AAC5C,IAAA,OAAO,uBAAuB,GAAG,KAAK,GAAG,4BAA4B;AACvE;AAEA;;;;AAIG;AACH,SAAS,wBAAwB,CAC/B,KAA8B,EAC9B,QAAsB,EAAA;AAEtB,IAAA,IAAI,UAA8B;AAElC,IAAA,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE;AACvE,QAAA,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAkD;AACzE,QAAA,MAAM,GAAG,GAAG,OAAO,QAAQ,KAAK,QAAQ,GAAG,QAAQ,GAAG,QAAQ,EAAE,GAAG;AACnE,QAAA,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;YACtD,UAAU,GAAG,GAAG;QAClB;aAAO;AACL,YAAA,OAAO,0BAA0B;QACnC;IACF;AAAO,SAAA,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE;AACjC,QAAA,MAAM,MAAM,GAAG,KAAK,CAAC,MAAsD;AAC3E,QAAA,IAAI,MAAM,EAAE,IAAI,KAAK,QAAQ,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE;AAChE,YAAA,UAAU,GAAG,MAAM,CAAC,IAAI;QAC1B;aAAO;AACL,YAAA,OAAO,0BAA0B;QACnC;IACF;SAAO;AACL,QAAA,OAAO,0BAA0B;IACnC;AAEA,IAAA,MAAM,IAAI,GAAG,sBAAsB,CAAC,UAAU,CAAC;AAC/C,IAAA,IAAI,IAAI,IAAI,IAAI,EAAE;AAChB,QAAA,OAAO,0BAA0B;IACnC;AAEA,IAAA,IAAI,QAAQ,KAAK,QAAQ,EAAE;QACzB,OAAO,4BAA4B,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;IAC9D;IACA,OAAO,yBAAyB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;AAC3D;AAEA;;;;;;;;AAQG;AACH,SAAS,2BAA2B,CAClC,KAA8B,EAC9B,QAAsB,EACtB,aAAuC,EAAA;AAEvC,IAAA,MAAM,gBAAgB,GACpB,QAAQ,KAAK;AACX,UAAE;UACA,0BAA0B;;AAGhC,IAAA,MAAM,UAAU,GAAG,KAAK,CAAC,WAAiC;AAC1D,IAAA,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE;AAClC,QAAA,MAAM,QAAQ,GAAG,CAAE,KAAK,CAAC,SAAgC,IAAI,EAAE,EAAE,KAAK,CACpE,GAAG,CACJ,CAAC,CAAC,CAAC;QAEJ,IAAI,UAAU,KAAK,MAAM,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE;AAC3D,YAAA,OAAO,aAAa,CAAC,KAAK,CAAC,IAAc,CAAC;QAC5C;QAEA,IAAI,UAAU,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE;YAC7D,IAAI,QAAQ,KAAK,iBAAiB,IAAI,QAAQ,KAAK,EAAE,EAAE;gBACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,CAAC,EACD,IAAI,CAAC,IAAI,CAAE,KAAK,CAAC,IAAe,CAAC,MAAM,GAAG,yBAAyB,CAAC,CACrE;gBACD,OAAO,YAAY,GAAG,gBAAgB;YACxC;;AAEA,YAAA,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;AACjC,gBAAA,OAAO,wBAAwB,CAC7B;AACE,oBAAA,GAAG,KAAK;AACR,oBAAA,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE;iBAC7C,EACD,QAAQ,CACT;YACH;AACA,YAAA,OAAO,aAAa,CAAC,KAAK,CAAC,IAAc,CAAC;QAC5C;AAEA,QAAA,IAAI,UAAU,KAAK,KAAK,EAAE;AACxB,YAAA,OAAO,4BAA4B;QACrC;AAEA,QAAA,OAAO,4BAA4B;IACrC;;AAGA,IAAA,MAAM,MAAM,GAAG,KAAK,CAAC,MAOR;AAEb,IAAA,IAAI,MAAM,IAAI,IAAI,EAAE;AAClB,QAAA,OAAO,4BAA4B;IACrC;AAEA,IAAA,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE;AAC7D,QAAA,OAAO,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC;IACnC;AAEA,IAAA,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE;AAC/D,QAAA,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACzD,IAAI,SAAS,KAAK,iBAAiB,IAAI,SAAS,KAAK,EAAE,EAAE;YACvD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,CAAC,EACD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,yBAAyB,CAAC,CAC1D;YACD,OAAO,YAAY,GAAG,gBAAgB;QACxC;AACA,QAAA,IAAI,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;YAClC,OAAO,wBAAwB,CAC7B,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,EAChE,QAAQ,CACT;QACH;AACA,QAAA,OAAO,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC;IACnC;AAEA,IAAA,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE;AACzB,QAAA,OAAO,4BAA4B;IACrC;;AAGA,IAAA,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;QAC9D,IAAI,KAAK,GAAG,CAAC;AACb,QAAA,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE;AAClC,YAAA,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,IAAI,KAAK,EAAE;gBACjE,MAAM,UAAU,GAAG,KAAgC;AACnD,gBAAA,IAAI,UAAU,CAAC,IAAI,KAAK,OAAO,EAAE;AAC/B,oBAAA,KAAK,IAAI,wBAAwB,CAAC,UAAU,EAAE,QAAQ,CAAC;gBACzD;YACF;QACF;QACA,OAAO,KAAK,GAAG,CAAC,GAAG,KAAK,GAAG,4BAA4B;IACzD;AAEA,IAAA,OAAO,4BAA4B;AACrC;AAEA,MAAM,UAAU,GAA6C,EAAE;AAE/D,eAAe,YAAY,CACzB,QAAA,GAAyB,YAAY,EAAA;AAErC,IAAA,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;IACnC,IAAI,MAAM,EAAE;AACV,QAAA,OAAO,MAAM;IACf;AACA,IAAA,MAAM,IAAI,GACR,QAAQ,KAAK;AACX,UAAE,MAAM,OAAO,8BAA8B;AAC7C,UAAE,MAAM,OAAO,kCAAkC,CAAC;AACtD,IAAA,MAAM,QAAQ,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC;AACpC,IAAA,UAAU,CAAC,QAAQ,CAAC,GAAG,QAAQ;AAC/B,IAAA,OAAO,QAAQ;AACjB;AAEM,SAAU,gBAAgB,CAAC,KAAa,EAAA;IAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;AAC1C,QAAA,OAAO,QAAQ;IACjB;AACA,IAAA,OAAO,YAAY;AACrB;AAEM,SAAU,uBAAuB,CACrC,OAAoB,EACpB,aAAuC,EACvC,WAAyB,YAAY,EAAA;IAErC,MAAM,gBAAgB,GAAG,CAAC;AAO1B,IAAA,MAAM,YAAY,GAAG,CAAC,KAAc,KAAU;AAC5C,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AACxB,YAAA,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE;gBACvB,MAAM,IAAI,GAAG,GAAsC;gBACnD,IAAI,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE;oBACjD;gBACF;gBACA,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,KAAK,EAAE;oBACpC;gBACF;AAEA,gBAAA,IACE,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS;oBACpC,IAAI,CAAC,IAAI,KAAK,WAAW;AACzB,oBAAA,IAAI,CAAC,IAAI,KAAK,OAAO,EACrB;AACA,oBAAA,SAAS,IAAI,IAAI,CAAC,IAAI,CACpB,wBAAwB,CAAC,IAAI,EAAE,QAAQ,CAAC,GAAG,yBAAyB,CACrE;oBACD;gBACF;AAEA,gBAAA,IACE,IAAI,CAAC,IAAI,KAAK,UAAU;oBACxB,IAAI,CAAC,IAAI,KAAK,MAAM;AACpB,oBAAA,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,UAAU,EACrC;AACA,oBAAA,SAAS,IAAI,IAAI,CAAC,IAAI,CACpB,2BAA2B,CAAC,IAAI,EAAE,QAAQ,EAAE,aAAa,CAAC;AACxD,wBAAA,yBAAyB,CAC5B;oBACD;gBACF;AAEA,gBAAA,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;AAClE,oBAAA,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI;oBACpC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;AACvD,wBAAA,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC;oBACtC;AACA,oBAAA,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI;oBAChC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AAC/C,wBAAA,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC;oBAClC;AACA,oBAAA,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM;oBACpC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;AACnD,wBAAA,SAAS,IAAI,aAAa,CAAC,MAAM,CAAC;oBACpC;oBACA;gBACF;gBAEA,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACnC,gBAAA,IAAI,WAAW,IAAI,IAAI,EAAE;oBACvB;gBACF;gBAEA,YAAY,CAAC,WAAW,CAAC;YAC3B;QACF;AAAO,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACpC,YAAA,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC;QACnC;AAAO,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YACpC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC9C;AAAO,aAAA,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;YACrC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC9C;AACF,IAAA,CAAC;IAED,IAAI,SAAS,GAAG,gBAAgB;AAChC,IAAA,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;AAC7B,IAAA,OAAO,SAAS;AAClB;AAEA;;;;AAIG;AACH,MAAM,uBAAuB,GAAG,GAAG;AAEnC;;;AAGG;MACU,kBAAkB,GAAG,OAChC,QAAA,GAAyB,YAAY,KACQ;AAC7C,IAAA,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC;AACxC,IAAA,MAAM,WAAW,GAAG,CAAC,IAAY,KAAa,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;AAC7D,IAAA,MAAM,QAAQ,GAAG,QAAQ,KAAK,QAAQ;IACtC,OAAO,CAAC,OAAoB,KAAY;QACtC,MAAM,KAAK,GAAG,uBAAuB,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC;AACrE,QAAA,OAAO,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,uBAAuB,CAAC,GAAG,KAAK;AACtE,IAAA,CAAC;AACH;AAEA;AACO,MAAM,mBAAmB,GAAG;AACjC,IAAA,MAAM,UAAU,GAAA;;IAEhB,CAAC;IAED,KAAK,GAAA;QACH,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;AACzC,YAAA,OAAO,UAAU,CAAC,GAAmB,CAAC;QACxC;IACF,CAAC;IAED,aAAa,GAAA;QACX,OAAO,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC;IAC3C,CAAC;;;;;"}
+{"version":3,"file":"tokens.mjs","names":[],"sources":["../../../src/utils/tokens.ts"],"sourcesContent":["import { Tokenizer } from 'ai-tokenizer';\nimport type { BaseMessage } from '@langchain/core/messages';\nimport { ContentTypes } from '@/common/enum';\n\nexport type EncodingName = 'o200k_base' | 'claude';\n\n/** Anthropic minimum image token cost. */\nconst ANTHROPIC_IMAGE_MIN_TOKENS = 1024;\n/** Anthropic divisor: tokens = width × height / 750. */\nconst ANTHROPIC_IMAGE_DIVISOR = 750;\n/** OpenAI low-detail fixed cost. */\nconst OPENAI_IMAGE_LOW_TOKENS = 85;\n/** OpenAI high-detail tile size. */\nconst OPENAI_IMAGE_TILE_SIZE = 512;\n/** OpenAI high-detail tokens per tile. */\nconst OPENAI_IMAGE_TOKENS_PER_TILE = 170;\n/** Google Gemini fixed per-image cost. */\nconst _GEMINI_IMAGE_TOKENS = 258;\n/** Safety margin for image and document token estimates (5% overestimate). */\nconst IMAGE_TOKEN_SAFETY_MARGIN = 1.05;\n\n/**\n * Anthropic PDF: each page costs image tokens + text tokens.\n * Typical range is 1500-3000 tokens/page. Using 2000 as midpoint.\n */\nconst ANTHROPIC_PDF_TOKENS_PER_PAGE = 2000;\n/** OpenAI PDF: each page rendered as high-detail image. ~1500 tokens typical. */\nconst OPENAI_PDF_TOKENS_PER_PAGE = 1500;\n/** Gemini PDF: fixed 258 tokens per page. */\nconst _GEMINI_PDF_TOKENS_PER_PAGE = 258;\n/** Approximate base64 bytes per PDF page for page count estimation. */\nconst BASE64_BYTES_PER_PDF_PAGE = 75_000;\n/** Fallback token cost for URL-referenced documents without local data. */\nconst URL_DOCUMENT_FALLBACK_TOKENS = 2000;\n\n/**\n * Extracts image dimensions from the first bytes of a base64-encoded\n * PNG, JPEG, GIF, or WebP without decoding the full image.\n * Returns null if the format is unrecognized or data is too short.\n */\nexport function extractImageDimensions(\n  base64Data: string\n): { width: number; height: number } | null {\n  const raw = base64Data.startsWith('data:')\n    ? base64Data.slice(base64Data.indexOf(',') + 1)\n    : base64Data;\n\n  if (raw.length < 32) {\n    return null;\n  }\n\n  const bytes = new Uint8Array(Buffer.from(raw.slice(0, 80), 'base64'));\n\n  if (bytes[0] === 0x89 && bytes[1] === 0x50) {\n    // PNG: width at bytes 16-19, height at 20-23 (big-endian)\n    const width =\n      (bytes[16] << 24) | (bytes[17] << 16) | (bytes[18] << 8) | bytes[19];\n    const height =\n      (bytes[20] << 24) | (bytes[21] << 16) | (bytes[22] << 8) | bytes[23];\n    return { width, height };\n  }\n\n  if (bytes[0] === 0xff && bytes[1] === 0xd8) {\n    // JPEG: scan for SOF0 (0xFFC0) or SOF2 (0xFFC2) marker\n    for (let i = 2; i < bytes.length - 9; i++) {\n      if (\n        bytes[i] === 0xff &&\n        (bytes[i + 1] === 0xc0 || bytes[i + 1] === 0xc2)\n      ) {\n        const height = (bytes[i + 5] << 8) | bytes[i + 6];\n        const width = (bytes[i + 7] << 8) | bytes[i + 8];\n        return { width, height };\n      }\n    }\n    return null;\n  }\n\n  if (bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46) {\n    // GIF: width at bytes 6-7, height at 8-9 (little-endian)\n    const width = bytes[6] | (bytes[7] << 8);\n    const height = bytes[8] | (bytes[9] << 8);\n    return { width, height };\n  }\n\n  if (\n    bytes[0] === 0x52 &&\n    bytes[1] === 0x49 &&\n    bytes[2] === 0x46 &&\n    bytes[3] === 0x46 &&\n    bytes[8] === 0x57 &&\n    bytes[9] === 0x45 &&\n    bytes[10] === 0x42 &&\n    bytes[11] === 0x50\n  ) {\n    // WebP VP8: width at bytes 26-27, height at 28-29\n    if (bytes.length > 29) {\n      const width = (bytes[26] | (bytes[27] << 8)) & 0x3fff;\n      const height = (bytes[28] | (bytes[29] << 8)) & 0x3fff;\n      return { width, height };\n    }\n    return null;\n  }\n\n  return null;\n}\n\n/** Estimates image token cost for Anthropic/Bedrock (Claude). */\nexport function estimateAnthropicImageTokens(\n  width: number,\n  height: number\n): number {\n  return Math.max(\n    ANTHROPIC_IMAGE_MIN_TOKENS,\n    Math.ceil((width * height) / ANTHROPIC_IMAGE_DIVISOR)\n  );\n}\n\n/** Estimates image token cost for OpenAI (high detail). */\nexport function estimateOpenAIImageTokens(\n  width: number,\n  height: number,\n  detail: string = 'high'\n): number {\n  if (detail === 'low') {\n    return OPENAI_IMAGE_LOW_TOKENS;\n  }\n  const tiles =\n    Math.ceil(width / OPENAI_IMAGE_TILE_SIZE) *\n    Math.ceil(height / OPENAI_IMAGE_TILE_SIZE);\n  return OPENAI_IMAGE_LOW_TOKENS + tiles * OPENAI_IMAGE_TOKENS_PER_TILE;\n}\n\n/**\n * Estimates token cost for an image content block.\n * Extracts dimensions from base64 header when available.\n * Falls back to Anthropic minimum (1024) when dimensions can't be determined.\n */\nfunction estimateImageBlockTokens(\n  block: Record<string, unknown>,\n  encoding: EncodingName\n): number {\n  let base64Data: string | undefined;\n\n  if (block.type === ContentTypes.IMAGE_URL || block.type === 'image_url') {\n    const imageUrl = block.image_url as string | { url?: string } | undefined;\n    const url = typeof imageUrl === 'string' ? imageUrl : imageUrl?.url;\n    if (typeof url === 'string' && url.startsWith('data:')) {\n      base64Data = url;\n    } else {\n      return ANTHROPIC_IMAGE_MIN_TOKENS;\n    }\n  } else if (block.type === 'image') {\n    const source = block.source as { type?: string; data?: string } | undefined;\n    if (source?.type === 'base64' && typeof source.data === 'string') {\n      base64Data = source.data;\n    } else {\n      return ANTHROPIC_IMAGE_MIN_TOKENS;\n    }\n  } else {\n    return ANTHROPIC_IMAGE_MIN_TOKENS;\n  }\n\n  const dims = extractImageDimensions(base64Data);\n  if (dims == null) {\n    return ANTHROPIC_IMAGE_MIN_TOKENS;\n  }\n\n  if (encoding === 'claude') {\n    return estimateAnthropicImageTokens(dims.width, dims.height);\n  }\n  return estimateOpenAIImageTokens(dims.width, dims.height);\n}\n\n/**\n * Estimates token cost for a document/file content block.\n * Handles both LangChain standard format (`type: 'file'` with `source_type`)\n * and Anthropic format (`type: 'document'` with `source`).\n *\n * - Plain text: tokenized directly via `getTokenCount`.\n * - Base64 PDF: page count estimated from base64 length × per-page cost.\n * - URL reference: conservative flat estimate.\n */\nfunction estimateDocumentBlockTokens(\n  block: Record<string, unknown>,\n  encoding: EncodingName,\n  getTokenCount: (text: string) => number\n): number {\n  const pdfTokensPerPage =\n    encoding === 'claude'\n      ? ANTHROPIC_PDF_TOKENS_PER_PAGE\n      : OPENAI_PDF_TOKENS_PER_PAGE;\n\n  // LangChain standard format: type='file', source_type, data/text/url, mime_type\n  const sourceType = block.source_type as string | undefined;\n  if (typeof sourceType === 'string') {\n    const mimeType = ((block.mime_type as string | undefined) ?? '').split(\n      ';'\n    )[0];\n\n    if (sourceType === 'text' && typeof block.text === 'string') {\n      return getTokenCount(block.text as string);\n    }\n\n    if (sourceType === 'base64' && typeof block.data === 'string') {\n      if (mimeType === 'application/pdf' || mimeType === '') {\n        const pageEstimate = Math.max(\n          1,\n          Math.ceil((block.data as string).length / BASE64_BYTES_PER_PDF_PAGE)\n        );\n        return pageEstimate * pdfTokensPerPage;\n      }\n      // Image inside a file block — delegate to image estimation\n      if (mimeType.startsWith('image/')) {\n        return estimateImageBlockTokens(\n          {\n            ...block,\n            type: 'image',\n            source: { type: 'base64', data: block.data },\n          },\n          encoding\n        );\n      }\n      return getTokenCount(block.data as string);\n    }\n\n    if (sourceType === 'url') {\n      return URL_DOCUMENT_FALLBACK_TOKENS;\n    }\n\n    return URL_DOCUMENT_FALLBACK_TOKENS;\n  }\n\n  // Anthropic format: type='document', source: { type, data, media_type }\n  const source = block.source as\n    | {\n        type?: string;\n        data?: string;\n        media_type?: string;\n        content?: unknown[];\n      }\n    | undefined;\n\n  if (source == null) {\n    return URL_DOCUMENT_FALLBACK_TOKENS;\n  }\n\n  if (source.type === 'text' && typeof source.data === 'string') {\n    return getTokenCount(source.data);\n  }\n\n  if (source.type === 'base64' && typeof source.data === 'string') {\n    const mediaType = (source.media_type ?? '').split(';')[0];\n    if (mediaType === 'application/pdf' || mediaType === '') {\n      const pageEstimate = Math.max(\n        1,\n        Math.ceil(source.data.length / BASE64_BYTES_PER_PDF_PAGE)\n      );\n      return pageEstimate * pdfTokensPerPage;\n    }\n    if (mediaType.startsWith('image/')) {\n      return estimateImageBlockTokens(\n        { type: 'image', source: { type: 'base64', data: source.data } },\n        encoding\n      );\n    }\n    return getTokenCount(source.data);\n  }\n\n  if (source.type === 'url') {\n    return URL_DOCUMENT_FALLBACK_TOKENS;\n  }\n\n  // content-type source (wraps other blocks like images)\n  if (source.type === 'content' && Array.isArray(source.content)) {\n    let total = 0;\n    for (const inner of source.content) {\n      if (inner != null && typeof inner === 'object' && 'type' in inner) {\n        const innerBlock = inner as Record<string, unknown>;\n        if (innerBlock.type === 'image') {\n          total += estimateImageBlockTokens(innerBlock, encoding);\n        }\n      }\n    }\n    return total > 0 ? total : URL_DOCUMENT_FALLBACK_TOKENS;\n  }\n\n  return URL_DOCUMENT_FALLBACK_TOKENS;\n}\n\nconst tokenizers: Partial<Record<EncodingName, Tokenizer>> = {};\n\nasync function getTokenizer(\n  encoding: EncodingName = 'o200k_base'\n): Promise<Tokenizer> {\n  const cached = tokenizers[encoding];\n  if (cached) {\n    return cached;\n  }\n  const data =\n    encoding === 'claude'\n      ? await import('ai-tokenizer/encoding/claude')\n      : await import('ai-tokenizer/encoding/o200k_base');\n  const instance = new Tokenizer(data);\n  tokenizers[encoding] = instance;\n  return instance;\n}\n\nexport function encodingForModel(model: string): EncodingName {\n  if (model.toLowerCase().includes('claude')) {\n    return 'claude';\n  }\n  return 'o200k_base';\n}\n\nexport function getTokenCountForMessage(\n  message: BaseMessage,\n  getTokenCount: (text: string) => number,\n  encoding: EncodingName = 'o200k_base'\n): number {\n  const tokensPerMessage = 3;\n\n  type ContentBlock = Record<string, unknown> & {\n    type?: string;\n    tool_call?: { name?: string; args?: string; output?: string };\n  };\n\n  const processValue = (value: unknown): void => {\n    if (Array.isArray(value)) {\n      for (const raw of value) {\n        const item = raw as ContentBlock | null | undefined;\n        if (item == null || typeof item.type !== 'string') {\n          continue;\n        }\n        if (item.type === ContentTypes.ERROR) {\n          continue;\n        }\n\n        if (\n          item.type === ContentTypes.IMAGE_URL ||\n          item.type === 'image_url' ||\n          item.type === 'image'\n        ) {\n          numTokens += Math.ceil(\n            estimateImageBlockTokens(item, encoding) * IMAGE_TOKEN_SAFETY_MARGIN\n          );\n          continue;\n        }\n\n        if (\n          item.type === 'document' ||\n          item.type === 'file' ||\n          item.type === ContentTypes.IMAGE_FILE\n        ) {\n          numTokens += Math.ceil(\n            estimateDocumentBlockTokens(item, encoding, getTokenCount) *\n              IMAGE_TOKEN_SAFETY_MARGIN\n          );\n          continue;\n        }\n\n        if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {\n          const toolName = item.tool_call.name;\n          if (typeof toolName === 'string' && toolName.length > 0) {\n            numTokens += getTokenCount(toolName);\n          }\n          const args = item.tool_call.args;\n          if (typeof args === 'string' && args.length > 0) {\n            numTokens += getTokenCount(args);\n          }\n          const output = item.tool_call.output;\n          if (typeof output === 'string' && output.length > 0) {\n            numTokens += getTokenCount(output);\n          }\n          continue;\n        }\n\n        const nestedValue = item[item.type];\n        if (nestedValue == null) {\n          continue;\n        }\n\n        processValue(nestedValue);\n      }\n    } else if (typeof value === 'string') {\n      numTokens += getTokenCount(value);\n    } else if (typeof value === 'number') {\n      numTokens += getTokenCount(value.toString());\n    } else if (typeof value === 'boolean') {\n      numTokens += getTokenCount(value.toString());\n    }\n  };\n\n  let numTokens = tokensPerMessage;\n  processValue(message.content);\n  return numTokens;\n}\n\n/**\n * Anthropic's API consistently reports ~10% more tokens than the local\n * claude tokenizer due to internal message framing and content encoding.\n * Verified empirically across content types via the count_tokens endpoint.\n */\nconst CLAUDE_TOKEN_CORRECTION = 1.1;\n\n/**\n * Creates a token counter function using the specified encoding.\n * Lazily loads the encoding data on first use via dynamic import.\n */\nexport const createTokenCounter = async (\n  encoding: EncodingName = 'o200k_base'\n): Promise<(message: BaseMessage) => number> => {\n  const tok = await getTokenizer(encoding);\n  const countTokens = (text: string): number => tok.count(text);\n  const isClaude = encoding === 'claude';\n  return (message: BaseMessage): number => {\n    const count = getTokenCountForMessage(message, countTokens, encoding);\n    return isClaude ? Math.ceil(count * CLAUDE_TOKEN_CORRECTION) : count;\n  };\n};\n\n/** Utility to manage the token encoder lifecycle explicitly. */\nexport const TokenEncoderManager = {\n  async initialize(): Promise<void> {\n    // No-op: ai-tokenizer is synchronously initialized from bundled data.\n  },\n\n  reset(): void {\n    for (const key of Object.keys(tokenizers)) {\n      delete tokenizers[key as EncodingName];\n    }\n  },\n\n  isInitialized(): boolean {\n    return Object.keys(tokenizers).length > 0;\n  },\n};\n"],"mappings":";;;;AAOA,MAAM,6BAA6B;;AAEnC,MAAM,0BAA0B;;AAEhC,MAAM,0BAA0B;;AAEhC,MAAM,yBAAyB;;AAE/B,MAAM,+BAA+B;;AAIrC,MAAM,4BAA4B;;;;;AAMlC,MAAM,gCAAgC;;AAEtC,MAAM,6BAA6B;;AAInC,MAAM,4BAA4B;;AAElC,MAAM,+BAA+B;;;;;;AAOrC,SAAgB,uBACd,YAC0C;CAC1C,MAAM,MAAM,WAAW,WAAW,OAAO,IACrC,WAAW,MAAM,WAAW,QAAQ,GAAG,IAAI,CAAC,IAC5C;CAEJ,IAAI,IAAI,SAAS,IACf,OAAO;CAGT,MAAM,QAAQ,IAAI,WAAW,OAAO,KAAK,IAAI,MAAM,GAAG,EAAE,GAAG,QAAQ,CAAC;CAEpE,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,IAMpC,OAAO;EAAE,OAHN,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;EAGnD,QADb,MAAM,OAAO,KAAO,MAAM,OAAO,KAAO,MAAM,OAAO,IAAK,MAAM;CAC5C;CAGzB,IAAI,MAAM,OAAO,OAAQ,MAAM,OAAO,KAAM;EAE1C,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,GAAG,KACpC,IACE,MAAM,OAAO,QACZ,MAAM,IAAI,OAAO,OAAQ,MAAM,IAAI,OAAO,MAC3C;GACA,MAAM,SAAU,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;GAE/C,OAAO;IAAE,OADM,MAAM,IAAI,MAAM,IAAK,MAAM,IAAI;IAC9B;GAAO;EACzB;EAEF,OAAO;CACT;CAEA,IAAI,MAAM,OAAO,MAAQ,MAAM,OAAO,MAAQ,MAAM,OAAO,IAIzD,OAAO;EAAE,OAFK,MAAM,KAAM,MAAM,MAAM;EAEtB,QADD,MAAM,KAAM,MAAM,MAAM;CAChB;CAGzB,IACE,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,OAAO,MACb,MAAM,QAAQ,MACd,MAAM,QAAQ,IACd;EAEA,IAAI,MAAM,SAAS,IAGjB,OAAO;GAAE,QAFM,MAAM,MAAO,MAAM,OAAO,KAAM;GAE/B,SADA,MAAM,MAAO,MAAM,OAAO,KAAM;EACzB;EAEzB,OAAO;CACT;CAEA,OAAO;AACT;;AAGA,SAAgB,6BACd,OACA,QACQ;CACR,OAAO,KAAK,IACV,4BACA,KAAK,KAAM,QAAQ,SAAU,uBAAuB,CACtD;AACF;;AAGA,SAAgB,0BACd,OACA,QACA,SAAiB,QACT;CACR,IAAI,WAAW,OACb,OAAO;CAKT,OAAO,0BAFL,KAAK,KAAK,QAAQ,sBAAsB,IACxC,KAAK,KAAK,SAAS,sBAAsB,IACF;AAC3C;;;;;;AAOA,SAAS,yBACP,OACA,UACQ;CACR,IAAI;CAEJ,IAAI,MAAM,SAAA,eAAmC,MAAM,SAAS,aAAa;EACvE,MAAM,WAAW,MAAM;EACvB,MAAM,MAAM,OAAO,aAAa,WAAW,WAAW,UAAU;EAChE,IAAI,OAAO,QAAQ,YAAY,IAAI,WAAW,OAAO,GACnD,aAAa;OAEb,OAAO;CAEX,OAAO,IAAI,MAAM,SAAS,SAAS;EACjC,MAAM,SAAS,MAAM;EACrB,IAAI,QAAQ,SAAS,YAAY,OAAO,OAAO,SAAS,UACtD,aAAa,OAAO;OAEpB,OAAO;CAEX,OACE,OAAO;CAGT,MAAM,OAAO,uBAAuB,UAAU;CAC9C,IAAI,QAAQ,MACV,OAAO;CAGT,IAAI,aAAa,UACf,OAAO,6BAA6B,KAAK,OAAO,KAAK,MAAM;CAE7D,OAAO,0BAA0B,KAAK,OAAO,KAAK,MAAM;AAC1D;;;;;;;;;;AAWA,SAAS,4BACP,OACA,UACA,eACQ;CACR,MAAM,mBACJ,aAAa,WACT,gCACA;CAGN,MAAM,aAAa,MAAM;CACzB,IAAI,OAAO,eAAe,UAAU;EAClC,MAAM,YAAa,MAAM,aAAoC,GAAA,CAAI,MAC/D,GACF,CAAC,CAAC;EAEF,IAAI,eAAe,UAAU,OAAO,MAAM,SAAS,UACjD,OAAO,cAAc,MAAM,IAAc;EAG3C,IAAI,eAAe,YAAY,OAAO,MAAM,SAAS,UAAU;GAC7D,IAAI,aAAa,qBAAqB,aAAa,IAKjD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAM,MAAM,KAAgB,SAAS,yBAAyB,CAEnD,IAAI;GAGxB,IAAI,SAAS,WAAW,QAAQ,GAC9B,OAAO,yBACL;IACE,GAAG;IACH,MAAM;IACN,QAAQ;KAAE,MAAM;KAAU,MAAM,MAAM;IAAK;GAC7C,GACA,QACF;GAEF,OAAO,cAAc,MAAM,IAAc;EAC3C;EAEA,IAAI,eAAe,OACjB,OAAO;EAGT,OAAO;CACT;CAGA,MAAM,SAAS,MAAM;CASrB,IAAI,UAAU,MACZ,OAAO;CAGT,IAAI,OAAO,SAAS,UAAU,OAAO,OAAO,SAAS,UACnD,OAAO,cAAc,OAAO,IAAI;CAGlC,IAAI,OAAO,SAAS,YAAY,OAAO,OAAO,SAAS,UAAU;EAC/D,MAAM,aAAa,OAAO,cAAc,GAAA,CAAI,MAAM,GAAG,CAAC,CAAC;EACvD,IAAI,cAAc,qBAAqB,cAAc,IAKnD,OAJqB,KAAK,IACxB,GACA,KAAK,KAAK,OAAO,KAAK,SAAS,yBAAyB,CAExC,IAAI;EAExB,IAAI,UAAU,WAAW,QAAQ,GAC/B,OAAO,yBACL;GAAE,MAAM;GAAS,QAAQ;IAAE,MAAM;IAAU,MAAM,OAAO;GAAK;EAAE,GAC/D,QACF;EAEF,OAAO,cAAc,OAAO,IAAI;CAClC;CAEA,IAAI,OAAO,SAAS,OAClB,OAAO;CAIT,IAAI,OAAO,SAAS,aAAa,MAAM,QAAQ,OAAO,OAAO,GAAG;EAC9D,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,OAAO,SACzB,IAAI,SAAS,QAAQ,OAAO,UAAU,YAAY,UAAU,OAAO;GACjE,MAAM,aAAa;GACnB,IAAI,WAAW,SAAS,SACtB,SAAS,yBAAyB,YAAY,QAAQ;EAE1D;EAEF,OAAO,QAAQ,IAAI,QAAQ;CAC7B;CAEA,OAAO;AACT;AAEA,MAAM,aAAuD,CAAC;AAE9D,eAAe,aACb,WAAyB,cACL;CACpB,MAAM,SAAS,WAAW;CAC1B,IAAI,QACF,OAAO;CAMT,MAAM,WAAW,IAAI,UAHnB,aAAa,WACT,MAAM,OAAO,kCACb,MAAM,OAAO,mCACgB;CACnC,WAAW,YAAY;CACvB,OAAO;AACT;AAEA,SAAgB,iBAAiB,OAA6B;CAC5D,IAAI,MAAM,YAAY,CAAC,CAAC,SAAS,QAAQ,GACvC,OAAO;CAET,OAAO;AACT;AAEA,SAAgB,wBACd,SACA,eACA,WAAyB,cACjB;CACR,MAAM,mBAAmB;CAOzB,MAAM,gBAAgB,UAAyB;EAC7C,IAAI,MAAM,QAAQ,KAAK,GACrB,KAAK,MAAM,OAAO,OAAO;GACvB,MAAM,OAAO;GACb,IAAI,QAAQ,QAAQ,OAAO,KAAK,SAAS,UACvC;GAEF,IAAI,KAAK,SAAA,SACP;GAGF,IACE,KAAK,SAAA,eACL,KAAK,SAAS,eACd,KAAK,SAAS,SACd;IACA,aAAa,KAAK,KAChB,yBAAyB,MAAM,QAAQ,IAAI,yBAC7C;IACA;GACF;GAEA,IACE,KAAK,SAAS,cACd,KAAK,SAAS,UACd,KAAK,SAAA,cACL;IACA,aAAa,KAAK,KAChB,4BAA4B,MAAM,UAAU,aAAa,IACvD,yBACJ;IACA;GACF;GAEA,IAAI,KAAK,SAAA,eAAmC,KAAK,aAAa,MAAM;IAClE,MAAM,WAAW,KAAK,UAAU;IAChC,IAAI,OAAO,aAAa,YAAY,SAAS,SAAS,GACpD,aAAa,cAAc,QAAQ;IAErC,MAAM,OAAO,KAAK,UAAU;IAC5B,IAAI,OAAO,SAAS,YAAY,KAAK,SAAS,GAC5C,aAAa,cAAc,IAAI;IAEjC,MAAM,SAAS,KAAK,UAAU;IAC9B,IAAI,OAAO,WAAW,YAAY,OAAO,SAAS,GAChD,aAAa,cAAc,MAAM;IAEnC;GACF;GAEA,MAAM,cAAc,KAAK,KAAK;GAC9B,IAAI,eAAe,MACjB;GAGF,aAAa,WAAW;EAC1B;OACK,IAAI,OAAO,UAAU,UAC1B,aAAa,cAAc,KAAK;OAC3B,IAAI,OAAO,UAAU,UAC1B,aAAa,cAAc,MAAM,SAAS,CAAC;OACtC,IAAI,OAAO,UAAU,WAC1B,aAAa,cAAc,MAAM,SAAS,CAAC;CAE/C;CAEA,IAAI,YAAY;CAChB,aAAa,QAAQ,OAAO;CAC5B,OAAO;AACT;;;;;;AAOA,MAAM,0BAA0B;;;;;AAMhC,MAAa,qBAAqB,OAChC,WAAyB,iBACqB;CAC9C,MAAM,MAAM,MAAM,aAAa,QAAQ;CACvC,MAAM,eAAe,SAAyB,IAAI,MAAM,IAAI;CAC5D,MAAM,WAAW,aAAa;CAC9B,QAAQ,YAAiC;EACvC,MAAM,QAAQ,wBAAwB,SAAS,aAAa,QAAQ;EACpE,OAAO,WAAW,KAAK,KAAK,QAAQ,uBAAuB,IAAI;CACjE;AACF;;AAGA,MAAa,sBAAsB;CACjC,MAAM,aAA4B,CAElC;CAEA,QAAc;EACZ,KAAK,MAAM,OAAO,OAAO,KAAK,UAAU,GACtC,OAAO,WAAW;CAEtB;CAEA,gBAAyB;EACvB,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,SAAS;CAC1C;AACF"}