@hyvmind/tiktoken-ts 0.0.1

package/dist/bpe.d.ts

@@ -0,0 +1,171 @@
+/**
+ * Byte-Pair Encoding (BPE) Implementation
+ * A pure TypeScript implementation of the BPE algorithm
+ *
+ * This implementation uses a hybrid approach:
+ * 1. Pattern-based tokenization for accuracy with special tokens
+ * 2. Statistical estimation for token counting (without full vocabulary)
+ *
+ * For production use with full accuracy, consider loading the actual
+ * vocabulary files from tiktoken.
+ */
+import type { EncodingConfig, EncodingName, SpecialTokens } from "./types.js";
+/**
+ * Default special tokens for OpenAI chat format
+ */
+export declare const DEFAULT_SPECIAL_TOKENS: SpecialTokens;
+/**
+ * cl100k_base encoding configuration
+ * Used by GPT-4, GPT-3.5-turbo
+ */
+export declare const CL100K_BASE_CONFIG: EncodingConfig;
+/**
+ * o200k_base encoding configuration
+ * Used by GPT-4o, GPT-4.1, GPT-5 models
+ */
+export declare const O200K_BASE_CONFIG: EncodingConfig;
+/**
+ * p50k_base encoding configuration (legacy)
+ * Used by older codex models
+ */
+export declare const P50K_BASE_CONFIG: EncodingConfig;
+/**
+ * Claude estimation encoding configuration
+ *
+ * IMPORTANT: This is an ESTIMATION encoding, not an exact BPE tokenizer.
+ *
+ * Claude uses a proprietary tokenizer that is NOT publicly available.
+ * This encoding provides "safe" estimates that intentionally over-count
+ * tokens to prevent API truncation issues.
+ *
+ * Based on research findings (see claude-tokenizer-research.md):
+ * - Claude 3+ uses ~22,000 token vocabulary (much smaller than OpenAI's 100K-200K)
+ * - Claude produces 16-30% MORE tokens than GPT-4 for equivalent content
+ * - Average ~3.5 characters per token (vs GPT-4's ~4)
+ *
+ * This encoding applies a 1.25x safety multiplier to cl100k_base estimates.
+ * For exact Claude token counts, use Anthropic's /v1/messages/count_tokens API.
+ *
+ * @see https://docs.anthropic.com/en/docs/build-with-claude/token-counting
+ */
+export declare const CLAUDE_ESTIMATION_CONFIG: EncodingConfig;
+/**
+ * Get encoding configuration by name
+ *
+ * @param name - Encoding name
+ * @returns Encoding configuration
+ */
+export declare function getEncodingConfig(name: EncodingName): EncodingConfig;
+/**
+ * Safety multiplier for Claude token estimation.
+ *
+ * Research shows Claude produces 16-30% more tokens than GPT-4 for equivalent content:
+ * - English articles: +16%
+ * - Math equations: +21%
+ * - Python code: +30%
+ *
+ * We use 1.25 (25% over-estimate) as a safe default that covers most cases
+ * without being excessively conservative.
+ */
+export declare const CLAUDE_SAFETY_MULTIPLIER = 1.25;
+/**
+ * BPE Tokenizer class
+ * Implements the core BPE algorithm with statistical estimation
+ */
+export declare class BPETokenizer {
+    private readonly config;
+    private readonly specialTokenPatterns;
+    private readonly bytePairCache;
+    private readonly isClaudeEstimation;
+    constructor(encoding?: EncodingName);
+    /**
+     * Initialize special token patterns
+     */
+    private initializeSpecialTokens;
+    /**
+     * Get the encoding name
+     */
+    get encodingName(): EncodingName;
+    /**
+     * Encode text into token IDs
+     * Uses a hybrid approach: special token detection + BPE encoding
+     *
+     * @param text - Input text
+     * @returns Array of token IDs
+     */
+    encode(text: string): number[];
+    /**
+     * Decode token IDs back to text
+     * Note: This is a best-effort decode for estimated tokens
+     *
+     * @param tokens - Array of token IDs
+     * @returns Decoded text
+     */
+    decode(tokens: number[]): string;
+    /**
+     * Count the number of tokens in text
+     * This uses statistical estimation for efficiency
+     *
+     * For Claude models (claude_estimation encoding), this applies a safety
+     * multiplier to ensure estimates err on the side of over-counting,
+     * preventing API truncation issues.
+     *
+     * @param text - Input text
+     * @returns Estimated token count
+     */
+    countTokens(text: string): number;
+    /**
+     * Split text on special tokens while preserving them
+     *
+     * @param text - Input text
+     * @returns Array of segments
+     */
+    private splitOnSpecialTokens;
+    /**
+     * Encode regular text using BPE
+     * Uses pattern-based splitting and byte-pair encoding
+     *
+     * @param text - Text to encode (no special tokens)
+     * @returns Array of token IDs
+     */
+    private encodeText;
+    /**
+     * Apply BPE to a byte sequence
+     * Uses a simplified hash-based approach for estimation
+     *
+     * @param bytes - UTF-8 bytes
+     * @returns Array of token IDs
+     */
+    private bytePairEncode;
+    /**
+     * Estimate the boundary of the next token
+     *
+     * @param bytes - Full byte sequence
+     * @param start - Starting position
+     * @returns Number of bytes for the next token
+     */
+    private estimateTokenBoundary;
+    /**
+     * Convert bytes to a deterministic token ID
+     *
+     * @param bytes - Token bytes
+     * @returns Token ID
+     */
+    private bytesToTokenId;
+    /**
+     * Estimate token count for text using statistical methods
+     * This is the primary method for fast token counting
+     *
+     * ACCURACY NOTES (validated against tiktoken-rs):
+     * - English text: +/-10-15% typically accurate
+     * - Numbers: Now properly models 1-3 digit grouping
+     * - CJK with o200k: Highly efficient (1 token per char typically)
+     * - CJK with cl100k: Less efficient (1-2 tokens per char)
+     * - Emoji: 2-3 tokens each
+     *
+     * @param text - Text to estimate
+     * @returns Estimated token count
+     */
+    private estimateTokenCount;
+}
+//# sourceMappingURL=bpe.d.ts.map

package/dist/bpe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bpe.d.ts","sourceRoot":"","sources":["../src/bpe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAU9E;;GAEG;AACH,eAAO,MAAM,sBAAsB,EAAE,aAOpC,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,cAOhC,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,cAO/B,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,gBAAgB,EAAE,cAO9B,CAAC;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,wBAAwB,EAAE,cAUtC,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,YAAY,GAAG,cAAc,CAapE;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,wBAAwB,OAAO,CAAC;AAE7C;;;GAGG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAsB;IAC3D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAwB;IACtD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;gBAEjC,QAAQ,GAAE,YAA2B;IAUjD;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAgB/B;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAE/B;IAED;;;;;;OAMG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAqB9B;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAoBhC;;;;;;;;;;OAUG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IA2BjC;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IAmC5B;;;;;;OAMG;IACH,OAAO,CAAC,UAAU;IA6BlB;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IA0BtB;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;IAyC7B;;;;;OAKG;IACH,OAAO,CAAC,cAAc;IAStB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,kBAAkB;CA6F3B"}

package/dist/bpe.js

@@ -0,0 +1,478 @@
+/**
+ * Byte-Pair Encoding (BPE) Implementation
+ * A pure TypeScript implementation of the BPE algorithm
+ *
+ * This implementation uses a hybrid approach:
+ * 1. Pattern-based tokenization for accuracy with special tokens
+ * 2. Statistical estimation for token counting (without full vocabulary)
+ *
+ * For production use with full accuracy, consider loading the actual
+ * vocabulary files from tiktoken.
+ */
+import { stringToBytes, bytesToString, containsEmoji, isAscii, countCodePoints, hashString, } from "./utils.js";
+/**
+ * Default special tokens for OpenAI chat format
+ */
+export const DEFAULT_SPECIAL_TOKENS = {
+    endOfText: "<|endoftext|>",
+    imStart: "<|im_start|>",
+    imEnd: "<|im_end|>",
+    imSep: "<|im_sep|>",
+    toolCall: "<|tool_call|>",
+    toolResult: "<|tool_result|>",
+};
+/**
+ * cl100k_base encoding configuration
+ * Used by GPT-4, GPT-3.5-turbo
+ */
+export const CL100K_BASE_CONFIG = {
+    name: "cl100k_base",
+    patternSplit: /(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/gu,
+    specialTokens: DEFAULT_SPECIAL_TOKENS,
+    vocabSize: 100256,
+    averageCharsPerToken: 3.8,
+};
+/**
+ * o200k_base encoding configuration
+ * Used by GPT-4o, GPT-4.1, GPT-5 models
+ */
+export const O200K_BASE_CONFIG = {
+    name: "o200k_base",
+    patternSplit: /[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/gu,
+    specialTokens: DEFAULT_SPECIAL_TOKENS,
+    vocabSize: 200000,
+    averageCharsPerToken: 4.0,
+};
+/**
+ * p50k_base encoding configuration (legacy)
+ * Used by older codex models
+ */
+export const P50K_BASE_CONFIG = {
+    name: "p50k_base",
+    patternSplit: /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu,
+    specialTokens: DEFAULT_SPECIAL_TOKENS,
+    vocabSize: 50257,
+    averageCharsPerToken: 3.5,
+};
+/**
+ * Claude estimation encoding configuration
+ *
+ * IMPORTANT: This is an ESTIMATION encoding, not an exact BPE tokenizer.
+ *
+ * Claude uses a proprietary tokenizer that is NOT publicly available.
+ * This encoding provides "safe" estimates that intentionally over-count
+ * tokens to prevent API truncation issues.
+ *
+ * Based on research findings (see claude-tokenizer-research.md):
+ * - Claude 3+ uses ~22,000 token vocabulary (much smaller than OpenAI's 100K-200K)
+ * - Claude produces 16-30% MORE tokens than GPT-4 for equivalent content
+ * - Average ~3.5 characters per token (vs GPT-4's ~4)
+ *
+ * This encoding applies a 1.25x safety multiplier to cl100k_base estimates.
+ * For exact Claude token counts, use Anthropic's /v1/messages/count_tokens API.
+ *
+ * @see https://docs.anthropic.com/en/docs/build-with-claude/token-counting
+ */
+export const CLAUDE_ESTIMATION_CONFIG = {
+    name: "claude_estimation",
+    // Use cl100k_base pattern as base (70% vocabulary overlap with Claude 1/2)
+    patternSplit: /(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/gu,
+    specialTokens: DEFAULT_SPECIAL_TOKENS,
+    // Claude's estimated vocabulary (~22K) but we use this for informational purposes
+    vocabSize: 22000,
+    // Claude averages ~3.5 chars per token, lower than cl100k_base
+    averageCharsPerToken: 3.5,
+};
+/**
+ * Get encoding configuration by name
+ *
+ * @param name - Encoding name
+ * @returns Encoding configuration
+ */
+export function getEncodingConfig(name) {
+    switch (name) {
+        case "cl100k_base":
+            return CL100K_BASE_CONFIG;
+        case "o200k_base":
+            return O200K_BASE_CONFIG;
+        case "p50k_base":
+            return P50K_BASE_CONFIG;
+        case "claude_estimation":
+            return CLAUDE_ESTIMATION_CONFIG;
+        default:
+            return O200K_BASE_CONFIG;
+    }
+}
+/**
+ * Safety multiplier for Claude token estimation.
+ *
+ * Research shows Claude produces 16-30% more tokens than GPT-4 for equivalent content:
+ * - English articles: +16%
+ * - Math equations: +21%
+ * - Python code: +30%
+ *
+ * We use 1.25 (25% over-estimate) as a safe default that covers most cases
+ * without being excessively conservative.
+ */
+export const CLAUDE_SAFETY_MULTIPLIER = 1.25;
+/**
+ * BPE Tokenizer class
+ * Implements the core BPE algorithm with statistical estimation
+ */
+export class BPETokenizer {
+    config;
+    specialTokenPatterns;
+    bytePairCache;
+    isClaudeEstimation;
+    constructor(encoding = "o200k_base") {
+        this.config = getEncodingConfig(encoding);
+        this.specialTokenPatterns = new Map();
+        this.bytePairCache = new Map();
+        this.isClaudeEstimation = encoding === "claude_estimation";
+        // Pre-compute special token patterns and their token IDs
+        this.initializeSpecialTokens();
+    }
+    /**
+     * Initialize special token patterns
+     */
+    initializeSpecialTokens() {
+        const tokens = this.config.specialTokens;
+        let tokenId = this.config.vocabSize - 10; // Reserve IDs at end of vocab
+        this.specialTokenPatterns.set(tokens.endOfText, tokenId++);
+        this.specialTokenPatterns.set(tokens.imStart, tokenId++);
+        this.specialTokenPatterns.set(tokens.imEnd, tokenId++);
+        this.specialTokenPatterns.set(tokens.imSep, tokenId++);
+        if (tokens.toolCall) {
+            this.specialTokenPatterns.set(tokens.toolCall, tokenId++);
+        }
+        if (tokens.toolResult) {
+            this.specialTokenPatterns.set(tokens.toolResult, tokenId++);
+        }
+    }
+    /**
+     * Get the encoding name
+     */
+    get encodingName() {
+        return this.config.name;
+    }
+    /**
+     * Encode text into token IDs
+     * Uses a hybrid approach: special token detection + BPE encoding
+     *
+     * @param text - Input text
+     * @returns Array of token IDs
+     */
+    encode(text) {
+        if (!text)
+            return [];
+        const tokens = [];
+        // First, split on special tokens
+        const segments = this.splitOnSpecialTokens(text);
+        for (const segment of segments) {
+            if (this.specialTokenPatterns.has(segment)) {
+                // Special token - use its pre-assigned ID
+                tokens.push(this.specialTokenPatterns.get(segment));
+            }
+            else {
+                // Regular text - apply BPE
+                tokens.push(...this.encodeText(segment));
+            }
+        }
+        return tokens;
+    }
+    /**
+     * Decode token IDs back to text
+     * Note: This is a best-effort decode for estimated tokens
+     *
+     * @param tokens - Array of token IDs
+     * @returns Decoded text
+     */
+    decode(tokens) {
+        const parts = [];
+        for (const token of tokens) {
+            // Check if it's a special token
+            for (const [text, id] of this.specialTokenPatterns) {
+                if (id === token) {
+                    parts.push(text);
+                    break;
+                }
+            }
+            // For regular tokens, we'd need the vocabulary to decode properly
+            // Since we don't have it, we return a placeholder
+            parts.push(`<token:${token}>`);
+        }
+        return parts.join("");
+    }
+    /**
+     * Count the number of tokens in text
+     * This uses statistical estimation for efficiency
+     *
+     * For Claude models (claude_estimation encoding), this applies a safety
+     * multiplier to ensure estimates err on the side of over-counting,
+     * preventing API truncation issues.
+     *
+     * @param text - Input text
+     * @returns Estimated token count
+     */
+    countTokens(text) {
+        if (!text)
+            return 0;
+        let count = 0;
+        // Split on special tokens first
+        const segments = this.splitOnSpecialTokens(text);
+        for (const segment of segments) {
+            if (this.specialTokenPatterns.has(segment)) {
+                count += 1;
+            }
+            else {
+                count += this.estimateTokenCount(segment);
+            }
+        }
+        let result = Math.max(1, Math.round(count));
+        // Apply Claude safety multiplier for Claude estimation encoding
+        // This ensures we over-estimate to prevent API truncation
+        if (this.isClaudeEstimation) {
+            result = Math.ceil(result * CLAUDE_SAFETY_MULTIPLIER);
+        }
+        return result;
+    }
+    /**
+     * Split text on special tokens while preserving them
+     *
+     * @param text - Input text
+     * @returns Array of segments
+     */
+    splitOnSpecialTokens(text) {
+        const segments = [];
+        // Build regex pattern for all special tokens
+        const specialTokensList = Array.from(this.specialTokenPatterns.keys());
+        if (specialTokensList.length === 0) {
+            return [text];
+        }
+        const pattern = new RegExp(`(${specialTokensList.map((t) => t.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})`, "g");
+        let lastIndex = 0;
+        let match;
+        while ((match = pattern.exec(text)) !== null) {
+            // Add text before the special token
+            if (match.index > lastIndex) {
+                segments.push(text.slice(lastIndex, match.index));
+            }
+            // Add the special token
+            segments.push(match[0]);
+            lastIndex = match.index + match[0].length;
+        }
+        // Add remaining text
+        if (lastIndex < text.length) {
+            segments.push(text.slice(lastIndex));
+        }
+        return segments.filter((s) => s.length > 0);
+    }
+    /**
+     * Encode regular text using BPE
+     * Uses pattern-based splitting and byte-pair encoding
+     *
+     * @param text - Text to encode (no special tokens)
+     * @returns Array of token IDs
+     */
+    encodeText(text) {
+        const tokens = [];
+        // Split text using the encoding's pattern
+        const matches = text.match(this.config.patternSplit) || [text];
+        for (const match of matches) {
+            // Check cache first
+            const cached = this.bytePairCache.get(match);
+            if (cached) {
+                tokens.push(...cached);
+                continue;
+            }
+            // Convert to bytes and apply BPE
+            const bytes = stringToBytes(match);
+            const encodedTokens = this.bytePairEncode(bytes);
+            // Cache the result
+            if (match.length < 100) {
+                this.bytePairCache.set(match, encodedTokens);
+            }
+            tokens.push(...encodedTokens);
+        }
+        return tokens;
+    }
+    /**
+     * Apply BPE to a byte sequence
+     * Uses a simplified hash-based approach for estimation
+     *
+     * @param bytes - UTF-8 bytes
+     * @returns Array of token IDs
+     */
+    bytePairEncode(bytes) {
+        if (bytes.length === 0)
+            return [];
+        // For accurate BPE, we'd need the full vocabulary and merge rules
+        // Instead, we use a deterministic hash-based approach that:
+        // 1. Groups bytes into likely token boundaries
+        // 2. Assigns consistent token IDs based on content
+        const tokens = [];
+        let i = 0;
+        while (i < bytes.length) {
+            // Determine likely token boundary (1-6 bytes typically)
+            const tokenLength = this.estimateTokenBoundary(bytes, i);
+            const tokenBytes = bytes.slice(i, i + tokenLength);
+            // Generate a consistent token ID from the bytes
+            const tokenId = this.bytesToTokenId(tokenBytes);
+            tokens.push(tokenId);
+            i += tokenLength;
+        }
+        return tokens;
+    }
+    /**
+     * Estimate the boundary of the next token
+     *
+     * @param bytes - Full byte sequence
+     * @param start - Starting position
+     * @returns Number of bytes for the next token
+     */
+    estimateTokenBoundary(bytes, start) {
+        const remaining = bytes.length - start;
+        if (remaining <= 0)
+            return 0;
+        // UTF-8 aware boundary detection
+        const firstByte = bytes[start];
+        // ASCII printable characters often merge together
+        if (firstByte >= 0x20 && firstByte <= 0x7e) {
+            // Check for common patterns
+            let end = start + 1;
+            while (end < bytes.length && end - start < 6) {
+                const b = bytes[end];
+                // Continue if same type of character
+                if (b >= 0x41 && b <= 0x5a) {
+                    // A-Z
+                    end++;
+                }
+                else if (b >= 0x61 && b <= 0x7a) {
+                    // a-z
+                    end++;
+                }
+                else if (b >= 0x30 && b <= 0x39) {
+                    // 0-9
+                    end++;
+                }
+                else {
+                    break;
+                }
+            }
+            return Math.min(end - start, remaining);
+        }
+        // Multi-byte UTF-8 sequences
+        if ((firstByte & 0xe0) === 0xc0)
+            return Math.min(2, remaining);
+        if ((firstByte & 0xf0) === 0xe0)
+            return Math.min(3, remaining);
+        if ((firstByte & 0xf8) === 0xf0)
+            return Math.min(4, remaining);
+        return 1;
+    }
+    /**
+     * Convert bytes to a deterministic token ID
+     *
+     * @param bytes - Token bytes
+     * @returns Token ID
+     */
+    bytesToTokenId(bytes) {
+        // Use a hash to generate consistent token IDs
+        const text = bytesToString(bytes);
+        const hash = hashString(text);
+        // Map to valid token range (avoid special tokens at the end)
+        return hash % (this.config.vocabSize - 20);
+    }
+    /**
+     * Estimate token count for text using statistical methods
+     * This is the primary method for fast token counting
+     *
+     * ACCURACY NOTES (validated against tiktoken-rs):
+     * - English text: +/-10-15% typically accurate
+     * - Numbers: Now properly models 1-3 digit grouping
+     * - CJK with o200k: Highly efficient (1 token per char typically)
+     * - CJK with cl100k: Less efficient (1-2 tokens per char)
+     * - Emoji: 2-3 tokens each
+     *
+     * @param text - Text to estimate
+     * @returns Estimated token count
+     */
+    estimateTokenCount(text) {
+        if (!text)
+            return 0;
+        const codePoints = countCodePoints(text);
+        const byteLength = stringToBytes(text).length;
+        // Start with base estimate, but we'll rebuild for mixed content
+        let estimate = 0;
+        // Handle numbers separately - tiktoken splits into 1-3 digit groups
+        // This is CRITICAL: "12345678901234567890" = 7 tokens, not 40
+        const numberMatches = text.match(/\d+/g);
+        let numberCharsCount = 0;
+        let numberTokenCount = 0;
+        if (numberMatches) {
+            for (const num of numberMatches) {
+                numberCharsCount += num.length;
+                // tiktoken groups digits into 1-3 digit tokens
+                numberTokenCount += Math.ceil(num.length / 3);
+            }
+        }
+        // Handle CJK characters - behavior differs by encoding
+        const cjkChars = text.match(/[\u4e00-\u9fff\u3400-\u4dbf\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/g);
+        let cjkCharsCount = 0;
+        let cjkTokenCount = 0;
+        if (cjkChars) {
+            cjkCharsCount = cjkChars.length;
+            if (this.config.name === "o200k_base") {
+                // o200k_base is HIGHLY efficient for CJK
+                // "こんにちは世界" = 2 tokens (not 6)
+                // Typically 1-3 CJK chars per token
+                cjkTokenCount = Math.ceil(cjkCharsCount / 2.5);
+            }
+            else {
+                // cl100k_base: CJK chars are typically 1-2 tokens each
+                cjkTokenCount = Math.ceil(cjkCharsCount * 1.2);
+            }
+        }
+        // Handle emoji - typically 2-3 tokens each
+        let emojiCount = 0;
+        let emojiTokenCount = 0;
+        if (containsEmoji(text)) {
+            const emojis = text.match(/\p{Emoji}/gu);
+            if (emojis) {
+                emojiCount = emojis.length;
+                // Each emoji is typically 2-3 tokens
+                emojiTokenCount = Math.ceil(emojiCount * 2.5);
+            }
+        }
+        // Calculate remaining non-special characters
+        const specialCharsCount = numberCharsCount + cjkCharsCount + emojiCount;
+        const remainingChars = codePoints - specialCharsCount;
+        // Base estimate for remaining (mostly ASCII/Latin) text
+        if (remainingChars > 0) {
+            estimate = remainingChars / this.config.averageCharsPerToken;
+            // Whitespace sequences add minimal overhead
+            const whitespaceSeqs = text.match(/\s+/g);
+            if (whitespaceSeqs) {
+                estimate += whitespaceSeqs.length * 0.1;
+            }
+        }
+        // Add special character token counts
+        estimate += numberTokenCount;
+        estimate += cjkTokenCount;
+        estimate += emojiTokenCount;
+        // Non-ASCII penalty for remaining text (excluding CJK which we handled)
+        if (!isAscii(text) && remainingChars > 0) {
+            const nonAsciiRatio = (byteLength - codePoints) / byteLength;
+            // Only apply penalty to the base estimate portion
+            const basePortion = remainingChars / this.config.averageCharsPerToken;
+            estimate += basePortion * nonAsciiRatio * 0.15;
+        }
+        // o200k_base tends to be slightly more efficient overall
+        if (this.config.name === "o200k_base" && remainingChars > 0) {
+            const basePortion = remainingChars / this.config.averageCharsPerToken;
+            estimate -= basePortion * 0.05;
+        }
+        return Math.max(1, Math.round(estimate));
+    }
+}
+//# sourceMappingURL=bpe.js.map

package/dist/bpe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bpe.js","sourceRoot":"","sources":["../src/bpe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EACL,aAAa,EACb,aAAa,EACb,aAAa,EACb,OAAO,EACP,eAAe,EACf,UAAU,GACX,MAAM,YAAY,CAAC;AAEpB;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAkB;IACnD,SAAS,EAAE,eAAe;IAC1B,OAAO,EAAE,cAAc;IACvB,KAAK,EAAE,YAAY;IACnB,KAAK,EAAE,YAAY;IACnB,QAAQ,EAAE,eAAe;IACzB,UAAU,EAAE,iBAAiB;CAC9B,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAmB;IAChD,IAAI,EAAE,aAAa;IACnB,YAAY,EACV,uHAAuH;IACzH,aAAa,EAAE,sBAAsB;IACrC,SAAS,EAAE,MAAM;IACjB,oBAAoB,EAAE,GAAG;CAC1B,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAmB;IAC/C,IAAI,EAAE,YAAY;IAClB,YAAY,EACV,qRAAqR;IACvR,aAAa,EAAE,sBAAsB;IACrC,SAAS,EAAE,MAAM;IACjB,oBAAoB,EAAE,GAAG;CAC1B,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAmB;IAC9C,IAAI,EAAE,WAAW;IACjB,YAAY,EACV,8EAA8E;IAChF,aAAa,EAAE,sBAAsB;IACrC,SAAS,EAAE,KAAK;IAChB,oBAAoB,EAAE,GAAG;CAC1B,CAAC;AAEF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAmB;IACtD,IAAI,EAAE,mBAAmB;IACzB,2EAA2E;IAC3E,YAAY,EACV,oJAAoJ;IACtJ,aAAa,EAAE,sBAAsB;IACrC,kFAAkF;IAClF,SAAS,EAAE,KAAK;IAChB,+DAA+D;IAC/D,oBAAoB,EAAE,GAAG;CAC1B,CAAC;AAEF;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAkB;IAClD,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,aAAa;YAChB,OAAO,kBAAkB,CAAC;QAC5B,KAAK,YAAY;YACf,OAAO,iBAAiB,CAAC;QAC3B,KAAK,WAAW;YACd,OAAO,gBAAgB,CAAC;QAC1B,KAAK,mBAAmB;YACtB,OAAO,wBAAwB,CAAC;QAClC;YACE,OAAO,iBAAiB,CAAC;IAC7B,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AAE7C;;;GAGG;AACH,MAAM,OAAO,YAAY;IACN,MAAM,CAAiB;IACvB,oBAAoB,CAAsB;IAC1C,aAAa,CAAwB;IACrC,kBAAkB,CAAU;IAE7C,YAAY,WAAyB,YAAY;QAC/C,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,oBAAoB,GAAG,IAAI,GAAG,EAAE,CAAC;QACtC,IAAI,CAAC,aAAa,GAAG,IAAI,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC,kBAAkB,GAAG,QAAQ,KAAK,mBAAmB,CAAC;QAE3D,yDAAyD;QACzD,IAAI,CAAC,uBAAuB,EAAE,CAAC;IACjC,CAAC;IAED;;OAEG;IACK,uBAAuB;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;QACzC,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,8BAA8B;QAExE,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QAC3D,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QACzD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACvD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAC5D,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IAC1B,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,IAAY;QACjB,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QAErB,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,iCAAiC;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAEjD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3C,0CAA0C;gBAC1C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,CAAC;YACvD,CAAC;iBAAM,CAAC;gBACN,2BAA2B;gBAC3B,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,MAAgB;QACrB,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,gCAAgC;YAChC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;gBACnD,IAAI,EAAE,KAAK,KAAK,EAAE,CAAC;oBACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACjB,MAAM;gBACR,CAAC;YACH,CAAC;YAED,kEAAkE;YAClE,kDAAkD;YAClD,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,GAAG,CAAC,CAAC;QACjC,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,CAAC;IAED;;;;;;;;;;OAUG;IACH,WAAW,CAAC,IAAY;QACtB,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAEpB,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,gCAAgC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAEjD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3C,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;iBAAM,CAAC;gBACN,KAAK,IAAI,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QAED,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAE5C,gEAAgE;QAChE,0DAA0D;QAC1D,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,wBAAwB,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;OAKG;IACK,oBAAoB,CAAC,IAAY;QACvC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,6CAA6C;QAC7C,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAC;QACvE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,MAAM,CACxB,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EACvF,GAAG,CACJ,CAAC;QAEF,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,KAAK,CAAC;QAEV,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,oCAAoC;YACpC,IAAI,KAAK,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;gBAC5B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YACpD,CAAC;YACD,wBAAwB;YACxB,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACxB,SAAS,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,qBAAqB;QACrB,IAAI,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;OAMG;IACK,UAAU,CAAC,IAAY;QAC7B,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,0CAA0C;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE/D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,oBAAoB;YACpB,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC7C,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;gBACvB,SAAS;YACX,CAAC;YAED,iCAAiC;YACjC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YAEjD,mBAAmB;YACnB,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACvB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;YAC/C,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACK,cAAc,CAAC,KAAiB;QACtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,kEAAkE;QAClE,4DAA4D;QAC5D,+CAA+C;QAC/C,mDAAmD;QAEnD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACxB,wDAAwD;YACxD,MAAM,WAAW,GAAG,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACzD,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;YAEnD,gDAAgD;YAChD,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAErB,CAAC,IAAI,WAAW,CAAC;QACnB,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACK,qBAAqB,CAAC,KAAiB,EAAE,KAAa;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC;QACvC,IAAI,SAAS,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC;QAE7B,iCAAiC;QACjC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAE/B,kDAAkD;QAClD,IAAI,SAAS,IAAI,IAAI,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YAC3C,4BAA4B;YAC5B,IAAI,GAAG,GAAG,KAAK,GAAG,CAAC,CAAC;YAEpB,OAAO,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;gBAC7C,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;gBAErB,qCAAqC;gBACrC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;oBAC3B,MAAM;oBACN,GAAG,EAAE,CAAC;gBACR,CAAC;qBAAM,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;oBAClC,MAAM;oBACN,GAAG,EAAE,CAAC;gBACR,CAAC;qBAAM,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;oBAClC,MAAM;oBACN,GAAG,EAAE,CAAC;gBACR,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,KAAK,EAAE,SAAS,CAAC,CAAC;QAC1C,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC/D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAE/D,OAAO,CAAC,CAAC;IACX,CAAC;IAED;;;;;OAKG;IACK,cAAc,CAAC,KAAiB;QACtC,8CAA8C;QAC9C,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAE9B,6DAA6D;QAC7D,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;;;;;;;;;OAaG;IACK,kBAAkB,CAAC,IAAY;QACrC,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAEpB,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAE9C,gEAAgE;QAChE,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,oEAAoE;QACpE,8DAA8D;QAC9D,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,IAAI,aAAa,EAAE,CAAC;YAClB,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBAChC,gBAAgB,IAAI,GAAG,CAAC,MAAM,CAAC;gBAC/B,+CAA+C;gBAC/C,gBAAgB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;QAED,uDAAuD;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,sEAAsE,CACvE,CAAC;QACF,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,IAAI,QAAQ,EAAE,CAAC;YACb,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACtC,yCAAyC;gBACzC,+BAA+B;gBAC/B,oCAAoC;gBACpC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC;YACjD,CAAC;iBAAM,CAAC;gBACN,uDAAuD;gBACvD,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,2CAA2C;QAC3C,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,eAAe,GAAG,CAAC,CAAC;QAExB,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YACzC,IAAI,MAAM,EAAE,CAAC;gBACX,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;gBAC3B,qCAAqC;gBACrC,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;QAED,6CAA6C;QAC7C,MAAM,iBAAiB,GAAG,gBAAgB,GAAG,aAAa,GAAG,UAAU,CAAC;QACxE,MAAM,cAAc,GAAG,UAAU,GAAG,iBAAiB,CAAC;QAEtD,wDAAwD;QACxD,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACvB,QAAQ,GAAG,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YAE7D,4CAA4C;YAC5C,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,cAAc,EAAE,CAAC;gBACnB,QAAQ,IAAI,cAAc,CAAC,MAAM,GAAG,GAAG,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,qCAAqC;QACrC,QAAQ,IAAI,gBAAgB,CAAC;QAC7B,QAAQ,IAAI,aAAa,CAAC;QAC1B,QAAQ,IAAI,eAAe,CAAC;QAE5B,wEAAwE;QACxE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,aAAa,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,UAAU,CAAC;YAC7D,kDAAkD;YAClD,MAAM,WAAW,GAAG,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YACtE,QAAQ,IAAI,WAAW,GAAG,aAAa,GAAG,IAAI,CAAC;QACjD,CAAC;QAED,yDAAyD;QACzD,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YAC5D,MAAM,WAAW,GAAG,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;YACtE,QAAQ,IAAI,WAAW,GAAG,IAAI,CAAC;QACjC,CAAC;QAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3C,CAAC;CACF"}