@hyvmind/tiktoken-ts 0.0.1

package/dist/core/tiktoken.js

@@ -0,0 +1,295 @@
+/**
+ * Core Tiktoken BPE Implementation
+ *
+ * This is an EXACT port of the CoreBPE struct from tiktoken-rs.
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/vendor_tiktoken.rs
+ *
+ * Provides the main tokenization API:
+ * - encode_ordinary(text) - Encode without special tokens
+ * - encode(text, allowed_special) - Encode with optional special tokens
+ * - encode_with_special_tokens(text) - Encode with all special tokens
+ * - decode(tokens) - Decode tokens to text
+ * - decode_bytes(tokens) - Decode tokens to raw bytes
+ */
+import { bytePairEncode, bytesToKey, keyToBytes, } from "./byte-pair-encoding.js";
+/**
+ * Error thrown when a token cannot be decoded
+ */
+export class DecodeKeyError extends Error {
+    token;
+    constructor(token) {
+        super(`Invalid token for decoding: ${token}`);
+        this.name = "DecodeKeyError";
+        this.token = token;
+    }
+}
+/**
+ * Escape a string for use in a regex
+ */
+function escapeRegex(str) {
+    return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/**
+ * CoreBPE - The main tokenizer class
+ *
+ * This class implements the complete BPE tokenization algorithm,
+ * matching the behavior of tiktoken-rs exactly.
+ */
+export class CoreBPE {
+    /** The vocabulary mapping byte sequences to ranks */
+    encoder;
+    /** The special tokens mapping strings to ranks */
+    specialTokensEncoder;
+    /** Reverse vocabulary mapping ranks to byte sequences */
+    decoder;
+    /** Reverse special tokens mapping ranks to byte sequences */
+    specialTokensDecoder;
+    /** Compiled regex for splitting text */
+    regex;
+    /** Compiled regex for finding special tokens */
+    specialRegex;
+    /** Text encoder for converting strings to bytes */
+    textEncoder;
+    /** Text decoder for converting bytes to strings */
+    textDecoder;
+    /**
+     * Create a new CoreBPE instance
+     *
+     * @param encoder - Vocabulary mapping byte sequences to ranks
+     * @param specialTokensEncoder - Special tokens mapping strings to ranks
+     * @param pattern - Regex pattern for splitting text into pieces
+     */
+    constructor(encoder, specialTokensEncoder, pattern) {
+        this.encoder = encoder;
+        this.specialTokensEncoder = specialTokensEncoder;
+        this.textEncoder = new TextEncoder();
+        this.textDecoder = new TextDecoder("utf-8", { fatal: false });
+        // Build the decoder (reverse of encoder)
+        this.decoder = new Map();
+        for (const [key, rank] of encoder) {
+            this.decoder.set(rank, keyToBytes(key));
+        }
+        // Verify encoder and decoder have same size
+        if (encoder.size !== this.decoder.size) {
+            throw new Error("Encoder and decoder must be of equal length; maybe you had duplicate token indices in your encoder?");
+        }
+        // Build special tokens decoder
+        this.specialTokensDecoder = new Map();
+        for (const [token, rank] of specialTokensEncoder) {
+            this.specialTokensDecoder.set(rank, this.textEncoder.encode(token));
+        }
+        // Compile the regex pattern
+        // Note: JavaScript regex flags differ from Rust - we use 'gu' for global + unicode
+        this.regex = new RegExp(pattern, "gu");
+        // Build special tokens regex
+        if (specialTokensEncoder.size > 0) {
+            const escapedTokens = Array.from(specialTokensEncoder.keys())
+                .map(escapeRegex)
+                .join("|");
+            this.specialRegex = new RegExp(escapedTokens, "g");
+        }
+        else {
+            this.specialRegex = null;
+        }
+    }
+    /**
+     * Decode tokens to raw bytes
+     *
+     * The bytes are not guaranteed to be valid UTF-8.
+     *
+     * @param tokens - Array of token ranks to decode
+     * @returns Decoded bytes
+     * @throws DecodeKeyError if a token is not found
+     */
+    decodeBytes(tokens) {
+        const parts = [];
+        let totalLength = 0;
+        for (const token of tokens) {
+            let tokenBytes = this.decoder.get(token);
+            if (tokenBytes === undefined) {
+                tokenBytes = this.specialTokensDecoder.get(token);
+            }
+            if (tokenBytes === undefined) {
+                throw new DecodeKeyError(token);
+            }
+            parts.push(tokenBytes);
+            totalLength += tokenBytes.length;
+        }
+        // Concatenate all parts
+        const result = new Uint8Array(totalLength);
+        let offset = 0;
+        for (const part of parts) {
+            result.set(part, offset);
+            offset += part.length;
+        }
+        return result;
+    }
+    /**
+     * Decode tokens to a string
+     *
+     * @param tokens - Array of token ranks to decode
+     * @returns Decoded string
+     * @throws DecodeKeyError if a token is not found
+     */
+    decode(tokens) {
+        const bytes = this.decodeBytes(tokens);
+        return this.textDecoder.decode(bytes);
+    }
+    /**
+     * Encode text without handling special tokens
+     *
+     * This is the core encoding logic. Special tokens are treated as regular text.
+     *
+     * @param text - Text to encode
+     * @returns Array of token ranks
+     */
+    encodeOrdinary(text) {
+        const tokens = [];
+        // Reset regex state
+        this.regex.lastIndex = 0;
+        let match;
+        while ((match = this.regex.exec(text)) !== null) {
+            const piece = match[0];
+            const pieceBytes = this.textEncoder.encode(piece);
+            const key = bytesToKey(pieceBytes);
+            // Check if the piece is a single token
+            const directRank = this.encoder.get(key);
+            if (directRank !== undefined) {
+                tokens.push(directRank);
+            }
+            else {
+                // Apply BPE
+                const bpeTokens = bytePairEncode(pieceBytes, this.encoder);
+                tokens.push(...bpeTokens);
+            }
+        }
+        return tokens;
+    }
+    /**
+     * Encode text with special token handling
+     *
+     * @param text - Text to encode
+     * @param allowedSpecial - Set of special tokens that are allowed
+     * @returns Tuple of [tokens, lastPieceTokenLen]
+     */
+    encode(text, allowedSpecial) {
+        if (!this.specialRegex || allowedSpecial.size === 0) {
+            // No special tokens to handle
+            return [this.encodeOrdinary(text), 0];
+        }
+        const tokens = [];
+        let start = 0;
+        let lastPieceTokenLen = 0;
+        while (start < text.length) {
+            // Find the next special token
+            let nextSpecial = null;
+            let searchStart = start;
+            while (searchStart < text.length) {
+                this.specialRegex.lastIndex = searchStart;
+                const match = this.specialRegex.exec(text);
+                if (!match) {
+                    break;
+                }
+                const matchedToken = match[0];
+                if (allowedSpecial.has(matchedToken)) {
+                    nextSpecial = { index: match.index, token: matchedToken };
+                    break;
+                }
+                // Token not allowed, continue searching
+                searchStart = match.index + 1;
+            }
+            const end = nextSpecial ? nextSpecial.index : text.length;
+            // Encode the text before the special token
+            if (end > start) {
+                const segment = text.slice(start, end);
+                // Reset regex state
+                this.regex.lastIndex = 0;
+                let match;
+                while ((match = this.regex.exec(segment)) !== null) {
+                    const piece = match[0];
+                    const pieceBytes = this.textEncoder.encode(piece);
+                    const key = bytesToKey(pieceBytes);
+                    const directRank = this.encoder.get(key);
+                    if (directRank !== undefined) {
+                        lastPieceTokenLen = 1;
+                        tokens.push(directRank);
+                    }
+                    else {
+                        const bpeTokens = bytePairEncode(pieceBytes, this.encoder);
+                        lastPieceTokenLen = bpeTokens.length;
+                        tokens.push(...bpeTokens);
+                    }
+                }
+            }
+            // Handle the special token
+            if (nextSpecial) {
+                const specialRank = this.specialTokensEncoder.get(nextSpecial.token);
+                if (specialRank !== undefined) {
+                    tokens.push(specialRank);
+                    lastPieceTokenLen = 0;
+                }
+                start = nextSpecial.index + nextSpecial.token.length;
+            }
+            else {
+                break;
+            }
+        }
+        return [tokens, lastPieceTokenLen];
+    }
+    /**
+     * Encode text with all special tokens allowed
+     *
+     * @param text - Text to encode
+     * @returns Array of token ranks
+     */
+    encodeWithSpecialTokens(text) {
+        const allowedSpecial = this.getSpecialTokens();
+        return this.encode(text, allowedSpecial)[0];
+    }
+    /**
+     * Get all special tokens
+     *
+     * @returns Set of special token strings
+     */
+    getSpecialTokens() {
+        return new Set(this.specialTokensEncoder.keys());
+    }
+    /**
+     * Get the vocabulary size (excluding special tokens)
+     */
+    get vocabSize() {
+        return this.encoder.size;
+    }
+    /**
+     * Get the total vocabulary size (including special tokens)
+     */
+    get totalVocabSize() {
+        return this.encoder.size + this.specialTokensEncoder.size;
+    }
+    /**
+     * Check if a token rank is a special token
+     */
+    isSpecialToken(token) {
+        return this.specialTokensDecoder.has(token);
+    }
+    /**
+     * Get the byte representation of a token
+     */
+    getTokenBytes(token) {
+        return this.decoder.get(token) ?? this.specialTokensDecoder.get(token);
+    }
+    /**
+     * Get the rank of a byte sequence
+     */
+    getRank(bytes) {
+        const key = bytesToKey(bytes);
+        return this.encoder.get(key);
+    }
+    /**
+     * Get the rank of a special token
+     */
+    getSpecialTokenRank(token) {
+        return this.specialTokensEncoder.get(token);
+    }
+}
+//# sourceMappingURL=tiktoken.js.map

package/dist/core/tiktoken.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tiktoken.js","sourceRoot":"","sources":["../../src/core/tiktoken.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAIL,cAAc,EACd,UAAU,EACV,UAAU,GACX,MAAM,yBAAyB,CAAC;AAEjC;;GAEG;AACH,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvB,KAAK,CAAO;IAE5B,YAAY,KAAW;QACrB,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;CACF;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO,GAAG,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,OAAO;IAClB,qDAAqD;IACpC,OAAO,CAAa;IAErC,kDAAkD;IACjC,oBAAoB,CAAoB;IAEzD,yDAAyD;IACxC,OAAO,CAAoB;IAE5C,6DAA6D;IAC5C,oBAAoB,CAAwB;IAE7D,wCAAwC;IACvB,KAAK,CAAS;IAE/B,gDAAgD;IAC/B,YAAY,CAAgB;IAE7C,mDAAmD;IAClC,WAAW,CAAmC;IAE/D,mDAAmD;IAClC,WAAW,CAAmC;IAE/D;;;;;;OAMG;IACH,YACE,OAAmB,EACnB,oBAAuC,EACvC,OAAe;QAEf,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAE9D,yCAAyC;QACzC,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1C,CAAC;QAED,4CAA4C;QAC5C,IAAI,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CACb,qGAAqG,CACtG,CAAC;QACJ,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC,oBAAoB,GAAG,IAAI,GAAG,EAAE,CAAC;QACtC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,oBAAoB,EAAE,CAAC;YACjD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACtE,CAAC;QAED,4BAA4B;QAC5B,mFAAmF;QACnF,IAAI,CAAC,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAEvC,6BAA6B;QAC7B,IAAI,oBAAoB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;iBAC1D,GAAG,CAAC,WAAW,CAAC;iBAChB,IAAI,CAAC,GAAG,CAAC,CAAC;YACb,IAAI,CAAC,YAAY,GAAG,IAAI,MAAM,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;QACrD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,WAAW,CAAC,MAAc;QACxB,MAAM,KAAK,GAAiB,EAAE,CAAC;QAC/B,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACzC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,UAAU,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACpD,CAAC;YACD,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,cAAc,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACvB,WAAW,IAAI,UAAU,CAAC,MAAM,CAAC;QACnC,CAAC;QAED,wBAAwB;QACxB,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACzB,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;QACxB,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,MAAc;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;IAED;;;;;;;OAOG;IACH,cAAc,CAAC,IAAY;QACzB,MAAM,MAAM,GAAW,EAAE,CAAC;QAE1B,oBAAoB;QACpB,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;QAEzB,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClD,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;YAEnC,uCAAuC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACzC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,YAAY;gBACZ,MAAM,SAAS,GAAG,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,IAAY,EAAE,cAA2B;QAC9C,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACpD,8BAA8B;YAC9B,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAE1B,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3B,8BAA8B;YAC9B,IAAI,WAAW,GAA4C,IAAI,CAAC;YAChE,IAAI,WAAW,GAAG,KAAK,CAAC;YAExB,OAAO,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjC,IAAI,CAAC,YAAY,CAAC,SAAS,GAAG,WAAW,CAAC;gBAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAE3C,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,MAAM;gBACR,CAAC;gBAED,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC9B,IAAI,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;oBACrC,WAAW,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;oBAC1D,MAAM;gBACR,CAAC;gBAED,wCAAwC;gBACxC,WAAW,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;YAChC,CAAC;YAED,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;YAE1D,2CAA2C;YAC3C,IAAI,GAAG,GAAG,KAAK,EAAE,CAAC;gBAChB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBAEvC,oBAAoB;gBACpB,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;gBAEzB,IAAI,KAAK,CAAC;gBACV,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;oBACnD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;oBACvB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAClD,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;oBAEnC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACzC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;wBAC7B,iBAAiB,GAAG,CAAC,CAAC;wBACtB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;oBAC1B,CAAC;yBAAM,CAAC;wBACN,MAAM,SAAS,GAAG,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;wBAC3D,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC;wBACrC,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;oBAC5B,CAAC;gBACH,CAAC;YACH,CAAC;YAED,2BAA2B;YAC3B,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,WAAW,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACrE,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;oBAC9B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBACzB,iBAAiB,GAAG,CAAC,CAAC;gBACxB,CAAC;gBACD,KAAK,GAAG,WAAW,CAAC,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC;YACvD,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;IACrC,CAAC;IAED;;;;;OAKG;IACH,uBAAuB,CAAC,IAAY;QAClC,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED;;;;OAIG;IACH,gBAAgB;QACd,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,KAAW;QACxB,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACzE,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,KAAiB;QACvB,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,KAAa;QAC/B,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;CACF"}

package/dist/core/vocab-loader.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Vocabulary Loader
+ *
+ * Handles loading and parsing of .tiktoken vocabulary files.
+ * These files contain base64-encoded tokens with their ranks.
+ *
+ * Format: Each line is "base64_token rank"
+ * Example: "SGVsbG8= 12345" means "Hello" has rank 12345
+ */
+import { type Vocabulary, type ReverseVocabulary, type Rank, keyToBytes } from "./byte-pair-encoding.js";
+export { keyToBytes };
+/**
+ * Base64 encode (works in both Node.js and browser)
+ */
+export declare function base64Encode(bytes: Uint8Array): string;
+/**
+ * Parse a .tiktoken vocabulary file content
+ *
+ * @param content - The raw content of the .tiktoken file
+ * @returns Vocabulary map and reverse vocabulary map
+ */
+export declare function parseVocabulary(content: string): {
+    encoder: Vocabulary;
+    decoder: ReverseVocabulary;
+};
+/**
+ * Vocabulary URLs for each encoding
+ */
+export declare const VOCABULARY_URLS: Record<string, string>;
+/**
+ * Load vocabulary from URL (with caching)
+ *
+ * @param encodingName - Name of the encoding
+ * @returns Promise resolving to vocabulary maps
+ */
+export declare function loadVocabularyFromUrl(encodingName: string): Promise<{
+    encoder: Vocabulary;
+    decoder: ReverseVocabulary;
+}>;
+/**
+ * Load vocabulary from a string (for embedded vocabularies)
+ *
+ * @param encodingName - Name of the encoding (for caching)
+ * @param content - The vocabulary file content
+ * @returns Vocabulary maps
+ */
+export declare function loadVocabularyFromString(encodingName: string, content: string): {
+    encoder: Vocabulary;
+    decoder: ReverseVocabulary;
+};
+/**
+ * Create special tokens map
+ *
+ * @param specialTokens - Map of special token strings to ranks
+ * @returns Maps for encoding and decoding special tokens
+ */
+export declare function createSpecialTokenMaps(specialTokens: Record<string, Rank>): {
+    encoder: Map<string, Rank>;
+    decoder: Map<Rank, Uint8Array>;
+};
+/**
+ * Clear the vocabulary cache
+ * Useful for testing or memory management
+ */
+export declare function clearVocabularyCache(): void;
+/**
+ * Get vocabulary from cache (sync, returns undefined if not loaded)
+ */
+export declare function getVocabularyFromCache(encodingName: string): {
+    encoder: Vocabulary;
+    decoder: ReverseVocabulary;
+} | undefined;
+/**
+ * Check if a vocabulary is cached
+ */
+export declare function isVocabularyCached(encodingName: string): boolean;
+//# sourceMappingURL=vocab-loader.d.ts.map

package/dist/core/vocab-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vocab-loader.d.ts","sourceRoot":"","sources":["../../src/core/vocab-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,KAAK,UAAU,EACf,KAAK,iBAAiB,EACtB,KAAK,IAAI,EAET,UAAU,EACX,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EAAE,UAAU,EAAE,CAAC;AAsBtB;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAWtD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG;IAChD,OAAO,EAAE,UAAU,CAAC;IACpB,OAAO,EAAE,iBAAiB,CAAC;CAC5B,CAgCA;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAalD,CAAC;AAUF;;;;;GAKG;AACH,wBAAsB,qBAAqB,CACzC,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC;IAAE,OAAO,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,CAAC,CA2B9D;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,GACd;IAAE,OAAO,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,CAarD;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG;IAC3E,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC3B,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;CAChC,CAYA;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,IAAI,IAAI,CAE3C;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CACpC,YAAY,EAAE,MAAM,GACnB;IAAE,OAAO,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,iBAAiB,CAAA;CAAE,GAAG,SAAS,CAEjE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAEhE"}

package/dist/core/vocab-loader.js

@@ -0,0 +1,176 @@
+/**
+ * Vocabulary Loader
+ *
+ * Handles loading and parsing of .tiktoken vocabulary files.
+ * These files contain base64-encoded tokens with their ranks.
+ *
+ * Format: Each line is "base64_token rank"
+ * Example: "SGVsbG8= 12345" means "Hello" has rank 12345
+ */
+import { bytesToKey, keyToBytes, } from "./byte-pair-encoding.js";
+// Re-export for convenience
+export { keyToBytes };
+/**
+ * Base64 decode (works in both Node.js and browser)
+ */
+function base64Decode(base64) {
+    // Use built-in atob for browser compatibility
+    // Node.js 18+ also supports atob globally
+    if (typeof atob === "function") {
+        const binaryString = atob(base64);
+        const bytes = new Uint8Array(binaryString.length);
+        for (let i = 0; i < binaryString.length; i++) {
+            bytes[i] = binaryString.charCodeAt(i);
+        }
+        return bytes;
+    }
+    // Fallback for older Node.js (should not be needed with Node 18+)
+    const buffer = Buffer.from(base64, "base64");
+    return new Uint8Array(buffer);
+}
+/**
+ * Base64 encode (works in both Node.js and browser)
+ */
+export function base64Encode(bytes) {
+    if (typeof btoa === "function") {
+        let binaryString = "";
+        for (let i = 0; i < bytes.length; i++) {
+            binaryString += String.fromCharCode(bytes[i]);
+        }
+        return btoa(binaryString);
+    }
+    // Fallback for older Node.js
+    return Buffer.from(bytes).toString("base64");
+}
+/**
+ * Parse a .tiktoken vocabulary file content
+ *
+ * @param content - The raw content of the .tiktoken file
+ * @returns Vocabulary map and reverse vocabulary map
+ */
+export function parseVocabulary(content) {
+    const encoder = new Map();
+    const decoder = new Map();
+    const lines = content.split("\n");
+    for (const line of lines) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        const spaceIndex = trimmed.indexOf(" ");
+        if (spaceIndex === -1)
+            continue;
+        const base64Token = trimmed.slice(0, spaceIndex);
+        const rankStr = trimmed.slice(spaceIndex + 1);
+        try {
+            const tokenBytes = base64Decode(base64Token);
+            const rank = parseInt(rankStr, 10);
+            if (!isNaN(rank)) {
+                const key = bytesToKey(tokenBytes);
+                encoder.set(key, rank);
+                decoder.set(rank, tokenBytes);
+            }
+        }
+        catch {
+            // Skip malformed lines
+            continue;
+        }
+    }
+    return { encoder, decoder };
+}
+/**
+ * Vocabulary URLs for each encoding
+ */
+export const VOCABULARY_URLS = {
+    r50k_base: "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken",
+    p50k_base: "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
+    p50k_edit: "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken", // Uses same vocab
+    cl100k_base: "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
+    o200k_base: "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
+    o200k_harmony: "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken", // Uses same vocab
+};
+/**
+ * Cache for loaded vocabularies
+ */
+const vocabularyCache = new Map();
+/**
+ * Load vocabulary from URL (with caching)
+ *
+ * @param encodingName - Name of the encoding
+ * @returns Promise resolving to vocabulary maps
+ */
+export async function loadVocabularyFromUrl(encodingName) {
+    // Check cache first
+    const cached = vocabularyCache.get(encodingName);
+    if (cached) {
+        return cached;
+    }
+    const url = VOCABULARY_URLS[encodingName];
+    if (!url) {
+        throw new Error(`Unknown encoding: ${encodingName}`);
+    }
+    // Fetch the vocabulary file
+    const response = await fetch(url);
+    if (!response.ok) {
+        throw new Error(`Failed to fetch vocabulary for ${encodingName}: ${response.statusText}`);
+    }
+    const content = await response.text();
+    const vocab = parseVocabulary(content);
+    // Cache the result
+    vocabularyCache.set(encodingName, vocab);
+    return vocab;
+}
+/**
+ * Load vocabulary from a string (for embedded vocabularies)
+ *
+ * @param encodingName - Name of the encoding (for caching)
+ * @param content - The vocabulary file content
+ * @returns Vocabulary maps
+ */
+export function loadVocabularyFromString(encodingName, content) {
+    // Check cache first
+    const cached = vocabularyCache.get(encodingName);
+    if (cached) {
+        return cached;
+    }
+    const vocab = parseVocabulary(content);
+    // Cache the result
+    vocabularyCache.set(encodingName, vocab);
+    return vocab;
+}
+/**
+ * Create special tokens map
+ *
+ * @param specialTokens - Map of special token strings to ranks
+ * @returns Maps for encoding and decoding special tokens
+ */
+export function createSpecialTokenMaps(specialTokens) {
+    const encoder = new Map();
+    const decoder = new Map();
+    for (const [token, rank] of Object.entries(specialTokens)) {
+        encoder.set(token, rank);
+        // Convert string to bytes for decoder
+        const textEncoder = new TextEncoder();
+        decoder.set(rank, textEncoder.encode(token));
+    }
+    return { encoder, decoder };
+}
+/**
+ * Clear the vocabulary cache
+ * Useful for testing or memory management
+ */
+export function clearVocabularyCache() {
+    vocabularyCache.clear();
+}
+/**
+ * Get vocabulary from cache (sync, returns undefined if not loaded)
+ */
+export function getVocabularyFromCache(encodingName) {
+    return vocabularyCache.get(encodingName);
+}
+/**
+ * Check if a vocabulary is cached
+ */
+export function isVocabularyCached(encodingName) {
+    return vocabularyCache.has(encodingName);
+}
+//# sourceMappingURL=vocab-loader.js.map

package/dist/core/vocab-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vocab-loader.js","sourceRoot":"","sources":["../../src/core/vocab-loader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAIL,UAAU,EACV,UAAU,GACX,MAAM,yBAAyB,CAAC;AAEjC,4BAA4B;AAC5B,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB;;GAEG;AACH,SAAS,YAAY,CAAC,MAAc;IAClC,8CAA8C;IAC9C,0CAA0C;IAC1C,IAAI,OAAO,IAAI,KAAK,UAAU,EAAE,CAAC;QAC/B,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,KAAK,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,kEAAkE;IAClE,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7C,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,KAAiB;IAC5C,IAAI,OAAO,IAAI,KAAK,UAAU,EAAE,CAAC;QAC/B,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,6BAA6B;IAC7B,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAI7C,MAAM,OAAO,GAAe,IAAI,GAAG,EAAE,CAAC;IACtC,MAAM,OAAO,GAAsB,IAAI,GAAG,EAAE,CAAC;IAE7C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,UAAU,KAAK,CAAC,CAAC;YAAE,SAAS;QAEhC,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QAE9C,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;YAC7C,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAEnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;gBACnC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;YACvB,SAAS;QACX,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAA2B;IACrD,SAAS,EACP,yEAAyE;IAC3E,SAAS,EACP,yEAAyE;IAC3E,SAAS,EACP,yEAAyE,EAAE,kBAAkB;IAC/F,WAAW,EACT,2EAA2E;IAC7E,UAAU,EACR,0EAA0E;IAC5E,aAAa,EACX,0EAA0E,EAAE,kBAAkB;CACjG,CAAC;AAEF;;GAEG;AACH,MAAM,eAAe,GAGjB,IAAI,GAAG,EAAE,CAAC;AAEd;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,YAAoB;IAEpB,oBAAoB;IACpB,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACjD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,GAAG,GAAG,eAAe,CAAC,YAAY,CAAC,CAAC;IAC1C,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,qBAAqB,YAAY,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,4BAA4B;IAC5B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CACb,kCAAkC,YAAY,KAAK,QAAQ,CAAC,UAAU,EAAE,CACzE,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAEvC,mBAAmB;IACnB,eAAe,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAEzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CACtC,YAAoB,EACpB,OAAe;IAEf,oBAAoB;IACpB,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACjD,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAEvC,mBAAmB;IACnB,eAAe,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IAEzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,aAAmC;IAIxE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAgB,CAAC;IACxC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IAE5C,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACzB,sCAAsC;QACtC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB;IAClC,eAAe,CAAC,KAAK,EAAE,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CACpC,YAAoB;IAEpB,OAAO,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,YAAoB;IACrD,OAAO,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;AAC3C,CAAC"}

package/dist/encodings/cl100k-base.d.ts

@@ -0,0 +1,43 @@
+/**
+ * cl100k_base Encoding
+ * Used by GPT-4, GPT-3.5-turbo, text-embedding-ada-002
+ *
+ * This encoding has:
+ * - 100,256 tokens
+ * - Better handling of code and programming languages
+ * - Improved whitespace handling
+ */
+import type { Tokenizer, EncodingName } from "../types.js";
+/**
+ * Encoding name constant
+ */
+export declare const ENCODING_NAME: EncodingName;
+/**
+ * Create a cl100k_base tokenizer instance
+ *
+ * @returns Tokenizer instance
+ */
+export declare function createCL100kTokenizer(): Tokenizer;
+/**
+ * Pre-computed token counts for common patterns
+ * Used to improve estimation accuracy
+ */
+export declare const COMMON_TOKEN_PATTERNS: Record<string, number>;
+/**
+ * Average characters per token for cl100k_base
+ * Based on empirical analysis of English text
+ */
+export declare const AVERAGE_CHARS_PER_TOKEN = 3.8;
+/**
+ * Token estimation adjustments for different content types
+ */
+export declare const CONTENT_TYPE_MULTIPLIERS: Record<string, number>;
+/**
+ * Get estimated token count with content-type awareness
+ *
+ * @param text - Input text
+ * @param contentType - Type of content
+ * @returns Estimated token count
+ */
+export declare function estimateTokensForContent(text: string, contentType?: keyof typeof CONTENT_TYPE_MULTIPLIERS): number;
+//# sourceMappingURL=cl100k-base.d.ts.map

package/dist/encodings/cl100k-base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cl100k-base.d.ts","sourceRoot":"","sources":["../../src/encodings/cl100k-base.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3D;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,YAA4B,CAAC;AAEzD;;;;GAIG;AACH,wBAAgB,qBAAqB,IAAI,SAAS,CASjD;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAmFxD,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAO3D,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,WAAW,GAAE,MAAM,OAAO,wBAAkC,GAC3D,MAAM,CAMR"}