@hyvmind/tiktoken-ts 0.0.1

package/dist/core/byte-pair-encoding.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Byte-Pair Encoding Core Algorithm
+ *
+ * This is an EXACT port of the BPE algorithm from tiktoken-rs.
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/vendor_tiktoken.rs
+ *
+ * The algorithm works as follows:
+ * 1. Split input text into pieces using a regex pattern
+ * 2. For each piece, convert to bytes and apply BPE merges
+ * 3. Look up each merged byte sequence in the vocabulary to get token IDs
+ */
+/**
+ * Token rank type (same as tiktoken-rs)
+ */
+export type Rank = number;
+/**
+ * Vocabulary type: maps byte sequences to ranks
+ */
+export type Vocabulary = Map<string, Rank>;
+/**
+ * Reverse vocabulary: maps ranks to byte sequences
+ */
+export type ReverseVocabulary = Map<Rank, Uint8Array>;
+/**
+ * Convert a byte array to a string key for Map lookup
+ * This is necessary because JavaScript Maps use reference equality for arrays
+ */
+export declare function bytesToKey(bytes: Uint8Array): string;
+/**
+ * Convert a string key back to bytes
+ */
+export declare function keyToBytes(key: string): Uint8Array;
+/**
+ * Encode a byte sequence using BPE
+ *
+ * @param piece - The byte sequence to encode
+ * @param ranks - The vocabulary mapping byte sequences to ranks
+ * @returns Array of token ranks
+ */
+export declare function bytePairEncode(piece: Uint8Array, ranks: Vocabulary): Rank[];
+/**
+ * Split a byte sequence into BPE tokens (returns byte slices, not ranks)
+ *
+ * @param piece - The byte sequence to split
+ * @param ranks - The vocabulary mapping byte sequences to ranks
+ * @returns Array of byte slices
+ */
+export declare function bytePairSplit(piece: Uint8Array, ranks: Vocabulary): Uint8Array[];
+//# sourceMappingURL=byte-pair-encoding.d.ts.map

package/dist/core/byte-pair-encoding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"byte-pair-encoding.d.ts","sourceRoot":"","sources":["../../src/core/byte-pair-encoding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH;;GAEG;AACH,MAAM,MAAM,IAAI,GAAG,MAAM,CAAC;AAE1B;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAE3C;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;AAEtD;;;GAGG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAQpD;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,CAMlD;AA8ED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,IAAI,EAAE,CA2B3E;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,UAAU,GAChB,UAAU,EAAE,CAed"}

package/dist/core/byte-pair-encoding.js

@@ -0,0 +1,154 @@
+/**
+ * Byte-Pair Encoding Core Algorithm
+ *
+ * This is an EXACT port of the BPE algorithm from tiktoken-rs.
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/vendor_tiktoken.rs
+ *
+ * The algorithm works as follows:
+ * 1. Split input text into pieces using a regex pattern
+ * 2. For each piece, convert to bytes and apply BPE merges
+ * 3. Look up each merged byte sequence in the vocabulary to get token IDs
+ */
+/**
+ * Convert a byte array to a string key for Map lookup
+ * This is necessary because JavaScript Maps use reference equality for arrays
+ */
+export function bytesToKey(bytes) {
+    // Use a more efficient encoding for the key
+    // Each byte becomes a character in the string
+    let key = "";
+    for (let i = 0; i < bytes.length; i++) {
+        key += String.fromCharCode(bytes[i]);
+    }
+    return key;
+}
+/**
+ * Convert a string key back to bytes
+ */
+export function keyToBytes(key) {
+    const bytes = new Uint8Array(key.length);
+    for (let i = 0; i < key.length; i++) {
+        bytes[i] = key.charCodeAt(i);
+    }
+    return bytes;
+}
+/**
+ * Internal BPE merge algorithm
+ *
+ * This is the core of the BPE algorithm. It finds the lowest-ranked pair
+ * of adjacent bytes and merges them, repeating until no more merges are possible.
+ *
+ * @param ranks - The vocabulary mapping byte sequences to ranks
+ * @param piece - The byte sequence to process
+ * @returns Array of (start_index, rank) tuples representing the merge points
+ */
+function _bytePairMerge(ranks, piece) {
+    // This is a vector of (start, rank).
+    // The rank is of the pair starting at position start.
+    const parts = [];
+    // Note that we hash bytes when indexing into `ranks`, not token pairs. As long as we train BPE
+    // the way we currently do, this is equivalent. An easy way to break this would be to decouple
+    // merge priority from token index or to prevent specific token merges.
+    let minRank = [
+        Number.MAX_SAFE_INTEGER,
+        Number.MAX_SAFE_INTEGER,
+    ];
+    for (let i = 0; i < piece.length - 1; i++) {
+        const pairKey = bytesToKey(piece.slice(i, i + 2));
+        const rank = ranks.get(pairKey) ?? Number.MAX_SAFE_INTEGER;
+        if (rank < minRank[0]) {
+            minRank = [rank, i];
+        }
+        parts.push([i, rank]);
+    }
+    parts.push([piece.length - 1, Number.MAX_SAFE_INTEGER]);
+    parts.push([piece.length, Number.MAX_SAFE_INTEGER]);
+    const getRank = (parts, i) => {
+        if (i + 3 < parts.length) {
+            // Similar to `piece[i..i + 2]` above. The +3 is because we haven't yet deleted
+            // parts[i + 1], see comment in the main loop.
+            const start = parts[i][0];
+            const end = parts[i + 3][0];
+            const pairKey = bytesToKey(piece.slice(start, end));
+            return ranks.get(pairKey) ?? Number.MAX_SAFE_INTEGER;
+        }
+        else {
+            return Number.MAX_SAFE_INTEGER;
+        }
+    };
+    // If you have n parts and m merges, this does O(mn) work.
+    // We could do something with a heap and do O(m log n) work.
+    // n is often very small so considerations like cache-locality outweigh the algorithmic
+    // complexity downsides of the `parts` vector.
+    while (minRank[0] !== Number.MAX_SAFE_INTEGER) {
+        const i = minRank[1];
+        // Update parts[i] and parts[i - 1] before removing parts[i + 1], since
+        // `parts.splice(i + 1, 1)` will thrash the cache.
+        if (i > 0) {
+            parts[i - 1][1] = getRank(parts, i - 1);
+        }
+        parts[i][1] = getRank(parts, i);
+        parts.splice(i + 1, 1);
+        minRank = [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER];
+        for (let j = 0; j < parts.length - 1; j++) {
+            const rank = parts[j][1];
+            if (rank < minRank[0]) {
+                minRank = [rank, j];
+            }
+        }
+    }
+    return parts;
+}
+/**
+ * Encode a byte sequence using BPE
+ *
+ * @param piece - The byte sequence to encode
+ * @param ranks - The vocabulary mapping byte sequences to ranks
+ * @returns Array of token ranks
+ */
+export function bytePairEncode(piece, ranks) {
+    if (piece.length === 1) {
+        const key = bytesToKey(piece);
+        const rank = ranks.get(key);
+        if (rank === undefined) {
+            throw new Error(`Unknown byte: ${piece[0]}`);
+        }
+        return [rank];
+    }
+    const parts = _bytePairMerge(ranks, piece);
+    // Convert parts to token ranks
+    const tokens = [];
+    for (let i = 0; i < parts.length - 1; i++) {
+        const start = parts[i][0];
+        const end = parts[i + 1][0];
+        const tokenBytes = piece.slice(start, end);
+        const key = bytesToKey(tokenBytes);
+        const rank = ranks.get(key);
+        if (rank === undefined) {
+            throw new Error(`Unknown token: ${Array.from(tokenBytes).join(",")}`);
+        }
+        tokens.push(rank);
+    }
+    return tokens;
+}
+/**
+ * Split a byte sequence into BPE tokens (returns byte slices, not ranks)
+ *
+ * @param piece - The byte sequence to split
+ * @param ranks - The vocabulary mapping byte sequences to ranks
+ * @returns Array of byte slices
+ */
+export function bytePairSplit(piece, ranks) {
+    if (piece.length <= 1) {
+        return [piece];
+    }
+    const parts = _bytePairMerge(ranks, piece);
+    const splits = [];
+    for (let i = 0; i < parts.length - 1; i++) {
+        const start = parts[i][0];
+        const end = parts[i + 1][0];
+        splits.push(piece.slice(start, end));
+    }
+    return splits;
+}
+//# sourceMappingURL=byte-pair-encoding.js.map

package/dist/core/byte-pair-encoding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"byte-pair-encoding.js","sourceRoot":"","sources":["../../src/core/byte-pair-encoding.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAiBH;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,KAAiB;IAC1C,4CAA4C;IAC5C,8CAA8C;IAC9C,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,GAAG,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,cAAc,CACrB,KAAiB,EACjB,KAAiB;IAEjB,qCAAqC;IACrC,sDAAsD;IACtD,MAAM,KAAK,GAA0B,EAAE,CAAC;IAExC,+FAA+F;IAC/F,8FAA8F;IAC9F,uEAAuE;IACvE,IAAI,OAAO,GAAmB;QAC5B,MAAM,CAAC,gBAAgB;QACvB,MAAM,CAAC,gBAAgB;KACxB,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,gBAAgB,CAAC;QAC3D,IAAI,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACtB,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACtB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IACxB,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACxD,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAEpD,MAAM,OAAO,GAAG,CAAC,KAA4B,EAAE,CAAS,EAAQ,EAAE;QAChE,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACzB,+EAA+E;YAC/E,8CAA8C;YAC9C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,gBAAgB,CAAC;QACvD,CAAC;aAAM,CAAC;YACN,OAAO,MAAM,CAAC,gBAAgB,CAAC;QACjC,CAAC;IACH,CAAC,CAAC;IAEF,0DAA0D;IAC1D,4DAA4D;IAC5D,uFAAuF;IACvF,8CAA8C;IAC9C,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAC9C,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,uEAAuE;QACvE,kDAAkD;QAClD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACV,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAEvB,OAAO,GAAG,CAAC,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtB,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,KAAiB,EAAE,KAAiB;IACjE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAE3C,+BAA+B;IAC/B,MAAM,MAAM,GAAW,EAAE,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAC3B,KAAiB,EACjB,KAAiB;IAEjB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,KAAK,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/core/encoding-definitions.d.ts

@@ -0,0 +1,95 @@
+/**
+ * Encoding Definitions
+ *
+ * EXACT definitions from tiktoken-rs for all supported encodings.
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/tiktoken_ext/openai_public.rs
+ */
+import type { Rank } from "./byte-pair-encoding.js";
+/**
+ * Special token constants
+ */
+export declare const SPECIAL_TOKENS: {
+    readonly ENDOFTEXT: "<|endoftext|>";
+    readonly FIM_PREFIX: "<|fim_prefix|>";
+    readonly FIM_MIDDLE: "<|fim_middle|>";
+    readonly FIM_SUFFIX: "<|fim_suffix|>";
+    readonly ENDOFPROMPT: "<|endofprompt|>";
+};
+/**
+ * Encoding definition interface
+ */
+export interface EncodingDefinition {
+    /** Encoding name */
+    name: string;
+    /**
+     * Regex pattern for splitting text into pieces
+     * Note: JavaScript regex patterns differ slightly from Rust
+     */
+    pattern: string;
+    /** Special tokens with their ranks */
+    specialTokens: Record<string, Rank>;
+    /** URL to download the vocabulary file */
+    vocabularyUrl: string;
+    /** Expected vocabulary size (for validation) */
+    expectedVocabSize: number;
+}
+/**
+ * r50k_base encoding (GPT-2)
+ *
+ * Used by: GPT-3 davinci, curie, babbage, ada models
+ * Vocabulary size: 50,257 tokens
+ */
+export declare const R50K_BASE: EncodingDefinition;
+/**
+ * p50k_base encoding
+ *
+ * Used by: Code models, text-davinci-002, text-davinci-003
+ * Vocabulary size: 50,281 tokens
+ */
+export declare const P50K_BASE: EncodingDefinition;
+/**
+ * p50k_edit encoding
+ *
+ * Used by: Edit models like text-davinci-edit-001, code-davinci-edit-001
+ * Same vocabulary as p50k_base but with FIM special tokens
+ */
+export declare const P50K_EDIT: EncodingDefinition;
+/**
+ * cl100k_base encoding
+ *
+ * Used by: ChatGPT models, GPT-4, GPT-3.5-turbo, text-embedding-ada-002
+ * Vocabulary size: 100,256 tokens
+ */
+export declare const CL100K_BASE: EncodingDefinition;
+/**
+ * o200k_base encoding
+ *
+ * Used by: GPT-4o, GPT-4.1, GPT-5, o-series models
+ * Vocabulary size: 200,000 tokens
+ *
+ * This encoding has a much larger vocabulary and is optimized for:
+ * - Better handling of non-English languages
+ * - More efficient tokenization of code
+ * - Better multimodal support
+ */
+export declare const O200K_BASE: EncodingDefinition;
+/**
+ * o200k_harmony encoding
+ *
+ * Used by: gpt-oss models (open source)
+ * Same vocabulary as o200k_base but with additional special tokens
+ */
+export declare const O200K_HARMONY: EncodingDefinition;
+/**
+ * All encoding definitions
+ */
+export declare const ENCODING_DEFINITIONS: Record<string, EncodingDefinition>;
+/**
+ * Get encoding definition by name
+ */
+export declare function getEncodingDefinition(name: string): EncodingDefinition | undefined;
+/**
+ * List all encoding names
+ */
+export declare function listEncodingNames(): string[];
+//# sourceMappingURL=encoding-definitions.d.ts.map

package/dist/core/encoding-definitions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoding-definitions.d.ts","sourceRoot":"","sources":["../../src/core/encoding-definitions.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAEpD;;GAEG;AACH,eAAO,MAAM,cAAc;;;;;;CAMjB,CAAC;AAEX;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB,sCAAsC;IACtC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAEpC,0CAA0C;IAC1C,aAAa,EAAE,MAAM,CAAC;IAEtB,gDAAgD;IAChD,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,eAAO,MAAM,SAAS,EAAE,kBAYvB,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,SAAS,EAAE,kBAWvB,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,SAAS,EAAE,kBAevB,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,WAAW,EAAE,kBAoBzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU,EAAE,kBA4BxB,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,aAAa,EAAE,kBA2B3B,CAAC;AAgBF;;GAEG;AACH,eAAO,MAAM,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CASnE,CAAC;AAEF;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,MAAM,GACX,kBAAkB,GAAG,SAAS,CAEhC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,EAAE,CAE5C"}

package/dist/core/encoding-definitions.js

@@ -0,0 +1,202 @@
+/**
+ * Encoding Definitions
+ *
+ * EXACT definitions from tiktoken-rs for all supported encodings.
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/tiktoken_ext/openai_public.rs
+ */
+/**
+ * Special token constants
+ */
+export const SPECIAL_TOKENS = {
+    ENDOFTEXT: "<|endoftext|>",
+    FIM_PREFIX: "<|fim_prefix|>",
+    FIM_MIDDLE: "<|fim_middle|>",
+    FIM_SUFFIX: "<|fim_suffix|>",
+    ENDOFPROMPT: "<|endofprompt|>",
+};
+/**
+ * r50k_base encoding (GPT-2)
+ *
+ * Used by: GPT-3 davinci, curie, babbage, ada models
+ * Vocabulary size: 50,257 tokens
+ */
+export const R50K_BASE = {
+    name: "r50k_base",
+    // Pattern from tiktoken-rs:
+    // 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+
+    pattern: "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+",
+    specialTokens: {
+        [SPECIAL_TOKENS.ENDOFTEXT]: 50256,
+    },
+    vocabularyUrl: "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken",
+    expectedVocabSize: 50256,
+};
+/**
+ * p50k_base encoding
+ *
+ * Used by: Code models, text-davinci-002, text-davinci-003
+ * Vocabulary size: 50,281 tokens
+ */
+export const P50K_BASE = {
+    name: "p50k_base",
+    // Same pattern as r50k_base
+    pattern: "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+",
+    specialTokens: {
+        [SPECIAL_TOKENS.ENDOFTEXT]: 50256,
+    },
+    vocabularyUrl: "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
+    expectedVocabSize: 50280,
+};
+/**
+ * p50k_edit encoding
+ *
+ * Used by: Edit models like text-davinci-edit-001, code-davinci-edit-001
+ * Same vocabulary as p50k_base but with FIM special tokens
+ */
+export const P50K_EDIT = {
+    name: "p50k_edit",
+    // Same pattern as p50k_base
+    pattern: "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+",
+    specialTokens: {
+        [SPECIAL_TOKENS.ENDOFTEXT]: 50256,
+        [SPECIAL_TOKENS.FIM_PREFIX]: 50281,
+        [SPECIAL_TOKENS.FIM_MIDDLE]: 50282,
+        [SPECIAL_TOKENS.FIM_SUFFIX]: 50283,
+    },
+    // Uses the same vocabulary file as p50k_base
+    vocabularyUrl: "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
+    expectedVocabSize: 50280,
+};
+/**
+ * cl100k_base encoding
+ *
+ * Used by: ChatGPT models, GPT-4, GPT-3.5-turbo, text-embedding-ada-002
+ * Vocabulary size: 100,256 tokens
+ */
+export const CL100K_BASE = {
+    name: "cl100k_base",
+    // Pattern from tiktoken-rs:
+    // (?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+
+    //
+    // JavaScript equivalent - (?i:...) is not supported, so we use case-insensitive alternatives
+    // Note: JavaScript doesn't support inline case-insensitive groups (?i:)
+    // We need to manually expand the contractions pattern
+    pattern: "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+    specialTokens: {
+        [SPECIAL_TOKENS.ENDOFTEXT]: 100257,
+        [SPECIAL_TOKENS.FIM_PREFIX]: 100258,
+        [SPECIAL_TOKENS.FIM_MIDDLE]: 100259,
+        [SPECIAL_TOKENS.FIM_SUFFIX]: 100260,
+        [SPECIAL_TOKENS.ENDOFPROMPT]: 100276,
+    },
+    vocabularyUrl: "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
+    expectedVocabSize: 100256,
+};
+/**
+ * o200k_base encoding
+ *
+ * Used by: GPT-4o, GPT-4.1, GPT-5, o-series models
+ * Vocabulary size: 200,000 tokens
+ *
+ * This encoding has a much larger vocabulary and is optimized for:
+ * - Better handling of non-English languages
+ * - More efficient tokenization of code
+ * - Better multimodal support
+ */
+export const O200K_BASE = {
+    name: "o200k_base",
+    // Pattern from tiktoken-rs O200K_BASE_PAT_STR:
+    // [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?
+    // |[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?
+    // |\p{N}{1,3}
+    // | ?[^\s\p{L}\p{N}]+[\r\n/]*   <-- Note: includes forward slash
+    // |\s*[\r\n]+
+    // |\s+(?!\S)
+    // |\s+
+    //
+    // JavaScript equivalent - expand the case-insensitive contractions
+    pattern: [
+        "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?",
+        "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])?",
+        "\\p{N}{1,3}",
+        " ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*", // Note: includes forward slash
+        "\\s*[\\r\\n]+",
+        "\\s+(?!\\S)",
+        "\\s+",
+    ].join("|"),
+    specialTokens: {
+        [SPECIAL_TOKENS.ENDOFTEXT]: 199999,
+        [SPECIAL_TOKENS.ENDOFPROMPT]: 200018,
+    },
+    vocabularyUrl: "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
+    expectedVocabSize: 199998,
+};
+/**
+ * o200k_harmony encoding
+ *
+ * Used by: gpt-oss models (open source)
+ * Same vocabulary as o200k_base but with additional special tokens
+ */
+export const O200K_HARMONY = {
+    name: "o200k_harmony",
+    // Same pattern as o200k_base
+    pattern: O200K_BASE.pattern,
+    specialTokens: {
+        "<|startoftext|>": 199998,
+        "<|endoftext|>": 199999,
+        "<|reserved_200000|>": 200000,
+        "<|reserved_200001|>": 200001,
+        "<|return|>": 200002,
+        "<|constrain|>": 200003,
+        "<|reserved_200004|>": 200004,
+        "<|channel|>": 200005,
+        "<|start|>": 200006,
+        "<|end|>": 200007,
+        "<|message|>": 200008,
+        "<|reserved_200009|>": 200009,
+        "<|reserved_200010|>": 200010,
+        "<|reserved_200011|>": 200011,
+        "<|call|>": 200012,
+        // Reserved tokens from 200013 to 201087 are generated dynamically
+        ...generateReservedTokens(200013, 201087),
+    },
+    // Uses the same vocabulary file as o200k_base
+    vocabularyUrl: "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
+    expectedVocabSize: 199998,
+};
+/**
+ * Generate reserved special tokens
+ */
+function generateReservedTokens(start, end) {
+    const tokens = {};
+    for (let i = start; i <= end; i++) {
+        tokens[`<|reserved_${i}|>`] = i;
+    }
+    return tokens;
+}
+/**
+ * All encoding definitions
+ */
+export const ENCODING_DEFINITIONS = {
+    r50k_base: R50K_BASE,
+    p50k_base: P50K_BASE,
+    p50k_edit: P50K_EDIT,
+    cl100k_base: CL100K_BASE,
+    o200k_base: O200K_BASE,
+    o200k_harmony: O200K_HARMONY,
+    // Aliases
+    gpt2: R50K_BASE,
+};
+/**
+ * Get encoding definition by name
+ */
+export function getEncodingDefinition(name) {
+    return ENCODING_DEFINITIONS[name];
+}
+/**
+ * List all encoding names
+ */
+export function listEncodingNames() {
+    return Object.keys(ENCODING_DEFINITIONS);
+}
+//# sourceMappingURL=encoding-definitions.js.map

package/dist/core/encoding-definitions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoding-definitions.js","sourceRoot":"","sources":["../../src/core/encoding-definitions.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,SAAS,EAAE,eAAe;IAC1B,UAAU,EAAE,gBAAgB;IAC5B,UAAU,EAAE,gBAAgB;IAC5B,UAAU,EAAE,gBAAgB;IAC5B,WAAW,EAAE,iBAAiB;CACtB,CAAC;AAyBX;;;;;GAKG;AACH,MAAM,CAAC,MAAM,SAAS,GAAuB;IAC3C,IAAI,EAAE,WAAW;IACjB,4BAA4B;IAC5B,6EAA6E;IAC7E,OAAO,EACL,oFAAoF;IACtF,aAAa,EAAE;QACb,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,KAAK;KAClC;IACD,aAAa,EACX,yEAAyE;IAC3E,iBAAiB,EAAE,KAAK;CACzB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,SAAS,GAAuB;IAC3C,IAAI,EAAE,WAAW;IACjB,4BAA4B;IAC5B,OAAO,EACL,oFAAoF;IACtF,aAAa,EAAE;QACb,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,KAAK;KAClC;IACD,aAAa,EACX,yEAAyE;IAC3E,iBAAiB,EAAE,KAAK;CACzB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,SAAS,GAAuB;IAC3C,IAAI,EAAE,WAAW;IACjB,4BAA4B;IAC5B,OAAO,EACL,oFAAoF;IACtF,aAAa,EAAE;QACb,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,KAAK;QACjC,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,KAAK;QAClC,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,KAAK;QAClC,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,KAAK;KACnC;IACD,6CAA6C;IAC7C,aAAa,EACX,yEAAyE;IAC3E,iBAAiB,EAAE,KAAK;CACzB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,WAAW,GAAuB;IAC7C,IAAI,EAAE,aAAa;IACnB,4BAA4B;IAC5B,sHAAsH;IACtH,EAAE;IACF,6FAA6F;IAC7F,wEAAwE;IACxE,sDAAsD;IACtD,OAAO,EACL,mKAAmK;IACrK,aAAa,EAAE;QACb,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM;QAClC,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,MAAM;QACnC,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,MAAM;QACnC,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,MAAM;QACnC,CAAC,cAAc,CAAC,WAAW,CAAC,EAAE,MAAM;KACrC;IACD,aAAa,EACX,2EAA2E;IAC7E,iBAAiB,EAAE,MAAM;CAC1B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,UAAU,GAAuB;IAC5C,IAAI,EAAE,YAAY;IAClB,+CAA+C;IAC/C,4GAA4G;IAC5G,6GAA6G;IAC7G,cAAc;IACd,iEAAiE;IACjE,cAAc;IACd,aAAa;IACb,OAAO;IACP,EAAE;IACF,mEAAmE;IACnE,OAAO,EAAE;QACP,qJAAqJ;QACrJ,qJAAqJ;QACrJ,aAAa;QACb,iCAAiC,EAAE,+BAA+B;QAClE,eAAe;QACf,aAAa;QACb,MAAM;KACP,CAAC,IAAI,CAAC,GAAG,CAAC;IACX,aAAa,EAAE;QACb,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM;QAClC,CAAC,cAAc,CAAC,WAAW,CAAC,EAAE,MAAM;KACrC;IACD,aAAa,EACX,0EAA0E;IAC5E,iBAAiB,EAAE,MAAM;CAC1B,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAuB;IAC/C,IAAI,EAAE,eAAe;IACrB,6BAA6B;IAC7B,OAAO,EAAE,UAAU,CAAC,OAAO;IAC3B,aAAa,EAAE;QACb,iBAAiB,EAAE,MAAM;QACzB,eAAe,EAAE,MAAM;QACvB,qBAAqB,EAAE,MAAM;QAC7B,qBAAqB,EAAE,MAAM;QAC7B,YAAY,EAAE,MAAM;QACpB,eAAe,EAAE,MAAM;QACvB,qBAAqB,EAAE,MAAM;QAC7B,aAAa,EAAE,MAAM;QACrB,WAAW,EAAE,MAAM;QACnB,SAAS,EAAE,MAAM;QACjB,aAAa,EAAE,MAAM;QACrB,qBAAqB,EAAE,MAAM;QAC7B,qBAAqB,EAAE,MAAM;QAC7B,qBAAqB,EAAE,MAAM;QAC7B,UAAU,EAAE,MAAM;QAClB,kEAAkE;QAClE,GAAG,sBAAsB,CAAC,MAAM,EAAE,MAAM,CAAC;KAC1C;IACD,8CAA8C;IAC9C,aAAa,EACX,0EAA0E;IAC5E,iBAAiB,EAAE,MAAM;CAC1B,CAAC;AAEF;;GAEG;AACH,SAAS,sBAAsB,CAC7B,KAAa,EACb,GAAW;IAEX,MAAM,MAAM,GAAyB,EAAE,CAAC;IACxC,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAuC;IACtE,SAAS,EAAE,SAAS;IACpB,SAAS,EAAE,SAAS;IACpB,SAAS,EAAE,SAAS;IACpB,WAAW,EAAE,WAAW;IACxB,UAAU,EAAE,UAAU;IACtB,aAAa,EAAE,aAAa;IAC5B,UAAU;IACV,IAAI,EAAE,SAAS;CAChB,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,IAAY;IAEZ,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB;IAC/B,OAAO,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAC3C,CAAC"}

package/dist/core/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Core Tiktoken Module
+ *
+ * This module provides the exact BPE implementation from tiktoken-rs,
+ * ported to TypeScript for use in both Node.js and browser environments.
+ */
+export { type Rank, type Vocabulary, type ReverseVocabulary, bytesToKey, keyToBytes, bytePairEncode, bytePairSplit, } from "./byte-pair-encoding.js";
+export { parseVocabulary, loadVocabularyFromUrl, loadVocabularyFromString, createSpecialTokenMaps, clearVocabularyCache, getVocabularyFromCache, isVocabularyCached, VOCABULARY_URLS, base64Encode, keyToBytes as vocabKeyToBytes, } from "./vocab-loader.js";
+export { CoreBPE, DecodeKeyError } from "./tiktoken.js";
+export { type EncodingDefinition, SPECIAL_TOKENS, R50K_BASE, P50K_BASE, P50K_EDIT, CL100K_BASE, O200K_BASE, O200K_HARMONY, ENCODING_DEFINITIONS, getEncodingDefinition, listEncodingNames, } from "./encoding-definitions.js";
+export { type TokenizerName, getTokenizerForModel, getContextSize, getExactContextSize, EXACT_CONTEXT_SIZES, } from "./model-to-encoding.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/core/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EACL,KAAK,IAAI,EACT,KAAK,UAAU,EACf,KAAK,iBAAiB,EACtB,UAAU,EACV,UAAU,EACV,cAAc,EACd,aAAa,GACd,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EACL,eAAe,EACf,qBAAqB,EACrB,wBAAwB,EACxB,sBAAsB,EACtB,oBAAoB,EACpB,sBAAsB,EACtB,kBAAkB,EAClB,eAAe,EACf,YAAY,EACZ,UAAU,IAAI,eAAe,GAC9B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAGxD,OAAO,EACL,KAAK,kBAAkB,EACvB,cAAc,EACd,SAAS,EACT,SAAS,EACT,SAAS,EACT,WAAW,EACX,UAAU,EACV,aAAa,EACb,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,2BAA2B,CAAC;AAGnC,OAAO,EACL,KAAK,aAAa,EAClB,oBAAoB,EACpB,cAAc,EACd,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,wBAAwB,CAAC"}

package/dist/core/index.js

@@ -0,0 +1,17 @@
+/**
+ * Core Tiktoken Module
+ *
+ * This module provides the exact BPE implementation from tiktoken-rs,
+ * ported to TypeScript for use in both Node.js and browser environments.
+ */
+// Byte-Pair Encoding algorithm
+export { bytesToKey, keyToBytes, bytePairEncode, bytePairSplit, } from "./byte-pair-encoding.js";
+// Vocabulary loading
+export { parseVocabulary, loadVocabularyFromUrl, loadVocabularyFromString, createSpecialTokenMaps, clearVocabularyCache, getVocabularyFromCache, isVocabularyCached, VOCABULARY_URLS, base64Encode, keyToBytes as vocabKeyToBytes, } from "./vocab-loader.js";
+// Core BPE tokenizer
+export { CoreBPE, DecodeKeyError } from "./tiktoken.js";
+// Encoding definitions
+export { SPECIAL_TOKENS, R50K_BASE, P50K_BASE, P50K_EDIT, CL100K_BASE, O200K_BASE, O200K_HARMONY, ENCODING_DEFINITIONS, getEncodingDefinition, listEncodingNames, } from "./encoding-definitions.js";
+// Model to encoding mappings
+export { getTokenizerForModel, getContextSize, getExactContextSize, EXACT_CONTEXT_SIZES, } from "./model-to-encoding.js";
+//# sourceMappingURL=index.js.map

package/dist/core/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,+BAA+B;AAC/B,OAAO,EAIL,UAAU,EACV,UAAU,EACV,cAAc,EACd,aAAa,GACd,MAAM,yBAAyB,CAAC;AAEjC,qBAAqB;AACrB,OAAO,EACL,eAAe,EACf,qBAAqB,EACrB,wBAAwB,EACxB,sBAAsB,EACtB,oBAAoB,EACpB,sBAAsB,EACtB,kBAAkB,EAClB,eAAe,EACf,YAAY,EACZ,UAAU,IAAI,eAAe,GAC9B,MAAM,mBAAmB,CAAC;AAE3B,qBAAqB;AACrB,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAExD,uBAAuB;AACvB,OAAO,EAEL,cAAc,EACd,SAAS,EACT,SAAS,EACT,SAAS,EACT,WAAW,EACX,UAAU,EACV,aAAa,EACb,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,2BAA2B,CAAC;AAEnC,6BAA6B;AAC7B,OAAO,EAEL,oBAAoB,EACpB,cAAc,EACd,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,wBAAwB,CAAC"}

package/dist/core/model-to-encoding.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Model to Encoding Mappings
+ *
+ * EXACT mappings from tiktoken-rs for model name to encoding.
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/tokenizer.rs
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/model.rs
+ */
+/**
+ * Tokenizer/Encoding enum values (matching tiktoken-rs)
+ */
+export type TokenizerName = "o200k_harmony" | "o200k_base" | "cl100k_base" | "p50k_base" | "r50k_base" | "p50k_edit" | "gpt2";
+/**
+ * Get the tokenizer/encoding for a model name
+ *
+ * This function matches the logic in tiktoken-rs get_tokenizer()
+ *
+ * @param modelName - The model name
+ * @returns The tokenizer name, or undefined if not found
+ */
+export declare function getTokenizerForModel(modelName: string): TokenizerName | undefined;
+/**
+ * Context size mapping from tiktoken-rs model.rs
+ *
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/model.rs
+ */
+export declare function getContextSize(model: string): number;
+/**
+ * Extended context limits including specific model versions
+ * from tiktoken-rs model.rs get_context_size match statement
+ */
+export declare const EXACT_CONTEXT_SIZES: Record<string, number>;
+/**
+ * Get context size with exact match support
+ */
+export declare function getExactContextSize(model: string): number;
+//# sourceMappingURL=model-to-encoding.d.ts.map

package/dist/core/model-to-encoding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-to-encoding.d.ts","sourceRoot":"","sources":["../../src/core/model-to-encoding.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,eAAe,GACf,YAAY,GACZ,aAAa,GACb,WAAW,GACX,WAAW,GACX,WAAW,GACX,MAAM,CAAC;AA8FX;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,MAAM,GAChB,aAAa,GAAG,SAAS,CAe3B;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CA0FpD;AAED;;;GAGG;AACH,eAAO,MAAM,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAoFtD,CAAC;AAEF;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CASzD"}