@hyvmind/tiktoken-ts 0.0.1

package/dist/core/model-to-encoding.js

@@ -0,0 +1,299 @@
+/**
+ * Model to Encoding Mappings
+ *
+ * EXACT mappings from tiktoken-rs for model name to encoding.
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/tokenizer.rs
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/model.rs
+ */
+/**
+ * Model prefix to tokenizer mapping
+ * Checked in order, so more specific prefixes should come first
+ *
+ * From tiktoken-rs MODEL_PREFIX_TO_TOKENIZER
+ */
+const MODEL_PREFIX_TO_TOKENIZER = [
+    // Reasoning models
+    ["o1-", "o200k_base"],
+    ["o3-", "o200k_base"],
+    ["o4-", "o200k_base"],
+    // Chat models
+    ["gpt-5-", "o200k_base"],
+    ["gpt-4.5-", "o200k_base"],
+    ["gpt-4.1-", "o200k_base"],
+    ["chatgpt-4o-", "o200k_base"],
+    ["gpt-4o-", "o200k_base"],
+    ["gpt-4-", "cl100k_base"],
+    ["gpt-3.5-turbo-", "cl100k_base"],
+    ["gpt-35-turbo-", "cl100k_base"], // Azure deployment name
+    ["gpt-oss-", "o200k_harmony"],
+    // Fine-tuned models
+    ["ft:gpt-4o", "o200k_base"],
+    ["ft:gpt-4", "cl100k_base"],
+    ["ft:gpt-3.5-turbo", "cl100k_base"],
+    ["ft:davinci-002", "cl100k_base"],
+    ["ft:babbage-002", "cl100k_base"],
+];
+/**
+ * Exact model name to tokenizer mapping
+ *
+ * From tiktoken-rs MODEL_TO_TOKENIZER
+ */
+const MODEL_TO_TOKENIZER = new Map([
+    // Reasoning models
+    ["o1", "o200k_base"],
+    ["o3", "o200k_base"],
+    ["o4", "o200k_base"],
+    // Chat models
+    ["gpt-5", "o200k_base"],
+    ["gpt-4.1", "o200k_base"],
+    ["chatgpt-4o-latest", "o200k_base"],
+    ["gpt-4o", "o200k_base"],
+    ["gpt-4", "cl100k_base"],
+    ["gpt-3.5-turbo", "cl100k_base"],
+    ["gpt-3.5", "cl100k_base"],
+    ["gpt-35-turbo", "cl100k_base"], // Azure deployment name
+    // Base models
+    ["davinci-002", "cl100k_base"],
+    ["babbage-002", "cl100k_base"],
+    // Embeddings
+    ["text-embedding-ada-002", "cl100k_base"],
+    ["text-embedding-3-small", "cl100k_base"],
+    ["text-embedding-3-large", "cl100k_base"],
+    // DEPRECATED MODELS - Text
+    ["text-davinci-003", "p50k_base"],
+    ["text-davinci-002", "p50k_base"],
+    ["text-davinci-001", "r50k_base"],
+    ["text-curie-001", "r50k_base"],
+    ["text-babbage-001", "r50k_base"],
+    ["text-ada-001", "r50k_base"],
+    ["davinci", "r50k_base"],
+    ["curie", "r50k_base"],
+    ["babbage", "r50k_base"],
+    ["ada", "r50k_base"],
+    // DEPRECATED MODELS - Code
+    ["code-davinci-002", "p50k_base"],
+    ["code-davinci-001", "p50k_base"],
+    ["code-cushman-002", "p50k_base"],
+    ["code-cushman-001", "p50k_base"],
+    ["davinci-codex", "p50k_base"],
+    ["cushman-codex", "p50k_base"],
+    // DEPRECATED MODELS - Edit
+    ["text-davinci-edit-001", "p50k_edit"],
+    ["code-davinci-edit-001", "p50k_edit"],
+    // DEPRECATED MODELS - Old Embeddings
+    ["text-similarity-davinci-001", "r50k_base"],
+    ["text-similarity-curie-001", "r50k_base"],
+    ["text-similarity-babbage-001", "r50k_base"],
+    ["text-similarity-ada-001", "r50k_base"],
+    ["text-search-davinci-doc-001", "r50k_base"],
+    ["text-search-curie-doc-001", "r50k_base"],
+    ["text-search-babbage-doc-001", "r50k_base"],
+    ["text-search-ada-doc-001", "r50k_base"],
+    ["code-search-babbage-code-001", "r50k_base"],
+    ["code-search-ada-code-001", "r50k_base"],
+    // Open source
+    ["gpt2", "gpt2"],
+    ["gpt-2", "gpt2"],
+]);
+/**
+ * Get the tokenizer/encoding for a model name
+ *
+ * This function matches the logic in tiktoken-rs get_tokenizer()
+ *
+ * @param modelName - The model name
+ * @returns The tokenizer name, or undefined if not found
+ */
+export function getTokenizerForModel(modelName) {
+    // First, check exact match
+    const exactMatch = MODEL_TO_TOKENIZER.get(modelName);
+    if (exactMatch) {
+        return exactMatch;
+    }
+    // Then, check prefix matches
+    for (const [prefix, tokenizer] of MODEL_PREFIX_TO_TOKENIZER) {
+        if (modelName.startsWith(prefix)) {
+            return tokenizer;
+        }
+    }
+    return undefined;
+}
+/**
+ * Context size mapping from tiktoken-rs model.rs
+ *
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/model.rs
+ */
+export function getContextSize(model) {
+    // Handle fine-tuned models
+    if (model.startsWith("ft:")) {
+        const rest = model.slice(3);
+        const base = rest.split(":")[0] ?? rest;
+        return getContextSize(base);
+    }
+    // Check prefixes (most specific first)
+    if (model.startsWith("gpt-5")) {
+        return 400_000;
+    }
+    if (model.startsWith("gpt-oss")) {
+        return 131_072;
+    }
+    if (model.startsWith("o1") ||
+        model.startsWith("o3") ||
+        model.startsWith("o4")) {
+        return 200_000;
+    }
+    if (model.startsWith("gpt-4.1")) {
+        return 1_047_576;
+    }
+    if (model.startsWith("gpt-4o")) {
+        return 128_000;
+    }
+    if (model.startsWith("gpt-4-turbo-")) {
+        return 128_000;
+    }
+    if (model.startsWith("gpt-4-0125")) {
+        return 128_000;
+    }
+    if (model.startsWith("gpt-4-1106")) {
+        return 128_000;
+    }
+    if (model.startsWith("gpt-4-32k")) {
+        return 32_768;
+    }
+    if (model.startsWith("gpt-4")) {
+        return 8192;
+    }
+    if (model.startsWith("gpt-3.5-turbo-0125")) {
+        return 16_385;
+    }
+    if (model.startsWith("gpt-3.5-turbo-1106")) {
+        return 16_385;
+    }
+    if (model.startsWith("gpt-3.5-turbo-16k")) {
+        return 16_385;
+    }
+    if (model.startsWith("gpt-3.5-turbo")) {
+        return 16_385;
+    }
+    if (model.startsWith("text-davinci-002") ||
+        model.startsWith("text-davinci-003")) {
+        return 4097;
+    }
+    if (model.startsWith("ada") ||
+        model.startsWith("babbage") ||
+        model.startsWith("curie")) {
+        return 2049;
+    }
+    if (model.startsWith("code-cushman-001")) {
+        return 2048;
+    }
+    if (model.startsWith("code-davinci-002")) {
+        return 8001;
+    }
+    if (model.startsWith("davinci")) {
+        return 2049;
+    }
+    if (model.startsWith("text-ada-001") ||
+        model.startsWith("text-babbage-001") ||
+        model.startsWith("text-curie-001")) {
+        return 2049;
+    }
+    if (model.startsWith("text-embedding-ada-002")) {
+        return 8192;
+    }
+    // Default
+    return 4096;
+}
+/**
+ * Extended context limits including specific model versions
+ * from tiktoken-rs model.rs get_context_size match statement
+ */
+export const EXACT_CONTEXT_SIZES = {
+    // o-series reasoning models
+    o1: 200_000,
+    "o1-2024-12-17": 200_000,
+    "o1-mini": 128_000,
+    "o1-mini-2024-09-12": 128_000,
+    "o1-preview": 128_000,
+    "o1-preview-2024-09-12": 128_000,
+    o3: 200_000,
+    "o3-mini": 200_000,
+    "o3-mini-2025-01-31": 200_000,
+    "o4-mini": 200_000,
+    "o4-mini-2025-04-16": 200_000,
+    // GPT-5 series
+    "gpt-5": 400_000,
+    "gpt-5-2025-03-22": 400_000,
+    "gpt-5-mini": 200_000,
+    "gpt-5-mini-2025-03-22": 200_000,
+    "gpt-5-turbo": 400_000,
+    "gpt-5-turbo-2025-06-09": 400_000,
+    "gpt-5-nano": 200_000,
+    "gpt-5-nano-2025-09-14": 200_000,
+    // GPT-4.5 preview
+    "gpt-4.5-preview": 128_000,
+    "gpt-4.5-preview-2025-02-27": 128_000,
+    // GPT-4.1 series (1M context!)
+    "gpt-4.1": 1_047_576,
+    "gpt-4.1-2025-04-14": 1_047_576,
+    "gpt-4.1-mini": 1_047_576,
+    "gpt-4.1-mini-2025-04-14": 1_047_576,
+    "gpt-4.1-nano": 1_047_576,
+    "gpt-4.1-nano-2025-04-14": 1_047_576,
+    // GPT-4o series
+    "chatgpt-4o-latest": 128_000,
+    "gpt-4o": 128_000,
+    "gpt-4o-2024-05-13": 128_000,
+    "gpt-4o-2024-08-06": 128_000,
+    "gpt-4o-2024-11-20": 128_000,
+    "gpt-4o-audio-preview": 128_000,
+    "gpt-4o-audio-preview-2024-10-01": 128_000,
+    "gpt-4o-audio-preview-2024-12-17": 128_000,
+    "gpt-4o-mini": 128_000,
+    "gpt-4o-mini-2024-07-18": 128_000,
+    "gpt-4o-mini-audio-preview": 128_000,
+    "gpt-4o-mini-audio-preview-2024-12-17": 128_000,
+    "gpt-4o-realtime-preview": 128_000,
+    "gpt-4o-realtime-preview-2024-10-01": 128_000,
+    "gpt-4o-realtime-preview-2024-12-17": 128_000,
+    // GPT-4 turbo series
+    "gpt-4-turbo": 128_000,
+    "gpt-4-turbo-2024-04-09": 128_000,
+    "gpt-4-turbo-preview": 128_000,
+    "gpt-4-0125-preview": 128_000,
+    "gpt-4-1106-preview": 128_000,
+    "gpt-4-vision-preview": 128_000,
+    "gpt-4-1106-vision-preview": 128_000,
+    // GPT-4 base
+    "gpt-4": 8_192,
+    "gpt-4-0613": 8_192,
+    "gpt-4-0314": 8_192,
+    "gpt-4-32k": 32_768,
+    "gpt-4-32k-0613": 32_768,
+    "gpt-4-32k-0314": 32_768,
+    // GPT-3.5 series
+    "gpt-3.5-turbo": 16_385,
+    "gpt-3.5-turbo-0125": 16_385,
+    "gpt-3.5-turbo-1106": 16_385,
+    "gpt-3.5-turbo-0613": 16_385,
+    "gpt-3.5-turbo-16k": 16_385,
+    "gpt-3.5-turbo-16k-0613": 16_385,
+    "gpt-3.5-turbo-instruct": 4_096,
+    "gpt-3.5-turbo-instruct-0914": 4_096,
+    "gpt-3.5-turbo-0301": 4_096,
+    // gpt-oss
+    "gpt-oss": 131_072,
+    "gpt-oss-2025-03-01": 131_072,
+};
+/**
+ * Get context size with exact match support
+ */
+export function getExactContextSize(model) {
+    // Check exact match first
+    const exactSize = EXACT_CONTEXT_SIZES[model];
+    if (exactSize !== undefined) {
+        return exactSize;
+    }
+    // Fall back to prefix matching
+    return getContextSize(model);
+}
+//# sourceMappingURL=model-to-encoding.js.map

package/dist/core/model-to-encoding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-to-encoding.js","sourceRoot":"","sources":["../../src/core/model-to-encoding.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAcH;;;;;GAKG;AACH,MAAM,yBAAyB,GAAmC;IAChE,mBAAmB;IACnB,CAAC,KAAK,EAAE,YAAY,CAAC;IACrB,CAAC,KAAK,EAAE,YAAY,CAAC;IACrB,CAAC,KAAK,EAAE,YAAY,CAAC;IACrB,cAAc;IACd,CAAC,QAAQ,EAAE,YAAY,CAAC;IACxB,CAAC,UAAU,EAAE,YAAY,CAAC;IAC1B,CAAC,UAAU,EAAE,YAAY,CAAC;IAC1B,CAAC,aAAa,EAAE,YAAY,CAAC;IAC7B,CAAC,SAAS,EAAE,YAAY,CAAC;IACzB,CAAC,QAAQ,EAAE,aAAa,CAAC;IACzB,CAAC,gBAAgB,EAAE,aAAa,CAAC;IACjC,CAAC,eAAe,EAAE,aAAa,CAAC,EAAE,wBAAwB;IAC1D,CAAC,UAAU,EAAE,eAAe,CAAC;IAC7B,oBAAoB;IACpB,CAAC,WAAW,EAAE,YAAY,CAAC;IAC3B,CAAC,UAAU,EAAE,aAAa,CAAC;IAC3B,CAAC,kBAAkB,EAAE,aAAa,CAAC;IACnC,CAAC,gBAAgB,EAAE,aAAa,CAAC;IACjC,CAAC,gBAAgB,EAAE,aAAa,CAAC;CAClC,CAAC;AAEF;;;;GAIG;AACH,MAAM,kBAAkB,GAA+B,IAAI,GAAG,CAAC;IAC7D,mBAAmB;IACnB,CAAC,IAAI,EAAE,YAAY,CAAC;IACpB,CAAC,IAAI,EAAE,YAAY,CAAC;IACpB,CAAC,IAAI,EAAE,YAAY,CAAC;IACpB,cAAc;IACd,CAAC,OAAO,EAAE,YAAY,CAAC;IACvB,CAAC,SAAS,EAAE,YAAY,CAAC;IACzB,CAAC,mBAAmB,EAAE,YAAY,CAAC;IACnC,CAAC,QAAQ,EAAE,YAAY,CAAC;IACxB,CAAC,OAAO,EAAE,aAAa,CAAC;IACxB,CAAC,eAAe,EAAE,aAAa,CAAC;IAChC,CAAC,SAAS,EAAE,aAAa,CAAC;IAC1B,CAAC,cAAc,EAAE,aAAa,CAAC,EAAE,wBAAwB;IACzD,cAAc;IACd,CAAC,aAAa,EAAE,aAAa,CAAC;IAC9B,CAAC,aAAa,EAAE,aAAa,CAAC;IAC9B,aAAa;IACb,CAAC,wBAAwB,EAAE,aAAa,CAAC;IACzC,CAAC,wBAAwB,EAAE,aAAa,CAAC;IACzC,CAAC,wBAAwB,EAAE,aAAa,CAAC;IACzC,2BAA2B;IAC3B,CAAC,kBAAkB,EAAE,WAAW,CAAC;IACjC,CAAC,kBAAkB,EAAE,WAAW,CAAC;IACjC,CAAC,kBAAkB,EAAE,WAAW,CAAC;IACjC,CAAC,gBAAgB,EAAE,WAAW,CAAC;IAC/B,CAAC,kBAAkB,EAAE,WAAW,CAAC;IACjC,CAAC,cAAc,EAAE,WAAW,CAAC;IAC7B,CAAC,SAAS,EAAE,WAAW,CAAC;IACxB,CAAC,OAAO,EAAE,WAAW,CAAC;IACtB,CAAC,SAAS,EAAE,WAAW,CAAC;IACxB,CAAC,KAAK,EAAE,WAAW,CAAC;IACpB,2BAA2B;IAC3B,CAAC,kBAAkB,EAAE,WAAW,CAAC;IACjC,CAAC,kBAAkB,EAAE,WAAW,CAAC;IACjC,CAAC,kBAAkB,EAAE,WAAW,CAAC;IACjC,CAAC,kBAAkB,EAAE,WAAW,CAAC;IACjC,CAAC,eAAe,EAAE,WAAW,CAAC;IAC9B,CAAC,eAAe,EAAE,WAAW,CAAC;IAC9B,2BAA2B;IAC3B,CAAC,uBAAuB,EAAE,WAAW,CAAC;IACtC,CAAC,uBAAuB,EAAE,WAAW,CAAC;IACtC,qCAAqC;IACrC,CAAC,6BAA6B,EAAE,WAAW,CAAC;IAC5C,CAAC,2BAA2B,EAAE,WAAW,CAAC;IAC1C,CAAC,6BAA6B,EAAE,WAAW,CAAC;IAC5C,CAAC,yBAAyB,EAAE,WAAW,CAAC;IACxC,CAAC,6BAA6B,EAAE,WAAW,CAAC;IAC5C,CAAC,2BAA2B,EAAE,WAAW,CAAC;IAC1C,CAAC,6BAA6B,EAAE,WAAW,CAAC;IAC5C,CAAC,yBAAyB,EAAE,WAAW,CAAC;IACxC,CAAC,8BAA8B,EAAE,WAAW,CAAC;IAC7C,CAAC,0BAA0B,EAAE,WAAW,CAAC;IACzC,cAAc;IACd,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,OAAO,EAAE,MAAM,CAAC;CAClB,CAAC,CAAC;AAEH;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAClC,SAAiB;IAEjB,2BAA2B;IAC3B,MAAM,UAAU,GAAG,kBAAkB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACrD,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,6BAA6B;IAC7B,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,yBAAyB,EAAE,CAAC;QAC5D,IAAI,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,2BAA2B;IAC3B,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACxC,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,uCAAuC;IACvC,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9B,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IACE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;QACtB,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;QACtB,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EACtB,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;QACrC,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACnC,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACnC,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAClC,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,EAAE,CAAC;QAC3C,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,EAAE,CAAC;QAC3C,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAC1C,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;QACtC,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IACE,KAAK,CAAC,UAAU,CAAC,kBAAkB,CAAC;QACpC,KAAK,CAAC,UAAU,CAAC,kBAAkB,CAAC,EACpC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IACE,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;QACvB,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC;QAC3B,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EACzB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IACE,KAAK,CAAC,UAAU,CAAC,cAAc,CAAC;QAChC,KAAK,CAAC,UAAU,CAAC,kBAAkB,CAAC;QACpC,KAAK,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAClC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,CAAC,wBAAwB,CAAC,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU;IACV,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAA2B;IACzD,4BAA4B;IAC5B,EAAE,EAAE,OAAO;IACX,eAAe,EAAE,OAAO;IACxB,SAAS,EAAE,OAAO;IAClB,oBAAoB,EAAE,OAAO;IAC7B,YAAY,EAAE,OAAO;IACrB,uBAAuB,EAAE,OAAO;IAChC,EAAE,EAAE,OAAO;IACX,SAAS,EAAE,OAAO;IAClB,oBAAoB,EAAE,OAAO;IAC7B,SAAS,EAAE,OAAO;IAClB,oBAAoB,EAAE,OAAO;IAE7B,eAAe;IACf,OAAO,EAAE,OAAO;IAChB,kBAAkB,EAAE,OAAO;IAC3B,YAAY,EAAE,OAAO;IACrB,uBAAuB,EAAE,OAAO;IAChC,aAAa,EAAE,OAAO;IACtB,wBAAwB,EAAE,OAAO;IACjC,YAAY,EAAE,OAAO;IACrB,uBAAuB,EAAE,OAAO;IAEhC,kBAAkB;IAClB,iBAAiB,EAAE,OAAO;IAC1B,4BAA4B,EAAE,OAAO;IAErC,+BAA+B;IAC/B,SAAS,EAAE,SAAS;IACpB,oBAAoB,EAAE,SAAS;IAC/B,cAAc,EAAE,SAAS;IACzB,yBAAyB,EAAE,SAAS;IACpC,cAAc,EAAE,SAAS;IACzB,yBAAyB,EAAE,SAAS;IAEpC,gBAAgB;IAChB,mBAAmB,EAAE,OAAO;IAC5B,QAAQ,EAAE,OAAO;IACjB,mBAAmB,EAAE,OAAO;IAC5B,mBAAmB,EAAE,OAAO;IAC5B,mBAAmB,EAAE,OAAO;IAC5B,sBAAsB,EAAE,OAAO;IAC/B,iCAAiC,EAAE,OAAO;IAC1C,iCAAiC,EAAE,OAAO;IAC1C,aAAa,EAAE,OAAO;IACtB,wBAAwB,EAAE,OAAO;IACjC,2BAA2B,EAAE,OAAO;IACpC,sCAAsC,EAAE,OAAO;IAC/C,yBAAyB,EAAE,OAAO;IAClC,oCAAoC,EAAE,OAAO;IAC7C,oCAAoC,EAAE,OAAO;IAE7C,qBAAqB;IACrB,aAAa,EAAE,OAAO;IACtB,wBAAwB,EAAE,OAAO;IACjC,qBAAqB,EAAE,OAAO;IAC9B,oBAAoB,EAAE,OAAO;IAC7B,oBAAoB,EAAE,OAAO;IAC7B,sBAAsB,EAAE,OAAO;IAC/B,2BAA2B,EAAE,OAAO;IAEpC,aAAa;IACb,OAAO,EAAE,KAAK;IACd,YAAY,EAAE,KAAK;IACnB,YAAY,EAAE,KAAK;IACnB,WAAW,EAAE,MAAM;IACnB,gBAAgB,EAAE,MAAM;IACxB,gBAAgB,EAAE,MAAM;IAExB,iBAAiB;IACjB,eAAe,EAAE,MAAM;IACvB,oBAAoB,EAAE,MAAM;IAC5B,oBAAoB,EAAE,MAAM;IAC5B,oBAAoB,EAAE,MAAM;IAC5B,mBAAmB,EAAE,MAAM;IAC3B,wBAAwB,EAAE,MAAM;IAChC,wBAAwB,EAAE,KAAK;IAC/B,6BAA6B,EAAE,KAAK;IACpC,oBAAoB,EAAE,KAAK;IAE3B,UAAU;IACV,SAAS,EAAE,OAAO;IAClB,oBAAoB,EAAE,OAAO;CAC9B,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAC/C,0BAA0B;IAC1B,MAAM,SAAS,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,+BAA+B;IAC/B,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC"}

package/dist/core/tiktoken.d.ts

@@ -0,0 +1,126 @@
+/**
+ * Core Tiktoken BPE Implementation
+ *
+ * This is an EXACT port of the CoreBPE struct from tiktoken-rs.
+ * Reference: https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/vendor_tiktoken.rs
+ *
+ * Provides the main tokenization API:
+ * - encode_ordinary(text) - Encode without special tokens
+ * - encode(text, allowed_special) - Encode with optional special tokens
+ * - encode_with_special_tokens(text) - Encode with all special tokens
+ * - decode(tokens) - Decode tokens to text
+ * - decode_bytes(tokens) - Decode tokens to raw bytes
+ */
+import { type Vocabulary, type Rank } from "./byte-pair-encoding.js";
+/**
+ * Error thrown when a token cannot be decoded
+ */
+export declare class DecodeKeyError extends Error {
+    readonly token: Rank;
+    constructor(token: Rank);
+}
+/**
+ * CoreBPE - The main tokenizer class
+ *
+ * This class implements the complete BPE tokenization algorithm,
+ * matching the behavior of tiktoken-rs exactly.
+ */
+export declare class CoreBPE {
+    /** The vocabulary mapping byte sequences to ranks */
+    private readonly encoder;
+    /** The special tokens mapping strings to ranks */
+    private readonly specialTokensEncoder;
+    /** Reverse vocabulary mapping ranks to byte sequences */
+    private readonly decoder;
+    /** Reverse special tokens mapping ranks to byte sequences */
+    private readonly specialTokensDecoder;
+    /** Compiled regex for splitting text */
+    private readonly regex;
+    /** Compiled regex for finding special tokens */
+    private readonly specialRegex;
+    /** Text encoder for converting strings to bytes */
+    private readonly textEncoder;
+    /** Text decoder for converting bytes to strings */
+    private readonly textDecoder;
+    /**
+     * Create a new CoreBPE instance
+     *
+     * @param encoder - Vocabulary mapping byte sequences to ranks
+     * @param specialTokensEncoder - Special tokens mapping strings to ranks
+     * @param pattern - Regex pattern for splitting text into pieces
+     */
+    constructor(encoder: Vocabulary, specialTokensEncoder: Map<string, Rank>, pattern: string);
+    /**
+     * Decode tokens to raw bytes
+     *
+     * The bytes are not guaranteed to be valid UTF-8.
+     *
+     * @param tokens - Array of token ranks to decode
+     * @returns Decoded bytes
+     * @throws DecodeKeyError if a token is not found
+     */
+    decodeBytes(tokens: Rank[]): Uint8Array;
+    /**
+     * Decode tokens to a string
+     *
+     * @param tokens - Array of token ranks to decode
+     * @returns Decoded string
+     * @throws DecodeKeyError if a token is not found
+     */
+    decode(tokens: Rank[]): string;
+    /**
+     * Encode text without handling special tokens
+     *
+     * This is the core encoding logic. Special tokens are treated as regular text.
+     *
+     * @param text - Text to encode
+     * @returns Array of token ranks
+     */
+    encodeOrdinary(text: string): Rank[];
+    /**
+     * Encode text with special token handling
+     *
+     * @param text - Text to encode
+     * @param allowedSpecial - Set of special tokens that are allowed
+     * @returns Tuple of [tokens, lastPieceTokenLen]
+     */
+    encode(text: string, allowedSpecial: Set<string>): [Rank[], number];
+    /**
+     * Encode text with all special tokens allowed
+     *
+     * @param text - Text to encode
+     * @returns Array of token ranks
+     */
+    encodeWithSpecialTokens(text: string): Rank[];
+    /**
+     * Get all special tokens
+     *
+     * @returns Set of special token strings
+     */
+    getSpecialTokens(): Set<string>;
+    /**
+     * Get the vocabulary size (excluding special tokens)
+     */
+    get vocabSize(): number;
+    /**
+     * Get the total vocabulary size (including special tokens)
+     */
+    get totalVocabSize(): number;
+    /**
+     * Check if a token rank is a special token
+     */
+    isSpecialToken(token: Rank): boolean;
+    /**
+     * Get the byte representation of a token
+     */
+    getTokenBytes(token: Rank): Uint8Array | undefined;
+    /**
+     * Get the rank of a byte sequence
+     */
+    getRank(bytes: Uint8Array): Rank | undefined;
+    /**
+     * Get the rank of a special token
+     */
+    getSpecialTokenRank(token: string): Rank | undefined;
+}
+//# sourceMappingURL=tiktoken.d.ts.map

package/dist/core/tiktoken.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tiktoken.d.ts","sourceRoot":"","sources":["../../src/core/tiktoken.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EACL,KAAK,UAAU,EAEf,KAAK,IAAI,EAIV,MAAM,yBAAyB,CAAC;AAEjC;;GAEG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,SAAgB,KAAK,EAAE,IAAI,CAAC;gBAEhB,KAAK,EAAE,IAAI;CAKxB;AASD;;;;;GAKG;AACH,qBAAa,OAAO;IAClB,qDAAqD;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAa;IAErC,kDAAkD;IAClD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAoB;IAEzD,yDAAyD;IACzD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAoB;IAE5C,6DAA6D;IAC7D,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAwB;IAE7D,wCAAwC;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAE/B,gDAAgD;IAChD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAE7C,mDAAmD;IACnD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmC;IAE/D,mDAAmD;IACnD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmC;IAE/D;;;;;;OAMG;gBAED,OAAO,EAAE,UAAU,EACnB,oBAAoB,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,EACvC,OAAO,EAAE,MAAM;IAyCjB;;;;;;;;OAQG;IACH,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,UAAU;IA2BvC;;;;;;OAMG;IACH,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM;IAK9B;;;;;;;OAOG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE;IA0BpC;;;;;;OAMG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC;IA4EnE;;;;;OAKG;IACH,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE;IAK7C;;;;OAIG;IACH,gBAAgB,IAAI,GAAG,CAAC,MAAM,CAAC;IAI/B;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED;;OAEG;IACH,cAAc,CAAC,KAAK,EAAE,IAAI,GAAG,OAAO;IAIpC;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,IAAI,GAAG,UAAU,GAAG,SAAS;IAIlD;;OAEG;IACH,OAAO,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,GAAG,SAAS;IAK5C;;OAEG;IACH,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;CAGrD"}