@hyvmind/tiktoken-ts 0.0.1

package/dist/tokenizer.js

@@ -0,0 +1,436 @@
+/**
+ * Main Tokenizer Implementation
+ * Provides high-level tokenization APIs for AI applications
+ */
+import { BPETokenizer } from "./bpe.js";
+import { getModelConfig, getEncodingForModel, getModelContextLimit, getModelMaxOutputTokens, } from "./models.js";
+// =============================================================================
+// Tokenizer Cache
+// =============================================================================
+/**
+ * Cache of tokenizer instances by encoding name
+ */
+const tokenizerCache = new Map();
+/**
+ * Get or create a tokenizer for the specified encoding
+ *
+ * @param encoding - Encoding name
+ * @returns Tokenizer instance
+ */
+function getOrCreateTokenizer(encoding) {
+    let tokenizer = tokenizerCache.get(encoding);
+    if (!tokenizer) {
+        const bpe = new BPETokenizer(encoding);
+        tokenizer = {
+            encodingName: encoding,
+            encode: (text) => bpe.encode(text),
+            decode: (tokens) => bpe.decode(tokens),
+            countTokens: (text) => bpe.countTokens(text),
+        };
+        tokenizerCache.set(encoding, tokenizer);
+    }
+    return tokenizer;
+}
+// =============================================================================
+// Factory Functions
+// =============================================================================
+/**
+ * Get a tokenizer for a specific encoding
+ *
+ * @param encodingName - Encoding name (cl100k_base, o200k_base, p50k_base)
+ * @returns Tokenizer instance
+ *
+ * @example
+ * ```typescript
+ * const tokenizer = getEncoding("o200k_base");
+ * const tokens = tokenizer.encode("Hello, world!");
+ * console.log(tokens.length); // ~4 tokens
+ * ```
+ */
+export function getEncoding(encodingName) {
+    return getOrCreateTokenizer(encodingName);
+}
+/**
+ * Get a tokenizer for a specific model
+ *
+ * @param model - Model name (e.g., "gpt-4o", "gpt-5-nano")
+ * @returns Tokenizer instance configured for the model
+ *
+ * @example
+ * ```typescript
+ * const tokenizer = getEncodingForModelName("gpt-4o");
+ * const count = tokenizer.countTokens("Hello, world!");
+ * console.log(count); // ~4 tokens
+ * ```
+ */
+export function getEncodingForModelName(model) {
+    const encoding = getEncodingForModel(model);
+    return getOrCreateTokenizer(encoding);
+}
+// =============================================================================
+// Token Counting Functions
+// =============================================================================
+/**
+ * Count tokens in text
+ *
+ * @param text - Input text
+ * @param options - Token counting options
+ * @returns Token count
+ *
+ * @example
+ * ```typescript
+ * // Count with default encoding (o200k_base)
+ * const count = countTokens("Hello, world!");
+ *
+ * // Count with specific model
+ * const count = countTokens("Hello, world!", { model: "gpt-4o" });
+ *
+ * // Count with specific encoding
+ * const count = countTokens("Hello, world!", { encoding: "cl100k_base" });
+ * ```
+ */
+export function countTokens(text, options) {
+    const encoding = options?.encoding ??
+        (options?.model ? getEncodingForModel(options.model) : "o200k_base");
+    const tokenizer = getOrCreateTokenizer(encoding);
+    return tokenizer.countTokens(text);
+}
+/**
+ * Count tokens in chat messages (includes message overhead)
+ *
+ * Chat messages have additional tokens for:
+ * - Role markers: <|im_start|>role\n ... <|im_end|>
+ * - Separator tokens between messages
+ *
+ * @param messages - Array of chat messages
+ * @param model - Model name for accurate counting
+ * @returns Total token count including overhead
+ *
+ * @example
+ * ```typescript
+ * const messages = [
+ *   { role: "system", content: "You are helpful." },
+ *   { role: "user", content: "Hello!" }
+ * ];
+ * const count = countChatTokens(messages, "gpt-4o");
+ * ```
+ */
+export function countChatTokens(messages, model = "gpt-4o") {
+    const encoding = getEncodingForModel(model);
+    const tokenizer = getOrCreateTokenizer(encoding);
+    let totalTokens = 0;
+    // Token overhead per message varies by model
+    // For GPT-4o family: 3 tokens per message (role + separators)
+    // For GPT-3.5/4: 4 tokens per message
+    const config = getModelConfig(model);
+    const tokensPerMessage = config?.family === "gpt-3.5" || config?.family === "gpt-4" ? 4 : 3;
+    for (const message of messages) {
+        // Message overhead
+        totalTokens += tokensPerMessage;
+        // Role token
+        totalTokens += 1;
+        // Content tokens
+        totalTokens += tokenizer.countTokens(message.content);
+        // Name field (if present) adds extra tokens
+        if (message.name) {
+            totalTokens += tokenizer.countTokens(message.name) + 1;
+        }
+    }
+    // Assistant priming (3 tokens for "<|im_start|>assistant\n")
+    totalTokens += 3;
+    return totalTokens;
+}
+/**
+ * Count tokens in a prompt with system message
+ * Convenience function for common AI playground use case
+ *
+ * @param systemPrompt - System prompt content
+ * @param userPrompt - User prompt content
+ * @param model - Model name
+ * @returns Total token count
+ */
+export function countPromptTokens(systemPrompt, userPrompt, model = "gpt-4o") {
+    const messages = [];
+    if (systemPrompt) {
+        messages.push({ role: "system", content: systemPrompt });
+    }
+    messages.push({ role: "user", content: userPrompt });
+    return countChatTokens(messages, model);
+}
+// =============================================================================
+// Max Tokens Estimation
+// =============================================================================
+/**
+ * Default options for max tokens estimation
+ */
+const DEFAULT_MAX_TOKENS_OPTIONS = {
+    desiredOutputTokens: 1000,
+    safetyMargin: 0.1,
+    minOutputTokens: 100,
+    maxOutputTokensCap: 16384,
+};
+/**
+ * Estimate a safe max_tokens value to avoid truncation
+ *
+ * This function helps prevent the common issue where max_tokens is set too low,
+ * causing the API to return an empty response with finish_reason: "length".
+ *
+ * @param promptText - The full prompt text (or use countChatTokens for messages)
+ * @param model - Model name (e.g., "gpt-4o", "gpt-5-nano")
+ * @param options - Estimation options
+ * @returns Recommended max_tokens value
+ *
+ * @example
+ * ```typescript
+ * const prompt = "Write a story about a robot.";
+ * const maxTokens = estimateMaxTokens(prompt, "gpt-5-nano", {
+ *   desiredOutputTokens: 500,
+ *   safetyMargin: 0.15
+ * });
+ * // Returns a value that ensures the response won't be truncated
+ * ```
+ */
+export function estimateMaxTokens(promptText, model, options) {
+    const opts = { ...DEFAULT_MAX_TOKENS_OPTIONS, ...options };
+    // Get model configuration
+    const contextLimit = getModelContextLimit(model);
+    const modelMaxOutput = getModelMaxOutputTokens(model);
+    // Count prompt tokens
+    const promptTokens = countTokens(promptText, { model });
+    // Calculate available tokens (context - prompt - safety buffer)
+    const safetyBuffer = Math.ceil(promptTokens * opts.safetyMargin);
+    const availableTokens = contextLimit - promptTokens - safetyBuffer;
+    // Apply constraints
+    let recommendedMaxTokens = Math.min(opts.desiredOutputTokens, availableTokens, modelMaxOutput, opts.maxOutputTokensCap);
+    // Ensure minimum
+    recommendedMaxTokens = Math.max(recommendedMaxTokens, opts.minOutputTokens);
+    return Math.round(recommendedMaxTokens);
+}
+/**
+ * Get detailed token estimation with warnings
+ *
+ * @param promptText - Full prompt text
+ * @param model - Model name
+ * @param options - Estimation options
+ * @returns Detailed token estimation
+ *
+ * @example
+ * ```typescript
+ * const estimation = getTokenEstimation(longPrompt, "gpt-4o", {
+ *   desiredOutputTokens: 2000
+ * });
+ *
+ * if (!estimation.fitsInContext) {
+ *   console.error(estimation.warning);
+ * }
+ *
+ * const response = await openai.chat.completions.create({
+ *   model: "gpt-4o",
+ *   messages: [...],
+ *   max_tokens: estimation.recommendedMaxTokens
+ * });
+ * ```
+ */
+export function getTokenEstimation(promptText, model, options) {
+    const opts = { ...DEFAULT_MAX_TOKENS_OPTIONS, ...options };
+    // Get model configuration
+    const contextLimit = getModelContextLimit(model);
+    const modelMaxOutput = getModelMaxOutputTokens(model);
+    // Count prompt tokens
+    const promptTokens = countTokens(promptText, { model });
+    // Calculate safety buffer
+    const safetyBuffer = Math.ceil(promptTokens * opts.safetyMargin);
+    // Calculate available tokens for output
+    const availableOutputTokens = Math.max(0, contextLimit - promptTokens - safetyBuffer);
+    // Check if prompt fits in context
+    const fitsInContext = promptTokens < contextLimit - opts.minOutputTokens;
+    // Calculate recommended max_tokens
+    let recommendedMaxTokens = Math.min(opts.desiredOutputTokens, availableOutputTokens, modelMaxOutput, opts.maxOutputTokensCap);
+    recommendedMaxTokens = Math.max(recommendedMaxTokens, opts.minOutputTokens);
+    // Generate warning if needed
+    let warning;
+    if (!fitsInContext) {
+        warning = `Prompt (${promptTokens} tokens) exceeds context limit (${contextLimit}) minus minimum output (${opts.minOutputTokens}). Response may be truncated or fail.`;
+    }
+    else if (availableOutputTokens < opts.desiredOutputTokens) {
+        warning = `Desired output (${opts.desiredOutputTokens} tokens) reduced to ${recommendedMaxTokens} due to context constraints.`;
+    }
+    else if (recommendedMaxTokens === opts.minOutputTokens) {
+        warning = `Output limited to minimum (${opts.minOutputTokens} tokens). Consider reducing prompt size.`;
+    }
+    return {
+        promptTokens,
+        recommendedMaxTokens: Math.round(recommendedMaxTokens),
+        contextLimit,
+        fitsInContext,
+        availableOutputTokens,
+        warning,
+    };
+}
+/**
+ * Get token estimation for chat messages
+ *
+ * @param messages - Array of chat messages
+ * @param model - Model name
+ * @param options - Estimation options
+ * @returns Detailed token estimation
+ */
+export function getChatTokenEstimation(messages, model, options) {
+    const opts = { ...DEFAULT_MAX_TOKENS_OPTIONS, ...options };
+    // Get model configuration
+    const contextLimit = getModelContextLimit(model);
+    const modelMaxOutput = getModelMaxOutputTokens(model);
+    // Count prompt tokens (with chat overhead)
+    const promptTokens = countChatTokens(messages, model);
+    // Calculate safety buffer
+    const safetyBuffer = Math.ceil(promptTokens * opts.safetyMargin);
+    // Calculate available tokens for output
+    const availableOutputTokens = Math.max(0, contextLimit - promptTokens - safetyBuffer);
+    // Check if messages fit in context
+    const fitsInContext = promptTokens < contextLimit - opts.minOutputTokens;
+    // Calculate recommended max_tokens
+    let recommendedMaxTokens = Math.min(opts.desiredOutputTokens, availableOutputTokens, modelMaxOutput, opts.maxOutputTokensCap);
+    recommendedMaxTokens = Math.max(recommendedMaxTokens, opts.minOutputTokens);
+    // Generate warning if needed
+    let warning;
+    if (!fitsInContext) {
+        warning = `Messages (${promptTokens} tokens) exceed context limit (${contextLimit}) minus minimum output (${opts.minOutputTokens}). Response may be truncated or fail.`;
+    }
+    else if (availableOutputTokens < opts.desiredOutputTokens) {
+        warning = `Desired output (${opts.desiredOutputTokens} tokens) reduced to ${recommendedMaxTokens} due to context constraints.`;
+    }
+    return {
+        promptTokens,
+        recommendedMaxTokens: Math.round(recommendedMaxTokens),
+        contextLimit,
+        fitsInContext,
+        availableOutputTokens,
+        warning,
+    };
+}
+// =============================================================================
+// Utility Functions
+// =============================================================================
+/**
+ * Check if text will fit within model context
+ *
+ * @param text - Input text
+ * @param model - Model name
+ * @param reservedOutputTokens - Tokens to reserve for output
+ * @returns True if text fits within context
+ */
+export function fitsInContext(text, model, reservedOutputTokens = 1000) {
+    const tokens = countTokens(text, { model });
+    const contextLimit = getModelContextLimit(model);
+    return tokens + reservedOutputTokens <= contextLimit;
+}
+/**
+ * Truncate text to fit within a token limit
+ *
+ * @param text - Input text
+ * @param maxTokens - Maximum tokens allowed
+ * @param model - Model name for accurate counting
+ * @returns Truncated text
+ *
+ * @example
+ * ```typescript
+ * const truncated = truncateToTokenLimit(longText, 1000, "gpt-4o");
+ * console.log(countTokens(truncated, { model: "gpt-4o" })); // <= 1000
+ * ```
+ */
+export function truncateToTokenLimit(text, maxTokens, model = "gpt-4o") {
+    const encoding = getEncodingForModel(model);
+    const tokenizer = getOrCreateTokenizer(encoding);
+    const currentTokens = tokenizer.countTokens(text);
+    if (currentTokens <= maxTokens) {
+        return text;
+    }
+    // Binary search for the right truncation point
+    let low = 0;
+    let high = text.length;
+    let result = "";
+    while (low < high) {
+        const mid = Math.floor((low + high + 1) / 2);
+        const truncated = text.slice(0, mid);
+        const tokens = tokenizer.countTokens(truncated);
+        if (tokens <= maxTokens) {
+            result = truncated;
+            low = mid;
+        }
+        else {
+            high = mid - 1;
+        }
+    }
+    // Add ellipsis if truncated
+    if (result.length < text.length) {
+        // Remove last few characters to make room for ellipsis
+        while (result.length > 0 &&
+            tokenizer.countTokens(result + "...") > maxTokens) {
+            result = result.slice(0, -1);
+        }
+        result += "...";
+    }
+    return result;
+}
+/**
+ * Split text into chunks that fit within token limits
+ *
+ * @param text - Input text
+ * @param maxTokensPerChunk - Maximum tokens per chunk
+ * @param overlap - Number of characters to overlap between chunks
+ * @param model - Model name
+ * @returns Array of text chunks
+ */
+export function splitIntoChunks(text, maxTokensPerChunk, overlap = 200, model = "gpt-4o") {
+    // If text fits in one chunk, return it directly
+    const totalTokens = countTokens(text, { model });
+    if (totalTokens <= maxTokensPerChunk) {
+        return [text.trim()].filter((s) => s.length > 0);
+    }
+    const chunks = [];
+    let remaining = text;
+    while (remaining.length > 0) {
+        // Find the maximum chunk that fits
+        let chunkEnd = remaining.length;
+        let chunk = remaining.slice(0, chunkEnd);
+        while (chunkEnd > 0 && countTokens(chunk, { model }) > maxTokensPerChunk) {
+            // Try to break at a sentence or paragraph boundary
+            const breakPoints = ["\n\n", "\n", ". ", "! ", "? ", ", ", " "];
+            let foundBreak = false;
+            for (const bp of breakPoints) {
+                const lastBreak = chunk.lastIndexOf(bp, chunkEnd - 1);
+                if (lastBreak > 0) {
+                    chunkEnd = lastBreak + bp.length;
+                    chunk = remaining.slice(0, chunkEnd);
+                    if (countTokens(chunk, { model }) <= maxTokensPerChunk) {
+                        foundBreak = true;
+                        break;
+                    }
+                }
+            }
+            if (!foundBreak) {
+                // No good break point, just truncate
+                chunkEnd = Math.floor(chunkEnd * 0.8);
+                chunk = remaining.slice(0, chunkEnd);
+            }
+        }
+        const trimmedChunk = chunk.trim();
+        if (trimmedChunk.length > 0) {
+            chunks.push(trimmedChunk);
+        }
+        // Move position forward, accounting for overlap
+        // Ensure we always make progress (minimum advance is half of chunk or 1)
+        const minAdvance = Math.max(Math.floor(chunkEnd / 2), 1);
+        const advance = Math.max(chunkEnd - overlap, minAdvance);
+        remaining = remaining.slice(advance);
+    }
+    return chunks;
+}
+/**
+ * Clear the tokenizer cache
+ * Useful for testing or memory management
+ */
+export function clearTokenizerCache() {
+    tokenizerCache.clear();
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAUH,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AACxC,OAAO,EACL,cAAc,EACd,mBAAmB,EACnB,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,aAAa,CAAC;AAErB,gFAAgF;AAChF,kBAAkB;AAClB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,cAAc,GAAiC,IAAI,GAAG,EAAE,CAAC;AAE/D;;;;;GAKG;AACH,SAAS,oBAAoB,CAAC,QAAsB;IAClD,IAAI,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAE7C,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,QAAQ,CAAC,CAAC;QACvC,SAAS,GAAG;YACV,YAAY,EAAE,QAAQ;YACtB,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;YAC1C,MAAM,EAAE,CAAC,MAAgB,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;YAChD,WAAW,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC;SACrD,CAAC;QACF,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,WAAW,CAAC,YAA0B;IACpD,OAAO,oBAAoB,CAAC,YAAY,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAa;IACnD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC5C,OAAO,oBAAoB,CAAC,QAAQ,CAAC,CAAC;AACxC,CAAC;AAED,gFAAgF;AAChF,2BAA2B;AAC3B,gFAAgF;AAEhF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,OAA2B;IACnE,MAAM,QAAQ,GACZ,OAAO,EAAE,QAAQ;QACjB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,mBAAmB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IACvE,MAAM,SAAS,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAEjD,OAAO,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AACrC,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,eAAe,CAC7B,QAAuB,EACvB,QAAgB,QAAQ;IAExB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAEjD,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,6CAA6C;IAC7C,8DAA8D;IAC9D,sCAAsC;IACtC,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACrC,MAAM,gBAAgB,GACpB,MAAM,EAAE,MAAM,KAAK,SAAS,IAAI,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAErE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,mBAAmB;QACnB,WAAW,IAAI,gBAAgB,CAAC;QAEhC,aAAa;QACb,WAAW,IAAI,CAAC,CAAC;QAEjB,iBAAiB;QACjB,WAAW,IAAI,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEtD,4CAA4C;QAC5C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,WAAW,IAAI,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,WAAW,IAAI,CAAC,CAAC;IAEjB,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAC/B,YAAoB,EACpB,UAAkB,EAClB,QAAgB,QAAQ;IAExB,MAAM,QAAQ,GAAkB,EAAE,CAAC;IAEnC,IAAI,YAAY,EAAE,CAAC;QACjB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;IAErD,OAAO,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;AAC1C,CAAC;AAED,gFAAgF;AAChF,wBAAwB;AACxB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,0BAA0B,GAA+B;IAC7D,mBAAmB,EAAE,IAAI;IACzB,YAAY,EAAE,GAAG;IACjB,eAAe,EAAE,GAAG;IACpB,kBAAkB,EAAE,KAAK;CAC1B,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,iBAAiB,CAC/B,UAAkB,EAClB,KAAa,EACb,OAA0B;IAE1B,MAAM,IAAI,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAC;IAE3D,0BAA0B;IAC1B,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IACjD,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;IAEtD,sBAAsB;IACtB,MAAM,YAAY,GAAG,WAAW,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAExD,gEAAgE;IAChE,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC;IACjE,MAAM,eAAe,GAAG,YAAY,GAAG,YAAY,GAAG,YAAY,CAAC;IAEnE,oBAAoB;IACpB,IAAI,oBAAoB,GAAG,IAAI,CAAC,GAAG,CACjC,IAAI,CAAC,mBAAmB,EACxB,eAAe,EACf,cAAc,EACd,IAAI,CAAC,kBAAkB,CACxB,CAAC;IAEF,iBAAiB;IACjB,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;IAE5E,OAAO,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,UAAU,kBAAkB,CAChC,UAAkB,EAClB,KAAa,EACb,OAA0B;IAE1B,MAAM,IAAI,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAC;IAE3D,0BAA0B;IAC1B,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IACjD,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;IAEtD,sBAAsB;IACtB,MAAM,YAAY,GAAG,WAAW,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAExD,0BAA0B;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC;IAEjE,wCAAwC;IACxC,MAAM,qBAAqB,GAAG,IAAI,CAAC,GAAG,CACpC,CAAC,EACD,YAAY,GAAG,YAAY,GAAG,YAAY,CAC3C,CAAC;IAEF,kCAAkC;IAClC,MAAM,aAAa,GAAG,YAAY,GAAG,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC;IAEzE,mCAAmC;IACnC,IAAI,oBAAoB,GAAG,IAAI,CAAC,GAAG,CACjC,IAAI,CAAC,mBAAmB,EACxB,qBAAqB,EACrB,cAAc,EACd,IAAI,CAAC,kBAAkB,CACxB,CAAC;IAEF,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;IAE5E,6BAA6B;IAC7B,IAAI,OAA2B,CAAC;IAEhC,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,WAAW,YAAY,mCAAmC,YAAY,2BAA2B,IAAI,CAAC,eAAe,uCAAuC,CAAC;IACzK,CAAC;SAAM,IAAI,qBAAqB,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC5D,OAAO,GAAG,mBAAmB,IAAI,CAAC,mBAAmB,uBAAuB,oBAAoB,8BAA8B,CAAC;IACjI,CAAC;SAAM,IAAI,oBAAoB,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;QACzD,OAAO,GAAG,8BAA8B,IAAI,CAAC,eAAe,0CAA0C,CAAC;IACzG,CAAC;IAED,OAAO;QACL,YAAY;QACZ,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC;QACtD,YAAY;QACZ,aAAa;QACb,qBAAqB;QACrB,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CACpC,QAAuB,EACvB,KAAa,EACb,OAA0B;IAE1B,MAAM,IAAI,GAAG,EAAE,GAAG,0BAA0B,EAAE,GAAG,OAAO,EAAE,CAAC;IAE3D,0BAA0B;IAC1B,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IACjD,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;IAEtD,2CAA2C;IAC3C,MAAM,YAAY,GAAG,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAEtD,0BAA0B;IAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC;IAEjE,wCAAwC;IACxC,MAAM,qBAAqB,GAAG,IAAI,CAAC,GAAG,CACpC,CAAC,EACD,YAAY,GAAG,YAAY,GAAG,YAAY,CAC3C,CAAC;IAEF,mCAAmC;IACnC,MAAM,aAAa,GAAG,YAAY,GAAG,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC;IAEzE,mCAAmC;IACnC,IAAI,oBAAoB,GAAG,IAAI,CAAC,GAAG,CACjC,IAAI,CAAC,mBAAmB,EACxB,qBAAqB,EACrB,cAAc,EACd,IAAI,CAAC,kBAAkB,CACxB,CAAC;IAEF,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;IAE5E,6BAA6B;IAC7B,IAAI,OAA2B,CAAC;IAEhC,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,aAAa,YAAY,kCAAkC,YAAY,2BAA2B,IAAI,CAAC,eAAe,uCAAuC,CAAC;IAC1K,CAAC;SAAM,IAAI,qBAAqB,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC5D,OAAO,GAAG,mBAAmB,IAAI,CAAC,mBAAmB,uBAAuB,oBAAoB,8BAA8B,CAAC;IACjI,CAAC;IAED,OAAO;QACL,YAAY;QACZ,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC;QACtD,YAAY;QACZ,aAAa;QACb,qBAAqB;QACrB,OAAO;KACR,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAC3B,IAAY,EACZ,KAAa,EACb,uBAA+B,IAAI;IAEnC,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5C,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAEjD,OAAO,MAAM,GAAG,oBAAoB,IAAI,YAAY,CAAC;AACvD,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAClC,IAAY,EACZ,SAAiB,EACjB,QAAgB,QAAQ;IAExB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAEjD,MAAM,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAElD,IAAI,aAAa,IAAI,SAAS,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,+CAA+C;IAC/C,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC;IACvB,IAAI,MAAM,GAAG,EAAE,CAAC;IAEhB,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAEhD,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YACxB,MAAM,GAAG,SAAS,CAAC;YACnB,GAAG,GAAG,GAAG,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,uDAAuD;QACvD,OACE,MAAM,CAAC,MAAM,GAAG,CAAC;YACjB,SAAS,CAAC,WAAW,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,SAAS,EACjD,CAAC;YACD,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;QACD,MAAM,IAAI,KAAK,CAAC;IAClB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAC7B,IAAY,EACZ,iBAAyB,EACzB,UAAkB,GAAG,EACrB,QAAgB,QAAQ;IAExB,gDAAgD;IAChD,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IACjD,IAAI,WAAW,IAAI,iBAAiB,EAAE,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,SAAS,GAAG,IAAI,CAAC;IAErB,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,mCAAmC;QACnC,IAAI,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;QAChC,IAAI,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAEzC,OAAO,QAAQ,GAAG,CAAC,IAAI,WAAW,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,iBAAiB,EAAE,CAAC;YACzE,mDAAmD;YACnD,MAAM,WAAW,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;YAEhE,IAAI,UAAU,GAAG,KAAK,CAAC;YACvB,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBAClB,QAAQ,GAAG,SAAS,GAAG,EAAE,CAAC,MAAM,CAAC;oBACjC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;oBACrC,IAAI,WAAW,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,IAAI,iBAAiB,EAAE,CAAC;wBACvD,UAAU,GAAG,IAAI,CAAC;wBAClB,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,qCAAqC;gBACrC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;gBACtC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAClC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5B,CAAC;QAED,gDAAgD;QAChD,yEAAyE;QACzE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,OAAO,EAAE,UAAU,CAAC,CAAC;QACzD,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB;IACjC,cAAc,CAAC,KAAK,EAAE,CAAC;AACzB,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,127 @@
+/**
+ * Tokenizer Types
+ * Core interfaces for the tiktoken-ts package
+ */
+/**
+ * Encoding names supported by the tokenizer
+ * - r50k_base: GPT-2, GPT-3 davinci/curie/babbage/ada
+ * - p50k_base: Codex models, text-davinci-002/003
+ * - p50k_edit: Edit models (text-davinci-edit-001, code-davinci-edit-001)
+ * - cl100k_base: GPT-4, GPT-3.5-turbo, text-embedding-ada-002
+ * - o200k_base: GPT-4o, GPT-4.1, GPT-5, o-series models
+ * - o200k_harmony: gpt-oss models
+ * - claude_estimation: Claude models (estimation only, errs on over-counting for safety)
+ */
+export type EncodingName = "r50k_base" | "p50k_base" | "p50k_edit" | "cl100k_base" | "o200k_base" | "o200k_harmony" | "claude_estimation" | "gpt2";
+/**
+ * Model family classification for encoding selection
+ */
+export type ModelFamily = "gpt-5" | "gpt-4.1" | "gpt-4o" | "gpt-4" | "gpt-3.5" | "o-series" | "codex" | "embedding" | "deepseek" | "gemini" | "claude" | "unknown";
+/**
+ * Special token types used in chat format
+ */
+export interface SpecialTokens {
+    readonly endOfText: string;
+    readonly imStart: string;
+    readonly imEnd: string;
+    readonly imSep: string;
+    readonly toolCall?: string;
+    readonly toolResult?: string;
+}
+/**
+ * Configuration for an encoding
+ */
+export interface EncodingConfig {
+    readonly name: EncodingName;
+    readonly patternSplit: RegExp;
+    readonly specialTokens: SpecialTokens;
+    readonly vocabSize: number;
+    readonly averageCharsPerToken: number;
+}
+/**
+ * Core tokenizer interface
+ */
+export interface Tokenizer {
+    /**
+     * Encode text into token IDs
+     * @param text - Input text to encode
+     * @returns Array of token IDs
+     */
+    encode(text: string): number[];
+    /**
+     * Decode token IDs back to text
+     * @param tokens - Array of token IDs
+     * @returns Decoded text
+     */
+    decode(tokens: number[]): string;
+    /**
+     * Count the number of tokens in text
+     * @param text - Input text
+     * @returns Token count
+     */
+    countTokens(text: string): number;
+    /**
+     * Get the encoding name
+     */
+    readonly encodingName: EncodingName;
+}
+/**
+ * Model context configuration
+ */
+export interface ModelConfig {
+    readonly name: string;
+    readonly encoding: EncodingName;
+    readonly contextLimit: number;
+    readonly maxOutputTokens: number;
+    readonly family: ModelFamily;
+}
+/**
+ * Token estimation result
+ */
+export interface TokenEstimation {
+    /** Estimated prompt tokens */
+    promptTokens: number;
+    /** Recommended max_tokens value */
+    recommendedMaxTokens: number;
+    /** Model context limit */
+    contextLimit: number;
+    /** Whether the prompt fits in context */
+    fitsInContext: boolean;
+    /** Available tokens for output */
+    availableOutputTokens: number;
+    /** Warning message if any */
+    warning?: string;
+}
+/**
+ * Chat message for token counting
+ */
+export interface ChatMessage {
+    role: "system" | "user" | "assistant" | "tool";
+    content: string;
+    name?: string;
+}
+/**
+ * Token counting options
+ */
+export interface TokenCountOptions {
+    /** Model name for accurate counting */
+    model?: string;
+    /** Encoding name override */
+    encoding?: EncodingName;
+    /** Include special tokens in count */
+    includeSpecialTokens?: boolean;
+}
+/**
+ * Max tokens estimation options
+ */
+export interface MaxTokensOptions {
+    /** Desired output token count */
+    desiredOutputTokens?: number;
+    /** Safety margin percentage (0.0 to 1.0) */
+    safetyMargin?: number;
+    /** Minimum output tokens to reserve */
+    minOutputTokens?: number;
+    /** Maximum output tokens cap */
+    maxOutputTokensCap?: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GACpB,WAAW,GACX,WAAW,GACX,WAAW,GACX,aAAa,GACb,YAAY,GACZ,eAAe,GACf,mBAAmB,GACnB,MAAM,CAAC;AAEX;;GAEG;AACH,MAAM,MAAM,WAAW,GACnB,OAAO,GACP,SAAS,GACT,QAAQ,GACR,OAAO,GACP,SAAS,GACT,UAAU,GACV,OAAO,GACP,WAAW,GACX,UAAU,GACV,QAAQ,GACR,QAAQ,GACR,SAAS,CAAC;AAEd;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,aAAa,CAAC;IACtC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;;OAIG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE/B;;;;OAIG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAEjC;;;;OAIG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAElC;;OAEG;IACH,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,8BAA8B;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,mCAAmC;IACnC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,0BAA0B;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,aAAa,EAAE,OAAO,CAAC;IACvB,kCAAkC;IAClC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,6BAA6B;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,MAAM,CAAC;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6BAA6B;IAC7B,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,sCAAsC;IACtC,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,iCAAiC;IACjC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,4CAA4C;IAC5C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uCAAuC;IACvC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gCAAgC;IAChC,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B"}

package/dist/types.js

@@ -0,0 +1,6 @@
+/**
+ * Tokenizer Types
+ * Core interfaces for the tiktoken-ts package
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG"}