@hyvmind/tiktoken-ts 0.0.1

package/dist/tiktoken.js

@@ -0,0 +1,331 @@
+/**
+ * High-Level Tiktoken API
+ *
+ * Provides easy-to-use tokenization APIs with automatic vocabulary loading.
+ *
+ * This module is the main entry point for most users. It provides:
+ * - Lazy loading of vocabularies
+ * - Caching of tokenizer instances
+ * - Support for both sync and async operations
+ * - Model name resolution
+ */
+import { CoreBPE, getEncodingDefinition, loadVocabularyFromUrl, loadVocabularyFromString, getVocabularyFromCache, isVocabularyCached, getTokenizerForModel, } from "./core/index.js";
+/**
+ * Tiktoken encoding instance
+ *
+ * This class wraps CoreBPE and provides:
+ * - Lazy initialization
+ * - Convenient API matching tiktoken-rs
+ * - Error handling for unloaded vocabularies
+ */
+export class Tiktoken {
+    /** The encoding name */
+    name;
+    /** The encoding definition */
+    definition;
+    /** The underlying CoreBPE instance (lazy loaded) */
+    coreBPE = null;
+    /** Promise for async initialization */
+    initPromise = null;
+    /** Whether the vocabulary has been loaded */
+    isLoaded = false;
+    /**
+     * Create a new Tiktoken instance
+     *
+     * Note: The vocabulary is NOT loaded until you call load() or one of the
+     * encoding methods with a loaded vocabulary.
+     *
+     * @param encodingName - The encoding name (e.g., "cl100k_base", "o200k_base")
+     */
+    constructor(encodingName) {
+        const definition = getEncodingDefinition(encodingName);
+        if (!definition) {
+            throw new Error(`Unknown encoding: ${encodingName}`);
+        }
+        this.name = encodingName;
+        this.definition = definition;
+        // Check if vocabulary is already cached
+        if (isVocabularyCached(encodingName)) {
+            this.initFromCache();
+        }
+    }
+    /**
+     * Initialize from cached vocabulary
+     */
+    initFromCache() {
+        const vocab = getVocabularyFromCache(this.name);
+        if (vocab) {
+            this.initCoreBPE(vocab.encoder, vocab.decoder);
+        }
+    }
+    /**
+     * Initialize the CoreBPE instance
+     */
+    initCoreBPE(encoder, _decoder) {
+        // Note: decoder is provided by vocab loader but CoreBPE builds its own from encoder
+        // The _decoder parameter is kept for API consistency with vocabulary loading
+        // Create special tokens map
+        const specialTokensEncoder = new Map();
+        for (const [token, rank] of Object.entries(this.definition.specialTokens)) {
+            specialTokensEncoder.set(token, rank);
+        }
+        this.coreBPE = new CoreBPE(encoder, specialTokensEncoder, this.definition.pattern);
+        this.isLoaded = true;
+    }
+    /**
+     * Load the vocabulary from URL
+     *
+     * @returns Promise that resolves when loaded
+     */
+    async load() {
+        if (this.isLoaded) {
+            return;
+        }
+        if (this.initPromise) {
+            return this.initPromise;
+        }
+        this.initPromise = (async () => {
+            const vocab = await loadVocabularyFromUrl(this.name);
+            this.initCoreBPE(vocab.encoder, vocab.decoder); // decoder used internally in initCoreBPE
+        })();
+        return this.initPromise;
+    }
+    /**
+     * Load the vocabulary from a string (for embedded vocabularies)
+     *
+     * @param content - The vocabulary file content
+     */
+    loadFromString(content) {
+        if (this.isLoaded) {
+            return;
+        }
+        const vocab = loadVocabularyFromString(this.name, content);
+        this.initCoreBPE(vocab.encoder, vocab.decoder);
+    }
+    /**
+     * Ensure the vocabulary is loaded
+     */
+    ensureLoaded() {
+        if (!this.coreBPE) {
+            throw new Error(`Vocabulary not loaded for ${this.name}. Call load() first or use the async API.`);
+        }
+        return this.coreBPE;
+    }
+    /**
+     * Check if the vocabulary is loaded
+     */
+    get loaded() {
+        return this.isLoaded;
+    }
+    // =========================================================================
+    // Encoding Methods
+    // =========================================================================
+    /**
+     * Encode text into token IDs without handling special tokens
+     *
+     * @param text - Text to encode
+     * @returns Array of token IDs
+     */
+    encodeOrdinary(text) {
+        return this.ensureLoaded().encodeOrdinary(text);
+    }
+    /**
+     * Encode text into token IDs
+     *
+     * @param text - Text to encode
+     * @param allowedSpecial - Special tokens to allow (default: none)
+     * @returns Array of token IDs
+     */
+    encode(text, allowedSpecial) {
+        const bpe = this.ensureLoaded();
+        if (allowedSpecial === "all") {
+            return bpe.encodeWithSpecialTokens(text);
+        }
+        const allowed = allowedSpecial ?? new Set();
+        return bpe.encode(text, allowed)[0];
+    }
+    /**
+     * Encode text with all special tokens allowed
+     *
+     * @param text - Text to encode
+     * @returns Array of token IDs
+     */
+    encodeWithSpecialTokens(text) {
+        return this.ensureLoaded().encodeWithSpecialTokens(text);
+    }
+    // =========================================================================
+    // Decoding Methods
+    // =========================================================================
+    /**
+     * Decode token IDs back to text
+     *
+     * @param tokens - Array of token IDs
+     * @returns Decoded text
+     */
+    decode(tokens) {
+        return this.ensureLoaded().decode(tokens);
+    }
+    /**
+     * Decode token IDs to raw bytes
+     *
+     * @param tokens - Array of token IDs
+     * @returns Decoded bytes
+     */
+    decodeBytes(tokens) {
+        return this.ensureLoaded().decodeBytes(tokens);
+    }
+    // =========================================================================
+    // Utility Methods
+    // =========================================================================
+    /**
+     * Count tokens in text (without returning the tokens)
+     *
+     * @param text - Text to count
+     * @returns Number of tokens
+     */
+    countTokens(text) {
+        return this.encodeOrdinary(text).length;
+    }
+    /**
+     * Get the vocabulary size (excluding special tokens)
+     */
+    get vocabSize() {
+        return this.ensureLoaded().vocabSize;
+    }
+    /**
+     * Get the total vocabulary size (including special tokens)
+     */
+    get totalVocabSize() {
+        return this.ensureLoaded().totalVocabSize;
+    }
+    /**
+     * Get all special tokens
+     */
+    getSpecialTokens() {
+        return this.ensureLoaded().getSpecialTokens();
+    }
+    /**
+     * Check if a token is a special token
+     */
+    isSpecialToken(token) {
+        return this.ensureLoaded().isSpecialToken(token);
+    }
+    /**
+     * Get the definition for this encoding
+     */
+    getDefinition() {
+        return this.definition;
+    }
+}
+// =========================================================================
+// Factory Functions
+// =========================================================================
+/**
+ * Cache of Tiktoken instances
+ */
+const tiktokenCache = new Map();
+/**
+ * Get or create a Tiktoken instance for an encoding
+ *
+ * @param encodingName - Encoding name
+ * @returns Tiktoken instance (may not be loaded yet)
+ */
+export function getEncoding(encodingName) {
+    let instance = tiktokenCache.get(encodingName);
+    if (!instance) {
+        instance = new Tiktoken(encodingName);
+        tiktokenCache.set(encodingName, instance);
+    }
+    return instance;
+}
+/**
+ * Get or create a Tiktoken instance for a model
+ *
+ * @param modelName - Model name
+ * @returns Tiktoken instance (may not be loaded yet)
+ * @throws Error if no encoding is found for the model
+ */
+export function getEncodingForModel(modelName) {
+    const encoding = getTokenizerForModel(modelName);
+    if (!encoding) {
+        // Default to o200k_base for unknown models (most modern models use this)
+        return getEncoding("o200k_base");
+    }
+    return getEncoding(encoding);
+}
+/**
+ * Get a loaded Tiktoken instance for an encoding (async)
+ *
+ * @param encodingName - Encoding name
+ * @returns Promise resolving to a loaded Tiktoken instance
+ */
+export async function getEncodingAsync(encodingName) {
+    const instance = getEncoding(encodingName);
+    await instance.load();
+    return instance;
+}
+/**
+ * Get a loaded Tiktoken instance for a model (async)
+ *
+ * @param modelName - Model name
+ * @returns Promise resolving to a loaded Tiktoken instance
+ */
+export async function getEncodingForModelAsync(modelName) {
+    const instance = getEncodingForModel(modelName);
+    await instance.load();
+    return instance;
+}
+/**
+ * Clear the Tiktoken instance cache
+ */
+export function clearTiktokenCache() {
+    tiktokenCache.clear();
+}
+// =========================================================================
+// Convenience Functions
+// =========================================================================
+/**
+ * Encode text using a specific encoding (async)
+ *
+ * @param text - Text to encode
+ * @param encodingName - Encoding name
+ * @returns Promise resolving to token IDs
+ */
+export async function encodeAsync(text, encodingName = "o200k_base") {
+    const tiktoken = await getEncodingAsync(encodingName);
+    return tiktoken.encodeOrdinary(text);
+}
+/**
+ * Decode tokens using a specific encoding (async)
+ *
+ * @param tokens - Token IDs
+ * @param encodingName - Encoding name
+ * @returns Promise resolving to decoded text
+ */
+export async function decodeAsync(tokens, encodingName = "o200k_base") {
+    const tiktoken = await getEncodingAsync(encodingName);
+    return tiktoken.decode(tokens);
+}
+/**
+ * Count tokens in text (async)
+ *
+ * @param text - Text to count
+ * @param encodingName - Encoding name
+ * @returns Promise resolving to token count
+ */
+export async function countTokensAsync(text, encodingName = "o200k_base") {
+    const tiktoken = await getEncodingAsync(encodingName);
+    return tiktoken.countTokens(text);
+}
+/**
+ * Count tokens for a model (async)
+ *
+ * @param text - Text to count
+ * @param modelName - Model name
+ * @returns Promise resolving to token count
+ */
+export async function countTokensForModelAsync(text, modelName) {
+    const tiktoken = await getEncodingForModelAsync(modelName);
+    return tiktoken.countTokens(text);
+}
+//# sourceMappingURL=tiktoken.js.map

package/dist/tiktoken.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tiktoken.js","sourceRoot":"","sources":["../src/tiktoken.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,OAAO,EAIP,qBAAqB,EAErB,qBAAqB,EACrB,wBAAwB,EACxB,sBAAsB,EACtB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,iBAAiB,CAAC;AAEzB;;;;;;;GAOG;AACH,MAAM,OAAO,QAAQ;IACnB,wBAAwB;IACR,IAAI,CAAS;IAE7B,8BAA8B;IACb,UAAU,CAAqB;IAEhD,oDAAoD;IAC5C,OAAO,GAAmB,IAAI,CAAC;IAEvC,uCAAuC;IAC/B,WAAW,GAAyB,IAAI,CAAC;IAEjD,6CAA6C;IACrC,QAAQ,GAAG,KAAK,CAAC;IAEzB;;;;;;;OAOG;IACH,YAAY,YAAoB;QAC9B,MAAM,UAAU,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;QACvD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,qBAAqB,YAAY,EAAE,CAAC,CAAC;QACvD,CAAC;QAED,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAE7B,wCAAwC;QACxC,IAAI,kBAAkB,CAAC,YAAY,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,OAAmB,EAAE,QAA2B;QAClE,oFAAoF;QACpF,6EAA6E;QAE7E,4BAA4B;QAC5B,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAgB,CAAC;QACrD,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;YAC1E,oBAAoB,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CACxB,OAAO,EACP,oBAAoB,EACpB,IAAI,CAAC,UAAU,CAAC,OAAO,CACxB,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,WAAW,CAAC;QAC1B,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,MAAM,KAAK,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrD,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,yCAAyC;QAC3F,CAAC,CAAC,EAAE,CAAC;QAEL,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;;;OAIG;IACH,cAAc,CAAC,OAAe;QAC5B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,MAAM,KAAK,GAAG,wBAAwB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC3D,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,CAAC,IAAI,2CAA2C,CAClF,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,4EAA4E;IAC5E,mBAAmB;IACnB,4EAA4E;IAE5E;;;;;OAKG;IACH,cAAc,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,IAAY,EAAE,cAAoC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAEhC,IAAI,cAAc,KAAK,KAAK,EAAE,CAAC;YAC7B,OAAO,GAAG,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC;QAED,MAAM,OAAO,GAAG,cAAc,IAAI,IAAI,GAAG,EAAU,CAAC;QACpD,OAAO,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,CAAC;IAED;;;;;OAKG;IACH,uBAAuB,CAAC,IAAY;QAClC,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED,4EAA4E;IAC5E,mBAAmB;IACnB,4EAA4E;IAE5E;;;;;OAKG;IACH,MAAM,CAAC,MAAc;QACnB,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,MAAc;QACxB,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IAED,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E;;;;;OAKG;IACH,WAAW,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,SAAS,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,cAAc,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,gBAAgB,EAAE,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,KAAW;QACxB,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;CACF;AAED,4EAA4E;AAC5E,oBAAoB;AACpB,4EAA4E;AAE5E;;GAEG;AACH,MAAM,aAAa,GAA0B,IAAI,GAAG,EAAE,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,YAAoB;IAC9C,IAAI,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAE/C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,IAAI,QAAQ,CAAC,YAAY,CAAC,CAAC;QACtC,aAAa,CAAC,GAAG,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,SAAiB;IACnD,MAAM,QAAQ,GAAG,oBAAoB,CAAC,SAAS,CAAC,CAAC;IACjD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,yEAAyE;QACzE,OAAO,WAAW,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC/B,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,YAAoB;IAEpB,MAAM,QAAQ,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;IAC3C,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtB,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,SAAiB;IAEjB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IAChD,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtB,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,aAAa,CAAC,KAAK,EAAE,CAAC;AACxB,CAAC;AAED,4EAA4E;AAC5E,wBAAwB;AACxB,4EAA4E;AAE5E;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAY,EACZ,eAAuB,YAAY;IAEnC,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,YAAY,CAAC,CAAC;IACtD,OAAO,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAc,EACd,eAAuB,YAAY;IAEnC,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,YAAY,CAAC,CAAC;IACtD,OAAO,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,eAAuB,YAAY;IAEnC,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,YAAY,CAAC,CAAC;IACtD,OAAO,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,IAAY,EACZ,SAAiB;IAEjB,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC,SAAS,CAAC,CAAC;IAC3D,OAAO,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AACpC,CAAC"}

package/dist/tokenizer.d.ts

@@ -0,0 +1,181 @@
+/**
+ * Main Tokenizer Implementation
+ * Provides high-level tokenization APIs for AI applications
+ */
+import type { Tokenizer, EncodingName, ChatMessage, TokenCountOptions, MaxTokensOptions, TokenEstimation } from "./types.js";
+/**
+ * Get a tokenizer for a specific encoding
+ *
+ * @param encodingName - Encoding name (cl100k_base, o200k_base, p50k_base)
+ * @returns Tokenizer instance
+ *
+ * @example
+ * ```typescript
+ * const tokenizer = getEncoding("o200k_base");
+ * const tokens = tokenizer.encode("Hello, world!");
+ * console.log(tokens.length); // ~4 tokens
+ * ```
+ */
+export declare function getEncoding(encodingName: EncodingName): Tokenizer;
+/**
+ * Get a tokenizer for a specific model
+ *
+ * @param model - Model name (e.g., "gpt-4o", "gpt-5-nano")
+ * @returns Tokenizer instance configured for the model
+ *
+ * @example
+ * ```typescript
+ * const tokenizer = getEncodingForModelName("gpt-4o");
+ * const count = tokenizer.countTokens("Hello, world!");
+ * console.log(count); // ~4 tokens
+ * ```
+ */
+export declare function getEncodingForModelName(model: string): Tokenizer;
+/**
+ * Count tokens in text
+ *
+ * @param text - Input text
+ * @param options - Token counting options
+ * @returns Token count
+ *
+ * @example
+ * ```typescript
+ * // Count with default encoding (o200k_base)
+ * const count = countTokens("Hello, world!");
+ *
+ * // Count with specific model
+ * const count = countTokens("Hello, world!", { model: "gpt-4o" });
+ *
+ * // Count with specific encoding
+ * const count = countTokens("Hello, world!", { encoding: "cl100k_base" });
+ * ```
+ */
+export declare function countTokens(text: string, options?: TokenCountOptions): number;
+/**
+ * Count tokens in chat messages (includes message overhead)
+ *
+ * Chat messages have additional tokens for:
+ * - Role markers: <|im_start|>role\n ... <|im_end|>
+ * - Separator tokens between messages
+ *
+ * @param messages - Array of chat messages
+ * @param model - Model name for accurate counting
+ * @returns Total token count including overhead
+ *
+ * @example
+ * ```typescript
+ * const messages = [
+ *   { role: "system", content: "You are helpful." },
+ *   { role: "user", content: "Hello!" }
+ * ];
+ * const count = countChatTokens(messages, "gpt-4o");
+ * ```
+ */
+export declare function countChatTokens(messages: ChatMessage[], model?: string): number;
+/**
+ * Count tokens in a prompt with system message
+ * Convenience function for common AI playground use case
+ *
+ * @param systemPrompt - System prompt content
+ * @param userPrompt - User prompt content
+ * @param model - Model name
+ * @returns Total token count
+ */
+export declare function countPromptTokens(systemPrompt: string, userPrompt: string, model?: string): number;
+/**
+ * Estimate a safe max_tokens value to avoid truncation
+ *
+ * This function helps prevent the common issue where max_tokens is set too low,
+ * causing the API to return an empty response with finish_reason: "length".
+ *
+ * @param promptText - The full prompt text (or use countChatTokens for messages)
+ * @param model - Model name (e.g., "gpt-4o", "gpt-5-nano")
+ * @param options - Estimation options
+ * @returns Recommended max_tokens value
+ *
+ * @example
+ * ```typescript
+ * const prompt = "Write a story about a robot.";
+ * const maxTokens = estimateMaxTokens(prompt, "gpt-5-nano", {
+ *   desiredOutputTokens: 500,
+ *   safetyMargin: 0.15
+ * });
+ * // Returns a value that ensures the response won't be truncated
+ * ```
+ */
+export declare function estimateMaxTokens(promptText: string, model: string, options?: MaxTokensOptions): number;
+/**
+ * Get detailed token estimation with warnings
+ *
+ * @param promptText - Full prompt text
+ * @param model - Model name
+ * @param options - Estimation options
+ * @returns Detailed token estimation
+ *
+ * @example
+ * ```typescript
+ * const estimation = getTokenEstimation(longPrompt, "gpt-4o", {
+ *   desiredOutputTokens: 2000
+ * });
+ *
+ * if (!estimation.fitsInContext) {
+ *   console.error(estimation.warning);
+ * }
+ *
+ * const response = await openai.chat.completions.create({
+ *   model: "gpt-4o",
+ *   messages: [...],
+ *   max_tokens: estimation.recommendedMaxTokens
+ * });
+ * ```
+ */
+export declare function getTokenEstimation(promptText: string, model: string, options?: MaxTokensOptions): TokenEstimation;
+/**
+ * Get token estimation for chat messages
+ *
+ * @param messages - Array of chat messages
+ * @param model - Model name
+ * @param options - Estimation options
+ * @returns Detailed token estimation
+ */
+export declare function getChatTokenEstimation(messages: ChatMessage[], model: string, options?: MaxTokensOptions): TokenEstimation;
+/**
+ * Check if text will fit within model context
+ *
+ * @param text - Input text
+ * @param model - Model name
+ * @param reservedOutputTokens - Tokens to reserve for output
+ * @returns True if text fits within context
+ */
+export declare function fitsInContext(text: string, model: string, reservedOutputTokens?: number): boolean;
+/**
+ * Truncate text to fit within a token limit
+ *
+ * @param text - Input text
+ * @param maxTokens - Maximum tokens allowed
+ * @param model - Model name for accurate counting
+ * @returns Truncated text
+ *
+ * @example
+ * ```typescript
+ * const truncated = truncateToTokenLimit(longText, 1000, "gpt-4o");
+ * console.log(countTokens(truncated, { model: "gpt-4o" })); // <= 1000
+ * ```
+ */
+export declare function truncateToTokenLimit(text: string, maxTokens: number, model?: string): string;
+/**
+ * Split text into chunks that fit within token limits
+ *
+ * @param text - Input text
+ * @param maxTokensPerChunk - Maximum tokens per chunk
+ * @param overlap - Number of characters to overlap between chunks
+ * @param model - Model name
+ * @returns Array of text chunks
+ */
+export declare function splitIntoChunks(text: string, maxTokensPerChunk: number, overlap?: number, model?: string): string[];
+/**
+ * Clear the tokenizer cache
+ * Useful for testing or memory management
+ */
+export declare function clearTokenizerCache(): void;
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,SAAS,EACT,YAAY,EACZ,WAAW,EACX,iBAAiB,EACjB,gBAAgB,EAChB,eAAe,EAChB,MAAM,YAAY,CAAC;AA6CpB;;;;;;;;;;;;GAYG;AACH,wBAAgB,WAAW,CAAC,YAAY,EAAE,YAAY,GAAG,SAAS,CAEjE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAGhE;AAMD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,MAAM,CAO7E;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,WAAW,EAAE,EACvB,KAAK,GAAE,MAAiB,GACvB,MAAM,CAiCR;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAC/B,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,MAAiB,GACvB,MAAM,CAUR;AAgBD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,iBAAiB,CAC/B,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,gBAAgB,GACzB,MAAM,CA0BR;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,kBAAkB,CAChC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,gBAAgB,GACzB,eAAe,CAmDjB;AAED;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,WAAW,EAAE,EACvB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,gBAAgB,GACzB,eAAe,CAiDjB;AAMD;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,oBAAoB,GAAE,MAAa,GAClC,OAAO,CAKT;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,MAAiB,GACvB,MAAM,CAyCR;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,MAAM,EACZ,iBAAiB,EAAE,MAAM,EACzB,OAAO,GAAE,MAAY,EACrB,KAAK,GAAE,MAAiB,GACvB,MAAM,EAAE,CAoDV;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,IAAI,IAAI,CAE1C"}