@hyvmind/tiktoken-ts 0.0.1

package/dist/encodings/o200k-base.js

@@ -0,0 +1,191 @@
+/**
+ * o200k_base Encoding
+ * Used by GPT-4o, GPT-4.1, GPT-5 models
+ *
+ * This encoding has:
+ * - 200,000 tokens (larger vocabulary)
+ * - Better handling of non-English languages
+ * - Improved efficiency for common patterns
+ * - Better support for multimodal inputs
+ */
+import { BPETokenizer } from "../bpe.js";
+/**
+ * Encoding name constant
+ */
+export const ENCODING_NAME = "o200k_base";
+/**
+ * Create an o200k_base tokenizer instance
+ *
+ * @returns Tokenizer instance
+ */
+export function createO200kTokenizer() {
+    const bpe = new BPETokenizer(ENCODING_NAME);
+    return {
+        encodingName: ENCODING_NAME,
+        encode: (text) => bpe.encode(text),
+        decode: (tokens) => bpe.decode(tokens),
+        countTokens: (text) => bpe.countTokens(text),
+    };
+}
+/**
+ * Pre-computed token counts for common patterns
+ * o200k_base has a larger vocabulary, so more words are single tokens
+ */
+export const COMMON_TOKEN_PATTERNS = {
+    // Contractions (same as cl100k)
+    "'s": 1,
+    "'t": 1,
+    "'re": 1,
+    "'ve": 1,
+    "'m": 1,
+    "'ll": 1,
+    "'d": 1,
+    // Common words (more are single tokens in o200k)
+    the: 1,
+    and: 1,
+    is: 1,
+    are: 1,
+    was: 1,
+    were: 1,
+    will: 1,
+    would: 1,
+    could: 1,
+    should: 1,
+    have: 1,
+    has: 1,
+    had: 1,
+    been: 1,
+    being: 1,
+    this: 1,
+    that: 1,
+    these: 1,
+    those: 1,
+    with: 1,
+    from: 1,
+    into: 1,
+    because: 1, // Single token in o200k
+    however: 1,
+    therefore: 1,
+    although: 1,
+    meanwhile: 1,
+    // Programming keywords
+    function: 1,
+    const: 1,
+    let: 1,
+    var: 1,
+    return: 1,
+    if: 1,
+    else: 1,
+    while: 1,
+    class: 1,
+    interface: 1,
+    type: 1,
+    import: 1,
+    export: 1,
+    async: 1,
+    await: 1,
+    true: 1,
+    false: 1,
+    null: 1,
+    undefined: 1,
+    // TypeScript/React keywords (more are single tokens)
+    React: 1,
+    useState: 1,
+    useEffect: 1,
+    useCallback: 1,
+    useMemo: 1,
+    useRef: 1,
+    useContext: 1,
+    Component: 1,
+    Fragment: 1,
+    children: 1,
+    props: 1,
+    // Common API terms
+    request: 1,
+    response: 1,
+    error: 1,
+    success: 1,
+    message: 1,
+    data: 1,
+    status: 1,
+    headers: 1,
+    // Common symbols
+    "=>": 1,
+    "===": 1,
+    "!==": 1,
+    "&&": 1,
+    "||": 1,
+    "++": 1,
+    "--": 1,
+    "+=": 1,
+    "-=": 1,
+    "...": 1,
+    "?.": 1, // Optional chaining
+    "??": 1, // Nullish coalescing
+};
+/**
+ * Average characters per token for o200k_base
+ * Slightly higher than cl100k due to larger vocabulary
+ */
+export const AVERAGE_CHARS_PER_TOKEN = 4.0;
+/**
+ * Token estimation adjustments for different content types
+ * o200k is generally more efficient across all types
+ */
+export const CONTENT_TYPE_MULTIPLIERS = {
+    prose: 0.95, // More efficient for English
+    code: 0.8, // Even better for code
+    json: 0.7, // Very efficient for JSON
+    markdown: 0.9, // Better for markdown
+    html: 0.75, // Better for HTML
+    math: 1.1, // Still needs more tokens for math
+    multilingual: 0.85, // Much better for non-English
+};
+/**
+ * Language-specific efficiency factors for o200k_base
+ * The larger vocabulary handles more languages efficiently
+ */
+export const LANGUAGE_EFFICIENCY = {
+    en: 1.0, // English (baseline)
+    es: 1.05, // Spanish
+    fr: 1.05, // French
+    de: 1.08, // German (compound words)
+    pt: 1.05, // Portuguese
+    it: 1.05, // Italian
+    nl: 1.08, // Dutch
+    ru: 1.2, // Russian (Cyrillic)
+    ja: 1.5, // Japanese (needs more tokens)
+    zh: 1.4, // Chinese (CJK characters)
+    ko: 1.4, // Korean (Hangul)
+    ar: 1.3, // Arabic
+    hi: 1.3, // Hindi
+    vi: 1.15, // Vietnamese
+    th: 1.3, // Thai
+};
+/**
+ * Get estimated token count with content-type awareness
+ *
+ * @param text - Input text
+ * @param contentType - Type of content
+ * @returns Estimated token count
+ */
+export function estimateTokensForContent(text, contentType = "prose") {
+    const tokenizer = createO200kTokenizer();
+    const baseCount = tokenizer.countTokens(text);
+    const multiplier = CONTENT_TYPE_MULTIPLIERS[contentType] ?? 0.95;
+    return Math.max(1, Math.round(baseCount * multiplier));
+}
+/**
+ * Get estimated token count with language awareness
+ *
+ * @param text - Input text
+ * @param language - ISO 639-1 language code
+ * @returns Estimated token count
+ */
+export function estimateTokensForLanguage(text, language) {
+    const tokenizer = createO200kTokenizer();
+    const baseCount = tokenizer.countTokens(text);
+    const efficiency = LANGUAGE_EFFICIENCY[language.toLowerCase()] ?? 1.0;
+    return Math.max(1, Math.round(baseCount * efficiency));
+}
+//# sourceMappingURL=o200k-base.js.map

package/dist/encodings/o200k-base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"o200k-base.js","sourceRoot":"","sources":["../../src/encodings/o200k-base.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAGzC;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAiB,YAAY,CAAC;AAExD;;;;GAIG;AACH,MAAM,UAAU,oBAAoB;IAClC,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAC;IAE5C,OAAO;QACL,YAAY,EAAE,aAAa;QAC3B,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAC1C,MAAM,EAAE,CAAC,MAAgB,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;QAChD,WAAW,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC;KACrD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAA2B;IAC3D,gCAAgC;IAChC,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IAEP,iDAAiD;IACjD,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,EAAE,EAAE,CAAC;IACL,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;IACT,IAAI,EAAE,CAAC;IACP,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,OAAO,EAAE,CAAC,EAAE,wBAAwB;IACpC,OAAO,EAAE,CAAC;IACV,SAAS,EAAE,CAAC;IACZ,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IAEZ,uBAAuB;IACvB,QAAQ,EAAE,CAAC;IACX,KAAK,EAAE,CAAC;IACR,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,MAAM,EAAE,CAAC;IACT,EAAE,EAAE,CAAC;IACL,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,SAAS,EAAE,CAAC;IACZ,IAAI,EAAE,CAAC;IACP,MAAM,EAAE,CAAC;IACT,MAAM,EAAE,CAAC;IACT,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,SAAS,EAAE,CAAC;IAEZ,qDAAqD;IACrD,KAAK,EAAE,CAAC;IACR,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,WAAW,EAAE,CAAC;IACd,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,CAAC;IACT,UAAU,EAAE,CAAC;IACb,SAAS,EAAE,CAAC;IACZ,QAAQ,EAAE,CAAC;IACX,QAAQ,EAAE,CAAC;IACX,KAAK,EAAE,CAAC;IAER,mBAAmB;IACnB,OAAO,EAAE,CAAC;IACV,QAAQ,EAAE,CAAC;IACX,KAAK,EAAE,CAAC;IACR,OAAO,EAAE,CAAC;IACV,OAAO,EAAE,CAAC;IACV,IAAI,EAAE,CAAC;IACP,MAAM,EAAE,CAAC;IACT,OAAO,EAAE,CAAC;IAEV,iBAAiB;IACjB,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC,EAAE,oBAAoB;IAC7B,IAAI,EAAE,CAAC,EAAE,qBAAqB;CAC/B,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAE3C;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAA2B;IAC9D,KAAK,EAAE,IAAI,EAAE,6BAA6B;IAC1C,IAAI,EAAE,GAAG,EAAE,uBAAuB;IAClC,IAAI,EAAE,GAAG,EAAE,0BAA0B;IACrC,QAAQ,EAAE,GAAG,EAAE,sBAAsB;IACrC,IAAI,EAAE,IAAI,EAAE,kBAAkB;IAC9B,IAAI,EAAE,GAAG,EAAE,mCAAmC;IAC9C,YAAY,EAAE,IAAI,EAAE,8BAA8B;CACnD,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAA2B;IACzD,EAAE,EAAE,GAAG,EAAE,qBAAqB;IAC9B,EAAE,EAAE,IAAI,EAAE,UAAU;IACpB,EAAE,EAAE,IAAI,EAAE,SAAS;IACnB,EAAE,EAAE,IAAI,EAAE,0BAA0B;IACpC,EAAE,EAAE,IAAI,EAAE,aAAa;IACvB,EAAE,EAAE,IAAI,EAAE,UAAU;IACpB,EAAE,EAAE,IAAI,EAAE,QAAQ;IAClB,EAAE,EAAE,GAAG,EAAE,qBAAqB;IAC9B,EAAE,EAAE,GAAG,EAAE,+BAA+B;IACxC,EAAE,EAAE,GAAG,EAAE,2BAA2B;IACpC,EAAE,EAAE,GAAG,EAAE,kBAAkB;IAC3B,EAAE,EAAE,GAAG,EAAE,SAAS;IAClB,EAAE,EAAE,GAAG,EAAE,QAAQ;IACjB,EAAE,EAAE,IAAI,EAAE,aAAa;IACvB,EAAE,EAAE,GAAG,EAAE,OAAO;CACjB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CACtC,IAAY,EACZ,cAAqD,OAAO;IAE5D,MAAM,SAAS,GAAG,oBAAoB,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,wBAAwB,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC;IAEjE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CACvC,IAAY,EACZ,QAAgB;IAEhB,MAAM,SAAS,GAAG,oBAAoB,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,mBAAmB,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,GAAG,CAAC;IAEtE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC;AACzD,CAAC"}

package/dist/encodings/p50k-base.d.ts

@@ -0,0 +1,44 @@
+/**
+ * p50k_base Encoding (Legacy)
+ * Used by older Codex models (code-davinci-002, code-cushman-001)
+ *
+ * This encoding has:
+ * - 50,257 tokens (smallest vocabulary)
+ * - Optimized for code completion
+ * - Less efficient for natural language
+ *
+ * @deprecated This encoding is used by deprecated models.
+ * Use o200k_base or cl100k_base for current models.
+ */
+import type { Tokenizer, EncodingName } from "../types.js";
+/**
+ * Encoding name constant
+ */
+export declare const ENCODING_NAME: EncodingName;
+/**
+ * Create a p50k_base tokenizer instance
+ *
+ * @returns Tokenizer instance
+ * @deprecated Use createO200kTokenizer or createCL100kTokenizer instead
+ */
+export declare function createP50kTokenizer(): Tokenizer;
+/**
+ * Average characters per token for p50k_base
+ * Lower than newer encodings due to smaller vocabulary
+ */
+export declare const AVERAGE_CHARS_PER_TOKEN = 3.5;
+/**
+ * Token estimation adjustments for different content types
+ * p50k was optimized primarily for code
+ */
+export declare const CONTENT_TYPE_MULTIPLIERS: Record<string, number>;
+/**
+ * Get estimated token count with content-type awareness
+ *
+ * @param text - Input text
+ * @param contentType - Type of content
+ * @returns Estimated token count
+ * @deprecated Use o200k_base or cl100k_base for current models
+ */
+export declare function estimateTokensForContent(text: string, contentType?: keyof typeof CONTENT_TYPE_MULTIPLIERS): number;
+//# sourceMappingURL=p50k-base.d.ts.map

package/dist/encodings/p50k-base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"p50k-base.d.ts","sourceRoot":"","sources":["../../src/encodings/p50k-base.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3D;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,YAA0B,CAAC;AAEvD;;;;;GAKG;AACH,wBAAgB,mBAAmB,IAAI,SAAS,CAS/C;AAED;;;GAGG;AACH,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C;;;GAGG;AACH,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAO3D,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,WAAW,GAAE,MAAM,OAAO,wBAAkC,GAC3D,MAAM,CAMR"}

package/dist/encodings/p50k-base.js

@@ -0,0 +1,64 @@
+/**
+ * p50k_base Encoding (Legacy)
+ * Used by older Codex models (code-davinci-002, code-cushman-001)
+ *
+ * This encoding has:
+ * - 50,257 tokens (smallest vocabulary)
+ * - Optimized for code completion
+ * - Less efficient for natural language
+ *
+ * @deprecated This encoding is used by deprecated models.
+ * Use o200k_base or cl100k_base for current models.
+ */
+import { BPETokenizer } from "../bpe.js";
+/**
+ * Encoding name constant
+ */
+export const ENCODING_NAME = "p50k_base";
+/**
+ * Create a p50k_base tokenizer instance
+ *
+ * @returns Tokenizer instance
+ * @deprecated Use createO200kTokenizer or createCL100kTokenizer instead
+ */
+export function createP50kTokenizer() {
+    const bpe = new BPETokenizer(ENCODING_NAME);
+    return {
+        encodingName: ENCODING_NAME,
+        encode: (text) => bpe.encode(text),
+        decode: (tokens) => bpe.decode(tokens),
+        countTokens: (text) => bpe.countTokens(text),
+    };
+}
+/**
+ * Average characters per token for p50k_base
+ * Lower than newer encodings due to smaller vocabulary
+ */
+export const AVERAGE_CHARS_PER_TOKEN = 3.5;
+/**
+ * Token estimation adjustments for different content types
+ * p50k was optimized primarily for code
+ */
+export const CONTENT_TYPE_MULTIPLIERS = {
+    prose: 1.15, // Less efficient for natural language
+    code: 0.9, // Better for code (its primary use case)
+    json: 0.85, // Reasonable for JSON
+    markdown: 1.05, // Less efficient for markdown
+    html: 0.9, // Reasonable for HTML
+    math: 1.25, // Less efficient for math
+};
+/**
+ * Get estimated token count with content-type awareness
+ *
+ * @param text - Input text
+ * @param contentType - Type of content
+ * @returns Estimated token count
+ * @deprecated Use o200k_base or cl100k_base for current models
+ */
+export function estimateTokensForContent(text, contentType = "prose") {
+    const tokenizer = createP50kTokenizer();
+    const baseCount = tokenizer.countTokens(text);
+    const multiplier = CONTENT_TYPE_MULTIPLIERS[contentType] ?? 1.15;
+    return Math.max(1, Math.round(baseCount * multiplier));
+}
+//# sourceMappingURL=p50k-base.js.map

package/dist/encodings/p50k-base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"p50k-base.js","sourceRoot":"","sources":["../../src/encodings/p50k-base.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAGzC;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAiB,WAAW,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB;IACjC,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAC;IAE5C,OAAO;QACL,YAAY,EAAE,aAAa;QAC3B,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAC1C,MAAM,EAAE,CAAC,MAAgB,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;QAChD,WAAW,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC;KACrD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAE3C;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAA2B;IAC9D,KAAK,EAAE,IAAI,EAAE,sCAAsC;IACnD,IAAI,EAAE,GAAG,EAAE,yCAAyC;IACpD,IAAI,EAAE,IAAI,EAAE,sBAAsB;IAClC,QAAQ,EAAE,IAAI,EAAE,8BAA8B;IAC9C,IAAI,EAAE,GAAG,EAAE,sBAAsB;IACjC,IAAI,EAAE,IAAI,EAAE,0BAA0B;CACvC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,UAAU,wBAAwB,CACtC,IAAY,EACZ,cAAqD,OAAO;IAE5D,MAAM,SAAS,GAAG,mBAAmB,EAAE,CAAC;IACxC,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,wBAAwB,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC;IAEjE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC;AACzD,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,61 @@
+/**
+ * tiktoken-ts - A pure TypeScript implementation of OpenAI's tiktoken
+ *
+ * A self-contained TypeScript tokenizer for AI applications.
+ * Implements EXACT BPE (Byte-Pair Encoding) algorithm from tiktoken-rs.
+ *
+ * @packageDocumentation
+ *
+ * @example Exact BPE Tokenization (Async)
+ * ```typescript
+ * import { getEncodingAsync, countTokensAsync } from "tiktoken-ts";
+ *
+ * // Load encoding and tokenize
+ * const tiktoken = await getEncodingAsync("cl100k_base");
+ * const tokens = tiktoken.encode("Hello, world!");
+ * console.log(tokens); // [15496, 11, 1917, 0]
+ *
+ * // Decode back to text
+ * const text = tiktoken.decode(tokens);
+ * console.log(text); // "Hello, world!"
+ *
+ * // Count tokens
+ * const count = await countTokensAsync("Hello, world!", "cl100k_base");
+ * console.log(count); // 4
+ * ```
+ *
+ * @example Token Estimation (Sync, uses heuristics)
+ * ```typescript
+ * import {
+ *   countTokens,
+ *   estimateMaxTokens,
+ *   getTokenEstimation,
+ *   getEncodingForModelName,
+ *   getModelContextLimit
+ * } from "tiktoken-ts";
+ *
+ * // Count tokens in text (estimation)
+ * const tokens = countTokens("Hello, world!", { model: "gpt-4o" });
+ *
+ * // Estimate safe max_tokens value
+ * const maxTokens = estimateMaxTokens(promptText, "gpt-5-nano", {
+ *   desiredOutputTokens: 1000,
+ *   safetyMargin: 0.1
+ * });
+ *
+ * // Get detailed estimation with warnings
+ * const estimation = getTokenEstimation(promptText, "gpt-4o");
+ * if (!estimation.fitsInContext) {
+ *   console.warn(estimation.warning);
+ * }
+ * ```
+ */
+export type { EncodingName, ModelFamily, SpecialTokens, EncodingConfig, Tokenizer, ModelConfig, TokenEstimation, ChatMessage, TokenCountOptions, MaxTokensOptions, } from "./types.js";
+export { type Rank, type Vocabulary, type ReverseVocabulary, bytesToKey, keyToBytes, bytePairEncode, bytePairSplit, parseVocabulary, loadVocabularyFromUrl, loadVocabularyFromString, clearVocabularyCache, isVocabularyCached, VOCABULARY_URLS, CoreBPE, DecodeKeyError, type EncodingDefinition, SPECIAL_TOKENS, R50K_BASE, P50K_BASE, P50K_EDIT, CL100K_BASE, O200K_BASE, O200K_HARMONY, ENCODING_DEFINITIONS, getEncodingDefinition, listEncodingNames, type TokenizerName, getTokenizerForModel, getContextSize, getExactContextSize, EXACT_CONTEXT_SIZES, } from "./core/index.js";
+export { Tiktoken, getEncoding as getTiktoken, getEncodingForModel as getTiktokenForModel, getEncodingAsync, getEncodingForModelAsync, clearTiktokenCache, encodeAsync, decodeAsync, countTokensAsync, countTokensForModelAsync, } from "./tiktoken.js";
+export { getEncoding, getEncodingForModelName, countTokens, countChatTokens, countPromptTokens, estimateMaxTokens, getTokenEstimation, getChatTokenEstimation, fitsInContext, truncateToTokenLimit, splitIntoChunks, clearTokenizerCache, } from "./tokenizer.js";
+export { MODEL_CONFIGS, MODEL_ALIASES, getModelConfig, getEncodingForModel, getModelContextLimit, getModelMaxOutputTokens, getModelFamily, usesO200kEncoding, usesClaudeEstimation, isClaudeModel, listModels, listModelsByFamily, } from "./models.js";
+export { BPETokenizer, DEFAULT_SPECIAL_TOKENS, CL100K_BASE_CONFIG, O200K_BASE_CONFIG, P50K_BASE_CONFIG, CLAUDE_ESTIMATION_CONFIG, CLAUDE_SAFETY_MULTIPLIER, getEncodingConfig, } from "./bpe.js";
+export { createCL100kTokenizer, CL100K_ENCODING_NAME, CL100K_AVG_CHARS, estimateCL100kTokens, createO200kTokenizer, O200K_ENCODING_NAME, O200K_AVG_CHARS, O200K_LANGUAGE_EFFICIENCY, estimateO200kTokens, estimateO200kTokensForLanguage, createP50kTokenizer, P50K_ENCODING_NAME, P50K_AVG_CHARS, estimateP50kTokens, createClaudeEstimationTokenizer, CLAUDE_ENCODING_NAME, CLAUDE_AVG_CHARS, CLAUDE_VOCAB_SIZE, CLAUDE_SAFETY_MULT, estimateClaudeTokens, estimateClaudeTokensConservative, claudeIncludesSafetyMultiplier, } from "./encodings/index.js";
+export { stringToBytes, bytesToString, countCodePoints, containsCJK, containsEmoji, isAscii, getTextComplexityMultiplier, normalizeWhitespace, } from "./utils.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAMH,YAAY,EACV,YAAY,EACZ,WAAW,EACX,aAAa,EACb,cAAc,EACd,SAAS,EACT,WAAW,EACX,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,YAAY,CAAC;AAMpB,OAAO,EAEL,KAAK,IAAI,EACT,KAAK,UAAU,EACf,KAAK,iBAAiB,EAGtB,UAAU,EACV,UAAU,EACV,cAAc,EACd,aAAa,EAGb,eAAe,EACf,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,EACpB,kBAAkB,EAClB,eAAe,EAGf,OAAO,EACP,cAAc,EAGd,KAAK,kBAAkB,EACvB,cAAc,EACd,SAAS,EACT,SAAS,EACT,SAAS,EACT,WAAW,EACX,UAAU,EACV,aAAa,EACb,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,EAGjB,KAAK,aAAa,EAClB,oBAAoB,EACpB,cAAc,EACd,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAMzB,OAAO,EAEL,QAAQ,EAGR,WAAW,IAAI,WAAW,EAC1B,mBAAmB,IAAI,mBAAmB,EAC1C,gBAAgB,EAChB,wBAAwB,EACxB,kBAAkB,EAGlB,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,wBAAwB,GACzB,MAAM,eAAe,CAAC;AAMvB,OAAO,EAEL,WAAW,EACX,uBAAuB,EAGvB,WAAW,EACX,eAAe,EACf,iBAAiB,EAGjB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EAGtB,aAAa,EACb,oBAAoB,EACpB,eAAe,EACf,mBAAmB,GACpB,MAAM,gBAAgB,CAAC;AAMxB,OAAO,EACL,aAAa,EACb,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,oBAAoB,EACpB,uBAAuB,EACvB,cAAc,EACd,iBAAiB,EACjB,oBAAoB,EACpB,aAAa,EACb,UAAU,EACV,kBAAkB,GACnB,MAAM,aAAa,CAAC;AAMrB,OAAO,EACL,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,iBAAiB,EACjB,gBAAgB,EAChB,wBAAwB,EACxB,wBAAwB,EACxB,iBAAiB,GAClB,MAAM,UAAU,CAAC;AAMlB,OAAO,EAEL,qBAAqB,EACrB,oBAAoB,EACpB,gBAAgB,EAChB,oBAAoB,EAGpB,oBAAoB,EACpB,mBAAmB,EACnB,eAAe,EACf,yBAAyB,EACzB,mBAAmB,EACnB,8BAA8B,EAG9B,mBAAmB,EACnB,kBAAkB,EAClB,cAAc,EACd,kBAAkB,EAGlB,+BAA+B,EAC/B,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,gCAAgC,EAChC,8BAA8B,GAC/B,MAAM,sBAAsB,CAAC;AAM9B,OAAO,EACL,aAAa,EACb,aAAa,EACb,eAAe,EACf,WAAW,EACX,aAAa,EACb,OAAO,EACP,2BAA2B,EAC3B,mBAAmB,GACpB,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,109 @@
+/**
+ * tiktoken-ts - A pure TypeScript implementation of OpenAI's tiktoken
+ *
+ * A self-contained TypeScript tokenizer for AI applications.
+ * Implements EXACT BPE (Byte-Pair Encoding) algorithm from tiktoken-rs.
+ *
+ * @packageDocumentation
+ *
+ * @example Exact BPE Tokenization (Async)
+ * ```typescript
+ * import { getEncodingAsync, countTokensAsync } from "tiktoken-ts";
+ *
+ * // Load encoding and tokenize
+ * const tiktoken = await getEncodingAsync("cl100k_base");
+ * const tokens = tiktoken.encode("Hello, world!");
+ * console.log(tokens); // [15496, 11, 1917, 0]
+ *
+ * // Decode back to text
+ * const text = tiktoken.decode(tokens);
+ * console.log(text); // "Hello, world!"
+ *
+ * // Count tokens
+ * const count = await countTokensAsync("Hello, world!", "cl100k_base");
+ * console.log(count); // 4
+ * ```
+ *
+ * @example Token Estimation (Sync, uses heuristics)
+ * ```typescript
+ * import {
+ *   countTokens,
+ *   estimateMaxTokens,
+ *   getTokenEstimation,
+ *   getEncodingForModelName,
+ *   getModelContextLimit
+ * } from "tiktoken-ts";
+ *
+ * // Count tokens in text (estimation)
+ * const tokens = countTokens("Hello, world!", { model: "gpt-4o" });
+ *
+ * // Estimate safe max_tokens value
+ * const maxTokens = estimateMaxTokens(promptText, "gpt-5-nano", {
+ *   desiredOutputTokens: 1000,
+ *   safetyMargin: 0.1
+ * });
+ *
+ * // Get detailed estimation with warnings
+ * const estimation = getTokenEstimation(promptText, "gpt-4o");
+ * if (!estimation.fitsInContext) {
+ *   console.warn(estimation.warning);
+ * }
+ * ```
+ */
+// =============================================================================
+// Core BPE Implementation (Exact tiktoken-rs port)
+// =============================================================================
+export { 
+// BPE algorithm
+bytesToKey, keyToBytes, bytePairEncode, bytePairSplit, 
+// Vocabulary loading
+parseVocabulary, loadVocabularyFromUrl, loadVocabularyFromString, clearVocabularyCache, isVocabularyCached, VOCABULARY_URLS, 
+// Core BPE tokenizer
+CoreBPE, DecodeKeyError, SPECIAL_TOKENS, R50K_BASE, P50K_BASE, P50K_EDIT, CL100K_BASE, O200K_BASE, O200K_HARMONY, ENCODING_DEFINITIONS, getEncodingDefinition, listEncodingNames, getTokenizerForModel, getContextSize, getExactContextSize, EXACT_CONTEXT_SIZES, } from "./core/index.js";
+// =============================================================================
+// High-Level Tiktoken API (Async, requires vocabulary loading)
+// =============================================================================
+export { 
+// Tiktoken class
+Tiktoken, 
+// Factory functions
+getEncoding as getTiktoken, getEncodingForModel as getTiktokenForModel, getEncodingAsync, getEncodingForModelAsync, clearTiktokenCache, 
+// Async convenience functions
+encodeAsync, decodeAsync, countTokensAsync, countTokensForModelAsync, } from "./tiktoken.js";
+// =============================================================================
+// Legacy/Estimation API (Sync, uses heuristics)
+// =============================================================================
+export { 
+// Factory functions
+getEncoding, getEncodingForModelName, 
+// Token counting (estimation)
+countTokens, countChatTokens, countPromptTokens, 
+// Max tokens estimation
+estimateMaxTokens, getTokenEstimation, getChatTokenEstimation, 
+// Utility functions
+fitsInContext, truncateToTokenLimit, splitIntoChunks, clearTokenizerCache, } from "./tokenizer.js";
+// =============================================================================
+// Model Configuration
+// =============================================================================
+export { MODEL_CONFIGS, MODEL_ALIASES, getModelConfig, getEncodingForModel, getModelContextLimit, getModelMaxOutputTokens, getModelFamily, usesO200kEncoding, usesClaudeEstimation, isClaudeModel, listModels, listModelsByFamily, } from "./models.js";
+// =============================================================================
+// Legacy BPE Implementation (Estimation-based)
+// =============================================================================
+export { BPETokenizer, DEFAULT_SPECIAL_TOKENS, CL100K_BASE_CONFIG, O200K_BASE_CONFIG, P50K_BASE_CONFIG, CLAUDE_ESTIMATION_CONFIG, CLAUDE_SAFETY_MULTIPLIER, getEncodingConfig, } from "./bpe.js";
+// =============================================================================
+// Encoding-Specific Exports
+// =============================================================================
+export { 
+// cl100k_base (GPT-4, GPT-3.5-turbo)
+createCL100kTokenizer, CL100K_ENCODING_NAME, CL100K_AVG_CHARS, estimateCL100kTokens, 
+// o200k_base (GPT-4o, GPT-4.1, GPT-5)
+createO200kTokenizer, O200K_ENCODING_NAME, O200K_AVG_CHARS, O200K_LANGUAGE_EFFICIENCY, estimateO200kTokens, estimateO200kTokensForLanguage, 
+// p50k_base (Legacy Codex)
+createP50kTokenizer, P50K_ENCODING_NAME, P50K_AVG_CHARS, estimateP50kTokens, 
+// claude_estimation (Anthropic Claude - estimation only with safety multiplier)
+createClaudeEstimationTokenizer, CLAUDE_ENCODING_NAME, CLAUDE_AVG_CHARS, CLAUDE_VOCAB_SIZE, CLAUDE_SAFETY_MULT, estimateClaudeTokens, estimateClaudeTokensConservative, claudeIncludesSafetyMultiplier, } from "./encodings/index.js";
+// =============================================================================
+// Utility Functions
+// =============================================================================
+export { stringToBytes, bytesToString, countCodePoints, containsCJK, containsEmoji, isAscii, getTextComplexityMultiplier, normalizeWhitespace, } from "./utils.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAmBH,gFAAgF;AAChF,mDAAmD;AACnD,gFAAgF;AAEhF,OAAO;AAML,gBAAgB;AAChB,UAAU,EACV,UAAU,EACV,cAAc,EACd,aAAa;AAEb,qBAAqB;AACrB,eAAe,EACf,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,EACpB,kBAAkB,EAClB,eAAe;AAEf,qBAAqB;AACrB,OAAO,EACP,cAAc,EAId,cAAc,EACd,SAAS,EACT,SAAS,EACT,SAAS,EACT,WAAW,EACX,UAAU,EACV,aAAa,EACb,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,EAIjB,oBAAoB,EACpB,cAAc,EACd,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAEzB,gFAAgF;AAChF,+DAA+D;AAC/D,gFAAgF;AAEhF,OAAO;AACL,iBAAiB;AACjB,QAAQ;AAER,oBAAoB;AACpB,WAAW,IAAI,WAAW,EAC1B,mBAAmB,IAAI,mBAAmB,EAC1C,gBAAgB,EAChB,wBAAwB,EACxB,kBAAkB;AAElB,8BAA8B;AAC9B,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,wBAAwB,GACzB,MAAM,eAAe,CAAC;AAEvB,gFAAgF;AAChF,gDAAgD;AAChD,gFAAgF;AAEhF,OAAO;AACL,oBAAoB;AACpB,WAAW,EACX,uBAAuB;AAEvB,8BAA8B;AAC9B,WAAW,EACX,eAAe,EACf,iBAAiB;AAEjB,wBAAwB;AACxB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB;AAEtB,oBAAoB;AACpB,aAAa,EACb,oBAAoB,EACpB,eAAe,EACf,mBAAmB,GACpB,MAAM,gBAAgB,CAAC;AAExB,gFAAgF;AAChF,sBAAsB;AACtB,gFAAgF;AAEhF,OAAO,EACL,aAAa,EACb,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,oBAAoB,EACpB,uBAAuB,EACvB,cAAc,EACd,iBAAiB,EACjB,oBAAoB,EACpB,aAAa,EACb,UAAU,EACV,kBAAkB,GACnB,MAAM,aAAa,CAAC;AAErB,gFAAgF;AAChF,+CAA+C;AAC/C,gFAAgF;AAEhF,OAAO,EACL,YAAY,EACZ,sBAAsB,EACtB,kBAAkB,EAClB,iBAAiB,EACjB,gBAAgB,EAChB,wBAAwB,EACxB,wBAAwB,EACxB,iBAAiB,GAClB,MAAM,UAAU,CAAC;AAElB,gFAAgF;AAChF,4BAA4B;AAC5B,gFAAgF;AAEhF,OAAO;AACL,qCAAqC;AACrC,qBAAqB,EACrB,oBAAoB,EACpB,gBAAgB,EAChB,oBAAoB;AAEpB,sCAAsC;AACtC,oBAAoB,EACpB,mBAAmB,EACnB,eAAe,EACf,yBAAyB,EACzB,mBAAmB,EACnB,8BAA8B;AAE9B,2BAA2B;AAC3B,mBAAmB,EACnB,kBAAkB,EAClB,cAAc,EACd,kBAAkB;AAElB,gFAAgF;AAChF,+BAA+B,EAC/B,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,gCAAgC,EAChC,8BAA8B,GAC/B,MAAM,sBAAsB,CAAC;AAE9B,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF,OAAO,EACL,aAAa,EACb,aAAa,EACb,eAAe,EACf,WAAW,EACX,aAAa,EACb,OAAO,EACP,2BAA2B,EAC3B,mBAAmB,GACpB,MAAM,YAAY,CAAC"}

package/dist/models.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Model Configuration
+ * Maps model names to encodings and context limits
+ *
+ * Context limits are synced with tiktoken-rs:
+ * https://github.com/zurawiki/tiktoken-rs/blob/main/tiktoken-rs/src/model.rs
+ */
+import type { EncodingName, ModelConfig, ModelFamily } from "./types.js";
+/**
+ * Model context limits and configurations
+ * Context limits from tiktoken-rs, max output tokens from OpenAI docs
+ */
+export declare const MODEL_CONFIGS: Record<string, ModelConfig>;
+/**
+ * Model name aliases for flexibility
+ */
+export declare const MODEL_ALIASES: Record<string, string>;
+/**
+ * Get model configuration by name
+ * Handles aliases and partial matches
+ *
+ * @param modelName - Model name or alias
+ * @returns Model configuration or undefined
+ */
+export declare function getModelConfig(modelName: string): ModelConfig | undefined;
+/**
+ * Get encoding name for a model
+ *
+ * @param modelName - Model name
+ * @returns Encoding name, defaults to o200k_base for unknown models
+ */
+export declare function getEncodingForModel(modelName: string): EncodingName;
+/**
+ * Get context limit for a model
+ *
+ * @param modelName - Model name
+ * @returns Context limit in tokens
+ */
+export declare function getModelContextLimit(modelName: string): number;
+/**
+ * Get max output tokens for a model
+ *
+ * @param modelName - Model name
+ * @returns Max output tokens
+ */
+export declare function getModelMaxOutputTokens(modelName: string): number;
+/**
+ * Get model family
+ *
+ * @param modelName - Model name
+ * @returns Model family classification
+ */
+export declare function getModelFamily(modelName: string): ModelFamily;
+/**
+ * Check if a model uses the newer o200k_base encoding
+ *
+ * @param modelName - Model name
+ * @returns True if model uses o200k_base
+ */
+export declare function usesO200kEncoding(modelName: string): boolean;
+/**
+ * Check if a model uses the Claude estimation encoding
+ *
+ * Claude models use a proprietary tokenizer. This encoding provides
+ * "safe" estimates that intentionally over-count tokens to prevent
+ * API truncation issues.
+ *
+ * @param modelName - Model name
+ * @returns True if model uses claude_estimation
+ */
+export declare function usesClaudeEstimation(modelName: string): boolean;
+/**
+ * Check if a model is a Claude model
+ *
+ * @param modelName - Model name
+ * @returns True if model is from Anthropic Claude family
+ */
+export declare function isClaudeModel(modelName: string): boolean;
+/**
+ * List all known models
+ *
+ * @returns Array of model names
+ */
+export declare function listModels(): string[];
+/**
+ * List models by family
+ *
+ * @param family - Model family to filter
+ * @returns Array of model configurations
+ */
+export declare function listModelsByFamily(family: ModelFamily): ModelConfig[];
+//# sourceMappingURL=models.d.ts.map

package/dist/models.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAsBzE;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CA2brD,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAgEhD,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAsBzE;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,YAAY,CAGnE;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAG9D;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAGjE;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,CAG7D;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAE5D;AAED;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAE/D;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAGxD;AAED;;;;GAIG;AACH,wBAAgB,UAAU,IAAI,MAAM,EAAE,CAErC;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,WAAW,GAAG,WAAW,EAAE,CAIrE"}