@hyvmind/tiktoken-ts 0.0.1

package/dist/encodings/cl100k-base.js

@@ -0,0 +1,142 @@
+/**
+ * cl100k_base Encoding
+ * Used by GPT-4, GPT-3.5-turbo, text-embedding-ada-002
+ *
+ * This encoding has:
+ * - 100,256 tokens
+ * - Better handling of code and programming languages
+ * - Improved whitespace handling
+ */
+import { BPETokenizer } from "../bpe.js";
+/**
+ * Encoding name constant
+ */
+export const ENCODING_NAME = "cl100k_base";
+/**
+ * Create a cl100k_base tokenizer instance
+ *
+ * @returns Tokenizer instance
+ */
+export function createCL100kTokenizer() {
+    const bpe = new BPETokenizer(ENCODING_NAME);
+    return {
+        encodingName: ENCODING_NAME,
+        encode: (text) => bpe.encode(text),
+        decode: (tokens) => bpe.decode(tokens),
+        countTokens: (text) => bpe.countTokens(text),
+    };
+}
+/**
+ * Pre-computed token counts for common patterns
+ * Used to improve estimation accuracy
+ */
+export const COMMON_TOKEN_PATTERNS = {
+    // Contractions
+    "'s": 1,
+    "'t": 1,
+    "'re": 1,
+    "'ve": 1,
+    "'m": 1,
+    "'ll": 1,
+    "'d": 1,
+    // Common words (1 token each in cl100k)
+    the: 1,
+    and: 1,
+    is: 1,
+    are: 1,
+    was: 1,
+    were: 1,
+    will: 1,
+    would: 1,
+    could: 1,
+    should: 1,
+    have: 1,
+    has: 1,
+    had: 1,
+    been: 1,
+    being: 1,
+    this: 1,
+    that: 1,
+    these: 1,
+    those: 1,
+    with: 1,
+    from: 1,
+    into: 1,
+    for: 1,
+    // Programming keywords (1 token each)
+    function: 1,
+    const: 1,
+    let: 1,
+    var: 1,
+    return: 1,
+    if: 1,
+    else: 1,
+    while: 1,
+    class: 1,
+    interface: 1,
+    type: 1,
+    import: 1,
+    export: 1,
+    async: 1,
+    await: 1,
+    true: 1,
+    false: 1,
+    null: 1,
+    undefined: 1,
+    // Common symbols
+    "=>": 1,
+    "===": 1,
+    "!==": 1,
+    "&&": 1,
+    "||": 1,
+    "++": 1,
+    "--": 1,
+    "+=": 1,
+    "-=": 1,
+    "...": 1,
+    // Punctuation
+    ".": 1,
+    ",": 1,
+    ";": 1,
+    ":": 1,
+    "!": 1,
+    "?": 1,
+    "(": 1,
+    ")": 1,
+    "[": 1,
+    "]": 1,
+    "{": 1,
+    "}": 1,
+    "<": 1,
+    ">": 1,
+};
+/**
+ * Average characters per token for cl100k_base
+ * Based on empirical analysis of English text
+ */
+export const AVERAGE_CHARS_PER_TOKEN = 3.8;
+/**
+ * Token estimation adjustments for different content types
+ */
+export const CONTENT_TYPE_MULTIPLIERS = {
+    prose: 1.0, // Normal English text
+    code: 0.85, // Code tends to be more token-efficient
+    json: 0.75, // JSON is very structured
+    markdown: 0.95, // Markdown has extra syntax tokens
+    html: 0.8, // HTML tags are often single tokens
+    math: 1.2, // Mathematical notation uses more tokens
+};
+/**
+ * Get estimated token count with content-type awareness
+ *
+ * @param text - Input text
+ * @param contentType - Type of content
+ * @returns Estimated token count
+ */
+export function estimateTokensForContent(text, contentType = "prose") {
+    const tokenizer = createCL100kTokenizer();
+    const baseCount = tokenizer.countTokens(text);
+    const multiplier = CONTENT_TYPE_MULTIPLIERS[contentType] ?? 1.0;
+    return Math.max(1, Math.round(baseCount * multiplier));
+}
+//# sourceMappingURL=cl100k-base.js.map

package/dist/encodings/cl100k-base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cl100k-base.js","sourceRoot":"","sources":["../../src/encodings/cl100k-base.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAGzC;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAiB,aAAa,CAAC;AAEzD;;;;GAIG;AACH,MAAM,UAAU,qBAAqB;IACnC,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAC;IAE5C,OAAO;QACL,YAAY,EAAE,aAAa;QAC3B,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAC1C,MAAM,EAAE,CAAC,MAAgB,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;QAChD,WAAW,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC;KACrD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAA2B;IAC3D,eAAe;IACf,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IAEP,wCAAwC;IACxC,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,EAAE,EAAE,CAAC;IACL,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;IACT,IAAI,EAAE,CAAC;IACP,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,GAAG,EAAE,CAAC;IAEN,sCAAsC;IACtC,QAAQ,EAAE,CAAC;IACX,KAAK,EAAE,CAAC;IACR,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,MAAM,EAAE,CAAC;IACT,EAAE,EAAE,CAAC;IACL,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,SAAS,EAAE,CAAC;IACZ,IAAI,EAAE,CAAC;IACP,MAAM,EAAE,CAAC;IACT,MAAM,EAAE,CAAC;IACT,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,SAAS,EAAE,CAAC;IAEZ,iBAAiB;IACjB,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;IAER,cAAc;IACd,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;IACN,GAAG,EAAE,CAAC;CACP,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAE3C;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAA2B;IAC9D,KAAK,EAAE,GAAG,EAAE,sBAAsB;IAClC,IAAI,EAAE,IAAI,EAAE,wCAAwC;IACpD,IAAI,EAAE,IAAI,EAAE,0BAA0B;IACtC,QAAQ,EAAE,IAAI,EAAE,mCAAmC;IACnD,IAAI,EAAE,GAAG,EAAE,oCAAoC;IAC/C,IAAI,EAAE,GAAG,EAAE,yCAAyC;CACrD,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CACtC,IAAY,EACZ,cAAqD,OAAO;IAE5D,MAAM,SAAS,GAAG,qBAAqB,EAAE,CAAC;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,wBAAwB,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC;IAEhE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC;AACzD,CAAC"}

package/dist/encodings/claude-estimation.d.ts

@@ -0,0 +1,136 @@
+/**
+ * Claude Estimation Encoding
+ *
+ * IMPORTANT: This is an ESTIMATION encoding, not an exact BPE tokenizer.
+ *
+ * Claude (Anthropic) uses a proprietary tokenizer that is NOT publicly available.
+ * This encoding provides "safe" estimates that intentionally over-count tokens
+ * to prevent API truncation issues when working with Claude models.
+ *
+ * Based on research findings (see claude-tokenizer-research.md):
+ * - Claude 3+ uses ~22,000 token vocabulary (much smaller than OpenAI's 100K-200K)
+ * - Claude produces 16-30% MORE tokens than GPT-4 for equivalent content:
+ *   - English articles: +16%
+ *   - Math equations: +21%
+ *   - Python code: +30%
+ * - Average ~3.5 characters per token (vs GPT-4's ~4)
+ *
+ * This encoding applies a 1.25x safety multiplier to ensure estimates err on
+ * the side of over-counting, preventing API truncation.
+ *
+ * For EXACT Claude token counts, use Anthropic's official API:
+ * @see https://docs.anthropic.com/en/docs/build-with-claude/token-counting
+ */
+import type { Tokenizer, EncodingName } from "../types.js";
+/**
+ * Encoding name constant
+ */
+export declare const ENCODING_NAME: EncodingName;
+/**
+ * Create a Claude estimation tokenizer instance
+ *
+ * This tokenizer provides SAFE estimates for Claude models by:
+ * 1. Using cl100k_base patterns as a base (70% vocabulary overlap with Claude 1/2)
+ * 2. Applying a 1.25x safety multiplier to account for Claude's higher token usage
+ *
+ * The estimates intentionally err on over-counting to prevent API truncation.
+ *
+ * @returns Tokenizer instance configured for Claude estimation
+ *
+ * @example
+ * ```typescript
+ * import { createClaudeEstimationTokenizer } from "tiktoken-ts";
+ *
+ * const tokenizer = createClaudeEstimationTokenizer();
+ * const count = tokenizer.countTokens("Hello, Claude!");
+ * // Returns a SAFE estimate (intentionally higher than actual)
+ * ```
+ */
+export declare function createClaudeEstimationTokenizer(): Tokenizer;
+/**
+ * Average characters per token for Claude models.
+ *
+ * Research indicates Claude averages ~3.5 characters per token,
+ * which is lower (less efficient) than GPT-4's ~4 characters per token.
+ */
+export declare const AVERAGE_CHARS_PER_TOKEN = 3.5;
+/**
+ * Estimated vocabulary size for Claude 3+.
+ *
+ * Based on reverse-engineering research by Sander Land, Claude 3's
+ * vocabulary is estimated at ~22,000 tokens - remarkably small compared
+ * to Mistral's 32K, GPT-4's 100K, or LLaMA 3's 128K.
+ */
+export declare const ESTIMATED_VOCAB_SIZE = 22000;
+/**
+ * Safety multiplier applied to token estimates.
+ *
+ * This multiplier ensures estimates err on over-counting to prevent
+ * API truncation. Based on research showing Claude produces 16-30%
+ * more tokens than GPT-4:
+ * - English: +16%
+ * - Math: +21%
+ * - Code: +30%
+ *
+ * We use 1.25 (25%) as a safe middle ground.
+ */
+export declare const SAFETY_MULTIPLIER = 1.25;
+/**
+ * Token estimation adjustments for different content types.
+ *
+ * Claude's tokenizer has different efficiency characteristics than OpenAI's:
+ * - Generally less efficient (more tokens per character)
+ * - Especially less efficient for code (+30% vs GPT-4)
+ * - CJK handling varies by input vs output context
+ *
+ * These multipliers are applied ON TOP of the base safety multiplier.
+ */
+export declare const CONTENT_TYPE_MULTIPLIERS: Record<string, number>;
+/**
+ * Get estimated token count with content-type awareness for Claude.
+ *
+ * The returned estimate includes both:
+ * 1. The base safety multiplier (1.25x)
+ * 2. Content-type specific adjustments
+ *
+ * @param text - Input text
+ * @param contentType - Type of content (affects estimation accuracy)
+ * @returns Safe estimated token count (intentionally over-counts)
+ *
+ * @example
+ * ```typescript
+ * import { estimateClaudeTokens } from "tiktoken-ts";
+ *
+ * // For English prose
+ * const count = estimateClaudeTokens("Hello, world!", "prose");
+ *
+ * // For Python code (applies additional code multiplier)
+ * const codeCount = estimateClaudeTokens(pythonCode, "code");
+ * ```
+ */
+export declare function estimateClaudeTokens(text: string, contentType?: keyof typeof CONTENT_TYPE_MULTIPLIERS): number;
+/**
+ * Get a CONSERVATIVE (high) token estimate for Claude.
+ *
+ * This function returns the highest reasonable estimate, suitable for:
+ * - Calculating worst-case API costs
+ * - Ensuring prompts fit within context limits
+ * - Setting very safe max_tokens values
+ *
+ * It applies the maximum content multiplier (1.1 for code/math) on top
+ * of the base safety multiplier (1.25x), resulting in ~1.375x over-estimation.
+ *
+ * @param text - Input text
+ * @returns Conservative (high) token estimate
+ */
+export declare function estimateClaudeTokensConservative(text: string): number;
+/**
+ * Check if Claude token estimation applies safety multiplier.
+ *
+ * This is a convenience function that always returns true, confirming
+ * that Claude estimates include a safety margin.
+ *
+ * @returns Always true (Claude estimation always includes safety multiplier)
+ */
+export declare function includesSafetyMultiplier(): boolean;
+//# sourceMappingURL=claude-estimation.d.ts.map

package/dist/encodings/claude-estimation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-estimation.d.ts","sourceRoot":"","sources":["../../src/encodings/claude-estimation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3D;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,YAAkC,CAAC;AAE/D;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,+BAA+B,IAAI,SAAS,CAS3D;AAED;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C;;;;;;GAMG;AACH,eAAO,MAAM,oBAAoB,QAAQ,CAAC;AAE1C;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,iBAAiB,OAA2B,CAAC;AAE1D;;;;;;;;;GASG;AACH,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAO3D,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,EACZ,WAAW,GAAE,MAAM,OAAO,wBAAkC,GAC3D,MAAM,CAMR;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,gCAAgC,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAErE;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,IAAI,OAAO,CAElD"}

package/dist/encodings/claude-estimation.js

@@ -0,0 +1,160 @@
+/**
+ * Claude Estimation Encoding
+ *
+ * IMPORTANT: This is an ESTIMATION encoding, not an exact BPE tokenizer.
+ *
+ * Claude (Anthropic) uses a proprietary tokenizer that is NOT publicly available.
+ * This encoding provides "safe" estimates that intentionally over-count tokens
+ * to prevent API truncation issues when working with Claude models.
+ *
+ * Based on research findings (see claude-tokenizer-research.md):
+ * - Claude 3+ uses ~22,000 token vocabulary (much smaller than OpenAI's 100K-200K)
+ * - Claude produces 16-30% MORE tokens than GPT-4 for equivalent content:
+ *   - English articles: +16%
+ *   - Math equations: +21%
+ *   - Python code: +30%
+ * - Average ~3.5 characters per token (vs GPT-4's ~4)
+ *
+ * This encoding applies a 1.25x safety multiplier to ensure estimates err on
+ * the side of over-counting, preventing API truncation.
+ *
+ * For EXACT Claude token counts, use Anthropic's official API:
+ * @see https://docs.anthropic.com/en/docs/build-with-claude/token-counting
+ */
+import { BPETokenizer, CLAUDE_SAFETY_MULTIPLIER } from "../bpe.js";
+/**
+ * Encoding name constant
+ */
+export const ENCODING_NAME = "claude_estimation";
+/**
+ * Create a Claude estimation tokenizer instance
+ *
+ * This tokenizer provides SAFE estimates for Claude models by:
+ * 1. Using cl100k_base patterns as a base (70% vocabulary overlap with Claude 1/2)
+ * 2. Applying a 1.25x safety multiplier to account for Claude's higher token usage
+ *
+ * The estimates intentionally err on over-counting to prevent API truncation.
+ *
+ * @returns Tokenizer instance configured for Claude estimation
+ *
+ * @example
+ * ```typescript
+ * import { createClaudeEstimationTokenizer } from "tiktoken-ts";
+ *
+ * const tokenizer = createClaudeEstimationTokenizer();
+ * const count = tokenizer.countTokens("Hello, Claude!");
+ * // Returns a SAFE estimate (intentionally higher than actual)
+ * ```
+ */
+export function createClaudeEstimationTokenizer() {
+    const bpe = new BPETokenizer(ENCODING_NAME);
+    return {
+        encodingName: ENCODING_NAME,
+        encode: (text) => bpe.encode(text),
+        decode: (tokens) => bpe.decode(tokens),
+        countTokens: (text) => bpe.countTokens(text),
+    };
+}
+/**
+ * Average characters per token for Claude models.
+ *
+ * Research indicates Claude averages ~3.5 characters per token,
+ * which is lower (less efficient) than GPT-4's ~4 characters per token.
+ */
+export const AVERAGE_CHARS_PER_TOKEN = 3.5;
+/**
+ * Estimated vocabulary size for Claude 3+.
+ *
+ * Based on reverse-engineering research by Sander Land, Claude 3's
+ * vocabulary is estimated at ~22,000 tokens - remarkably small compared
+ * to Mistral's 32K, GPT-4's 100K, or LLaMA 3's 128K.
+ */
+export const ESTIMATED_VOCAB_SIZE = 22000;
+/**
+ * Safety multiplier applied to token estimates.
+ *
+ * This multiplier ensures estimates err on over-counting to prevent
+ * API truncation. Based on research showing Claude produces 16-30%
+ * more tokens than GPT-4:
+ * - English: +16%
+ * - Math: +21%
+ * - Code: +30%
+ *
+ * We use 1.25 (25%) as a safe middle ground.
+ */
+export const SAFETY_MULTIPLIER = CLAUDE_SAFETY_MULTIPLIER;
+/**
+ * Token estimation adjustments for different content types.
+ *
+ * Claude's tokenizer has different efficiency characteristics than OpenAI's:
+ * - Generally less efficient (more tokens per character)
+ * - Especially less efficient for code (+30% vs GPT-4)
+ * - CJK handling varies by input vs output context
+ *
+ * These multipliers are applied ON TOP of the base safety multiplier.
+ */
+export const CONTENT_TYPE_MULTIPLIERS = {
+    prose: 1.0, // Base English text
+    code: 1.1, // Code is particularly inefficient on Claude (+30% vs GPT-4)
+    json: 1.0, // JSON follows prose patterns
+    markdown: 1.0, // Markdown similar to prose
+    html: 0.95, // HTML tags slightly more efficient
+    math: 1.1, // Math equations use more tokens on Claude (+21% vs GPT-4)
+};
+/**
+ * Get estimated token count with content-type awareness for Claude.
+ *
+ * The returned estimate includes both:
+ * 1. The base safety multiplier (1.25x)
+ * 2. Content-type specific adjustments
+ *
+ * @param text - Input text
+ * @param contentType - Type of content (affects estimation accuracy)
+ * @returns Safe estimated token count (intentionally over-counts)
+ *
+ * @example
+ * ```typescript
+ * import { estimateClaudeTokens } from "tiktoken-ts";
+ *
+ * // For English prose
+ * const count = estimateClaudeTokens("Hello, world!", "prose");
+ *
+ * // For Python code (applies additional code multiplier)
+ * const codeCount = estimateClaudeTokens(pythonCode, "code");
+ * ```
+ */
+export function estimateClaudeTokens(text, contentType = "prose") {
+    const tokenizer = createClaudeEstimationTokenizer();
+    const baseCount = tokenizer.countTokens(text); // Already includes safety multiplier
+    const contentMultiplier = CONTENT_TYPE_MULTIPLIERS[contentType] ?? 1.0;
+    return Math.max(1, Math.ceil(baseCount * contentMultiplier));
+}
+/**
+ * Get a CONSERVATIVE (high) token estimate for Claude.
+ *
+ * This function returns the highest reasonable estimate, suitable for:
+ * - Calculating worst-case API costs
+ * - Ensuring prompts fit within context limits
+ * - Setting very safe max_tokens values
+ *
+ * It applies the maximum content multiplier (1.1 for code/math) on top
+ * of the base safety multiplier (1.25x), resulting in ~1.375x over-estimation.
+ *
+ * @param text - Input text
+ * @returns Conservative (high) token estimate
+ */
+export function estimateClaudeTokensConservative(text) {
+    return estimateClaudeTokens(text, "code"); // Uses highest multiplier
+}
+/**
+ * Check if Claude token estimation applies safety multiplier.
+ *
+ * This is a convenience function that always returns true, confirming
+ * that Claude estimates include a safety margin.
+ *
+ * @returns Always true (Claude estimation always includes safety multiplier)
+ */
+export function includesSafetyMultiplier() {
+    return true;
+}
+//# sourceMappingURL=claude-estimation.js.map

package/dist/encodings/claude-estimation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-estimation.js","sourceRoot":"","sources":["../../src/encodings/claude-estimation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,WAAW,CAAC;AAGnE;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAiB,mBAAmB,CAAC;AAE/D;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,+BAA+B;IAC7C,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAC;IAE5C,OAAO;QACL,YAAY,EAAE,aAAa;QAC3B,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAC1C,MAAM,EAAE,CAAC,MAAgB,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;QAChD,WAAW,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC;KACrD,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAE3C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,CAAC;AAE1C;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,wBAAwB,CAAC;AAE1D;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAA2B;IAC9D,KAAK,EAAE,GAAG,EAAE,oBAAoB;IAChC,IAAI,EAAE,GAAG,EAAE,6DAA6D;IACxE,IAAI,EAAE,GAAG,EAAE,8BAA8B;IACzC,QAAQ,EAAE,GAAG,EAAE,4BAA4B;IAC3C,IAAI,EAAE,IAAI,EAAE,oCAAoC;IAChD,IAAI,EAAE,GAAG,EAAE,2DAA2D;CACvE,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,oBAAoB,CAClC,IAAY,EACZ,cAAqD,OAAO;IAE5D,MAAM,SAAS,GAAG,+BAA+B,EAAE,CAAC;IACpD,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,qCAAqC;IACpF,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC;IAEvE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,iBAAiB,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,gCAAgC,CAAC,IAAY;IAC3D,OAAO,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,0BAA0B;AACvE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,wBAAwB;IACtC,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/encodings/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Encoding Exports
+ * Central export for all encoding implementations
+ */
+export { createCL100kTokenizer, ENCODING_NAME as CL100K_ENCODING_NAME, COMMON_TOKEN_PATTERNS as CL100K_TOKEN_PATTERNS, AVERAGE_CHARS_PER_TOKEN as CL100K_AVG_CHARS, CONTENT_TYPE_MULTIPLIERS as CL100K_CONTENT_MULTIPLIERS, estimateTokensForContent as estimateCL100kTokens, } from "./cl100k-base.js";
+export { createO200kTokenizer, ENCODING_NAME as O200K_ENCODING_NAME, COMMON_TOKEN_PATTERNS as O200K_TOKEN_PATTERNS, AVERAGE_CHARS_PER_TOKEN as O200K_AVG_CHARS, CONTENT_TYPE_MULTIPLIERS as O200K_CONTENT_MULTIPLIERS, LANGUAGE_EFFICIENCY as O200K_LANGUAGE_EFFICIENCY, estimateTokensForContent as estimateO200kTokens, estimateTokensForLanguage as estimateO200kTokensForLanguage, } from "./o200k-base.js";
+export { createP50kTokenizer, ENCODING_NAME as P50K_ENCODING_NAME, AVERAGE_CHARS_PER_TOKEN as P50K_AVG_CHARS, CONTENT_TYPE_MULTIPLIERS as P50K_CONTENT_MULTIPLIERS, estimateTokensForContent as estimateP50kTokens, } from "./p50k-base.js";
+export { createClaudeEstimationTokenizer, ENCODING_NAME as CLAUDE_ENCODING_NAME, AVERAGE_CHARS_PER_TOKEN as CLAUDE_AVG_CHARS, ESTIMATED_VOCAB_SIZE as CLAUDE_VOCAB_SIZE, SAFETY_MULTIPLIER as CLAUDE_SAFETY_MULT, CONTENT_TYPE_MULTIPLIERS as CLAUDE_CONTENT_MULTIPLIERS, estimateClaudeTokens, estimateClaudeTokensConservative, includesSafetyMultiplier as claudeIncludesSafetyMultiplier, } from "./claude-estimation.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/encodings/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/encodings/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EACL,qBAAqB,EACrB,aAAa,IAAI,oBAAoB,EACrC,qBAAqB,IAAI,qBAAqB,EAC9C,uBAAuB,IAAI,gBAAgB,EAC3C,wBAAwB,IAAI,0BAA0B,EACtD,wBAAwB,IAAI,oBAAoB,GACjD,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EACL,oBAAoB,EACpB,aAAa,IAAI,mBAAmB,EACpC,qBAAqB,IAAI,oBAAoB,EAC7C,uBAAuB,IAAI,eAAe,EAC1C,wBAAwB,IAAI,yBAAyB,EACrD,mBAAmB,IAAI,yBAAyB,EAChD,wBAAwB,IAAI,mBAAmB,EAC/C,yBAAyB,IAAI,8BAA8B,GAC5D,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,mBAAmB,EACnB,aAAa,IAAI,kBAAkB,EACnC,uBAAuB,IAAI,cAAc,EACzC,wBAAwB,IAAI,wBAAwB,EACpD,wBAAwB,IAAI,kBAAkB,GAC/C,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,+BAA+B,EAC/B,aAAa,IAAI,oBAAoB,EACrC,uBAAuB,IAAI,gBAAgB,EAC3C,oBAAoB,IAAI,iBAAiB,EACzC,iBAAiB,IAAI,kBAAkB,EACvC,wBAAwB,IAAI,0BAA0B,EACtD,oBAAoB,EACpB,gCAAgC,EAChC,wBAAwB,IAAI,8BAA8B,GAC3D,MAAM,wBAAwB,CAAC"}

package/dist/encodings/index.js

@@ -0,0 +1,13 @@
+/**
+ * Encoding Exports
+ * Central export for all encoding implementations
+ */
+// cl100k_base (GPT-4, GPT-3.5-turbo)
+export { createCL100kTokenizer, ENCODING_NAME as CL100K_ENCODING_NAME, COMMON_TOKEN_PATTERNS as CL100K_TOKEN_PATTERNS, AVERAGE_CHARS_PER_TOKEN as CL100K_AVG_CHARS, CONTENT_TYPE_MULTIPLIERS as CL100K_CONTENT_MULTIPLIERS, estimateTokensForContent as estimateCL100kTokens, } from "./cl100k-base.js";
+// o200k_base (GPT-4o, GPT-4.1, GPT-5)
+export { createO200kTokenizer, ENCODING_NAME as O200K_ENCODING_NAME, COMMON_TOKEN_PATTERNS as O200K_TOKEN_PATTERNS, AVERAGE_CHARS_PER_TOKEN as O200K_AVG_CHARS, CONTENT_TYPE_MULTIPLIERS as O200K_CONTENT_MULTIPLIERS, LANGUAGE_EFFICIENCY as O200K_LANGUAGE_EFFICIENCY, estimateTokensForContent as estimateO200kTokens, estimateTokensForLanguage as estimateO200kTokensForLanguage, } from "./o200k-base.js";
+// p50k_base (Legacy Codex models)
+export { createP50kTokenizer, ENCODING_NAME as P50K_ENCODING_NAME, AVERAGE_CHARS_PER_TOKEN as P50K_AVG_CHARS, CONTENT_TYPE_MULTIPLIERS as P50K_CONTENT_MULTIPLIERS, estimateTokensForContent as estimateP50kTokens, } from "./p50k-base.js";
+// claude_estimation (Anthropic Claude models - estimation only)
+export { createClaudeEstimationTokenizer, ENCODING_NAME as CLAUDE_ENCODING_NAME, AVERAGE_CHARS_PER_TOKEN as CLAUDE_AVG_CHARS, ESTIMATED_VOCAB_SIZE as CLAUDE_VOCAB_SIZE, SAFETY_MULTIPLIER as CLAUDE_SAFETY_MULT, CONTENT_TYPE_MULTIPLIERS as CLAUDE_CONTENT_MULTIPLIERS, estimateClaudeTokens, estimateClaudeTokensConservative, includesSafetyMultiplier as claudeIncludesSafetyMultiplier, } from "./claude-estimation.js";
+//# sourceMappingURL=index.js.map

package/dist/encodings/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/encodings/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,qCAAqC;AACrC,OAAO,EACL,qBAAqB,EACrB,aAAa,IAAI,oBAAoB,EACrC,qBAAqB,IAAI,qBAAqB,EAC9C,uBAAuB,IAAI,gBAAgB,EAC3C,wBAAwB,IAAI,0BAA0B,EACtD,wBAAwB,IAAI,oBAAoB,GACjD,MAAM,kBAAkB,CAAC;AAE1B,sCAAsC;AACtC,OAAO,EACL,oBAAoB,EACpB,aAAa,IAAI,mBAAmB,EACpC,qBAAqB,IAAI,oBAAoB,EAC7C,uBAAuB,IAAI,eAAe,EAC1C,wBAAwB,IAAI,yBAAyB,EACrD,mBAAmB,IAAI,yBAAyB,EAChD,wBAAwB,IAAI,mBAAmB,EAC/C,yBAAyB,IAAI,8BAA8B,GAC5D,MAAM,iBAAiB,CAAC;AAEzB,kCAAkC;AAClC,OAAO,EACL,mBAAmB,EACnB,aAAa,IAAI,kBAAkB,EACnC,uBAAuB,IAAI,cAAc,EACzC,wBAAwB,IAAI,wBAAwB,EACpD,wBAAwB,IAAI,kBAAkB,GAC/C,MAAM,gBAAgB,CAAC;AAExB,gEAAgE;AAChE,OAAO,EACL,+BAA+B,EAC/B,aAAa,IAAI,oBAAoB,EACrC,uBAAuB,IAAI,gBAAgB,EAC3C,oBAAoB,IAAI,iBAAiB,EACzC,iBAAiB,IAAI,kBAAkB,EACvC,wBAAwB,IAAI,0BAA0B,EACtD,oBAAoB,EACpB,gCAAgC,EAChC,wBAAwB,IAAI,8BAA8B,GAC3D,MAAM,wBAAwB,CAAC"}

package/dist/encodings/o200k-base.d.ts

@@ -0,0 +1,58 @@
+/**
+ * o200k_base Encoding
+ * Used by GPT-4o, GPT-4.1, GPT-5 models
+ *
+ * This encoding has:
+ * - 200,000 tokens (larger vocabulary)
+ * - Better handling of non-English languages
+ * - Improved efficiency for common patterns
+ * - Better support for multimodal inputs
+ */
+import type { Tokenizer, EncodingName } from "../types.js";
+/**
+ * Encoding name constant
+ */
+export declare const ENCODING_NAME: EncodingName;
+/**
+ * Create an o200k_base tokenizer instance
+ *
+ * @returns Tokenizer instance
+ */
+export declare function createO200kTokenizer(): Tokenizer;
+/**
+ * Pre-computed token counts for common patterns
+ * o200k_base has a larger vocabulary, so more words are single tokens
+ */
+export declare const COMMON_TOKEN_PATTERNS: Record<string, number>;
+/**
+ * Average characters per token for o200k_base
+ * Slightly higher than cl100k due to larger vocabulary
+ */
+export declare const AVERAGE_CHARS_PER_TOKEN = 4;
+/**
+ * Token estimation adjustments for different content types
+ * o200k is generally more efficient across all types
+ */
+export declare const CONTENT_TYPE_MULTIPLIERS: Record<string, number>;
+/**
+ * Language-specific efficiency factors for o200k_base
+ * The larger vocabulary handles more languages efficiently
+ */
+export declare const LANGUAGE_EFFICIENCY: Record<string, number>;
+/**
+ * Get estimated token count with content-type awareness
+ *
+ * @param text - Input text
+ * @param contentType - Type of content
+ * @returns Estimated token count
+ */
+export declare function estimateTokensForContent(text: string, contentType?: keyof typeof CONTENT_TYPE_MULTIPLIERS): number;
+/**
+ * Get estimated token count with language awareness
+ *
+ * @param text - Input text
+ * @param language - ISO 639-1 language code
+ * @returns Estimated token count
+ */
+export declare function estimateTokensForLanguage(text: string, language: string): number;
+//# sourceMappingURL=o200k-base.d.ts.map

package/dist/encodings/o200k-base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"o200k-base.d.ts","sourceRoot":"","sources":["../../src/encodings/o200k-base.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3D;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,YAA2B,CAAC;AAExD;;;;GAIG;AACH,wBAAgB,oBAAoB,IAAI,SAAS,CAShD;AAED;;;GAGG;AACH,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAgGxD,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,uBAAuB,IAAM,CAAC;AAE3C;;;GAGG;AACH,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAQ3D,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAgBtD,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,WAAW,GAAE,MAAM,OAAO,wBAAkC,GAC3D,MAAM,CAMR;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,GACf,MAAM,CAMR"}