ai-token-estimator 1.0.3 → 1.1.0

package/README.md

@@ -39,6 +39,29 @@ console.log(getAvailableModels());
 // ['gpt-5.2', 'gpt-4o', 'claude-opus-4.5', 'gemini-3-pro', ...]
 ```
 
+## Exact OpenAI tokenization (BPE)
+
+This package includes **exact tokenization for OpenAI models** using a tiktoken-compatible BPE tokenizer (via `gpt-tokenizer`).
+
+Notes:
+- Encodings are **lazy-loaded on first use** (one-time cost per encoding).
+- Exact tokenization is **slower** than heuristic estimation; `estimate()` defaults to `'heuristic'` to keep existing behavior fast.
+- `encode` / `decode` and `estimate({ tokenizer: 'openai_exact' })` require **Node.js** (uses `node:module` under the hood).
+
+```ts
+import { encode, decode } from 'ai-token-estimator';
+
+const text = 'Hello, world!';
+const tokens = encode(text, { model: 'gpt-5.1' }); // exact OpenAI token IDs
+const roundTrip = decode(tokens, { model: 'gpt-5.1' });
+
+console.log(tokens.length);
+console.log(roundTrip); // "Hello, world!"
+```
+
+Supported encodings:
+`r50k_base`, `p50k_base`, `p50k_edit`, `cl100k_base`, `o200k_base`, `o200k_harmony`
+
 ## API Reference
 
 ### `estimate(input: EstimateInput): EstimateOutput`
@@ -52,6 +75,7 @@ interface EstimateInput {
   text: string;           // The text to estimate tokens for
   model: string;          // Model ID (e.g., 'gpt-4o', 'claude-opus-4.5')
   rounding?: 'ceil' | 'round' | 'floor';  // Rounding strategy (default: 'ceil')
+  tokenizer?: 'heuristic' | 'openai_exact' | 'auto'; // Token counting strategy (default: 'heuristic')
 }
 ```
 
@@ -64,13 +88,36 @@ interface EstimateOutput {
   estimatedTokens: number; // Estimated token count (integer)
   estimatedInputCost: number; // Estimated cost in USD
   charsPerToken: number;   // The ratio used for this model
+  tokenizerMode?: 'heuristic' | 'openai_exact' | 'auto'; // Which strategy was used
+  encodingUsed?: string;   // OpenAI encoding when using exact tokenization
 }
 ```
 
+### `countTokens(input: TokenCountInput): TokenCountOutput`
+
+Counts tokens for a given model:
+- OpenAI models: **exact** BPE tokenization
+- Other providers: heuristic estimate
+
+```ts
+import { countTokens } from 'ai-token-estimator';
+
+const result = countTokens({ text: 'Hello, world!', model: 'gpt-5.1' });
+// { tokens: 4, exact: true, encoding: 'o200k_base' }
+```
+
 ### `getAvailableModels(): string[]`
 
 Returns an array of all supported model IDs.
 
+### `encode(text: string, options?: EncodeOptions): number[]`
+
+Encodes text into **OpenAI token IDs** using tiktoken-compatible BPE tokenization.
+
+### `decode(tokens: Iterable<number>, options?: { encoding?: OpenAIEncoding; model?: string }): string`
+
+Decodes OpenAI token IDs back into text using the selected encoding/model.
+
 ### `getModelConfig(model: string): ModelConfig`
 
 Returns the configuration for a specific model. Throws if the model is not found.
@@ -108,6 +155,14 @@ This package counts Unicode code points, not UTF-16 code units. This means:
 - Accented characters count correctly
 - Most source code characters count as 1
 
+## Benchmarks (repo only)
+
+This repository includes a small benchmark script to compare heuristic vs exact OpenAI tokenization:
+
+```bash
+npm run benchmark:tokenizer
+```
+
 <!-- SUPPORTED_MODELS_START -->
 ## Supported Models
 

package/dist/index.cjs

@@ -22,6 +22,9 @@ var index_exports = {};
 __export(index_exports, {
   DEFAULT_MODELS: () => DEFAULT_MODELS,
   LAST_UPDATED: () => LAST_UPDATED,
+  countTokens: () => countTokens,
+  decode: () => decode,
+  encode: () => encode,
   estimate: () => estimate,
   getAvailableModels: () => getAvailableModels,
   getModelConfig: () => getModelConfig
@@ -404,6 +407,79 @@ function getAvailableModels() {
   return Object.keys(DEFAULT_MODELS);
 }
 
+// src/openai-bpe.ts
+var import_node_module = require("module");
+var import_constants = require("gpt-tokenizer/constants");
+var import_mapping = require("gpt-tokenizer/mapping");
+var import_meta = {};
+var requireBase = typeof __filename === "string" && __filename.length > 0 ? __filename : import_meta.url;
+var NODE_REQUIRE = (0, import_node_module.createRequire)(requireBase);
+var ENCODING_MODULES = {
+  r50k_base: "gpt-tokenizer/cjs/encoding/r50k_base",
+  p50k_base: "gpt-tokenizer/cjs/encoding/p50k_base",
+  p50k_edit: "gpt-tokenizer/cjs/encoding/p50k_edit",
+  cl100k_base: "gpt-tokenizer/cjs/encoding/cl100k_base",
+  o200k_base: "gpt-tokenizer/cjs/encoding/o200k_base",
+  o200k_harmony: "gpt-tokenizer/cjs/encoding/o200k_harmony"
+};
+var encodingApiCache = /* @__PURE__ */ new Map();
+function getEncodingApi(encoding) {
+  const cached = encodingApiCache.get(encoding);
+  if (cached) return cached;
+  const modulePath = ENCODING_MODULES[encoding];
+  const mod = NODE_REQUIRE(modulePath);
+  const api = { encode: mod.encode, decode: mod.decode };
+  encodingApiCache.set(encoding, api);
+  return api;
+}
+function resolveEncoding(selector) {
+  if (selector?.encoding) {
+    return selector.encoding;
+  }
+  const model = selector?.model?.trim();
+  if (model) {
+    const mapped = import_mapping.modelToEncodingMap[model];
+    if (mapped) {
+      return mapped;
+    }
+  }
+  return import_mapping.DEFAULT_ENCODING;
+}
+function getOpenAIEncoding(selector) {
+  return resolveEncoding(selector);
+}
+function toGptTokenizerEncodeOptions(allowSpecial) {
+  const mode = allowSpecial ?? "none_raise";
+  switch (mode) {
+    case "all":
+      return {
+        allowedSpecial: import_constants.ALL_SPECIAL_TOKENS,
+        disallowedSpecial: /* @__PURE__ */ new Set()
+      };
+    case "none":
+      return {
+        allowedSpecial: /* @__PURE__ */ new Set(),
+        disallowedSpecial: /* @__PURE__ */ new Set()
+      };
+    case "none_raise":
+    default:
+      return {
+        disallowedSpecial: import_constants.ALL_SPECIAL_TOKENS
+      };
+  }
+}
+function encode(text, options) {
+  const encoding = resolveEncoding(options);
+  const api = getEncodingApi(encoding);
+  const encodeOptions = toGptTokenizerEncodeOptions(options?.allowSpecial);
+  return api.encode(text, encodeOptions);
+}
+function decode(tokens, options) {
+  const encoding = resolveEncoding(options);
+  const api = getEncodingApi(encoding);
+  return api.decode(tokens);
+}
+
 // src/estimator.ts
 function countCodePoints(text) {
   let count = 0;
@@ -413,21 +489,43 @@ function countCodePoints(text) {
   return count;
 }
 function estimate(input) {
-  const { text, model, rounding = "ceil" } = input;
+  const { text, model, rounding = "ceil", tokenizer = "heuristic" } = input;
   const config = getModelConfig(model);
   const characterCount = countCodePoints(text);
-  const rawTokens = characterCount / config.charsPerToken;
+  const isNonOpenAIModel2 = model.startsWith("claude-") || model.startsWith("gemini-");
   let estimatedTokens;
-  switch (rounding) {
-    case "floor":
-      estimatedTokens = Math.floor(rawTokens);
-      break;
-    case "round":
-      estimatedTokens = Math.round(rawTokens);
-      break;
-    case "ceil":
-    default:
-      estimatedTokens = Math.ceil(rawTokens);
+  let tokenizerModeUsed = "heuristic";
+  let encodingUsed;
+  const shouldTryExact = tokenizer === "openai_exact" || tokenizer === "auto";
+  if (shouldTryExact && !isNonOpenAIModel2) {
+    try {
+      estimatedTokens = encode(text, { model, allowSpecial: "none" }).length;
+      tokenizerModeUsed = "openai_exact";
+      encodingUsed = getOpenAIEncoding({ model });
+    } catch (error) {
+      if (tokenizer === "openai_exact") {
+        throw error;
+      }
+    }
+  } else if (tokenizer === "openai_exact" && isNonOpenAIModel2) {
+    throw new Error(
+      `Tokenizer mode "openai_exact" requested for non-OpenAI model: "${model}"`
+    );
+  }
+  if (estimatedTokens === void 0) {
+    const rawTokens = characterCount / config.charsPerToken;
+    switch (rounding) {
+      case "floor":
+        estimatedTokens = Math.floor(rawTokens);
+        break;
+      case "round":
+        estimatedTokens = Math.round(rawTokens);
+        break;
+      case "ceil":
+      default:
+        estimatedTokens = Math.ceil(rawTokens);
+    }
+    tokenizerModeUsed = "heuristic";
   }
   const estimatedInputCost = estimatedTokens * config.inputCostPerMillion / 1e6;
   return {
@@ -435,13 +533,44 @@ function estimate(input) {
     characterCount,
     estimatedTokens,
     estimatedInputCost,
-    charsPerToken: config.charsPerToken
+    charsPerToken: config.charsPerToken,
+    tokenizerMode: tokenizerModeUsed,
+    encodingUsed
   };
 }
+
+// src/token-counter.ts
+function isNonOpenAIModel(model) {
+  return model.startsWith("claude-") || model.startsWith("gemini-");
+}
+function countTokens(input) {
+  const { text, model } = input;
+  if (isNonOpenAIModel(model)) {
+    return {
+      tokens: estimate({ text, model }).estimatedTokens,
+      exact: false
+    };
+  }
+  try {
+    return {
+      tokens: encode(text, { model, allowSpecial: "none" }).length,
+      exact: true,
+      encoding: getOpenAIEncoding({ model })
+    };
+  } catch {
+    return {
+      tokens: estimate({ text, model }).estimatedTokens,
+      exact: false
+    };
+  }
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   DEFAULT_MODELS,
   LAST_UPDATED,
+  countTokens,
+  decode,
+  encode,
   estimate,
   getAvailableModels,
   getModelConfig

package/dist/index.d.cts

@@ -7,6 +7,7 @@ interface ModelConfig {
     /** Cost in USD per 1 million input tokens */
     inputCostPerMillion: number;
 }
+type TokenizerMode = 'heuristic' | 'openai_exact' | 'auto';
 /**
  * Input parameters for the estimate function.
  */
@@ -17,6 +18,13 @@ interface EstimateInput {
     model: string;
     /** Rounding strategy for token count (default: 'ceil') */
     rounding?: 'ceil' | 'round' | 'floor';
+    /**
+     * Token counting strategy.
+     * - `heuristic` (default): use chars-per-token ratios
+     * - `openai_exact`: use OpenAI BPE tokenization (throws if non-OpenAI model)
+     * - `auto`: use OpenAI BPE for OpenAI models, otherwise heuristic
+     */
+    tokenizer?: TokenizerMode;
 }
 /**
  * Output from the estimate function.
@@ -32,6 +40,10 @@ interface EstimateOutput {
     estimatedInputCost: number;
     /** The chars-per-token ratio used */
     charsPerToken: number;
+    /** Which tokenizer strategy was used */
+    tokenizerMode?: TokenizerMode;
+    /** OpenAI encoding used when tokenizerMode is `openai_exact` */
+    encodingUsed?: string;
 }
 
 /**
@@ -81,4 +93,52 @@ declare function getModelConfig(model: string): ModelConfig;
  */
 declare function getAvailableModels(): string[];
 
-export { DEFAULT_MODELS, type EstimateInput, type EstimateOutput, LAST_UPDATED, type ModelConfig, estimate, getAvailableModels, getModelConfig };
+type OpenAIEncoding = 'r50k_base' | 'p50k_base' | 'p50k_edit' | 'cl100k_base' | 'o200k_base' | 'o200k_harmony';
+type SpecialTokenHandling = 'all' | 'none' | 'none_raise';
+interface EncodeOptions {
+    /**
+     * Explicit OpenAI encoding override.
+     * When provided, this takes precedence over `model`.
+     */
+    encoding?: OpenAIEncoding;
+    /**
+     * OpenAI model ID used to select the appropriate encoding.
+     */
+    model?: string;
+    /**
+     * How special tokens are handled.
+     * - `none_raise` (default): throw if special tokens appear
+     * - `none`: treat special tokens as regular text
+     * - `all`: allow special tokens and encode them as special token IDs
+     */
+    allowSpecial?: SpecialTokenHandling;
+}
+/**
+ * Encode text into OpenAI token IDs using tiktoken-compatible BPE encoding.
+ *
+ * This is exact tokenization for OpenAI models (unlike heuristic estimators).
+ */
+declare function encode(text: string, options?: EncodeOptions): number[];
+/**
+ * Decode OpenAI token IDs into text using tiktoken-compatible BPE encoding.
+ */
+declare function decode(tokens: Iterable<number>, options?: Pick<EncodeOptions, 'encoding' | 'model'>): string;
+
+interface TokenCountInput {
+    text: string;
+    model: string;
+}
+interface TokenCountOutput {
+    tokens: number;
+    exact: boolean;
+    encoding?: OpenAIEncoding;
+}
+/**
+ * Count tokens for a given model.
+ *
+ * - OpenAI models: exact BPE tokenization
+ * - Other providers: heuristic estimate (chars-per-token)
+ */
+declare function countTokens(input: TokenCountInput): TokenCountOutput;
+
+export { DEFAULT_MODELS, type EncodeOptions, type EstimateInput, type EstimateOutput, LAST_UPDATED, type ModelConfig, type OpenAIEncoding, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, countTokens, decode, encode, estimate, getAvailableModels, getModelConfig };

package/dist/index.d.ts

@@ -7,6 +7,7 @@ interface ModelConfig {
     /** Cost in USD per 1 million input tokens */
     inputCostPerMillion: number;
 }
+type TokenizerMode = 'heuristic' | 'openai_exact' | 'auto';
 /**
  * Input parameters for the estimate function.
  */
@@ -17,6 +18,13 @@ interface EstimateInput {
     model: string;
     /** Rounding strategy for token count (default: 'ceil') */
     rounding?: 'ceil' | 'round' | 'floor';
+    /**
+     * Token counting strategy.
+     * - `heuristic` (default): use chars-per-token ratios
+     * - `openai_exact`: use OpenAI BPE tokenization (throws if non-OpenAI model)
+     * - `auto`: use OpenAI BPE for OpenAI models, otherwise heuristic
+     */
+    tokenizer?: TokenizerMode;
 }
 /**
  * Output from the estimate function.
@@ -32,6 +40,10 @@ interface EstimateOutput {
     estimatedInputCost: number;
     /** The chars-per-token ratio used */
     charsPerToken: number;
+    /** Which tokenizer strategy was used */
+    tokenizerMode?: TokenizerMode;
+    /** OpenAI encoding used when tokenizerMode is `openai_exact` */
+    encodingUsed?: string;
 }
 
 /**
@@ -81,4 +93,52 @@ declare function getModelConfig(model: string): ModelConfig;
  */
 declare function getAvailableModels(): string[];
 
-export { DEFAULT_MODELS, type EstimateInput, type EstimateOutput, LAST_UPDATED, type ModelConfig, estimate, getAvailableModels, getModelConfig };
+type OpenAIEncoding = 'r50k_base' | 'p50k_base' | 'p50k_edit' | 'cl100k_base' | 'o200k_base' | 'o200k_harmony';
+type SpecialTokenHandling = 'all' | 'none' | 'none_raise';
+interface EncodeOptions {
+    /**
+     * Explicit OpenAI encoding override.
+     * When provided, this takes precedence over `model`.
+     */
+    encoding?: OpenAIEncoding;
+    /**
+     * OpenAI model ID used to select the appropriate encoding.
+     */
+    model?: string;
+    /**
+     * How special tokens are handled.
+     * - `none_raise` (default): throw if special tokens appear
+     * - `none`: treat special tokens as regular text
+     * - `all`: allow special tokens and encode them as special token IDs
+     */
+    allowSpecial?: SpecialTokenHandling;
+}
+/**
+ * Encode text into OpenAI token IDs using tiktoken-compatible BPE encoding.
+ *
+ * This is exact tokenization for OpenAI models (unlike heuristic estimators).
+ */
+declare function encode(text: string, options?: EncodeOptions): number[];
+/**
+ * Decode OpenAI token IDs into text using tiktoken-compatible BPE encoding.
+ */
+declare function decode(tokens: Iterable<number>, options?: Pick<EncodeOptions, 'encoding' | 'model'>): string;
+
+interface TokenCountInput {
+    text: string;
+    model: string;
+}
+interface TokenCountOutput {
+    tokens: number;
+    exact: boolean;
+    encoding?: OpenAIEncoding;
+}
+/**
+ * Count tokens for a given model.
+ *
+ * - OpenAI models: exact BPE tokenization
+ * - Other providers: heuristic estimate (chars-per-token)
+ */
+declare function countTokens(input: TokenCountInput): TokenCountOutput;
+
+export { DEFAULT_MODELS, type EncodeOptions, type EstimateInput, type EstimateOutput, LAST_UPDATED, type ModelConfig, type OpenAIEncoding, type SpecialTokenHandling, type TokenCountInput, type TokenCountOutput, type TokenizerMode, countTokens, decode, encode, estimate, getAvailableModels, getModelConfig };

package/dist/index.js

@@ -374,6 +374,78 @@ function getAvailableModels() {
   return Object.keys(DEFAULT_MODELS);
 }
 
+// src/openai-bpe.ts
+import { createRequire } from "module";
+import { ALL_SPECIAL_TOKENS } from "gpt-tokenizer/constants";
+import { DEFAULT_ENCODING, modelToEncodingMap } from "gpt-tokenizer/mapping";
+var requireBase = typeof __filename === "string" && __filename.length > 0 ? __filename : import.meta.url;
+var NODE_REQUIRE = createRequire(requireBase);
+var ENCODING_MODULES = {
+  r50k_base: "gpt-tokenizer/cjs/encoding/r50k_base",
+  p50k_base: "gpt-tokenizer/cjs/encoding/p50k_base",
+  p50k_edit: "gpt-tokenizer/cjs/encoding/p50k_edit",
+  cl100k_base: "gpt-tokenizer/cjs/encoding/cl100k_base",
+  o200k_base: "gpt-tokenizer/cjs/encoding/o200k_base",
+  o200k_harmony: "gpt-tokenizer/cjs/encoding/o200k_harmony"
+};
+var encodingApiCache = /* @__PURE__ */ new Map();
+function getEncodingApi(encoding) {
+  const cached = encodingApiCache.get(encoding);
+  if (cached) return cached;
+  const modulePath = ENCODING_MODULES[encoding];
+  const mod = NODE_REQUIRE(modulePath);
+  const api = { encode: mod.encode, decode: mod.decode };
+  encodingApiCache.set(encoding, api);
+  return api;
+}
+function resolveEncoding(selector) {
+  if (selector?.encoding) {
+    return selector.encoding;
+  }
+  const model = selector?.model?.trim();
+  if (model) {
+    const mapped = modelToEncodingMap[model];
+    if (mapped) {
+      return mapped;
+    }
+  }
+  return DEFAULT_ENCODING;
+}
+function getOpenAIEncoding(selector) {
+  return resolveEncoding(selector);
+}
+function toGptTokenizerEncodeOptions(allowSpecial) {
+  const mode = allowSpecial ?? "none_raise";
+  switch (mode) {
+    case "all":
+      return {
+        allowedSpecial: ALL_SPECIAL_TOKENS,
+        disallowedSpecial: /* @__PURE__ */ new Set()
+      };
+    case "none":
+      return {
+        allowedSpecial: /* @__PURE__ */ new Set(),
+        disallowedSpecial: /* @__PURE__ */ new Set()
+      };
+    case "none_raise":
+    default:
+      return {
+        disallowedSpecial: ALL_SPECIAL_TOKENS
+      };
+  }
+}
+function encode(text, options) {
+  const encoding = resolveEncoding(options);
+  const api = getEncodingApi(encoding);
+  const encodeOptions = toGptTokenizerEncodeOptions(options?.allowSpecial);
+  return api.encode(text, encodeOptions);
+}
+function decode(tokens, options) {
+  const encoding = resolveEncoding(options);
+  const api = getEncodingApi(encoding);
+  return api.decode(tokens);
+}
+
 // src/estimator.ts
 function countCodePoints(text) {
   let count = 0;
@@ -383,21 +455,43 @@ function countCodePoints(text) {
   return count;
 }
 function estimate(input) {
-  const { text, model, rounding = "ceil" } = input;
+  const { text, model, rounding = "ceil", tokenizer = "heuristic" } = input;
   const config = getModelConfig(model);
   const characterCount = countCodePoints(text);
-  const rawTokens = characterCount / config.charsPerToken;
+  const isNonOpenAIModel2 = model.startsWith("claude-") || model.startsWith("gemini-");
   let estimatedTokens;
-  switch (rounding) {
-    case "floor":
-      estimatedTokens = Math.floor(rawTokens);
-      break;
-    case "round":
-      estimatedTokens = Math.round(rawTokens);
-      break;
-    case "ceil":
-    default:
-      estimatedTokens = Math.ceil(rawTokens);
+  let tokenizerModeUsed = "heuristic";
+  let encodingUsed;
+  const shouldTryExact = tokenizer === "openai_exact" || tokenizer === "auto";
+  if (shouldTryExact && !isNonOpenAIModel2) {
+    try {
+      estimatedTokens = encode(text, { model, allowSpecial: "none" }).length;
+      tokenizerModeUsed = "openai_exact";
+      encodingUsed = getOpenAIEncoding({ model });
+    } catch (error) {
+      if (tokenizer === "openai_exact") {
+        throw error;
+      }
+    }
+  } else if (tokenizer === "openai_exact" && isNonOpenAIModel2) {
+    throw new Error(
+      `Tokenizer mode "openai_exact" requested for non-OpenAI model: "${model}"`
+    );
+  }
+  if (estimatedTokens === void 0) {
+    const rawTokens = characterCount / config.charsPerToken;
+    switch (rounding) {
+      case "floor":
+        estimatedTokens = Math.floor(rawTokens);
+        break;
+      case "round":
+        estimatedTokens = Math.round(rawTokens);
+        break;
+      case "ceil":
+      default:
+        estimatedTokens = Math.ceil(rawTokens);
+    }
+    tokenizerModeUsed = "heuristic";
   }
   const estimatedInputCost = estimatedTokens * config.inputCostPerMillion / 1e6;
   return {
@@ -405,12 +499,43 @@ function estimate(input) {
     characterCount,
     estimatedTokens,
     estimatedInputCost,
-    charsPerToken: config.charsPerToken
+    charsPerToken: config.charsPerToken,
+    tokenizerMode: tokenizerModeUsed,
+    encodingUsed
   };
 }
+
+// src/token-counter.ts
+function isNonOpenAIModel(model) {
+  return model.startsWith("claude-") || model.startsWith("gemini-");
+}
+function countTokens(input) {
+  const { text, model } = input;
+  if (isNonOpenAIModel(model)) {
+    return {
+      tokens: estimate({ text, model }).estimatedTokens,
+      exact: false
+    };
+  }
+  try {
+    return {
+      tokens: encode(text, { model, allowSpecial: "none" }).length,
+      exact: true,
+      encoding: getOpenAIEncoding({ model })
+    };
+  } catch {
+    return {
+      tokens: estimate({ text, model }).estimatedTokens,
+      exact: false
+    };
+  }
+}
 export {
   DEFAULT_MODELS,
   LAST_UPDATED,
+  countTokens,
+  decode,
+  encode,
   estimate,
   getAvailableModels,
   getModelConfig

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-token-estimator",
-  "version": "1.0.3",
+  "version": "1.1.0",
   "description": "Estimate token counts and costs for LLM API calls",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -23,13 +23,17 @@
     "LICENSE",
     "README.md"
   ],
+  "dependencies": {
+    "gpt-tokenizer": "^3.4.0"
+  },
   "scripts": {
     "build": "tsup src/index.ts --format cjs,esm --dts",
     "test": "vitest run",
     "test:watch": "vitest",
     "lint": "eslint src tests",
     "prepublishOnly": "npm run lint && npm run test && npm run build",
-    "update-pricing": "tsx scripts/update-pricing.ts"
+    "update-pricing": "tsx scripts/update-pricing.ts",
+    "benchmark:tokenizer": "tsx benchmark/tokenizer.ts"
   },
   "keywords": [
     "llm",