bpe-lite 0.1.0 → 0.3.0

package/README.md

@@ -1,13 +1,14 @@
 # bpe-lite
 
-Offline BPE tokenizer for OpenAI, Anthropic, and Gemini. Zero dependencies, no network calls at runtime. Works in any Node 18+ environment including Docker and edge runtimes.
+Offline BPE tokenizer for OpenAI, Anthropic, and Gemini. Zero dependencies, no network calls at runtime. Works in any Node 18+ environment including Docker and edge runtimes. Ships CJS and ESM. TypeScript types included.
 
 ```js
 const { countTokens } = require('bpe-lite');
 
-countTokens('Hello, world!', 'openai')    // → 4
-countTokens('Hello, world!', 'anthropic') // → 4
-countTokens('Hello, world!', 'gemini')    // → 4
+countTokens('Hello, world!', 'openai-o200k') // → 4  (GPT-4o, o1, o3, o4, GPT-4.1, GPT-5)
+countTokens('Hello, world!', 'openai')       // → 4  (GPT-4, GPT-3.5)
+countTokens('Hello, world!', 'anthropic')    // → 4
+countTokens('Hello, world!', 'gemini')       // → 4
 ```
 
 ## Install
@@ -18,33 +19,71 @@ npm install bpe-lite
 
 ## Usage
 
+Both CommonJS and ESM are supported:
+
+```js
+// CommonJS
+const { countTokens, encode, decode, isWithinTokenLimit, openai, openaiO200k, anthropic, gemini } = require('bpe-lite');
+
+// ESM
+import { countTokens, encode, decode, isWithinTokenLimit, openai, openaiO200k, anthropic, gemini } from 'bpe-lite';
+```
+
 ```js
-const { countTokens, encode, decode, openai, anthropic, gemini } = require('bpe-lite');
 
 // Count tokens
-countTokens('Your text here', 'openai');    // → number
+countTokens('Your text here', 'openai-o200k'); // → number (GPT-4o, o1, o3, o4, GPT-4.1, GPT-5)
+countTokens('Your text here', 'openai');       // → number (GPT-4, GPT-3.5)
 
 // Encode / decode
-const ids = encode('Hello', 'openai');      // → [9906]
-decode(ids, 'openai');                      // → 'Hello'
-
-// Tokenizer instances (lazy-loaded, cached per provider)
-const tok = openai();
-tok.encode('Hello');                        // → [9906]
-tok.decode([9906]);                         // → 'Hello'
-tok.count('Hello, world!');                 // → 4
+const ids = encode('Hello', 'openai-o200k');   // → [13225]
+decode(ids, 'openai-o200k');                   // → 'Hello'
+
+// Check token limit — short-circuits early on long text, more efficient than encode()
+// Returns the token count if within the limit, false if exceeded
+isWithinTokenLimit('Hello, world!', 10, 'openai-o200k'); // → 4
+isWithinTokenLimit('Hello, world!', 3,  'openai-o200k'); // → false
+
+// Tokenizer instances — lazy-loaded and cached per provider
+const tok = openaiO200k();
+tok.encode('Hello');          // → [13225]
+tok.decode([13225]);          // → 'Hello'
+tok.count('Hello, world!');   // → 4
 ```
 
 ## Providers
 
-| Provider | Vocab | Tokens | Accuracy |
-|----------|-------|--------|----------|
-| `openai` | cl100k_base | 100,256 | Exact — vocab sourced directly from OpenAI's CDN (same source as tiktoken) |
-| `anthropic` | cl100k approximation | 100,256 | ~95% — Claude 3+ tokenizer has not been publicly released |
-| `gemini` | Gemma 3 | 262,144 | Exact — Gemini uses the same tokenizer as Gemma 3 open-weights |
+| Provider | Vocab | Tokens | Models | Accuracy |
+|----------|-------|--------|--------|----------|
+| `openai-o200k` | o200k_base | 199,998 | GPT-4o, o1, o3, o4, GPT-4.1, GPT-5 | Exact — vocab sourced directly from OpenAI's CDN |
+| `openai` | cl100k_base | 100,256 | GPT-4, GPT-3.5 | Exact — vocab sourced directly from OpenAI's CDN |
+| `anthropic` | cl100k approximation | 100,256 | Claude 3+ | ~95% — Anthropic has not released the Claude 3+ tokenizer |
+| `gemini` | Gemma 3 | 262,144 | Gemini | Exact — Gemini uses the same tokenizer as Gemma 3 open-weights |
 
 Vocab files are bundled in the package — no network required at runtime or install time.
 
+## API
+
+### `countTokens(text, provider?)`
+
+Returns the number of tokens in `text`. Default provider: `'openai'`.
+
+### `encode(text, provider?)`
+
+Returns an array of token ids.
+
+### `decode(ids, provider?)`
+
+Decodes an array of token ids back to a string.
+
+### `isWithinTokenLimit(text, limit, provider?)`
+
+Returns the token count if `text` is within `limit` tokens, or `false` if exceeded. More efficient than `encode()` for long texts — the tiktoken engine short-circuits as soon as the limit is crossed.
+
+### Tokenizer instances
+
+`openai()`, `openaiO200k()`, `anthropic()`, `gemini()` each return a cached `Tokenizer` object with `.encode()`, `.decode()`, and `.count()` methods. Instances are created once per provider per process.
+
 ## Why not tiktoken?
 
 `tiktoken` is accurate for OpenAI but requires Rust/WASM native bindings, which can break in Docker containers, edge runtimes, and serverless environments. `bpe-lite` is pure JavaScript — it runs anywhere Node 18+ runs.
@@ -52,7 +91,7 @@ Vocab files are bundled in the package — no network required at runtime or ins
 ## Caveats
 
 - **Anthropic**: Anthropic has not released the Claude 3+ tokenizer. The cl100k approximation is accurate to ~95% for most text.
-- **Speed**: Pure JS is slower than tiktoken's native implementation. For token *counting* (not bulk processing) the difference is negligible.
+- **Speed**: Pure JS is slower than tiktoken's native implementation. For token counting (not bulk processing) the difference is negligible.
 - **Node version**: Requires Node 18+ for Unicode property escapes (`\p{L}`, `\p{N}`) in the pre-tokenization regex.
 
 ## License

package/index.d.ts

@@ -0,0 +1,42 @@
+export type Provider = 'openai' | 'openai-o200k' | 'anthropic' | 'gemini';
+
+export interface Tokenizer {
+  encode(text: string): number[];
+  decode(ids: number[]): string;
+  count(text: string): number;
+  countUpTo(text: string, limit: number): number;
+}
+
+/**
+ * Count the number of tokens in text for the given provider.
+ */
+export function countTokens(text: string, provider?: Provider): number;
+
+/**
+ * Encode text to token ids.
+ */
+export function encode(text: string, provider?: Provider): number[];
+
+/**
+ * Decode token ids back to text.
+ */
+export function decode(ids: number[], provider?: Provider): string;
+
+/**
+ * Check if text is within a token limit.
+ * Returns the token count if within the limit, or false if exceeded.
+ * More efficient than encode() for long texts since it short-circuits.
+ */
+export function isWithinTokenLimit(text: string, limit: number, provider?: Provider): number | false;
+
+/** Tokenizer instance for OpenAI cl100k_base (GPT-4, GPT-3.5). */
+export function openai(): Tokenizer;
+
+/** Tokenizer instance for OpenAI o200k_base (GPT-4o, o1, o3, o4, GPT-4.1, GPT-5). */
+export function openaiO200k(): Tokenizer;
+
+/** Tokenizer instance for Anthropic (cl100k approximation, ~95% accurate). */
+export function anthropic(): Tokenizer;
+
+/** Tokenizer instance for Gemini (Gemma 3 vocab, exact). */
+export function gemini(): Tokenizer;

package/package.json

@@ -1,12 +1,21 @@
 {
   "name": "bpe-lite",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Offline BPE tokenizer for OpenAI, Anthropic, and Gemini — zero dependencies",
   "main": "src/index.js",
+  "types": "index.d.ts",
   "type": "commonjs",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "import": "./src/index.mjs",
+      "require": "./src/index.js"
+    }
+  },
   "files": [
     "src/",
-    "vocabs/"
+    "vocabs/",
+    "index.d.ts"
   ],
   "scripts": {
     "build": "node scripts/build-vocabs.js",

package/src/bpe.js

@@ -194,4 +194,35 @@ function mergeB64(a, b) {
   return buf.toString('base64');
 }
 
-module.exports = { encodeTiktoken, decodeTiktoken };
+/**
+ * Count tokens up to a limit, short-circuiting once exceeded.
+ * @param {string} text
+ * @param {Object} vocabData
+ * @param {number} limit
+ * @returns {number} token count (may exceed limit by up to one chunk)
+ */
+function countTiktokenUpTo(text, vocabData, limit) {
+  if (!text) return 0;
+
+  const { vocab, specialTokens = {}, pattern } = vocabData;
+  const specials = Object.entries(specialTokens);
+  const pieces = splitOnSpecials(text, specials);
+
+  let count = 0;
+  for (const piece of pieces) {
+    if (piece.isSpecial) {
+      count++;
+    } else {
+      const chunks = pretokenize(piece.text, pattern);
+      for (const chunk of chunks) {
+        count += bpeEncode(chunk, vocab).length;
+        if (count > limit) return count;
+      }
+    }
+    if (count > limit) return count;
+  }
+
+  return count;
+}
+
+module.exports = { encodeTiktoken, decodeTiktoken, countTiktokenUpTo };

package/src/index.js

@@ -2,6 +2,7 @@
 
 const path = require('path');
 const fs = require('fs');
+const zlib = require('zlib');
 const { Tokenizer } = require('./tokenizer');
 
 const VOCABS_DIR = path.join(__dirname, '..', 'vocabs');
@@ -12,15 +13,21 @@ const _cache = {};
 function loadTokenizer(provider) {
   if (_cache[provider]) return _cache[provider];
 
-  const filePath = path.join(VOCABS_DIR, `${provider}.json`);
-  if (!fs.existsSync(filePath)) {
+  const gzPath   = path.join(VOCABS_DIR, `${provider}.json.gz`);
+  const jsonPath  = path.join(VOCABS_DIR, `${provider}.json`);
+
+  let data;
+  if (fs.existsSync(gzPath)) {
+    data = JSON.parse(zlib.gunzipSync(fs.readFileSync(gzPath)).toString('utf8'));
+  } else if (fs.existsSync(jsonPath)) {
+    data = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
+  } else {
     throw new Error(
-      `Vocab file not found for provider "${provider}": ${filePath}\n` +
+      `Vocab file not found for provider "${provider}".\n` +
       'Run "node scripts/build-vocabs.js" to build vocab files.'
     );
   }
 
-  const data = JSON.parse(fs.readFileSync(filePath, 'utf8'));
   _cache[provider] = new Tokenizer(data);
   return _cache[provider];
 }
@@ -48,20 +55,36 @@ function encode(text, provider = 'openai') {
 /**
  * Decode token ids back to text.
  * @param {number[]} ids
- * @param {'openai'|'anthropic'|'gemini'} provider
+ * @param {'openai'|'openai-o200k'|'anthropic'|'gemini'} provider
  * @returns {string}
  */
 function decode(ids, provider = 'openai') {
   return loadTokenizer(provider).decode(ids);
 }
 
-/** Get a Tokenizer instance for OpenAI (cl100k_base). */
+/**
+ * Check if text is within a token limit without necessarily encoding the whole string.
+ * Returns false if the limit is exceeded, otherwise returns the token count.
+ * @param {string} text
+ * @param {number} limit
+ * @param {'openai'|'openai-o200k'|'anthropic'|'gemini'} provider
+ * @returns {number|false}
+ */
+function isWithinTokenLimit(text, limit, provider = 'openai') {
+  const count = loadTokenizer(provider).countUpTo(text, limit);
+  return count <= limit ? count : false;
+}
+
+/** Get a Tokenizer instance for OpenAI (cl100k_base — GPT-4, GPT-3.5). */
 function openai() { return loadTokenizer('openai'); }
 
+/** Get a Tokenizer instance for OpenAI modern models (o200k_base — GPT-4o, o1, o3, o4, GPT-4.1, GPT-5). */
+function openaiO200k() { return loadTokenizer('openai-o200k'); }
+
 /** Get a Tokenizer instance for Anthropic (cl100k approximation). */
 function anthropic() { return loadTokenizer('anthropic'); }
 
 /** Get a Tokenizer instance for Gemini (Gemma3 vocab). */
 function gemini() { return loadTokenizer('gemini'); }
 
-module.exports = { countTokens, encode, decode, openai, anthropic, gemini };
+module.exports = { countTokens, encode, decode, isWithinTokenLimit, openai, openaiO200k, anthropic, gemini };

package/src/index.mjs

@@ -0,0 +1,24 @@
+import { createRequire } from 'module';
+
+const require = createRequire(import.meta.url);
+const {
+  countTokens,
+  encode,
+  decode,
+  isWithinTokenLimit,
+  openai,
+  openaiO200k,
+  anthropic,
+  gemini,
+} = require('./index.js');
+
+export {
+  countTokens,
+  encode,
+  decode,
+  isWithinTokenLimit,
+  openai,
+  openaiO200k,
+  anthropic,
+  gemini,
+};

package/src/tokenizer.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { encodeTiktoken, decodeTiktoken } = require('./bpe');
+const { encodeTiktoken, decodeTiktoken, countTiktokenUpTo } = require('./bpe');
 const { encodeSPM, decodeSPM } = require('./spm');
 
 class Tokenizer {
@@ -26,6 +26,19 @@ class Tokenizer {
   count(text) {
     return this.encode(text).length;
   }
+
+  /**
+   * Count tokens, stopping as soon as the count exceeds limit.
+   * More efficient than encode() for token limit checks on long text.
+   * @param {string} text
+   * @param {number} limit
+   * @returns {number}
+   */
+  countUpTo(text, limit) {
+    if (this._engine === 'tiktoken') return countTiktokenUpTo(text, this._data, limit);
+    // SPM encodes the whole text as one unit — no clean early exit, just encode and count
+    return this.encode(text).length;
+  }
 }
 
 module.exports = { Tokenizer };