bpe-lite 0.4.2 → 0.5.1

package/README.md

@@ -57,8 +57,8 @@ tok.count('Hello, world!');   // → 4
 |----------|-------|--------|--------|----------|
 | `openai-o200k` | o200k_base | 199,998 | GPT-4o, o1, o3, o4, GPT-4.1, GPT-5 | Exact — vocab sourced directly from OpenAI's CDN |
 | `openai` | cl100k_base | 100,256 | GPT-4, GPT-3.5 | Exact — vocab sourced directly from OpenAI's CDN |
-| `anthropic` | cl100k approximation | 100,256 | Claude 3+ | ~95% — Anthropic has not released the Claude 3+ tokenizer |
-| `gemini` | Gemma 3 | 262,144 | Gemini | Exact — Gemini uses the same tokenizer as Gemma 3 open-weights |
+| `anthropic` | Xenova/claude-tokenizer | 65,000 | Claude | ~80% within 10%, mean error 7.0% — Anthropic has not released the tokenizer. Vocab sourced from Xenova's reverse-engineered 65k BPE (HuggingFace) with NFKC normalization, symbol/emoji byte-level corrections, whitespace sequence injections, and probe-based merge adjustments. Typical prose and code: well within 10%. Outliers: repeated chars (39%), Arabic (20%), symbols (16%), currency (14%). Tested against Claude 4 API across 25 diverse samples. |
+| `gemini` | Gemma 3 | 262,144 | Gemini | ~99.7% — 92% exact, mean error 0.34% across 25 diverse samples vs Gemini 2.0 Flash API |
 
 Vocab files are bundled in the package — no network required at runtime or install time.
 
@@ -70,23 +70,23 @@ Benchmarked on Node v24 (win32/x64). Benchmark command: `node --expose-gc script
 
 | impl | ops/s | tokens/s | MB/s |
 |------|------:|---------:|-----:|
-| bpe-lite | **257** | **3.15M** | **13.6** |
-| ai-tokenizer | 201 | 2.46M | 10.7 |
-| js-tiktoken | 23 | 282k | 1.2 |
+| bpe-lite | **291** | **3.56M** | **15.4** |
+| ai-tokenizer | 291 | 3.56M | 15.4 |
+| js-tiktoken | 30 | 370k | 1.6 |
 
 **Anthropic — large text (~54 KB)**
 
 | impl | ops/s | tokens/s | MB/s |
 |------|------:|---------:|-----:|
-| bpe-lite | **257** | 3.15M | **13.6** |
-| ai-tokenizer | 253 | **4.62M** | 13.4 |
+| bpe-lite | **289** | **5.34M** | **15.3** |
+| ai-tokenizer | 291 | 5.31M | 15.4 |
 
-**Gemini — large text (8 KB)**
+**Gemini — large text (~54 KB)**
 
 | impl | ops/s | tokens/s | MB/s | note |
 |------|------:|---------:|-----:|------|
-| bpe-lite | **3,800** | **6.23M** | **29.7** | actual Gemma3 SPM |
-| ai-tokenizer | 1,220 | 2.01M | 9.6 | o200k BPE — different algorithm, different results |
+| bpe-lite | **998** | **11.1M** | **52.9** | actual Gemma3 SPM |
+| ai-tokenizer | 215 | 2.40M | 11.4 | o200k BPE — different algorithm, different results |
 
 ai-tokenizer does not implement Gemini tokenization. The row above uses their o200k encoding on the same input string; it produces different token ids and counts than the Gemini tokenizer, so it is not a real comparison.
 
@@ -120,7 +120,7 @@ Returns the token count if `text` is within `limit` tokens, or `false` if exceed
 
 ## Caveats
 
-- **Anthropic**: Anthropic has not released the Claude 3+ tokenizer. The cl100k approximation is accurate to ~95% for most text.
+- **Anthropic**: Anthropic has not released the Claude tokenizer. The vocab is sourced from [Xenova/claude-tokenizer](https://huggingface.co/Xenova/claude-tokenizer), a community reverse-engineering of the ~65k BPE vocab. NFKC normalization and probe-based merge corrections are applied. Accuracy varies by text type — common prose and code are usually within 10%, but Arabic, Japanese, repeated characters, and some symbol/emoji combinations diverge significantly.
 - **Node version**: Requires Node 18+ for Unicode property escapes (`\p{L}`, `\p{N}`) in the pre-tokenization regex.
 
 ## License

package/index.d.ts

@@ -1,42 +1,42 @@
-export type Provider = 'openai' | 'openai-o200k' | 'anthropic' | 'gemini';
-
-export interface Tokenizer {
-  encode(text: string): number[];
-  decode(ids: number[]): string;
-  count(text: string): number;
-  countUpTo(text: string, limit: number): number;
-}
-
-/**
- * Count the number of tokens in text for the given provider.
- */
-export function countTokens(text: string, provider?: Provider): number;
-
-/**
- * Encode text to token ids.
- */
-export function encode(text: string, provider?: Provider): number[];
-
-/**
- * Decode token ids back to text.
- */
-export function decode(ids: number[], provider?: Provider): string;
-
-/**
- * Check if text is within a token limit.
- * Returns the token count if within the limit, or false if exceeded.
- * More efficient than encode() for long texts since it short-circuits.
- */
-export function isWithinTokenLimit(text: string, limit: number, provider?: Provider): number | false;
-
-/** Tokenizer instance for OpenAI cl100k_base (GPT-4, GPT-3.5). */
-export function openai(): Tokenizer;
-
-/** Tokenizer instance for OpenAI o200k_base (GPT-4o, o1, o3, o4, GPT-4.1, GPT-5). */
-export function openaiO200k(): Tokenizer;
-
-/** Tokenizer instance for Anthropic (cl100k approximation, ~95% accurate). */
-export function anthropic(): Tokenizer;
-
-/** Tokenizer instance for Gemini (Gemma 3 vocab, exact). */
-export function gemini(): Tokenizer;
+export type Provider = 'openai' | 'openai-o200k' | 'anthropic' | 'gemini';
+
+export interface Tokenizer {
+  encode(text: string): number[];
+  decode(ids: number[]): string;
+  count(text: string): number;
+  countUpTo(text: string, limit: number): number;
+}
+
+/**
+ * Count the number of tokens in text for the given provider.
+ */
+export function countTokens(text: string, provider?: Provider): number;
+
+/**
+ * Encode text to token ids.
+ */
+export function encode(text: string, provider?: Provider): number[];
+
+/**
+ * Decode token ids back to text.
+ */
+export function decode(ids: number[], provider?: Provider): string;
+
+/**
+ * Check if text is within a token limit.
+ * Returns the token count if within the limit, or false if exceeded.
+ * More efficient than encode() for long texts since it short-circuits.
+ */
+export function isWithinTokenLimit(text: string, limit: number, provider?: Provider): number | false;
+
+/** Tokenizer instance for OpenAI cl100k_base (GPT-4, GPT-3.5). */
+export function openai(): Tokenizer;
+
+/** Tokenizer instance for OpenAI o200k_base (GPT-4o, o1, o3, o4, GPT-4.1, GPT-5). */
+export function openaiO200k(): Tokenizer;
+
+/** Tokenizer instance for Anthropic (cl100k approximation, ~95% accurate). */
+export function anthropic(): Tokenizer;
+
+/** Tokenizer instance for Gemini (Gemma 3 vocab, exact). */
+export function gemini(): Tokenizer;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bpe-lite",
-  "version": "0.4.2",
+  "version": "0.5.1",
   "description": "Offline BPE tokenizer for OpenAI, Anthropic, and Gemini — zero dependencies",
   "main": "src/index.js",
   "types": "index.d.ts",
@@ -36,6 +36,11 @@
   "license": "MIT",
   "devDependencies": {
     "ai-tokenizer": "^1.0.6",
+    "gpt-tokenizer": "^3.4.0",
     "js-tiktoken": "^1.0.21"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/jschoemaker/bpe-lite"
   }
 }

package/src/bpe.js

@@ -80,7 +80,8 @@ class MinHeap {
   _siftUp(i) {
     while (i > 0) {
       const p = (i - 1) >> 1;
-      if (this.ranks[p] <= this.ranks[i]) break;
+      if (this.ranks[p] < this.ranks[i]) break;
+      if (this.ranks[p] === this.ranks[i] && this.left[p] <= this.left[i]) break;
       this._swap(i, p);
       i = p;
     }
@@ -93,8 +94,12 @@ class MinHeap {
       if (l >= n) break;
       const r = l + 1;
       let m = l;
-      if (r < n && this.ranks[r] < this.ranks[l]) m = r;
-      if (this.ranks[i] <= this.ranks[m]) break;
+      if (r < n) {
+        if (this.ranks[r] < this.ranks[l] ||
+            (this.ranks[r] === this.ranks[l] && this.left[r] < this.left[l])) m = r;
+      }
+      if (this.ranks[i] < this.ranks[m]) break;
+      if (this.ranks[i] === this.ranks[m] && this.left[i] <= this.left[m]) break;
       this._swap(i, m);
       i = m;
     }
@@ -136,7 +141,7 @@ function pretokenize(text, compiled) {
 }
 
 function buildPreparedTiktoken(vocabData) {
-  const { vocab, specialTokens = {}, pattern } = vocabData;
+  const { vocab, specialTokens = {}, pattern, normalize } = vocabData;
 
   const vocabBin = new Map();
   let maxId = -1;
@@ -164,6 +169,7 @@ function buildPreparedTiktoken(vocabData) {
     idToBytes,
     specials,
     patternCompiled: compilePretokenizer(pattern),
+    normalize: normalize || null,
     // opt A — per-instance chunk cache: chunk string → ids[]
     cache: new Map(),
     // opt B — per-instance grow-only scratch (reused across chunks)
@@ -302,6 +308,7 @@ function splitOnSpecials(text, specials) {
 
 function encodeTiktokenPrepared(text, prepared) {
   if (!text) return [];
+  if (prepared.normalize) text = text.normalize(prepared.normalize);
 
   const ids = [];
   const { vocabBin, scratch, cache, patternCompiled, specials } = prepared;
@@ -355,6 +362,7 @@ function decodeTiktokenPrepared(ids, prepared) {
 
 function countTiktokenPrepared(text, prepared) {
   if (!text) return 0;
+  if (prepared.normalize) text = text.normalize(prepared.normalize);
 
   const { vocabBin, scratch, cache, patternCompiled, specials } = prepared;
   const pieces = splitOnSpecials(text, specials);
@@ -396,6 +404,7 @@ function countTiktokenPrepared(text, prepared) {
 
 function countTiktokenUpToPrepared(text, prepared, limit) {
   if (!text) return 0;
+  if (prepared.normalize) text = text.normalize(prepared.normalize);
 
   const { vocabBin, scratch, cache, patternCompiled, specials } = prepared;
   const pieces = splitOnSpecials(text, specials);

package/src/index.js

@@ -1,90 +1,90 @@
-'use strict';
-
-const path = require('path');
-const fs = require('fs');
-const zlib = require('zlib');
-const { Tokenizer } = require('./tokenizer');
-
-const VOCABS_DIR = path.join(__dirname, '..', 'vocabs');
-
-// Lazy-loaded tokenizer instances (created once per provider per process)
-const _cache = {};
-
-function loadTokenizer(provider) {
-  if (_cache[provider]) return _cache[provider];
-
-  const gzPath   = path.join(VOCABS_DIR, `${provider}.json.gz`);
-  const jsonPath  = path.join(VOCABS_DIR, `${provider}.json`);
-
-  let data;
-  if (fs.existsSync(gzPath)) {
-    data = JSON.parse(zlib.gunzipSync(fs.readFileSync(gzPath)).toString('utf8'));
-  } else if (fs.existsSync(jsonPath)) {
-    data = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
-  } else {
-    throw new Error(
-      `Vocab file not found for provider "${provider}".\n` +
-      'Run "node scripts/build-vocabs.js" to build vocab files.'
-    );
-  }
-
-  _cache[provider] = new Tokenizer(data);
-  return _cache[provider];
-}
-
-/**
- * Count tokens in text for a given provider.
- * @param {string} text
- * @param {'openai'|'anthropic'|'gemini'} provider
- * @returns {number}
- */
-function countTokens(text, provider = 'openai') {
-  return loadTokenizer(provider).count(text);
-}
-
-/**
- * Encode text to token ids.
- * @param {string} text
- * @param {'openai'|'anthropic'|'gemini'} provider
- * @returns {number[]}
- */
-function encode(text, provider = 'openai') {
-  return loadTokenizer(provider).encode(text);
-}
-
-/**
- * Decode token ids back to text.
- * @param {number[]} ids
- * @param {'openai'|'openai-o200k'|'anthropic'|'gemini'} provider
- * @returns {string}
- */
-function decode(ids, provider = 'openai') {
-  return loadTokenizer(provider).decode(ids);
-}
-
-/**
- * Check if text is within a token limit without necessarily encoding the whole string.
- * Returns false if the limit is exceeded, otherwise returns the token count.
- * @param {string} text
- * @param {number} limit
- * @param {'openai'|'openai-o200k'|'anthropic'|'gemini'} provider
- * @returns {number|false}
- */
-function isWithinTokenLimit(text, limit, provider = 'openai') {
-  const count = loadTokenizer(provider).countUpTo(text, limit);
-  return count <= limit ? count : false;
-}
-
-/** Get a Tokenizer instance for OpenAI (cl100k_base — GPT-4, GPT-3.5). */
-function openai() { return loadTokenizer('openai'); }
-
-/** Get a Tokenizer instance for OpenAI modern models (o200k_base — GPT-4o, o1, o3, o4, GPT-4.1, GPT-5). */
-function openaiO200k() { return loadTokenizer('openai-o200k'); }
-
-/** Get a Tokenizer instance for Anthropic (cl100k approximation). */
-function anthropic() { return loadTokenizer('anthropic'); }
-
-/** Get a Tokenizer instance for Gemini (Gemma3 vocab). */
-function gemini() { return loadTokenizer('gemini'); }
-
-module.exports = { countTokens, encode, decode, isWithinTokenLimit, openai, openaiO200k, anthropic, gemini };
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const zlib = require('zlib');
+const { Tokenizer } = require('./tokenizer');
+
+const VOCABS_DIR = path.join(__dirname, '..', 'vocabs');
+
+// Lazy-loaded tokenizer instances (created once per provider per process)
+const _cache = {};
+
+function loadTokenizer(provider) {
+  if (_cache[provider]) return _cache[provider];
+
+  const gzPath   = path.join(VOCABS_DIR, `${provider}.json.gz`);
+  const jsonPath  = path.join(VOCABS_DIR, `${provider}.json`);
+
+  let data;
+  if (fs.existsSync(gzPath)) {
+    data = JSON.parse(zlib.gunzipSync(fs.readFileSync(gzPath)).toString('utf8'));
+  } else if (fs.existsSync(jsonPath)) {
+    data = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
+  } else {
+    throw new Error(
+      `Vocab file not found for provider "${provider}".\n` +
+      'Run "node scripts/build-vocabs.js" to build vocab files.'
+    );
+  }
+
+  _cache[provider] = new Tokenizer(data);
+  return _cache[provider];
+}
+
+/**
+ * Count tokens in text for a given provider.
+ * @param {string} text
+ * @param {'openai'|'anthropic'|'gemini'} provider
+ * @returns {number}
+ */
+function countTokens(text, provider = 'openai') {
+  return loadTokenizer(provider).count(text);
+}
+
+/**
+ * Encode text to token ids.
+ * @param {string} text
+ * @param {'openai'|'anthropic'|'gemini'} provider
+ * @returns {number[]}
+ */
+function encode(text, provider = 'openai') {
+  return loadTokenizer(provider).encode(text);
+}
+
+/**
+ * Decode token ids back to text.
+ * @param {number[]} ids
+ * @param {'openai'|'openai-o200k'|'anthropic'|'gemini'} provider
+ * @returns {string}
+ */
+function decode(ids, provider = 'openai') {
+  return loadTokenizer(provider).decode(ids);
+}
+
+/**
+ * Check if text is within a token limit without necessarily encoding the whole string.
+ * Returns false if the limit is exceeded, otherwise returns the token count.
+ * @param {string} text
+ * @param {number} limit
+ * @param {'openai'|'openai-o200k'|'anthropic'|'gemini'} provider
+ * @returns {number|false}
+ */
+function isWithinTokenLimit(text, limit, provider = 'openai') {
+  const count = loadTokenizer(provider).countUpTo(text, limit);
+  return count <= limit ? count : false;
+}
+
+/** Get a Tokenizer instance for OpenAI (cl100k_base — GPT-4, GPT-3.5). */
+function openai() { return loadTokenizer('openai'); }
+
+/** Get a Tokenizer instance for OpenAI modern models (o200k_base — GPT-4o, o1, o3, o4, GPT-4.1, GPT-5). */
+function openaiO200k() { return loadTokenizer('openai-o200k'); }
+
+/** Get a Tokenizer instance for Anthropic (cl100k approximation). */
+function anthropic() { return loadTokenizer('anthropic'); }
+
+/** Get a Tokenizer instance for Gemini (Gemma3 vocab). */
+function gemini() { return loadTokenizer('gemini'); }
+
+module.exports = { countTokens, encode, decode, isWithinTokenLimit, openai, openaiO200k, anthropic, gemini };

package/src/index.mjs

@@ -1,24 +1,24 @@
-import { createRequire } from 'module';
-
-const require = createRequire(import.meta.url);
-const {
-  countTokens,
-  encode,
-  decode,
-  isWithinTokenLimit,
-  openai,
-  openaiO200k,
-  anthropic,
-  gemini,
-} = require('./index.js');
-
-export {
-  countTokens,
-  encode,
-  decode,
-  isWithinTokenLimit,
-  openai,
-  openaiO200k,
-  anthropic,
-  gemini,
-};
+import { createRequire } from 'module';
+
+const require = createRequire(import.meta.url);
+const {
+  countTokens,
+  encode,
+  decode,
+  isWithinTokenLimit,
+  openai,
+  openaiO200k,
+  anthropic,
+  gemini,
+} = require('./index.js');
+
+export {
+  countTokens,
+  encode,
+  decode,
+  isWithinTokenLimit,
+  openai,
+  openaiO200k,
+  anthropic,
+  gemini,
+};

package/src/spm.js

@@ -23,12 +23,31 @@ function buildPreparedSPM(vocabData) {
     idToStr.set(id, str);
   }
 
+  // Seed tokens: multi-char vocab entries with no producing merge.
+  // In the original SentencePiece model these were user-defined symbols
+  // (HTML tags, special tokens, etc.) that the encoder recognizes atomically
+  // before BPE. We handle the common '<' case with a greedy longest-match
+  // lookup keyed by '<' — the only first-char with multiple seed tokens.
+  const producible = new Set();
+  for (const m of merges) {
+    const sp = m.indexOf(' ');
+    if (sp !== -1) producible.add(m.slice(0, sp) + m.slice(sp + 1));
+  }
+  const seedsByAngleBracket = [];
+  for (const [str, id] of Object.entries(vocab)) {
+    if (str.length > 1 && str[0] === '<' && !producible.has(str)) {
+      seedsByAngleBracket.push({ str, id, chars: [...str] });
+    }
+  }
+  seedsByAngleBracket.sort((a, b) => b.chars.length - a.chars.length);
+
   return {
     vocab,
     merges,
     mergeRank,
     idToStr,
-    // opt A — segment-level cache: each ▁-prefixed word segment → ids[]
+    seedsByAngleBracket,
+    // opt A — segment-level cache: each word segment → ids[]
     // Generalises across different inputs (same words reused across texts).
     // Note: 1 of 514,906 Gemma merges crosses a ▁ boundary ("> ▁</"),
     // making this negligibly imprecise for that HTML pattern.
@@ -66,24 +85,55 @@ function encodeSPM(text, vocabData) {
  * Returns true if every segment was a cache hit; false on first miss.
  * Isolated from encodeSegment so V8 can keep this function optimised
  * even when encodeSPMPrepared is called with wildly different text lengths.
+ *
+ * Segmentation: for a run of N consecutive ▁ chars before a word:
+ *   N=1 → one segment [▁word]
+ *   N>1 → two segments [▁×(N-1), ▁word]
+ * This matches how Gemma BPE merges multi-space tokens (▁▁=138, ▁▁▁=139, …).
  */
 function _scanFromCache(normalized, cache, result) {
-  let segStart = 0;
-  for (let i = 1; i <= normalized.length; i++) {
-    if (i === normalized.length || normalized[i] === SPACE_CHAR) {
-      const seg = normalized.slice(segStart, i);
-      const segIds = cache.get(seg);
-      if (segIds === undefined) return false; // cache miss → caller handles cold path
-      for (let j = 0; j < segIds.length; j++) result.push(segIds[j]);
-      segStart = i;
+  let i = 0;
+  while (i < normalized.length) {
+    // Count leading ▁ chars
+    const segStart = i;
+    while (i < normalized.length && normalized[i] === SPACE_CHAR) i++;
+    const spaceCount = i - segStart;
+
+    if (i === normalized.length) {
+      // Trailing spaces only
+      if (spaceCount > 0) {
+        const seg = normalized.slice(segStart, i);
+        const segIds = cache.get(seg);
+        if (segIds === undefined) return false;
+        for (let j = 0; j < segIds.length; j++) result.push(segIds[j]);
+      }
+      break;
+    }
+
+    // Collect word chars (non-▁)
+    while (i < normalized.length && normalized[i] !== SPACE_CHAR) i++;
+
+    // If N>1 spaces, emit (N-1) ▁s as standalone segment first
+    if (spaceCount > 1) {
+      const spaceSeg = normalized.slice(segStart, segStart + spaceCount - 1);
+      const spaceIds = cache.get(spaceSeg);
+      if (spaceIds === undefined) return false;
+      for (let j = 0; j < spaceIds.length; j++) result.push(spaceIds[j]);
     }
+
+    // Emit ▁ + word as one segment
+    const wordStart = spaceCount > 0 ? segStart + spaceCount - 1 : segStart;
+    const wordSeg = normalized.slice(wordStart, i);
+    const wordIds = cache.get(wordSeg);
+    if (wordIds === undefined) return false;
+    for (let j = 0; j < wordIds.length; j++) result.push(wordIds[j]);
   }
   return true;
 }
 
 // Cold-path helper — kept separate so it is never inlined into the hot loop.
-function _encodeAndCache(seg, vocab, mergeRank, scratch, cache) {
-  const ids = encodeSegment(seg, vocab, mergeRank, scratch);
+function _encodeAndCache(seg, vocab, mergeRank, scratch, cache, seeds) {
+  const ids = encodeSegment(seg, vocab, mergeRank, scratch, seeds);
   cache.set(seg, ids);
   return ids;
 }
@@ -91,10 +141,10 @@ function _encodeAndCache(seg, vocab, mergeRank, scratch, cache) {
 function encodeSPMPrepared(text, prepared) {
   if (!text) return [];
 
-  const { vocab, mergeRank, scratch, cache } = prepared;
+  const { vocab, mergeRank, scratch, cache, seedsByAngleBracket } = prepared;
 
-  // Normalize: replace spaces with ▁, prepend ▁
-  const normalized = SPACE_CHAR + text.replace(/ /g, SPACE_CHAR);
+  // Normalize: replace spaces with ▁ (Gemma3: no ▁ prepend for first char)
+  const normalized = text.replace(/ /g, SPACE_CHAR);
 
   // Fast path: serve every segment from the segment cache.
   // After the first call, this path handles all subsequent calls for common text.
@@ -104,20 +154,41 @@ function encodeSPMPrepared(text, prepared) {
   // Cold path: at least one segment is missing — encode everything from scratch.
   // (Simpler to re-scan than to continue from the miss point.)
   result.length = 0;
-  let segStart = 0;
-  for (let i = 1; i <= normalized.length; i++) {
-    if (i === normalized.length || normalized[i] === SPACE_CHAR) {
-      const seg = normalized.slice(segStart, i);
-      const segIds = cache.get(seg) ?? _encodeAndCache(seg, vocab, mergeRank, scratch, cache);
-      for (let j = 0; j < segIds.length; j++) result.push(segIds[j]);
-      segStart = i;
+  let i = 0;
+  while (i < normalized.length) {
+    const segStart = i;
+    while (i < normalized.length && normalized[i] === SPACE_CHAR) i++;
+    const spaceCount = i - segStart;
+
+    if (i === normalized.length) {
+      if (spaceCount > 0) {
+        const seg = normalized.slice(segStart, i);
+        const segIds = cache.get(seg) ?? _encodeAndCache(seg, vocab, mergeRank, scratch, cache);
+        for (let j = 0; j < segIds.length; j++) result.push(segIds[j]);
+      }
+      break;
+    }
+
+    while (i < normalized.length && normalized[i] !== SPACE_CHAR) i++;
+
+    if (spaceCount > 1) {
+      const spaceSeg = normalized.slice(segStart, segStart + spaceCount - 1);
+      const spaceIds = cache.get(spaceSeg) ?? _encodeAndCache(spaceSeg, vocab, mergeRank, scratch, cache);
+      for (let j = 0; j < spaceIds.length; j++) result.push(spaceIds[j]);
     }
+
+    const wordStart = spaceCount > 0 ? segStart + spaceCount - 1 : segStart;
+    const wordSeg = normalized.slice(wordStart, i);
+    const wordIds = cache.get(wordSeg) ?? _encodeAndCache(wordSeg, vocab, mergeRank, scratch, cache);
+    for (let j = 0; j < wordIds.length; j++) result.push(wordIds[j]);
   }
   return result;
 }
 
-// Encode a single ▁-prefixed segment using MinHeap BPE.
-function encodeSegment(seg, vocab, mergeRank, scratch) {
+// Encode a single segment using MinHeap BPE.
+// seeds: sorted array of {str, id, chars[]} for '<'-prefixed vocab entries that have
+// no producing merge (SentencePiece user-defined symbols); matched greedily before BPE.
+function encodeSegment(seg, vocab, mergeRank, scratch, seeds) {
   const chars = [...seg];
   const n = chars.length;
 
@@ -125,32 +196,59 @@ function encodeSegment(seg, vocab, mergeRank, scratch) {
   const { str, ids, prev, next, ver, alive, heap } = scratch;
   heap.reset();
 
-  for (let i = 0; i < n; i++) {
-    const c = chars[i];
-    prev[i]  = i - 1;
-    next[i]  = i + 1;
-    ver[i]   = 0;
-    alive[i] = 1;
+  // Initialize nodes: one node per Unicode char, except seed tokens which collapse
+  // multiple chars into a single node (longest match at each '<' position).
+  let nNodes = 0;
+  let pos = 0;
+  while (pos < n) {
+    const node = nNodes++;
+    prev[node] = node - 1;
+    next[node] = node + 1;
+    ver[node]  = 0;
+    alive[node] = 1;
+
+    let matched = false;
+    if (seeds && chars[pos] === '<') {
+      for (const seed of seeds) {
+        const sl = seed.chars.length;
+        if (pos + sl > n) continue;
+        let ok = true;
+        for (let k = 0; k < sl; k++) {
+          if (chars[pos + k] !== seed.chars[k]) { ok = false; break; }
+        }
+        if (ok) {
+          str[node] = seed.str;
+          ids[node] = seed.id;
+          pos += sl;
+          matched = true;
+          break;
+        }
+      }
+    }
 
-    if (vocab[c] !== undefined) {
-      str[i] = c;
-      ids[i] = vocab[c];
-    } else {
-      const codePoint = c.codePointAt(0);
-      const hex = codePoint.toString(16).toUpperCase().padStart(2, '0');
-      const byteKey = `<0x${hex}>`;
-      if (vocab[byteKey] !== undefined) {
-        str[i] = byteKey;
-        ids[i] = vocab[byteKey];
+    if (!matched) {
+      const c = chars[pos];
+      if (vocab[c] !== undefined) {
+        str[node] = c;
+        ids[node] = vocab[c];
       } else {
-        str[i] = c;
-        ids[i] = vocab['<unk>'] ?? 0;
+        const codePoint = c.codePointAt(0);
+        const hex = codePoint.toString(16).toUpperCase().padStart(2, '0');
+        const byteKey = `<0x${hex}>`;
+        if (vocab[byteKey] !== undefined) {
+          str[node] = byteKey;
+          ids[node] = vocab[byteKey];
+        } else {
+          str[node] = c;
+          ids[node] = vocab['<unk>'] ?? 0;
+        }
       }
+      pos++;
     }
   }
-  next[n - 1] = -1;
+  next[nNodes - 1] = -1;
 
-  for (let i = 0; i < n - 1; i++) {
+  for (let i = 0; i < nNodes - 1; i++) {
     const rank = mergeRank.get(`${str[i]} ${str[i + 1]}`);
     if (rank !== undefined) heap.push(rank, i, i + 1, ver[i], ver[i + 1]);
   }

package/src/tokenizer.js

@@ -1,59 +1,59 @@
-'use strict';
-
-const {
-  buildPreparedTiktoken,
-  encodeTiktokenPrepared,
-  decodeTiktokenPrepared,
-  countTiktokenPrepared,
-  countTiktokenUpToPrepared,
-} = require('./bpe');
-const { buildPreparedSPM, encodeSPMPrepared, decodeSPMPrepared } = require('./spm');
-
-class Tokenizer {
-  constructor(vocabData) {
-    this._data = vocabData;
-    this._engine = vocabData.engine;
-    this._preparedTiktoken = null;
-    this._preparedSPM = null;
-
-    if (this._engine !== 'tiktoken' && this._engine !== 'spm') {
-      throw new Error(`Unknown tokenizer engine: ${this._engine}`);
-    }
-
-    if (this._engine === 'tiktoken') {
-      this._preparedTiktoken = buildPreparedTiktoken(vocabData);
-    } else {
-      this._preparedSPM = buildPreparedSPM(vocabData);
-    }
-  }
-
-  encode(text) {
-    if (this._engine === 'tiktoken') return encodeTiktokenPrepared(text, this._preparedTiktoken);
-    return encodeSPMPrepared(text, this._preparedSPM);
-  }
-
-  decode(ids) {
-    if (this._engine === 'tiktoken') return decodeTiktokenPrepared(ids, this._preparedTiktoken);
-    return decodeSPMPrepared(ids, this._preparedSPM);
-  }
-
-  count(text) {
-    if (this._engine === 'tiktoken') return countTiktokenPrepared(text, this._preparedTiktoken);
-    return this.encode(text).length;
-  }
-
-  /**
-   * Count tokens, stopping as soon as the count exceeds limit.
-   * More efficient than encode() for token limit checks on long text.
-   * @param {string} text
-   * @param {number} limit
-   * @returns {number}
-   */
-  countUpTo(text, limit) {
-    if (this._engine === 'tiktoken') return countTiktokenUpToPrepared(text, this._preparedTiktoken, limit);
-    // SPM encodes the whole text as one unit — no clean early exit, just encode and count
-    return this.encode(text).length;
-  }
-}
-
-module.exports = { Tokenizer };
+'use strict';
+
+const {
+  buildPreparedTiktoken,
+  encodeTiktokenPrepared,
+  decodeTiktokenPrepared,
+  countTiktokenPrepared,
+  countTiktokenUpToPrepared,
+} = require('./bpe');
+const { buildPreparedSPM, encodeSPMPrepared, decodeSPMPrepared } = require('./spm');
+
+class Tokenizer {
+  constructor(vocabData) {
+    this._data = vocabData;
+    this._engine = vocabData.engine;
+    this._preparedTiktoken = null;
+    this._preparedSPM = null;
+
+    if (this._engine !== 'tiktoken' && this._engine !== 'spm') {
+      throw new Error(`Unknown tokenizer engine: ${this._engine}`);
+    }
+
+    if (this._engine === 'tiktoken') {
+      this._preparedTiktoken = buildPreparedTiktoken(vocabData);
+    } else {
+      this._preparedSPM = buildPreparedSPM(vocabData);
+    }
+  }
+
+  encode(text) {
+    if (this._engine === 'tiktoken') return encodeTiktokenPrepared(text, this._preparedTiktoken);
+    return encodeSPMPrepared(text, this._preparedSPM);
+  }
+
+  decode(ids) {
+    if (this._engine === 'tiktoken') return decodeTiktokenPrepared(ids, this._preparedTiktoken);
+    return decodeSPMPrepared(ids, this._preparedSPM);
+  }
+
+  count(text) {
+    if (this._engine === 'tiktoken') return countTiktokenPrepared(text, this._preparedTiktoken);
+    return this.encode(text).length;
+  }
+
+  /**
+   * Count tokens, stopping as soon as the count exceeds limit.
+   * More efficient than encode() for token limit checks on long text.
+   * @param {string} text
+   * @param {number} limit
+   * @returns {number}
+   */
+  countUpTo(text, limit) {
+    if (this._engine === 'tiktoken') return countTiktokenUpToPrepared(text, this._preparedTiktoken, limit);
+    // SPM encodes the whole text as one unit — no clean early exit, just encode and count
+    return this.encode(text).length;
+  }
+}
+
+module.exports = { Tokenizer };