@simulatte/doppler 0.1.9 → 0.2.0

package/src/inference/tokenizers/bundled.js

@@ -1,1009 +0,0 @@
-
-
-import { BaseTokenizer } from './base.js';
-import { log } from '../../debug/index.js';
-import { getRuntimeConfig } from '../../config/runtime.js';
-
-function pickCandidate(...values) {
-  for (const value of values) {
-    if (value != null) return value;
-  }
-  return null;
-}
-
-function resolveTokenId(value, vocab, label) {
-  if (value == null) return null;
-  if (typeof value === 'number') return value;
-  if (typeof value === 'string') {
-    const id = vocab.get(value);
-    if (id === undefined) {
-      throw new Error(`[Tokenizer] Special token "${label}" not found in vocab: "${value}"`);
-    }
-    return id;
-  }
-  return null;
-}
-
-function resolveSpecialTokens(specialTokensRaw, fallbackTokens, vocab) {
-  const padCandidate = pickCandidate(
-    specialTokensRaw?.pad,
-    specialTokensRaw?.pad_token,
-    specialTokensRaw?.pad_token_id,
-    fallbackTokens?.pad
-  );
-  const bosCandidate = pickCandidate(
-    specialTokensRaw?.bos,
-    specialTokensRaw?.bos_token,
-    specialTokensRaw?.bos_token_id,
-    fallbackTokens?.bos
-  );
-  const eosCandidate = pickCandidate(
-    specialTokensRaw?.eos,
-    specialTokensRaw?.eos_token,
-    specialTokensRaw?.eos_token_id,
-    fallbackTokens?.eos
-  );
-  const unkCandidate = pickCandidate(
-    specialTokensRaw?.unk,
-    specialTokensRaw?.unk_token,
-    specialTokensRaw?.unk_token_id,
-    fallbackTokens?.unk
-  );
-
-  const resolved = {
-    pad: resolveTokenId(padCandidate, vocab, 'pad'),
-    bos: resolveTokenId(bosCandidate, vocab, 'bos'),
-    eos: resolveTokenId(eosCandidate, vocab, 'eos'),
-    unk: resolveTokenId(unkCandidate, vocab, 'unk'),
-  };
-
-  if (resolved.eos == null) {
-    throw new Error('[Tokenizer] Missing EOS token in tokenizer.json and runtime config.');
-  }
-
-  return resolved;
-}
-
-function resolveByteLevelPretokenizerConfig(preTokenizer) {
-  if (!preTokenizer || typeof preTokenizer !== 'object') {
-    return {
-      useByteLevel: false,
-      addPrefixSpace: null,
-    };
-  }
-
-  if (preTokenizer.type === 'ByteLevel') {
-    return {
-      useByteLevel: true,
-      addPrefixSpace: preTokenizer.add_prefix_space === true,
-    };
-  }
-
-  if (preTokenizer.type === 'Sequence' && Array.isArray(preTokenizer.pretokenizers)) {
-    for (const entry of preTokenizer.pretokenizers) {
-      const resolved = resolveByteLevelPretokenizerConfig(entry);
-      if (resolved.useByteLevel) {
-        return resolved;
-      }
-    }
-  }
-
-  return {
-    useByteLevel: false,
-    addPrefixSpace: null,
-  };
-}
-
-function registerAddedTokens(addedTokens, vocab, reverseVocab, patterns, specialTokenIds, derivedSpecialTokens = null) {
-  let maxId = -1;
-  for (const token of addedTokens) {
-    const content = token?.content;
-    const id = typeof token?.id === 'number' ? token.id : parseInt(token?.id, 10);
-    if (!Number.isFinite(id) || !content) continue;
-    if (!vocab.has(content)) {
-      vocab.set(content, id);
-      reverseVocab.set(id, content);
-    }
-    if (id > maxId) maxId = id;
-    if (content.length > 1) {
-      patterns.push({ content, id });
-    }
-    if (token.special) {
-      specialTokenIds.add(id);
-      if (derivedSpecialTokens) {
-        if (derivedSpecialTokens.bos == null && (content === '<bos>' || content === '<s>' || content.includes('bos'))) {
-          derivedSpecialTokens.bos = id;
-        } else if (derivedSpecialTokens.eos == null && (content === '<eos>' || content === '</s>' || content.includes('eos'))) {
-          derivedSpecialTokens.eos = id;
-        } else if (derivedSpecialTokens.pad == null && (content === '<pad>' || content.includes('pad'))) {
-          derivedSpecialTokens.pad = id;
-        } else if (derivedSpecialTokens.unk == null && (content === '<unk>' || content.includes('unk'))) {
-          derivedSpecialTokens.unk = id;
-        }
-      }
-    }
-  }
-  return maxId;
-}
-
-
-export class TransformersTokenizer extends BaseTokenizer {
-
-  #tokenizer = null;
-
-  #modelId;
-
-
-  constructor(config = {}) {
-    // TransformersTokenizer gets vocabSize from setTokenizer(), so defer validation
-    super({
-      ...config,
-    });
-    this.#modelId = config.modelId;
-  }
-
-  
-  setTokenizer(tokenizer) {
-    this.#tokenizer = tokenizer;
-    if (tokenizer.model?.vocab) {
-      this.vocabSize = Object.keys(tokenizer.model.vocab).length;
-    }
-  }
-
-  
-  async load(_modelId) {
-    // DOPPLER uses bundled tokenizers only - no external CDN dependencies
-    throw new Error(
-      '[Tokenizer] TransformersTokenizer is deprecated. ' +
-      'Use bundled tokenizer (type: "bundled" or "huggingface" with file). ' +
-      'DOPPLER requires no external runtime dependencies.'
-    );
-  }
-
-  
-  encode(text) {
-    if (!this.#tokenizer) {
-      throw new Error('Tokenizer not initialized');
-    }
-
-    const result = this.#tokenizer.encode(text, {
-      add_special_tokens: this.addBosToken
-    });
-
-    return Array.from(result);
-  }
-
-  
-  decode(ids, skipSpecialTokens = true, trim = true) {
-    if (!this.#tokenizer) {
-      throw new Error('Tokenizer not initialized');
-    }
-
-    const result = this.#tokenizer.decode(ids, { skip_special_tokens: skipSpecialTokens });
-    return trim ? result.trim() : result;
-  }
-
-  
-  batchEncode(texts) {
-    return texts.map(t => this.encode(t));
-  }
-}
-
-
-export class BundledTokenizer extends BaseTokenizer {
-  
-  #vocab = new Map();
-  
-  #reverseVocab = new Map();
-  
-  #merges = [];
-  
-  #mergeRanks = new Map();
-  
-  #scores = [];
-  
-  #tokenTypes = [];
-  
-  #type = null;
-  
-  #byteTokens = new Map();
-  
-  #specialTokenPatterns = [];
-  
-  #specialTokenIds = new Set();
-  
-  #addSpacePrefix = true;
-  
-  #spacePrefixChar = '▁';
-  
-  #byteDecoder = null;
-
-  #byteEncoder = null;
-
-  #useByteLevelEncoding = false;
-
-  
-  constructor(config = {}) {
-    // BundledTokenizer gets vocabSize from load(), so defer validation
-    super({
-      ...config,
-    });
-  }
-
-  #resetState() {
-    this.#vocab.clear();
-    this.#reverseVocab.clear();
-    this.#merges = [];
-    this.#mergeRanks.clear();
-    this.#scores = [];
-    this.#tokenTypes = [];
-    this.#type = null;
-    this.#byteTokens.clear();
-    this.#specialTokenPatterns = [];
-    this.#specialTokenIds = new Set();
-    this.#addSpacePrefix = true;
-    this.#spacePrefixChar = '▁';
-    this.#byteDecoder = null;
-    this.#byteEncoder = null;
-    this.#useByteLevelEncoding = false;
-    this.vocabSize = 0;
-  }
-
-  
-  isSpecialToken(tokenId) {
-    if (this.#specialTokenIds.size > 0) {
-      return this.#specialTokenIds.has(tokenId);
-    }
-    return super.isSpecialToken(tokenId);
-  }
-
-  
-  #getUnkTokenId() {
-    if (this.specialTokens.unk == null) {
-      throw new Error('[Tokenizer] Missing unk token in tokenizer metadata.');
-    }
-    return this.specialTokens.unk;
-  }
-
-  #initializeByteDecoder() {
-    // GPT2-style byte <-> unicode reversible mapping used by many BPE tokenizers
-    // (including Qwen-family tokenizers).
-    const base = [];
-    for (let i = 33; i <= 126; i++) base.push(i);
-    for (let i = 161; i <= 172; i++) base.push(i);
-    for (let i = 174; i <= 255; i++) base.push(i);
-
-    const chars = [...base];
-    let extra = 0;
-    for (let b = 0; b <= 255; b++) {
-      if (!base.includes(b)) {
-        base.push(b);
-        chars.push(256 + extra);
-        extra += 1;
-      }
-    }
-
-    this.#byteDecoder = new Map();
-    this.#byteEncoder = new Map();
-    for (let i = 0; i < base.length; i++) {
-      this.#byteDecoder.set(String.fromCodePoint(chars[i]), base[i]);
-      this.#byteEncoder.set(base[i], String.fromCodePoint(chars[i]));
-    }
-  }
-
-  #encodeByteLevelText(text) {
-    const bytes = new TextEncoder().encode(text);
-    let out = '';
-    for (const byte of bytes) {
-      out += this.#byteEncoder?.get(byte) ?? String.fromCharCode(byte);
-    }
-    return out;
-  }
-
-  
-  load(tokenizerJson) {
-    this.#resetState();
-    // Detect format: HuggingFace has model.vocab, bundled has top-level vocab
-    const isHuggingFace = 'model' in tokenizerJson && tokenizerJson.model?.vocab !== undefined;
-
-    if (isHuggingFace) {
-      this.#loadHuggingFaceFormat( (tokenizerJson));
-    } else {
-      this.#loadBundledFormat( (tokenizerJson));
-    }
-  }
-
-  
-  #loadHuggingFaceFormat(hf) {
-    const model = hf.model;
-    if (typeof model.type !== 'string') {
-      throw new Error('[Tokenizer] Missing model.type in HuggingFace tokenizer JSON.');
-    }
-    this.#type = model.type.toLowerCase();
-    if (this.#type !== 'bpe' && this.#type !== 'unigram') {
-      throw new Error(`[Tokenizer] Unsupported tokenizer type: ${model.type}`);
-    }
-    log.info('Tokenizer', `HuggingFace model.type="${model.type}", using type="${this.#type}"`);
-    this.#byteDecoder = null;
-    if (this.#type === 'bpe') {
-      this.#initializeByteDecoder();
-    }
-    let maxId = -1;
-
-    // Handle vocab based on type
-    if (this.#type === 'unigram' && Array.isArray(model.vocab)) {
-      // Unigram format: [[token, score], ...]
-      for (let i = 0; i < model.vocab.length; i++) {
-        const [token, score] = model.vocab[i];
-        this.#vocab.set(token, i);
-        this.#reverseVocab.set(i, token);
-        this.#scores.push(score);
-        if (i > maxId) maxId = i;
-
-        // Track byte tokens
-        if (token.match(/^<0x[0-9A-Fa-f]{2}>$/)) {
-          const byteVal = parseInt(token.slice(3, 5), 16);
-          this.#byteTokens.set(byteVal, i);
-        }
-      }
-    } else if (this.#type === 'bpe' && model.vocab && typeof model.vocab === 'object') {
-      // BPE format: { token: id }
-      for (const [token, id] of Object.entries(model.vocab)) {
-        const numId = typeof id === 'number' ? id : parseInt( (id), 10);
-        this.#vocab.set(token, numId);
-        this.#reverseVocab.set(numId, token);
-        if (Number.isFinite(numId) && numId > maxId) maxId = numId;
-
-        // Track byte tokens
-        if (token.match(/^<0x[0-9A-Fa-f]{2}>$/)) {
-          const byteVal = parseInt(token.slice(3, 5), 16);
-          this.#byteTokens.set(byteVal, numId);
-        }
-      }
-    } else {
-      throw new Error(`[Tokenizer] Missing vocab for tokenizer type: ${model.type}`);
-    }
-
-    this.vocabSize = this.#vocab.size;
-
-    // Load merges from model.merges
-    // Handle both string format ("token1 token2") and array format (["token1", "token2"])
-    if (model.merges && model.merges.length > 0) {
-      for (let i = 0; i < model.merges.length; i++) {
-        const merge = model.merges[i];
-        // Convert array format to string format for consistent lookup
-        const mergeKey = Array.isArray(merge) ? merge.join(' ') : merge;
-        this.#merges.push(mergeKey);
-        this.#mergeRanks.set(mergeKey, i);
-      }
-    }
-
-    const addedTokens = Array.isArray(hf.added_tokens) ? hf.added_tokens : [];
-    const specialTokenIds = new Set();
-    const specialTokenPatterns = [];
-    const derivedSpecialTokens = {
-      pad: null,
-      bos: null,
-      eos: null,
-      unk: null,
-    };
-    const addedMaxId = registerAddedTokens(
-      addedTokens,
-      this.#vocab,
-      this.#reverseVocab,
-      specialTokenPatterns,
-      specialTokenIds,
-      derivedSpecialTokens
-    );
-    if (addedMaxId > maxId) {
-      maxId = addedMaxId;
-    }
-
-    const specialTokensRaw = hf.special_tokens_map || hf.specialTokens || hf.special_tokens || null;
-    const fallbackTokens = {
-      ...this.specialTokens,
-      pad: model.pad_id ?? this.specialTokens.pad ?? derivedSpecialTokens.pad,
-      bos: model.bos_id ?? this.specialTokens.bos ?? derivedSpecialTokens.bos,
-      eos: model.eos_id ?? this.specialTokens.eos ?? derivedSpecialTokens.eos,
-      unk: model.unk_id ?? this.specialTokens.unk ?? derivedSpecialTokens.unk,
-    };
-    this.specialTokens = resolveSpecialTokens(specialTokensRaw, fallbackTokens, this.#vocab);
-    this.#specialTokenIds = specialTokenIds;
-    this.#specialTokenPatterns = specialTokenPatterns;
-    const builtinSpecials = [
-      this.specialTokens.pad,
-      this.specialTokens.bos,
-      this.specialTokens.eos,
-      this.specialTokens.unk,
-    ];
-    for (const id of builtinSpecials) {
-      if (typeof id === 'number' && Number.isFinite(id)) {
-        this.#specialTokenIds.add(id);
-      }
-    }
-    // Sort special tokens by length (longest first) for greedy matching
-    this.#specialTokenPatterns.sort((a, b) => b.content.length - a.content.length);
-    // Debug: log special tokens
-    log.debug('Tokenizer', `Special token patterns: ${this.#specialTokenPatterns.map(t => `${t.id}:"${t.content}"`).join(', ')}`);
-
-    // Some models add special tokens with IDs above the base vocab range.
-    // Keep vocabSize aligned to the maximum ID + 1 to match embedding/LM-head shapes.
-    if (maxId >= 0) {
-      this.vocabSize = Math.max(this.vocabSize, maxId + 1);
-    }
-
-    // Handle behavior flags (use HF config if present, else runtime defaults)
-    const runtimeDefaults = getRuntimeConfig().inference.tokenizer;
-    const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(hf.pre_tokenizer);
-    const configuredAddBosToken = this.addBosToken;
-    const configuredAddEosToken = this.addEosToken;
-    this.addBosToken =
-      hf.add_bos_token
-      ?? hf.addBosToken
-      ?? configuredAddBosToken
-      ?? runtimeDefaults.addBosToken;
-    this.addEosToken =
-      hf.add_eos_token
-      ?? hf.addEosToken
-      ?? configuredAddEosToken
-      ?? runtimeDefaults.addEosToken;
-    if (this.addBosToken && this.specialTokens.bos == null) {
-      throw new Error('[Tokenizer] addBosToken is enabled but bos token is missing.');
-    }
-    if (this.addEosToken && this.specialTokens.eos == null) {
-      throw new Error('[Tokenizer] addEosToken is enabled but eos token is missing.');
-    }
-    // Determine if we should add a space prefix to the input
-    // Check multiple locations where HuggingFace stores this:
-    // - model.add_prefix_space / model.add_dummy_prefix (GPT-style)
-    // - decoder.add_prefix_space (SentencePiece decoder)
-    // - decoder.prepend_scheme === "always" (Metaspace decoder, used by Gemma)
-    // - normalizer.prepend_scheme === "always" (Metaspace normalizer)
-    // - runtime config addSpacePrefix (user override or null for auto-detect)
-    const decoderPrepend = hf.decoder?.prepend_scheme === 'always' || hf.decoder?.add_prefix_space === true;
-    const normalizerPrepend = hf.normalizer?.prepend_scheme === 'always' || hf.normalizer?.add_prefix_space === true;
-    this.#useByteLevelEncoding = byteLevelPretokenizer.useByteLevel;
-    const runtimeSpacePrefix = runtimeDefaults.addSpacePrefix;
-    // Use explicit runtime config if set (non-null), otherwise auto-detect from tokenizer.json
-    this.#addSpacePrefix = runtimeSpacePrefix
-      ?? byteLevelPretokenizer.addPrefixSpace
-      ?? model.add_prefix_space
-      ?? model.add_dummy_prefix
-      ?? decoderPrepend
-      ?? normalizerPrepend
-      ?? false;
-    log.debug('Tokenizer', `addSpacePrefix=${this.#addSpacePrefix} (runtime=${runtimeSpacePrefix}, model=${model.add_prefix_space ?? model.add_dummy_prefix}, decoder=${decoderPrepend}, normalizer=${normalizerPrepend})`);
-
-    // Detect space prefix style by checking which WORD tokens exist in vocab
-    // GPT-style uses 'Ġ' (U+0120), SentencePiece uses '▁' (U+2581)
-    // IMPORTANT: Only check for actual word tokens like 'Ġthe', not single char 'Ġ'
-    // because some models (Gemma) have 'Ġ' as a token but use '▁' for actual word prefixes
-    const hasGptStyle = this.#vocab.has('Ġthe') || this.#vocab.has('Ġa') || this.#vocab.has('Ġis');
-    const hasSentencePieceStyle = this.#vocab.has('▁the') || this.#vocab.has('▁a') || this.#vocab.has('▁is');
-    if (hasGptStyle && !hasSentencePieceStyle) {
-      this.#spacePrefixChar = 'Ġ';
-      log.debug('Tokenizer', 'Detected GPT-style space prefix');
-    } else if (hasSentencePieceStyle && !hasGptStyle) {
-      this.#spacePrefixChar = '▁';
-      log.debug('Tokenizer', 'Detected SentencePiece-style space prefix');
-    } else if (hasGptStyle && hasSentencePieceStyle) {
-      // Both styles exist - prefer GPT style for Llama-family models
-      this.#spacePrefixChar = 'Ġ';
-      log.debug('Tokenizer', 'Both space styles found, defaulting to GPT-style');
-    } else {
-      // Neither style found - might be byte-level or no space prefix needed
-      this.#addSpacePrefix = false;
-      log.debug('Tokenizer', 'No space prefix tokens found, disabling space prefix');
-    }
-
-    log.info('Tokenizer', `Loaded HuggingFace ${this.vocabSize} tokens (${this.#type}), ${this.#specialTokenPatterns.length} special patterns, ${this.#merges.length} merges`);
-    // Debug: show sample vocab entries (look for common words)
-    const commonWords = ['the', '▁the', 'Ġthe', 'a', '▁a', 'is', '▁is', 'user', '▁user', 'u', 's', 'e', 'r'];
-    const foundTokens = commonWords.map(w => {
-      const id = this.#vocab.get(w);
-      return id !== undefined ? `"${w}"=${id}` : null;
-    }).filter(Boolean);
-    log.debug('Tokenizer', `Common tokens in vocab: ${foundTokens.join(', ') || 'NONE FOUND'}`);
-    // Show first few merges (escape whitespace)
-    if (this.#merges.length > 0) {
-      const escapedMerges = this.#merges.slice(0, 5).map(m =>
-        String(m).replace(/\n/g, '\\n').replace(/\r/g, '\\r').replace(/ /g, '␣')
-      );
-      log.debug('Tokenizer', `First 5 merges: ${escapedMerges.join(' | ')}`);
-    }
-  }
-
-  
-  #loadBundledFormat(tokenizerJson) {
-    if (typeof tokenizerJson.type !== 'string') {
-      throw new Error('[Tokenizer] Missing tokenizer.type in bundled tokenizer JSON.');
-    }
-    this.#type = tokenizerJson.type.toLowerCase();
-    if (this.#type !== 'bpe' && this.#type !== 'unigram') {
-      throw new Error(`[Tokenizer] Unsupported tokenizer type: ${tokenizerJson.type}`);
-    }
-    this.#byteDecoder = null;
-    if (this.#type === 'bpe') {
-      this.#initializeByteDecoder();
-    }
-
-    // Build vocab maps
-    for (const [token, id] of Object.entries(tokenizerJson.vocab)) {
-      const numId = typeof id === 'number' ? id : parseInt( (id), 10);
-      this.#vocab.set(token, numId);
-      this.#reverseVocab.set(numId, token);
-
-      // Track byte tokens for fallback
-      if (token.match(/^<0x[0-9A-Fa-f]{2}>$/)) {
-        const byteVal = parseInt(token.slice(3, 5), 16);
-        this.#byteTokens.set(byteVal, numId);
-      }
-    }
-
-    this.vocabSize = this.#vocab.size;
-
-    // Load merges for BPE
-    if (tokenizerJson.merges && tokenizerJson.merges.length > 0) {
-      this.#merges = tokenizerJson.merges;
-      for (let i = 0; i < this.#merges.length; i++) {
-        this.#mergeRanks.set(this.#merges[i], i);
-      }
-    }
-
-    // Load scores for Unigram
-    if (tokenizerJson.scores && tokenizerJson.scores.length > 0) {
-      this.#scores = tokenizerJson.scores;
-    }
-
-    // Load token types if available
-    if (tokenizerJson.tokenTypes) {
-      this.#tokenTypes = tokenizerJson.tokenTypes;
-    }
-
-    let maxId = -1;
-    for (const id of this.#vocab.values()) {
-      if (Number.isFinite(id) && id > maxId) {
-        maxId = id;
-      }
-    }
-
-    const addedTokens = Array.isArray(tokenizerJson.added_tokens) ? tokenizerJson.added_tokens : [];
-    const tokenPatterns = [];
-    const specialTokenIds = new Set();
-    const derivedSpecialTokens = {
-      pad: null,
-      bos: null,
-      eos: null,
-      unk: null,
-    };
-    const addedMaxId = registerAddedTokens(
-      addedTokens,
-      this.#vocab,
-      this.#reverseVocab,
-      tokenPatterns,
-      specialTokenIds,
-      derivedSpecialTokens
-    );
-    if (addedMaxId > maxId) {
-      maxId = addedMaxId;
-    }
-
-    // Set special tokens - support both camelCase and snake_case formats
-    const specialTokensRaw =  (tokenizerJson.specialTokens ||  (tokenizerJson).special_tokens);
-    this.specialTokens = resolveSpecialTokens(
-      specialTokensRaw,
-      {
-        ...derivedSpecialTokens,
-        ...this.specialTokens,
-      },
-      this.#vocab
-    );
-    log.debug('Tokenizer', `Special tokens: BOS=${this.specialTokens.bos}, EOS=${this.specialTokens.eos}`);
-    this.#specialTokenIds = specialTokenIds;
-    this.#specialTokenPatterns = tokenPatterns;
-    const builtinSpecials = [
-      this.specialTokens.pad,
-      this.specialTokens.bos,
-      this.specialTokens.eos,
-      this.specialTokens.unk,
-    ];
-    for (const id of builtinSpecials) {
-      if (typeof id === 'number' && Number.isFinite(id)) {
-        this.#specialTokenIds.add(id);
-      }
-    }
-    this.#specialTokenPatterns.sort((a, b) => b.content.length - a.content.length);
-    if (maxId >= 0) {
-      this.vocabSize = Math.max(this.vocabSize, maxId + 1);
-    }
-
-    const runtimeDefaults = getRuntimeConfig().inference.tokenizer;
-    const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(tokenizerJson.pre_tokenizer);
-    const configuredAddBosToken = this.addBosToken;
-    const configuredAddEosToken = this.addEosToken;
-    this.addBosToken =
-      tokenizerJson.addBosToken
-      ?? tokenizerJson.add_bos_token
-      ?? configuredAddBosToken
-      ?? runtimeDefaults.addBosToken;
-    this.addEosToken =
-      tokenizerJson.addEosToken
-      ?? tokenizerJson.add_eos_token
-      ?? configuredAddEosToken
-      ?? runtimeDefaults.addEosToken;
-    if (this.addBosToken && this.specialTokens.bos == null) {
-      throw new Error('[Tokenizer] addBosToken is enabled but bos token is missing.');
-    }
-    if (this.addEosToken && this.specialTokens.eos == null) {
-      throw new Error('[Tokenizer] addEosToken is enabled but eos token is missing.');
-    }
-    this.#useByteLevelEncoding = byteLevelPretokenizer.useByteLevel;
-    // NOTE: Default to FALSE - first word shouldn't get space prefix
-    // Space prefixes are only for words that follow a space in original text
-    this.#addSpacePrefix = tokenizerJson.addSpacePrefix === true
-      || byteLevelPretokenizer.addPrefixSpace === true;
-
-    // Detect space prefix style based on vocab tokens
-    // GPT-style uses 'Ġ' (U+0120), SentencePiece uses '▁' (U+2581)
-    // IMPORTANT: Only check for actual word tokens like 'Ġthe', not single char 'Ġ'
-    // because some models (Gemma) have 'Ġ' as a token but use '▁' for actual word prefixes
-    const hasGptStyle = this.#vocab.has('Ġthe') || this.#vocab.has('Ġa') || this.#vocab.has('Ġis');
-    const hasSentencePieceStyle = this.#vocab.has('▁the') || this.#vocab.has('▁a') || this.#vocab.has('▁is');
-
-    if (hasGptStyle && !hasSentencePieceStyle) {
-      this.#spacePrefixChar = 'Ġ';
-      log.debug('Tokenizer', 'Detected GPT-style space prefix');
-    } else if (hasSentencePieceStyle && !hasGptStyle) {
-      this.#spacePrefixChar = '▁';
-      log.debug('Tokenizer', 'Detected SentencePiece-style space prefix');
-    } else if (hasGptStyle && hasSentencePieceStyle) {
-      // Both exist - prefer GPT-style for Llama 3 compatibility
-      this.#spacePrefixChar = 'Ġ';
-      log.debug('Tokenizer', 'Both space prefix styles found, using GPT-style');
-    } else {
-      // Default to SentencePiece style
-      this.#spacePrefixChar = '▁';
-      log.debug('Tokenizer', 'No space prefix tokens found, defaulting to SentencePiece-style');
-    }
-
-    log.info('Tokenizer', `Loaded ${this.vocabSize} tokens (${this.#type})`);
-  }
-
-  
-  encode(text) {
-    if (this.#vocab.size === 0) {
-      throw new Error('BundledTokenizer not loaded');
-    }
-
-    
-    const ids = [];
-
-    if (this.addBosToken) {
-      ids.push(this.specialTokens.bos);
-    }
-
-    // Split text around literal added tokens and special tokens, then tokenize
-    // the remaining plain-text segments normally.
-    const segments = this.#splitOnSpecialTokens(text);
-    for (const seg of segments) {
-      if (seg.isSpecial && seg.id !== undefined) {
-        ids.push(seg.id);
-      } else if (seg.text && seg.text.length > 0) {
-        if (this.#type === 'unigram') {
-          ids.push(...this.#encodeUnigram(seg.text));
-        } else {
-          ids.push(...this.#encodeBPE(seg.text));
-        }
-      }
-    }
-
-    if (this.addEosToken) {
-      ids.push(this.specialTokens.eos);
-    }
-
-    return ids;
-  }
-
-  
-  #splitOnSpecialTokens(text) {
-    if (this.#specialTokenPatterns.length === 0) {
-      return [{ text, isSpecial: false }];
-    }
-
-    
-    const segments = [];
-    let remaining = text;
-
-    while (remaining.length > 0) {
-      // Find the EARLIEST special token match
-      let earliestIdx = Infinity;
-      
-      let earliestToken = null;
-
-      for (const { content, id } of this.#specialTokenPatterns) {
-        const idx = remaining.indexOf(content);
-        if (idx !== -1 && idx < earliestIdx) {
-          earliestIdx = idx;
-          earliestToken = { content, id };
-        }
-      }
-
-      if (earliestToken === null) {
-        // No special tokens found, rest is plain text
-        segments.push({ text: remaining, isSpecial: false });
-        break;
-      }
-
-      if (earliestIdx === 0) {
-        // Special token at start
-        segments.push({ id: earliestToken.id, isSpecial: true });
-        remaining = remaining.slice(earliestToken.content.length);
-      } else {
-        // Text before special token
-        segments.push({ text: remaining.slice(0, earliestIdx), isSpecial: false });
-        segments.push({ id: earliestToken.id, isSpecial: true });
-        remaining = remaining.slice(earliestIdx + earliestToken.content.length);
-      }
-    }
-
-    return segments;
-  }
-
-  
-  #encodeUnigram(text) {
-    if (text.length === 0) return [];
-
-    // Add space prefix at start if configured (SentencePiece add_dummy_prefix)
-    // This matches HuggingFace behavior: "The" -> " The" -> "▁The"
-    let normalized = text;
-    if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
-      normalized = ` ${normalized}`;
-    }
-
-    // Normalize: convert spaces to the model's space prefix character
-    // This turns " The color of" into "▁The▁color▁of"
-    const sp = this.#spacePrefixChar;
-    const prefixed = normalized.replace(/ /g, sp);
-
-    const n = prefixed.length;
-    if (n === 0) return [];
-
-    // Viterbi: best[i] = {score, prev, tokenLen} for position i
-    
-    const best = new Array(n + 1).fill(null);
-    best[0] = { score: 0, prev: -1, tokenLen: 0 };
-
-    for (let i = 0; i < n; i++) {
-      if (best[i] === null) continue;
-
-      // Try all possible tokens starting at position i
-      for (let len = 1; len <= Math.min(n - i, 32); len++) {
-        const substr = prefixed.slice(i, i + len);
-        const tokenId = this.#vocab.get(substr);
-
-        if (tokenId !== undefined) {
-          const score = this.#scores[tokenId] || 0;
-          const newScore = best[i].score + score;
-          if (best[i + len] === null || newScore > best[i + len].score) {
-            best[i + len] = { score: newScore, prev: i, tokenLen: len };
-          }
-        }
-      }
-
-      // Byte fallback for single character
-      if (best[i + 1] === null) {
-        const bytes = new TextEncoder().encode(prefixed[i]);
-        const byteScore = best[i].score - 10 * bytes.length;
-        best[i + 1] = { score: byteScore, prev: i, tokenLen: 1, isBytes: true, bytes };
-      }
-    }
-
-    // Backtrack to get tokens
-    
-    const tokens = [];
-    let pos = n;
-    while (pos > 0) {
-      const state = best[pos];
-      if (state.isBytes && state.bytes) {
-        for (let j = state.bytes.length - 1; j >= 0; j--) {
-          const byteId = this.#byteTokens.get(state.bytes[j]);
-          tokens.push(byteId ?? this.#getUnkTokenId());
-        }
-      } else {
-        const substr = prefixed.slice(state.prev, pos);
-        const tokenId = this.#vocab.get(substr);
-        tokens.push(tokenId ?? this.#getUnkTokenId());
-      }
-      pos = state.prev;
-    }
-
-    return tokens.reverse();
-  }
-
-  
-  #encodeBPE(text) {
-    if (text.length === 0) return [];
-
-    let normalized = text;
-    let prefixed;
-    if (this.#useByteLevelEncoding) {
-      if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
-        normalized = ` ${normalized}`;
-      }
-      prefixed = this.#encodeByteLevelText(normalized);
-    } else {
-      if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
-        normalized = ` ${normalized}`;
-      }
-      const sp = this.#spacePrefixChar;
-      prefixed = normalized.replace(/ /g, sp);
-    }
-
-    if (this.#mergeRanks.size === 0) {
-      return this.#encodeBPEGreedy(prefixed);
-    }
-
-    const tokens = this.#bpeTokenize(prefixed);
-    
-    const ids = [];
-    for (const token of tokens) {
-      const id = this.#vocab.get(token);
-      if (id !== undefined) {
-        ids.push(id);
-        continue;
-      }
-      const bytes = new TextEncoder().encode(token);
-      for (const b of bytes) {
-        const byteId = this.#byteTokens.get(b);
-        if (byteId !== undefined) {
-          ids.push(byteId);
-          continue;
-        }
-        const byteToken = `<0x${b.toString(16).padStart(2, '0').toUpperCase()}>`;
-        ids.push(this.#vocab.get(byteToken) ?? this.#getUnkTokenId());
-      }
-    }
-
-    return ids;
-  }
-
-  
-  #encodeBPEGreedy(text) {
-    
-    const ids = [];
-    let pos = 0;
-
-    while (pos < text.length) {
-      let bestLen = 0;
-      let bestId = this.#getUnkTokenId();
-
-      const maxLen = Math.min(32, text.length - pos);
-      for (let len = maxLen; len >= 1; len--) {
-        const substr = text.slice(pos, pos + len);
-        const id = this.#vocab.get(substr);
-        if (id !== undefined) {
-          bestLen = len;
-          bestId = id;
-          break;
-        }
-      }
-
-      if (bestLen === 0) {
-        const char = text[pos];
-        const bytes = new TextEncoder().encode(char);
-        for (const b of bytes) {
-          const byteId = this.#byteTokens.get(b);
-          if (byteId !== undefined) {
-            ids.push(byteId);
-            continue;
-          }
-          const byteToken = `<0x${b.toString(16).padStart(2, '0').toUpperCase()}>`;
-          ids.push(this.#vocab.get(byteToken) ?? this.#getUnkTokenId());
-        }
-        pos += 1;
-      } else {
-        ids.push(bestId);
-        pos += bestLen;
-      }
-    }
-
-    return ids;
-  }
-
-  
-  #bpeTokenize(text) {
-    if (text.length === 0) return [];
-
-    let tokens = text.split('');
-    if (tokens.length === 1) return tokens;
-
-    while (tokens.length > 1) {
-      let minRank = Infinity;
-      
-      let minPair = null;
-
-      for (let i = 0; i < tokens.length - 1; i++) {
-        const pair = `${tokens[i]} ${tokens[i + 1]}`;
-        const rank = this.#mergeRanks.get(pair);
-        if (rank !== undefined && rank < minRank) {
-          minRank = rank;
-          minPair = pair;
-        }
-      }
-
-      if (!minPair) break;
-
-      const [first, second] = minPair.split(' ');
-      
-      const newTokens = [];
-      let i = 0;
-      while (i < tokens.length) {
-        if (i < tokens.length - 1 && tokens[i] === first && tokens[i + 1] === second) {
-          newTokens.push(first + second);
-          i += 2;
-        } else {
-          newTokens.push(tokens[i]);
-          i += 1;
-        }
-      }
-      tokens = newTokens;
-    }
-
-    return tokens;
-  }
-
-  
-  decode(ids, skipSpecialTokens = true, trim = true) {
-    if (this.#vocab.size === 0) {
-      throw new Error('BundledTokenizer not loaded');
-    }
-
-    
-    const tokens = [];
-    for (const id of ids) {
-      if (skipSpecialTokens && this.isSpecialToken(id)) {
-        continue;
-      }
-
-      const token = this.#reverseVocab.get(id);
-      if (token !== undefined) {
-        // Handle byte tokens
-        if (token.match(/^<0x[0-9A-Fa-f]{2}>$/)) {
-          const byteVal = parseInt(token.slice(3, 5), 16);
-          tokens.push(String.fromCharCode(byteVal));
-        } else {
-          tokens.push(token);
-        }
-      }
-    }
-
-    let result;
-    if (this.#type === 'bpe' && this.#byteDecoder instanceof Map && this.#byteDecoder.size > 0) {
-      const merged = tokens.join('');
-      const bytes = [];
-      for (const ch of merged) {
-        const mapped = this.#byteDecoder.get(ch);
-        if (mapped != null) {
-          bytes.push(mapped);
-          continue;
-        }
-        const fallbackBytes = new TextEncoder().encode(ch);
-        for (const byte of fallbackBytes) {
-          bytes.push(byte);
-        }
-      }
-      result = new TextDecoder('utf-8', { fatal: false }).decode(Uint8Array.from(bytes));
-      // SentencePiece-style markers can still appear in some mixed vocabularies.
-      result = result.replace(/▁/g, ' ');
-    } else {
-      // Join and convert ▁ back to spaces, handle GPT-style markers
-      result = tokens.join('')
-        .replace(/▁/g, ' ')
-        .replace(/Ġ/g, ' ')
-        .replace(/Ċ/g, '\n');
-    }
-
-    // Only trim when requested (not during streaming where spaces matter)
-    return trim ? result.trim() : result;
-  }
-}