@simulatte/doppler 0.1.9 → 0.2.0

package/src/inference/test-harness.js

@@ -1,361 +0,0 @@
-
-
-import { initDevice, getDevice, getKernelCapabilities } from '../gpu/device.js';
-import { parseManifest, getExpectedShardHash } from '../formats/rdrr/index.js';
-import { createPipeline } from './pipelines/text.js';
-import { log as debugLog } from '../debug/index.js';
-import { getRuntimeConfig, setRuntimeConfig } from '../config/runtime.js';
-import { downloadShard as downloadShardFromDistribution } from '../distribution/shard-delivery.js';
-import {
-  fetchHotSwapManifest,
-  verifyHotSwapManifest,
-} from '../hotswap/manifest.js';
-import { evaluateHotSwapRollout, setHotSwapManifest } from '../hotswap/runtime.js';
-import {
-  fetchIntentBundle,
-  getKernelRegistryVersion,
-  verifyIntentBundle,
-} from '../hotswap/intent-bundle.js';
-
-
-
-// ============================================================================
-// Model Discovery
-// ============================================================================
-
-
-export async function discoverModels(
-  fallbackModels
-) {
-  try {
-    const resp = await fetch('/models/catalog.json');
-    if (resp.ok) {
-      const payload = await resp.json();
-      const catalogModels = Array.isArray(payload.models) ? payload.models : [];
-      if (catalogModels.length > 0) {
-        return catalogModels.map((m) => ({
-          id: m.modelId || m.id || 'unknown',
-          name: m.label || m.name || m.modelId || 'Unknown',
-          ...m,
-        }));
-      }
-    }
-  } catch (e) {}
-
-  if (Array.isArray(fallbackModels) && fallbackModels.length > 0) {
-    return fallbackModels.map((id) => ({ id, name: id }));
-  }
-
-  throw new Error('discoverModels: failed to fetch /models/catalog.json and no explicit fallback model list was provided.');
-}
-
-// ============================================================================
-// URL Parameter Parsing
-// ============================================================================
-
-
-export function parseRuntimeOverridesFromURL(searchParams) {
-  const query = typeof globalThis.location !== 'undefined' ? globalThis.location.search : '';
-  const params = searchParams || new URLSearchParams(query);
-
-  
-  const runtime = {};
-
-  // Runtime config (full or partial)
-  const runtimeConfigRaw = params.get('runtimeConfig');
-  if (runtimeConfigRaw) {
-    try {
-      const parsed = JSON.parse(runtimeConfigRaw);
-      if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
-        throw new Error('runtimeConfig must be a JSON object');
-      }
-      runtime.runtimeConfig = parsed;
-    } catch (e) {
-      throw new Error(`Failed to parse runtimeConfig URL parameter: ${e?.message}`);
-    }
-  }
-
-  // Config chain (for debugging)
-  const configChainRaw = params.get('configChain');
-  if (configChainRaw) {
-    try {
-      const parsed = JSON.parse(configChainRaw);
-      if (!Array.isArray(parsed) || parsed.some((entry) => typeof entry !== 'string' || !entry.trim())) {
-        throw new Error('configChain must be an array of non-empty strings');
-      }
-      runtime.configChain = parsed;
-      debugLog.info('TestHarness', `Config chain: ${parsed.join(' -> ')}`);
-    } catch (e) {
-      throw new Error(`Failed to parse configChain URL parameter: ${e?.message}`);
-    }
-  }
-
-  return runtime;
-}
-
-// ============================================================================
-// Shard Loading
-// ============================================================================
-
-function buildManifestVersionSet(manifest) {
-  if (!manifest || typeof manifest !== 'object') return 'manifest:invalid';
-  const shards = Array.isArray(manifest.shards)
-    ? manifest.shards.map((shard, index) => ({
-      index,
-      filename: shard?.filename ?? null,
-      size: shard?.size ?? null,
-      hash: shard?.hash ?? null,
-    }))
-    : [];
-  const payload = {
-    modelId: manifest.modelId ?? null,
-    version: manifest.version ?? null,
-    hashAlgorithm: manifest.hashAlgorithm ?? null,
-    tensorCount: manifest.tensorCount ?? null,
-    totalSize: manifest.totalSize ?? null,
-    shards,
-  };
-  return JSON.stringify(payload);
-}
-
-function toShardBytes(buffer, shardIndex) {
-  if (buffer instanceof ArrayBuffer) {
-    return new Uint8Array(buffer);
-  }
-  if (ArrayBuffer.isView(buffer)) {
-    const view = buffer;
-    return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
-  }
-  throw new Error(`Shard ${shardIndex} did not return an in-memory buffer`);
-}
-
-
-export function createHttpShardLoader(baseUrl, manifest, log) {
-  const algorithm = manifest.hashAlgorithm;
-  if (!algorithm) {
-    throw new Error('Manifest missing hashAlgorithm for shard delivery.');
-  }
-
-  const runtimeConfig = getRuntimeConfig();
-  const distributionConfig = runtimeConfig.loading?.distribution || {};
-  const totalShards = manifest.shards?.length || 0;
-  const requiredEncoding = distributionConfig.requiredContentEncoding ?? null;
-  const manifestVersionSet = buildManifestVersionSet(manifest);
-  const shardCache = new Map();
-  const pendingLoads = new Map();
-  let shardsLoaded = 0;
-  let totalBytesLoaded = 0;
-  const loadStartTime = Date.now();
-
-  return async ( idx) => {
-    const shard = manifest.shards[idx];
-    if (!shard) {
-      throw new Error(`No shard at index ${idx}`);
-    }
-
-    // Return cached shard if already loaded
-    if (shardCache.has(idx)) {
-      return  (shardCache.get(idx));
-    }
-
-    // Wait for pending load if one is in progress (avoid duplicate fetches)
-    if (pendingLoads.has(idx)) {
-      return  (pendingLoads.get(idx));
-    }
-
-    // Start new load and track it as pending
-    const loadPromise = (async () => {
-      try {
-        const result = await downloadShardFromDistribution(baseUrl, idx, shard, {
-          distributionConfig,
-          algorithm,
-          requiredEncoding,
-          expectedHash: getExpectedShardHash(shard, algorithm) || null,
-          expectedSize: Number.isFinite(shard.size) ? Math.floor(shard.size) : null,
-          expectedManifestVersionSet: manifestVersionSet,
-          writeToStore: false,
-          enableSourceCache: true,
-        });
-
-        const data = toShardBytes(result.buffer, idx);
-        shardCache.set(idx, data);
-        shardsLoaded++;
-        totalBytesLoaded += data.byteLength;
-
-        // Note: Individual shard progress is now reported through pipeline onProgress callback
-        // to avoid noisy duplicate logging. Log summary only when all shards loaded.
-        if (log && shardsLoaded === totalShards) {
-          const totalElapsed = (Date.now() - loadStartTime) / 1000;
-          const avgSpeed = totalElapsed > 0 ? totalBytesLoaded / totalElapsed : 0;
-          log(`All ${totalShards} shards loaded: ${(totalBytesLoaded / 1024 / 1024).toFixed(1)}MB in ${totalElapsed.toFixed(1)}s (${(avgSpeed / 1024 / 1024).toFixed(0)} MB/s avg)`);
-        }
-
-        return data;
-      } finally {
-        pendingLoads.delete(idx);
-      }
-    })();
-
-    pendingLoads.set(idx, loadPromise);
-    return loadPromise;
-  };
-}
-
-// ============================================================================
-// Pipeline Initialization
-// ============================================================================
-
-
-export async function fetchManifest(manifestUrl) {
-  const response = await fetch(manifestUrl);
-  if (!response.ok) {
-    throw new Error(`Failed to fetch manifest: ${response.status}`);
-  }
-  return parseManifest(await response.text());
-}
-
-
-export async function initializeDevice() {
-  await initDevice();
-  return getKernelCapabilities();
-}
-
-
-export async function initializeInference(modelUrl, options = {}) {
-  const log = options.log || (( msg) => debugLog.info('TestHarness', msg));
-  const onProgress = options.onProgress || (() => {});
-  if (options.runtime?.runtimeConfig) {
-    setRuntimeConfig(options.runtime.runtimeConfig);
-  }
-
-  const hotSwapConfig = getRuntimeConfig().shared.hotSwap;
-  const intentBundleConfig = getRuntimeConfig().shared.intentBundle;
-  if (hotSwapConfig.enabled && hotSwapConfig.manifestUrl) {
-    const rolloutDecision = evaluateHotSwapRollout(hotSwapConfig, {
-      modelUrl,
-      subjectId: options.modelId || null,
-      sessionId: options.sessionId || null,
-      optInTag: options.hotSwapOptInTag || null,
-    });
-    if (!rolloutDecision.allowed) {
-      log(`Hot-swap: rollout skipped (${rolloutDecision.reason})`);
-    } else {
-      onProgress('hotswap', 0.05, 'Loading hot-swap manifest...');
-      log(`Hot-swap: loading manifest ${hotSwapConfig.manifestUrl}`);
-      const hotSwapManifest = await fetchHotSwapManifest(hotSwapConfig.manifestUrl);
-      const verification = await verifyHotSwapManifest(hotSwapManifest, hotSwapConfig, {
-        source: {
-          kind: 'remote',
-          isLocal: false,
-          url: hotSwapConfig.manifestUrl,
-        },
-      });
-      if (!verification.ok) {
-        throw new Error(`Hot-swap manifest rejected: ${verification.reason}`);
-      }
-      setHotSwapManifest(hotSwapManifest);
-      log(
-        `Hot-swap manifest accepted: ${hotSwapManifest.bundleId} (${verification.reason}, rollout=${rolloutDecision.reason})`
-      );
-    }
-  }
-
-  // 1. Initialize WebGPU
-  onProgress('init', 0, 'Initializing WebGPU...');
-  log('Initializing WebGPU...');
-
-  await initDevice();
-  const device = getDevice();
-  const capabilities = getKernelCapabilities();
-
-  log(`GPU: hasF16=${capabilities.hasF16}, hasSubgroups=${capabilities.hasSubgroups}`);
-
-  // 2. Fetch manifest
-  onProgress('manifest', 0.1, 'Fetching manifest...');
-  log('Fetching manifest...');
-
-  const manifestUrl = `${modelUrl}/manifest.json`;
-  const manifest = await fetchManifest(manifestUrl);
-
-  if (intentBundleConfig.enabled && intentBundleConfig.bundleUrl) {
-    onProgress('intent', 0.12, 'Loading intent bundle...');
-    log(`Intent bundle: loading ${intentBundleConfig.bundleUrl}`);
-    const bundle = await fetchIntentBundle(intentBundleConfig.bundleUrl);
-    const kernelRegistryVersion = intentBundleConfig.requireKernelRegistryVersion
-      ? await getKernelRegistryVersion()
-      : null;
-    const verification = await verifyIntentBundle(bundle, {
-      manifest: intentBundleConfig.requireBaseModelHash ? manifest : null,
-      kernelRegistryVersion,
-      enforceDeterministicOutput: intentBundleConfig.enforceDeterministicOutput,
-    });
-    if (!verification.ok) {
-      const reason = verification.reasons?.length
-        ? `${verification.reason}: ${verification.reasons.join('; ')}`
-        : verification.reason;
-      throw new Error(`Intent bundle rejected: ${reason}`);
-    }
-    log(`Intent bundle accepted (${verification.reason})`);
-    intentBundleConfig.bundle = bundle;
-  }
-
-  const modelLabel = typeof manifest.architecture === 'string'
-    ? manifest.architecture
-    : (manifest.modelType || manifest.modelId || 'unknown');
-  log(`Model: ${modelLabel}`);
-
-  // 3. Create shard loader
-  const loadShard = createHttpShardLoader(modelUrl, manifest, log);
-
-  // 4. Build runtime options
-  
-  const runtime = {
-    ...options.runtime,
-  };
-
-  // 5. Create pipeline
-  onProgress('pipeline', 0.2, 'Creating pipeline...');
-  log('Creating pipeline...');
-
-  const pipeline = await createPipeline( ( (manifest)), {
-    storage: { loadShard },
-    gpu: { device },
-    runtime,
-    baseUrl: modelUrl,
-    onProgress: ( progress) => {
-      const pct = 0.2 + progress.percent * 0.8;
-      onProgress(progress.stage || 'loading', pct, progress.message);
-    },
-  });
-
-  onProgress('complete', 1, 'Ready');
-  log('Pipeline ready');
-
-  // Snapshot active configuration for diffing
-  const configSnapshot = {
-     kernelPathId: pipeline.resolvedKernelPath?.id || null,
-     kernelPathName: pipeline.resolvedKernelPath?.name || null,
-     // Detailed per-op view could be expanded here if needed
-  };
-
-  return { pipeline, manifest, capabilities, configSnapshot };
-}
-
-// ============================================================================
-// Test State (for browser automation)
-// ============================================================================
-
-
-export function createTestState() {
-  return {
-    ready: false,
-    loading: false,
-    loaded: false,
-    generating: false,
-    done: false,
-    output: '',
-    tokens: [],
-    errors: [],
-    model: null,
-  };
-}

package/src/inference/tokenizer.d.ts

@@ -1,72 +0,0 @@
-/**
- * Tokenizer Wrapper
- *
- * Provides a unified interface for tokenization across different backends.
- *
- * @module inference/tokenizer
- */
-
-export type { TokenizerConfig, ModelManifest, SpecialTokens } from './tokenizers/types.js';
-
-import type { TokenizerConfig, ModelManifest, SpecialTokens } from './tokenizers/types.js';
-
-/**
- * Options for tokenizer initialization
- */
-export interface TokenizerInitOptions {
-  /** Base URL for loading tokenizer files */
-  baseUrl?: string;
-  /** Preset tokenizer config as fallback hints (manifest takes precedence) */
-  presetTokenizer?: {
-    bosToken?: string;
-    eosTokens?: string[];
-    padToken?: string;
-    addBosToken?: boolean;
-    addEosToken?: boolean;
-    hfModel?: string;
-    allowArchFallback?: boolean;
-  };
-  /** Optional direct loader for tokenizer.json */
-  loadTokenizerJson?: (() => Promise<Record<string, unknown> | string | null | undefined>) | null;
-  /** Optional direct loader for tokenizer.model (SentencePiece) */
-  loadTokenizerModel?: ((path?: string) => Promise<ArrayBuffer | Uint8Array | null | undefined>) | null;
-}
-
-/**
- * Tokenizer wrapper that auto-detects backend from model manifest
- * This is a thin wrapper over the backend implementations
- */
-export declare class Tokenizer {
-  private backend;
-  private config;
-
-  /**
-   * Initialize from model manifest.
-   * Preset tokenizer provides fallback hints when manifest tokenizer is missing fields.
-   */
-  initialize(manifest: ModelManifest, options?: TokenizerInitOptions): Promise<void>;
-
-  /**
-   * Encode text to token IDs
-   */
-  encode(text: string): number[];
-
-  /**
-   * Decode token IDs to text
-   * @param skipSpecialTokens - Whether to skip special tokens in output
-   * @param trim - Whether to trim whitespace (default true, set false for streaming)
-   */
-  decode(ids: number[], skipSpecialTokens?: boolean, trim?: boolean): string;
-
-  /**
-   * Get special tokens
-   */
-  getSpecialTokens(): SpecialTokens;
-
-  /**
-   * Get vocabulary size
-   */
-  getVocabSize(): number;
-}
-
-export default Tokenizer;

package/src/inference/tokenizer.js

@@ -1,239 +0,0 @@
-
-
-import { log } from '../debug/index.js';
-import { BaseTokenizer } from './tokenizers/base.js';
-import { TransformersTokenizer, BundledTokenizer } from './tokenizers/bundled.js';
-import { SentencePieceTokenizer } from './tokenizers/sentencepiece.js';
-import { BPETokenizer } from './tokenizers/bpe.js';
-
-function hasScheme(value) {
-  return /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(value);
-}
-
-function toArrayBuffer(value, label) {
-  if (value instanceof ArrayBuffer) {
-    return value;
-  }
-  if (value instanceof Uint8Array) {
-    return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
-  }
-  throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
-}
-
-function parseTokenizerJsonPayload(value) {
-  if (value == null) {
-    return null;
-  }
-  if (typeof value === 'string') {
-    return JSON.parse(value);
-  }
-  if (typeof value === 'object') {
-    return value;
-  }
-  throw new Error('Tokenizer JSON loader must return an object, JSON string, or null.');
-}
-
-export class Tokenizer {
-  
-  backend = null;
-
-  
-  config = null;
-
-  
-  async initialize(manifest, options = {}) {
-    const modelId = manifest?.modelId || 'unknown';
-    // Merge tokenizer config: preset provides fallback hints, manifest takes precedence
-    const presetTokenizer = options.presetTokenizer || {};
-    const tokenizerConfig = { ...presetTokenizer, ...(manifest.tokenizer || {}) };
-    const eosTokenId = Array.isArray(manifest.eos_token_id)
-      ? manifest.eos_token_id[0]
-      : manifest.eos_token_id;
-
-    if (tokenizerConfig.eosToken == null && Array.isArray(tokenizerConfig.eosTokens) && tokenizerConfig.eosTokens.length > 0) {
-      tokenizerConfig.eosToken = tokenizerConfig.eosTokens[0];
-    }
-    if (tokenizerConfig.eosToken == null && typeof eosTokenId === 'number') {
-      tokenizerConfig.eosToken = eosTokenId;
-    }
-    if (tokenizerConfig.bosToken == null && tokenizerConfig.bosTokenId != null) {
-      tokenizerConfig.bosToken = tokenizerConfig.bosTokenId;
-    }
-    if (tokenizerConfig.padToken == null && tokenizerConfig.padTokenId != null) {
-      tokenizerConfig.padToken = tokenizerConfig.padTokenId;
-    }
-    if (tokenizerConfig.unkToken == null && tokenizerConfig.unkTokenId != null) {
-      tokenizerConfig.unkToken = tokenizerConfig.unkTokenId;
-    }
-
-    // Check for bundled or HuggingFace tokenizer first (eliminates transformers.js dependency)
-    const isBundled = tokenizerConfig.type === 'bundled' || tokenizerConfig.type === 'huggingface';
-    if (isBundled && tokenizerConfig.file) {
-      tokenizerConfig.deferSpecialTokens = true;
-      log.info('Tokenizer', `Loading ${tokenizerConfig.type} tokenizer from ${tokenizerConfig.file}`);
-      this.backend = new BundledTokenizer(tokenizerConfig);
-
-      const baseUrl = options.baseUrl;
-      
-      let tokenizerJson = null;
-
-      // Try to load tokenizer.json
-      if (typeof options.loadTokenizerJson === 'function') {
-        try {
-          const loaded = await options.loadTokenizerJson();
-          tokenizerJson = parseTokenizerJsonPayload(loaded);
-        } catch (err) {
-          const message = err instanceof Error ? err.message : String(err);
-          log.warn('Tokenizer', `Failed to load bundled tokenizer from custom loader: ${message}`);
-        }
-      }
-
-      if (!tokenizerJson && baseUrl) {
-        // Load from remote URL
-        const tokenizerUrl = `${baseUrl}/${tokenizerConfig.file}`;
-        try {
-          const response = await fetch(tokenizerUrl);
-          if (!response.ok) {
-            throw new Error(`Failed to fetch tokenizer: ${response.status}`);
-          }
-          tokenizerJson = await response.json();
-        } catch (err) {
-          const message = err instanceof Error ? err.message : String(err);
-          log.warn('Tokenizer', `Failed to fetch bundled tokenizer from URL: ${message}`);
-        }
-      } else {
-        // Try to load from OPFS (for cached models)
-        try {
-          const { loadTokenizerFromStore } = await import('../storage/shard-manager.js');
-          const tokenizerStr = await loadTokenizerFromStore();
-          if (tokenizerStr) {
-            tokenizerJson = JSON.parse(tokenizerStr);
-          }
-        } catch (err) {
-          const message = err instanceof Error ? err.message : String(err);
-          log.warn('Tokenizer', `Failed to load bundled tokenizer from OPFS: ${message}`);
-        }
-      }
-
-      if (tokenizerJson) {
-         (this.backend).load(tokenizerJson);
-        this.config = tokenizerConfig;
-        return;
-      }
-
-      // No external fallback - bundled tokenizer is required
-      throw new Error(
-        `[Tokenizer] Bundled tokenizer not found for model "${modelId}". ` +
-        `Expected tokenizer file: "${tokenizerConfig.file}". ` +
-        'Ensure tokenizer.json is in OPFS or model directory. ' +
-        'Clear browser storage and re-download the model.'
-      );
-    }
-
-    let hfModel = tokenizerConfig.hfModel ?? tokenizerConfig.modelId ?? null;
-    const allowArchFallback = tokenizerConfig.allowArchFallback === true;
-    if (allowArchFallback && !hfModel) {
-      throw new Error(
-        `[Tokenizer] tokenizer.allowArchFallback requires explicit tokenizer.hfModel or tokenizer.modelId for model "${modelId}".`
-      );
-    }
-
-    if (hfModel) {
-      // Use Transformers.js for HuggingFace models (fallback)
-      log.info('Tokenizer', `Loading from HuggingFace: ${hfModel}`);
-      this.backend = new TransformersTokenizer({
-        modelId: hfModel,
-        ...tokenizerConfig
-      });
-      await  (this.backend).load(hfModel);
-    } else if (tokenizerConfig.sentencepieceModel) {
-      // Load SentencePiece model
-      this.backend = new SentencePieceTokenizer(tokenizerConfig);
-
-      // Load the model data from the provided source
-      
-      let modelData;
-      if (typeof options.loadTokenizerModel === 'function') {
-        try {
-          const loaded = await options.loadTokenizerModel(
-            typeof tokenizerConfig.sentencepieceModel === 'string'
-              ? tokenizerConfig.sentencepieceModel
-              : undefined
-          );
-          if (loaded != null) {
-            modelData = toArrayBuffer(loaded, 'options.loadTokenizerModel');
-          }
-        } catch (err) {
-          const message = err instanceof Error ? err.message : String(err);
-          log.warn('Tokenizer', `Failed to load sentencepiece model from custom loader: ${message}`);
-        }
-      }
-      if (!modelData && tokenizerConfig.sentencepieceModel instanceof ArrayBuffer) {
-        modelData = tokenizerConfig.sentencepieceModel;
-      } else if (!modelData && tokenizerConfig.loadShard) {
-        // Use provided shard loader
-        modelData = await tokenizerConfig.loadShard(tokenizerConfig.sentencepieceModel);
-      } else if (!modelData && typeof tokenizerConfig.sentencepieceModel === 'string') {
-        if (options.baseUrl && !hasScheme(tokenizerConfig.sentencepieceModel)) {
-          const url = `${options.baseUrl}/${tokenizerConfig.sentencepieceModel}`;
-          const response = await fetch(url);
-          modelData = await response.arrayBuffer();
-        } else if (hasScheme(tokenizerConfig.sentencepieceModel)) {
-          const response = await fetch(tokenizerConfig.sentencepieceModel);
-          modelData = await response.arrayBuffer();
-        } else {
-          try {
-            const { loadTokenizerModelFromStore } = await import('../storage/shard-manager.js');
-            modelData = await loadTokenizerModelFromStore();
-          } catch (err) {
-            const message = err instanceof Error ? err.message : String(err);
-            log.warn('Tokenizer', `Failed to load tokenizer.model from OPFS: ${message}`);
-          }
-        }
-      }
-
-      if (modelData) {
-        await  (this.backend).load(modelData);
-      } else {
-        throw new Error('Could not load SentencePiece model data');
-      }
-    } else if (tokenizerConfig.vocab && tokenizerConfig.merges) {
-      // BPE with vocab + merges
-      this.backend = new BPETokenizer(tokenizerConfig);
-       (this.backend).load(tokenizerConfig.vocab, tokenizerConfig.merges);
-    } else {
-      throw new Error(
-        `[Tokenizer] No valid tokenizer configuration in manifest for model "${modelId}". ` +
-        'Provide tokenizer.hfModel or bundle tokenizer.json (tokenizer.type="bundled", tokenizer.file="tokenizer.json").'
-      );
-    }
-
-    this.config = tokenizerConfig;
-  }
-  encode(text) {
-    if (!this.backend) {
-      throw new Error('Tokenizer not initialized');
-    }
-    return this.backend.encode(text);
-  }
-
-  
-  decode(ids, skipSpecialTokens = true, trim = true) {
-    if (!this.backend) {
-      throw new Error('Tokenizer not initialized');
-    }
-    return this.backend.decode(ids, skipSpecialTokens, trim);
-  }
-
-  
-  getSpecialTokens() {
-    return this.backend?.specialTokens || {};
-  }
-
-  
-  getVocabSize() {
-    return this.backend?.getVocabSize() || 0;
-  }
-}
-
-export default Tokenizer;