@genai-fi/nanogpt 0.9.0 → 0.10.0

package/README.md

@@ -1,28 +1,366 @@
 # GenAI NanoGPT
 
-Developed as a part of the Finnish Generation AI research project. This is an implementation of [NanoGPT](https://github.com/karpathy/nanoGPT) for Tensorflow.js. It allows GPT models to be training and loaded within a web browser and exposes some XAI functionality.
+A browser-native implementation of GPT language models built on TensorFlow.js, developed as part of the Finnish Generation AI research project. This library enables training, fine-tuning, and inference of transformer-based language models entirely in the browser with support for explainable AI (XAI) features. It is intended to be used as an educational tool for learning about the model training process since it targets mostly tiny models. In principle it could be adapted to load other pre-trained models from Hugging Face.
 
-Work in progress...
+Live version available here: https://lm.gen-ai.fi
 
-# Install
+## Overview
 
-```
+GenAI NanoGPT is inspired by [Andrej Karpathy's NanoGPT](https://github.com/karpathy/nanoGPT) but reimagined for the browser using TensorFlow.js. It provides a complete pipeline for:
+
+-   **Training** language models from scratch in the browser
+-   **Loading** pre-trained models from various sources (Hugging Face, local files)
+-   **Generating** text efficiently on a wide range of devices
+-   **Analyzing** model behavior through attention visualization and embeddings
+-   **Optimizing** performance across CPU, WebGL, and WebGPU backends
+
+### Key Features
+
+-   🚀 **Browser-Native**: No server required - train and run models entirely client-side
+-   📱 **Works on Small Devices**: Train models on iPads, phones, and Chromebooks - no powerful hardware needed
+-   🎯 **Multiple Backends**: Automatic backend selection (CPU, WebGL, WebGPU) for optimal performance
+-   🔧 **Flexible Tokenization**: Support for both character-level and BPE tokenizers
+-   📊 **XAI Support**: Attention score visualization, gradient analysis, and embedding extraction
+-   💾 **Model Persistence**: Save and load models in SafeTensors format
+-   ⚡ **Performance Optimizations**: Custom WebGPU kernels, gradient checkpointing, and mixed precision training
+-   🎨 **Real-time Training**: Live training metrics and generation during training
+
+## Installation
+
+```bash
 npm install @genai-fi/nanogpt
 ```
 
-# Usage
+## Quick Start
+
+### Creating and Training a Model
+
+```javascript
+import { TeachableLLM, selectBackend } from '@genai-fi/nanogpt';
+
+// Select the best available backend
+await selectBackend('webgpu'); // or 'webgl', 'cpu'
+
+// Create a new model
+const model = TeachableLLM.create('char', {
+    vocabSize: 200,
+    blockSize: 128, // Context window size
+    nLayer: 4, // Number of transformer layers
+    nHead: 4, // Number of attention heads
+    nEmbed: 192, // Embedding dimension
+    dropout: 0.1,
+    useRope: true, // Use Rotary Position Embeddings
+});
+
+// Training data
+const trainingText = [
+    'The quick brown fox jumps over the lazy dog.',
+    'A journey of a thousand miles begins with a single step.',
+    // ... more text
+];
+
+// Train the model
+await model.train(trainingText, {
+    batchSize: 16,
+    learningRate: 3e-4,
+    maxSteps: 1000,
+    logInterval: 10,
+    validationSplit: 0.1,
+});
 
+// Generate text
+const output = await model.generateText('Once upon a time', {
+    maxLength: 100,
+    temperature: 0.8,
+    topP: 0.9,
+});
+
+console.log(output);
+```
+
+### Loading a Pre-trained Model
+
+```javascript
+import { TeachableLLM, waitForModel } from '@genai-fi/nanogpt';
+
+// Load from Hugging Face
+const model = TeachableLLM.loadModel('username/model-name');
+
+// Or load from a file
+const fileInput = document.getElementById('fileInput');
+fileInput.addEventListener('change', async (event) => {
+    const file = event.target.files[0];
+    const model = TeachableLLM.loadModel(file);
+    await waitForModel(model);
+
+    const text = await model.generateText('Hello');
+    console.log(text);
+});
 ```
-import { TeachableLLM, CharTokeniser } from '@genai-fi/nanogpt';
-import * as tf from '@tensorflow/tfjs';
 
-const tokeniser = new CharTokeniser();
-const model = TeachableLLM.create(tf, tokeniser, {
+## Event Handlers and Real-time Updates
+
+### Monitoring Training Progress
+
+Track training metrics in real-time with event handlers:
+
+```javascript
+const model = TeachableLLM.create('char', config);
+
+// Listen for training step updates
+model.on('trainStep', (step, progress) => {
+    console.log(`Step ${step.step}/${progress.totalSteps}`);
+    console.log(`Loss: ${step.loss.toFixed(4)}`);
+    console.log(`Validation Loss: ${step.valLoss?.toFixed(4) || 'N/A'}`);
+    console.log(`Progress: ${(progress.progress * 100).toFixed(1)}%`);
+    console.log(`Time Remaining: ${progress.timeRemaining}s`);
+
+    // Update UI progress bar
+    updateProgressBar(progress.progress);
+    updateLossChart(step.loss, step.valLoss);
+});
+
+await model.train(trainingText, options);
+```
+
+### Real-time Token Generation
+
+Stream generated tokens as they're produced:
+
+```javascript
+const generator = model.generator();
+
+// Listen for generated tokens
+generator.on('tokens', (tokens) => {
+    // tokens is an array of new token IDs
+    const text = model.tokeniser.decode(tokens);
+    console.log('New tokens:', text);
+
+    // Update UI incrementally
+    appendToOutput(text);
+});
+
+// Generation lifecycle events
+generator.on('start', () => {
+    console.log('Generation started');
+    showSpinner();
+});
+
+generator.on('stop', () => {
+    console.log('Generation complete');
+    hideSpinner();
+});
+
+generator.on('error', (error) => {
+    console.error('Generation error:', error);
+});
+
+// Start generation
+await generator.generate('Once upon a time', {
+    maxLength: 200,
+    temperature: 0.8,
+});
+```
+
+## Training on Small Devices
+
+GenAI NanoGPT is designed to work efficiently on resource-constrained devices like iPads, phones, and Chromebooks:
+
+### Recommended Settings for Small Devices
+
+```javascript
+// Smaller model configuration for mobile devices
+const mobileModel = TeachableLLM.create('char', {
     vocabSize: 200,
-    blockSize: 128,
-    nLayer: 4,
-    nHead: 3,
-    nEmbed: 192,
-    dropout: 0.0,
+    blockSize: 128, // Smaller context window
+    nLayer: 4, // Fewer layers
+    nHead: 3, // Fewer attention heads
+    nEmbed: 192, // Smaller embeddings
+});
+
+// Training options optimized for limited memory
+await mobileModel.train(trainingText, {
+    batchSize: 8, // Smaller batch size
+    learningRate: 3e-4,
+    maxSteps: 500,
+    validationSplit: 0.1,
+    logInterval: 50,
+    gradientCheckpointing: true,
+    mixedPrecision: true,
+});
+```
+
+### Tips for Training on Mobile Devices
+
+1. **Start Small**: Use smaller models (4 layers) and shorter context windows (128 tokens)
+2. **Reduce Batch Size**: Use batch sizes of 8-16 depending on available memory
+3. **Use Character Tokenization**: Character-level tokenizers use less memory than BPE
+4. **Optimize Training Data**: Use smaller datasets or train in stages
+
+## Advanced Usage
+
+### Attention Visualization
+
+```javascript
+const generator = model.generator();
+
+const text = await generator.generate('Prompt', {
+    attentionScores: true,
+    maxLength: 50,
 });
+
+// Get attention data for visualization
+const attentionData = generator.getAttentionData();
+// Shape: [num_tokens][num_layers][num_heads][seq_len][seq_len]
+
+const probabilities = generator.getProbabilitiesData();
+// Shape: [num_tokens][seq_len][vocab_size]
+```
+
+### Streaming Generation
+
+```javascript
+const generator = model.generator();
+
+generator.on('tokens', (tokens) => {
+    // Update UI with new tokens in real-time
+    updateDisplay(tokens);
+});
+
+generator.on('start', () => console.log('Generation started'));
+generator.on('stop', () => console.log('Generation complete'));
+
+await generator.generate('Once upon a time', {
+    maxLength: 200,
+});
+```
+
+### Memory Management
+
+```javascript
+// Enable profiling
+model.enableProfiler = true;
+
+// After training/generation
+const profiler = model.getProfiler();
+if (profiler) {
+    console.log('Memory stats:', profiler.getStats());
+}
+
+// Clean up
+model.dispose();
+```
+
+## Examples
+
+See the [`browser-tests`](browser-tests/) directory for complete examples:
+
+-   [`generate.html`](browser-tests/generate.html): Text generation with UI
+-   [`rope-train.html`](browser-tests/rope-train.html): Training a model with RoPE
+-   [`hf.html`](browser-tests/hf.html): Loading from Hugging Face
+-   [`loader.html`](browser-tests/loader.html): Loading different file formats
+-   [`perf.html`](browser-tests/perf.html): Performance testing
+
+## Development
+
+### Setup
+
+```bash
+git clone https://github.com/knicos/genai-nanogpt.git
+cd genai-nanogpt
+npm install
+```
+
+### Building
+
+```bash
+npm run build       # Build for production
+npm run dev         # Development mode with watch
+```
+
+### Testing
+
+```bash
+npm test            # Run all tests
+```
+
+### Browser Tests
+
+```bash
+npm run test:gl       # Start dev server
+```
+
+### Project Structure
+
+```
+lib/
+├── models/          # Model architectures (NanoGPT)
+├── layers/          # Transformer layers (attention, MLP, etc.)
+├── ops/             # Custom TensorFlow.js operations
+│   ├── cpu/         # CPU kernels
+│   ├── webgl/       # WebGL kernels
+│   └── webgpu/      # WebGPU kernels
+├── training/        # Training utilities and optimizers
+├── tokeniser/       # Tokenization implementations
+├── loader/          # Model loading/saving
+├── utilities/       # Helper functions
+└── TeachableLLM.ts  # Main API
+```
+
+### Custom Operations
+
+This library implements several custom TensorFlow.js operations optimized for transformer models:
+
+-   **RoPE**: Rotary Position Embeddings
+-   **Attention Mask**: Causal attention masking
+-   **RMS Norm**: Root Mean Square normalization
+-   **Adam Optimizer**: Extended Adam with weight decay
+-   **16-bit Operators**: To enable mixed-precision training
+
+See [`lib/ops`](lib/ops/) for implementations.
+
+### Contributing
+
+1. Fork the repository
+2. Create a feature branch: `git checkout -b feature/amazing-feature`
+3. Commit your changes: `git commit -m 'Add amazing feature'`
+4. Push to the branch: `git push origin feature/amazing-feature`
+5. Open a Pull Request
+
+### Code Style
+
+This project uses ESLint and Prettier for code formatting:
+
+```bash
+npm run lint        # Check code style
+```
+
+## Performance Tips
+
+1. **Use WebGPU**: Provides the best performance for training and inference
+2. **Batch Size**: Larger batches improve GPU utilization but require more memory
+3. **Mixed Precision**: Enable for faster training on supported hardware (coming soon)
+4. **Gradient Checkpointing**: Reduce memory usage during training, but slower
+5. **Use RoPE**: More efficient than absolute position embeddings
+6. **Start Small on Mobile**: Use 2-4 layers and batch size 2-8 on phones/tablets
+
+## Acknowledgments
+
+-   Inspired by [Andrej Karpathy's NanoGPT](https://github.com/karpathy/nanoGPT)
+-   Built with [TensorFlow.js](https://www.tensorflow.org/js)
+-   Developed as part of the Finnish [Generation AI research project](https://generation-ai-stn.fi)
+
+## Citation
+
+If you use this library in your research, please cite:
+
+```bibtex
+@inproceedings{10.1145/3769994.3770061,
+author = {Pope, Nicolas and Tedre, Matti},
+title = {A Teachable Machine for Transformers},
+year = {2025},
+publisher = {Association for Computing Machinery},
+doi = {10.1145/3769994.3770061},
+booktitle = {Proceedings of the 25th Koli Calling International Conference on Computing Education Research},
+}
 ```

package/dist/Generator.js

@@ -1,82 +1,73 @@
-import { E as C } from "./index-Dwqa6Zy2.js";
-import { E as _, F as I, G as O, a6 as R, t as q, k as K } from "./index-BzFyqcy-.js";
+import { E as C } from "./index-DvYrXKkX.js";
+import { A as _, B as I, E as O, t as R, k as q } from "./index-ZyQhjEPo.js";
+import "./utilities/packed.js";
 import "./ops/cpu/attentionMask.js";
 import "./ops/webgl/attentionMask.js";
 import "./ops/grads/attentionMask.js";
-import "./ops/cpu/qkv.js";
-import "./ops/webgl/qkv.js";
-import "./ops/grads/qkv.js";
-import { p as j } from "./random_width-CXVRloNK.js";
-import { t as G } from "./register_all_kernels-DIGpEwcf.js";
-import "./index-Tf7vU29b.js";
-import "./dataset-DlZtKmBq.js";
+import { p as K } from "./random_width-DY6Kk2Dl.js";
+import { t as j } from "./register_all_kernels-Bwu1PTuU.js";
+import "./index-Cp39cXWe.js";
+import "./dataset-0xP8GjwI.js";
 import "./ops/cpu/rope.js";
 import "./ops/webgl/rope.js";
-import "./ops/grads/rope.js";
+import "./rope-B5UUMsPi.js";
 import "./ops/cpu/appendCache.js";
 import "./ops/webgl/appendCache.js";
-import "./ops/cpu/fusedSoftmax.js";
-import "./ops/webgl/fusedSoftmax.js";
-import "./ops/grads/fusedSoftmax.js";
-import "./ops/cpu/matMulGelu.js";
-import "./ops/webgl/matMulGelu.js";
-import "./ops/grads/matMulGelu.js";
+import "./ops/grads/softmax16.js";
+import "./matMul16--R5hOwDG.js";
+import "./ops/webgl/matMul16.js";
+import "./ops/cpu/matMul16.js";
+import "./pack16-CFUqumar.js";
+import "./ops/transpose16.js";
+import "./ops/reshape16.js";
+import "./ops/cpu/qkv.js";
+import "./ops/webgl/qkv.js";
+import "./ops/grads/qkv.js";
 import "./ops/cpu/normRMS.js";
 import "./ops/webgl/normRMS.js";
 import "./ops/grads/normRMS.js";
+import "./ops/grads/add16.js";
 import { sparseSoftmaxCrossEntropy as V } from "./training/sparseCrossEntropy.js";
-import "./jszip.min-CjP2V1VV.js";
+import "./jszip.min-Bz5-11Bk.js";
 import $ from "./tokeniser/CharTokeniser.js";
 import "./ops/cpu/adamAdjust.js";
 import "./ops/webgl/adamAdjust.js";
 import "./ops/cpu/adamMoments.js";
 import "./ops/webgl/adamMoments.js";
-import "./papaparse.min-C8l2Kvo1.js";
-import M from "./utilities/topP.js";
+import "./papaparse.min-C0cScC2i.js";
+import G from "./utilities/topP.js";
 import "./ops/cpu/scatterSub.js";
 import "./ops/webgl/scatterSub.js";
 import "./ops/cpu/gatherSub.js";
 import "./ops/webgl/gatherSub.js";
+import "./ops/cpu/matMulGelu.js";
+import "./ops/webgl/matMulGelu.js";
+import "./ops/grads/matMulGelu.js";
 import "./ops/cpu/gelu.js";
 import "./ops/webgl/gelu.js";
-import "./gelu-Bp_-935b.js";
+import "./gelu-CNLFZWea.js";
 import "./ops/webgl/log.js";
 import "./checks/normRMS.js";
 import "./checks/normRMSGrad.js";
-import N from "./utilities/multinomialCPU.js";
-import { r as E } from "./reshape-CnIwVG1c.js";
-import { t as P } from "./tensor2d-D76QGjF3.js";
-import { s as S } from "./softmax-D7Jj3p_P.js";
-import { g as F } from "./gather-Dnpgw-YQ.js";
-import { c as H } from "./concat-B912vBbo.js";
-/**
- * @license
- * Copyright 2020 Google LLC. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * =============================================================================
- */
-function U(p, t, s, e = !1) {
-  const o = I(p, "logits", "multinomial"), i = o.size, c = o.rank;
+import M from "./utilities/multinomialCPU.js";
+import { i as N } from "./tensor_util-DV-FP5Q3.js";
+import { r as E } from "./reshape-DevtBWtf.js";
+import { t as P } from "./tensor2d-G4Ys2GxX.js";
+import { s as S } from "./softmax-ZHVebtR1.js";
+import { g as B } from "./gather-DykLGqmW.js";
+import { c as H } from "./concat-BHlIJeyT.js";
+function U(l, t, s, e = !1) {
+  const o = I(l, "logits", "multinomial"), i = o.size, c = o.rank;
   if (i < 2)
     throw new Error(`Error in multinomial: you need at least 2 outcomes, but got ${i}.`);
   if (c > 2)
     throw new Error(`Rank of probabilities must be 1 or 2, but is ${c}`);
   s = s || Math.random();
-  const n = { logits: c === 1 ? E(o, [1, -1]) : o }, l = { numSamples: t, seed: s, normalized: e }, d = O.runKernel(R, n, l);
+  const n = { logits: c === 1 ? E(o, [1, -1]) : o }, p = { numSamples: t, seed: s, normalized: e }, d = O.runKernel(N, n, p);
   return c === 1 ? E(d, [d.size]) : d;
 }
 const z = /* @__PURE__ */ _({ multinomial_: U }), W = [
-  ...Array.from({ length: 95 }, (p, t) => String.fromCharCode(t + 32)),
+  ...Array.from({ length: 95 }, (l, t) => String.fromCharCode(t + 32)),
   // ASCII
   // Spanish accented letters and punctuation
   ..."áéíóúüñ¿¡",
@@ -87,10 +78,10 @@ const z = /* @__PURE__ */ _({ multinomial_: U }), W = [
   // Cyrillic letters
   ..."абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ"
 ];
-function B(p, t) {
-  return p.length === t ? p : p.length > t ? p.slice(0, t) : p.concat(Array(t - p.length).fill(""));
+function F(l, t) {
+  return l.length === t ? l : l.length > t ? l.slice(0, t) : l.concat(Array(t - l.length).fill(""));
 }
-class Wt extends C {
+class te extends C {
   constructor(t, s) {
     super(), this.model = t, this.tokeniser = s, this.actualTokeniser = s;
   }
@@ -116,7 +107,7 @@ class Wt extends C {
     const c = await t.decode([i]);
     if (e) {
       const T = await Promise.all(
-        e.map((n) => n.array().then((l) => l))
+        e.map((n) => n.array().then((p) => p))
       );
       e.forEach((n) => n.dispose()), this.attentionData.push(T);
     }
@@ -131,14 +122,14 @@ class Wt extends C {
       } : void 0,
       cache: s,
       outputEmbeddings: !!e?.embeddings
-    }, [l, d] = q(() => {
-      const a = t, m = a.shape[1], h = m <= this.model.config.blockSize ? a : a.slice(
+    }, [p, d] = R(() => {
+      const r = t, m = r.shape[1], h = m <= this.model.config.blockSize ? r : r.slice(
         [0, m - this.model.config.blockSize],
-        [a.shape[0], this.model.config.blockSize]
-      ), r = T ? this.model.config.blockSize - h.shape[1] : 0, v = r > 0 ? j(h, [
+        [r.shape[0], this.model.config.blockSize]
+      ), a = T ? this.model.config.blockSize - h.shape[1] : 0, v = a > 0 ? K(h, [
         [0, 0],
-        [0, r]
-      ]) : h, [g] = this.model.forward(n, v), u = g.shape[1] - 1 - r, f = g.slice([0, u, 0], [g.shape[0], 1, g.shape[2]]);
+        [0, a]
+      ]) : h, [g] = this.model.forward(n, v), u = g.shape[1] - 1 - a, f = g.slice([0, u, 0], [g.shape[0], 1, g.shape[2]]);
       let y;
       if (e?.targets) {
         const k = e.targets.shift();
@@ -148,46 +139,46 @@ class Wt extends C {
         }
       }
       return n.attentionScores?.attentionOut && n.attentionScores.attentionOut.forEach((k, w) => {
-        k.shape[1] !== 1 && (n.attentionScores.attentionOut[w] = K(
+        k.shape[1] !== 1 && (n.attentionScores.attentionOut[w] = q(
           k.slice([0, u, 0], [k.shape[0], 1, k.shape[2]])
         ), k.dispose());
       }), g.dispose(), [f.div(o).squeeze([1]), y];
     });
     let b, x;
     if (c) {
-      const a = S(l), m = await a.array();
-      a.dispose();
-      const h = M(m, c);
-      e?.includeProbabilities && (x = m), b = N(h);
+      const r = S(p), m = await r.array();
+      r.dispose();
+      const h = G(m, c);
+      e?.includeProbabilities && (x = m), b = M(h);
     } else if (i) {
-      const { values: a, indices: m } = G(l, i), h = z(a, 1);
-      b = F(m, h, 1), a.dispose(), m.dispose(), h.dispose();
-    } else if (b = z(l, 1), e?.includeProbabilities) {
-      const a = S(l);
-      x = await a.array(), a.dispose();
+      const { values: r, indices: m } = j(p, i), h = z(r, 1);
+      b = B(m, h, 1), r.dispose(), m.dispose(), h.dispose();
+    } else if (b = z(p, 1), e?.includeProbabilities) {
+      const r = S(p);
+      x = await r.array(), r.dispose();
     }
     if (n.embeddings) {
-      const m = (e?.embeddings === "all" ? n.embeddings : n.embeddings.filter((r) => r.name.startsWith("block_output_"))).map(async (r) => {
-        const v = r.tensor.shape[1], g = r.tensor.slice([0, v - 1, 0], [r.tensor.shape[0], 1, r.tensor.shape[2]]);
-        r.tensor.dispose();
+      const m = (e?.embeddings === "all" ? n.embeddings : n.embeddings.filter((a) => a.name.startsWith("block_output_"))).map(async (a) => {
+        const v = a.tensor.shape[1], g = a.tensor.slice([0, v - 1, 0], [a.tensor.shape[0], 1, a.tensor.shape[2]]);
+        a.tensor.dispose();
         const u = g.squeeze([1]);
         if (g.dispose(), e?.embeddings === "softmax") {
           const f = this.model.project(u);
           u.dispose();
           const y = S(f, -1);
-          return f.dispose(), { name: r.name, tensor: await y.array() };
+          return f.dispose(), { name: a.name, tensor: await y.array() };
         } else if (e?.embeddings === "logits") {
           const f = this.model.project(u);
-          return u.dispose(), { name: r.name, tensor: await f.array() };
+          return u.dispose(), { name: a.name, tensor: await f.array() };
         } else {
           const f = await u.array();
-          return u.dispose(), { name: r.name, tensor: f };
+          return u.dispose(), { name: a.name, tensor: f };
         }
       }), h = await Promise.all(m);
       this.embeddingsData.push(h);
     }
     const A = b.reshape([1, 1]);
-    b.dispose(), b = A, l.dispose();
+    b.dispose(), b = A, p.dispose();
     let L;
     return d && (L = await d.array(), d.dispose()), { output: b, probabilities: x, attention: n.attentionScores?.attentionOut, loss: L };
   }
@@ -211,10 +202,10 @@ class Wt extends C {
         const d = s;
         s = H([s, i], 1), d.dispose();
       }
-      const l = await this.processResponse(this.actualTokeniser, i, T, c);
-      if (this.cache || i.dispose(), l === null)
+      const p = await this.processResponse(this.actualTokeniser, i, T, c);
+      if (this.cache || i.dispose(), p === null)
         break;
-      this.outputText += l;
+      this.outputText += p;
     }
     return s.dispose(), this.outputText;
   }
@@ -233,7 +224,7 @@ class Wt extends C {
         o[i] = { k: void 0, v: void 0, length: 0, cumulativeLength: 0 };
       this.cache = o, this.lastToken = -1;
     }
-    const e = this.tokeniser.trained ? this.tokeniser : new $(B(W, this.tokeniser.vocabSize));
+    const e = this.tokeniser.trained ? this.tokeniser : new $(F(W, this.tokeniser.vocabSize));
     this.actualTokeniser = e;
   }
   async step(t, s) {
@@ -268,5 +259,5 @@ class Wt extends C {
   }
 }
 export {
-  Wt as default
+  te as default
 };