@genai-fi/nanogpt 0.3.0 → 0.3.1

package/dist/NanoGPTModel.js

@@ -5,14 +5,14 @@ import C from "./layers/RoPECache.js";
 import q from "./layers/RMSNorm.js";
 import { estimateParameterCount as K } from "./utilities/parameters.js";
 import { createSoftmaxCrossEntropyWithGrad as N } from "./training/sparseCrossEntropy.js";
-import R from "./layers/BaseLayer.js";
-import { r as T, e as D, t as A, o as B, p as G } from "./random_width-PbCt7RXv.js";
-import { o as v, h as E, p as O, E as y, W as H, X as Q, Y as X, t as w, Z as j, f as _ } from "./index-pWA4_lUh.js";
-import { e as U, a as V } from "./exports_layers-7idKoYqh.js";
-import { r as I } from "./reshape-C8CR_Bad.js";
-import { r as Y } from "./range-CcDl05lo.js";
-import { g as Z } from "./gather-BPGW8RsB.js";
-import { s as J } from "./softmax-Be_lsqUc.js";
+import T from "./layers/BaseLayer.js";
+import { r as R, e as D, p as A } from "./random_width-oeUIlUZj.js";
+import { o as y, h as E, p as B, E as z, W as G, X as O, Y as Q, t as w, Z as X, f as _ } from "./index-pWA4_lUh.js";
+import { e as j, a as U } from "./exports_layers-tbTBcwMM.js";
+import { r as S } from "./reshape-C8CR_Bad.js";
+import { r as V } from "./range-CcDl05lo.js";
+import { g as Y } from "./gather-BPGW8RsB.js";
+import { s as Z } from "./softmax-Be_lsqUc.js";
 /**
  * @license
  * Copyright 2020 Google LLC. All Rights Reserved.
@@ -29,13 +29,13 @@ import { s as J } from "./softmax-Be_lsqUc.js";
  * limitations under the License.
  * =============================================================================
  */
-function tt(m, t) {
-  let e = E(m, "a", "mod"), s = E(t, "b", "mod");
-  [e, s] = O(e, s);
-  const i = { a: e, b: s };
-  return y.runKernel(H, i);
+function H(m, t) {
+  let e = E(m, "a", "mod"), o = E(t, "b", "mod");
+  [e, o] = B(e, o);
+  const i = { a: e, b: o };
+  return z.runKernel(G, i);
 }
-const et = /* @__PURE__ */ v({ mod_: tt });
+const J = /* @__PURE__ */ y({ mod_: H });
 /**
  * @license
  * Copyright 2020 Google LLC. All Rights Reserved.
@@ -52,17 +52,17 @@ const et = /* @__PURE__ */ v({ mod_: tt });
  * limitations under the License.
  * =============================================================================
  */
-function ot(m, t, e, s = !1) {
-  const i = E(m, "logits", "multinomial"), o = i.size, n = i.rank;
-  if (o < 2)
-    throw new Error(`Error in multinomial: you need at least 2 outcomes, but got ${o}.`);
-  if (n > 2)
-    throw new Error(`Rank of probabilities must be 1 or 2, but is ${n}`);
+function tt(m, t, e, o = !1) {
+  const i = E(m, "logits", "multinomial"), s = i.size, r = i.rank;
+  if (s < 2)
+    throw new Error(`Error in multinomial: you need at least 2 outcomes, but got ${s}.`);
+  if (r > 2)
+    throw new Error(`Rank of probabilities must be 1 or 2, but is ${r}`);
   e = e || Math.random();
-  const r = { logits: n === 1 ? I(i, [1, -1]) : i }, a = { numSamples: t, seed: e, normalized: s }, h = y.runKernel(Q, r, a);
-  return n === 1 ? I(h, [h.size]) : h;
+  const n = { logits: r === 1 ? S(i, [1, -1]) : i }, h = { numSamples: t, seed: e, normalized: o }, a = z.runKernel(O, n, h);
+  return r === 1 ? S(a, [a.size]) : a;
 }
-const S = /* @__PURE__ */ v({ multinomial_: ot });
+const I = /* @__PURE__ */ y({ multinomial_: tt });
 /**
  * @license
  * Copyright 2018 Google LLC. All Rights Reserved.
@@ -79,20 +79,20 @@ const S = /* @__PURE__ */ v({ multinomial_: ot });
  * limitations under the License.
  * =============================================================================
  */
-function st(m, t = 1, e = !0) {
-  const s = E(m, "x", "topk");
-  if (s.rank === 0)
+function et(m, t = 1, e = !0) {
+  const o = E(m, "x", "topk");
+  if (o.rank === 0)
     throw new Error("topk() expects the input to be of rank 1 or higher");
-  const i = s.shape[s.shape.length - 1];
+  const i = o.shape[o.shape.length - 1];
   if (t < 0)
     throw new Error(`'k' passed to topk() must be >= 0 but got ${t}`);
   if (t > i)
     throw new Error(`'k' passed to topk() must be <= the last dimension (${i}) but got ${t}`);
-  const o = { x: s }, n = { k: t, sorted: e }, [l, r] = y.runKernel(X, o, n);
-  return { values: l, indices: r };
+  const s = { x: o }, r = { k: t, sorted: e }, [l, n] = z.runKernel(Q, s, r);
+  return { values: l, indices: n };
 }
-const it = /* @__PURE__ */ v({ topk_: st });
-class Et extends R {
+const ot = /* @__PURE__ */ y({ topk_: et });
+class kt extends T {
   config;
   wte;
   // Token embeddings
@@ -111,12 +111,12 @@ class Et extends R {
       vocabSize: this.config.vocabSize,
       embedDim: this.config.nEmbed,
       name: "token_embedding"
-    }), this.config.useRope === !1 ? this.wpe = U({
+    }), this.config.useRope === !1 ? this.wpe = j({
       inputDim: this.config.blockSize,
       outputDim: this.config.nEmbed,
       name: "positional_embedding",
-      embeddingsInitializer: T({ mean: 0, stddev: 0.02 })
-    }) : this.ropeCache = new C(this.config), this.drop = V({ rate: this.config.dropout }), this.blocks = [];
+      embeddingsInitializer: R({ mean: 0, stddev: 0.02 })
+    }) : this.ropeCache = new C(this.config), this.drop = U({ rate: this.config.dropout }), this.blocks = [];
     for (let e = 0; e < this.config.nLayer; e++)
       this.blocks.push(new L(e, this.config, this.ropeCache));
     this.lnF = new q([this.config.nEmbed], 1e-8, "final_rms_norm");
@@ -142,14 +142,14 @@ class Et extends R {
       this.blocks[e].loadWeights(t);
     this.lnF.setWeights(t.get("final_rms_norm") || []);
   }
-  inputPhase(t, e, s = !1) {
+  inputPhase(t, e, o = !1) {
     return w(() => {
       const i = this.wte.embed(t);
       if (this.config.useRope === !1) {
-        const [, o] = t.shape, n = this.config.blockSize, l = Y(0, o, 1, "int32"), r = et(j(l, _(e, "int32")), _(n, "int32")), a = this.wpe.apply(r), h = i.add(a);
-        return this.drop.apply(h, { training: s });
+        const [, s] = t.shape, r = this.config.blockSize, l = V(0, s, 1, "int32"), n = J(X(l, _(e, "int32")), _(r, "int32")), h = this.wpe.apply(n), a = i.add(h);
+        return this.drop.apply(a, { training: o });
       } else
-        return this.drop.apply(i, { training: s });
+        return this.drop.apply(i, { training: o });
     });
   }
   setSkipMask(t) {
@@ -186,8 +186,8 @@ class Et extends R {
   calculateLoss(t, e) {
     try {
       return N()(t, e).mean();
-    } catch (s) {
-      throw console.error("Error computing loss:", s), new Error(`Loss computation failed: ${s}`);
+    } catch (o) {
+      throw console.error("Error computing loss:", o), new Error(`Loss computation failed: ${o}`);
     }
   }
   // Attention rollout per Abnar & Zuidema (2020)
@@ -196,77 +196,67 @@ class Et extends R {
     return w(() => {
       if (t.length === 0)
         throw new Error("No attentions for rollout");
-      const [e, s, i] = t[0].shape;
-      for (const o of t) {
-        const [n, l, r] = o.shape;
-        if (n !== e || l !== s || r !== i)
+      const [e, o, i] = t[0].shape;
+      for (const s of t) {
+        const [r, l, n] = s.shape;
+        if (r !== e || l !== o || n !== i)
           throw new Error(
-            `Inconsistent attention shapes in rollout: expected [${e},${s},${i}] got [${n},${l},${r}]`
+            `Inconsistent attention shapes in rollout: expected [${e},${o},${i}] got [${r},${l},${n}]`
           );
       }
-      if (s === i) {
-        const o = D(i, i).expandDims(0);
-        let n = o.tile([e, 1, 1]);
+      if (o === i) {
+        const s = D(i, i).expandDims(0);
+        let r = s.tile([e, 1, 1]);
         for (const l of t) {
-          const r = l.add(o);
-          n = r.div(r.sum(-1, !0)).matMul(n);
+          const n = l.add(s);
+          r = n.div(n.sum(-1, !0)).matMul(r);
         }
-        return n;
+        return r;
       }
-      if (s === 1) {
-        let o = null;
-        const n = A([i - 1], "int32"), l = B(n, i).reshape([1, 1, i]).tile([e, 1, 1]);
-        n.dispose();
-        for (const r of t) {
-          let a = r.add(l);
-          a = a.div(a.sum(-1, !0)), o == null ? o = a : (o = o.mul(a), o = o.div(o.sum(-1, !0)));
-        }
-        return o;
-      }
-      throw new Error(`Unsupported attention shapes for rollout: [B=${e}, Q=${s}, K=${i}]`);
+      throw new Error(`Unsupported attention shapes for rollout: [B=${e}, Q=${o}, K=${i}]`);
     });
   }
-  forward(t, e, s = !1, i = !1, o) {
+  forward(t, e, o = !1, i = !1, s) {
     return this.validateInput(t), w(() => {
       this.startMemory();
-      const n = o?.[0]?.length ?? 0;
-      let l = this.inputPhase(t, n, s);
-      const r = [];
-      if (o && o.length !== this.blocks.length)
-        throw console.error("Cache", o), new Error(`Cache length ${o.length} does not match number of blocks ${this.blocks.length}`);
+      const r = s?.[0]?.length ?? 0;
+      let l = this.inputPhase(t, r, o);
+      const n = [];
+      if (s && s.length !== this.blocks.length)
+        throw console.error("Cache", s), new Error(`Cache length ${s.length} does not match number of blocks ${this.blocks.length}`);
       for (let c = 0; c < this.blocks.length; c++) {
         const u = l, d = this.blocks[c], {
           output: b,
           attention: k,
           cache: f
-        } = d.call(l, s, i, o ? o[c] : void 0);
-        l = b, u.dispose(), i && k && r.push(k), o && f ? (o[c]?.k.dispose(), o[c]?.v.dispose(), o[c] = f) : f && (f.k.dispose(), f.v.dispose());
+        } = d.call(l, o, i, s ? s[c] : void 0);
+        l = b, u.dispose(), i && k && n.push(k), s && f ? (s[c]?.k.dispose(), s[c]?.v.dispose(), s[c] = f) : f && (f.k.dispose(), f.v.dispose());
       }
-      let a;
-      i && r.length > 0 && (a = this.computeAttentionRollout(r)), l = this.lnF.apply(l);
-      const h = this.wte.project(l);
+      let h;
+      i && n.length > 0 && (h = this.computeAttentionRollout(n)), l = this.lnF.apply(l);
+      const a = this.wte.project(l);
       let p;
-      return e && (p = this.calculateLoss(h, e)), this.endMemory("Forward"), { logits: h, loss: p, attention: i ? a : void 0 };
+      return e && (p = this.calculateLoss(a, e)), this.endMemory("Forward"), { logits: a, loss: p, attention: i ? h : void 0 };
     });
   }
-  generate(t, e, s) {
-    const i = s?.temperature ?? 1, o = s?.topK, n = s?.usePadding ?? !1, l = s?.includeAttention ?? !1;
+  generate(t, e, o) {
+    const i = o?.temperature ?? 1, s = o?.topK, r = o?.usePadding ?? !1, l = o?.includeAttention ?? !1;
     return w(() => {
-      const r = t, a = r.shape[1], h = a <= this.config.blockSize ? r : r.slice(
-        [0, a - this.config.blockSize],
-        [r.shape[0], this.config.blockSize]
-      ), p = n ? this.config.blockSize - h.shape[1] : 0, c = p > 0 ? G(h, [
+      const n = t, h = n.shape[1], a = h <= this.config.blockSize ? n : n.slice(
+        [0, h - this.config.blockSize],
+        [n.shape[0], this.config.blockSize]
+      ), p = r ? this.config.blockSize - a.shape[1] : 0, c = p > 0 ? A(a, [
         [0, 0],
         [0, p]
-      ]) : h, { logits: u, attention: d } = this.forward(c, void 0, !1, l, e), b = u.shape[1] - 1 - p, k = u.slice([0, b, 0], [u.shape[0], 1, u.shape[2]]), f = d ? d.slice([0, b, 0], [d.shape[0], 1, d.shape[2]]) : void 0, $ = k.div(i);
+      ]) : a, { logits: u, attention: d } = this.forward(c, void 0, !1, l, e), b = u.shape[1] - 1 - p, k = u.slice([0, b, 0], [u.shape[0], 1, u.shape[2]]), f = d ? d.slice([0, b, 0], [d.shape[0], 1, d.shape[2]]) : void 0, $ = k.div(i);
       let g;
-      if (o) {
-        const { values: x, indices: M } = it($, o), W = S(x.squeeze([1]), 1);
-        g = Z(M.squeeze([1]), W, 1);
+      if (s) {
+        const { values: M, indices: x } = ot($, s), W = I(M.squeeze([1]), 1);
+        g = Y(x.squeeze([1]), W, 1);
       } else
-        g = S($.squeeze([1]), 1);
-      let z;
-      return s?.includeProbabilities && (z = J($.squeeze([1]))), g = g.reshape([1, 1]), { output: g, attention: f?.squeeze([1]), probabilities: z };
+        g = I($.squeeze([1]), 1);
+      let v;
+      return o?.includeProbabilities && (v = Z($.squeeze([1]))), g = g.reshape([1, 1]), { output: g, attention: f?.squeeze([1]), probabilities: v };
     });
   }
   getNumParams() {
@@ -277,5 +267,5 @@ class Et extends R {
   }
 }
 export {
-  Et as default
+  kt as default
 };

package/dist/TeachableLLM.d.ts

@@ -4,15 +4,17 @@ import { default as NanoGPT } from './NanoGPTModel';
 import { SaveOptions } from './utilities/save';
 import { default as Generator, IGenerateOptions } from './Generator';
 import { default as Trainer, ITrainerOptions } from './Trainer';
-import { default as EE } from 'eventemitter3';
 import { default as MemoryProfiler } from './utilities/profile';
 type TeachableLLMStatus = 'warmup' | 'awaitingTokens' | 'ready' | 'training' | 'loading' | 'busy' | 'error';
-export default class TeachableLLM extends EE<'status' | 'error' | 'trainStep'> {
+export default class TeachableLLM {
+    private ee;
     private _config?;
     private _model?;
     private _tokeniser?;
     private _status;
     constructor(tokeniser?: ITokeniser, model?: NanoGPT);
+    get vocab(): string[];
+    get loaded(): boolean;
     get config(): GPTConfig;
     get model(): NanoGPT;
     get tokeniser(): ITokeniser;
@@ -28,8 +30,17 @@ export default class TeachableLLM extends EE<'status' | 'error' | 'trainStep'> {
     getNumParams(): number;
     trainer(): Trainer;
     train(text: string[], options?: ITrainerOptions): Promise<void>;
+    trainTokeniser(text: string[]): Promise<number>;
     generator(): Generator;
     generateText(prompt?: string, options?: IGenerateOptions): Promise<string>;
     dispose(): void;
+    on(event: 'status', listener: (status: TeachableLLMStatus) => void): void;
+    on(event: 'error', listener: (error: Error) => void): void;
+    on(event: 'trainStep', listener: (step: number) => void): void;
+    on(event: 'loaded', listener: () => void): void;
+    off(event: 'status', listener: (status: TeachableLLMStatus) => void): void;
+    off(event: 'error', listener: (error: Error) => void): void;
+    off(event: 'trainStep', listener: (step: number) => void): void;
+    off(event: 'loaded', listener: () => void): void;
 }
 export {};

package/dist/TeachableLLM.js

@@ -1,12 +1,12 @@
-import { defaultConfig as l } from "./config.js";
+import { defaultConfig as d } from "./config.js";
 import h from "./NanoGPTModel.js";
-import { saveModel as d } from "./utilities/save.js";
+import { saveModel as l } from "./utilities/save.js";
 import { loadModel as f } from "./utilities/load.js";
 import u from "./Generator.js";
 import _ from "./Trainer.js";
-import { E as p } from "./index-Dwqa6Zy2.js";
+import { E as c } from "./index-Dwqa6Zy2.js";
 import { dummyPassAsync as m } from "./utilities/dummy.js";
-import c from "./tokeniser/CharTokeniser.js";
+import p from "./tokeniser/CharTokeniser.js";
 import g from "./tokeniser/bpe.js";
 import "./papaparse.min-C8l2Kvo1.js";
 import "./index-Tf7vU29b.js";
@@ -29,13 +29,20 @@ import "./ops/grads/rope.js";
 import "./ops/cpu/appendCache.js";
 import "./ops/webgl/appendCache.js";
 import w from "./utilities/profile.js";
-class a extends p {
+class a {
+  ee = new c();
   _config;
   _model;
   _tokeniser;
   _status = "loading";
   constructor(t, e) {
-    super(), this._config = e?.config, this._tokeniser = t, this._model = e;
+    this._config = e?.config, this._tokeniser = t, this._model = e;
+  }
+  get vocab() {
+    return this._tokeniser?.getVocab() || [];
+  }
+  get loaded() {
+    return !!this._model && !!this._tokeniser && !!this._config;
   }
   get config() {
     if (!this._config)
@@ -59,34 +66,34 @@ class a extends p {
     return this._status === "ready" && !!this._model && !!this._tokeniser && this.tokeniser.trained;
   }
   setStatus(t) {
-    this._status !== t && (this._status = t, this.emit("status", t));
+    this._status !== t && (this._status = t, this.ee.emit("status", t));
   }
   saveModel(t) {
     if (!this._model || !this._tokeniser)
       throw new Error("Model or tokeniser is not initialized.");
-    return d(this._model, this._tokeniser, t);
+    return l(this._model, this._tokeniser, t);
   }
   static loadModel(t) {
     const e = new a();
-    return f(t).then(({ model: r, tokeniser: o }) => {
-      e._model = r, e._tokeniser = o, e._config = r.config, e.setStatus("warmup"), m(r).then(() => {
-        e.setStatus("ready");
+    return f(t).then(({ model: i, tokeniser: o }) => {
+      e._model = i, e._tokeniser = o, e._config = i.config, e.setStatus("warmup"), m(i).then(() => {
+        e.setStatus("ready"), e.ee.emit("loaded");
       }).catch((s) => {
-        e.setStatus("error"), e.emit("error", s);
+        e.setStatus("error"), e.ee.emit("error", s);
       });
-    }).catch((r) => {
-      e.setStatus("error"), e.emit("error", r);
+    }).catch((i) => {
+      e.setStatus("error"), e.ee.emit("error", i);
     }), e;
   }
   static create(t, e = {}) {
-    const r = { ...l, ...e }, o = t === "char" ? new c(r.vocabSize) : new g(r.vocabSize), s = new h(r), i = new a(o, s);
-    return i.setStatus("warmup"), m(s).then(() => {
-      i.tokeniser.trained ? i.setStatus("ready") : (i.setStatus("awaitingTokens"), i.tokeniser.once("trainStatus", (n) => {
-        n === "trained" && i.setStatus("ready");
+    const i = { ...d, ...e }, o = t === "char" ? new p(i.vocabSize) : new g(i.vocabSize), s = new h(i), r = new a(o, s);
+    return r.setStatus("warmup"), m(s).then(() => {
+      r.tokeniser.trained ? (r.setStatus("ready"), r.ee.emit("loaded")) : (r.setStatus("awaitingTokens"), r.ee.emit("loaded"), r.tokeniser.once("trainStatus", (n) => {
+        n === "trained" && r.setStatus("ready");
       }));
     }).catch((n) => {
-      i.setStatus("error"), i.emit("error", n);
-    }), i;
+      r.setStatus("error"), r.ee.emit("error", n);
+    }), r;
   }
   getProfiler() {
     return this._model?.getProfiler();
@@ -112,17 +119,23 @@ class a extends p {
       throw new Error("Model or tokeniser is not initialized.");
     const t = new _(this._model, this._tokeniser);
     return t.on("start", () => this.setStatus("training")), t.on("stop", () => this.setStatus("ready")), t.on("log", async (e) => {
-      const r = this.listeners("trainStep");
-      for (const o of r)
+      const i = this.ee.listeners("trainStep");
+      for (const o of i)
         await o(e);
     }), t;
   }
   train(t, e) {
     return this.trainer().train(t, e);
   }
+  async trainTokeniser(t) {
+    if (!this._tokeniser)
+      throw new Error("tokeniser_not_initialized.");
+    const e = await this._tokeniser.train(t);
+    return this._status === "awaitingTokens" && this.setStatus("ready"), e;
+  }
   generator() {
     if (!this._model || !this._tokeniser)
-      throw new Error("Model or tokeniser is not initialized.");
+      throw new Error("model_or_tokeniser_not_initialized.");
     const t = new u(this._model, this._tokeniser);
     return t.on("start", () => {
       this.status === "ready" && this.setStatus("busy");
@@ -136,6 +149,18 @@ class a extends p {
   dispose() {
     this._model?.dispose();
   }
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  on(t, e) {
+    if (t === "loaded" && this.loaded) {
+      setTimeout(() => e(), 0);
+      return;
+    }
+    this.ee.on(t, e);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  off(t, e) {
+    this.ee.off(t, e);
+  }
 }
 export {
   a as default

package/dist/{exports_layers-7idKoYqh.js → exports_layers-tbTBcwMM.js}

@@ -1,4 +1,4 @@
-import { E as e, D as r, a as o } from "./random_width-PbCt7RXv.js";
+import { E as e, D as r, a as o } from "./random_width-oeUIlUZj.js";
 import "./index-pWA4_lUh.js";
 /**
  * @license

package/dist/layers/CausalSelfAttention.js

@@ -4,8 +4,8 @@ import { qkv as y } from "../ops/qkv.js";
 import { rope as m } from "../ops/rope.js";
 import { appendCache as b } from "../ops/appendCache.js";
 import { w as j, x as f, t as z } from "../index-pWA4_lUh.js";
-import { r as w, l as E, w as D, b as T } from "../random_width-PbCt7RXv.js";
-import { d as L, a as k } from "../exports_layers-7idKoYqh.js";
+import { r as w, l as E, w as D, b as T } from "../random_width-oeUIlUZj.js";
+import { d as L, a as k } from "../exports_layers-tbTBcwMM.js";
 import { o as W } from "../ones-Cog-G2ag.js";
 import { z as M } from "../zeros-CCy9C3uU.js";
 import { v as A } from "../variable-Dl_ub3pk.js";

package/dist/layers/MLP.js

@@ -1,7 +1,7 @@
 import { t as n } from "../index-pWA4_lUh.js";
 import l from "./BaseLayer.js";
-import { r as s } from "../random_width-PbCt7RXv.js";
-import { d as i, a as c } from "../exports_layers-7idKoYqh.js";
+import { r as s } from "../random_width-oeUIlUZj.js";
+import { d as i, a as c } from "../exports_layers-tbTBcwMM.js";
 class u extends l {
   cFc;
   cProj;

package/dist/layers/TiedEmbedding.js

@@ -1,4 +1,4 @@
-import { r as t, d as s } from "../random_width-PbCt7RXv.js";
+import { r as t, d as s } from "../random_width-oeUIlUZj.js";
 import "../index-pWA4_lUh.js";
 import { v as r } from "../variable-Dl_ub3pk.js";
 import { g as d } from "../gather-BPGW8RsB.js";

package/dist/main.d.ts

@@ -7,5 +7,6 @@ export { default as loadTextData } from './data/textLoader';
 export type { ITrainerOptions } from './Trainer';
 export type { IGenerateOptions } from './Generator';
 export type { TrainingLogEntry } from './NanoGPTModel';
+export type { ITokeniser } from './tokeniser/type';
 export type { GPTConfig } from './config';
 export { estimateParameterCount, estimateMemoryUsage, estimateTrainingMemoryUsage, estimateResources, validateConfig, } from './utilities/parameters';

package/dist/{random_width-PbCt7RXv.js → random_width-oeUIlUZj.js}

@@ -15481,9 +15481,7 @@ export {
   Yt as d,
   Jr as e,
   Zd as l,
-  cp as o,
   Yr as p,
   wy as r,
-  At as t,
   $t as w
 };

package/dist/tokeniser/bpe.js

@@ -53,7 +53,7 @@ function m(o, e) {
     o.tokens[s] = n;
   }), o.pairs.delete(u(e.a, e.b));
 }
-class w extends g {
+class S extends g {
   targetSize;
   vocab = /* @__PURE__ */ new Set();
   vocabIndex = /* @__PURE__ */ new Map();
@@ -93,7 +93,7 @@ class w extends g {
     let f = 0;
     for (const i of this.vocab.keys())
       this.vocabIndex.set(i, f++);
-    return this.vocab.size;
+    return this.emit("trainStatus", "trained"), this.vocab.size;
   }
   getVocab() {
     return Array.from(this.vocab);
@@ -126,5 +126,5 @@ class w extends g {
   }
 }
 export {
-  w as default
+  S as default
 };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.3.0",
+    "version": "0.3.1",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",