@genai-fi/nanogpt 0.17.4 → 0.18.0

package/dist/{tfjs_backend-BGnG-ppu.js → tfjs_backend-Bk3PmK91.js}

@@ -1,26 +1,26 @@
-import { o as g, q as w, x as A, ag as Te, i as ke, j as M, m as Q, E as J, B as ae, U as ue, _ as le, a2 as fe, bb as he, aD as pe, bc as Ie, t as S, L as $e, a_ as Ee } from "./index-CUXkjxiT.js";
-import { t as be } from "./tensor1d-LMGMIUlr.js";
-import { r as Le } from "./random_normal-IBRrha8a.js";
-import { s as P } from "./slice-BzS11Qh0.js";
-import { r as c } from "./reshape-BDOuCSNW.js";
-import { g as Ne } from "./gather-qIqEqaGn.js";
-import { e as Pe } from "./step-BTxPtq1r.js";
-import { c as Ce } from "./clip_by_value-DDA7rrcT.js";
-import { t as Fe } from "./tile-CFy-xTO6.js";
-import { s as ve, b as Me, c as je, g as Ue } from "./selu_util-BLhIqRkw.js";
-import { m as $ } from "./mat_mul-DhG0Newp.js";
-import { t as Ve } from "./transpose-9kRxIXWR.js";
-import { c as j } from "./concat-CAQpCret.js";
-import { g as xe, r as Be } from "./dropout_util--NxWuYg2.js";
-import { f as Ge } from "./floor-BQtb-Azg.js";
-function qe(e) {
+import { o as g, n as w, v as A, ag as Te, d as ke, h as M, m as Q, E as q, z as ae, N as ue, _ as le, a2 as fe, bb as he, aD as pe, bc as Ie, t as S, J as $e, a_ as Ee } from "./index-DSGwv2Yx.js";
+import { t as be } from "./tensor1d-BMl0eZYV.js";
+import { r as Le } from "./random_normal-CIm8lk2-.js";
+import { s as P } from "./slice-BvItlgXu.js";
+import { r as c } from "./reshape-BIN71H3p.js";
+import { g as Ne } from "./gather-DEyjXNb1.js";
+import { e as Pe } from "./step-DQY6_ABw.js";
+import { c as ve } from "./clip_by_value-CLAD4h_I.js";
+import { t as Ce } from "./tile-CsWlVKKz.js";
+import { s as Fe, b as Me, c as je, g as Ve } from "./selu_util-BXdhy_W6.js";
+import { m as $ } from "./mat_mul-DP86qZtZ.js";
+import { t as Ue } from "./transpose-Qxz-4os3.js";
+import { c as j } from "./concat-Dqk7Xk7h.js";
+import { g as xe, r as Be } from "./dropout_util-N0z8Os-K.js";
+import { f as Ge } from "./floor-CymuCmTO.js";
+function Je(e) {
   return j(
     e,
     0
     /* axis */
   );
 }
-const Je = /* @__PURE__ */ g({ concat1d_: qe });
+const qe = /* @__PURE__ */ g({ concat1d_: Je });
 function Ke(e, n) {
   return j(e, n);
 }
@@ -52,7 +52,7 @@ function en(e, n, t) {
   const s = w(e, "x", "slice4d");
   return A(s.rank === 4, () => `slice4d expects a rank-4 tensor, but got a rank-${s.rank} tensor`), P(s, n, t);
 }
-const V = /* @__PURE__ */ g({ slice4d_: en });
+const U = /* @__PURE__ */ g({ slice4d_: en });
 function nn(e, n, t, s) {
   const r = w(e, "x", "dropout");
   if (A(r.dtype === "float32", () => `x has to be a floating point tensor since it's going to be scaled, but got a ${r.dtype} tensor instead.`), A(n >= 0 && n < 1, () => `rate must be a float in the range [0, 1), but got ${n}.`), n === 0)
@@ -62,7 +62,7 @@ function nn(e, n, t, s) {
 }
 const tn = /* @__PURE__ */ g({ dropout_: nn });
 function sn({ a: e, b: n, transposeA: t = !1, transposeB: s = !1, bias: r, activation: o = "linear", preluActivationWeights: i, leakyreluAlpha: f = 0.2 }) {
-  if (ve(J.state.gradientDepth, o) === !1) {
+  if (Fe(q.state.gradientDepth, o) === !1) {
     let _ = $(e, n, t, s);
     return r != null && (_ = M(_, r)), Me(_, o, i, f);
   }
@@ -70,38 +70,38 @@ function sn({ a: e, b: n, transposeA: t = !1, transposeB: s = !1, bias: r, activ
   [a, u] = ae(a, u);
   const m = t ? a.shape[a.rank - 2] : a.shape[a.rank - 1], d = s ? u.shape[u.rank - 1] : u.shape[u.rank - 2], T = t ? a.shape[a.rank - 1] : a.shape[a.rank - 2], h = s ? u.shape[u.rank - 2] : u.shape[u.rank - 1], ne = a.shape.slice(0, -2), I = u.shape.slice(0, -2), te = ue(ne), se = ue(I);
   A(m === d, () => `Error in fused matMul: inner shapes (${m}) and (${d}) of Tensors with shapes ${a.shape} and ${u.shape} and transposeA=${t} and transposeB=${s} must match.`);
-  const B = le(a.shape.slice(0, -2), u.shape.slice(0, -2)).concat([T, h]), G = t ? c(a, [te, m, T]) : c(a, [te, T, m]), q = s ? c(u, [se, h, d]) : c(u, [se, d, h]);
+  const B = le(a.shape.slice(0, -2), u.shape.slice(0, -2)).concat([T, h]), G = t ? c(a, [te, m, T]) : c(a, [te, T, m]), J = s ? c(u, [se, h, d]) : c(u, [se, d, h]);
   let E;
   r != null && (E = w(r, "bias", "fused matMul"), [E] = ae(E, a), le(B, E.shape));
   let re;
   i != null && (re = w(i, "prelu weights", "fused matMul"));
-  const oe = (_, C) => {
-    const [O, D, y, U] = C, k = je(c(_, y.shape), y, o);
+  const oe = (_, v) => {
+    const [O, D, y, V] = v, k = je(c(_, y.shape), y, o);
     let b, L;
     if (!t && !s ? (b = $(k, D, !1, !0), L = $(O, k, !0, !1)) : !t && s ? (b = $(k, D, !1, !1), L = $(k, O, !0, !1)) : t && !s ? (b = $(D, k, !1, !0), L = $(O, k, !1, !1)) : (b = $(D, k, !0, !0), L = $(k, O, !0, !0)), r != null) {
-      const ye = Ue(U, k);
+      const ye = Ve(V, k);
       return [b, L, ye];
     } else
       return [b, L];
   }, ie = {
     a: G,
-    b: q,
+    b: J,
     bias: E,
     preluActivationWeights: re
   }, ce = { transposeA: t, transposeB: s, activation: o, leakyreluAlpha: f };
-  return r == null ? fe((C, O, D) => {
+  return r == null ? fe((v, O, D) => {
     const y = (
       // tslint:disable-next-line: no-unnecessary-type-assertion
-      J.runKernel(he, ie, ce)
+      q.runKernel(he, ie, ce)
     );
-    return D([C, O, y]), { value: c(y, B), gradFunc: oe };
-  })(G, q) : fe((C, O, D, y) => {
-    const U = (
+    return D([v, O, y]), { value: c(y, B), gradFunc: oe };
+  })(G, J) : fe((v, O, D, y) => {
+    const V = (
       // tslint:disable-next-line: no-unnecessary-type-assertion
-      J.runKernel(he, ie, ce)
+      q.runKernel(he, ie, ce)
     );
-    return y([C, O, U, D]), { value: c(U, B), gradFunc: oe };
-  })(G, q, E);
+    return y([v, O, V, D]), { value: c(V, B), gradFunc: oe };
+  })(G, J, E);
 }
 const de = /* @__PURE__ */ g({ fusedMatMul_: sn });
 class Ae extends Error {
@@ -119,9 +119,9 @@ class l extends Error {
     super(n), Object.setPrototypeOf(this, l.prototype);
   }
 }
-class v extends Error {
+class F extends Error {
   constructor(n) {
-    super(n), Object.setPrototypeOf(this, v.prototype);
+    super(n), Object.setPrototypeOf(this, F.prototype);
   }
 }
 class ee extends Error {
@@ -144,16 +144,16 @@ function me(e, n) {
   if (!e)
     throw new ee(n);
 }
-function Cn(e, n) {
+function vn(e, n) {
   let t = 0;
   for (const s of e)
     s === n && t++;
   return t;
 }
-function Fn(e) {
+function Cn(e) {
   return e.length === 1 ? e[0] : e;
 }
-function vn(e) {
+function Fn(e) {
   return Array.isArray(e) ? e : [e];
 }
 function Mn(e) {
@@ -164,7 +164,7 @@ function jn(e) {
   return e.length <= 1 || e.indexOf("_") === -1 ? e : e.replace(/[_]+(\w|$)/g, (n, t) => t.toUpperCase());
 }
 let p = {};
-function Un(e) {
+function Vn(e) {
   if (e == null)
     return null;
   const n = {};
@@ -182,7 +182,7 @@ function W(e) {
       }
     }
 }
-function Vn(e, n = {}, t = {}, s = "object", r = !1) {
+function Un(e, n = {}, t = {}, s = "object", r = !1) {
   if (typeof e == "string") {
     const o = e;
     let i;
@@ -255,7 +255,7 @@ function x(e, n, t) {
   if (t != null && e.indexOf(t) < 0)
     throw new l(`${t} is not a valid ${n}.  Valid values are ${e} or null/undefined.`);
 }
-function qn(e, n, t = 0, s = 1 / 0) {
+function Jn(e, n, t = 0, s = 1 / 0) {
   return me(t >= 0), me(s >= t), Array.isArray(e) && e.length >= t && e.length <= s && e.every((r) => typeof r === n);
 }
 function on(e, n) {
@@ -264,7 +264,7 @@ function on(e, n) {
 function Oe(e) {
   return e === null ? "null" : Array.isArray(e) ? "[" + e.map((n) => Oe(n)).join(",") + "]" : typeof e == "string" ? `"${e}"` : `${e}`;
 }
-function Jn(e, n, t) {
+function qn(e, n, t) {
   let s = t != null ? t() : pe(), r;
   return (...i) => {
     const f = t != null ? t() : pe();
@@ -288,18 +288,18 @@ function Wn(e) {
 function Yn(e) {
   x(ln, "PoolMode", e);
 }
-const F = [], ge = "/";
+const C = [], ge = "/";
 function Hn(e, n) {
-  F.push(e);
+  C.push(e);
   try {
     const t = n();
-    return F.pop(), t;
+    return C.pop(), t;
   } catch (t) {
-    throw F.pop(), t;
+    throw C.pop(), t;
   }
 }
 function hn() {
-  return F.length === 0 ? "" : F.join(ge) + ge;
+  return C.length === 0 ? "" : C.join(ge) + ge;
 }
 function Qn(e) {
   if (!De(e))
@@ -401,7 +401,7 @@ function R(e, n, t) {
       case 3:
         return z(e, [n, 0, 0], [t, e.shape[1], e.shape[2]]);
       case 4:
-        return V(e, [n, 0, 0, 0], [t, e.shape[1], e.shape[2], e.shape[3]]);
+        return U(e, [n, 0, 0, 0], [t, e.shape[1], e.shape[2], e.shape[3]]);
       case 5:
         return P(e, [n, 0, 0, 0, 0], [
           t,
@@ -434,7 +434,7 @@ function Z(e, n, t) {
       case 3:
         return z(e, [0, 0, n], [e.shape[0], e.shape[1], t]);
       case 4:
-        return V(e, [0, 0, 0, n], [e.shape[0], e.shape[1], e.shape[2], t]);
+        return U(e, [0, 0, 0, n], [e.shape[0], e.shape[1], e.shape[2], t]);
       default:
         throw new l(`sliceAlongLastAxis() received an unsupported tensor rank: ${e.rank}`);
     }
@@ -470,9 +470,9 @@ function at(e, n, t, s) {
           case 1:
             return R(e, n, t);
           case 2:
-            return V(e, [0, n, 0, 0], [e.shape[0], t, e.shape[2], e.shape[3]]);
+            return U(e, [0, n, 0, 0], [e.shape[0], t, e.shape[2], e.shape[3]]);
           case 3:
-            return V(e, [0, 0, n, 0], [e.shape[0], e.shape[1], t, e.shape[3]]);
+            return U(e, [0, 0, n, 0], [e.shape[0], e.shape[1], t, e.shape[3]]);
           case 4:
             return Z(e, n, t);
           default:
@@ -490,7 +490,7 @@ function ut(e, n = -1) {
 function lt(e, n) {
   switch (e.rank) {
     case 1:
-      return Je([e, n]);
+      return qe([e, n]);
     case 2:
       return Re([e, n], 0);
     case 3:
@@ -504,18 +504,18 @@ function lt(e, n) {
 function mn(e, n) {
   if (Array.isArray(n) || (n = [n]), e.rank !== n.length)
     throw new l(`The length of input n (${n.length}) does not match the number of dimensions in input x (${e.rank})`);
-  return Fe(e, n);
+  return Ce(e, n);
 }
 function ft(e, n = 0, t = 1, s, r) {
   return Le(e, n, t, s, r);
 }
 function ht(e, n, t, s) {
   if (e.rank < 2 || n.rank < 2)
-    throw new v(`dot requires both inputs to be rank >= 2 but got x shape = ${e.shape} and y shape = ${n.shape}`);
+    throw new F(`dot requires both inputs to be rank >= 2 but got x shape = ${e.shape} and y shape = ${n.shape}`);
   if (n.rank >= 3) {
     const r = e.shape.slice(-1)[0], o = n.shape.slice(-2)[0];
     if (r !== o)
-      throw new v(`If rank y >= 3, then the second last dim of y must equal the last dim of x but got x shape = ${e.shape} and  y shape = ${n.shape}`);
+      throw new F(`If rank y >= 3, then the second last dim of y must equal the last dim of x but got x shape = ${e.shape} and  y shape = ${n.shape}`);
   }
   if (e.rank === 2 && n.rank === 2)
     return de({
@@ -530,7 +530,7 @@ function ht(e, n, t, s) {
     const r = e.shape.slice(), o = r.pop();
     e = c(e, [-1, o]);
     const i = n.shape.slice(), f = i.pop(), a = i.pop(), u = [...i, f], m = Array.from({ length: n.rank }, (ne, I) => I === 0 ? n.rank - 2 : I <= n.rank - 2 ? I - 1 : I);
-    n = c(Ve(n, m), [a, -1]);
+    n = c(Ue(n, m), [a, -1]);
     const d = [...r, ...u];
     return c(de({
       a: e,
@@ -576,7 +576,7 @@ function mt(e, n, t) {
 }
 function gt(e, n = 1) {
   if (n !== 1)
-    throw new v(`Support for alpha values other than 1 (${n}) is not implemented yet.`);
+    throw new F(`Support for alpha values other than 1 (${n}) is not implemented yet.`);
   return Pe(e);
 }
 function kt(e) {
@@ -588,7 +588,7 @@ function $t(e, n, t, s) {
 function wt(e) {
   return S(() => {
     const n = M(0.5, Q(0.2, e));
-    return Ce(n, 0, 1);
+    return ve(n, 0, 1);
   });
 }
 function At(e, n, t = !1) {
@@ -599,17 +599,17 @@ export {
   Ae as A,
   pt as B,
   tt as C,
-  Cn as D,
+  vn as D,
   gt as E,
   wt as F,
   kt as G,
   nt as H,
   zn as I,
-  qn as J,
+  Jn as J,
   mt as K,
   at as L,
   Zn as M,
-  v as N,
+  F as N,
   on as O,
   Kn as P,
   Wn as Q,
@@ -629,13 +629,13 @@ export {
   _e as b,
   x as c,
   ht as d,
-  Vn as e,
+  Un as e,
   Xn as f,
   Qn as g,
-  Fn as h,
-  vn as i,
+  Cn as h,
+  Fn as i,
   st as j,
-  Jn as k,
+  qn as k,
   it as l,
   dt as m,
   Hn as n,
@@ -643,7 +643,7 @@ export {
   rt as p,
   jn as q,
   ft as r,
-  Un as s,
+  Vn as s,
   Mn as t,
   Bn as u,
   xn as v,

package/dist/{tile-CFy-xTO6.js → tile-CsWlVKKz.js}

@@ -1,4 +1,4 @@
-import { o as e, q as a, x as i, E as c, T as l } from "./index-CUXkjxiT.js";
+import { o as e, n as a, v as i, E as c, T as l } from "./index-DSGwv2Yx.js";
 function u(r, t) {
   const n = a(r, "x", "tile", "string_or_numeric");
   i(n.rank === t.length, () => `Error in transpose: rank of input ${n.rank} must match length of reps ${t}.`);

package/dist/tokeniser/BaseTokeniser.d.ts

@@ -2,6 +2,8 @@ import { Conversation, ITokeniser } from './type';
 import { default as EE } from 'eventemitter3';
 export declare const SPECIALS: string[];
 export default abstract class BaseTokeniser extends EE<'trainStatus'> implements ITokeniser {
+    id: string;
+    datasetID?: string;
     protected specialTokens: Map<string, number>;
     protected specialTokenSet: Set<number>;
     abstract vocabSize: number;
@@ -12,7 +14,8 @@ export default abstract class BaseTokeniser extends EE<'trainStatus'> implements
     isSpecialToken(index: number): boolean;
     protected addSpecialTokens(): void;
     protected addSpecialToken(token: string, index: number): void;
-    abstract train(text: Conversation[][], cb?: (vocab: number) => void): Promise<number>;
+    protected generateID(): void;
+    abstract train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
     abstract getVocab(): string[];
     abstract getMerges(): [string, string][];
     abstract destroy(): void;

package/dist/tokeniser/BaseTokeniser.js

@@ -1,5 +1,5 @@
 import { E as r } from "../index-DvYrXKkX.js";
-const h = [
+const l = [
   "<eos>",
   "<bos>",
   "",
@@ -11,20 +11,36 @@ const h = [
   "<|system_start|>",
   "<|system_end|>"
 ];
-class k extends r {
+class T extends r {
+  id = "untrained";
+  datasetID;
   specialTokens = /* @__PURE__ */ new Map();
   specialTokenSet = /* @__PURE__ */ new Set();
   isSpecialToken(s) {
     return this.specialTokenSet.has(s);
   }
   addSpecialTokens() {
-    h.forEach((s, t) => {
+    l.forEach((s, t) => {
       this.addToken(s, t), this.specialTokens.set(s, t), this.specialTokenSet.add(t);
     });
   }
   addSpecialToken(s, t) {
     this.specialTokens.set(s, t), this.specialTokenSet.add(t);
   }
+  generateID() {
+    const s = this.getVocab();
+    let t = 2166136261, e = 2654435769;
+    for (let a = 0; a < s.length; a++) {
+      const i = s[a];
+      t ^= i.length, t = Math.imul(t, 16777619), e ^= a, e = Math.imul(e, 2246822507);
+      for (let c = 0; c < i.length; c++) {
+        const h = i.charCodeAt(c);
+        t ^= h, t = Math.imul(t, 16777619), e ^= h, e = Math.imul(e, 3266489909);
+      }
+    }
+    const o = (t >>> 0).toString(36), n = (e >>> 0).toString(36);
+    this.id = "tokeniser_" + o + "_" + n;
+  }
   encodeSequence(s) {
     const t = this.encode(s);
     return [this.bosToken, ...t, this.eosToken];
@@ -94,6 +110,6 @@ class k extends r {
   }
 }
 export {
-  h as SPECIALS,
-  k as default
+  l as SPECIALS,
+  T as default
 };

package/dist/tokeniser/CharTokeniser.d.ts

@@ -12,7 +12,7 @@ export default class CharTokeniser extends BaseTokeniser {
     addToken(token: string, index?: number): number;
     get trained(): boolean;
     destroy(): void;
-    train(text: Conversation[][]): Promise<number>;
+    train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
     tokenise(text: string[], numeric: true): number[][];
     tokenise(text: string[]): string[][];
     detokenise(tokens: (number[] | Uint16Array)[]): string[];

package/dist/tokeniser/CharTokeniser.js

@@ -1,6 +1,7 @@
-import k, { SPECIALS as d } from "./BaseTokeniser.js";
-const u = ["<eos>", "<unk>"];
-class T extends k {
+import { yieldIfNeeded as u } from "../utilities/yielder.js";
+import b, { SPECIALS as T } from "./BaseTokeniser.js";
+const l = ["<eos>", "<unk>"];
+class x extends b {
   vocabSize = 0;
   eosToken = 0;
   bosToken = 0;
@@ -8,30 +9,30 @@ class T extends k {
   vocab = [];
   cache = /* @__PURE__ */ new Map();
   _trained = !1;
-  constructor(i) {
-    if (super(), Array.isArray(i)) {
-      if (this.vocab = i, this.vocab.length > 0)
-        this.vocabSize = this.vocab.length, d.forEach((t) => {
-          const e = this.vocab.indexOf(t);
-          e !== -1 && this.addSpecialToken(t, e);
-        }), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex("") ?? -1, this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<unk>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<pad>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("_")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf(" ")), this.unkToken === -1 && (this.unkToken = this.eosToken), this.vocab = this.vocab.map((t) => t === "<pad>" ? "" : t), this.vocab.forEach((t, e) => {
-          this.cache.set(t, e);
+  constructor(t) {
+    if (super(), Array.isArray(t)) {
+      if (this.vocab = t, this.vocab.length > 0)
+        this.vocabSize = this.vocab.length, T.forEach((i) => {
+          const e = this.vocab.indexOf(i);
+          e !== -1 && this.addSpecialToken(i, e);
+        }), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex("") ?? -1, this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<unk>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<pad>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("_")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf(" ")), this.unkToken === -1 && (this.unkToken = this.eosToken), this.vocab = this.vocab.map((i) => i === "<pad>" ? "" : i), this.vocab.forEach((i, e) => {
+          this.cache.set(i, e);
         });
       else
         throw new Error("Vocab cannot be empty");
       this._trained = !0;
     } else
-      this.vocabSize = i, this.vocab = new Array(this.vocabSize).fill(""), this.addSpecialTokens(), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex(""), this.vocab.forEach((t, e) => {
-        this.cache.set(t, e);
+      this.vocabSize = t, this.vocab = new Array(this.vocabSize).fill(""), this.addSpecialTokens(), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex(""), this.vocab.forEach((i, e) => {
+        this.cache.set(i, e);
       }), this.cache.set("", this.unkToken);
   }
-  addToken(i, t) {
-    if (this.cache.has(i))
-      return this.cache.get(i);
+  addToken(t, i) {
+    if (this.cache.has(t))
+      return this.cache.get(t);
     let e;
-    if (t !== void 0 ? e = t : (e = this.vocab.indexOf("", this.unkToken + 1), e === -1 && (e = this.vocabSize)), e >= this.vocabSize)
+    if (i !== void 0 ? e = i : (e = this.vocab.indexOf("", this.unkToken + 1), e === -1 && (e = this.vocabSize)), e >= this.vocabSize)
       throw new Error("Vocab size exceeded");
-    return this.vocab[e] = i, this.cache.set(i, e), e;
+    return this.vocab[e] = t, this.cache.set(t, e), e;
   }
   get trained() {
     return this.vocab.length === this.vocabSize && this._trained;
@@ -39,43 +40,54 @@ class T extends k {
   destroy() {
     this.cache.clear(), this.vocab = [];
   }
-  async train(i) {
-    const t = i.map((o) => o.map((n) => n.content.split(""))).flat(2), e = new Set(t), s = Array.from(e), h = this.vocab.indexOf("", this.unkToken + 1), a = this.vocabSize - u.length;
-    if (h === -1)
-      return this.vocabSize;
-    if (this._trained = !0, s.length > a) {
-      const o = /* @__PURE__ */ new Map();
-      t.forEach((n) => {
-        o.set(n, (o.get(n) || 0) + 1);
-      }), s.sort((n, r) => (o.get(n) || 0) - (o.get(r) || 0)), s.splice(0, s.length - a);
+  async train(t, i, e) {
+    this.datasetID = e;
+    const a = /* @__PURE__ */ new Set();
+    let h = performance.now();
+    for (const n of t)
+      n.forEach((o) => {
+        for (const r of o.content)
+          a.add(r);
+      }), h = await u(h, i, 0);
+    const s = Array.from(a), k = this.vocab.indexOf("", this.unkToken + 1), d = this.vocabSize - l.length;
+    if (k === -1)
+      return this.generateID(), this.vocabSize;
+    if (this._trained = !0, s.length > d) {
+      const n = /* @__PURE__ */ new Map();
+      t.forEach((o) => {
+        o.forEach((r) => {
+          for (const f of r.content)
+            n.set(f, (n.get(f) || 0) + 1);
+        });
+      }), s.sort((o, r) => (n.get(o) || 0) - (n.get(r) || 0)), s.splice(0, s.length - d);
     }
-    let c = h;
+    let c = k;
     if (c !== -1) {
-      const o = new Set(this.vocab);
-      for (const n of s)
-        if (!o.has(n) && (this.vocab[c] = n, o.add(n), c = this.vocab.indexOf("", c + 1), c === -1))
+      const n = new Set(this.vocab);
+      for (const o of s)
+        if (!n.has(o) && (this.vocab[c] = o, n.add(o), c = this.vocab.indexOf("", c + 1), c === -1))
           break;
     }
-    return this.cache.clear(), this.vocab.forEach((o, n) => {
-      this.cache.set(o, n);
-    }), this.emit("trainStatus", "trained"), this.vocabSize;
+    return this.cache.clear(), this.vocab.forEach((n, o) => {
+      this.cache.set(n, o);
+    }), this.generateID(), this.emit("trainStatus", "trained"), this.vocabSize;
   }
-  tokenise(i, t) {
+  tokenise(t, i) {
     if (!this.trained)
       throw new Error("Tokeniser not trained");
-    return i.map((s) => t ? s.split("").map((h) => this.cache.get(h) ?? this.unkToken) : s.split("").map((h) => {
-      const a = this.cache.get(h);
-      return a !== void 0 ? this.vocab[a] : "";
+    return t.map((a) => i ? a.split("").map((h) => this.cache.get(h) ?? this.unkToken) : a.split("").map((h) => {
+      const s = this.cache.get(h);
+      return s !== void 0 ? this.vocab[s] : "";
     }));
   }
-  detokenise(i) {
-    return i.map((e) => Array.from(e).map((s) => this.vocab[s] || "").join(""));
+  detokenise(t) {
+    return t.map((e) => Array.from(e).map((a) => this.vocab[a] || "").join(""));
   }
-  encode(i) {
-    return this.tokenise([i], !0)[0];
+  encode(t) {
+    return this.tokenise([t], !0)[0];
   }
-  decode(i) {
-    return this.detokenise([i])[0];
+  decode(t) {
+    return this.detokenise([t])[0];
   }
   getVocab() {
     return this.vocab;
@@ -83,13 +95,13 @@ class T extends k {
   getMerges() {
     return [];
   }
-  async createTrainingData(i, t = 5) {
-    const e = await this.tokenise(i, !0), s = [], h = [];
-    for (let a = 0; a < e.length - t; a++)
-      s.push(...e[a].slice(0, t)), h.push(e[a + 1][0]);
-    return [s, h];
+  async createTrainingData(t, i = 5) {
+    const e = await this.tokenise(t, !0), a = [], h = [];
+    for (let s = 0; s < e.length - i; s++)
+      a.push(...e[s].slice(0, i)), h.push(e[s + 1][0]);
+    return [a, h];
   }
 }
 export {
-  T as default
+  x as default
 };

package/dist/tokeniser/bpe.d.ts

@@ -15,7 +15,7 @@ export default class BPETokeniser extends BaseTokeniser {
     get eosToken(): number;
     get bosToken(): number;
     get unkToken(): number;
-    train(text?: Conversation[][], cb?: (vocab: number) => void): Promise<number>;
+    train(text?: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
     getVocab(): string[];
     getMerges(): [string, string][];
     private tokeniseWord;