@genai-fi/nanogpt 0.10.3 → 0.12.0

package/dist/{step-CINUs5QB.js → step-BS5JXRR6.js}

@@ -1,7 +1,7 @@
-import { l as _, aG as P, A as k, B as M, E as v, aH as F, aI as J, aJ as X, aK as q, aL as Q, aM as Y } from "./index-DOvlwCh-.js";
+import { x as _, aG as P, o as A, q as M, E as v, aH as q, aI as F, aJ as J, aK as X, aL as Q, aM as Y } from "./index-Duu1Lvvv.js";
 function it(t, n, e, o, s = "NHWC", f) {
   const l = t[3], r = [...n, l], c = nt(s);
-  return B(t, r, e, f, o, null, null, c);
+  return j(t, r, e, f, o, null, null, c);
 }
 function ft(t, n, e, o, s, f, l = "channelsLast") {
   const [r, c] = T(n);
@@ -12,7 +12,7 @@ function ft(t, n, e, o, s, f, l = "channelsLast") {
     u = [r, c, t[1], t[1]];
   else
     throw new Error(`Unknown dataFormat ${l}`);
-  return B(t, u, e, o, s, f, !1, l);
+  return j(t, u, e, o, s, f, !1, l);
 }
 function ht(t, n, e, o, s, f, l = "NDHWC") {
   const [r, c, u] = K(n);
@@ -25,7 +25,7 @@ function ht(t, n, e, o, s, f, l = "NDHWC") {
     throw new Error(`Unknown dataFormat ${l}`);
   return Z(t, h, e, o, s, !1, i, f);
 }
-function B(t, n, e, o, s, f, l = !1, r = "channelsLast") {
+function j(t, n, e, o, s, f, l = !1, r = "channelsLast") {
   let [c, u, h, i] = [-1, -1, -1, -1];
   if (r === "channelsLast")
     [c, u, h, i] = t;
@@ -33,7 +33,7 @@ function B(t, n, e, o, s, f, l = !1, r = "channelsLast") {
     [c, i, u, h] = t;
   else
     throw new Error(`Unknown dataFormat ${r}`);
-  const [a, p, , $] = n, [w, g] = T(e), [x, y] = T(o), L = A(a, x), b = A(p, y), { padInfo: C, outHeight: E, outWidth: D } = d(s, u, h, w, g, L, b, f, r), I = l ? $ * i : $;
+  const [a, p, , $] = n, [w, g] = T(e), [x, y] = T(o), L = k(a, x), b = k(p, y), { padInfo: C, outHeight: E, outWidth: D } = d(s, u, h, w, g, L, b, f, r), I = l ? $ * i : $;
   let m;
   return r === "channelsFirst" ? m = [c, I, E, D] : r === "channelsLast" && (m = [c, E, D, I]), {
     batchSize: c,
@@ -66,7 +66,7 @@ function Z(t, n, e, o, s, f = !1, l = "channelsLast", r) {
     [c, a, u, h, i] = t;
   else
     throw new Error(`Unknown dataFormat ${l}`);
-  const [p, $, w, , g] = n, [x, y, L] = K(e), [b, C, E] = K(o), D = A(p, b), I = A($, C), m = A(w, E), { padInfo: G, outDepth: N, outHeight: R, outWidth: W } = tt(s, u, h, i, x, y, L, D, I, m, r), H = f ? g * a : g;
+  const [p, $, w, , g] = n, [x, y, L] = K(e), [b, C, E] = K(o), D = k(p, b), I = k($, C), m = k(w, E), { padInfo: G, outDepth: N, outHeight: R, outWidth: W } = tt(s, u, h, i, x, y, L, D, I, m, r), H = f ? g * a : g;
   let O;
   return l === "channelsFirst" ? O = [c, H, N, R, W] : l === "channelsLast" && (O = [c, N, R, W, H]), {
     batchSize: c,
@@ -98,19 +98,19 @@ function Z(t, n, e, o, s, f = !1, l = "channelsLast", r) {
   };
 }
 function z(t, n, e, o, s) {
-  o == null && (o = j(t, n, e));
+  o == null && (o = B(t, n, e));
   const f = t[0], l = t[1], r = U((f - n + 2 * o) / e + 1, s), c = U((l - n + 2 * o) / e + 1, s);
   return [r, c];
 }
 function S(t, n, e, o, s, f) {
-  s == null && (s = j(t, n[0], o[0]));
+  s == null && (s = B(t, n[0], o[0]));
   const l = [0, 0, 0, e];
   for (let r = 0; r < 3; r++)
     t[r] + 2 * s >= n[r] && (l[r] = U((t[r] - n[r] + 2 * s) / o[r] + 1, f));
   return l;
 }
-function j(t, n, e, o = 1) {
-  const s = A(n, o);
+function B(t, n, e, o = 1) {
+  const s = k(n, o);
   return Math.floor((t[0] * (e - 1) - e + s) / 2);
 }
 function T(t) {
@@ -119,7 +119,7 @@ function T(t) {
 function K(t) {
   return typeof t == "number" ? [t, t, t] : t;
 }
-function A(t, n) {
+function k(t, n) {
   return n <= 1 ? t : t + (t - 1) * (n - 1);
 }
 function d(t, n, e, o, s, f, l, r, c) {
@@ -212,41 +212,41 @@ function $t(t, n, e) {
 }
 function et(t) {
   const e = { x: M(t, "x", "sigmoid", "float32") };
-  return v.runKernel(F, e);
+  return v.runKernel(q, e);
 }
-const gt = /* @__PURE__ */ k({ sigmoid_: et });
+const gt = /* @__PURE__ */ A({ sigmoid_: et });
 function ot(t) {
   const e = { x: M(t, "x", "elu", "float32") };
-  return v.runKernel(J, e);
+  return v.runKernel(F, e);
 }
-const wt = /* @__PURE__ */ k({ elu_: ot });
+const wt = /* @__PURE__ */ A({ elu_: ot });
 function st(t, n = 0.2) {
   const o = { x: M(t, "x", "leakyRelu") }, s = { alpha: n };
-  return v.runKernel(X, o, s);
+  return v.runKernel(J, o, s);
 }
-const xt = /* @__PURE__ */ k({ leakyRelu_: st });
+const xt = /* @__PURE__ */ A({ leakyRelu_: st });
 function rt(t, n) {
   const e = M(t, "x", "prelu"), o = M(n, "alpha", "prelu"), s = { x: e, alpha: o };
-  return v.runKernel(q, s);
+  return v.runKernel(X, s);
 }
-const Et = /* @__PURE__ */ k({ prelu_: rt });
+const Et = /* @__PURE__ */ A({ prelu_: rt });
 function lt(t) {
   const e = { x: M(t, "x", "relu6") };
   return v.runKernel(Q, e);
 }
-const yt = /* @__PURE__ */ k({ relu6_: lt });
+const yt = /* @__PURE__ */ A({ relu6_: lt });
 function ct(t, n = 0) {
   const o = { x: M(t, "x", "step") }, s = { alpha: n };
   return v.runKernel(Y, o, s);
 }
-const bt = /* @__PURE__ */ k({ step_: ct });
+const bt = /* @__PURE__ */ A({ step_: ct });
 export {
   bt as a,
-  B as b,
+  j as b,
   $t as c,
   Z as d,
   wt as e,
-  j as f,
+  B as f,
   it as g,
   ft as h,
   ht as i,

package/dist/{sum-DWAtNGez.js → sum-BPUfDB2X.js}

@@ -1,8 +1,8 @@
-import { A as e, B as u, X as c, E as l, Y as m } from "./index-DOvlwCh-.js";
-function i(t, n = null, o = !1) {
+import { o as e, q as u, L as c, E as l, N as m } from "./index-Duu1Lvvv.js";
+function i(t, o = null, n = !1) {
   let s = u(t, "x", "sum");
   s.dtype === "bool" && (s = c(s, "int32"));
-  const r = { x: s }, a = { axis: n, keepDims: o };
+  const r = { x: s }, a = { axis: o, keepDims: n };
   return l.runKernel(m, r, a);
 }
 const f = /* @__PURE__ */ e({ sum_: i });

package/dist/tensor-CEt9Nm2s.js

@@ -0,0 +1,8 @@
+import { X as t, Y as a } from "./index-Duu1Lvvv.js";
+function f(r, n, e) {
+  const o = t(r, e);
+  return a(r, n, o, e);
+}
+export {
+  f as t
+};

package/dist/tensor1d-Cc_KCIDg.js

@@ -0,0 +1,11 @@
+import { W as o, X as s, Y as t } from "./index-Duu1Lvvv.js";
+function h(r, e) {
+  o(r);
+  const n = s(r, e);
+  if (n.length !== 1)
+    throw new Error("tensor1d() requires values to be a flat/TypedArray");
+  return t(r, null, n, e);
+}
+export {
+  h as t
+};

package/dist/{tensor2d-Bs9wZRc7.js → tensor2d-BN97fF71.js}

@@ -1,13 +1,13 @@
-import { $ as t, a0 as a, a1 as s } from "./index-DOvlwCh-.js";
+import { W as t, X as s, Y as a } from "./index-Duu1Lvvv.js";
 function i(n, r, o) {
   if (t(n), r != null && r.length !== 2)
     throw new Error("tensor2d() requires shape to have two numbers");
-  const e = a(n, o);
+  const e = s(n, o);
   if (e.length !== 2 && e.length !== 1)
     throw new Error("tensor2d() requires values to be number[][] or flat/TypedArray");
   if (e.length === 1 && r == null)
     throw new Error("tensor2d() requires shape to be provided when `values` are a flat/TypedArray");
-  return s(n, r, e, o);
+  return a(n, r, e, o);
 }
 export {
   i as t

package/dist/{tensor4d-BARPdTaS.js → tensor4d-vuDDgdUI.js}

@@ -1,4 +1,4 @@
-import { $ as t, a0 as a, a1 as s } from "./index-DOvlwCh-.js";
+import { W as t, X as a, Y as s } from "./index-Duu1Lvvv.js";
 function i(n, r, o) {
   if (t(n), r != null && r.length !== 4)
     throw new Error("tensor4d() requires shape to have four numbers");

package/dist/{tfjs_backend-y1cvNhLA.js → tfjs_backend-806hyYve.js}

@@ -1,16 +1,16 @@
-import { A as w, B as S, l as T, E as q, x as U, L as ie, h as ue, a3 as ae, a6 as le, aX as fe, aF as he, aY as Te, t as $, X as ge, m as ke, w as _e, aZ as Ie } from "./index-DOvlwCh-.js";
-import { t as Ee } from "./tensor1d-D11P_7Dp.js";
-import { r as Le, d as Ne } from "./dropout-BcvN9JYi.js";
-import { s as F } from "./slice-BfEGSH82.js";
-import { r as c } from "./reshape-ByE68wS9.js";
-import { g as be } from "./gather-D3JcZUaI.js";
-import { e as Fe } from "./step-CINUs5QB.js";
-import { c as Ce } from "./clip_by_value-BnO7-a88.js";
-import { t as Pe } from "./tile-mbfagpsB.js";
-import { s as ve, b as Me, c as je, g as Ve } from "./selu_util-kbhpTdYD.js";
-import { m as k } from "./mat_mul-SjpJRLyL.js";
-import { t as Ue } from "./transpose-ClWiBS_b.js";
-import { c as M } from "./concat-BV8bt5H-.js";
+import { o as w, q as S, x as y, E as J, l as U, B as ie, U as ue, _ as ae, a1 as le, aX as fe, aF as he, aY as ye, t as $, L as ge, m as ke, j as Te, aZ as Ie } from "./index-Duu1Lvvv.js";
+import { t as Ee } from "./tensor1d-Cc_KCIDg.js";
+import { r as Le, d as Ne } from "./dropout-DLhSMNTZ.js";
+import { s as F } from "./slice-wBNvzVyz.js";
+import { r as c } from "./reshape-BI0yzp1T.js";
+import { g as be } from "./gather-BSULDalH.js";
+import { e as Fe } from "./step-BS5JXRR6.js";
+import { c as Ce } from "./clip_by_value-fg2aKzUy.js";
+import { t as Pe } from "./tile-OWUvpIVt.js";
+import { s as ve, b as Me, c as je, g as Ve } from "./selu_util-DyW0X1WG.js";
+import { m as k } from "./mat_mul-Bn2BDpT4.js";
+import { t as Ue } from "./transpose-BUkQCJp9.js";
+import { c as M } from "./concat-CSm2rMwe.js";
 function Be(e) {
   return M(
     e,
@@ -22,76 +22,76 @@ const xe = /* @__PURE__ */ w({ concat1d_: Be });
 function Ge(e, n) {
   return M(e, n);
 }
-const Je = /* @__PURE__ */ w({ concat2d_: Ge });
-function qe(e, n) {
+const qe = /* @__PURE__ */ w({ concat2d_: Ge });
+function Je(e, n) {
   return M(e, n);
 }
-const Ke = /* @__PURE__ */ w({ concat3d_: qe });
+const Ke = /* @__PURE__ */ w({ concat3d_: Je });
 function Ze(e, n) {
   return M(e, n);
 }
 const Re = /* @__PURE__ */ w({ concat4d_: Ze });
 function Ye(e, n, t) {
   const s = S(e, "x", "slice1d");
-  return T(s.rank === 1, () => `slice1d expects a rank-1 tensor, but got a rank-${s.rank} tensor`), F(s, [n], [t]);
+  return y(s.rank === 1, () => `slice1d expects a rank-1 tensor, but got a rank-${s.rank} tensor`), F(s, [n], [t]);
 }
 const H = /* @__PURE__ */ w({ slice1d_: Ye });
 function We(e, n, t) {
   const s = S(e, "x", "slice2d");
-  return T(s.rank === 2, () => `slice2d expects a rank-2 tensor, but got a rank-${s.rank} tensor`), F(s, n, t);
+  return y(s.rank === 2, () => `slice2d expects a rank-2 tensor, but got a rank-${s.rank} tensor`), F(s, n, t);
 }
 const we = /* @__PURE__ */ w({ slice2d_: We });
 function Xe(e, n, t) {
   const s = S(e, "x", "slice3d");
-  return T(s.rank === 3, () => `slice3d expects a rank-3 tensor, but got a rank-${s.rank} tensor`), F(s, n, t);
+  return y(s.rank === 3, () => `slice3d expects a rank-3 tensor, but got a rank-${s.rank} tensor`), F(s, n, t);
 }
 const Q = /* @__PURE__ */ w({ slice3d_: Xe });
 function He(e, n, t) {
   const s = S(e, "x", "slice4d");
-  return T(s.rank === 4, () => `slice4d expects a rank-4 tensor, but got a rank-${s.rank} tensor`), F(s, n, t);
+  return y(s.rank === 4, () => `slice4d expects a rank-4 tensor, but got a rank-${s.rank} tensor`), F(s, n, t);
 }
 const V = /* @__PURE__ */ w({ slice4d_: He });
 function Qe({ a: e, b: n, transposeA: t = !1, transposeB: s = !1, bias: r, activation: o = "linear", preluActivationWeights: a, leakyreluAlpha: f = 0.2 }) {
-  if (ve(q.state.gradientDepth, o) === !1) {
+  if (ve(J.state.gradientDepth, o) === !1) {
     let D = k(e, n, t, s);
     return r != null && (D = U(D, r)), Me(D, o, a, f);
   }
   let i = S(e, "a", "fused matMul"), u = S(n, "b", "fused matMul");
   [i, u] = ie(i, u);
-  const m = t ? i.shape[i.rank - 2] : i.shape[i.rank - 1], d = s ? u.shape[u.rank - 1] : u.shape[u.rank - 2], _ = t ? i.shape[i.rank - 1] : i.shape[i.rank - 2], h = s ? u.shape[u.rank - 2] : u.shape[u.rank - 1], ee = i.shape.slice(0, -2), I = u.shape.slice(0, -2), ne = ue(ee), te = ue(I);
-  T(m === d, () => `Error in fused matMul: inner shapes (${m}) and (${d}) of Tensors with shapes ${i.shape} and ${u.shape} and transposeA=${t} and transposeB=${s} must match.`);
-  const x = ae(i.shape.slice(0, -2), u.shape.slice(0, -2)).concat([_, h]), G = t ? c(i, [ne, m, _]) : c(i, [ne, _, m]), J = s ? c(u, [te, h, d]) : c(u, [te, d, h]);
+  const m = t ? i.shape[i.rank - 2] : i.shape[i.rank - 1], d = s ? u.shape[u.rank - 1] : u.shape[u.rank - 2], T = t ? i.shape[i.rank - 1] : i.shape[i.rank - 2], h = s ? u.shape[u.rank - 2] : u.shape[u.rank - 1], ee = i.shape.slice(0, -2), I = u.shape.slice(0, -2), ne = ue(ee), te = ue(I);
+  y(m === d, () => `Error in fused matMul: inner shapes (${m}) and (${d}) of Tensors with shapes ${i.shape} and ${u.shape} and transposeA=${t} and transposeB=${s} must match.`);
+  const x = ae(i.shape.slice(0, -2), u.shape.slice(0, -2)).concat([T, h]), G = t ? c(i, [ne, m, T]) : c(i, [ne, T, m]), q = s ? c(u, [te, h, d]) : c(u, [te, d, h]);
   let E;
   r != null && (E = S(r, "bias", "fused matMul"), [E] = ie(E, i), ae(x, E.shape));
   let se;
   a != null && (se = S(a, "prelu weights", "fused matMul"));
   const re = (D, C) => {
-    const [A, O, y, j] = C, g = je(c(D, y.shape), y, o);
+    const [A, O, _, j] = C, g = je(c(D, _.shape), _, o);
     let L, N;
     if (!t && !s ? (L = k(g, O, !1, !0), N = k(A, g, !0, !1)) : !t && s ? (L = k(g, O, !1, !1), N = k(g, A, !0, !1)) : t && !s ? (L = k(O, g, !1, !0), N = k(A, g, !1, !1)) : (L = k(O, g, !0, !0), N = k(g, A, !0, !0)), r != null) {
-      const ye = Ve(j, g);
-      return [L, N, ye];
+      const _e = Ve(j, g);
+      return [L, N, _e];
     } else
       return [L, N];
   }, oe = {
     a: G,
-    b: J,
+    b: q,
     bias: E,
     preluActivationWeights: se
   }, ce = { transposeA: t, transposeB: s, activation: o, leakyreluAlpha: f };
   return r == null ? le((C, A, O) => {
-    const y = (
+    const _ = (
       // tslint:disable-next-line: no-unnecessary-type-assertion
-      q.runKernel(fe, oe, ce)
+      J.runKernel(fe, oe, ce)
     );
-    return O([C, A, y]), { value: c(y, x), gradFunc: re };
-  })(G, J) : le((C, A, O, y) => {
+    return O([C, A, _]), { value: c(_, x), gradFunc: re };
+  })(G, q) : le((C, A, O, _) => {
     const j = (
       // tslint:disable-next-line: no-unnecessary-type-assertion
-      q.runKernel(fe, oe, ce)
+      J.runKernel(fe, oe, ce)
     );
-    return y([C, A, j, O]), { value: c(j, x), gradFunc: re };
-  })(G, J, E);
+    return _([C, A, j, O]), { value: c(j, x), gradFunc: re };
+  })(G, q, E);
 }
 const pe = /* @__PURE__ */ w({ fusedMatMul_: Qe });
 class $e extends Error {
@@ -119,7 +119,7 @@ class z extends Error {
     super(n), Object.setPrototypeOf(this, z.prototype);
   }
 }
-function Tn(e, n) {
+function yn(e, n) {
   if (Array.isArray(e)) {
     let t = [];
     for (let s = 0; s < n; s++)
@@ -134,7 +134,7 @@ function de(e, n) {
   if (!e)
     throw new z(n);
 }
-function _n(e, n) {
+function Tn(e, n) {
   let t = 0;
   for (const s of e)
     s === n && t++;
@@ -208,8 +208,8 @@ function Fn(e, n = {}, t = {}, s = "object", r = !1) {
       for (const h of Object.keys(t))
         p[h] = t[h];
       Y(o.config);
-      const _ = i(f, o.config, t, r);
-      return p = Object.assign({}, d), _;
+      const T = i(f, o.config, t, r);
+      return p = Object.assign({}, d), T;
     } else {
       const u = Object.assign({}, p);
       for (const d of Object.keys(t))
@@ -249,7 +249,7 @@ function Mn(e, n, t = 0, s = 1 / 0) {
   return de(t >= 0), de(s >= t), Array.isArray(e) && e.length >= t && e.length <= s && e.every((r) => typeof r === n);
 }
 function en(e, n) {
-  Array.isArray(e) ? (T(e.length > 0, () => `${n} is unexpectedly an empty array.`), e.forEach((t, s) => en(t, `element ${s + 1} of ${n}`))) : T(Number.isInteger(e) && e > 0, () => `Expected ${n} to be a positive integer, but got ${Oe(e)}.`);
+  Array.isArray(e) ? (y(e.length > 0, () => `${n} is unexpectedly an empty array.`), e.forEach((t, s) => en(t, `element ${s + 1} of ${n}`))) : y(Number.isInteger(e) && e > 0, () => `Expected ${n} to be a positive integer, but got ${Oe(e)}.`);
 }
 function Oe(e) {
   return e === null ? "null" : Array.isArray(e) ? "[" + e.map((n) => Oe(n)).join(",") + "]" : typeof e == "string" ? `"${e}"` : `${e}`;
@@ -279,7 +279,7 @@ function Gn(e) {
   B(rn, "PoolMode", e);
 }
 const P = [], me = "/";
-function Jn(e, n) {
+function qn(e, n) {
   P.push(e);
   try {
     const t = n();
@@ -291,7 +291,7 @@ function Jn(e, n) {
 function cn() {
   return P.length === 0 ? "" : P.join(me) + me;
 }
-function qn(e) {
+function Jn(e) {
   if (!Se(e))
     throw new Error("Not a valid tensor name: '" + e + "'");
   return cn() + e;
@@ -351,7 +351,7 @@ function Wn(e, n) {
 }
 let K;
 function Xn() {
-  return K == null && (K = Te().epsilon()), K;
+  return K == null && (K = ye().epsilon()), K;
 }
 function W() {
   return "channelsLast";
@@ -482,7 +482,7 @@ function st(e, n) {
     case 1:
       return xe([e, n]);
     case 2:
-      return Je([e, n], 0);
+      return qe([e, n], 0);
     case 3:
       return Ke([e, n], 0);
     case 4:
@@ -570,7 +570,7 @@ function at(e, n = 1) {
   return Fe(e);
 }
 function lt(e) {
-  return $(() => _e(e, U(Ie(e), 1)));
+  return $(() => Te(e, U(Ie(e), 1)));
 }
 function ft(e, n, t, s) {
   return $(() => Ne(e, n, t, s));
@@ -589,7 +589,7 @@ export {
   $e as A,
   ct as B,
   Wn as C,
-  _n as D,
+  Tn as D,
   at as E,
   ht as F,
   lt as G,
@@ -621,14 +621,14 @@ export {
   ot as d,
   Fn as e,
   Kn as f,
-  qn as g,
+  Jn as g,
   In as h,
   En as i,
   Xn as j,
   jn as k,
   zn as l,
   it as m,
-  Jn as n,
+  qn as n,
   de as o,
   Hn as p,
   Nn as q,
@@ -637,7 +637,7 @@ export {
   Ln as t,
   Pn as u,
   Cn as v,
-  Tn as w,
+  yn as w,
   vn as x,
   Z as y,
   an as z

package/dist/{tile-mbfagpsB.js → tile-OWUvpIVt.js}

@@ -1,9 +1,9 @@
-import { A as e, B as a, l as i, E as c, T as l } from "./index-DOvlwCh-.js";
+import { o as e, q as a, x as i, E as c, T as l } from "./index-Duu1Lvvv.js";
 function u(r, t) {
   const n = a(r, "x", "tile", "string_or_numeric");
   i(n.rank === t.length, () => `Error in transpose: rank of input ${n.rank} must match length of reps ${t}.`);
-  const s = { x: n }, o = { reps: t };
-  return c.runKernel(l, s, o);
+  const o = { x: n }, s = { reps: t };
+  return c.runKernel(l, o, s);
 }
 const p = /* @__PURE__ */ e({ tile_: u });
 export {

package/dist/tokeniser/BaseTokeniser.d.ts

@@ -0,0 +1,25 @@
+import { Conversation, ITokeniser } from './type';
+import { default as EE } from 'eventemitter3';
+export declare const SPECIALS: string[];
+export default abstract class BaseTokeniser extends EE<'trainStatus'> implements ITokeniser {
+    protected specialTokens: Map<string, number>;
+    protected specialTokenSet: Set<number>;
+    abstract vocabSize: number;
+    abstract eosToken: number;
+    abstract bosToken: number;
+    abstract trained: boolean;
+    abstract addToken(token: string, index?: number): number;
+    isSpecialToken(index: number): boolean;
+    protected addSpecialTokens(): void;
+    protected addSpecialToken(token: string, index: number): void;
+    abstract train(text: string[]): Promise<number>;
+    abstract getVocab(): string[];
+    abstract getMerges(): [string, string][];
+    abstract destroy(): void;
+    abstract encode(text: string): number[];
+    encodeSequence(text: string): number[];
+    encodeConversation(conversation: Conversation[], completion?: boolean): number[];
+    abstract decode(tokens: number[]): string;
+    decodeConversation(tokens: number[]): Conversation[];
+    getSpecialTokenIndex(token: string): number | undefined;
+}

package/dist/tokeniser/BaseTokeniser.js

@@ -0,0 +1,94 @@
+import { E as r } from "../index-DvYrXKkX.js";
+const h = [
+  "<eos>",
+  "<bos>",
+  "",
+  "<|user_start|>",
+  "<|user_end|>",
+  "<|assistant_start|>",
+  "<|assistant_end|>",
+  "<|system_start|>",
+  "<|system_end|>"
+];
+class k extends r {
+  specialTokens = /* @__PURE__ */ new Map();
+  specialTokenSet = /* @__PURE__ */ new Set();
+  isSpecialToken(e) {
+    return this.specialTokenSet.has(e);
+  }
+  addSpecialTokens() {
+    h.forEach((e, t) => {
+      this.addToken(e, t), this.specialTokens.set(e, t), this.specialTokenSet.add(t);
+    });
+  }
+  addSpecialToken(e, t) {
+    this.specialTokens.set(e, t), this.specialTokenSet.add(t);
+  }
+  encodeSequence(e) {
+    const t = this.encode(e);
+    return [this.bosToken, ...t, this.eosToken];
+  }
+  encodeConversation(e, t) {
+    const s = [[this.bosToken]], a = [
+      this.getSpecialTokenIndex("<|user_start|>"),
+      this.getSpecialTokenIndex("<|assistant_start|>"),
+      this.getSpecialTokenIndex("<|system_start|>")
+    ], n = [
+      this.getSpecialTokenIndex("<|user_end|>"),
+      this.getSpecialTokenIndex("<|assistant_end|>"),
+      this.getSpecialTokenIndex("<|system_end|>")
+    ];
+    for (const i of e) {
+      const c = this.encode(i.content);
+      switch (i.role) {
+        case "user":
+          s.push([a[0]]);
+          break;
+        case "assistant":
+          s.push([a[1]]);
+          break;
+        case "system":
+          s.push([a[2]]);
+          break;
+      }
+      switch (s.push(c), i.role) {
+        case "user":
+          s.push([n[0]]);
+          break;
+        case "assistant":
+          s.push([n[1]]);
+          break;
+        case "system":
+          s.push([n[2]]);
+          break;
+      }
+    }
+    const o = s.flat();
+    return t ? o.push(a[1]) : o.push(this.eosToken), o;
+  }
+  decodeConversation(e) {
+    const t = [];
+    let s = 0;
+    for (; s < e.length; ) {
+      const a = e[s];
+      let n = null;
+      if (a === this.getSpecialTokenIndex("<|user_start|>") ? n = "user" : a === this.getSpecialTokenIndex("<|assistant_start|>") ? n = "assistant" : a === this.getSpecialTokenIndex("<|system_start|>") && (n = "system"), n) {
+        s++;
+        const o = [];
+        for (; s < e.length && e[s] !== this.getSpecialTokenIndex(`<|${n}_end|>`); )
+          o.push(e[s]), s++;
+        const i = this.decode(o);
+        t.push({ role: n, content: i });
+      }
+      s++;
+    }
+    return t;
+  }
+  getSpecialTokenIndex(e) {
+    return this.specialTokens.get(e);
+  }
+}
+export {
+  h as SPECIALS,
+  k as default
+};

package/dist/tokeniser/CharTokeniser.d.ts

@@ -1,23 +1,24 @@
-import { default as EE } from 'eventemitter3';
-import { ITokeniser } from './type';
-export default class CharTokeniser extends EE<'trainStatus'> implements ITokeniser {
+import { default as BaseTokeniser } from './BaseTokeniser';
+export default class CharTokeniser extends BaseTokeniser {
     vocabSize: number;
     eosToken: number;
+    bosToken: number;
     unkToken: number;
     vocab: string[];
     private cache;
     private _trained;
     constructor(vocabSize: number);
     constructor(vocab: string[]);
+    addToken(token: string, index?: number): number;
     get trained(): boolean;
     destroy(): void;
     train(text: string[]): Promise<number>;
-    tokenise(text: string[], numeric: true): Promise<number[][]>;
-    tokenise(text: string[]): Promise<string[][]>;
-    detokenise(tokens: number[][]): Promise<string[]>;
-    encode(text: string): Promise<number[]>;
-    decode(tokens: number[]): Promise<string>;
+    tokenise(text: string[], numeric: true): number[][];
+    tokenise(text: string[]): string[][];
+    detokenise(tokens: (number[] | Uint16Array)[]): string[];
+    encode(text: string): number[];
+    decode(tokens: number[] | Uint16Array): string;
     getVocab(): string[];
-    getMerges(): Promise<[string, string][]>;
+    getMerges(): [string, string][];
     createTrainingData(text: string[], windowSize?: number): Promise<[number[], number[]]>;
 }