@genai-fi/nanogpt 0.10.3 → 0.11.0

package/dist/{step-CINUs5QB.js → step-dXR33iOg.js}

@@ -1,7 +1,7 @@
-import { l as _, aG as P, A as k, B as M, E as v, aH as F, aI as J, aJ as X, aK as q, aL as Q, aM as Y } from "./index-DOvlwCh-.js";
+import { y as _, aG as P, q as A, u as M, E as v, aH as q, aI as F, aJ as J, aK as X, aL as Q, aM as Y } from "./index-D0RBWjq8.js";
 function it(t, n, e, o, s = "NHWC", f) {
   const l = t[3], r = [...n, l], c = nt(s);
-  return B(t, r, e, f, o, null, null, c);
+  return j(t, r, e, f, o, null, null, c);
 }
 function ft(t, n, e, o, s, f, l = "channelsLast") {
   const [r, c] = T(n);
@@ -12,7 +12,7 @@ function ft(t, n, e, o, s, f, l = "channelsLast") {
     u = [r, c, t[1], t[1]];
   else
     throw new Error(`Unknown dataFormat ${l}`);
-  return B(t, u, e, o, s, f, !1, l);
+  return j(t, u, e, o, s, f, !1, l);
 }
 function ht(t, n, e, o, s, f, l = "NDHWC") {
   const [r, c, u] = K(n);
@@ -25,7 +25,7 @@ function ht(t, n, e, o, s, f, l = "NDHWC") {
     throw new Error(`Unknown dataFormat ${l}`);
   return Z(t, h, e, o, s, !1, i, f);
 }
-function B(t, n, e, o, s, f, l = !1, r = "channelsLast") {
+function j(t, n, e, o, s, f, l = !1, r = "channelsLast") {
   let [c, u, h, i] = [-1, -1, -1, -1];
   if (r === "channelsLast")
     [c, u, h, i] = t;
@@ -33,15 +33,15 @@ function B(t, n, e, o, s, f, l = !1, r = "channelsLast") {
     [c, i, u, h] = t;
   else
     throw new Error(`Unknown dataFormat ${r}`);
-  const [a, p, , $] = n, [w, g] = T(e), [x, y] = T(o), L = A(a, x), b = A(p, y), { padInfo: C, outHeight: E, outWidth: D } = d(s, u, h, w, g, L, b, f, r), I = l ? $ * i : $;
+  const [a, p, , $] = n, [w, g] = T(e), [x, E] = T(o), L = k(a, x), b = k(p, E), { padInfo: C, outHeight: y, outWidth: D } = d(s, u, h, w, g, L, b, f, r), I = l ? $ * i : $;
   let m;
-  return r === "channelsFirst" ? m = [c, I, E, D] : r === "channelsLast" && (m = [c, E, D, I]), {
+  return r === "channelsFirst" ? m = [c, I, y, D] : r === "channelsLast" && (m = [c, y, D, I]), {
     batchSize: c,
     dataFormat: r,
     inHeight: u,
     inWidth: h,
     inChannels: i,
-    outHeight: E,
+    outHeight: y,
     outWidth: D,
     outChannels: I,
     padInfo: C,
@@ -52,7 +52,7 @@ function B(t, n, e, o, s, f, l = !1, r = "channelsLast") {
     effectiveFilterHeight: L,
     effectiveFilterWidth: b,
     dilationHeight: x,
-    dilationWidth: y,
+    dilationWidth: E,
     inShape: t,
     outShape: m,
     filterShape: n
@@ -66,7 +66,7 @@ function Z(t, n, e, o, s, f = !1, l = "channelsLast", r) {
     [c, a, u, h, i] = t;
   else
     throw new Error(`Unknown dataFormat ${l}`);
-  const [p, $, w, , g] = n, [x, y, L] = K(e), [b, C, E] = K(o), D = A(p, b), I = A($, C), m = A(w, E), { padInfo: G, outDepth: N, outHeight: R, outWidth: W } = tt(s, u, h, i, x, y, L, D, I, m, r), H = f ? g * a : g;
+  const [p, $, w, , g] = n, [x, E, L] = K(e), [b, C, y] = K(o), D = k(p, b), I = k($, C), m = k(w, y), { padInfo: G, outDepth: N, outHeight: R, outWidth: W } = tt(s, u, h, i, x, E, L, D, I, m, r), H = f ? g * a : g;
   let O;
   return l === "channelsFirst" ? O = [c, H, N, R, W] : l === "channelsLast" && (O = [c, N, R, W, H]), {
     batchSize: c,
@@ -81,7 +81,7 @@ function Z(t, n, e, o, s, f = !1, l = "channelsLast", r) {
     outChannels: H,
     padInfo: G,
     strideDepth: x,
-    strideHeight: y,
+    strideHeight: E,
     strideWidth: L,
     filterDepth: p,
     filterHeight: $,
@@ -91,26 +91,26 @@ function Z(t, n, e, o, s, f = !1, l = "channelsLast", r) {
     effectiveFilterWidth: m,
     dilationDepth: b,
     dilationHeight: C,
-    dilationWidth: E,
+    dilationWidth: y,
     inShape: t,
     outShape: O,
     filterShape: n
   };
 }
 function z(t, n, e, o, s) {
-  o == null && (o = j(t, n, e));
+  o == null && (o = B(t, n, e));
   const f = t[0], l = t[1], r = U((f - n + 2 * o) / e + 1, s), c = U((l - n + 2 * o) / e + 1, s);
   return [r, c];
 }
 function S(t, n, e, o, s, f) {
-  s == null && (s = j(t, n[0], o[0]));
+  s == null && (s = B(t, n[0], o[0]));
   const l = [0, 0, 0, e];
   for (let r = 0; r < 3; r++)
     t[r] + 2 * s >= n[r] && (l[r] = U((t[r] - n[r] + 2 * s) / o[r] + 1, f));
   return l;
 }
-function j(t, n, e, o = 1) {
-  const s = A(n, o);
+function B(t, n, e, o = 1) {
+  const s = k(n, o);
   return Math.floor((t[0] * (e - 1) - e + s) / 2);
 }
 function T(t) {
@@ -119,7 +119,7 @@ function T(t) {
 function K(t) {
   return typeof t == "number" ? [t, t, t] : t;
 }
-function A(t, n) {
+function k(t, n) {
   return n <= 1 ? t : t + (t - 1) * (n - 1);
 }
 function d(t, n, e, o, s, f, l, r, c) {
@@ -157,8 +157,8 @@ function tt(t, n, e, o, s, f, l, r, c, u, h) {
     a = g[0], p = g[1], $ = g[2];
   } else if (t === "same") {
     a = Math.ceil(n / s), p = Math.ceil(e / f), $ = Math.ceil(o / l);
-    const w = (a - 1) * s + r - n, g = (p - 1) * f + c - e, x = ($ - 1) * l + u - o, y = Math.floor(w / 2), L = w - y, b = Math.floor(g / 2), C = g - b, E = Math.floor(x / 2), D = x - E;
-    i = { top: b, bottom: C, left: E, right: D, front: y, back: L, type: "SAME" };
+    const w = (a - 1) * s + r - n, g = (p - 1) * f + c - e, x = ($ - 1) * l + u - o, E = Math.floor(w / 2), L = w - E, b = Math.floor(g / 2), C = g - b, y = Math.floor(x / 2), D = x - y;
+    i = { top: b, bottom: C, left: y, right: D, front: E, back: L, type: "SAME" };
   } else
     throw Error(`Unknown padding parameter: ${t}`);
   return { padInfo: i, outDepth: a, outHeight: p, outWidth: $ };
@@ -212,41 +212,41 @@ function $t(t, n, e) {
 }
 function et(t) {
   const e = { x: M(t, "x", "sigmoid", "float32") };
-  return v.runKernel(F, e);
+  return v.runKernel(q, e);
 }
-const gt = /* @__PURE__ */ k({ sigmoid_: et });
+const gt = /* @__PURE__ */ A({ sigmoid_: et });
 function ot(t) {
   const e = { x: M(t, "x", "elu", "float32") };
-  return v.runKernel(J, e);
+  return v.runKernel(F, e);
 }
-const wt = /* @__PURE__ */ k({ elu_: ot });
+const wt = /* @__PURE__ */ A({ elu_: ot });
 function st(t, n = 0.2) {
   const o = { x: M(t, "x", "leakyRelu") }, s = { alpha: n };
-  return v.runKernel(X, o, s);
+  return v.runKernel(J, o, s);
 }
-const xt = /* @__PURE__ */ k({ leakyRelu_: st });
+const xt = /* @__PURE__ */ A({ leakyRelu_: st });
 function rt(t, n) {
   const e = M(t, "x", "prelu"), o = M(n, "alpha", "prelu"), s = { x: e, alpha: o };
-  return v.runKernel(q, s);
+  return v.runKernel(X, s);
 }
-const Et = /* @__PURE__ */ k({ prelu_: rt });
+const yt = /* @__PURE__ */ A({ prelu_: rt });
 function lt(t) {
   const e = { x: M(t, "x", "relu6") };
   return v.runKernel(Q, e);
 }
-const yt = /* @__PURE__ */ k({ relu6_: lt });
+const Et = /* @__PURE__ */ A({ relu6_: lt });
 function ct(t, n = 0) {
   const o = { x: M(t, "x", "step") }, s = { alpha: n };
   return v.runKernel(Y, o, s);
 }
-const bt = /* @__PURE__ */ k({ step_: ct });
+const bt = /* @__PURE__ */ A({ step_: ct });
 export {
   bt as a,
-  B as b,
+  j as b,
   $t as c,
   Z as d,
   wt as e,
-  j as f,
+  B as f,
   it as g,
   ft as h,
   ht as i,
@@ -254,8 +254,8 @@ export {
   at as k,
   xt as l,
   pt as m,
-  Et as p,
-  yt as r,
+  yt as p,
+  Et as r,
   gt as s,
   V as t
 };

package/dist/sum-BdplSvq_.js

@@ -0,0 +1,11 @@
+import { q as a, u as e, N as c, E as l, Q as m } from "./index-D0RBWjq8.js";
+function i(t, n = null, o = !1) {
+  let s = e(t, "x", "sum");
+  s.dtype === "bool" && (s = c(s, "int32"));
+  const r = { x: s }, u = { axis: n, keepDims: o };
+  return l.runKernel(m, r, u);
+}
+const f = /* @__PURE__ */ a({ sum_: i });
+export {
+  f as s
+};

package/dist/tensor-BQqrDvpx.js

@@ -0,0 +1,8 @@
+import { Y as t, Z as a } from "./index-D0RBWjq8.js";
+function f(r, n, e) {
+  const o = t(r, e);
+  return a(r, n, o, e);
+}
+export {
+  f as t
+};

package/dist/tensor1d-LxP9asMm.js

@@ -0,0 +1,11 @@
+import { X as o, Y as s, Z as t } from "./index-D0RBWjq8.js";
+function h(r, e) {
+  o(r);
+  const n = s(r, e);
+  if (n.length !== 1)
+    throw new Error("tensor1d() requires values to be a flat/TypedArray");
+  return t(r, null, n, e);
+}
+export {
+  h as t
+};

package/dist/{tensor2d-Bs9wZRc7.js → tensor2d-BN1sSfQO.js}

@@ -1,13 +1,13 @@
-import { $ as t, a0 as a, a1 as s } from "./index-DOvlwCh-.js";
+import { X as t, Y as s, Z as a } from "./index-D0RBWjq8.js";
 function i(n, r, o) {
   if (t(n), r != null && r.length !== 2)
     throw new Error("tensor2d() requires shape to have two numbers");
-  const e = a(n, o);
+  const e = s(n, o);
   if (e.length !== 2 && e.length !== 1)
     throw new Error("tensor2d() requires values to be number[][] or flat/TypedArray");
   if (e.length === 1 && r == null)
     throw new Error("tensor2d() requires shape to be provided when `values` are a flat/TypedArray");
-  return s(n, r, e, o);
+  return a(n, r, e, o);
 }
 export {
   i as t

package/dist/{tensor4d-BARPdTaS.js → tensor4d-DVwr7pLF.js}

@@ -1,4 +1,4 @@
-import { $ as t, a0 as a, a1 as s } from "./index-DOvlwCh-.js";
+import { X as t, Y as a, Z as s } from "./index-D0RBWjq8.js";
 function i(n, r, o) {
   if (t(n), r != null && r.length !== 4)
     throw new Error("tensor4d() requires shape to have four numbers");

package/dist/{tfjs_backend-y1cvNhLA.js → tfjs_backend-Vi4JfLzT.js}

@@ -1,16 +1,16 @@
-import { A as w, B as S, l as T, E as q, x as U, L as ie, h as ue, a3 as ae, a6 as le, aX as fe, aF as he, aY as Te, t as $, X as ge, m as ke, w as _e, aZ as Ie } from "./index-DOvlwCh-.js";
-import { t as Ee } from "./tensor1d-D11P_7Dp.js";
-import { r as Le, d as Ne } from "./dropout-BcvN9JYi.js";
-import { s as F } from "./slice-BfEGSH82.js";
-import { r as c } from "./reshape-ByE68wS9.js";
-import { g as be } from "./gather-D3JcZUaI.js";
-import { e as Fe } from "./step-CINUs5QB.js";
-import { c as Ce } from "./clip_by_value-BnO7-a88.js";
-import { t as Pe } from "./tile-mbfagpsB.js";
-import { s as ve, b as Me, c as je, g as Ve } from "./selu_util-kbhpTdYD.js";
-import { m as k } from "./mat_mul-SjpJRLyL.js";
-import { t as Ue } from "./transpose-ClWiBS_b.js";
-import { c as M } from "./concat-BV8bt5H-.js";
+import { q as w, u as S, y as T, E as J, n as U, D as ie, V as ue, $ as ae, a2 as le, aX as fe, aF as he, aY as Te, t as $, N as ge, m as ke, l as _e, aZ as Ie } from "./index-D0RBWjq8.js";
+import { t as Ee } from "./tensor1d-LxP9asMm.js";
+import { r as Le, d as Ne } from "./dropout-OxuaJz6z.js";
+import { s as F } from "./slice-Aqy7KbJh.js";
+import { r as c } from "./reshape-CkjKPPqB.js";
+import { g as be } from "./gather-BcO5UQNJ.js";
+import { e as Fe } from "./step-dXR33iOg.js";
+import { c as Ce } from "./clip_by_value-Dn5tzexi.js";
+import { t as Pe } from "./tile-CvN_LyVr.js";
+import { s as ve, b as Me, c as je, g as Ve } from "./selu_util-BuLbmbrl.js";
+import { m as k } from "./mat_mul-DxpNTCRz.js";
+import { t as Ue } from "./transpose-JawVKyZy.js";
+import { c as M } from "./concat-C6X3AAlQ.js";
 function Be(e) {
   return M(
     e,
@@ -22,11 +22,11 @@ const xe = /* @__PURE__ */ w({ concat1d_: Be });
 function Ge(e, n) {
   return M(e, n);
 }
-const Je = /* @__PURE__ */ w({ concat2d_: Ge });
-function qe(e, n) {
+const qe = /* @__PURE__ */ w({ concat2d_: Ge });
+function Je(e, n) {
   return M(e, n);
 }
-const Ke = /* @__PURE__ */ w({ concat3d_: qe });
+const Ke = /* @__PURE__ */ w({ concat3d_: Je });
 function Ze(e, n) {
   return M(e, n);
 }
@@ -52,7 +52,7 @@ function He(e, n, t) {
 }
 const V = /* @__PURE__ */ w({ slice4d_: He });
 function Qe({ a: e, b: n, transposeA: t = !1, transposeB: s = !1, bias: r, activation: o = "linear", preluActivationWeights: a, leakyreluAlpha: f = 0.2 }) {
-  if (ve(q.state.gradientDepth, o) === !1) {
+  if (ve(J.state.gradientDepth, o) === !1) {
     let D = k(e, n, t, s);
     return r != null && (D = U(D, r)), Me(D, o, a, f);
   }
@@ -60,7 +60,7 @@ function Qe({ a: e, b: n, transposeA: t = !1, transposeB: s = !1, bias: r, activ
   [i, u] = ie(i, u);
   const m = t ? i.shape[i.rank - 2] : i.shape[i.rank - 1], d = s ? u.shape[u.rank - 1] : u.shape[u.rank - 2], _ = t ? i.shape[i.rank - 1] : i.shape[i.rank - 2], h = s ? u.shape[u.rank - 2] : u.shape[u.rank - 1], ee = i.shape.slice(0, -2), I = u.shape.slice(0, -2), ne = ue(ee), te = ue(I);
   T(m === d, () => `Error in fused matMul: inner shapes (${m}) and (${d}) of Tensors with shapes ${i.shape} and ${u.shape} and transposeA=${t} and transposeB=${s} must match.`);
-  const x = ae(i.shape.slice(0, -2), u.shape.slice(0, -2)).concat([_, h]), G = t ? c(i, [ne, m, _]) : c(i, [ne, _, m]), J = s ? c(u, [te, h, d]) : c(u, [te, d, h]);
+  const x = ae(i.shape.slice(0, -2), u.shape.slice(0, -2)).concat([_, h]), G = t ? c(i, [ne, m, _]) : c(i, [ne, _, m]), q = s ? c(u, [te, h, d]) : c(u, [te, d, h]);
   let E;
   r != null && (E = S(r, "bias", "fused matMul"), [E] = ie(E, i), ae(x, E.shape));
   let se;
@@ -75,23 +75,23 @@ function Qe({ a: e, b: n, transposeA: t = !1, transposeB: s = !1, bias: r, activ
       return [L, N];
   }, oe = {
     a: G,
-    b: J,
+    b: q,
     bias: E,
     preluActivationWeights: se
   }, ce = { transposeA: t, transposeB: s, activation: o, leakyreluAlpha: f };
   return r == null ? le((C, A, O) => {
     const y = (
       // tslint:disable-next-line: no-unnecessary-type-assertion
-      q.runKernel(fe, oe, ce)
+      J.runKernel(fe, oe, ce)
     );
     return O([C, A, y]), { value: c(y, x), gradFunc: re };
-  })(G, J) : le((C, A, O, y) => {
+  })(G, q) : le((C, A, O, y) => {
     const j = (
       // tslint:disable-next-line: no-unnecessary-type-assertion
-      q.runKernel(fe, oe, ce)
+      J.runKernel(fe, oe, ce)
     );
     return y([C, A, j, O]), { value: c(j, x), gradFunc: re };
-  })(G, J, E);
+  })(G, q, E);
 }
 const pe = /* @__PURE__ */ w({ fusedMatMul_: Qe });
 class $e extends Error {
@@ -279,7 +279,7 @@ function Gn(e) {
   B(rn, "PoolMode", e);
 }
 const P = [], me = "/";
-function Jn(e, n) {
+function qn(e, n) {
   P.push(e);
   try {
     const t = n();
@@ -291,7 +291,7 @@ function Jn(e, n) {
 function cn() {
   return P.length === 0 ? "" : P.join(me) + me;
 }
-function qn(e) {
+function Jn(e) {
   if (!Se(e))
     throw new Error("Not a valid tensor name: '" + e + "'");
   return cn() + e;
@@ -482,7 +482,7 @@ function st(e, n) {
     case 1:
       return xe([e, n]);
     case 2:
-      return Je([e, n], 0);
+      return qe([e, n], 0);
     case 3:
       return Ke([e, n], 0);
     case 4:
@@ -621,14 +621,14 @@ export {
   ot as d,
   Fn as e,
   Kn as f,
-  qn as g,
+  Jn as g,
   In as h,
   En as i,
   Xn as j,
   jn as k,
   zn as l,
   it as m,
-  Jn as n,
+  qn as n,
   de as o,
   Hn as p,
   Nn as q,

package/dist/{tile-mbfagpsB.js → tile-CvN_LyVr.js}

@@ -1,11 +1,11 @@
-import { A as e, B as a, l as i, E as c, T as l } from "./index-DOvlwCh-.js";
-function u(r, t) {
+import { q as e, u as a, y as i, E as c, T as u } from "./index-D0RBWjq8.js";
+function l(r, t) {
   const n = a(r, "x", "tile", "string_or_numeric");
   i(n.rank === t.length, () => `Error in transpose: rank of input ${n.rank} must match length of reps ${t}.`);
   const s = { x: n }, o = { reps: t };
-  return c.runKernel(l, s, o);
+  return c.runKernel(u, s, o);
 }
-const p = /* @__PURE__ */ e({ tile_: u });
+const p = /* @__PURE__ */ e({ tile_: l });
 export {
   p as t
 };

package/dist/tokeniser/BaseTokeniser.d.ts

@@ -0,0 +1,27 @@
+import { Conversation, ITokeniser } from './type';
+import { default as EE } from 'eventemitter3';
+export declare const SPECIALS: string[];
+export default abstract class BaseTokeniser extends EE<'trainStatus'> implements ITokeniser {
+    protected specialTokens: Map<string, number>;
+    protected specialTokenSet: Set<number>;
+    abstract vocabSize: number;
+    abstract eosToken: number;
+    abstract bosToken: number;
+    abstract trained: boolean;
+    abstract addToken(token: string, index?: number): number;
+    isSpecialToken(index: number): boolean;
+    protected addSpecialTokens(): void;
+    protected addSpecialToken(token: string, index: number): void;
+    abstract train(text: string[]): Promise<number>;
+    abstract tokenise(text: string[], numeric?: boolean): Promise<string[][] | number[][]>;
+    abstract detokenise(tokens: string[][] | number[][]): Promise<string[]>;
+    abstract getVocab(): string[];
+    abstract getMerges(): Promise<[string, string][]>;
+    abstract destroy(): void;
+    abstract encode(text: string): Promise<number[]>;
+    encodeSequence(text: string): Promise<number[]>;
+    encodeConversation(conversation: Conversation[], completion?: boolean): Promise<number[]>;
+    abstract decode(tokens: number[]): Promise<string>;
+    decodeConversation(tokens: number[]): Promise<Conversation[]>;
+    getSpecialTokenIndex(token: string): number | undefined;
+}

package/dist/tokeniser/BaseTokeniser.js

@@ -0,0 +1,94 @@
+import { E as r } from "../index-DvYrXKkX.js";
+const h = [
+  "<eos>",
+  "<bos>",
+  "",
+  "<|user_start|>",
+  "<|user_end|>",
+  "<|assistant_start|>",
+  "<|assistant_end|>",
+  "<|system_start|>",
+  "<|system_end|>"
+];
+class k extends r {
+  specialTokens = /* @__PURE__ */ new Map();
+  specialTokenSet = /* @__PURE__ */ new Set();
+  isSpecialToken(e) {
+    return this.specialTokenSet.has(e);
+  }
+  addSpecialTokens() {
+    h.forEach((e, t) => {
+      this.addToken(e, t), this.specialTokens.set(e, t), this.specialTokenSet.add(t);
+    });
+  }
+  addSpecialToken(e, t) {
+    this.specialTokens.set(e, t), this.specialTokenSet.add(t);
+  }
+  async encodeSequence(e) {
+    const t = await this.encode(e);
+    return [this.bosToken, ...t, this.eosToken];
+  }
+  async encodeConversation(e, t) {
+    const s = [[this.bosToken]], a = [
+      this.getSpecialTokenIndex("<|user_start|>"),
+      this.getSpecialTokenIndex("<|assistant_start|>"),
+      this.getSpecialTokenIndex("<|system_start|>")
+    ], n = [
+      this.getSpecialTokenIndex("<|user_end|>"),
+      this.getSpecialTokenIndex("<|assistant_end|>"),
+      this.getSpecialTokenIndex("<|system_end|>")
+    ];
+    for (const i of e) {
+      const c = await this.encode(i.content);
+      switch (i.role) {
+        case "user":
+          s.push([a[0]]);
+          break;
+        case "assistant":
+          s.push([a[1]]);
+          break;
+        case "system":
+          s.push([a[2]]);
+          break;
+      }
+      switch (s.push(c), i.role) {
+        case "user":
+          s.push([n[0]]);
+          break;
+        case "assistant":
+          s.push([n[1]]);
+          break;
+        case "system":
+          s.push([n[2]]);
+          break;
+      }
+    }
+    const o = s.flat();
+    return t ? o.push(a[1]) : o.push(this.eosToken), o;
+  }
+  async decodeConversation(e) {
+    const t = [];
+    let s = 0;
+    for (; s < e.length; ) {
+      const a = e[s];
+      let n = null;
+      if (a === this.getSpecialTokenIndex("<|user_start|>") ? n = "user" : a === this.getSpecialTokenIndex("<|assistant_start|>") ? n = "assistant" : a === this.getSpecialTokenIndex("<|system_start|>") && (n = "system"), n) {
+        s++;
+        const o = [];
+        for (; s < e.length && e[s] !== this.getSpecialTokenIndex(`<|${n}_end|>`); )
+          o.push(e[s]), s++;
+        const i = await this.decode(o);
+        t.push({ role: n, content: i });
+      }
+      s++;
+    }
+    return t;
+  }
+  getSpecialTokenIndex(e) {
+    return this.specialTokens.get(e);
+  }
+}
+export {
+  h as SPECIALS,
+  k as default
+};

package/dist/tokeniser/CharTokeniser.d.ts

@@ -1,14 +1,15 @@
-import { default as EE } from 'eventemitter3';
-import { ITokeniser } from './type';
-export default class CharTokeniser extends EE<'trainStatus'> implements ITokeniser {
+import { default as BaseTokeniser } from './BaseTokeniser';
+export default class CharTokeniser extends BaseTokeniser {
     vocabSize: number;
     eosToken: number;
+    bosToken: number;
     unkToken: number;
     vocab: string[];
     private cache;
     private _trained;
     constructor(vocabSize: number);
     constructor(vocab: string[]);
+    addToken(token: string, index?: number): number;
     get trained(): boolean;
     destroy(): void;
     train(text: string[]): Promise<number>;