@genai-fi/nanogpt 0.2.11 → 0.3.0

package/dist/range-CcDl05lo.js

@@ -0,0 +1,26 @@
+import { E as e, R as f } from "./index-pWA4_lUh.js";
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function E(n, o, r = 1, a = "float32") {
+  if (r === 0)
+    throw new Error("Cannot have a step of zero");
+  const t = { start: n, stop: o, step: r, dtype: a };
+  return e.runKernel(f, {}, t);
+}
+export {
+  E as r
+};

package/dist/{reshape-DmnmKT6r.js → reshape-C8CR_Bad.js}

@@ -1,4 +1,4 @@
-import { o, d as t, E as a, R as p } from "./index-YPKosni4.js";
+import { o, h as t, E as a, q as p } from "./index-pWA4_lUh.js";
 /**
  * @license
  * Copyright 2020 Google LLC. All Rights Reserved.
@@ -19,7 +19,7 @@ function c(r, s) {
   const e = { x: t(r, "x", "reshape", "string_or_numeric") }, n = { shape: s };
   return a.runKernel(p, e, n);
 }
-const x = /* @__PURE__ */ o({ reshape_: c });
+const u = /* @__PURE__ */ o({ reshape_: c });
 export {
-  x as r
+  u as r
 };

package/dist/sin-BJIrfnj7.js

@@ -0,0 +1,47 @@
+import { o, h as t, E as c, K as a, L as e } from "./index-pWA4_lUh.js";
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function i(n) {
+  const s = { x: t(n, "x", "cos", "float32") };
+  return c.runKernel(a, s);
+}
+const f = /* @__PURE__ */ o({ cos_: i });
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function x(n) {
+  const s = { x: t(n, "x", "sin", "float32") };
+  return c.runKernel(e, s);
+}
+const p = /* @__PURE__ */ o({ sin_: x });
+export {
+  f as c,
+  p as s
+};

package/dist/softmax-Be_lsqUc.js

@@ -0,0 +1,105 @@
+import { o as r, h as e, E as c, _ as $, $ as h, a0 as E, s as S, Z as _, a1 as d } from "./index-pWA4_lUh.js";
+import { m as k, s as w, e as K } from "./sum-C7Mgy9Bw.js";
+import { r as l } from "./reshape-C8CR_Bad.js";
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function T(n) {
+  const s = { x: e(n, "x", "exp") };
+  return c.runKernel($, s);
+}
+const b = /* @__PURE__ */ r({ exp_: T });
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function L(n) {
+  const s = { x: e(n, "x", "log", "float32") };
+  return c.runKernel(h, s);
+}
+const N = /* @__PURE__ */ r({ log_: L });
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function v(n, o = null, s = !1) {
+  const t = e(n, "x", "logSumExp"), a = E(o, t.shape), p = k(
+    t,
+    a,
+    !0
+    /* keepDims */
+  ), i = S(t, p), f = b(i), m = w(f, a), u = N(m), x = _(l(p, u.shape), u);
+  if (s) {
+    const g = K(x.shape, a);
+    return l(x, g);
+  }
+  return x;
+}
+const I = /* @__PURE__ */ r({ logSumExp_: v });
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function y(n, o = -1) {
+  const s = e(n, "logits", "softmax", "float32");
+  if (o === -1 && (o = s.rank - 1), o !== s.rank - 1)
+    throw Error(`Softmax along a non-last dimension is not yet supported. Logits was rank ${s.rank} and dim was ${o}`);
+  const t = { logits: s }, a = { dim: o };
+  return c.runKernel(d, t, a);
+}
+const M = /* @__PURE__ */ r({ softmax_: y });
+export {
+  N as a,
+  b as e,
+  I as l,
+  M as s
+};

package/dist/{complex-CJ-qCcLB.js → split-DZbvruEP.js}

@@ -1,4 +1,4 @@
-import { o as c, d as s, g as n, E as m, C as r } from "./index-YPKosni4.js";
+import { o as p, h as i, E as a, S as c } from "./index-pWA4_lUh.js";
 /**
  * @license
  * Copyright 2020 Google LLC. All Rights Reserved.
@@ -15,13 +15,11 @@ import { o as c, d as s, g as n, E as m, C as r } from "./index-YPKosni4.js";
  * limitations under the License.
  * =============================================================================
  */
-function l(o, p) {
-  const a = s(o, "real", "complex"), e = s(p, "imag", "complex");
-  n(a.shape, e.shape, `real and imag shapes, ${a.shape} and ${e.shape}, must match in call to tf.complex().`);
-  const t = { real: a, imag: e };
-  return m.runKernel(r, t);
+function e(t, s, o = 0) {
+  const n = { x: i(t, "x", "split") }, r = { numOrSizeSplits: s, axis: o };
+  return a.runKernel(c, n, r);
 }
-const i = /* @__PURE__ */ c({ complex_: l });
+const u = /* @__PURE__ */ p({ split_: e });
 export {
-  i as c
+  u as s
 };

package/dist/stack-BMm-efee.js

@@ -0,0 +1,27 @@
+import { o as e, i as c, j as n, E as i, P as k } from "./index-pWA4_lUh.js";
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function u(r, t = 0) {
+  const s = c(r, "tensors", "stack", "string_or_numeric");
+  n(s.length >= 1, () => "Pass at least one tensor to tf.stack"), s.length > 0 && n(t <= s[0].rank, () => "Axis must be <= rank of the tensor");
+  const o = s, a = { axis: t };
+  return i.runKernel(k, o, a);
+}
+const l = /* @__PURE__ */ e({ stack_: u });
+export {
+  l as s
+};

package/dist/sum-C7Mgy9Bw.js

@@ -0,0 +1,104 @@
+import { o as c, h as i, E as a, T as f, U as m, V as p } from "./index-pWA4_lUh.js";
+/**
+ * @license
+ * Copyright 2017 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function d(e, n) {
+  for (let t = 0; t < e.length; ++t)
+    if (e[e.length - t - 1] !== n - 1 - t)
+      return !1;
+  return !0;
+}
+function x(e, n, t) {
+  const s = e.length + n.length, r = [];
+  let o = 0, l = 0;
+  for (let u = 0; u < s; u++)
+    t.indexOf(u) === -1 ? r.push(e[o++]) : r.push(n[l++]);
+  return r;
+}
+function S(e, n) {
+  const t = [], s = e.length;
+  for (let o = 0; o < s; o++)
+    n.indexOf(o) === -1 && t.push(e[o]);
+  const r = n.map((o) => e[o]);
+  return [t, r];
+}
+function A(e, n) {
+  const t = n.map((s) => 1);
+  return x(e, t, n);
+}
+function E(e, n) {
+  if (d(e, n))
+    return null;
+  const t = [];
+  for (let s = 0; s < n; ++s)
+    e.indexOf(s) === -1 && t.push(s);
+  return e.forEach((s) => t.push(s)), t;
+}
+function O(e) {
+  return e.map((n, t) => [t, n]).sort((n, t) => n[1] - t[1]).map((n) => n[0]);
+}
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function h(e, n = null, t = !1) {
+  const r = { x: i(e, "x", "max") }, o = { reductionIndices: n, keepDims: t };
+  return a.runKernel(f, r, o);
+}
+const T = /* @__PURE__ */ c({ max_: h });
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function g(e, n = null, t = !1) {
+  let s = i(e, "x", "sum");
+  s.dtype === "bool" && (s = m(s, "int32"));
+  const r = { x: s }, o = { axis: n, keepDims: t };
+  return a.runKernel(p, r, o);
+}
+const b = /* @__PURE__ */ c({ sum_: g });
+export {
+  O as a,
+  S as c,
+  A as e,
+  E as g,
+  T as m,
+  b as s
+};

package/dist/tensor-DJVbYhh1.js

@@ -0,0 +1,24 @@
+import { D as t, F as a } from "./index-pWA4_lUh.js";
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function f(r, n, e) {
+  const o = t(r, e);
+  return a(r, n, o, e);
+}
+export {
+  f as t
+};

package/dist/tensor2d-ZuQSh2D-.js

@@ -0,0 +1,30 @@
+import { y as t, D as s, F as a } from "./index-pWA4_lUh.js";
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function i(n, r, o) {
+  if (t(n), r != null && r.length !== 2)
+    throw new Error("tensor2d() requires shape to have two numbers");
+  const e = s(n, o);
+  if (e.length !== 2 && e.length !== 1)
+    throw new Error("tensor2d() requires values to be number[][] or flat/TypedArray");
+  if (e.length === 1 && r == null)
+    throw new Error("tensor2d() requires shape to be provided when `values` are a flat/TypedArray");
+  return a(n, r, e, o);
+}
+export {
+  i as t
+};

package/dist/tokeniser/bpe.d.ts

@@ -1,14 +1,25 @@
-export default class BPE {
+import { default as EE } from 'eventemitter3';
+import { ITokeniser } from './type';
+export default class BPETokeniser extends EE<'trainStatus'> implements ITokeniser {
+    private targetSize;
     private vocab;
     private vocabIndex;
     private merges;
     private pretokenMap;
-    constructor(vocab?: string[], merges?: [string, string][]);
-    train(text: string[], vocabSize: number, onUpdate?: (progress: number, vocabSize: number) => void): void;
+    constructor(vocabSize: number);
+    constructor(vocab: string[], merges?: [string, string][]);
+    destroy(): void;
+    get trained(): boolean;
+    get vocabSize(): number;
+    get eosToken(): number;
+    train(text: string[]): Promise<number>;
     getVocab(): string[];
-    getMerges(): [string, string][];
+    getMerges(): Promise<[string, string][]>;
     private tokeniseWord;
     private tokeniseStrings;
-    tokenise(text: string[], numeric: true): number[][];
-    tokenise(text: string[]): string[][];
+    tokenise(text: string[], numeric: true): Promise<number[][]>;
+    tokenise(text: string[]): Promise<string[][]>;
+    detokenise(tokens: number[][]): Promise<string[]>;
+    encode(text: string): Promise<number[]>;
+    decode(tokens: number[]): Promise<string>;
 }

package/dist/tokeniser/bpe.js

@@ -1,100 +1,128 @@
-import f from "../utilities/tokenParse.js";
-function b(r) {
-  const s = /* @__PURE__ */ new Map();
-  for (let e = 0; e < r.length; e++) {
-    const t = r[e];
+import p from "../utilities/tokenParse.js";
+import { E as g } from "../index-Dwqa6Zy2.js";
+function u(o, e) {
+  return `${o}-::-${e}`;
+}
+function k(o) {
+  const e = /* @__PURE__ */ new Map();
+  for (let s = 0; s < o.length; s++) {
+    const t = o[s];
     for (let n = 0; n < t.length - 1; n++) {
-      const o = `${t[n]}${t[n + 1]}`, i = s.get(o) || {
+      const r = u(t[n], t[n + 1]), a = e.get(r) || {
         a: t[n],
         b: t[n + 1],
         count: 0,
         instances: /* @__PURE__ */ new Set()
       };
-      i.count += 1, i.instances.add(e), s.set(o, i);
+      a.count += 1, a.instances.add(s), e.set(r, a);
     }
   }
-  return { pairs: s, tokens: r };
+  return { pairs: e, tokens: o };
 }
-function h(r, s, e, t, n) {
-  const o = `${s}${e}`;
-  if (r.pairs.has(o)) {
-    const i = r.pairs.get(o);
-    i.count += n, i.instances.add(t);
+function h(o, e, s, t, n) {
+  const r = u(e, s);
+  if (o.pairs.has(r)) {
+    const a = o.pairs.get(r);
+    a.count += n, n > 0 ? a.instances.add(t) : a.count <= 0 ? o.pairs.delete(r) : a.instances.delete(t);
   } else
-    r.pairs.set(o, { a: s, b: e, count: n, instances: /* @__PURE__ */ new Set([t]) });
+    o.pairs.set(r, { a: e, b: s, count: n, instances: /* @__PURE__ */ new Set([t]) });
 }
-function g(r) {
-  let s = null, e = 0;
-  for (const t of r.pairs.values())
-    t.count > e && (e = t.count, s = t);
-  return s;
+function b(o) {
+  let e = null, s = 0;
+  for (const t of o.pairs.values())
+    t.count > s && (s = t.count, e = t);
+  return e;
 }
-function m(r, s) {
-  return r.map((e) => {
+function d(o, e) {
+  return o.map((s) => {
     const t = [];
-    for (let n = 0; n < e.length; n++)
-      n < e.length - 1 && e[n] === s[0] && e[n + 1] === s[1] ? (t.push(s[0] + s[1]), n++) : t.push(e[n]);
+    for (let n = 0; n < s.length; n++)
+      n < s.length - 1 && s[n] === e[0] && s[n + 1] === e[1] ? (t.push(e[0] + e[1]), n++) : t.push(s[n]);
     return t;
   });
 }
-function d(r, s) {
-  s.instances.forEach((e) => {
-    const t = r.tokens[e], n = [];
-    for (let o = 0; o < t.length; o++)
-      if (o < t.length - 1 && t[o] === s.a && t[o + 1] === s.b) {
-        const i = s.a + s.b;
-        n.push(i), o > 0 && (h(r, t[o - 1], s.a, e, -1), h(r, t[o - 1], i, e, 1)), o++, o < t.length - 1 && (h(r, s.b, t[o + 1], e, -1), h(r, i, t[o + 1], e, 1));
+function m(o, e) {
+  e.instances.forEach((s) => {
+    const t = o.tokens[s], n = [];
+    for (let r = 0; r < t.length; r++)
+      if (r < t.length - 1 && t[r] === e.a && t[r + 1] === e.b) {
+        const a = e.a + e.b;
+        n.push(a), r > 0 && (h(o, t[r - 1], e.a, s, -1), h(o, t[r - 1], a, s, 1)), r++, r < t.length - 1 && (h(o, e.b, t[r + 1], s, -1), h(o, a, t[r + 1], s, 1));
       } else
-        n.push(t[o]);
-    r.tokens[e] = n;
-  }), r.pairs.delete(`${s.a}${s.b}`);
+        n.push(t[r]);
+    o.tokens[s] = n;
+  }), o.pairs.delete(u(e.a, e.b));
 }
-class w {
+class w extends g {
+  targetSize;
   vocab = /* @__PURE__ */ new Set();
   vocabIndex = /* @__PURE__ */ new Map();
   merges = [];
   pretokenMap = /* @__PURE__ */ new Map();
-  constructor(s, e) {
-    s && s.forEach((t, n) => {
+  constructor(e, s) {
+    super(), Array.isArray(e) ? (e.forEach((t, n) => {
       this.vocab.add(t), this.vocabIndex.set(t, n);
-    }), e && (this.merges = e);
+    }), s && (this.merges = s), this.targetSize = e.length) : (this.vocab.add("<eos>"), this.vocab.add("<unk>"), this.targetSize = e);
+  }
+  destroy() {
+    this.vocab.clear(), this.vocabIndex.clear(), this.merges = [], this.pretokenMap.clear();
+  }
+  get trained() {
+    return this.vocab.size === this.targetSize && this.merges.length > 0;
+  }
+  get vocabSize() {
+    return this.vocab.size;
   }
-  train(s, e, t) {
-    const n = s.map((a) => f(a, !0)).flat(1), o = new Set(n);
+  get eosToken() {
+    return this.vocabIndex.get("<eos>") ?? 0;
+  }
+  async train(e) {
+    const s = e.map((i) => p(i)).flat(1), t = new Set(s);
     this.vocab = /* @__PURE__ */ new Set(), this.pretokenMap.clear(), this.merges = [], this.vocab.add("<eos>");
-    const i = Array.from(o), u = i.map((a) => a.split("").map((c) => (this.vocab.add(c), c))), p = b(u);
-    for (; this.vocab.size < e && this.merges.length < e; ) {
-      const a = g(p);
-      if (!a)
+    const n = Array.from(t), r = n.map((i) => Array.from(i).map((c) => (this.vocab.add(c), c))), a = k(r);
+    for (; this.vocab.size < this.targetSize && this.merges.length < this.targetSize; ) {
+      const i = b(a);
+      if (!i)
         break;
-      this.merges.push([a.a, a.b]), this.vocab.add(a.a + a.b), d(p, a), t && this.vocab.size % 100 === 0 && t(this.vocab.size / e, this.vocab.size);
+      this.merges.push([i.a, i.b]), this.vocab.add(i.a + i.b), m(a, i);
     }
-    i.forEach((a, l) => {
-      const c = u[l];
-      this.pretokenMap.set(a, c);
+    n.forEach((i, l) => {
+      const c = r[l];
+      this.pretokenMap.set(i, c);
     }), this.vocabIndex.clear();
-    let k = 0;
-    for (const a of this.vocab.keys())
-      this.vocabIndex.set(a, k++);
+    let f = 0;
+    for (const i of this.vocab.keys())
+      this.vocabIndex.set(i, f++);
+    return this.vocab.size;
   }
   getVocab() {
     return Array.from(this.vocab);
   }
-  getMerges() {
+  async getMerges() {
     return this.merges;
   }
-  tokeniseWord(s) {
-    let e = s.split("");
+  tokeniseWord(e) {
+    let s = Array.from(e);
     return this.merges.forEach((t) => {
-      e = m([e], t)[0];
-    }), this.pretokenMap.set(s, e), e;
+      s = d([s], t)[0];
+    }), this.pretokenMap.set(e, s), s;
+  }
+  tokeniseStrings(e) {
+    return e.map((s) => p(s).map((r) => this.pretokenMap.has(r) ? this.pretokenMap.get(r) : this.tokeniseWord(r)).flat(1));
+  }
+  async tokenise(e, s) {
+    const t = this.tokeniseStrings(e);
+    return s ? t.map((n) => n.map((r) => this.vocabIndex.get(r) ?? -1)) : t;
+  }
+  async detokenise(e) {
+    const s = this.getVocab();
+    return e.map((n) => n.map((r) => s[r]).join(""));
   }
-  tokeniseStrings(s) {
-    return s.map((e) => f(e, !0).map((o) => this.pretokenMap.has(o) ? this.pretokenMap.get(o) : this.tokeniseWord(o)).flat(1));
+  async encode(e) {
+    return (await this.tokenise([e], !0))[0];
   }
-  tokenise(s, e) {
-    const t = this.tokeniseStrings(s);
-    return e ? t.map((n) => n.map((o) => this.vocabIndex.get(o) ?? -1)) : t;
+  async decode(e) {
+    return (await this.detokenise([e]))[0];
   }
 }
 export {

package/dist/training/AdamExt.js

@@ -1,4 +1,4 @@
-import { A as r, a as c, s as h, b as g, e as o } from "../index-YPKosni4.js";
+import { A as r, b as c, f as h, s as g, e as o } from "../index-pWA4_lUh.js";
 class u extends r {
   constructor(t, e, s, a, i) {
     super(t, e, s, a), this.config = i, this.startLearningRate = t;

package/dist/training/DatasetBuilder.d.ts

@@ -1,12 +1,12 @@
+import { Tensor } from '@tensorflow/tfjs-core';
 import { ITokeniser } from '../tokeniser/type';
-import { default as TF } from '@tensorflow/tfjs';
+import { Dataset } from '@tensorflow/tfjs-data';
 export declare class DatasetBuilder {
     tokenizer: ITokeniser;
     blockSize: number;
-    private tf;
-    constructor(tf: typeof TF, tokenizer: ITokeniser, blockSize?: number);
-    createTextDataset(textData: string[], batchSize?: number, start?: number, end?: number): Promise<TF.data.Dataset<{
-        xs: TF.Tensor;
-        ys: TF.Tensor;
+    constructor(tokenizer: ITokeniser, blockSize?: number);
+    createTextDataset(textData: string[], batchSize?: number, start?: number, end?: number): Promise<Dataset<{
+        xs: Tensor;
+        ys: Tensor;
     }>>;
 }