@genai-fi/nanogpt 0.2.12 → 0.3.0

package/dist/utilities/dummy.js

@@ -1,12 +1,14 @@
-async function f(s) {
-  const o = s.tf.zeros([1, s.config.blockSize], "int32"), { logits: i, loss: t } = s.forward(o, void 0, !1);
-  await i.data(), i.dispose(), t && t.dispose(), o.dispose();
+import "../index-pWA4_lUh.js";
+import { z as n } from "../zeros-CCy9C3uU.js";
+async function c(s) {
+  const o = n([1, s.config.blockSize], "int32"), { logits: t, loss: i } = s.forward(o, void 0, !1);
+  await t.data(), t.dispose(), i && i.dispose(), o.dispose();
 }
-function c(s) {
-  const o = s.tf.zeros([1, s.config.blockSize], "int32"), { logits: i, loss: t } = s.forward(o, void 0, !1);
-  i.dispose(), t && t.dispose(), o.dispose();
+function f(s) {
+  const o = n([1, s.config.blockSize], "int32"), { logits: t, loss: i } = s.forward(o, void 0, !1);
+  t.dispose(), i && i.dispose(), o.dispose();
 }
 export {
-  c as dummyPass,
-  f as dummyPassAsync
+  f as dummyPass,
+  c as dummyPassAsync
 };

package/dist/utilities/generate.js

@@ -1,20 +1,23 @@
-async function h(r, t, a, c, g) {
+import { t as y } from "../index-pWA4_lUh.js";
+import { t as x } from "../tensor2d-ZuQSh2D-.js";
+import { c as g } from "../concat-BIZS_td9.js";
+async function A(o, r, a, c, T) {
   if (c <= 0)
     throw new Error("Length must be a positive integer");
   if (a.length === 0)
     throw new Error("Prompt cannot be an empty string");
-  const p = await r.tokenise([a], !0), s = t.config.useRope ? new Array(t.config.nLayer).fill(void 0) : void 0, u = t.tf.tidy(() => {
-    let e = t.tf.tensor2d(p, [1, p[0].length], "int32"), n = e;
+  const d = await o.tokenise([a], !0), s = r.config.useRope ? new Array(r.config.nLayer).fill(void 0) : void 0, p = y(() => {
+    let e = x(d, [1, d[0].length], "int32"), t = e;
     for (let f = 0; f < c; f++) {
-      const { output: o } = t.generate(e, s, g), w = e, y = n;
-      n = t.tf.concat([n, o], 1), e = s ? o : t.tf.concat([e, o], 1), w.dispose(), y.dispose(), s || o.dispose();
+      const { output: n } = r.generate(e, s, T), l = e, m = t;
+      t = g([t, n], 1), e = s ? n : g([e, n], 1), l.dispose(), m.dispose(), s || n.dispose();
     }
-    return n;
-  }), T = await u.array();
-  u.dispose();
-  const i = T[0], d = i.indexOf(r.eosToken);
-  return d !== -1 && i.splice(d), await r.decode(i);
+    return t;
+  }), w = await p.array();
+  p.dispose();
+  const i = w[0], u = i.indexOf(o.eosToken);
+  return u !== -1 && i.splice(u), await o.decode(i);
 }
 export {
-  h as generateText
+  A as generateText
 };

package/dist/utilities/load.d.ts

@@ -1,7 +1,6 @@
-import { default as TF } from '@tensorflow/tfjs';
 import { default as NanoGPT } from '../NanoGPTModel';
 import { ITokeniser } from '../tokeniser/type';
-export declare function loadModel(tf: typeof TF, data: Blob | Buffer | string): Promise<{
+export declare function loadModel(data: Blob | Buffer | string): Promise<{
     model: NanoGPT;
     tokeniser: ITokeniser;
 }>;

package/dist/utilities/load.js

@@ -1,45 +1,47 @@
-import { j as u } from "../jszip.min-CjP2V1VV.js";
-import { importWeights as F } from "./weights.js";
-import k from "../tokeniser/CharTokeniser.js";
-import j from "../NanoGPTModel.js";
-import { dummyPassAsync as A } from "./dummy.js";
-async function E(o) {
-  const e = await fetch(o);
-  if (!e.ok)
-    throw new Error(`Failed to fetch ${o}: ${e.statusText}`);
-  return e.arrayBuffer();
+import { j as k } from "../jszip.min-CjP2V1VV.js";
+import { importWeights as b } from "./weights.js";
+import u from "../tokeniser/CharTokeniser.js";
+import F from "../NanoGPTModel.js";
+import { dummyPassAsync as j } from "./dummy.js";
+import { d as T } from "../index-pWA4_lUh.js";
+import E from "../tokeniser/bpe.js";
+async function A(t) {
+  const o = await fetch(t);
+  if (!o.ok)
+    throw new Error(`Failed to fetch ${t}: ${o.statusText}`);
+  return o.arrayBuffer();
 }
-async function T(o, e) {
-  const m = typeof e == "string" ? await E(e) : e, n = await u.loadAsync(m), s = /* @__PURE__ */ new Map(), c = await n.file("manifest.json")?.async("string");
-  if (!c)
+async function P(t) {
+  const o = typeof t == "string" ? await A(t) : t, n = await k.loadAsync(o), i = /* @__PURE__ */ new Map(), f = await n.file("manifest.json")?.async("string");
+  if (!f)
     throw new Error("Manifest file not found in the zip archive");
-  const f = JSON.parse(c);
-  for (const [t, r] of Object.entries(f.weightSpec))
-    s.set(t, { spec: r, data: new Float32Array() });
-  const l = await n.file("tokeniser.json")?.async("string");
-  if (!l)
+  const l = JSON.parse(f);
+  for (const [e, r] of Object.entries(l.weightSpec))
+    i.set(e, { spec: r, data: new Float32Array() });
+  const p = await n.file("tokeniser.json")?.async("string");
+  if (!p)
     throw new Error("Tokeniser file not found in the zip archive");
-  const g = JSON.parse(l), y = new k(g.vocab), p = /* @__PURE__ */ new Map();
-  for (const t of Object.keys(n.files))
-    if (t.endsWith(".bin")) {
-      const r = t.replace(".bin", ""), h = await n.file(t).async("arraybuffer"), d = new Float32Array(h), i = s.get(r) || { spec: [], data: new Float32Array() };
-      i.data = d, s.set(r, i);
-      const b = await F(i, o);
-      p.set(r, b);
+  const s = JSON.parse(p), y = (s.type ?? "char") === "char" ? new u(s.vocab) : new E(s.vocab, s.merges), w = /* @__PURE__ */ new Map();
+  for (const e of Object.keys(n.files))
+    if (e.endsWith(".bin")) {
+      const r = e.replace(".bin", ""), g = await n.file(e).async("arraybuffer"), h = new Float32Array(g), c = i.get(r) || { spec: [], data: new Float32Array() };
+      c.data = h, i.set(r, c);
+      const d = await b(c);
+      w.set(r, d);
     }
-  o.disposeVariables();
-  const a = new j(o, f.config);
-  await A(a), a.loadWeights(p);
-  const w = await n.file("log.json")?.async("string");
-  if (w)
+  T();
+  const a = new F(l.config);
+  await j(a), a.loadWeights(w);
+  const m = await n.file("log.json")?.async("string");
+  if (m)
     try {
-      const t = JSON.parse(w);
-      a.log = t;
-    } catch (t) {
-      throw console.error("Error parsing training log:", t), new Error(`Failed to parse training log: ${t}`);
+      const e = JSON.parse(m);
+      a.log = e;
+    } catch (e) {
+      throw console.error("Error parsing training log:", e), new Error(`Failed to parse training log: ${e}`);
     }
   return { model: a, tokeniser: y };
 }
 export {
-  T as loadModel
+  P as loadModel
 };

package/dist/utilities/profile.js

@@ -1,4 +1,4 @@
-import { m as s } from "../index-YPKosni4.js";
+import { m as s } from "../index-pWA4_lUh.js";
 const m = 1024 * 1024;
 class i {
   log = /* @__PURE__ */ new Map();

package/dist/utilities/save.js

@@ -1,10 +1,11 @@
 import { j as g } from "../jszip.min-CjP2V1VV.js";
 import { exportWeights as l } from "./weights.js";
-const b = "1.0.0";
-async function p(t, s, i) {
+import b from "../tokeniser/CharTokeniser.js";
+const y = "1.0.0";
+async function h(t, a, i) {
   const o = i?.includeLog ?? !0, c = t.saveWeights(), e = new g(), f = {};
-  for (const [n, a] of c) {
-    const r = await l(a);
+  for (const [n, s] of c) {
+    const r = await l(s);
     f[n] = r.spec, e.file(`${n}.bin`, r.data.buffer, { binary: !0 });
   }
   if (e.file(
@@ -12,7 +13,7 @@ async function p(t, s, i) {
     JSON.stringify({
       weightSpec: f,
       config: t.config,
-      version: b,
+      version: y,
       application: "@genai-fi/nanogpt",
       meta: i?.metadata,
       name: i?.name
@@ -22,15 +23,19 @@ async function p(t, s, i) {
     }
   ), e.file(
     "tokeniser.json",
-    JSON.stringify({ vocab: s.getVocab(), merges: await s.getMerges() }),
+    JSON.stringify({
+      type: a instanceof b ? "char" : "bpe",
+      vocab: a.getVocab(),
+      merges: await a.getMerges()
+    }),
     {
       binary: !1
     }
   ), o && e.file("log.json", JSON.stringify(t.log), { binary: !1 }), i?.files)
-    for (const [n, a] of Object.entries(i.files))
-      e.file(n, JSON.stringify(a), { binary: !1 });
+    for (const [n, s] of Object.entries(i.files))
+      e.file(n, JSON.stringify(s), { binary: !1 });
   return e.generateAsync({ type: "blob" });
 }
 export {
-  p as saveModel
+  h as saveModel
 };

package/dist/utilities/tokenParse.d.ts

@@ -1 +1 @@
-export default function parseTokens(text: string, raw?: boolean): string[];
+export default function parseTokens(text: string): string[];

package/dist/utilities/tokenParse.js

@@ -1,66 +1,12 @@
-function o(a, c) {
-  a.length !== 0 && c.push(` ${a.trim()}`);
-}
-function l(a, c) {
-  const r = c ? a : a.toLocaleLowerCase(), s = [];
+function h(o) {
+  const r = Array.from(o), t = [], l = new RegExp("(\\p{P}|\\p{S}|\\s)", "gu");
   let e = "";
-  for (let t = 0; t < r.length; t++) {
-    const n = r[t];
-    switch (n) {
-      case "0":
-      case "1":
-      case "2":
-      case "3":
-      case "4":
-      case "5":
-      case "6":
-      case "7":
-      case "8":
-      case "9":
-      case ":":
-      case ";":
-      case ",":
-      case ".":
-      case "?":
-      case "!":
-      case '"':
-      case "'":
-      case "`":
-      case "(":
-      case ")":
-      case "[":
-      case "]":
-      case "{":
-      case "}":
-      case "-":
-      case "_":
-      case "/":
-      case "\\":
-      case "%":
-      case "<":
-      case ">":
-      case "=":
-      case "+":
-      case "*":
-      case "&":
-      case "^":
-      case "|":
-      case "~":
-      case "@":
-      case "#":
-      case "$":
-        c ? s.push(e) : o(e, s), s.push(n), e = "";
-        break;
-      case " ":
-        c ? s.push(e) : o(e, s), e = n;
-        break;
-      default:
-        e += n;
-        break;
-    }
+  for (let n = 0; n < r.length; n++) {
+    const s = r[n];
+    s === " " ? (t.push(e), e = s) : s.match(l) ? (t.push(e), t.push(s), e = "") : e += s;
   }
-  return e.length > 0 && (c ? s.push(e) : o(e, s)), s;
+  return e.length > 0 && t.push(e), t.filter((n) => n.length > 0);
 }
 export {
-  l as default
+  h as default
 };

package/dist/utilities/weights.d.ts

@@ -1,4 +1,4 @@
-import { default as TF } from '@tensorflow/tfjs';
+import { Tensor } from '@tensorflow/tfjs-core';
 export interface ITensorSpec {
     shape: number[];
     min?: number;
@@ -8,5 +8,5 @@ export interface IWeightManifest {
     spec: ITensorSpec[];
     data: Float32Array;
 }
-export declare function exportWeights(weights: TF.Tensor[]): Promise<IWeightManifest>;
-export declare function importWeights(manifest: IWeightManifest, tf: typeof TF): Promise<TF.Tensor[]>;
+export declare function exportWeights(weights: Tensor[]): Promise<IWeightManifest>;
+export declare function importWeights(manifest: IWeightManifest): Promise<Tensor[]>;

package/dist/utilities/weights.js

@@ -1,22 +1,24 @@
-function h(e) {
-  const n = e.reduce((s, o) => s + o.length, 0), a = new Float32Array(n);
+import "../index-pWA4_lUh.js";
+import { t as p } from "../tensor-DJVbYhh1.js";
+function h(n) {
+  const e = n.reduce((s, o) => s + o.length, 0), a = new Float32Array(e);
   let t = 0;
-  for (const s of e)
+  for (const s of n)
     a.set(s, t), t += s.length;
   return a;
 }
-async function f(e) {
-  const n = {
+async function d(n) {
+  const e = {
     spec: [],
     data: new Float32Array()
   }, a = [];
-  for (const t of e) {
+  for (const t of n) {
     if (!t || !Array.isArray(t.shape) || t.shape.length === 0) {
       console.warn("Skipping weight with invalid shape:", t);
       continue;
     }
     const s = t.min(), o = t.max(), c = (await s.data())[0], r = (await o.data())[0] - c;
-    n.spec.push({
+    e.spec.push({
       shape: t.shape,
       min: c,
       scale: r
@@ -24,20 +26,20 @@ async function f(e) {
     const i = await t.data();
     a.push(i);
   }
-  return n.data = h(a), n;
+  return e.data = h(a), e;
 }
-async function l(e, n) {
-  const a = [];
-  let t = 0;
-  for (const s of e.spec) {
-    const o = s.shape.reduce((i, p) => i * p, 1), c = e.data.slice(t, t + o);
-    t += o;
-    const r = n.tensor(c, s.shape, "float32");
-    a.push(r);
+async function u(n) {
+  const e = [];
+  let a = 0;
+  for (const t of n.spec) {
+    const s = t.shape.reduce((r, i) => r * i, 1), o = n.data.slice(a, a + s);
+    a += s;
+    const c = p(o, t.shape, "float32");
+    e.push(c);
   }
-  return a;
+  return e;
 }
 export {
-  f as exportWeights,
-  l as importWeights
+  d as exportWeights,
+  u as importWeights
 };

package/dist/variable-Dl_ub3pk.js

@@ -0,0 +1,23 @@
+import { E as i } from "./index-pWA4_lUh.js";
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function m(r, a = !0, e, t) {
+  return i.makeVariable(r, a, e, t);
+}
+export {
+  m as v
+};

package/dist/{stack-BtKpB0Ry.js → zeros-CCy9C3uU.js}

@@ -1,7 +1,7 @@
-import { E as e, j as c, o as f, k as u, l as a, P as i } from "./index-YPKosni4.js";
+import { o as m, h as r, M as l, E as c, N as i, k as p, O as u, n as f } from "./index-pWA4_lUh.js";
 /**
  * @license
- * Copyright 2018 Google LLC. All Rights Reserved.
+ * Copyright 2020 Google LLC. All Rights Reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -15,15 +15,16 @@ import { E as e, j as c, o as f, k as u, l as a, P as i } from "./index-YPKosni4
  * limitations under the License.
  * =============================================================================
  */
-function g(n, s, t = 1, r = "float32") {
-  if (t === 0)
-    throw new Error("Cannot have a step of zero");
-  const o = { start: n, stop: s, step: t, dtype: r };
-  return e.runKernel(c, {}, o);
+function x(a, e) {
+  const o = r(a, "real", "complex"), s = r(e, "imag", "complex");
+  l(o.shape, s.shape, `real and imag shapes, ${o.shape} and ${s.shape}, must match in call to tf.complex().`);
+  const n = { real: o, imag: s };
+  return c.runKernel(i, n);
 }
+const g = /* @__PURE__ */ m({ complex_: x });
 /**
  * @license
- * Copyright 2020 Google LLC. All Rights Reserved.
+ * Copyright 2018 Google LLC. All Rights Reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -37,14 +38,15 @@ function g(n, s, t = 1, r = "float32") {
  * limitations under the License.
  * =============================================================================
  */
-function k(n, s = 0) {
-  const t = u(n, "tensors", "stack", "string_or_numeric");
-  a(t.length >= 1, () => "Pass at least one tensor to tf.stack"), t.length > 0 && a(s <= t[0].rank, () => "Axis must be <= rank of the tensor");
-  const r = t, o = { axis: s };
-  return e.runKernel(i, r, o);
+function t(a, e = "float32") {
+  if (p(a), e === "complex64") {
+    const s = t(a, "float32"), n = t(a, "float32");
+    return g(s, n);
+  }
+  const o = u(f(a), e);
+  return c.makeTensor(o, a, e);
 }
-const h = /* @__PURE__ */ f({ stack_: k });
 export {
-  g as r,
-  h as s
+  g as c,
+  t as z
 };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.2.12",
+    "version": "0.3.0",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",
@@ -21,6 +21,7 @@
         "preview": "vite preview",
         "lint": "eslint",
         "test": "vitest",
+        "test:gl": "playwright test",
         "ci:test": "vitest --coverage --reporter=junit --outputFile=junit.xml",
         "coverage": "vitest run --coverage",
         "train": "tsx scripts/train.ts",

package/dist/assets/worker-BYeSPNkq.js

@@ -1 +0,0 @@
-(function(){"use strict";function u(n,t){const e=n,s=[];let a="";for(let o=0;o<e.length;o++){const c=e[o];switch(c){case"0":case"1":case"2":case"3":case"4":case"5":case"6":case"7":case"8":case"9":case":":case";":case",":case".":case"?":case"!":case'"':case"'":case"`":case"(":case")":case"[":case"]":case"{":case"}":case"-":case"_":case"/":case"\\":case"%":case"<":case">":case"=":case"+":case"*":case"&":case"^":case"|":case"~":case"@":case"#":case"$":s.push(a),s.push(c),a="";break;case" ":s.push(a),a=c;break;default:a+=c;break}}return a.length>0&&s.push(a),s}function k(n){const t=new Map;for(let e=0;e<n.length;e++){const s=n[e];for(let a=0;a<s.length-1;a++){const o=`${s[a]}${s[a+1]}`,c=t.get(o)||{a:s[a],b:s[a+1],count:0,instances:new Set};c.count+=1,c.instances.add(e),t.set(o,c)}}return{pairs:t,tokens:n}}function p(n,t,e,s,a){const o=`${t}${e}`;if(n.pairs.has(o)){const c=n.pairs.get(o);c.count+=a,c.instances.add(s)}else n.pairs.set(o,{a:t,b:e,count:a,instances:new Set([s])})}function b(n){let t=null,e=0;for(const s of n.pairs.values())s.count>e&&(e=s.count,t=s);return t}function m(n,t){return n.map(e=>{const s=[];for(let a=0;a<e.length;a++)a<e.length-1&&e[a]===t[0]&&e[a+1]===t[1]?(s.push(t[0]+t[1]),a++):s.push(e[a]);return s})}function y(n,t){t.instances.forEach(e=>{const s=n.tokens[e],a=[];for(let o=0;o<s.length;o++)if(o<s.length-1&&s[o]===t.a&&s[o+1]===t.b){const c=t.a+t.b;a.push(c),o>0&&(p(n,s[o-1],t.a,e,-1),p(n,s[o-1],c,e,1)),o++,o<s.length-1&&(p(n,t.b,s[o+1],e,-1),p(n,c,s[o+1],e,1))}else a.push(s[o]);n.tokens[e]=a}),n.pairs.delete(`${t.a}${t.b}`)}class l{vocab=new Set;vocabIndex=new Map;merges=[];pretokenMap=new Map;constructor(t,e){t&&t.forEach((s,a)=>{this.vocab.add(s),this.vocabIndex.set(s,a)}),e&&(this.merges=e)}train(t,e,s){const a=t.map(i=>u(i)).flat(1),o=new Set(a);this.vocab=new Set,this.pretokenMap.clear(),this.merges=[],this.vocab.add("<eos>");const c=Array.from(o),d=c.map(i=>i.split("").map(h=>(this.vocab.add(h),h))),f=k(d);for(;this.vocab.size<e&&this.merges.length<e;){const i=b(f);if(!i)break;this.merges.push([i.a,i.b]),this.vocab.add(i.a+i.b),y(f,i),s&&this.vocab.size%100===0&&s(this.vocab.size/e,this.vocab.size)}c.forEach((i,g)=>{const h=d[g];this.pretokenMap.set(i,h)}),this.vocabIndex.clear();let M=0;for(const i of this.vocab.keys())this.vocabIndex.set(i,M++)}getVocab(){return Array.from(this.vocab)}getMerges(){return this.merges}tokeniseWord(t){let e=t.split("");return this.merges.forEach(s=>{e=m([e],s)[0]}),this.pretokenMap.set(t,e),e}tokeniseStrings(t){return t.map(e=>u(e).map(o=>this.pretokenMap.has(o)?this.pretokenMap.get(o):this.tokeniseWord(o)).flat(1))}tokenise(t,e){const s=this.tokeniseStrings(t);return e?s.map(a=>a.map(o=>this.vocabIndex.get(o)??-1)):s}}let r=new l;onmessage=async n=>{if(n.data.type==="tokenise")if(n.data.numeric){const t=r.tokenise(n.data.text,!0),e={type:"tokeniseResponse",id:n.data.id,tokens:t,numeric:!0};postMessage(e)}else{const t=r.tokenise(n.data.text),e={type:"tokeniseResponse",id:n.data.id,tokens:t,numeric:!1};postMessage(e)}else if(n.data.type==="detokenise"){const t=r.getVocab(),e=n.data.tokens.map(a=>a.map(o=>t[o]).join("")),s={type:"detokeniseResponse",id:n.data.id,text:e};postMessage(s)}else if(n.data.type==="train"){r=new l,r.train(n.data.text,n.data.vocabSize??100,(e,s)=>{const a={type:"trainStatus",id:n.data.id,progress:e,vocabSize:s};postMessage(a)});const t={type:"trainResponse",id:n.data.id,vocabSize:r.getVocab().length};postMessage(t)}else if(n.data.type==="tokens"){const t=r.getVocab(),e={type:"tokensResponse",id:n.data.id,tokens:t};postMessage(e)}}})();

package/dist/tokeniser/NodeTokeniser.d.ts

@@ -1,20 +0,0 @@
-import { default as EE } from 'eventemitter3';
-import { ITokeniser } from './type';
-export default class NodeTokeniser extends EE<'trainStatus'> implements ITokeniser {
-    vocabSize: number;
-    eosToken: number;
-    private bpe;
-    constructor(vocabSize: number);
-    constructor(vocab: string[], merges: [string, string][]);
-    get trained(): boolean;
-    destroy(): void;
-    train(text: string[]): Promise<number>;
-    tokenise(text: string[], numeric: true): Promise<number[][]>;
-    tokenise(text: string[]): Promise<string[][]>;
-    detokenise(tokens: number[][]): Promise<string[]>;
-    encode(text: string): Promise<number[]>;
-    decode(tokens: number[]): Promise<string>;
-    getVocab(): string[];
-    getMerges(): Promise<[string, string][]>;
-    createTrainingData(text: string[], windowSize?: number): Promise<[number[], number[]]>;
-}

package/dist/tokeniser/NodeTokeniser.js

@@ -1,46 +0,0 @@
-import { E as a } from "../index-Dwqa6Zy2.js";
-import o from "./bpe.js";
-class p extends a {
-  vocabSize = 0;
-  eosToken = 0;
-  bpe = new o();
-  constructor(e, t) {
-    super(), Array.isArray(e) ? (this.bpe = new o(e, t), this.vocabSize = e.length) : this.vocabSize = e;
-  }
-  get trained() {
-    return this.vocabSize > 0;
-  }
-  destroy() {
-  }
-  async train(e) {
-    return this.bpe.train(e, this.vocabSize), this.vocabSize = this.bpe.getVocab().length, this.vocabSize;
-  }
-  async tokenise(e, t) {
-    return t ? this.bpe.tokenise(e, !0) : this.bpe.tokenise(e);
-  }
-  async detokenise(e) {
-    const t = this.bpe.getVocab();
-    return e.map((n) => n.map((i) => t[i]).join(""));
-  }
-  async encode(e) {
-    return (await this.tokenise([e], !0))[0];
-  }
-  async decode(e) {
-    return (await this.detokenise([e]))[0];
-  }
-  getVocab() {
-    return this.bpe.getVocab();
-  }
-  async getMerges() {
-    return this.bpe.getMerges();
-  }
-  async createTrainingData(e, t = 5) {
-    const s = this.bpe.tokenise(e, !0), n = [], i = [];
-    for (let r = 0; r < s.length - t; r++)
-      n.push(...s[r].slice(0, t)), i.push(s[r + 1][0]);
-    return [n, i];
-  }
-}
-export {
-  p as default
-};

package/dist/tokeniser/WebTokeniser.d.ts

@@ -1,18 +0,0 @@
-import { default as EE } from 'eventemitter3';
-import { ITokeniser } from './type';
-export default class WebTokeniser extends EE<'trainStatus'> implements ITokeniser {
-    private id;
-    vocabSize: number;
-    private handler?;
-    constructor();
-    destroy(): void;
-    private post;
-    train(text: string[], vocabSize: number): Promise<number>;
-    tokenise(text: string[], numeric: true): Promise<number[][]>;
-    tokenise(text: string[]): Promise<string[][]>;
-    detokenise(tokens: number[][]): Promise<string[]>;
-    encode(text: string): Promise<number[]>;
-    decode(tokens: number[]): Promise<string>;
-    getVocab(): Promise<string[]>;
-    createTrainingData(text: string[], windowSize?: number): Promise<[number[], number[]]>;
-}