@genai-fi/nanogpt 0.2.12 → 0.3.1

package/dist/tokeniser/WebTokeniser.js

@@ -1,96 +0,0 @@
-import { E as d } from "../index-Dwqa6Zy2.js";
-const t = new Worker(new URL(
-  /* @vite-ignore */
-  "/assets/worker-BYeSPNkq.js",
-  import.meta.url
-), {
-  type: "module"
-});
-let r = 0;
-class m extends d {
-  id;
-  vocabSize = 0;
-  handler;
-  constructor() {
-    super(), this.id = r++, this.handler = (e) => {
-      e.data.type === "trainStatus" && e.data.id === this.id && (this.vocabSize = e.data.vocabSize, this.emit("trainStatus", e.data.progress, e.data.vocabSize));
-    }, t.addEventListener("message", this.handler);
-  }
-  destroy() {
-    this.handler && (t.removeEventListener("message", this.handler), this.handler = void 0);
-  }
-  post(e) {
-    t.postMessage(e);
-  }
-  async train(e, n) {
-    return new Promise((s) => {
-      const i = (a) => {
-        a.data.type === "trainResponse" && a.data.id === this.id && (t.removeEventListener("message", i), this.vocabSize = a.data.vocabSize, s(this.vocabSize));
-      };
-      t.addEventListener("message", i), this.post({
-        type: "train",
-        id: this.id,
-        text: e,
-        vocabSize: n
-      });
-    });
-  }
-  async tokenise(e, n) {
-    return new Promise((s) => {
-      const i = (a) => {
-        a.data.type === "tokeniseResponse" && a.data.id === this.id && (t.removeEventListener("message", i), s(a.data.tokens));
-      };
-      t.addEventListener("message", i), this.post({
-        type: "tokenise",
-        id: this.id,
-        text: e,
-        numeric: n
-      });
-    });
-  }
-  async detokenise(e) {
-    return new Promise((n) => {
-      const s = (i) => {
-        i.data.type === "detokeniseResponse" && i.data.id === this.id && (t.removeEventListener("message", s), n(i.data.text));
-      };
-      t.addEventListener("message", s), this.post({
-        type: "detokenise",
-        id: this.id,
-        tokens: e
-      });
-    });
-  }
-  async encode(e) {
-    return (await this.tokenise([e], !0))[0];
-  }
-  async decode(e) {
-    return (await this.detokenise([e]))[0];
-  }
-  async getVocab() {
-    return new Promise((e) => {
-      const n = (s) => {
-        s.data.type === "tokensResponse" && s.data.id === this.id && (t.removeEventListener("message", n), e(s.data.tokens));
-      };
-      t.addEventListener("message", n), this.post({
-        type: "tokens",
-        id: this.id
-      });
-    });
-  }
-  async createTrainingData(e, n = 5) {
-    return new Promise((s) => {
-      const i = (a) => {
-        a.data.type === "buildTrainingDataResponse" && a.data.id === this.id && (t.removeEventListener("message", i), s(a.data.trainingData));
-      };
-      t.addEventListener("message", i), this.post({
-        type: "buildTrainingData",
-        id: this.id,
-        text: e,
-        windowSize: n
-      });
-    });
-  }
-}
-export {
-  m as default
-};

package/dist/tokeniser/worker.js

@@ -1,53 +0,0 @@
-import d from "./bpe.js";
-let e = new d();
-onmessage = async (s) => {
-  if (s.data.type === "tokenise")
-    if (s.data.numeric) {
-      const t = e.tokenise(s.data.text, !0), a = {
-        type: "tokeniseResponse",
-        id: s.data.id,
-        tokens: t,
-        numeric: !0
-      };
-      postMessage(a);
-    } else {
-      const t = e.tokenise(s.data.text), a = {
-        type: "tokeniseResponse",
-        id: s.data.id,
-        tokens: t,
-        numeric: !1
-      };
-      postMessage(a);
-    }
-  else if (s.data.type === "detokenise") {
-    const t = e.getVocab(), a = s.data.tokens.map((i) => i.map((n) => t[n]).join("")), o = {
-      type: "detokeniseResponse",
-      id: s.data.id,
-      text: a
-    };
-    postMessage(o);
-  } else if (s.data.type === "train") {
-    e = new d(), e.train(s.data.text, s.data.vocabSize ?? 100, (a, o) => {
-      const i = {
-        type: "trainStatus",
-        id: s.data.id,
-        progress: a,
-        vocabSize: o
-      };
-      postMessage(i);
-    });
-    const t = {
-      type: "trainResponse",
-      id: s.data.id,
-      vocabSize: e.getVocab().length
-    };
-    postMessage(t);
-  } else if (s.data.type === "tokens") {
-    const t = e.getVocab(), a = {
-      type: "tokensResponse",
-      id: s.data.id,
-      tokens: t
-    };
-    postMessage(a);
-  }
-};