@genai-fi/nanogpt 0.2.2 → 0.2.4

package/dist/TeachableLLM.js

@@ -1,6 +1,6 @@
 import { defaultConfig as d } from "./config.js";
-import u from "./NanoGPTModel.js";
-import { saveModel as m } from "./utilities/save.js";
+import m from "./NanoGPTModel.js";
+import { saveModel as u } from "./utilities/save.js";
 import { loadModel as l } from "./utilities/load.js";
 import f from "./Generator.js";
 import _ from "./Trainer.js";
@@ -8,6 +8,7 @@ import { E as c } from "./index-Dwqa6Zy2.js";
 import { dummyPassAsync as h } from "./utilities/dummy.js";
 import g from "./tokeniser/CharTokeniser.js";
 import "./papaparse.min-C8l2Kvo1.js";
+import "./index-Tf7vU29b.js";
 class a extends c {
   _config;
   _model;
@@ -44,7 +45,7 @@ class a extends c {
   saveModel(t) {
     if (!this._model || !this._tokeniser)
       throw new Error("Model or tokeniser is not initialized.");
-    return m(this._model, this._tokeniser, t);
+    return u(this._model, this._tokeniser, t);
   }
   static loadModel(t, r) {
     const e = new a(t);
@@ -59,7 +60,7 @@ class a extends c {
     }), e;
   }
   static create(t, r = {}) {
-    const e = { ...d, ...r }, s = new g(e.vocabSize), o = new u(t, e), i = new a(t, s, o);
+    const e = { ...d, ...r }, s = new g(e.vocabSize), o = new m(t, e), i = new a(t, s, o);
     return i.setStatus("warmup"), h(o).then(() => {
       i.tokeniser.trained ? i.setStatus("ready") : (i.setStatus("awaitingTokens"), i.tokeniser.once("trainStatus", (n) => {
         n === "trained" && i.setStatus("ready");

package/dist/data/parquet.js

@@ -1,12 +1,14 @@
-async function p(s, f = 104857600, e = "text") {
-  const r = await (await import("../parquet-DpcqBLb0.js").then((t) => t.p)).ParquetReader.openBuffer(Buffer.from(await s.arrayBuffer())), a = [], i = r.getCursor([[e]]);
+import { B as n } from "../index-Tf7vU29b.js";
+const p = 100 * 1024 * 1024;
+async function w(s, f = p, r = "text") {
+  const e = await (await import("../parquet-BRl5lE_I.js").then((t) => t.p)).ParquetReader.openBuffer(n.from(await s.arrayBuffer())), a = [], i = e.getCursor([[r]]);
   let o = 0;
   for (; ; ) {
     const t = await i.next();
-    if (!t || !t[e] || typeof t[e] != "string" || (a.push(t[e]), o += t[e].length, o > f)) break;
+    if (!t || !t[r] || typeof t[r] != "string" || (a.push(t[r]), o += t[r].length, o > f)) break;
   }
-  return r.close(), a;
+  return e.close(), a;
 }
 export {
-  p as loadParquet
+  w as loadParquet
 };

package/dist/data/pdf.d.ts

@@ -0,0 +1 @@
+export declare function loadPDF(file: File, maxSize?: number): Promise<string[]>;

package/dist/data/pdf.js

@@ -0,0 +1,12 @@
+async function u(s, i = 104857600) {
+  const e = await (await import("../pdf-kJD-f258.js")).getDocument({ data: await s.arrayBuffer() }).promise, c = e.numPages, a = [];
+  let n = 0;
+  for (let t = 1; t <= c; t++) {
+    const o = (await (await e.getPage(t)).getTextContent()).items.map((g) => g.str).join(" ");
+    if (a.push(o), n += o.length, n > i) break;
+  }
+  return a;
+}
+export {
+  u as loadPDF
+};

package/dist/data/textLoader.js

@@ -1,19 +1,39 @@
 import { p as s } from "../papaparse.min-C8l2Kvo1.js";
-import { loadParquet as l } from "./parquet.js";
-function m(e, t) {
+import { loadParquet as d } from "./parquet.js";
+import { loadPDF as m } from "./pdf.js";
+function x(e, t) {
   const a = e.findIndex((n) => n.toLowerCase() === t.toLowerCase());
   return a === -1 ? 0 : a;
 }
-function u(e) {
+function f(e) {
   return e.every((t) => t.length < 64);
 }
-async function w(e, t) {
-  const a = e.type;
+function l(e) {
+  return e.split(".").pop() || "";
+}
+function w(e) {
+  switch (l(e)) {
+    case "parquet":
+      return "application/parquet";
+    case "csv":
+      return "text/csv";
+    case "txt":
+      return "text/plain";
+    case "pdf":
+      return "application/pdf";
+    default:
+      return "unknown";
+  }
+}
+async function q(e, t) {
+  const a = e.type !== "" ? e.type : w(e.name);
   if (a === "application/parquet")
-    return l(e, t?.maxSize, t?.column);
+    return d(e, t?.maxSize, t?.column);
+  if (a === "application/pdf")
+    return m(e, t?.maxSize);
   if (a === "text/csv") {
-    const n = "FileReaderSync" in global ? e : await e.text();
-    return new Promise((c, o) => {
+    const n = await e.text();
+    return new Promise((p, o) => {
       s.parse(n, {
         header: !1,
         skipEmptyLines: !0,
@@ -21,8 +41,8 @@ async function w(e, t) {
           if (r.errors.length > 0)
             o(new Error("Error parsing file"));
           else {
-            const i = m(r.data[0], t?.column || "text"), d = t?.hasHeader ?? u(r.data[0]) ? r.data.slice(1) : r.data;
-            c(d.map((p) => p[i]));
+            const c = x(r.data[0], t?.column || "text"), i = t?.hasHeader ?? f(r.data[0]) ? r.data.slice(1) : r.data;
+            p(i.map((u) => u[c]));
           }
         },
         error: (r) => {
@@ -35,5 +55,5 @@ async function w(e, t) {
   throw new Error(`Unsupported file type: ${a}`);
 }
 export {
-  w as default
+  q as default
 };

package/dist/index-D5v913EJ.js

@@ -0,0 +1,4 @@
+const l = globalThis || void 0 || self;
+export {
+  l as g
+};