@genai-fi/nanogpt 0.2.3 → 0.2.4

package/dist/TeachableLLM.js

@@ -1,6 +1,6 @@
 import { defaultConfig as d } from "./config.js";
-import u from "./NanoGPTModel.js";
-import { saveModel as m } from "./utilities/save.js";
+import m from "./NanoGPTModel.js";
+import { saveModel as u } from "./utilities/save.js";
 import { loadModel as l } from "./utilities/load.js";
 import f from "./Generator.js";
 import _ from "./Trainer.js";
@@ -8,6 +8,7 @@ import { E as c } from "./index-Dwqa6Zy2.js";
 import { dummyPassAsync as h } from "./utilities/dummy.js";
 import g from "./tokeniser/CharTokeniser.js";
 import "./papaparse.min-C8l2Kvo1.js";
+import "./index-Tf7vU29b.js";
 class a extends c {
   _config;
   _model;
@@ -44,7 +45,7 @@ class a extends c {
   saveModel(t) {
     if (!this._model || !this._tokeniser)
       throw new Error("Model or tokeniser is not initialized.");
-    return m(this._model, this._tokeniser, t);
+    return u(this._model, this._tokeniser, t);
   }
   static loadModel(t, r) {
     const e = new a(t);
@@ -59,7 +60,7 @@ class a extends c {
     }), e;
   }
   static create(t, r = {}) {
-    const e = { ...d, ...r }, s = new g(e.vocabSize), o = new u(t, e), i = new a(t, s, o);
+    const e = { ...d, ...r }, s = new g(e.vocabSize), o = new m(t, e), i = new a(t, s, o);
     return i.setStatus("warmup"), h(o).then(() => {
       i.tokeniser.trained ? i.setStatus("ready") : (i.setStatus("awaitingTokens"), i.tokeniser.once("trainStatus", (n) => {
         n === "trained" && i.setStatus("ready");

package/dist/data/parquet.js

@@ -1,12 +1,14 @@
-async function p(s, f = 104857600, e = "text") {
-  const r = await (await import("../parquet-DpcqBLb0.js").then((t) => t.p)).ParquetReader.openBuffer(Buffer.from(await s.arrayBuffer())), a = [], i = r.getCursor([[e]]);
+import { B as n } from "../index-Tf7vU29b.js";
+const p = 100 * 1024 * 1024;
+async function w(s, f = p, r = "text") {
+  const e = await (await import("../parquet-BRl5lE_I.js").then((t) => t.p)).ParquetReader.openBuffer(n.from(await s.arrayBuffer())), a = [], i = e.getCursor([[r]]);
   let o = 0;
   for (; ; ) {
     const t = await i.next();
-    if (!t || !t[e] || typeof t[e] != "string" || (a.push(t[e]), o += t[e].length, o > f)) break;
+    if (!t || !t[r] || typeof t[r] != "string" || (a.push(t[r]), o += t[r].length, o > f)) break;
   }
-  return r.close(), a;
+  return e.close(), a;
 }
 export {
-  p as loadParquet
+  w as loadParquet
 };

package/dist/data/pdf.d.ts

@@ -0,0 +1 @@
+export declare function loadPDF(file: File, maxSize?: number): Promise<string[]>;

package/dist/data/pdf.js

@@ -0,0 +1,12 @@
+async function u(s, i = 104857600) {
+  const e = await (await import("../pdf-kJD-f258.js")).getDocument({ data: await s.arrayBuffer() }).promise, c = e.numPages, a = [];
+  let n = 0;
+  for (let t = 1; t <= c; t++) {
+    const o = (await (await e.getPage(t)).getTextContent()).items.map((g) => g.str).join(" ");
+    if (a.push(o), n += o.length, n > i) break;
+  }
+  return a;
+}
+export {
+  u as loadPDF
+};

package/dist/data/textLoader.js

@@ -1,54 +1,59 @@
-import { p as u } from "../papaparse.min-C8l2Kvo1.js";
+import { p as s } from "../papaparse.min-C8l2Kvo1.js";
 import { loadParquet as d } from "./parquet.js";
+import { loadPDF as m } from "./pdf.js";
 function x(e, t) {
-  const r = e.findIndex((n) => n.toLowerCase() === t.toLowerCase());
-  return r === -1 ? 0 : r;
+  const a = e.findIndex((n) => n.toLowerCase() === t.toLowerCase());
+  return a === -1 ? 0 : a;
 }
-function l(e) {
+function f(e) {
   return e.every((t) => t.length < 64);
 }
-function m(e) {
+function l(e) {
   return e.split(".").pop() || "";
 }
-function f(e) {
-  switch (m(e)) {
+function w(e) {
+  switch (l(e)) {
     case "parquet":
       return "application/parquet";
     case "csv":
       return "text/csv";
     case "txt":
       return "text/plain";
+    case "pdf":
+      return "application/pdf";
     default:
       return "unknown";
   }
 }
-async function g(e, t) {
-  const r = e.type !== "" ? e.type : f(e.name);
-  if (r === "application/parquet")
+async function q(e, t) {
+  const a = e.type !== "" ? e.type : w(e.name);
+  if (a === "application/parquet")
     return d(e, t?.maxSize, t?.column);
-  if (r === "text/csv") {
+  if (a === "application/pdf")
+    return m(e, t?.maxSize);
+  if (a === "text/csv") {
     const n = await e.text();
-    return new Promise((c, o) => {
-      u.parse(n, {
+    return new Promise((p, o) => {
+      s.parse(n, {
         header: !1,
         skipEmptyLines: !0,
-        complete: (a) => {
-          if (a.errors.length > 0)
+        complete: (r) => {
+          if (r.errors.length > 0)
             o(new Error("Error parsing file"));
           else {
-            const p = x(a.data[0], t?.column || "text"), i = t?.hasHeader ?? l(a.data[0]) ? a.data.slice(1) : a.data;
-            c(i.map((s) => s[p]));
+            const c = x(r.data[0], t?.column || "text"), i = t?.hasHeader ?? f(r.data[0]) ? r.data.slice(1) : r.data;
+            p(i.map((u) => u[c]));
           }
         },
-        error: (a) => {
-          o(a);
+        error: (r) => {
+          o(r);
         }
       });
     });
-  } else if (r === "text/plain")
+  } else if (a === "text/plain")
     return [await e.text()];
-  throw new Error(`Unsupported file type: ${r}`);
+  throw new Error(`Unsupported file type: ${a}`);
 }
 export {
-  g as default
+  q as default
 };

package/dist/index-D5v913EJ.js

@@ -0,0 +1,4 @@
+const l = globalThis || void 0 || self;
+export {
+  l as g
+};