@genai-fi/nanogpt 0.2.3 → 0.2.5

package/dist/TeachableLLM.js

@@ -1,6 +1,6 @@
 import { defaultConfig as d } from "./config.js";
-import u from "./NanoGPTModel.js";
-import { saveModel as m } from "./utilities/save.js";
+import m from "./NanoGPTModel.js";
+import { saveModel as u } from "./utilities/save.js";
 import { loadModel as l } from "./utilities/load.js";
 import f from "./Generator.js";
 import _ from "./Trainer.js";
@@ -8,6 +8,8 @@ import { E as c } from "./index-Dwqa6Zy2.js";
 import { dummyPassAsync as h } from "./utilities/dummy.js";
 import g from "./tokeniser/CharTokeniser.js";
 import "./papaparse.min-C8l2Kvo1.js";
+import "./index-Tf7vU29b.js";
+import "./jszip.min-CjP2V1VV.js";
 class a extends c {
   _config;
   _model;
@@ -44,7 +46,7 @@ class a extends c {
   saveModel(t) {
     if (!this._model || !this._tokeniser)
       throw new Error("Model or tokeniser is not initialized.");
-    return m(this._model, this._tokeniser, t);
+    return u(this._model, this._tokeniser, t);
   }
   static loadModel(t, r) {
     const e = new a(t);
@@ -59,7 +61,7 @@ class a extends c {
     }), e;
   }
   static create(t, r = {}) {
-    const e = { ...d, ...r }, s = new g(e.vocabSize), o = new u(t, e), i = new a(t, s, o);
+    const e = { ...d, ...r }, s = new g(e.vocabSize), o = new m(t, e), i = new a(t, s, o);
     return i.setStatus("warmup"), h(o).then(() => {
       i.tokeniser.trained ? i.setStatus("ready") : (i.setStatus("awaitingTokens"), i.tokeniser.once("trainStatus", (n) => {
         n === "trained" && i.setStatus("ready");

package/dist/data/docx.d.ts

@@ -0,0 +1 @@
+export declare function loadDOCX(file: Blob | Uint8Array): Promise<string[]>;

package/dist/data/docx.js

@@ -0,0 +1,15 @@
+import { j as a } from "../jszip.min-CjP2V1VV.js";
+async function c(n) {
+  const t = await (await a.loadAsync(n)).file("word/document.xml")?.async("string");
+  if (!t) throw new Error("Failed to load document.xml");
+  return s(t).split(`
+`).filter((r) => r.trim().length > 10);
+}
+function s(n) {
+  const t = new DOMParser().parseFromString(n, "application/xml");
+  return Array.from(t.getElementsByTagName("w:t")).map((r) => r.textContent).join(`
+`);
+}
+export {
+  c as loadDOCX
+};

package/dist/data/parquet.js

@@ -1,12 +1,14 @@
-async function p(s, f = 104857600, e = "text") {
-  const r = await (await import("../parquet-DpcqBLb0.js").then((t) => t.p)).ParquetReader.openBuffer(Buffer.from(await s.arrayBuffer())), a = [], i = r.getCursor([[e]]);
+import { B as n } from "../index-Tf7vU29b.js";
+const p = 100 * 1024 * 1024;
+async function w(s, f = p, r = "text") {
+  const e = await (await import("../parquet-BRl5lE_I.js").then((t) => t.p)).ParquetReader.openBuffer(n.from(await s.arrayBuffer())), a = [], i = e.getCursor([[r]]);
   let o = 0;
   for (; ; ) {
     const t = await i.next();
-    if (!t || !t[e] || typeof t[e] != "string" || (a.push(t[e]), o += t[e].length, o > f)) break;
+    if (!t || !t[r] || typeof t[r] != "string" || (a.push(t[r]), o += t[r].length, o > f)) break;
   }
-  return r.close(), a;
+  return e.close(), a;
 }
 export {
-  p as loadParquet
+  w as loadParquet
 };

package/dist/data/pdf.d.ts

@@ -0,0 +1 @@
+export declare function loadPDF(file: Blob | Uint8Array, maxSize?: number): Promise<string[]>;