@genai-fi/nanogpt 0.19.0 → 0.19.1

package/dist/data/textLoader.d.ts

@@ -4,4 +4,4 @@ export interface DataOptions {
     column?: string;
     hasHeader?: boolean;
 }
-export default function loadTextData(file: File, options?: DataOptions): Promise<Conversation[][]>;
+export default function loadTextData(file: File, options?: DataOptions, cb?: (progress: number) => void): Promise<Conversation[][]>;

package/dist/data/textLoader.js

@@ -1,20 +1,20 @@
-import { p as u } from "../papaparse.min-C0cScC2i.js";
-import { loadParquet as f } from "./parquet.js";
-import { loadPDF as d } from "./pdf.js";
-import { loadDOCX as x } from "./docx.js";
-import { z as m } from "../jszip.min-BZhlzntC.js";
-function y(t, r) {
-  const n = t.findIndex((i) => i.toLowerCase() === r.toLowerCase());
-  return n === -1 ? 0 : n;
+import { p as x } from "../papaparse.min-C0cScC2i.js";
+import { loadParquet as m } from "./parquet.js";
+import { loadPDF as y } from "./pdf.js";
+import { loadDOCX as g } from "./docx.js";
+import { z as w } from "../jszip.min-BZhlzntC.js";
+function h(t, e) {
+  const o = t.findIndex((s) => s.toLowerCase() === e.toLowerCase());
+  return o === -1 ? 0 : o;
 }
-function w(t) {
-  return t.every((r) => r.length < 64);
+function z(t) {
+  return t.every((e) => e.length < 64);
 }
-function h(t) {
+function j(t) {
   return t.split(".").pop() || "";
 }
-function g(t) {
-  switch (h(t)) {
+function v(t) {
+  switch (j(t)) {
     case "json":
       return "application/json";
     case "jsonl":
@@ -35,74 +35,84 @@ function g(t) {
       return "unknown";
   }
 }
-function z(t) {
+function E(t) {
   if (!Array.isArray(t)) return !1;
-  const r = t[0];
-  return typeof r == "object" && r !== null && "role" in r && "content" in r && typeof r.role == "string" && typeof r.content == "string";
+  const e = t[0];
+  return typeof e == "object" && e !== null && "role" in e && "content" in e && typeof e.role == "string" && typeof e.content == "string";
 }
-async function j(t, r) {
-  const n = t.type !== "" ? t.type : g(t.name);
-  if (n === "application/parquet")
-    return f(t, r?.maxSize, r?.column);
-  if (n === "application/pdf")
-    return d(t, r?.maxSize);
-  if (n === "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
-    return x(t);
-  if (n === "application/json") {
-    const i = await t.text(), o = JSON.parse(i);
-    if (Array.isArray(o))
-      return o.map((e) => [
-        typeof e == "string" ? { role: "text", content: e } : "text" in e ? { role: "text", content: e.text } : { role: "text", content: JSON.stringify(e) }
+async function O(t, e, o) {
+  const s = t.type !== "" ? t.type : v(t.name);
+  if (s === "application/parquet")
+    return m(t, e?.maxSize, e?.column);
+  if (s === "application/pdf")
+    return y(t, e?.maxSize);
+  if (s === "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
+    return g(t);
+  if (s === "application/json") {
+    const c = await t.text(), a = JSON.parse(c);
+    if (Array.isArray(a))
+      return a.map((r) => [
+        typeof r == "string" ? { role: "text", content: r } : "text" in r ? { role: "text", content: r.text } : { role: "text", content: JSON.stringify(r) }
       ]);
     throw new Error("Expected JSON array");
   }
-  if (n === "application/jsonl")
-    return (await t.text()).split(`
-`).filter((o) => o.trim() !== "").map((o) => {
+  if (s === "application/jsonl") {
+    const c = await t.text();
+    return o && o(0.1), c.split(`
+`).filter((a) => a.trim() !== "").map((a, r, n) => {
+      o && r % 1e3 === 0 && o(0.1 + r / n.length * 0.9);
       try {
-        const e = JSON.parse(o);
-        return z(e) ? e : [
-          typeof e == "string" ? { role: "text", content: e } : "text" in e ? { role: "text", content: e.text } : { role: "text", content: JSON.stringify(e) }
+        const i = JSON.parse(a);
+        return E(i) ? i : [
+          typeof i == "string" ? { role: "text", content: i } : "text" in i ? { role: "text", content: i.text } : { role: "text", content: JSON.stringify(i) }
         ];
       } catch {
-        return [{ role: "text", content: o }];
+        return [{ role: "text", content: a }];
       }
     });
-  if (n === "application/zip") {
-    const i = await m.loadAsync(t), o = [];
-    for (const e of Object.keys(i.files)) {
-      const a = i.file(e);
-      if (a) {
-        const c = await a.async("blob"), s = await j(new File([c], e), r);
-        o.push(...s);
+  }
+  if (s === "application/zip") {
+    const c = await w.loadAsync(t);
+    let a = [];
+    const r = Object.keys(c.files);
+    for (let n = 0; n < r.length; n++) {
+      const i = r[n], p = c.file(i);
+      if (p) {
+        const l = await p.async("blob", (u) => {
+          if (o) {
+            const d = 0.1 + (u.percent / 100 * 0.9 / r.length + n / r.length * 0.9);
+            o(d);
+          }
+        }), f = await O(new File([l], i), e);
+        o && o(0.1 + (n + 1) / r.length * 0.9), a = a.concat(f);
       }
     }
-    return o;
+    return a;
   }
-  if (n === "text/csv") {
-    const i = await t.text();
-    return new Promise((o, e) => {
-      u.parse(i, {
+  if (s === "text/csv") {
+    const c = await t.text();
+    return o && o(0.1), new Promise((a, r) => {
+      x.parse(c, {
         header: !1,
         skipEmptyLines: !0,
         delimiter: ",",
-        complete: (a) => {
-          if (a.errors.length > 0)
-            console.error(a.errors), e(new Error("Error parsing file"));
+        complete: (n) => {
+          if (n.errors.length > 0)
+            console.error(n.errors), r(new Error("Error parsing file"));
           else {
-            const c = y(a.data[0], r?.column || "text"), p = r?.hasHeader ?? w(a.data[0]) ? a.data.slice(1) : a.data;
-            o(p.map((l) => [{ role: "text", content: l[c] }]));
+            const i = h(n.data[0], e?.column || "text"), l = e?.hasHeader ?? z(n.data[0]) ? n.data.slice(1) : n.data;
+            a(l.map((f) => [{ role: "text", content: f[i] }]));
           }
         },
-        error: (a) => {
-          e(a);
+        error: (n) => {
+          r(n);
         }
       });
     });
-  } else if (n === "text/plain")
+  } else if (s === "text/plain")
     return [[{ role: "text", content: await t.text() }]];
-  throw new Error(`Unsupported file type: ${n}`);
+  throw new Error(`Unsupported file type: ${s}`);
 }
 export {
-  j as default
+  O as default
 };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.19.0",
+    "version": "0.19.1",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",