@genai-fi/nanogpt 0.19.0 → 0.20.0

package/dist/data/textLoader.d.ts

@@ -1,7 +0,0 @@
-import { Conversation } from '../tokeniser/type';
-export interface DataOptions {
-    maxSize?: number;
-    column?: string;
-    hasHeader?: boolean;
-}
-export default function loadTextData(file: File, options?: DataOptions): Promise<Conversation[][]>;

package/dist/data/textLoader.js

@@ -1,108 +0,0 @@
-import { p as u } from "../papaparse.min-C0cScC2i.js";
-import { loadParquet as f } from "./parquet.js";
-import { loadPDF as d } from "./pdf.js";
-import { loadDOCX as x } from "./docx.js";
-import { z as m } from "../jszip.min-BZhlzntC.js";
-function y(t, r) {
-  const n = t.findIndex((i) => i.toLowerCase() === r.toLowerCase());
-  return n === -1 ? 0 : n;
-}
-function w(t) {
-  return t.every((r) => r.length < 64);
-}
-function h(t) {
-  return t.split(".").pop() || "";
-}
-function g(t) {
-  switch (h(t)) {
-    case "json":
-      return "application/json";
-    case "jsonl":
-      return "application/jsonl";
-    case "parquet":
-      return "application/parquet";
-    case "csv":
-      return "text/csv";
-    case "txt":
-      return "text/plain";
-    case "pdf":
-      return "application/pdf";
-    case "docx":
-      return "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
-    case "zip":
-      return "application/zip";
-    default:
-      return "unknown";
-  }
-}
-function z(t) {
-  if (!Array.isArray(t)) return !1;
-  const r = t[0];
-  return typeof r == "object" && r !== null && "role" in r && "content" in r && typeof r.role == "string" && typeof r.content == "string";
-}
-async function j(t, r) {
-  const n = t.type !== "" ? t.type : g(t.name);
-  if (n === "application/parquet")
-    return f(t, r?.maxSize, r?.column);
-  if (n === "application/pdf")
-    return d(t, r?.maxSize);
-  if (n === "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
-    return x(t);
-  if (n === "application/json") {
-    const i = await t.text(), o = JSON.parse(i);
-    if (Array.isArray(o))
-      return o.map((e) => [
-        typeof e == "string" ? { role: "text", content: e } : "text" in e ? { role: "text", content: e.text } : { role: "text", content: JSON.stringify(e) }
-      ]);
-    throw new Error("Expected JSON array");
-  }
-  if (n === "application/jsonl")
-    return (await t.text()).split(`
-`).filter((o) => o.trim() !== "").map((o) => {
-      try {
-        const e = JSON.parse(o);
-        return z(e) ? e : [
-          typeof e == "string" ? { role: "text", content: e } : "text" in e ? { role: "text", content: e.text } : { role: "text", content: JSON.stringify(e) }
-        ];
-      } catch {
-        return [{ role: "text", content: o }];
-      }
-    });
-  if (n === "application/zip") {
-    const i = await m.loadAsync(t), o = [];
-    for (const e of Object.keys(i.files)) {
-      const a = i.file(e);
-      if (a) {
-        const c = await a.async("blob"), s = await j(new File([c], e), r);
-        o.push(...s);
-      }
-    }
-    return o;
-  }
-  if (n === "text/csv") {
-    const i = await t.text();
-    return new Promise((o, e) => {
-      u.parse(i, {
-        header: !1,
-        skipEmptyLines: !0,
-        delimiter: ",",
-        complete: (a) => {
-          if (a.errors.length > 0)
-            console.error(a.errors), e(new Error("Error parsing file"));
-          else {
-            const c = y(a.data[0], r?.column || "text"), p = r?.hasHeader ?? w(a.data[0]) ? a.data.slice(1) : a.data;
-            o(p.map((l) => [{ role: "text", content: l[c] }]));
-          }
-        },
-        error: (a) => {
-          e(a);
-        }
-      });
-    });
-  } else if (n === "text/plain")
-    return [[{ role: "text", content: await t.text() }]];
-  throw new Error(`Unsupported file type: ${n}`);
-}
-export {
-  j as default
-};