@genai-fi/nanogpt 0.0.1

package/dist/utilities/textLoader.js

@@ -0,0 +1,438 @@
+import { g as ke } from "../_commonjsHelpers-DaMA6jEr.js";
+var ne = { exports: {} };
+/* @license
+Papa Parse
+v5.5.3
+https://github.com/mholt/PapaParse
+License: MIT
+*/
+var ve = ne.exports, ge;
+function Ee() {
+  return ge || (ge = 1, function(se, fe) {
+    ((J, y) => {
+      se.exports = y();
+    })(ve, function J() {
+      var y = typeof self < "u" ? self : typeof window < "u" ? window : y !== void 0 ? y : {}, H, Z = !y.document && !!y.postMessage, ae = y.IS_PAPA_WORKER || !1, ee = {}, me = 0, u = {};
+      function N(e) {
+        this._handle = null, this._finished = !1, this._completed = !1, this._halted = !1, this._input = null, this._baseIndex = 0, this._partialLine = "", this._rowCount = 0, this._start = 0, this._nextChunk = null, this.isFirstChunk = !0, this._completeResults = { data: [], errors: [], meta: {} }, (function(t) {
+          var r = ue(t);
+          r.chunkSize = parseInt(r.chunkSize), t.step || t.chunk || (r.chunkSize = null), this._handle = new le(r), (this._handle.streamer = this)._config = r;
+        }).call(this, e), this.parseChunk = function(t, r) {
+          var n = parseInt(this._config.skipFirstNLines) || 0;
+          if (this.isFirstChunk && 0 < n) {
+            let l = this._config.newline;
+            l || (i = this._config.quoteChar || '"', l = this._handle.guessLineEndings(t, i)), t = [...t.split(l).slice(n)].join(l);
+          }
+          this.isFirstChunk && k(this._config.beforeFirstChunk) && (i = this._config.beforeFirstChunk(t)) !== void 0 && (t = i), this.isFirstChunk = !1, this._halted = !1;
+          var n = this._partialLine + t, i = (this._partialLine = "", this._handle.parse(n, this._baseIndex, !this._finished));
+          if (!this._handle.paused() && !this._handle.aborted()) {
+            if (t = i.meta.cursor, n = (this._finished || (this._partialLine = n.substring(t - this._baseIndex), this._baseIndex = t), i && i.data && (this._rowCount += i.data.length), this._finished || this._config.preview && this._rowCount >= this._config.preview), ae) y.postMessage({ results: i, workerId: u.WORKER_ID, finished: n });
+            else if (k(this._config.chunk) && !r) {
+              if (this._config.chunk(i, this._handle), this._handle.paused() || this._handle.aborted()) return void (this._halted = !0);
+              this._completeResults = i = void 0;
+            }
+            return this._config.step || this._config.chunk || (this._completeResults.data = this._completeResults.data.concat(i.data), this._completeResults.errors = this._completeResults.errors.concat(i.errors), this._completeResults.meta = i.meta), this._completed || !n || !k(this._config.complete) || i && i.meta.aborted || (this._config.complete(this._completeResults, this._input), this._completed = !0), n || i && i.meta.paused || this._nextChunk(), i;
+          }
+          this._halted = !0;
+        }, this._sendError = function(t) {
+          k(this._config.error) ? this._config.error(t) : ae && this._config.error && y.postMessage({ workerId: u.WORKER_ID, error: t, finished: !1 });
+        };
+      }
+      function te(e) {
+        var t;
+        (e = e || {}).chunkSize || (e.chunkSize = u.RemoteChunkSize), N.call(this, e), this._nextChunk = Z ? function() {
+          this._readChunk(), this._chunkLoaded();
+        } : function() {
+          this._readChunk();
+        }, this.stream = function(r) {
+          this._input = r, this._nextChunk();
+        }, this._readChunk = function() {
+          if (this._finished) this._chunkLoaded();
+          else {
+            if (t = new XMLHttpRequest(), this._config.withCredentials && (t.withCredentials = this._config.withCredentials), Z || (t.onload = K(this._chunkLoaded, this), t.onerror = K(this._chunkError, this)), t.open(this._config.downloadRequestBody ? "POST" : "GET", this._input, !Z), this._config.downloadRequestHeaders) {
+              var r, n = this._config.downloadRequestHeaders;
+              for (r in n) t.setRequestHeader(r, n[r]);
+            }
+            var i;
+            this._config.chunkSize && (i = this._start + this._config.chunkSize - 1, t.setRequestHeader("Range", "bytes=" + this._start + "-" + i));
+            try {
+              t.send(this._config.downloadRequestBody);
+            } catch (l) {
+              this._chunkError(l.message);
+            }
+            Z && t.status === 0 && this._chunkError();
+          }
+        }, this._chunkLoaded = function() {
+          t.readyState === 4 && (t.status < 200 || 400 <= t.status ? this._chunkError() : (this._start += this._config.chunkSize || t.responseText.length, this._finished = !this._config.chunkSize || this._start >= ((r) => (r = r.getResponseHeader("Content-Range")) !== null ? parseInt(r.substring(r.lastIndexOf("/") + 1)) : -1)(t), this.parseChunk(t.responseText)));
+        }, this._chunkError = function(r) {
+          r = t.statusText || r, this._sendError(new Error(r));
+        };
+      }
+      function re(e) {
+        (e = e || {}).chunkSize || (e.chunkSize = u.LocalChunkSize), N.call(this, e);
+        var t, r, n = typeof FileReader < "u";
+        this.stream = function(i) {
+          this._input = i, r = i.slice || i.webkitSlice || i.mozSlice, n ? ((t = new FileReader()).onload = K(this._chunkLoaded, this), t.onerror = K(this._chunkError, this)) : t = new FileReaderSync(), this._nextChunk();
+        }, this._nextChunk = function() {
+          this._finished || this._config.preview && !(this._rowCount < this._config.preview) || this._readChunk();
+        }, this._readChunk = function() {
+          var i = this._input, l = (this._config.chunkSize && (l = Math.min(this._start + this._config.chunkSize, this._input.size), i = r.call(i, this._start, l)), t.readAsText(i, this._config.encoding));
+          n || this._chunkLoaded({ target: { result: l } });
+        }, this._chunkLoaded = function(i) {
+          this._start += this._config.chunkSize, this._finished = !this._config.chunkSize || this._start >= this._input.size, this.parseChunk(i.target.result);
+        }, this._chunkError = function() {
+          this._sendError(t.error);
+        };
+      }
+      function G(e) {
+        var t;
+        N.call(this, e = e || {}), this.stream = function(r) {
+          return t = r, this._nextChunk();
+        }, this._nextChunk = function() {
+          var r, n;
+          if (!this._finished) return r = this._config.chunkSize, t = r ? (n = t.substring(0, r), t.substring(r)) : (n = t, ""), this._finished = !t, this.parseChunk(n);
+        };
+      }
+      function ie(e) {
+        N.call(this, e = e || {});
+        var t = [], r = !0, n = !1;
+        this.pause = function() {
+          N.prototype.pause.apply(this, arguments), this._input.pause();
+        }, this.resume = function() {
+          N.prototype.resume.apply(this, arguments), this._input.resume();
+        }, this.stream = function(i) {
+          this._input = i, this._input.on("data", this._streamData), this._input.on("end", this._streamEnd), this._input.on("error", this._streamError);
+        }, this._checkIsFinished = function() {
+          n && t.length === 1 && (this._finished = !0);
+        }, this._nextChunk = function() {
+          this._checkIsFinished(), t.length ? this.parseChunk(t.shift()) : r = !0;
+        }, this._streamData = K(function(i) {
+          try {
+            t.push(typeof i == "string" ? i : i.toString(this._config.encoding)), r && (r = !1, this._checkIsFinished(), this.parseChunk(t.shift()));
+          } catch (l) {
+            this._streamError(l);
+          }
+        }, this), this._streamError = K(function(i) {
+          this._streamCleanUp(), this._sendError(i);
+        }, this), this._streamEnd = K(function() {
+          this._streamCleanUp(), n = !0, this._streamData("");
+        }, this), this._streamCleanUp = K(function() {
+          this._input.removeListener("data", this._streamData), this._input.removeListener("end", this._streamEnd), this._input.removeListener("error", this._streamError);
+        }, this);
+      }
+      function le(e) {
+        var t, r, n, i, l = Math.pow(2, 53), x = -l, F = /^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/, j = /^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/, d = this, w = 0, a = 0, T = !1, h = !1, c = [], s = { data: [], errors: [], meta: {} };
+        function C(p) {
+          return e.skipEmptyLines === "greedy" ? p.join("").trim() === "" : p.length === 1 && p[0].length === 0;
+        }
+        function S() {
+          if (s && n && (z("Delimiter", "UndetectableDelimiter", "Unable to auto-detect delimiting character; defaulted to '" + u.DefaultDelimiter + "'"), n = !1), e.skipEmptyLines && (s.data = s.data.filter(function(o) {
+            return !C(o);
+          })), A()) {
+            let o = function(R, O) {
+              k(e.transformHeader) && (R = e.transformHeader(R, O)), c.push(R);
+            };
+            if (s) if (Array.isArray(s.data[0])) {
+              for (var p = 0; A() && p < s.data.length; p++) s.data[p].forEach(o);
+              s.data.splice(0, 1);
+            } else s.data.forEach(o);
+          }
+          function g(o, R) {
+            for (var O = e.header ? {} : [], m = 0; m < o.length; m++) {
+              var v = m, _ = o[m], _ = ((M, f) => ((E) => (e.dynamicTypingFunction && e.dynamicTyping[E] === void 0 && (e.dynamicTyping[E] = e.dynamicTypingFunction(E)), (e.dynamicTyping[E] || e.dynamicTyping) === !0))(M) ? f === "true" || f === "TRUE" || f !== "false" && f !== "FALSE" && (((E) => {
+                if (F.test(E) && (E = parseFloat(E), x < E && E < l))
+                  return 1;
+              })(f) ? parseFloat(f) : j.test(f) ? new Date(f) : f === "" ? null : f) : f)(v = e.header ? m >= c.length ? "__parsed_extra" : c[m] : v, _ = e.transform ? e.transform(_, v) : _);
+              v === "__parsed_extra" ? (O[v] = O[v] || [], O[v].push(_)) : O[v] = _;
+            }
+            return e.header && (m > c.length ? z("FieldMismatch", "TooManyFields", "Too many fields: expected " + c.length + " fields but parsed " + m, a + R) : m < c.length && z("FieldMismatch", "TooFewFields", "Too few fields: expected " + c.length + " fields but parsed " + m, a + R)), O;
+          }
+          var b;
+          s && (e.header || e.dynamicTyping || e.transform) && (b = 1, !s.data.length || Array.isArray(s.data[0]) ? (s.data = s.data.map(g), b = s.data.length) : s.data = g(s.data, 0), e.header && s.meta && (s.meta.fields = c), a += b);
+        }
+        function A() {
+          return e.header && c.length === 0;
+        }
+        function z(p, g, b, o) {
+          p = { type: p, code: g, message: b }, o !== void 0 && (p.row = o), s.errors.push(p);
+        }
+        k(e.step) && (i = e.step, e.step = function(p) {
+          s = p, A() ? S() : (S(), s.data.length !== 0 && (w += p.data.length, e.preview && w > e.preview ? r.abort() : (s.data = s.data[0], i(s, d))));
+        }), this.parse = function(p, g, b) {
+          var o = e.quoteChar || '"', o = (e.newline || (e.newline = this.guessLineEndings(p, o)), n = !1, e.delimiter ? k(e.delimiter) && (e.delimiter = e.delimiter(p), s.meta.delimiter = e.delimiter) : ((o = ((R, O, m, v, _) => {
+            var M, f, E, W;
+            _ = _ || [",", "	", "|", ";", u.RECORD_SEP, u.UNIT_SEP];
+            for (var Q = 0; Q < _.length; Q++) {
+              for (var P, X = _[Q], D = 0, U = 0, I = 0, L = (E = void 0, new oe({ comments: v, delimiter: X, newline: O, preview: 10 }).parse(R)), B = 0; B < L.data.length; B++) m && C(L.data[B]) ? I++ : (P = L.data[B].length, U += P, E === void 0 ? E = P : 0 < P && (D += Math.abs(P - E), E = P));
+              0 < L.data.length && (U /= L.data.length - I), (f === void 0 || D <= f) && (W === void 0 || W < U) && 1.99 < U && (f = D, M = X, W = U);
+            }
+            return { successful: !!(e.delimiter = M), bestDelimiter: M };
+          })(p, e.newline, e.skipEmptyLines, e.comments, e.delimitersToGuess)).successful ? e.delimiter = o.bestDelimiter : (n = !0, e.delimiter = u.DefaultDelimiter), s.meta.delimiter = e.delimiter), ue(e));
+          return e.preview && e.header && o.preview++, t = p, r = new oe(o), s = r.parse(t, g, b), S(), T ? { meta: { paused: !0 } } : s || { meta: { paused: !1 } };
+        }, this.paused = function() {
+          return T;
+        }, this.pause = function() {
+          T = !0, r.abort(), t = k(e.chunk) ? "" : t.substring(r.getCharIndex());
+        }, this.resume = function() {
+          d.streamer._halted ? (T = !1, d.streamer.parseChunk(t, !0)) : setTimeout(d.resume, 3);
+        }, this.aborted = function() {
+          return h;
+        }, this.abort = function() {
+          h = !0, r.abort(), s.meta.aborted = !0, k(e.complete) && e.complete(s), t = "";
+        }, this.guessLineEndings = function(R, o) {
+          R = R.substring(0, 1048576);
+          var o = new RegExp(Y(o) + "([^]*?)" + Y(o), "gm"), b = (R = R.replace(o, "")).split("\r"), o = R.split(`
+`), R = 1 < o.length && o[0].length < b[0].length;
+          if (b.length === 1 || R) return `
+`;
+          for (var O = 0, m = 0; m < b.length; m++) b[m][0] === `
+` && O++;
+          return O >= b.length / 2 ? `\r
+` : "\r";
+        };
+      }
+      function Y(e) {
+        return e.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+      }
+      function oe(e) {
+        var t = (e = e || {}).delimiter, r = e.newline, n = e.comments, i = e.step, l = e.preview, x = e.fastMode, F = null, j = !1, d = e.quoteChar == null ? '"' : e.quoteChar, w = d;
+        if (e.escapeChar !== void 0 && (w = e.escapeChar), (typeof t != "string" || -1 < u.BAD_DELIMITERS.indexOf(t)) && (t = ","), n === t) throw new Error("Comment character same as delimiter");
+        n === !0 ? n = "#" : (typeof n != "string" || -1 < u.BAD_DELIMITERS.indexOf(n)) && (n = !1), r !== `
+` && r !== "\r" && r !== `\r
+` && (r = `
+`);
+        var a = 0, T = !1;
+        this.parse = function(h, c, s) {
+          if (typeof h != "string") throw new Error("Input must be a string");
+          var C = h.length, S = t.length, A = r.length, z = n.length, p = k(i), g = [], b = [], o = [], R = a = 0;
+          if (!h) return D();
+          if (x || x !== !1 && h.indexOf(d) === -1) {
+            for (var O = h.split(r), m = 0; m < O.length; m++) {
+              if (o = O[m], a += o.length, m !== O.length - 1) a += r.length;
+              else if (s) return D();
+              if (!n || o.substring(0, z) !== n) {
+                if (p) {
+                  if (g = [], W(o.split(t)), U(), T) return D();
+                } else W(o.split(t));
+                if (l && l <= m) return g = g.slice(0, l), D(!0);
+              }
+            }
+            return D();
+          }
+          for (var v = h.indexOf(t, a), _ = h.indexOf(r, a), M = new RegExp(Y(w) + Y(d), "g"), f = h.indexOf(d, a); ; ) if (h[a] === d) for (f = a, a++; ; ) {
+            if ((f = h.indexOf(d, f + 1)) === -1) return s || b.push({ type: "Quotes", code: "MissingQuotes", message: "Quoted field unterminated", row: g.length, index: a }), P();
+            if (f === C - 1) return P(h.substring(a, f).replace(M, d));
+            if (d === w && h[f + 1] === w) f++;
+            else if (d === w || f === 0 || h[f - 1] !== w) {
+              v !== -1 && v < f + 1 && (v = h.indexOf(t, f + 1));
+              var E = Q((_ = _ !== -1 && _ < f + 1 ? h.indexOf(r, f + 1) : _) === -1 ? v : Math.min(v, _));
+              if (h.substr(f + 1 + E, S) === t) {
+                o.push(h.substring(a, f).replace(M, d)), h[a = f + 1 + E + S] !== d && (f = h.indexOf(d, a)), v = h.indexOf(t, a), _ = h.indexOf(r, a);
+                break;
+              }
+              if (E = Q(_), h.substring(f + 1 + E, f + 1 + E + A) === r) {
+                if (o.push(h.substring(a, f).replace(M, d)), X(f + 1 + E + A), v = h.indexOf(t, a), f = h.indexOf(d, a), p && (U(), T)) return D();
+                if (l && g.length >= l) return D(!0);
+                break;
+              }
+              b.push({ type: "Quotes", code: "InvalidQuotes", message: "Trailing quote on quoted field is malformed", row: g.length, index: a }), f++;
+            }
+          }
+          else if (n && o.length === 0 && h.substring(a, a + z) === n) {
+            if (_ === -1) return D();
+            a = _ + A, _ = h.indexOf(r, a), v = h.indexOf(t, a);
+          } else if (v !== -1 && (v < _ || _ === -1)) o.push(h.substring(a, v)), a = v + S, v = h.indexOf(t, a);
+          else {
+            if (_ === -1) break;
+            if (o.push(h.substring(a, _)), X(_ + A), p && (U(), T)) return D();
+            if (l && g.length >= l) return D(!0);
+          }
+          return P();
+          function W(I) {
+            g.push(I), R = a;
+          }
+          function Q(I) {
+            var L = 0;
+            return L = I !== -1 && (I = h.substring(f + 1, I)) && I.trim() === "" ? I.length : L;
+          }
+          function P(I) {
+            return s || (I === void 0 && (I = h.substring(a)), o.push(I), a = C, W(o), p && U()), D();
+          }
+          function X(I) {
+            a = I, W(o), o = [], _ = h.indexOf(r, a);
+          }
+          function D(I) {
+            if (e.header && !c && g.length && !j) {
+              var L = g[0], B = /* @__PURE__ */ Object.create(null), he = new Set(L);
+              let pe = !1;
+              for (let $ = 0; $ < L.length; $++) {
+                let q = L[$];
+                if (B[q = k(e.transformHeader) ? e.transformHeader(q, $) : q]) {
+                  let V, _e = B[q];
+                  for (; V = q + "_" + _e, _e++, he.has(V); ) ;
+                  he.add(V), L[$] = V, B[q]++, pe = !0, (F = F === null ? {} : F)[V] = q;
+                } else B[q] = 1, L[$] = q;
+                he.add(q);
+              }
+              pe && console.warn("Duplicate headers found and renamed."), j = !0;
+            }
+            return { data: g, errors: b, meta: { delimiter: t, linebreak: r, aborted: T, truncated: !!I, cursor: R + (c || 0), renamedHeaders: F } };
+          }
+          function U() {
+            i(D()), g = [], b = [];
+          }
+        }, this.abort = function() {
+          T = !0;
+        }, this.getCharIndex = function() {
+          return a;
+        };
+      }
+      function ye(e) {
+        var t = e.data, r = ee[t.workerId], n = !1;
+        if (t.error) r.userError(t.error, t.file);
+        else if (t.results && t.results.data) {
+          var i = { abort: function() {
+            n = !0, de(t.workerId, { data: [], errors: [], meta: { aborted: !0 } });
+          }, pause: ce, resume: ce };
+          if (k(r.userStep)) {
+            for (var l = 0; l < t.results.data.length && (r.userStep({ data: t.results.data[l], errors: t.results.errors, meta: t.results.meta }, i), !n); l++) ;
+            delete t.results;
+          } else k(r.userChunk) && (r.userChunk(t.results, i, t.file), delete t.results);
+        }
+        t.finished && !n && de(t.workerId, t.results);
+      }
+      function de(e, t) {
+        var r = ee[e];
+        k(r.userComplete) && r.userComplete(t), r.terminate(), delete ee[e];
+      }
+      function ce() {
+        throw new Error("Not implemented.");
+      }
+      function ue(e) {
+        if (typeof e != "object" || e === null) return e;
+        var t, r = Array.isArray(e) ? [] : {};
+        for (t in e) r[t] = ue(e[t]);
+        return r;
+      }
+      function K(e, t) {
+        return function() {
+          e.apply(t, arguments);
+        };
+      }
+      function k(e) {
+        return typeof e == "function";
+      }
+      return u.parse = function(e, t) {
+        var r = (t = t || {}).dynamicTyping || !1;
+        if (k(r) && (t.dynamicTypingFunction = r, r = {}), t.dynamicTyping = r, t.transform = !!k(t.transform) && t.transform, !t.worker || !u.WORKERS_SUPPORTED) return r = null, u.NODE_STREAM_INPUT, typeof e == "string" ? (e = ((n) => n.charCodeAt(0) !== 65279 ? n : n.slice(1))(e), r = new (t.download ? te : G)(t)) : e.readable === !0 && k(e.read) && k(e.on) ? r = new ie(t) : (y.File && e instanceof File || e instanceof Object) && (r = new re(t)), r.stream(e);
+        (r = (() => {
+          var n;
+          return !!u.WORKERS_SUPPORTED && (n = (() => {
+            var i = y.URL || y.webkitURL || null, l = J.toString();
+            return u.BLOB_URL || (u.BLOB_URL = i.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ", "(", l, ")();"], { type: "text/javascript" })));
+          })(), (n = new y.Worker(n)).onmessage = ye, n.id = me++, ee[n.id] = n);
+        })()).userStep = t.step, r.userChunk = t.chunk, r.userComplete = t.complete, r.userError = t.error, t.step = k(t.step), t.chunk = k(t.chunk), t.complete = k(t.complete), t.error = k(t.error), delete t.worker, r.postMessage({ input: e, config: t, workerId: r.id });
+      }, u.unparse = function(e, t) {
+        var r = !1, n = !0, i = ",", l = `\r
+`, x = '"', F = x + x, j = !1, d = null, w = !1, a = ((() => {
+          if (typeof t == "object") {
+            if (typeof t.delimiter != "string" || u.BAD_DELIMITERS.filter(function(c) {
+              return t.delimiter.indexOf(c) !== -1;
+            }).length || (i = t.delimiter), typeof t.quotes != "boolean" && typeof t.quotes != "function" && !Array.isArray(t.quotes) || (r = t.quotes), typeof t.skipEmptyLines != "boolean" && typeof t.skipEmptyLines != "string" || (j = t.skipEmptyLines), typeof t.newline == "string" && (l = t.newline), typeof t.quoteChar == "string" && (x = t.quoteChar), typeof t.header == "boolean" && (n = t.header), Array.isArray(t.columns)) {
+              if (t.columns.length === 0) throw new Error("Option columns is empty");
+              d = t.columns;
+            }
+            t.escapeChar !== void 0 && (F = t.escapeChar + x), t.escapeFormulae instanceof RegExp ? w = t.escapeFormulae : typeof t.escapeFormulae == "boolean" && t.escapeFormulae && (w = /^[=+\-@\t\r].*$/);
+          }
+        })(), new RegExp(Y(x), "g"));
+        if (typeof e == "string" && (e = JSON.parse(e)), Array.isArray(e)) {
+          if (!e.length || Array.isArray(e[0])) return T(null, e, j);
+          if (typeof e[0] == "object") return T(d || Object.keys(e[0]), e, j);
+        } else if (typeof e == "object") return typeof e.data == "string" && (e.data = JSON.parse(e.data)), Array.isArray(e.data) && (e.fields || (e.fields = e.meta && e.meta.fields || d), e.fields || (e.fields = Array.isArray(e.data[0]) ? e.fields : typeof e.data[0] == "object" ? Object.keys(e.data[0]) : []), Array.isArray(e.data[0]) || typeof e.data[0] == "object" || (e.data = [e.data])), T(e.fields || [], e.data || [], j);
+        throw new Error("Unable to serialize unrecognized input");
+        function T(c, s, C) {
+          var S = "", A = (typeof c == "string" && (c = JSON.parse(c)), typeof s == "string" && (s = JSON.parse(s)), Array.isArray(c) && 0 < c.length), z = !Array.isArray(s[0]);
+          if (A && n) {
+            for (var p = 0; p < c.length; p++) 0 < p && (S += i), S += h(c[p], p);
+            0 < s.length && (S += l);
+          }
+          for (var g = 0; g < s.length; g++) {
+            var b = (A ? c : s[g]).length, o = !1, R = A ? Object.keys(s[g]).length === 0 : s[g].length === 0;
+            if (C && !A && (o = C === "greedy" ? s[g].join("").trim() === "" : s[g].length === 1 && s[g][0].length === 0), C === "greedy" && A) {
+              for (var O = [], m = 0; m < b; m++) {
+                var v = z ? c[m] : m;
+                O.push(s[g][v]);
+              }
+              o = O.join("").trim() === "";
+            }
+            if (!o) {
+              for (var _ = 0; _ < b; _++) {
+                0 < _ && !R && (S += i);
+                var M = A && z ? c[_] : _;
+                S += h(s[g][M], _);
+              }
+              g < s.length - 1 && (!C || 0 < b && !R) && (S += l);
+            }
+          }
+          return S;
+        }
+        function h(c, s) {
+          var C, S;
+          return c == null ? "" : c.constructor === Date ? JSON.stringify(c).slice(1, 25) : (S = !1, w && typeof c == "string" && w.test(c) && (c = "'" + c, S = !0), C = c.toString().replace(a, F), (S = S || r === !0 || typeof r == "function" && r(c, s) || Array.isArray(r) && r[s] || ((A, z) => {
+            for (var p = 0; p < z.length; p++) if (-1 < A.indexOf(z[p])) return !0;
+            return !1;
+          })(C, u.BAD_DELIMITERS) || -1 < C.indexOf(i) || C.charAt(0) === " " || C.charAt(C.length - 1) === " ") ? x + C + x : C);
+        }
+      }, u.RECORD_SEP = "", u.UNIT_SEP = "", u.BYTE_ORDER_MARK = "\uFEFF", u.BAD_DELIMITERS = ["\r", `
+`, '"', u.BYTE_ORDER_MARK], u.WORKERS_SUPPORTED = !Z && !!y.Worker, u.NODE_STREAM_INPUT = 1, u.LocalChunkSize = 10485760, u.RemoteChunkSize = 5242880, u.DefaultDelimiter = ",", u.Parser = oe, u.ParserHandle = le, u.NetworkStreamer = te, u.FileStreamer = re, u.StringStreamer = G, u.ReadableStreamStreamer = ie, y.jQuery && ((H = y.jQuery).fn.parse = function(e) {
+        var t = e.config || {}, r = [];
+        return this.each(function(l) {
+          if (!(H(this).prop("tagName").toUpperCase() === "INPUT" && H(this).attr("type").toLowerCase() === "file" && y.FileReader) || !this.files || this.files.length === 0) return !0;
+          for (var x = 0; x < this.files.length; x++) r.push({ file: this.files[x], inputElem: this, instanceConfig: H.extend({}, t) });
+        }), n(), this;
+        function n() {
+          if (r.length === 0) k(e.complete) && e.complete();
+          else {
+            var l, x, F, j, d = r[0];
+            if (k(e.before)) {
+              var w = e.before(d.file, d.inputElem);
+              if (typeof w == "object") {
+                if (w.action === "abort") return l = "AbortError", x = d.file, F = d.inputElem, j = w.reason, void (k(e.error) && e.error({ name: l }, x, F, j));
+                if (w.action === "skip") return void i();
+                typeof w.config == "object" && (d.instanceConfig = H.extend(d.instanceConfig, w.config));
+              } else if (w === "skip") return void i();
+            }
+            var a = d.instanceConfig.complete;
+            d.instanceConfig.complete = function(T) {
+              k(a) && a(T, d.file, d.inputElem), i();
+            }, u.parse(d.file, d.instanceConfig);
+          }
+        }
+        function i() {
+          r.splice(0, 1), n();
+        }
+      }), ae && (y.onmessage = function(e) {
+        e = e.data, u.WORKER_ID === void 0 && e && (u.WORKER_ID = e.workerId), typeof e.input == "string" ? y.postMessage({ workerId: u.WORKER_ID, results: u.parse(e.input, e.config), finished: !0 }) : (y.File && e.input instanceof File || e.input instanceof Object) && (e = u.parse(e.input, e.config)) && y.postMessage({ workerId: u.WORKER_ID, results: e, finished: !0 });
+      }), (te.prototype = Object.create(N.prototype)).constructor = te, (re.prototype = Object.create(N.prototype)).constructor = re, (G.prototype = Object.create(G.prototype)).constructor = G, (ie.prototype = Object.create(N.prototype)).constructor = ie, u;
+    });
+  }(ne)), ne.exports;
+}
+var we = Ee();
+const be = /* @__PURE__ */ ke(we);
+async function Se(se) {
+  return new Promise((fe, J) => {
+    be.parse(se, {
+      header: !1,
+      skipEmptyLines: !0,
+      complete: (y) => {
+        y.errors.length > 0 ? J(new Error("Error parsing file")) : fe(y.data.slice(1).map((H) => H[0]));
+      },
+      error: (y) => {
+        J(y);
+      }
+    });
+  });
+}
+export {
+  Se as default
+};

package/dist/utilities/tokenParse.d.ts

@@ -0,0 +1 @@
+export default function parseTokens(text: string, raw?: boolean): string[];

package/dist/utilities/tokenParse.js

@@ -0,0 +1,66 @@
+function o(a, c) {
+  a.length !== 0 && c.push(` ${a.trim()}`);
+}
+function l(a, c) {
+  const r = c ? a : a.toLocaleLowerCase(), s = [];
+  let e = "";
+  for (let t = 0; t < r.length; t++) {
+    const n = r[t];
+    switch (n) {
+      case "0":
+      case "1":
+      case "2":
+      case "3":
+      case "4":
+      case "5":
+      case "6":
+      case "7":
+      case "8":
+      case "9":
+      case ":":
+      case ";":
+      case ",":
+      case ".":
+      case "?":
+      case "!":
+      case '"':
+      case "'":
+      case "`":
+      case "(":
+      case ")":
+      case "[":
+      case "]":
+      case "{":
+      case "}":
+      case "-":
+      case "_":
+      case "/":
+      case "\\":
+      case "%":
+      case "<":
+      case ">":
+      case "=":
+      case "+":
+      case "*":
+      case "&":
+      case "^":
+      case "|":
+      case "~":
+      case "@":
+      case "#":
+      case "$":
+        c ? s.push(e) : o(e, s), s.push(n), e = "";
+        break;
+      case " ":
+        c ? s.push(e) : o(e, s), e = n;
+        break;
+      default:
+        e += n;
+        break;
+    }
+  }
+  return e.length > 0 && (c ? s.push(e) : o(e, s)), s;
+}
+export {
+  l as default
+};

package/dist/utilities/weights.d.ts

@@ -0,0 +1,12 @@
+import { default as TF } from '@tensorflow/tfjs';
+export interface ITensorSpec {
+    shape: number[];
+    min?: number;
+    scale?: number;
+}
+export interface IWeightManifest {
+    spec: ITensorSpec[];
+    data: Float32Array;
+}
+export declare function exportWeights(weights: TF.Tensor[]): Promise<IWeightManifest>;
+export declare function importWeights(manifest: IWeightManifest, tf: typeof TF): Promise<TF.Tensor[]>;

package/dist/utilities/weights.js

@@ -0,0 +1,43 @@
+function h(e) {
+  const n = e.reduce((s, o) => s + o.length, 0), a = new Float32Array(n);
+  let t = 0;
+  for (const s of e)
+    a.set(s, t), t += s.length;
+  return a;
+}
+async function f(e) {
+  const n = {
+    spec: [],
+    data: new Float32Array()
+  }, a = [];
+  for (const t of e) {
+    if (!t || !Array.isArray(t.shape) || t.shape.length === 0) {
+      console.warn("Skipping weight with invalid shape:", t);
+      continue;
+    }
+    const s = t.min(), o = t.max(), c = (await s.data())[0], r = (await o.data())[0] - c;
+    n.spec.push({
+      shape: t.shape,
+      min: c,
+      scale: r
+    }), s.dispose(), o.dispose();
+    const i = await t.data();
+    a.push(i);
+  }
+  return n.data = h(a), n;
+}
+async function l(e, n) {
+  const a = [];
+  let t = 0;
+  for (const s of e.spec) {
+    const o = s.shape.reduce((i, p) => i * p, 1), c = e.data.slice(t, t + o);
+    t += o;
+    const r = n.tensor(c, s.shape, "float32");
+    a.push(r);
+  }
+  return a;
+}
+export {
+  f as exportWeights,
+  l as importWeights
+};

package/package.json

@@ -0,0 +1,59 @@
+{
+    "name": "@genai-fi/nanogpt",
+    "version": "0.0.1",
+    "type": "module",
+    "main": "dist/main.js",
+    "types": "dist/main.d.ts",
+    "files": [
+        "dist"
+    ],
+    "exports": {
+        ".": "./dist/main.js"
+    },
+    "repository": {
+        "url": "https://github.com/knicos/genai-nanogpt.git",
+        "type": "git"
+    },
+    "scripts": {
+        "dev": "vite",
+        "build": "tsc && vite build",
+        "build:script": "tsc -p scripts/tsconfig.json",
+        "preview": "vite preview",
+        "lint": "eslint",
+        "test": "vitest",
+        "ci:test": "vitest --coverage --reporter=junit --outputFile=junit.xml",
+        "coverage": "vitest run --coverage",
+        "train": "tsx scripts/train.ts --epochs 2 --batch 64",
+        "generate": "tsx scripts/generate.ts",
+        "evaluate": "tsx scripts/evaluate.ts",
+        "debug": "tsx scripts/debug.ts"
+    },
+    "peerDependencies": {
+        "@tensorflow/tfjs": "^4.22.0"
+    },
+    "devDependencies": {
+        "@eslint/js": "^9.32.0",
+        "@tensorflow/tfjs-node-gpu": "^4.22.0",
+        "@types/node": "^22.13.14",
+        "@types/papaparse": "^5.3.16",
+        "@types/yargs": "^17.0.33",
+        "@vitest/coverage-v8": "^3.2.4",
+        "chalk": "^5.4.1",
+        "dayjs": "^1.11.13",
+        "eslint": "^9.32.0",
+        "glob": "^11.0.3",
+        "jsdom": "^26.1.0",
+        "tsx": "^4.20.3",
+        "typescript": "^5.8.3",
+        "typescript-eslint": "^8.38.0",
+        "vite": "^7.0.6",
+        "vite-plugin-dts": "^4.5.4",
+        "vitest": "^3.2.4",
+        "yargs": "^18.0.0"
+    },
+    "dependencies": {
+        "eventemitter3": "^5.0.1",
+        "jszip": "^3.10.1",
+        "papaparse": "^5.5.3"
+    }
+}