@genai-fi/nanogpt 0.21.0 → 0.22.0

package/dist/data/textLoader.js

@@ -1,9 +1,10 @@
 import { i as e, t } from "../chunk-CWhphoD1.js";
 import { loadPDF as n } from "./pdf.js";
-import { loadDOCX as r, t as i } from "./docx.js";
+import { loadDOCX as r } from "./docx.js";
+import { n as i, r as a, t as o } from "../stream-DKl3GTDL.js";
 //#endregion
 //#region lib/data/textLoader.ts
-var a = /* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {
+var s = /* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {
 	((n, r) => {
 		typeof define == "function" && define.amd ? define([], r) : typeof t == "object" && e !== void 0 ? t.exports = r() : n.Papa = r();
 	})(e, function e() {
@@ -496,19 +497,19 @@ var a = /* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {
 			});
 		}), (l.prototype = Object.create(c.prototype)).constructor = l, (u.prototype = Object.create(c.prototype)).constructor = u, (d.prototype = Object.create(d.prototype)).constructor = d, (f.prototype = Object.create(c.prototype)).constructor = f, s;
 	});
-})))(), 1), o = /* @__PURE__ */ e(i(), 1);
-function s(e, t) {
+})))(), 1);
+function c(e, t) {
 	let n = e.findIndex((e) => e.toLowerCase() === t.toLowerCase());
 	return n === -1 ? 0 : n;
 }
-function c(e) {
+function l(e) {
 	return e.every((e) => e.length < 64);
 }
-function l(e) {
+function u(e) {
 	return e.split(".").pop() || "";
 }
-function u(e) {
-	switch (l(e)) {
+function d(e) {
+	switch (u(e)) {
 		case "json": return "application/json";
 		case "jsonl": return "application/jsonl";
 		case "parquet": return "application/parquet";
@@ -520,19 +521,14 @@ function u(e) {
 		default: return "unknown";
 	}
 }
-function d(e) {
-	if (!Array.isArray(e)) return !1;
-	let t = e[0];
-	return typeof t == "object" && !!t && "role" in t && "content" in t && typeof t.role == "string" && typeof t.content == "string";
-}
-async function f(e, t, i) {
-	let l = e.type === "" ? u(e.name) : e.type;
-	if (l === "application/parquet") throw Error("Parquet loading is not currently supported in the browser. Please convert your data to JSONL format.");
-	if (l === "application/pdf") return n(e, t?.maxSize);
-	if (l === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") return r(e);
-	if (l === "application/json") {
+async function f(e, t) {
+	let u = e.type === "" ? d(e.name) : e.type;
+	if (u === "application/parquet") throw Error("Parquet loading is not currently supported in the browser. Please convert your data to JSONL format.");
+	if (u === "application/pdf") return new i(await n(e, t?.maxSize));
+	if (u === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") return new i(await r(e));
+	if (u === "application/json") {
 		let t = await e.text(), n = JSON.parse(t);
-		if (Array.isArray(n)) return n.map((e) => [typeof e == "string" ? {
+		if (Array.isArray(n)) return new i(n.map((e) => [typeof e == "string" ? {
 			role: "text",
 			content: e
 		} : "text" in e ? {
@@ -541,61 +537,26 @@ async function f(e, t, i) {
 		} : {
 			role: "text",
 			content: JSON.stringify(e)
-		}]);
+		}]));
 		throw Error("Expected JSON array");
 	}
-	if (l === "application/jsonl") {
-		let t = await e.text();
-		return i && i(.1), t.split("\n").filter((e) => e.trim() !== "").map((e, t, n) => {
-			i && t % 1e3 == 0 && i(.1 + t / n.length * .9);
-			try {
-				let t = JSON.parse(e);
-				return d(t) ? t : [typeof t == "string" ? {
-					role: "text",
-					content: t
-				} : "text" in t ? {
-					role: "text",
-					content: t.text
-				} : {
-					role: "text",
-					content: JSON.stringify(t)
-				}];
-			} catch {
-				return [{
-					role: "text",
-					content: e
-				}];
-			}
-		});
-	}
-	if (l === "application/zip") {
-		let n = await o.default.loadAsync(e), r = [], a = Object.keys(n.files);
-		for (let e = 0; e < a.length; e++) {
-			let o = a[e], s = n.file(o);
-			if (s) {
-				let n = await s.async("blob", (t) => {
-					i && i(.1 + (t.percent / 100 * .9 / a.length + e / a.length * .9));
-				}), c = await f(new File([n], o), t);
-				i && i(.1 + (e + 1) / a.length * .9), r = r.concat(c);
-			}
-		}
-		return r;
-	}
-	if (l === "text/csv") {
+	if (u === "application/jsonl") return new o(e);
+	if (u === "application/zip") return new a(e);
+	if (u === "text/csv") {
 		let n = await e.text();
-		return i && i(.1), new Promise((e, r) => {
-			a.default.parse(n, {
+		return new Promise((e, r) => {
+			s.default.parse(n, {
 				header: !1,
 				skipEmptyLines: !0,
 				delimiter: ",",
 				complete: (n) => {
 					if (n.errors.length > 0) console.error(n.errors), r(/* @__PURE__ */ Error("Error parsing file"));
 					else {
-						let r = s(n.data[0], t?.column || "text");
-						e((t?.hasHeader ?? c(n.data[0]) ? n.data.slice(1) : n.data).map((e) => [{
+						let r = c(n.data[0], t?.column || "text");
+						e(new i((t?.hasHeader ?? l(n.data[0]) ? n.data.slice(1) : n.data).map((e) => [{
 							role: "text",
 							content: e[r]
-						}]));
+						}])));
 					}
 				},
 				error: (e) => {
@@ -603,11 +564,11 @@ async function f(e, t, i) {
 				}
 			});
 		});
-	} else if (l === "text/plain") return [[{
+	} else if (u === "text/plain") return new i([[{
 		role: "text",
 		content: await e.text()
-	}]];
-	throw Error(`Unsupported file type: ${l}`);
+	}]]);
+	throw Error(`Unsupported file type: ${u}`);
 }
 //#endregion
 export { f as default };

package/dist/loader/load.js

@@ -1,2 +1,2 @@
-import { d as e, u as t } from "../main-DdsoTJAW.js";
+import { d as e, u as t } from "../main-Bgc7_9kb.js";
 export { t as VERSION, e as loadModel };

package/dist/loader/loadHF.js

@@ -1,2 +1,2 @@
-import { f as e } from "../main-DdsoTJAW.js";
+import { f as e } from "../main-Bgc7_9kb.js";
 export { e as default };

package/dist/loader/loadTransformers.js

@@ -1,2 +1,2 @@
-import { h as e, m as t } from "../main-DdsoTJAW.js";
+import { h as e, m as t } from "../main-Bgc7_9kb.js";
 export { t as default, e as mapTransformersConfigToGPTConfig };

package/dist/loader/newZipLoad.js

@@ -1,2 +1,2 @@
-import { p as e } from "../main-DdsoTJAW.js";
+import { p as e } from "../main-Bgc7_9kb.js";
 export { e as default };

package/dist/loader/oldZipLoad.js

@@ -1,2 +1,2 @@
-import { g as e } from "../main-DdsoTJAW.js";
+import { g as e } from "../main-Bgc7_9kb.js";
 export { e as default };

package/dist/loader/save.js

@@ -1,2 +1,2 @@
-import { l as e } from "../main-DdsoTJAW.js";
+import { l as e } from "../main-Bgc7_9kb.js";
 export { e as saveModel };

package/dist/{main-DdsoTJAW.js → main-Bgc7_9kb.js}

@@ -6,7 +6,7 @@ import { A as Er, C as Dr, D as Or, E as kr, F as Ar, I as jr, L as Mr, M as Nr,
 import { a as Xr, i as Zr, n as Qr, o as $r, r as ei, t as ti } from "./slice_util-Dve4TkDn.js";
 import { n as ni, t as ri } from "./concat_util-CWzvQQlK.js";
 import { a as ii, c as ai, i as oi, n as si, o as ci, r as li, s as ui, t as di } from "./complex_util-CkazZsaH.js";
-import { i as fi } from "./DatasetBuilder-B9UGNQT5.js";
+import { i as fi } from "./DatasetBuilder-Ctb425Id.js";
 import { $ as pi, A as mi, B as hi, C as gi, D as _i, E as vi, F as yi, G as bi, H as xi, I as Si, J as Ci, K as wi, L as Ti, M as Ei, N as Di, O as Oi, P as ki, Q as Ai, R as ji, S as Mi, T as Ni, U as Pi, V as Fi, W as Ii, X as Li, Y as Ri, Z as zi, _ as Bi, _t as Vi, a as Hi, at as Ui, b as Wi, c as Gi, ct as Ki, d as qi, dt as Ji, et as Yi, f as Xi, ft as Zi, g as Qi, gt as $i, h as ea, ht as ta, i as na, it as ra, j as ia, k as aa, l as oa, lt as sa, m as ca, mt as la, n as ua, nt as da, o as fa, ot as pa, p as ma, pt as ha, q as ga, r as _a, rt as va, s as ya, st as ba, tt as H, u as xa, ut as Sa, v as Ca, vt as wa, w as Ta, x as Ea, y as Da, yt as U, z as Oa } from "./shared-24EoeHJl.js";
 import { $ as ka, A as Aa, B as ja, C as Ma, D as Na, E as Pa, F as Fa, G as Ia, H as La, I as Ra, J as za, K as Ba, L as Va, M as Ha, N as Ua, O as Wa, P as Ga, Q as Ka, R as qa, S as Ja, T as Ya, U as Xa, V as Za, W as Qa, X as $a, Y as W, Z as eo, _ as to, a as no, b as ro, c as io, et as ao, f as oo, g as so, h as co, i as lo, it as uo, j as fo, k as po, l as mo, m as ho, n as go, nt as _o, o as G, p as vo, q as yo, r as bo, rt as xo, s as So, t as Co, tt as wo, u as To, v as K, w as Eo, x as Do, y as Oo, z as ko } from "./gpgpu_math-DBYEAAdI.js";
 import { A as Ao, B as jo, C as Mo, D as No, E as Po, F as Fo, G as Io, H as Lo, I as Ro, J as zo, K as Bo, L as Vo, M as Ho, N as Uo, O as Wo, P as Go, R as Ko, S as qo, T as Jo, U as Yo, V as Xo, W as Zo, Y as Qo, _ as $o, a as es, c as ts, d as ns, f as rs, g as is, h as as, i as os, j as ss, k as cs, l as ls, m as us, n as ds, o as fs, p as ps, q as ms, r as hs, s as gs, t as _s, u as vs, v as ys, w as bs, x as xs, y as Ss, z as Cs } from "./shared-CfTzpULd.js";
@@ -56,6 +56,7 @@ import Dc from "./utilities/profile.js";
 import Oc from "./training/PreTrainer.js";
 import kc from "./training/SFTTrainer.js";
 import { t as Ac } from "./data/docx.js";
+import "./stream-DKl3GTDL.js";
 import "./data/textLoader.js";
 import { estimateParameterCount as jc } from "./utilities/parameters.js";
 import { tokensFromTasks as Mc } from "./training/tasks/Task.js";
@@ -13077,24 +13078,24 @@ var yS = class extends mc {
 //#endregion
 //#region lib/training/validation.ts
 async function bS(e, t, n, r, i = .1, a) {
-	let o = e instanceof Uint16Array ? e : await Mc(e, t, void 0, a), s = o instanceof Uint16Array ? o : o.tokens, c = Math.ceil(s.length / n.blockSize), l = o instanceof Uint16Array ? void 0 : o.mask, u = /* @__PURE__ */ new Set();
+	let o = e[0] instanceof Uint16Array ? e : await Mc(e, t, void 0, a), s = Array.isArray(o) ? o : o.tokens, c = s.reduce((e, t) => e + t.length, 0), l = Math.ceil(c / n.blockSize), u = Array.isArray(o) ? void 0 : o.mask, d = /* @__PURE__ */ new Set();
 	if (i > 0) {
-		let e = Math.max(1, Math.floor(c * i));
-		for (; u.size < e;) {
-			let e = Math.floor(Math.random() * c);
-			u.add(e);
+		let e = Math.max(1, Math.floor(l * i));
+		for (; d.size < e;) {
+			let e = Math.floor(Math.random() * l);
+			d.add(e);
 		}
 	}
-	let d = new Uint32Array(c - u.size), f = new Uint32Array(u.size), p = 0, m = 0;
-	for (let e = 0; e < c; e++) u.has(e) ? m < f.length && (f[m++] = e) : p < d.length && (d[p++] = e);
-	u.clear(), fi(f);
-	let { dataset: h, state: g } = await n.createTextDataset(s, r, d, l || void 0), { dataset: _, state: v } = await n.createTextDataset(s, r, f);
+	let f = new Uint32Array(l - d.size), p = new Uint32Array(d.size), m = 0, h = 0;
+	for (let e = 0; e < l; e++) d.has(e) ? h < p.length && (p[h++] = e) : m < f.length && (f[m++] = e);
+	d.clear(), fi(p);
+	let { dataset: g, state: _ } = await n.createTextDataset(s, r, f, u || void 0), { dataset: v, state: y } = await n.createTextDataset(s, r, p);
 	return {
-		trainDataset: h,
-		validationDataset: _,
-		size: s.length,
-		validationState: v,
-		trainState: g
+		trainDataset: g,
+		validationDataset: v,
+		size: c,
+		validationState: y,
+		trainState: _
 	};
 }
 //#endregion
@@ -13221,7 +13222,7 @@ var ES = class e extends mc {
 		} else this.model.metaData.mode !== "conversational" && (this.model.metaData.mode = "completion");
 		let i = n.maskedLoss ?? this.trainingType === "sft";
 		if (this.trainingType === "sft" && this.trainer instanceof kc && e instanceof Uint16Array) throw Error("SFT training requires Task[] input");
-		let { trainDataset: a, validationDataset: o, size: s } = await bS(e, this.trainer.tokenizer, this.trainer.datasetBuilder, n?.batchSize || 32, n?.validationSplit || .1, i), c = s * (1 - (n?.validationSplit || 0));
+		let { trainDataset: a, validationDataset: o, size: s } = await bS(e, this.trainer.tokenizer, this.trainer.datasetBuilder, n?.batchSize || 32, n?.validationSplit || .1, i), c = Math.floor(s * (1 - (n?.validationSplit || .1)));
 		this.trainDataset = a, this.validationDataset = o, this.totalTokens = c, this.options.epochSteps = Math.ceil(this.totalTokens / ((n?.batchSize || 32) * this.model.config.blockSize)), this.trainer.updateOptimizer(this.options);
 	}
 	configureModel(e) {
@@ -13229,7 +13230,7 @@ var ES = class e extends mc {
 		if (this.trainingType === "pretraining" && (this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA(), this.trainer.model.weightStore.setTrainable(["*"])), this.trainingType === "sft") {
 			if (t === "lora") {
 				let t = this.trainer.model;
-				if (console.log("Configuring model for LoRA fine-tuning with options:", e), e?.loraName) {
+				if (e?.loraName) {
 					if (!t.hasLoRA(e.loraName)) if (e.loraConfig) t.createLoRA(e.loraName, e.loraConfig), t.attachLoRA(e.loraName);
 					else throw Error(`LoRA configuration must be provided to create LoRA with name ${e.loraName}`);
 					else if (t.attachLoRA(e.loraName), e.loraConfig) {

package/dist/main.d.ts

@@ -14,6 +14,8 @@ export { default as BPETokeniser } from './tokeniser/bpe';
 export { default as waitForModel } from './utilities/waitForModel';
 export { default as generateDatasetID } from './utilities/datasetID';
 export { default as loadTextData } from './data/textLoader';
+export type { ConversationStream, ConversationCursor } from './data/stream';
+export { MemoryConversationStream } from './data/stream';
 export type { DatasetMetadata, ModelMode } from './loader/types';
 export { default as Generator, type IGenerator } from './Generator';
 export { default as Evaluator } from './training/Evaluator';
@@ -25,6 +27,7 @@ export type { TrainingOptions, TrainingLogEntry } from './training/types';
 export type { GPTConfig } from './models/config';
 export { estimateParameterCount, estimateMemoryUsage, estimateTrainingMemoryUsage, estimateResources, validateConfig, } from './utilities/parameters';
 export { default as topP } from './utilities/topP';
+export { sliceUint16Shards, sliceUint8Shards } from './utilities/tokens';
 export { Task, tokensFromTasks } from './training/tasks/Task';
 export declare const tasks: {
     PretrainingTask: typeof PretrainingTask;

package/dist/main.js

@@ -1,16 +1,18 @@
-import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-DdsoTJAW.js";
+import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-Bgc7_9kb.js";
 import c from "./tokeniser/CharTokeniser.js";
 import l from "./tokeniser/bpe.js";
 import { AdamWOptimizer as u } from "./training/AdamW.js";
 import d from "./utilities/topP.js";
 import f from "./training/Evaluator.js";
-import p from "./utilities/waitForModel.js";
-import m from "./utilities/datasetID.js";
-import h from "./data/textLoader.js";
-import { estimateMemoryUsage as g, estimateParameterCount as _, estimateResources as v, estimateTrainingMemoryUsage as y, validateConfig as b } from "./utilities/parameters.js";
-import { Task as x, tokensFromTasks as S } from "./training/tasks/Task.js";
-import { selectBackend as C } from "./backend.js";
-import w from "./utilities/performance.js";
-import T from "./checks/index.js";
-import { sentenceEmbeddings as E, sentenceEmbeddingsTensor as D } from "./utilities/sentences.js";
-export { u as AdamWOptimizer, l as BPETokeniser, c as CharTokeniser, f as Evaluator, a as Generator, t as Model, s as NanoGPT, x as Task, n as TeachableLLM, e as Trainer, T as checks, g as estimateMemoryUsage, _ as estimateParameterCount, v as estimateResources, y as estimateTrainingMemoryUsage, m as generateDatasetID, o as layers, h as loadTextData, r as ops, w as performanceTest, C as selectBackend, E as sentenceEmbeddings, D as sentenceEmbeddingsTensor, i as tasks, S as tokensFromTasks, d as topP, b as validateConfig, p as waitForModel };
+import { sliceUint16Shards as p, sliceUint8Shards as m } from "./utilities/tokens.js";
+import h from "./utilities/waitForModel.js";
+import g from "./utilities/datasetID.js";
+import { n as _ } from "./stream-DKl3GTDL.js";
+import v from "./data/textLoader.js";
+import { estimateMemoryUsage as y, estimateParameterCount as b, estimateResources as x, estimateTrainingMemoryUsage as S, validateConfig as C } from "./utilities/parameters.js";
+import { Task as w, tokensFromTasks as T } from "./training/tasks/Task.js";
+import { selectBackend as E } from "./backend.js";
+import D from "./utilities/performance.js";
+import O from "./checks/index.js";
+import { sentenceEmbeddings as k, sentenceEmbeddingsTensor as A } from "./utilities/sentences.js";
+export { u as AdamWOptimizer, l as BPETokeniser, c as CharTokeniser, f as Evaluator, a as Generator, _ as MemoryConversationStream, t as Model, s as NanoGPT, w as Task, n as TeachableLLM, e as Trainer, O as checks, y as estimateMemoryUsage, b as estimateParameterCount, x as estimateResources, S as estimateTrainingMemoryUsage, g as generateDatasetID, o as layers, v as loadTextData, r as ops, D as performanceTest, E as selectBackend, k as sentenceEmbeddings, A as sentenceEmbeddingsTensor, p as sliceUint16Shards, m as sliceUint8Shards, i as tasks, T as tokensFromTasks, d as topP, C as validateConfig, h as waitForModel };

package/dist/models/NanoGPTV1.js

@@ -1,2 +1,2 @@
-import { y as e } from "../main-DdsoTJAW.js";
+import { y as e } from "../main-Bgc7_9kb.js";
 export { e as default };

package/dist/models/NanoGPTV2.js

@@ -1,2 +1,2 @@
-import { v as e } from "../main-DdsoTJAW.js";
+import { v as e } from "../main-Bgc7_9kb.js";
 export { e as default };

package/dist/models/factory.js

@@ -1,2 +1,2 @@
-import { _ as e } from "../main-DdsoTJAW.js";
+import { _ as e } from "../main-Bgc7_9kb.js";
 export { e as default };

package/dist/models/model.js

@@ -1,2 +1,2 @@
-import { b as e } from "../main-DdsoTJAW.js";
+import { b as e } from "../main-Bgc7_9kb.js";
 export { e as default };