@genai-fi/nanogpt 1.0.1 → 1.0.3

package/dist/training/prepareData.d.ts

@@ -18,5 +18,5 @@ interface PrepareDataResult {
     totalTokens: number;
 }
 /** Take our training options, model, tokeniser, and tasks, and prepare the training and validation datasets in Tensorflow format. */
-export default function prepareData(options: TrainingOptions, model: Model<ModelForwardAttributes>, tokeniser: ITokeniser, tasks: ConversationStream[] | Uint16Array[] | TokenStore, validation?: Uint16Array[] | TokenStore, datasets?: DatasetMetadata[]): Promise<PrepareDataResult>;
+export default function prepareData(options: TrainingOptions, model: Model<ModelForwardAttributes>, tokeniser: ITokeniser, tasks: ConversationStream[] | Uint16Array[] | TokenStore, datasetId: string, validation?: Uint16Array[] | TokenStore, datasets?: DatasetMetadata[]): Promise<PrepareDataResult>;
 export {};

package/dist/training/prepareData.js

@@ -3,45 +3,45 @@ import { tokensFromStreams as t } from "./tasks/tokenStream.js";
 import { t as n } from "../DatasetBuilder-DU1G1OKX.js";
 import { createTrainValidationDatasets as r, storeFromArray as i } from "./validation.js";
 //#region lib/training/prepareData.ts
-async function a(a, o, s, c, l, u) {
-	let d = a.loraName || a.loraConfig;
-	if (u && d) throw Error("Cannot specify datasets when using LoRA fine-tuning");
-	if (!u && !d) throw Error("Must specify datasets for non-LoRA training");
-	if (u) {
+async function a(a, o, s, c, l, u, d) {
+	let f = a.loraName || a.loraConfig;
+	if (d && f) throw Error("Cannot specify datasets when using LoRA fine-tuning");
+	if (!d && !f) throw Error("Must specify datasets for non-LoRA training");
+	if (d) {
 		let e = o.metaData.pretrainingData || [], t = [...e], n = !1;
-		for (let r of u) e.some((e) => e.id === r.id) || t.push({
+		for (let r of d) e.some((e) => e.id === r.id) || t.push({
 			id: r.id,
 			name: r.name,
 			conversational: r.conversational
 		}), r.conversational && (n = !0);
 		o.metaData.pretrainingData = t, n ? o.metaData.mode = "conversational" : o.metaData.mode !== "conversational" && (o.metaData.mode = "completion");
 	} else o.metaData.mode !== "conversational" && (o.metaData.mode = "completion");
-	let f = a.maskedLoss ?? a.method.type === "supervised", p, m = l;
-	if (Array.isArray(c)) if (c[0] instanceof Uint16Array) p = c;
+	let p = a.maskedLoss ?? a.method.type === "supervised", m, h = u;
+	if (Array.isArray(c)) if (c[0] instanceof Uint16Array) m = c;
 	else {
-		let e = await t(c, s, {
-			masking: f,
+		let e = await t(c, s, l, {
+			masking: p,
 			validationSplit: a.validationSplit
 		});
-		p = e.trainingTokens, l || (m = e.validationTokens);
+		m = e.trainingTokens, u || (h = e.validationTokens);
 	}
-	else p = c;
-	let h = p instanceof e ? p.getTokenCount() : p.reduce((e, t) => e + t.length, 0);
-	a.epochSteps = Math.ceil(h / ((a?.batchSize || 32) * o.config.blockSize));
-	let g = new n(s, o.config.blockSize);
-	if (m) {
-		let { trainDataset: e, validationDataset: t } = await r(p, m, s, g, a?.batchSize || 32);
+	else m = c;
+	let g = m instanceof e ? m.getTokenCount() : m.reduce((e, t) => e + t.length, 0);
+	a.epochSteps = Math.ceil(g / ((a?.batchSize || 32) * o.config.blockSize));
+	let _ = new n(s, o.config.blockSize);
+	if (h) {
+		let { trainDataset: e, validationDataset: t } = await r(m, h, s, _, a?.batchSize || 32);
 		return {
 			trainDataset: e,
 			validationDataset: t,
-			totalTokens: h
+			totalTokens: g
 		};
 	} else {
-		let t = p instanceof e ? p : await i(p, s);
+		let t = m instanceof e ? m : await i(m, s);
 		return {
-			trainDataset: (await g.createTextDataset(t, a)).dataset,
+			trainDataset: (await _.createTextDataset(t, a)).dataset,
 			validationDataset: void 0,
-			totalTokens: h
+			totalTokens: g
 		};
 	}
 }

package/dist/training/tasks/tokenStream.d.ts

@@ -10,7 +10,7 @@ interface TokensFromTasksOptions {
     validationSeed?: string | number;
     cb?: (tokens: number) => void;
 }
-export declare function tokensFromStreams(tasks: ConversationStream[], tokenizer: ITokeniser, options?: TokensFromTasksOptions): Promise<{
+export declare function tokensFromStreams(tasks: ConversationStream[], tokenizer: ITokeniser, datasetId: string, options?: TokensFromTasksOptions): Promise<{
     trainingTokens: TokenStore;
     validationTokens?: TokenStore;
 }>;

package/dist/training/tasks/tokenStream.js

@@ -22,24 +22,24 @@ function r(e, t, n, r, i, a) {
 		} else n.set(e, r.offset), s && !Array.isArray(o) && s.set(o.mask.map((e) => +!!e), r.offset), r.offset += e.length;
 	}
 }
-async function i(i, a, o) {
+async function i(i, a, o, s) {
 	await t("training-tokens");
-	let s = await e("training-tokens", a.id, a.datasetID ?? "", o);
+	let c = await e("training-tokens", a.id, o, s);
 	await t("validation-tokens");
-	let c = o?.validationSplit && o.validationSplit > 0 ? await e("validation-tokens", a.id, a.datasetID ?? "", o) : void 0, l = [new Uint16Array(s.shardSize)], u = o?.masking ? [new Uint8Array(s.shardSize)] : null, d = {
+	let l = s?.validationSplit && s.validationSplit > 0 ? await e("validation-tokens", a.id, o, s) : void 0, u = [new Uint16Array(c.shardSize)], d = s?.masking ? [new Uint8Array(c.shardSize)] : null, f = {
 		offset: 0,
 		total: 0
-	}, f = o?.validationSplit && o.validationSplit > 0 ? [new Uint16Array(c.shardSize)] : void 0, p = o?.masking && f ? [new Uint8Array(c.shardSize)] : null, m = {
+	}, p = s?.validationSplit && s.validationSplit > 0 ? [new Uint16Array(l.shardSize)] : void 0, m = s?.masking && p ? [new Uint8Array(l.shardSize)] : null, h = {
 		offset: 0,
 		total: 0
-	}, h = 0, g = o?.cb, _ = o?.validationSeed === void 0 ? Math.random : n(o.validationSeed);
-	for (; h < i.length;) await i[h++].begin((e) => {
-		let t = o?.validationSplit && o.validationSplit > 0 && _() < o.validationSplit, n = t ? f : l, i = t ? m : d, h = t ? p : u, g = t ? c : s;
-		r(e, n, a, i, g.shardSize, h || void 0), n.length > 1 && (g.appendShard(n[0], h ? h[0] : void 0), n.shift(), h && h.shift());
-	}, g ? () => g(d.total) : void 0);
-	return l.length === 1 && (l[0] = l[0].subarray(0, d.offset), s.appendShard(l[0], u ? u[0].subarray(0, d.offset) : void 0)), f && f.length === 1 && (f[0] = f[0].subarray(0, m.offset), c.appendShard(f[0], p ? p[0].subarray(0, m.offset) : void 0)), await s.finish(), c && await c.finish(), {
-		trainingTokens: s,
-		validationTokens: f ? c : void 0
+	}, g = 0, _ = s?.cb, v = s?.validationSeed === void 0 ? Math.random : n(s.validationSeed);
+	for (; g < i.length;) await i[g++].begin((e) => {
+		let t = s?.validationSplit && s.validationSplit > 0 && v() < s.validationSplit, n = t ? p : u, i = t ? h : f, o = t ? m : d, g = t ? l : c;
+		r(e, n, a, i, g.shardSize, o || void 0), n.length > 1 && (g.appendShard(n[0], o ? o[0] : void 0), n.shift(), o && o.shift());
+	}, _ ? () => _(f.total) : void 0);
+	return u.length === 1 && (u[0] = u[0].subarray(0, f.offset), c.appendShard(u[0], d ? d[0].subarray(0, f.offset) : void 0)), p && p.length === 1 && (p[0] = p[0].subarray(0, h.offset), l.appendShard(p[0], m ? m[0].subarray(0, h.offset) : void 0)), await c.finish(), l && await l.finish(), {
+		trainingTokens: c,
+		validationTokens: p ? l : void 0
 	};
 }
 //#endregion

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "1.0.1",
+    "version": "1.0.3",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",
@@ -57,9 +57,9 @@
         "jszip": "^3.10.1",
         "papaparse": "^5.5.3",
         "patch-package": "^8.0.1",
-        "pdfjs-dist": "^5.4.149",
         "picomatch": "^4.0.3",
-        "uuid": "^14.0.0"
+        "uuid": "^14.0.0",
+        "pdfjs-dist": "^6.2.108"
     },
     "allowScripts": {
         "webgpu@0.4.0": true,