@genai-fi/nanogpt 1.0.0 → 1.0.2

package/dist/api/training.js

@@ -115,17 +115,17 @@ var o = class {
 			this.setState(e, "error", t);
 		}));
 	}
-	async job(e, a = [], o, s) {
-		if (this.assertValidOptions(e), this.assertValidDataInput(a), this.assertValidDatasets(o), s && s instanceof t && s.tokeniserId !== this._tokeniser.id) throw Error("tokeniser_mismatch");
-		let c = Array.from(this._jobs.keys());
-		for (let t of c) {
+	async job(e, o = [], s, c) {
+		if (this.assertValidOptions(e), this.assertValidDataInput(o), this.assertValidDatasets(s), c && c instanceof t && c.tokeniserId !== this._tokeniser.id) throw Error("tokeniser_mismatch");
+		let l = Array.from(this._jobs.keys());
+		for (let t of l) {
 			let n = this._jobs.get(t);
 			if (n && n.id !== e.previous_job_id) {
 				if (n.options.method.type === e.method.type && n.state !== "running") n.trainer.dispose(), this._jobs.delete(n.id);
 				else if (n.options.method.type === e.method.type && n.state === "running") throw Error("training_in_progress");
 			}
 		}
-		let l = e.previous_job_id ? this.getJob(e.previous_job_id) : {
+		let u = e.previous_job_id ? this.getJob(e.previous_job_id) : {
 			id: n(),
 			state: "pending",
 			trainer: r(this._model, this._tokeniser, e),
@@ -134,40 +134,36 @@ var o = class {
 			remaining: 0,
 			options: e,
 			totalTokens: 0,
-			datasets: o,
+			datasets: s,
 			breakOnLog: !1
 		};
-		if (!l) throw Error("invalid_previous_job_id");
-		if (this._jobs.set(l.id, l), this.setState(l, "pending"), a instanceof t) {
-			if (l.datasetId && a.datasetId !== l.datasetId) {
-				let e = /* @__PURE__ */ Error("dataset_mismatch");
-				throw this.setState(l, "error", e), e;
-			}
-			if (a.tokeniserId !== this._tokeniser.id) {
+		if (!u) throw Error("invalid_previous_job_id");
+		if (this._jobs.set(u.id, u), this.setState(u, "pending"), o instanceof t) {
+			if (u.datasetId && o.datasetId !== u.datasetId && console.warn("TokenStore dataset ID does not match job dataset ID, proceeding anyway"), o.tokeniserId !== this._tokeniser.id) {
 				let e = /* @__PURE__ */ Error("tokeniser_mismatch");
-				throw this.setState(l, "error", e), e;
+				throw this.setState(u, "error", e), e;
 			}
-			l.datasetId = a.datasetId;
+			u.datasetId = o.datasetId;
 		}
-		if (!l.trainDataset) try {
-			let t = await i(e, this._model, this._tokeniser, a, s, o);
-			l.trainDataset = t.trainDataset, l.validationDataset = t.validationDataset, l.totalTokens = t.totalTokens;
+		if (!u.trainDataset) try {
+			let t = await i(e, this._model, this._tokeniser, o, u.datasetId ?? a(s), c, s);
+			u.trainDataset = t.trainDataset, u.validationDataset = t.validationDataset, u.totalTokens = t.totalTokens;
 		} catch (e) {
-			throw this.setState(l, "error", /* @__PURE__ */ Error("prepare_data_failed")), l.trainer.dispose(), this._jobs.delete(l.id), e;
+			throw this.setState(u, "error", /* @__PURE__ */ Error("prepare_data_failed")), u.trainer.dispose(), this._jobs.delete(u.id), e;
 		}
 		if (e.previous_job_id) {
-			this.assertValidOptions(e), l.options = e;
+			this.assertValidOptions(e), u.options = e;
 			try {
-				l.trainer.configure(e);
+				u.trainer.configure(e);
 			} catch (e) {
-				throw this.setState(l, "error", /* @__PURE__ */ Error("trainer_configuration_failed")), l.trainer.dispose(), this._jobs.delete(l.id), e;
+				throw this.setState(u, "error", /* @__PURE__ */ Error("trainer_configuration_failed")), u.trainer.dispose(), this._jobs.delete(u.id), e;
 			}
 		} else try {
-			l.trainer.configure(e);
+			u.trainer.configure(e);
 		} catch (e) {
-			throw this.setState(l, "error", /* @__PURE__ */ Error("trainer_configuration_failed")), l.trainer.dispose(), this._jobs.delete(l.id), e;
+			throw this.setState(u, "error", /* @__PURE__ */ Error("trainer_configuration_failed")), u.trainer.dispose(), this._jobs.delete(u.id), e;
 		}
-		return this.launchJob(l), l;
+		return this.launchJob(u), u;
 	}
 	getJob(e) {
 		return this.assertValidJobId(e), this._jobs.get(e) ?? null;

package/dist/training/prepareData.d.ts

@@ -18,5 +18,5 @@ interface PrepareDataResult {
     totalTokens: number;
 }
 /** Take our training options, model, tokeniser, and tasks, and prepare the training and validation datasets in Tensorflow format. */
-export default function prepareData(options: TrainingOptions, model: Model<ModelForwardAttributes>, tokeniser: ITokeniser, tasks: ConversationStream[] | Uint16Array[] | TokenStore, validation?: Uint16Array[] | TokenStore, datasets?: DatasetMetadata[]): Promise<PrepareDataResult>;
+export default function prepareData(options: TrainingOptions, model: Model<ModelForwardAttributes>, tokeniser: ITokeniser, tasks: ConversationStream[] | Uint16Array[] | TokenStore, datasetId: string, validation?: Uint16Array[] | TokenStore, datasets?: DatasetMetadata[]): Promise<PrepareDataResult>;
 export {};

package/dist/training/prepareData.js

@@ -3,45 +3,45 @@ import { tokensFromStreams as t } from "./tasks/tokenStream.js";
 import { t as n } from "../DatasetBuilder-DU1G1OKX.js";
 import { createTrainValidationDatasets as r, storeFromArray as i } from "./validation.js";
 //#region lib/training/prepareData.ts
-async function a(a, o, s, c, l, u) {
-	let d = a.loraName || a.loraConfig;
-	if (u && d) throw Error("Cannot specify datasets when using LoRA fine-tuning");
-	if (!u && !d) throw Error("Must specify datasets for non-LoRA training");
-	if (u) {
+async function a(a, o, s, c, l, u, d) {
+	let f = a.loraName || a.loraConfig;
+	if (d && f) throw Error("Cannot specify datasets when using LoRA fine-tuning");
+	if (!d && !f) throw Error("Must specify datasets for non-LoRA training");
+	if (d) {
 		let e = o.metaData.pretrainingData || [], t = [...e], n = !1;
-		for (let r of u) e.some((e) => e.id === r.id) || t.push({
+		for (let r of d) e.some((e) => e.id === r.id) || t.push({
 			id: r.id,
 			name: r.name,
 			conversational: r.conversational
 		}), r.conversational && (n = !0);
 		o.metaData.pretrainingData = t, n ? o.metaData.mode = "conversational" : o.metaData.mode !== "conversational" && (o.metaData.mode = "completion");
 	} else o.metaData.mode !== "conversational" && (o.metaData.mode = "completion");
-	let f = a.maskedLoss ?? a.method.type === "supervised", p, m = l;
-	if (Array.isArray(c)) if (c[0] instanceof Uint16Array) p = c;
+	let p = a.maskedLoss ?? a.method.type === "supervised", m, h = u;
+	if (Array.isArray(c)) if (c[0] instanceof Uint16Array) m = c;
 	else {
-		let e = await t(c, s, {
-			masking: f,
+		let e = await t(c, s, l, {
+			masking: p,
 			validationSplit: a.validationSplit
 		});
-		p = e.trainingTokens, l || (m = e.validationTokens);
+		m = e.trainingTokens, u || (h = e.validationTokens);
 	}
-	else p = c;
-	let h = p instanceof e ? p.getTokenCount() : p.reduce((e, t) => e + t.length, 0);
-	a.epochSteps = Math.ceil(h / ((a?.batchSize || 32) * o.config.blockSize));
-	let g = new n(s, o.config.blockSize);
-	if (m) {
-		let { trainDataset: e, validationDataset: t } = await r(p, m, s, g, a?.batchSize || 32);
+	else m = c;
+	let g = m instanceof e ? m.getTokenCount() : m.reduce((e, t) => e + t.length, 0);
+	a.epochSteps = Math.ceil(g / ((a?.batchSize || 32) * o.config.blockSize));
+	let _ = new n(s, o.config.blockSize);
+	if (h) {
+		let { trainDataset: e, validationDataset: t } = await r(m, h, s, _, a?.batchSize || 32);
 		return {
 			trainDataset: e,
 			validationDataset: t,
-			totalTokens: h
+			totalTokens: g
 		};
 	} else {
-		let t = p instanceof e ? p : await i(p, s);
+		let t = m instanceof e ? m : await i(m, s);
 		return {
-			trainDataset: (await g.createTextDataset(t, a)).dataset,
+			trainDataset: (await _.createTextDataset(t, a)).dataset,
 			validationDataset: void 0,
-			totalTokens: h
+			totalTokens: g
 		};
 	}
 }

package/dist/training/tasks/tokenStream.d.ts

@@ -10,7 +10,7 @@ interface TokensFromTasksOptions {
     validationSeed?: string | number;
     cb?: (tokens: number) => void;
 }
-export declare function tokensFromStreams(tasks: ConversationStream[], tokenizer: ITokeniser, options?: TokensFromTasksOptions): Promise<{
+export declare function tokensFromStreams(tasks: ConversationStream[], tokenizer: ITokeniser, datasetId: string, options?: TokensFromTasksOptions): Promise<{
     trainingTokens: TokenStore;
     validationTokens?: TokenStore;
 }>;

package/dist/training/tasks/tokenStream.js

@@ -22,24 +22,24 @@ function r(e, t, n, r, i, a) {
 		} else n.set(e, r.offset), s && !Array.isArray(o) && s.set(o.mask.map((e) => +!!e), r.offset), r.offset += e.length;
 	}
 }
-async function i(i, a, o) {
+async function i(i, a, o, s) {
 	await t("training-tokens");
-	let s = await e("training-tokens", a.id, a.datasetID ?? "", o);
+	let c = await e("training-tokens", a.id, o, s);
 	await t("validation-tokens");
-	let c = o?.validationSplit && o.validationSplit > 0 ? await e("validation-tokens", a.id, a.datasetID ?? "", o) : void 0, l = [new Uint16Array(s.shardSize)], u = o?.masking ? [new Uint8Array(s.shardSize)] : null, d = {
+	let l = s?.validationSplit && s.validationSplit > 0 ? await e("validation-tokens", a.id, o, s) : void 0, u = [new Uint16Array(c.shardSize)], d = s?.masking ? [new Uint8Array(c.shardSize)] : null, f = {
 		offset: 0,
 		total: 0
-	}, f = o?.validationSplit && o.validationSplit > 0 ? [new Uint16Array(c.shardSize)] : void 0, p = o?.masking && f ? [new Uint8Array(c.shardSize)] : null, m = {
+	}, p = s?.validationSplit && s.validationSplit > 0 ? [new Uint16Array(l.shardSize)] : void 0, m = s?.masking && p ? [new Uint8Array(l.shardSize)] : null, h = {
 		offset: 0,
 		total: 0
-	}, h = 0, g = o?.cb, _ = o?.validationSeed === void 0 ? Math.random : n(o.validationSeed);
-	for (; h < i.length;) await i[h++].begin((e) => {
-		let t = o?.validationSplit && o.validationSplit > 0 && _() < o.validationSplit, n = t ? f : l, i = t ? m : d, h = t ? p : u, g = t ? c : s;
-		r(e, n, a, i, g.shardSize, h || void 0), n.length > 1 && (g.appendShard(n[0], h ? h[0] : void 0), n.shift(), h && h.shift());
-	}, g ? () => g(d.total) : void 0);
-	return l.length === 1 && (l[0] = l[0].subarray(0, d.offset), s.appendShard(l[0], u ? u[0].subarray(0, d.offset) : void 0)), f && f.length === 1 && (f[0] = f[0].subarray(0, m.offset), c.appendShard(f[0], p ? p[0].subarray(0, m.offset) : void 0)), await s.finish(), c && await c.finish(), {
-		trainingTokens: s,
-		validationTokens: f ? c : void 0
+	}, g = 0, _ = s?.cb, v = s?.validationSeed === void 0 ? Math.random : n(s.validationSeed);
+	for (; g < i.length;) await i[g++].begin((e) => {
+		let t = s?.validationSplit && s.validationSplit > 0 && v() < s.validationSplit, n = t ? p : u, i = t ? h : f, o = t ? m : d, g = t ? l : c;
+		r(e, n, a, i, g.shardSize, o || void 0), n.length > 1 && (g.appendShard(n[0], o ? o[0] : void 0), n.shift(), o && o.shift());
+	}, _ ? () => _(f.total) : void 0);
+	return u.length === 1 && (u[0] = u[0].subarray(0, f.offset), c.appendShard(u[0], d ? d[0].subarray(0, f.offset) : void 0)), p && p.length === 1 && (p[0] = p[0].subarray(0, h.offset), l.appendShard(p[0], m ? m[0].subarray(0, h.offset) : void 0)), await c.finish(), l && await l.finish(), {
+		trainingTokens: c,
+		validationTokens: p ? l : void 0
 	};
 }
 //#endregion

package/dist/training/validation.d.ts

@@ -9,11 +9,11 @@ export declare function createTrainValidationDatasets(trainingTokens: Uint16Arra
         xs: Tensor;
         ys: Tensor;
     }>;
-    validationDataset: Dataset<{
+    validationDataset?: Dataset<{
         xs: Tensor;
         ys: Tensor;
     }>;
     size: number;
-    validationState: DatasetState;
+    validationState?: DatasetState;
     trainState: DatasetState;
 }>;

package/dist/training/validation.js

@@ -7,7 +7,15 @@ async function n(e, n) {
 	}), await r.finish(), r;
 }
 async function r(t, r, i, a, o) {
-	let s = t instanceof e ? t : await n(t, i), c = r instanceof e ? r : await n(r, i), l = s.getTokenCount(), { dataset: u, state: d } = await a.createTextDataset(s, { batchSize: o }), { dataset: f, state: p } = await a.createTextDataset(c, {
+	let s = t instanceof e ? t : await n(t, i), c = r instanceof e ? r : await n(r, i), l = s.getTokenCount(), { dataset: u, state: d } = await a.createTextDataset(s, { batchSize: o });
+	if (c.getTokenCount() === 0) return {
+		trainDataset: u,
+		validationDataset: void 0,
+		size: l,
+		validationState: void 0,
+		trainState: d
+	};
+	let { dataset: f, state: p } = await a.createTextDataset(c, {
 		batchSize: o,
 		shuffleFirst: !0
 	});

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "1.0.0",
+    "version": "1.0.2",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",