@genai-fi/nanogpt 0.22.0 → 0.24.0

package/dist/Trainer-Cr7csbTD.js

@@ -0,0 +1,228 @@
+import { packingSupported as e } from "./utilities/packed.js";
+import { t } from "./eventemitter3-D_qV3Lof.js";
+import n from "./training/PreTrainer.js";
+import { TokenStore as r } from "./training/tasks/TokenStore.js";
+import { tokensFromStreams as i } from "./training/tasks/tokenStream.js";
+import { createTrainValidationDatasets as a, storeFromArray as o } from "./training/validation.js";
+import s from "./training/SFTTrainer.js";
+//#region node_modules/uuid/dist/stringify.js
+var c = [];
+for (let e = 0; e < 256; ++e) c.push((e + 256).toString(16).slice(1));
+function l(e, t = 0) {
+	return (c[e[t + 0]] + c[e[t + 1]] + c[e[t + 2]] + c[e[t + 3]] + "-" + c[e[t + 4]] + c[e[t + 5]] + "-" + c[e[t + 6]] + c[e[t + 7]] + "-" + c[e[t + 8]] + c[e[t + 9]] + "-" + c[e[t + 10]] + c[e[t + 11]] + c[e[t + 12]] + c[e[t + 13]] + c[e[t + 14]] + c[e[t + 15]]).toLowerCase();
+}
+//#endregion
+//#region node_modules/uuid/dist/rng.js
+var u = new Uint8Array(16);
+function d() {
+	return crypto.getRandomValues(u);
+}
+//#endregion
+//#region node_modules/uuid/dist/v4.js
+function f(e, t, n) {
+	return !t && !e && crypto.randomUUID ? crypto.randomUUID() : p(e, t, n);
+}
+function p(e, t, n) {
+	e ||= {};
+	let r = e.random ?? e.rng?.() ?? d();
+	if (r.length < 16) throw Error("Random bytes length must be >= 16");
+	if (r[6] = r[6] & 15 | 64, r[8] = r[8] & 63 | 128, t) {
+		if (n ||= 0, n < 0 || n + 16 > t.length) throw RangeError(`UUID byte range ${n}:${n + 15} is out of buffer bounds`);
+		for (let e = 0; e < 16; ++e) t[n + e] = r[e];
+		return t;
+	}
+	return l(r);
+}
+//#endregion
+//#region lib/Trainer.ts
+var m = class c extends t {
+	trainer;
+	trainingType = "pretraining";
+	hasTrained = !1;
+	trainDataset;
+	validationDataset;
+	totalTokens = 0;
+	tokensProcessed = 0;
+	log = [];
+	progress = null;
+	options = {
+		batchSize: 32,
+		sftMode: "full",
+		logInterval: 10
+	};
+	tokenizer;
+	constructor(t, r, i, a, o) {
+		if (super(), t instanceof c) {
+			let e = r || t.options, a = t.options, o = !1;
+			if (t.trainingType === "sft" && e.sftMode !== a.sftMode && (o = !0), t.trainer instanceof s && t.trainer.loraName && e.loraName !== t.trainer.loraName && (o = !0), i !== void 0 && i !== t.trainingType && (o = !0), o) {
+				if (t.trainingType === "sft") {
+					let n = new s(t.model, t.tokenizer, e);
+					this.trainer = n, n.loraName = e.loraName;
+				} else this.trainer = new n(t.model, t.tokenizer, e);
+				this.trainingType = i || t.trainingType, this.options = e, this.tokenizer = t.tokenizer;
+			} else this.trainer = t.trainer, this.trainingType = i || t.trainingType, this.options = e, this.trainer.updateOptimizer(this.options), this.log = t.log, this.progress = t.progress, this.totalTokens = t.totalTokens, this.tokenizer = t.tokenizer, e.batchSize === a.batchSize && (this.trainDataset = t.trainDataset, this.validationDataset = t.validationDataset);
+			return;
+		}
+		if (!r) throw Error("Tokeniser must be provided when initializing Trainer with a model");
+		if (!t) throw Error("Model must be provided when initializing Trainer");
+		this.options = a || {
+			batchSize: 32,
+			sftMode: "full"
+		};
+		let l = this.options.mixedPrecision && e();
+		if (this.options.lossScaling = l ? t.lossScaling : 1, i === "sft") {
+			let e = new s(t, r, this.options, o);
+			this.trainer = e, e.loraName = a?.loraName;
+		} else this.trainer = new n(t, r, this.options, o);
+		this.trainingType = i || "pretraining", this.tokenizer = r;
+	}
+	get model() {
+		return this.trainer.model;
+	}
+	get optimizer() {
+		return this.trainer.optimizer;
+	}
+	get isTraining() {
+		return this.trainer.isRunning;
+	}
+	stop() {
+		this.trainer.stop();
+	}
+	reset() {
+		this.hasTrained = !1, this.log = [], this.trainer.reset();
+	}
+	dispose() {
+		this.trainer.dispose(), this.removeAllListeners();
+	}
+	getTotalTokens() {
+		return this.totalTokens;
+	}
+	setOptions(e) {
+		let t = new Set(Object.keys(e).filter((t) => e[t] !== this.options[t]));
+		if (this.trainer.isRunning) {
+			if (t.has("batchSize")) throw Error("Cannot change batch size during training");
+			if (t.has("sftMode")) throw Error("Cannot change SFT mode during training");
+			if (t.has("loraConfig")) throw Error("Cannot change LoRA configuration during training");
+			if (t.has("validationSplit")) throw Error("Cannot change validation split during training");
+			if (t.has("trainableWeights")) throw Error("Cannot change trainable weights during training");
+			if (t.has("mixedPrecision")) throw Error("Cannot change mixed precision setting during training");
+			if (t.has("gradientCheckpointing")) throw Error("Cannot change gradient checkpointing setting during training");
+		}
+		this.options = {
+			...this.options,
+			...e
+		}, this.trainer.updateOptimizer(this.options), t.has("metrics") && this.trainer.setMetrics(e.metrics || []);
+	}
+	async prepare(e = [], t, n) {
+		let c = this.options, l = c.loraName || c.loraConfig;
+		if (n && l) throw Error("Cannot specify datasets when using LoRA fine-tuning");
+		if (!n && !l) throw Error("Must specify datasets for non-LoRA training");
+		if (n) {
+			let e = this.model.metaData.pretrainingData || [], t = [...e], r = !1;
+			for (let i of n) e.some((e) => e.id === i.id) || t.push({
+				id: i.id,
+				name: i.name,
+				conversational: i.conversational
+			}), i.conversational && (r = !0);
+			this.model.metaData.pretrainingData = t, r ? this.model.metaData.mode = "conversational" : this.model.metaData.mode !== "conversational" && (this.model.metaData.mode = "completion");
+		} else this.model.metaData.mode !== "conversational" && (this.model.metaData.mode = "completion");
+		let u = c.maskedLoss ?? this.trainingType === "sft";
+		if (this.trainingType === "sft" && this.trainer instanceof s && e instanceof Uint16Array) throw Error("SFT training requires Task[] input");
+		let d, f = t;
+		if (Array.isArray(e)) if (e[0] instanceof Uint16Array) d = e;
+		else {
+			let n = await i(e, this.trainer.tokenizer, {
+				masking: u,
+				validationSplit: c.validationSplit
+			});
+			d = n.trainingTokens, t || (f = n.validationTokens);
+		}
+		else d = e;
+		let p = d instanceof r ? d.getTokenCount() : d.reduce((e, t) => e + t.length, 0);
+		if (f) {
+			let { trainDataset: e, validationDataset: t } = await a(d, f, this.trainer.tokenizer, this.trainer.datasetBuilder, c?.batchSize || 32);
+			this.trainDataset = e, this.validationDataset = t;
+		} else {
+			let e = d instanceof r ? d : await o(d, this.trainer.tokenizer);
+			this.trainDataset = (await this.trainer.datasetBuilder.createTextDataset(e, c)).dataset;
+		}
+		this.totalTokens = p, this.options.epochSteps = Math.ceil(this.totalTokens / ((c?.batchSize || 32) * this.model.config.blockSize)), this.trainer.updateOptimizer(this.options);
+	}
+	configureModel(e) {
+		let t = e?.sftMode || "full";
+		if (this.trainingType === "pretraining" && (this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA(), this.trainer.model.weightStore.setTrainable(["*"])), this.trainingType === "sft") {
+			if (t === "lora") {
+				let t = this.trainer.model;
+				if (e?.loraName) {
+					if (!t.hasLoRA(e.loraName)) if (e.loraConfig) t.createLoRA(e.loraName, e.loraConfig), t.attachLoRA(e.loraName);
+					else throw Error(`LoRA configuration must be provided to create LoRA with name ${e.loraName}`);
+					else if (t.attachLoRA(e.loraName), e.loraConfig) {
+						let n = t.lora;
+						(n.alpha !== e.loraConfig.alpha || n.rank !== e.loraConfig.rank) && (t.detachLoRA(), t.deleteLoRA(e.loraName), t.createLoRA(e.loraName, e.loraConfig), t.attachLoRA(e.loraName), console.warn("Resetting LoRA with new configuration."));
+					}
+				} else if (e?.loraConfig) if (t.hasLoRA()) {
+					let n = t.lora;
+					if (n.alpha !== e.loraConfig.alpha || n.rank !== e.loraConfig.rank) {
+						t.detachLoRA();
+						let n = e.loraName || f();
+						t.createLoRA(n, e.loraConfig), t.attachLoRA(n);
+					}
+				} else {
+					let n = e.loraName || f();
+					t.createLoRA(n, e.loraConfig), t.attachLoRA(n);
+				}
+				else if (!t.hasLoRA()) throw Error("LoRA configuration must be provided for lora SFT mode");
+			} else this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA();
+			t === "last-layer" ? this.trainer.model.weightStore.setTrainable([`block_${this.trainer.model.config.nLayer - 1}_*`, "token_embedding"]) : t === "full" && this.trainer.model.weightStore.setTrainable(["*"]);
+		}
+		e?.trainableWeights && this.trainer.model.weightStore.setTrainable(e.trainableWeights);
+	}
+	async train() {
+		let e = this.options;
+		if (!this.trainDataset) throw Error("Dataset not prepared");
+		this.hasTrained || this.trainer.setLearningRate(e?.learningRate || .001), this.hasTrained = !0, this.emit("start"), this.model.metaData.pretrainingSettings = e;
+		let t = Date.now();
+		this.log.length > 0 && this.trainer.resumeFromLog(this.log[this.log.length - 1]), this.trainer.setGradientCheckpointing(e?.gradientCheckpointing || !1), this.trainer.setMixedPrecision(e?.mixedPrecision || !1), this.trainer.setLabelSmoothing(e?.labelSmoothing || 0), this.trainer.setDropout(e?.dropout || 0), this.trainer.setLayerDrop(e?.layerDrop || 0), this.configureModel(e), await this.trainer.trainOnDataset(this.trainDataset, {
+			...e,
+			onStep: async (e) => {
+				this.log.push(e), this.progress = {
+					lastLog: e,
+					progress: e.totalTokens / this.totalTokens,
+					remaining: Math.max(0, (this.totalTokens - e.totalTokens) / e.totalTokens * e.duration)
+				}, this.tokensProcessed = e.totalTokens;
+				let t = this.listeners("log");
+				for (let n of t) await n(e, this.progress);
+			}
+		}, this.validationDataset), this.model.metaData.actionLog = this.model.metaData.actionLog || [];
+		let n = Date.now();
+		this.model.metaData.actionLog.push({
+			action: "pretrain",
+			timestamp: n,
+			duration: n - t,
+			tokensProcessed: this.tokensProcessed,
+			options: e
+		}), this.emit("stop");
+	}
+	async step(e) {
+		if (!this.trainDataset) throw Error("Dataset not prepared");
+		this.hasTrained || this.trainer.setLearningRate(e?.learningRate || .001), this.hasTrained = !0, this.emit("start");
+		let { log: t } = await this.trainer.stepDataset(this.trainDataset, e || {}, this.validationDataset), n = this.listeners("log");
+		for (let e of n) await e(t, {
+			lastLog: t,
+			progress: t.totalTokens / this.totalTokens,
+			remaining: Math.max(0, (this.totalTokens - t.totalTokens) / t.totalTokens * t.duration)
+		});
+		this.emit("stop");
+	}
+	getLog() {
+		return this.log;
+	}
+	getProgress() {
+		return this.progress;
+	}
+	isPrepared() {
+		return this.trainDataset !== void 0 && this.validationDataset !== void 0;
+	}
+};
+//#endregion
+export { m as t };

package/dist/Trainer.d.ts

@@ -1,10 +1,11 @@
 import { ITokeniser } from './tokeniser/type';
 import { default as EE } from 'eventemitter3';
 import { default as Model, ModelForwardAttributes } from './models/model';
-import { Task } from './training/tasks/Task';
 import { TrainingOptions, TrainingLogEntry } from './training/types';
 import { AdamWOptimizer } from './training/AdamW';
 import { DatasetMetadata } from './loader/types';
+import { TokenStore } from './training/tasks/TokenStore';
+import { ConversationStream } from './data/stream';
 interface TrainingProgress {
     lastLog: TrainingLogEntry;
     progress: number;
@@ -33,7 +34,7 @@ export default class Trainer extends EE<'start' | 'stop' | 'log'> {
     dispose(): void;
     getTotalTokens(): number;
     setOptions(options: TrainingOptions): void;
-    prepare(tasks?: Task[] | Uint16Array[], datasets?: DatasetMetadata[]): Promise<void>;
+    prepare(tasks?: ConversationStream[] | Uint16Array[] | TokenStore, validation?: Uint16Array[] | TokenStore, datasets?: DatasetMetadata[]): Promise<void>;
     private configureModel;
     train(): Promise<void>;
     step(options?: TrainingOptions): Promise<void>;

package/dist/Trainer.js

@@ -1,2 +1,2 @@
-import { a as e } from "./main-Bgc7_9kb.js";
+import { t as e } from "./Trainer-Cr7csbTD.js";
 export { e as default };

package/dist/data/stream.d.ts

@@ -1,19 +1,19 @@
 import { Conversation } from '../../tokeniser/type';
-export interface ConversationCursor {
-    next(): Promise<Conversation[] | null>;
-}
 export interface ConversationStream {
-    cursor(): ConversationCursor;
+    begin(cb: (conv: Conversation[]) => void, yieldCb?: () => void): Promise<void>;
+    step(cb: (conv: Conversation[]) => void): Promise<() => Promise<boolean>>;
 }
 export declare class MemoryConversationStream implements ConversationStream {
     private conversations;
     constructor(conversations: Conversation[][]);
-    cursor(): ConversationCursor;
+    step(cb: (conv: Conversation[]) => void): Promise<() => Promise<boolean>>;
+    begin(cb: (conv: Conversation[]) => void, yieldCb?: () => void): Promise<void>;
 }
 declare class JSONLFromReadableStream implements ConversationStream {
     private sourceFactory;
     constructor(sourceFactory: () => Promise<ReadableStream<Uint8Array>>);
-    cursor(): ConversationCursor;
+    step(cb: (conv: Conversation[]) => void): Promise<() => Promise<boolean>>;
+    begin(cb: (conv: Conversation[]) => void, yieldCb?: () => void): Promise<void>;
 }
 export declare class JSONLConversationStream extends JSONLFromReadableStream {
     constructor(file: File);

package/dist/data/stream.js

@@ -1,2 +1,2 @@
-import { n as e, r as t, t as n } from "../stream-DKl3GTDL.js";
+import { n as e, r as t, t as n } from "../stream-BpAwcvHz.js";
 export { n as JSONLConversationStream, e as MemoryConversationStream, t as ZipJSONLConversationStream };

package/dist/data/textLoader.js

@@ -1,7 +1,7 @@
 import { i as e, t } from "../chunk-CWhphoD1.js";
 import { loadPDF as n } from "./pdf.js";
 import { loadDOCX as r } from "./docx.js";
-import { n as i, r as a, t as o } from "../stream-DKl3GTDL.js";
+import { n as i, r as a, t as o } from "../stream-BpAwcvHz.js";
 //#endregion
 //#region lib/data/textLoader.ts
 var s = /* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {

package/dist/{BaseTokeniser-C9TSv4th.js → eventemitter3-D_qV3Lof.js}

@@ -86,136 +86,6 @@ var n = (/* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {
 		var t;
 		return e ? (t = r ? r + e : e, this._events[t] && s(this, t)) : (this._events = new i(), this._eventsCount = 0), this;
 	}, c.prototype.off = c.prototype.removeListener, c.prototype.addListener = c.prototype.on, c.prefixed = r, c.EventEmitter = c, t !== void 0 && (t.exports = c);
-})))(), 1)).default, r = [
-	"<eos>",
-	"<bos>",
-	"",
-	"<pad>",
-	"<|user_start|>",
-	"<|user_end|>",
-	"<|assistant_start|>",
-	"<|assistant_end|>",
-	"<|system_start|>",
-	"<|system_end|>"
-], i = class extends n {
-	id = "untrained";
-	datasetID;
-	specialTokens = /* @__PURE__ */ new Map();
-	specialTokenSet = /* @__PURE__ */ new Set();
-	isSpecialToken(e) {
-		return this.specialTokenSet.has(e);
-	}
-	addSpecialTokens() {
-		r.forEach((e, t) => {
-			this.addToken(e, t), this.specialTokens.set(e, t), this.specialTokenSet.add(t);
-		});
-	}
-	addSpecialToken(e, t) {
-		this.specialTokens.set(e, t), this.specialTokenSet.add(t);
-	}
-	generateID() {
-		let e = this.getVocab(), t = 2166136261, n = 2654435769;
-		if (e.length === 0) {
-			this.id = "untrained";
-			return;
-		}
-		for (let r = 0; r < e.length; r++) {
-			let i = e[r];
-			t ^= i.length, t = Math.imul(t, 16777619), n ^= r, n = Math.imul(n, 2246822507);
-			for (let e = 0; e < i.length; e++) {
-				let r = i.charCodeAt(e);
-				t ^= r, t = Math.imul(t, 16777619), n ^= r, n = Math.imul(n, 3266489909);
-			}
-		}
-		let r = (t >>> 0).toString(36), i = (n >>> 0).toString(36);
-		this.id = "tokeniser_" + r + "_" + i;
-	}
-	encodeSequence(e) {
-		let t = this.encode(e);
-		return [
-			this.bosToken,
-			...t,
-			this.eosToken
-		];
-	}
-	encodeAsSequence(e, t) {
-		let n = e.flatMap((e) => this.encode(e.content));
-		return t ? [
-			this.bosToken,
-			...n,
-			this.eosToken,
-			this.bosToken
-		] : [
-			this.bosToken,
-			...n,
-			this.eosToken
-		];
-	}
-	encodeConversation(e, t, n) {
-		let r = [[this.bosToken]], i;
-		n && (i = [[!1]]);
-		let a = [
-			this.getSpecialTokenIndex("<|user_start|>"),
-			this.getSpecialTokenIndex("<|assistant_start|>"),
-			this.getSpecialTokenIndex("<|system_start|>")
-		], o = [
-			this.getSpecialTokenIndex("<|user_end|>"),
-			this.getSpecialTokenIndex("<|assistant_end|>"),
-			this.getSpecialTokenIndex("<|system_end|>")
-		];
-		for (let t of e) {
-			let e = !1, s = this.encode(t.content);
-			switch (t.role) {
-				case "user":
-					r.push([a[0]]), e = !0;
-					break;
-				case "assistant":
-					r.push([a[1]]);
-					break;
-				case "system":
-					r.push([a[2]]), e = !0;
-					break;
-			}
-			switch (r.push(s), t.role) {
-				case "user":
-					r.push([o[0]]);
-					break;
-				case "assistant":
-					r.push([o[1]]);
-					break;
-				case "system":
-					r.push([o[2]]);
-					break;
-			}
-			n && i && e ? (i.push([!1]), i.push(s.map(() => !1)), i.push([!1])) : n && i && (i.push([!1]), i.push(s.map(() => !0)), i.push([!0]));
-		}
-		let s = r.flat();
-		return t ? (s.push(a[1]), n && i && i.push([!1])) : (s.push(this.eosToken), n && i && i.push([!0])), n && i ? {
-			tokens: s,
-			mask: i.flat()
-		} : s;
-	}
-	decodeConversation(e) {
-		let t = [], n = 0;
-		for (; n < e.length;) {
-			let r = e[n], i = null;
-			if (r === this.getSpecialTokenIndex("<|user_start|>") ? i = "user" : r === this.getSpecialTokenIndex("<|assistant_start|>") ? i = "assistant" : r === this.getSpecialTokenIndex("<|system_start|>") ? i = "system" : r === this.bosToken || (r === this.eosToken ? i = null : (i = "text", n--)), i) {
-				n++;
-				let r = [];
-				for (; n < e.length && e[n] !== this.getSpecialTokenIndex(`<|${i}_end|>`) && e[n] !== this.eosToken;) r.push(e[n]), n++;
-				let a = this.decode(r);
-				t.push({
-					role: i,
-					content: a
-				});
-			}
-			n++;
-		}
-		return t;
-	}
-	getSpecialTokenIndex(e) {
-		return this.specialTokens.get(e);
-	}
-};
+})))(), 1)).default;
 //#endregion
-export { r as n, n as r, i as t };
+export { n as t };

package/dist/loader/load.js

@@ -1,2 +1,2 @@
-import { d as e, u as t } from "../main-Bgc7_9kb.js";
+import { c as e, s as t } from "../main-Dz72vadm.js";
 export { t as VERSION, e as loadModel };

package/dist/loader/loadHF.js

@@ -1,2 +1,2 @@
-import { f as e } from "../main-Bgc7_9kb.js";
+import { l as e } from "../main-Dz72vadm.js";
 export { e as default };

package/dist/loader/loadTransformers.js

@@ -1,2 +1,2 @@
-import { h as e, m as t } from "../main-Bgc7_9kb.js";
-export { t as default, e as mapTransformersConfigToGPTConfig };
+import { d as e, f as t } from "../main-Dz72vadm.js";
+export { e as default, t as mapTransformersConfigToGPTConfig };

package/dist/loader/newZipLoad.js

@@ -1,2 +1,2 @@
-import { p as e } from "../main-Bgc7_9kb.js";
+import { u as e } from "../main-Dz72vadm.js";
 export { e as default };

package/dist/loader/oldZipLoad.js

@@ -1,2 +1,2 @@
-import { g as e } from "../main-Bgc7_9kb.js";
+import { p as e } from "../main-Dz72vadm.js";
 export { e as default };

package/dist/loader/save.js

@@ -1,2 +1,2 @@
-import { l as e } from "../main-Bgc7_9kb.js";
+import { o as e } from "../main-Dz72vadm.js";
 export { e as saveModel };