@genai-fi/nanogpt 0.23.0 → 1.0.0

package/dist/Trainer-DBsyWJ4s.js

@@ -1,228 +0,0 @@
-import { packingSupported as e } from "./utilities/packed.js";
-import { t } from "./eventemitter3-D_qV3Lof.js";
-import n from "./training/PreTrainer.js";
-import { TokenStore as r } from "./training/tasks/TokenStore.js";
-import { tokensFromTasks as i } from "./training/tasks/Task.js";
-import { createTrainValidationDatasets as a, storeFromArray as o } from "./training/validation.js";
-import s from "./training/SFTTrainer.js";
-//#region node_modules/uuid/dist/stringify.js
-var c = [];
-for (let e = 0; e < 256; ++e) c.push((e + 256).toString(16).slice(1));
-function l(e, t = 0) {
-	return (c[e[t + 0]] + c[e[t + 1]] + c[e[t + 2]] + c[e[t + 3]] + "-" + c[e[t + 4]] + c[e[t + 5]] + "-" + c[e[t + 6]] + c[e[t + 7]] + "-" + c[e[t + 8]] + c[e[t + 9]] + "-" + c[e[t + 10]] + c[e[t + 11]] + c[e[t + 12]] + c[e[t + 13]] + c[e[t + 14]] + c[e[t + 15]]).toLowerCase();
-}
-//#endregion
-//#region node_modules/uuid/dist/rng.js
-var u = new Uint8Array(16);
-function d() {
-	return crypto.getRandomValues(u);
-}
-//#endregion
-//#region node_modules/uuid/dist/v4.js
-function f(e, t, n) {
-	return !t && !e && crypto.randomUUID ? crypto.randomUUID() : p(e, t, n);
-}
-function p(e, t, n) {
-	e ||= {};
-	let r = e.random ?? e.rng?.() ?? d();
-	if (r.length < 16) throw Error("Random bytes length must be >= 16");
-	if (r[6] = r[6] & 15 | 64, r[8] = r[8] & 63 | 128, t) {
-		if (n ||= 0, n < 0 || n + 16 > t.length) throw RangeError(`UUID byte range ${n}:${n + 15} is out of buffer bounds`);
-		for (let e = 0; e < 16; ++e) t[n + e] = r[e];
-		return t;
-	}
-	return l(r);
-}
-//#endregion
-//#region lib/Trainer.ts
-var m = class c extends t {
-	trainer;
-	trainingType = "pretraining";
-	hasTrained = !1;
-	trainDataset;
-	validationDataset;
-	totalTokens = 0;
-	tokensProcessed = 0;
-	log = [];
-	progress = null;
-	options = {
-		batchSize: 32,
-		sftMode: "full",
-		logInterval: 10
-	};
-	tokenizer;
-	constructor(t, r, i, a, o) {
-		if (super(), t instanceof c) {
-			let e = r || t.options, a = t.options, o = !1;
-			if (t.trainingType === "sft" && e.sftMode !== a.sftMode && (o = !0), t.trainer instanceof s && t.trainer.loraName && e.loraName !== t.trainer.loraName && (o = !0), i !== void 0 && i !== t.trainingType && (o = !0), o) {
-				if (t.trainingType === "sft") {
-					let n = new s(t.model, t.tokenizer, e);
-					this.trainer = n, n.loraName = e.loraName;
-				} else this.trainer = new n(t.model, t.tokenizer, e);
-				this.trainingType = i || t.trainingType, this.options = e, this.tokenizer = t.tokenizer;
-			} else this.trainer = t.trainer, this.trainingType = i || t.trainingType, this.options = e, this.trainer.updateOptimizer(this.options), this.log = t.log, this.progress = t.progress, this.totalTokens = t.totalTokens, this.tokenizer = t.tokenizer, e.batchSize === a.batchSize && (this.trainDataset = t.trainDataset, this.validationDataset = t.validationDataset);
-			return;
-		}
-		if (!r) throw Error("Tokeniser must be provided when initializing Trainer with a model");
-		if (!t) throw Error("Model must be provided when initializing Trainer");
-		this.options = a || {
-			batchSize: 32,
-			sftMode: "full"
-		};
-		let l = this.options.mixedPrecision && e();
-		if (this.options.lossScaling = l ? t.lossScaling : 1, i === "sft") {
-			let e = new s(t, r, this.options, o);
-			this.trainer = e, e.loraName = a?.loraName;
-		} else this.trainer = new n(t, r, this.options, o);
-		this.trainingType = i || "pretraining", this.tokenizer = r;
-	}
-	get model() {
-		return this.trainer.model;
-	}
-	get optimizer() {
-		return this.trainer.optimizer;
-	}
-	get isTraining() {
-		return this.trainer.isRunning;
-	}
-	stop() {
-		this.trainer.stop();
-	}
-	reset() {
-		this.hasTrained = !1, this.log = [], this.trainer.reset();
-	}
-	dispose() {
-		this.trainer.dispose(), this.removeAllListeners();
-	}
-	getTotalTokens() {
-		return this.totalTokens;
-	}
-	setOptions(e) {
-		let t = new Set(Object.keys(e).filter((t) => e[t] !== this.options[t]));
-		if (this.trainer.isRunning) {
-			if (t.has("batchSize")) throw Error("Cannot change batch size during training");
-			if (t.has("sftMode")) throw Error("Cannot change SFT mode during training");
-			if (t.has("loraConfig")) throw Error("Cannot change LoRA configuration during training");
-			if (t.has("validationSplit")) throw Error("Cannot change validation split during training");
-			if (t.has("trainableWeights")) throw Error("Cannot change trainable weights during training");
-			if (t.has("mixedPrecision")) throw Error("Cannot change mixed precision setting during training");
-			if (t.has("gradientCheckpointing")) throw Error("Cannot change gradient checkpointing setting during training");
-		}
-		this.options = {
-			...this.options,
-			...e
-		}, this.trainer.updateOptimizer(this.options), t.has("metrics") && this.trainer.setMetrics(e.metrics || []);
-	}
-	async prepare(e = [], t, n) {
-		let c = this.options, l = c.loraName || c.loraConfig;
-		if (n && l) throw Error("Cannot specify datasets when using LoRA fine-tuning");
-		if (!n && !l) throw Error("Must specify datasets for non-LoRA training");
-		if (n) {
-			let e = this.model.metaData.pretrainingData || [], t = [...e], r = !1;
-			for (let i of n) e.some((e) => e.id === i.id) || t.push({
-				id: i.id,
-				name: i.name,
-				conversational: i.conversational
-			}), i.conversational && (r = !0);
-			this.model.metaData.pretrainingData = t, r ? this.model.metaData.mode = "conversational" : this.model.metaData.mode !== "conversational" && (this.model.metaData.mode = "completion");
-		} else this.model.metaData.mode !== "conversational" && (this.model.metaData.mode = "completion");
-		let u = c.maskedLoss ?? this.trainingType === "sft";
-		if (this.trainingType === "sft" && this.trainer instanceof s && e instanceof Uint16Array) throw Error("SFT training requires Task[] input");
-		let d, f = t;
-		if (Array.isArray(e)) if (e[0] instanceof Uint16Array) d = e;
-		else {
-			let n = await i(e, this.trainer.tokenizer, {
-				masking: u,
-				validationSplit: c.validationSplit
-			});
-			d = n.trainingTokens, t || (f = n.validationTokens);
-		}
-		else d = e;
-		let p = d instanceof r ? d.getTokenCount() : d.reduce((e, t) => e + t.length, 0);
-		if (f) {
-			let { trainDataset: e, validationDataset: t } = await a(d, f, this.trainer.tokenizer, this.trainer.datasetBuilder, c?.batchSize || 32);
-			this.trainDataset = e, this.validationDataset = t;
-		} else {
-			let e = d instanceof r ? d : await o(d, this.trainer.tokenizer);
-			this.trainDataset = (await this.trainer.datasetBuilder.createTextDataset(e, c)).dataset;
-		}
-		this.totalTokens = p, this.options.epochSteps = Math.ceil(this.totalTokens / ((c?.batchSize || 32) * this.model.config.blockSize)), this.trainer.updateOptimizer(this.options);
-	}
-	configureModel(e) {
-		let t = e?.sftMode || "full";
-		if (this.trainingType === "pretraining" && (this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA(), this.trainer.model.weightStore.setTrainable(["*"])), this.trainingType === "sft") {
-			if (t === "lora") {
-				let t = this.trainer.model;
-				if (e?.loraName) {
-					if (!t.hasLoRA(e.loraName)) if (e.loraConfig) t.createLoRA(e.loraName, e.loraConfig), t.attachLoRA(e.loraName);
-					else throw Error(`LoRA configuration must be provided to create LoRA with name ${e.loraName}`);
-					else if (t.attachLoRA(e.loraName), e.loraConfig) {
-						let n = t.lora;
-						(n.alpha !== e.loraConfig.alpha || n.rank !== e.loraConfig.rank) && (t.detachLoRA(), t.deleteLoRA(e.loraName), t.createLoRA(e.loraName, e.loraConfig), t.attachLoRA(e.loraName), console.warn("Resetting LoRA with new configuration."));
-					}
-				} else if (e?.loraConfig) if (t.hasLoRA()) {
-					let n = t.lora;
-					if (n.alpha !== e.loraConfig.alpha || n.rank !== e.loraConfig.rank) {
-						t.detachLoRA();
-						let n = e.loraName || f();
-						t.createLoRA(n, e.loraConfig), t.attachLoRA(n);
-					}
-				} else {
-					let n = e.loraName || f();
-					t.createLoRA(n, e.loraConfig), t.attachLoRA(n);
-				}
-				else if (!t.hasLoRA()) throw Error("LoRA configuration must be provided for lora SFT mode");
-			} else this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA();
-			t === "last-layer" ? this.trainer.model.weightStore.setTrainable([`block_${this.trainer.model.config.nLayer - 1}_*`, "token_embedding"]) : t === "full" && this.trainer.model.weightStore.setTrainable(["*"]);
-		}
-		e?.trainableWeights && this.trainer.model.weightStore.setTrainable(e.trainableWeights);
-	}
-	async train() {
-		let e = this.options;
-		if (!this.trainDataset) throw Error("Dataset not prepared");
-		this.hasTrained || this.trainer.setLearningRate(e?.learningRate || .001), this.hasTrained = !0, this.emit("start"), this.model.metaData.pretrainingSettings = e;
-		let t = Date.now();
-		this.log.length > 0 && this.trainer.resumeFromLog(this.log[this.log.length - 1]), this.trainer.setGradientCheckpointing(e?.gradientCheckpointing || !1), this.trainer.setMixedPrecision(e?.mixedPrecision || !1), this.trainer.setLabelSmoothing(e?.labelSmoothing || 0), this.trainer.setDropout(e?.dropout || 0), this.trainer.setLayerDrop(e?.layerDrop || 0), this.configureModel(e), await this.trainer.trainOnDataset(this.trainDataset, {
-			...e,
-			onStep: async (e) => {
-				this.log.push(e), this.progress = {
-					lastLog: e,
-					progress: e.totalTokens / this.totalTokens,
-					remaining: Math.max(0, (this.totalTokens - e.totalTokens) / e.totalTokens * e.duration)
-				}, this.tokensProcessed = e.totalTokens;
-				let t = this.listeners("log");
-				for (let n of t) await n(e, this.progress);
-			}
-		}, this.validationDataset), this.model.metaData.actionLog = this.model.metaData.actionLog || [];
-		let n = Date.now();
-		this.model.metaData.actionLog.push({
-			action: "pretrain",
-			timestamp: n,
-			duration: n - t,
-			tokensProcessed: this.tokensProcessed,
-			options: e
-		}), this.emit("stop");
-	}
-	async step(e) {
-		if (!this.trainDataset) throw Error("Dataset not prepared");
-		this.hasTrained || this.trainer.setLearningRate(e?.learningRate || .001), this.hasTrained = !0, this.emit("start");
-		let { log: t } = await this.trainer.stepDataset(this.trainDataset, e || {}, this.validationDataset), n = this.listeners("log");
-		for (let e of n) await e(t, {
-			lastLog: t,
-			progress: t.totalTokens / this.totalTokens,
-			remaining: Math.max(0, (this.totalTokens - t.totalTokens) / t.totalTokens * t.duration)
-		});
-		this.emit("stop");
-	}
-	getLog() {
-		return this.log;
-	}
-	getProgress() {
-		return this.progress;
-	}
-	isPrepared() {
-		return this.trainDataset !== void 0 && this.validationDataset !== void 0;
-	}
-};
-//#endregion
-export { m as t };

package/dist/Trainer.d.ts

@@ -1,45 +0,0 @@
-import { ITokeniser } from './tokeniser/type';
-import { default as EE } from 'eventemitter3';
-import { default as Model, ModelForwardAttributes } from './models/model';
-import { Task } from './training/tasks/Task';
-import { TrainingOptions, TrainingLogEntry } from './training/types';
-import { AdamWOptimizer } from './training/AdamW';
-import { DatasetMetadata } from './loader/types';
-import { TokenStore } from './training/tasks/TokenStore';
-interface TrainingProgress {
-    lastLog: TrainingLogEntry;
-    progress: number;
-    remaining: number;
-}
-export type TrainingType = 'pretraining' | 'sft';
-export default class Trainer extends EE<'start' | 'stop' | 'log'> {
-    private trainer;
-    readonly trainingType: TrainingType;
-    private hasTrained;
-    private trainDataset?;
-    private validationDataset?;
-    private totalTokens;
-    private tokensProcessed;
-    log: TrainingLogEntry[];
-    private progress;
-    options: TrainingOptions;
-    protected tokenizer: ITokeniser;
-    constructor(model: Model<ModelForwardAttributes>, tokeniser: ITokeniser, trainingType?: TrainingType, options?: TrainingOptions, optimizer?: AdamWOptimizer);
-    constructor(trainer: Trainer, options?: TrainingOptions);
-    get model(): Model<ModelForwardAttributes>;
-    get optimizer(): AdamWOptimizer;
-    get isTraining(): boolean;
-    stop(): void;
-    reset(): void;
-    dispose(): void;
-    getTotalTokens(): number;
-    setOptions(options: TrainingOptions): void;
-    prepare(tasks?: Task[] | Uint16Array[] | TokenStore, validation?: Uint16Array[] | TokenStore, datasets?: DatasetMetadata[]): Promise<void>;
-    private configureModel;
-    train(): Promise<void>;
-    step(options?: TrainingOptions): Promise<void>;
-    getLog(): TrainingLogEntry[];
-    getProgress(): TrainingProgress | null;
-    isPrepared(): boolean;
-}
-export {};

package/dist/Trainer.js

@@ -1,2 +0,0 @@
-import { t as e } from "./Trainer-DBsyWJ4s.js";
-export { e as default };