@genai-fi/nanogpt 1.0.3 → 1.1.0

package/dist/api/training.d.ts

@@ -29,7 +29,8 @@ export interface ITrainingJob {
     totalTokens: number;
     datasets: DatasetMetadata[];
     datasetId?: string;
-    breakOnLog: boolean;
+    breakOnLog: Set<number>;
+    resumeCounter: number;
 }
 interface TrainingEvents {
     error: (id: string, error: Error) => void;
@@ -46,6 +47,7 @@ export default class Training {
     private _model;
     private _tokeniser;
     private _jobs;
+    private _breakCounter;
     constructor(model: Model<ModelForwardAttributes, GPTConfig>, tokeniser: ITokeniser);
     private setState;
     get activeJobs(): number;
@@ -63,7 +65,8 @@ export default class Training {
     /** Resume a paused job and optionally change some options. */
     resume(id: string): Promise<void>;
     cancel(id: string): void;
-    breakpoints(id: string, enabled: boolean): void;
+    addBreak(jobid: string): number | undefined;
+    deleteBreak(jobid: string, breakId: number): void;
     getPretrainingJob(): ITrainingJob | null;
     dispose(): void;
 }

package/dist/api/training.js

@@ -10,6 +10,7 @@ var o = class {
 	_model;
 	_tokeniser;
 	_jobs = /* @__PURE__ */ new Map();
+	_breakCounter = 1;
 	constructor(t, n) {
 		this.ee = new e(), this._model = t, this._tokeniser = n;
 	}
@@ -19,32 +20,31 @@ var o = class {
 		};
 		switch (t) {
 			case "pending":
-				e.state !== "completed" && e.state !== "paused" && e.state !== "cancelled" && e.state !== "pending" && r();
+				e.state !== "completed" && e.state !== "paused" && e.state !== "cancelled" && e.state !== "pending" && r(), e.state = t;
 				break;
 			case "running":
-				e.state !== "pending" && e.state !== "paused" && r(), this.ee.emit("running", e.id);
+				e.state !== "pending" && e.state !== "paused" && r(), e.state = t, this.ee.emit("running", e.id);
 				break;
 			case "paused":
-				e.state !== "pausing" && r(), this.ee.emit("paused", e.id);
+				e.state !== "pausing" && r(), e.state = t, this.ee.emit("paused", e.id);
 				break;
 			case "completed":
-				e.state !== "running" && r(), this.ee.emit("completed", e.id);
+				e.state !== "running" && r(), e.state = t, this.ee.emit("completed", e.id);
 				break;
 			case "pausing":
-				e.state !== "running" && r(), this.ee.emit("pausing", e.id);
+				e.state !== "running" && r(), e.state = t, this.ee.emit("pausing", e.id);
 				break;
 			case "cancelling":
-				e.state !== "running" && e.state !== "paused" && e.state !== "pausing" && r(), this.ee.emit("cancelling", e.id);
+				e.state !== "running" && e.state !== "paused" && e.state !== "pausing" && r(), e.state = t, this.ee.emit("cancelling", e.id);
 				break;
 			case "cancelled":
-				e.state !== "paused" && e.state !== "cancelling" && r(), this.ee.emit("cancelled", e.id);
+				e.state !== "paused" && e.state !== "cancelling" && r(), e.state = t, this.ee.emit("cancelled", e.id);
 				break;
 			case "error":
-				n && this.ee.emit("error", e.id, n);
+				e.state = t, n && this.ee.emit("error", e.id, n);
 				break;
 			default: throw Error("invalid_state");
 		}
-		e.state = t;
 	}
 	get activeJobs() {
 		return this._jobs.values().reduce((e, t) => t.state === "running" ? e + 1 : e, 0);
@@ -102,13 +102,14 @@ var o = class {
 			totalTokens: t[t.length - 1]?.totalTokens || 0,
 			datasets: o,
 			datasetId: a(o),
-			breakOnLog: !1
+			breakOnLog: /* @__PURE__ */ new Set(),
+			resumeCounter: 0
 		};
 		return s.trainer.log = t, s.trainer.resumeFromLog(t[t.length - 1]), this._jobs.set(s.id, s), s;
 	}
 	launchJob(e) {
 		e.trainDataset && (this.setState(e, "running"), e.trainer.trainOnDataset(e.trainDataset, e.options, e.validationDataset, (t) => {
-			e.history = e.trainer.log, e.progress = t.totalTokens / e.totalTokens, e.remaining = Math.max(0, (e.totalTokens - t.totalTokens) / t.totalTokens * t.duration), e.breakOnLog && e.state === "running" && (this.setState(e, "pausing"), e.trainer.stop()), this.ee.emit("progress", e);
+			e.history = e.trainer.log, e.progress = t.totalTokens / e.totalTokens, e.remaining = Math.max(0, (e.totalTokens - t.totalTokens) / t.totalTokens * t.duration), e.breakOnLog.size > 0 && e.state === "running" && (this.setState(e, "pausing"), e.trainer.stop()), this.ee.emit("progress", e);
 		}).then(() => {
 			e.state === "running" ? this.setState(e, "completed") : e.state === "pausing" ? this.setState(e, "paused") : e.state === "cancelling" && this.setState(e, "cancelled");
 		}).catch((t) => {
@@ -135,7 +136,8 @@ var o = class {
 			options: e,
 			totalTokens: 0,
 			datasets: s,
-			breakOnLog: !1
+			breakOnLog: /* @__PURE__ */ new Set(),
+			resumeCounter: 0
 		};
 		if (!u) throw Error("invalid_previous_job_id");
 		if (this._jobs.set(u.id, u), this.setState(u, "pending"), o instanceof t) {
@@ -175,7 +177,7 @@ var o = class {
 	async resume(e) {
 		this.assertValidJobId(e);
 		let t = this.getJob(e);
-		t && t.state === "paused" && (this.setState(t, "pending"), this.launchJob(t));
+		t && t.state === "paused" && (t.resumeCounter++, !(t.resumeCounter < t.breakOnLog.size) && (t.resumeCounter = 0, this.setState(t, "pending"), this.launchJob(t)));
 	}
 	cancel(e) {
 		this.assertValidJobId(e);
@@ -185,9 +187,16 @@ var o = class {
 			return;
 		} else (t.state === "running" || t.state === "pausing") && (this.setState(t, "cancelling"), t.trainer.stop());
 	}
-	breakpoints(e, t) {
+	addBreak(e) {
+		let t = this.getJob(e);
+		if (t) {
+			let e = this._breakCounter++;
+			return t.breakOnLog.add(e), e;
+		}
+	}
+	deleteBreak(e, t) {
 		let n = this.getJob(e);
-		n && (n.breakOnLog = t);
+		n && n.breakOnLog.delete(t);
 	}
 	getPretrainingJob() {
 		for (let e of this._jobs.values()) if (e.options.method.type === "pretraining") return e;

package/dist/inference/Generator.d.ts

@@ -34,6 +34,7 @@ export default class Generator extends EE<'start' | 'stop' | 'tokens' | 'reset'>
     private startTime;
     private tokenCount;
     constructor(model: Model<ModelForwardAttributes>, tokeniser: ITokeniser);
+    private shouldTerminate;
     /** Generate logits and select a token. */
     private _generateToken;
     /** Generate multiple tokens in a loop and produce text */

package/dist/inference/Generator.js

@@ -40,6 +40,11 @@ var b = class extends e {
 	constructor(e, t) {
 		super(), this.model = e, this.tokeniser = t, this.actualTokeniser = t;
 	}
+	shouldTerminate(e, t) {
+		if (e) return !1;
+		let n = this.tokeniser.getSpecialTokenIndex("<|assistant_end|>");
+		return t === this.actualTokeniser.eosToken || t === n;
+	}
 	async _generateToken(e, t, n) {
 		let s = n?.temperature ?? 1, h = n?.topK, _ = n?.topP, v = n?.usePadding ?? !1, y = {
 			training: !1,
@@ -132,7 +137,7 @@ var b = class extends e {
 		S.dispose(), S = E;
 		let D = (await S.array())[0][0], O = this.actualTokeniser.decode([D]);
 		this.lastToken = D;
-		let k = !n?.allowSpecial && this.tokeniser.isSpecialToken(D), A = {
+		let k = this.shouldTerminate(n?.allowSpecial ?? !1, D), A = {
 			outputTensor: S,
 			token: D,
 			text: O,

package/dist/loader/types.d.ts

@@ -36,6 +36,7 @@ export interface DatasetMetadata {
     id: string;
     name: string;
     conversational: boolean;
+    url?: string;
 }
 export interface ActionLogEntry {
     action: 'pretrain' | 'generate' | 'finetune';

package/dist/training/BasicTrainer.js

@@ -234,7 +234,9 @@ var p = {
 					if (isNaN(e) || !isFinite(e)) throw console.error("Invalid loss value:", e), console.error("Batch xs:", await n.xs.array()), console.error("Batch ys:", await n.ys.array()), console.error("State:", l), Error("Loss is NaN or Infinity");
 					console.log(`Step ${l.step}: Loss = ${e}`);
 				}
-				n.xs.dispose(), n.ys.dispose(), l.step++, l.totalSteps++, r ? await this.performLogging(s, n.xs.shape[0], f, i) : (l.gradientNorm &&= (l.gradientNorm.dispose(), void 0), l.accuracy &&= (l.accuracy.dispose(), void 0)), s.dispose(), l.step >= c && this.stop();
+				n.xs.dispose(), n.ys.dispose(), l.step++, l.totalSteps++;
+				let u = l.step >= c;
+				r || u ? await this.performLogging(s, n.xs.shape[0], f, i) : (l.gradientNorm &&= (l.gradientNorm.dispose(), void 0), l.accuracy &&= (l.accuracy.dispose(), void 0)), s.dispose(), u && this.stop();
 			}
 		} catch (e) {
 			throw console.error("Training error:", e), r(), e;

package/dist/training/prepareData.js

@@ -8,13 +8,14 @@ async function a(a, o, s, c, l, u, d) {
 	if (d && f) throw Error("Cannot specify datasets when using LoRA fine-tuning");
 	if (!d && !f) throw Error("Must specify datasets for non-LoRA training");
 	if (d) {
-		let e = o.metaData.pretrainingData || [], t = [...e], n = !1;
-		for (let r of d) e.some((e) => e.id === r.id) || t.push({
-			id: r.id,
-			name: r.name,
-			conversational: r.conversational
-		}), r.conversational && (n = !0);
-		o.metaData.pretrainingData = t, n ? o.metaData.mode = "conversational" : o.metaData.mode !== "conversational" && (o.metaData.mode = "completion");
+		let e = !1;
+		for (let t of d) t.conversational && (e = !0);
+		o.metaData.pretrainingData = d.map((e) => ({
+			id: e.id,
+			name: e.name,
+			conversational: e.conversational,
+			url: e.url
+		})), e ? o.metaData.mode = "conversational" : o.metaData.mode !== "conversational" && (o.metaData.mode = "completion");
 	} else o.metaData.mode !== "conversational" && (o.metaData.mode = "completion");
 	let p = a.maskedLoss ?? a.method.type === "supervised", m, h = u;
 	if (Array.isArray(c)) if (c[0] instanceof Uint16Array) m = c;
@@ -27,7 +28,7 @@ async function a(a, o, s, c, l, u, d) {
 	}
 	else m = c;
 	let g = m instanceof e ? m.getTokenCount() : m.reduce((e, t) => e + t.length, 0);
-	a.epochSteps = Math.ceil(g / ((a?.batchSize || 32) * o.config.blockSize));
+	a.epochSteps = Math.max(1, Math.floor(g / ((a?.batchSize || 32) * o.config.blockSize)));
 	let _ = new n(s, o.config.blockSize);
 	if (h) {
 		let { trainDataset: e, validationDataset: t } = await r(m, h, s, _, a?.batchSize || 32);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "1.0.3",
+    "version": "1.1.0",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",