@genai-fi/nanogpt 1.0.4 → 1.1.0

package/dist/api/training.d.ts

@@ -29,7 +29,8 @@ export interface ITrainingJob {
     totalTokens: number;
     datasets: DatasetMetadata[];
     datasetId?: string;
-    breakOnLog: boolean;
+    breakOnLog: Set<number>;
+    resumeCounter: number;
 }
 interface TrainingEvents {
     error: (id: string, error: Error) => void;
@@ -46,6 +47,7 @@ export default class Training {
     private _model;
     private _tokeniser;
     private _jobs;
+    private _breakCounter;
     constructor(model: Model<ModelForwardAttributes, GPTConfig>, tokeniser: ITokeniser);
     private setState;
     get activeJobs(): number;
@@ -63,7 +65,8 @@ export default class Training {
     /** Resume a paused job and optionally change some options. */
     resume(id: string): Promise<void>;
     cancel(id: string): void;
-    breakpoints(id: string, enabled: boolean): void;
+    addBreak(jobid: string): number | undefined;
+    deleteBreak(jobid: string, breakId: number): void;
     getPretrainingJob(): ITrainingJob | null;
     dispose(): void;
 }

package/dist/api/training.js

@@ -10,6 +10,7 @@ var o = class {
 	_model;
 	_tokeniser;
 	_jobs = /* @__PURE__ */ new Map();
+	_breakCounter = 1;
 	constructor(t, n) {
 		this.ee = new e(), this._model = t, this._tokeniser = n;
 	}
@@ -19,32 +20,31 @@ var o = class {
 		};
 		switch (t) {
 			case "pending":
-				e.state !== "completed" && e.state !== "paused" && e.state !== "cancelled" && e.state !== "pending" && r();
+				e.state !== "completed" && e.state !== "paused" && e.state !== "cancelled" && e.state !== "pending" && r(), e.state = t;
 				break;
 			case "running":
-				e.state !== "pending" && e.state !== "paused" && r(), this.ee.emit("running", e.id);
+				e.state !== "pending" && e.state !== "paused" && r(), e.state = t, this.ee.emit("running", e.id);
 				break;
 			case "paused":
-				e.state !== "pausing" && r(), this.ee.emit("paused", e.id);
+				e.state !== "pausing" && r(), e.state = t, this.ee.emit("paused", e.id);
 				break;
 			case "completed":
-				e.state !== "running" && r(), this.ee.emit("completed", e.id);
+				e.state !== "running" && r(), e.state = t, this.ee.emit("completed", e.id);
 				break;
 			case "pausing":
-				e.state !== "running" && r(), this.ee.emit("pausing", e.id);
+				e.state !== "running" && r(), e.state = t, this.ee.emit("pausing", e.id);
 				break;
 			case "cancelling":
-				e.state !== "running" && e.state !== "paused" && e.state !== "pausing" && r(), this.ee.emit("cancelling", e.id);
+				e.state !== "running" && e.state !== "paused" && e.state !== "pausing" && r(), e.state = t, this.ee.emit("cancelling", e.id);
 				break;
 			case "cancelled":
-				e.state !== "paused" && e.state !== "cancelling" && r(), this.ee.emit("cancelled", e.id);
+				e.state !== "paused" && e.state !== "cancelling" && r(), e.state = t, this.ee.emit("cancelled", e.id);
 				break;
 			case "error":
-				n && this.ee.emit("error", e.id, n);
+				e.state = t, n && this.ee.emit("error", e.id, n);
 				break;
 			default: throw Error("invalid_state");
 		}
-		e.state = t;
 	}
 	get activeJobs() {
 		return this._jobs.values().reduce((e, t) => t.state === "running" ? e + 1 : e, 0);
@@ -102,13 +102,14 @@ var o = class {
 			totalTokens: t[t.length - 1]?.totalTokens || 0,
 			datasets: o,
 			datasetId: a(o),
-			breakOnLog: !1
+			breakOnLog: /* @__PURE__ */ new Set(),
+			resumeCounter: 0
 		};
 		return s.trainer.log = t, s.trainer.resumeFromLog(t[t.length - 1]), this._jobs.set(s.id, s), s;
 	}
 	launchJob(e) {
 		e.trainDataset && (this.setState(e, "running"), e.trainer.trainOnDataset(e.trainDataset, e.options, e.validationDataset, (t) => {
-			e.history = e.trainer.log, e.progress = t.totalTokens / e.totalTokens, e.remaining = Math.max(0, (e.totalTokens - t.totalTokens) / t.totalTokens * t.duration), e.breakOnLog && e.state === "running" && (this.setState(e, "pausing"), e.trainer.stop()), this.ee.emit("progress", e);
+			e.history = e.trainer.log, e.progress = t.totalTokens / e.totalTokens, e.remaining = Math.max(0, (e.totalTokens - t.totalTokens) / t.totalTokens * t.duration), e.breakOnLog.size > 0 && e.state === "running" && (this.setState(e, "pausing"), e.trainer.stop()), this.ee.emit("progress", e);
 		}).then(() => {
 			e.state === "running" ? this.setState(e, "completed") : e.state === "pausing" ? this.setState(e, "paused") : e.state === "cancelling" && this.setState(e, "cancelled");
 		}).catch((t) => {
@@ -135,7 +136,8 @@ var o = class {
 			options: e,
 			totalTokens: 0,
 			datasets: s,
-			breakOnLog: !1
+			breakOnLog: /* @__PURE__ */ new Set(),
+			resumeCounter: 0
 		};
 		if (!u) throw Error("invalid_previous_job_id");
 		if (this._jobs.set(u.id, u), this.setState(u, "pending"), o instanceof t) {
@@ -175,7 +177,7 @@ var o = class {
 	async resume(e) {
 		this.assertValidJobId(e);
 		let t = this.getJob(e);
-		t && t.state === "paused" && (this.setState(t, "pending"), this.launchJob(t));
+		t && t.state === "paused" && (t.resumeCounter++, !(t.resumeCounter < t.breakOnLog.size) && (t.resumeCounter = 0, this.setState(t, "pending"), this.launchJob(t)));
 	}
 	cancel(e) {
 		this.assertValidJobId(e);
@@ -185,9 +187,16 @@ var o = class {
 			return;
 		} else (t.state === "running" || t.state === "pausing") && (this.setState(t, "cancelling"), t.trainer.stop());
 	}
-	breakpoints(e, t) {
+	addBreak(e) {
+		let t = this.getJob(e);
+		if (t) {
+			let e = this._breakCounter++;
+			return t.breakOnLog.add(e), e;
+		}
+	}
+	deleteBreak(e, t) {
 		let n = this.getJob(e);
-		n && (n.breakOnLog = t);
+		n && n.breakOnLog.delete(t);
 	}
 	getPretrainingJob() {
 		for (let e of this._jobs.values()) if (e.options.method.type === "pretraining") return e;

package/dist/training/BasicTrainer.js

@@ -234,7 +234,9 @@ var p = {
 					if (isNaN(e) || !isFinite(e)) throw console.error("Invalid loss value:", e), console.error("Batch xs:", await n.xs.array()), console.error("Batch ys:", await n.ys.array()), console.error("State:", l), Error("Loss is NaN or Infinity");
 					console.log(`Step ${l.step}: Loss = ${e}`);
 				}
-				n.xs.dispose(), n.ys.dispose(), l.step++, l.totalSteps++, r ? await this.performLogging(s, n.xs.shape[0], f, i) : (l.gradientNorm &&= (l.gradientNorm.dispose(), void 0), l.accuracy &&= (l.accuracy.dispose(), void 0)), s.dispose(), l.step >= c && this.stop();
+				n.xs.dispose(), n.ys.dispose(), l.step++, l.totalSteps++;
+				let u = l.step >= c;
+				r || u ? await this.performLogging(s, n.xs.shape[0], f, i) : (l.gradientNorm &&= (l.gradientNorm.dispose(), void 0), l.accuracy &&= (l.accuracy.dispose(), void 0)), s.dispose(), u && this.stop();
 			}
 		} catch (e) {
 			throw console.error("Training error:", e), r(), e;

package/dist/training/prepareData.js

@@ -28,7 +28,7 @@ async function a(a, o, s, c, l, u, d) {
 	}
 	else m = c;
 	let g = m instanceof e ? m.getTokenCount() : m.reduce((e, t) => e + t.length, 0);
-	a.epochSteps = Math.ceil(g / ((a?.batchSize || 32) * o.config.blockSize));
+	a.epochSteps = Math.max(1, Math.floor(g / ((a?.batchSize || 32) * o.config.blockSize)));
 	let _ = new n(s, o.config.blockSize);
 	if (h) {
 		let { trainDataset: e, validationDataset: t } = await r(m, h, s, _, a?.batchSize || 32);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "1.0.4",
+    "version": "1.1.0",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",