@genai-fi/nanogpt 0.20.5 → 0.20.7

package/dist/Generator.js

@@ -1,2 +1,2 @@
-import { c as e, s as t } from "./main-PgJdpFTC.js";
+import { c as e, s as t } from "./main-BTmbwF1h.js";
 export { t as default, e as isConversation };

package/dist/TeachableLLM.js

@@ -1,2 +1,2 @@
-import { i as e } from "./main-PgJdpFTC.js";
+import { i as e } from "./main-BTmbwF1h.js";
 export { e as default };

package/dist/Trainer.js

@@ -1,2 +1,2 @@
-import { a as e } from "./main-PgJdpFTC.js";
+import { a as e } from "./main-BTmbwF1h.js";
 export { e as default };

package/dist/checks/rope.js

@@ -19,11 +19,11 @@ async function i(i) {
 		useRope: !0
 	});
 	o.ensureRopeCache(120);
-	let s = t().runKernel("Rope", {
-		x: a,
-		sin: o.getSin(),
-		cos: o.getCos()
-	}, { pastLen: 20 });
+	let s = t().runKernel("Rope", { x: a }, {
+		pastLen: 20,
+		ropeCache: o,
+		negSin: !1
+	});
 	return Array.isArray(s) ? s.map((e) => e.array()) : s.array();
 }
 //#endregion

package/dist/loader/load.js

@@ -1,2 +1,2 @@
-import { d as e, u as t } from "../main-PgJdpFTC.js";
+import { d as e, u as t } from "../main-BTmbwF1h.js";
 export { t as VERSION, e as loadModel };

package/dist/loader/loadHF.js

@@ -1,2 +1,2 @@
-import { f as e } from "../main-PgJdpFTC.js";
+import { f as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/loader/loadTransformers.js

@@ -1,2 +1,2 @@
-import { h as e, m as t } from "../main-PgJdpFTC.js";
+import { h as e, m as t } from "../main-BTmbwF1h.js";
 export { t as default, e as mapTransformersConfigToGPTConfig };

package/dist/loader/newZipLoad.js

@@ -1,2 +1,2 @@
-import { p as e } from "../main-PgJdpFTC.js";
+import { p as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/loader/oldZipLoad.js

@@ -1,2 +1,2 @@
-import { g as e } from "../main-PgJdpFTC.js";
+import { g as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/loader/save.js

@@ -1,2 +1,2 @@
-import { l as e } from "../main-PgJdpFTC.js";
+import { l as e } from "../main-BTmbwF1h.js";
 export { e as saveModel };

package/dist/{main-PgJdpFTC.js → main-BTmbwF1h.js}

@@ -12829,7 +12829,7 @@ var bS = class extends hc {
 	}
 	async tokenisePrompt(e, t, n) {
 		if (t) {
-			let r = t.length > 0 && t[t.length - 1].role === "assistant", i = [];
+			let r = t.length > 0 && t[t.length - 1].role === "text", i = [];
 			return i = n?.nonConversational ? r && n?.continuation ? [e.bosToken, ...e.encode(t[t.length - 1].content)] : e.encodeAsSequence(t, !0) : e.encodeConversation(t, !0), i.length > this.model.config.blockSize && (i = i.slice(-this.model.config.blockSize)), ce([i], [1, i.length], "int32");
 		} else {
 			let t = n?.nonConversational ? void 0 : e.getSpecialTokenIndex("<|assistant_start|>"), r = t === void 0 ? [e.bosToken] : [e.bosToken, t];
@@ -12947,32 +12947,32 @@ var bS = class extends hc {
 			multinomialRand: p
 		};
 	}
-	async _generate(e) {
-		let t = !1;
-		this.outputConversation.length === 0 || this.outputConversation[this.outputConversation.length - 1]._completed || this.outputConversation[this.outputConversation.length - 1].role !== "assistant" ? (this.outputConversation.push({
-			role: "assistant",
+	async _generate(e, t) {
+		let n = !1;
+		this.outputConversation.length === 0 || this.outputConversation[this.outputConversation.length - 1]._completed || this.outputConversation[this.outputConversation.length - 1].role !== "assistant" && e?.nonConversational !== !0 || this.outputConversation[this.outputConversation.length - 1].role !== "text" && e?.nonConversational === !0 ? (this.outputConversation.push({
+			role: e?.nonConversational === !0 ? "text" : "assistant",
 			content: "",
 			_timestamp: Date.now()
-		}), t = !0, this.resetCache(!e?.noCache)) : this.lastToken < 0 && this.resetCache(!e?.noCache);
-		let n = this.lastToken >= 0 && this.cache ? ce([this.lastToken], [1, 1], "int32") : await this.tokenisePrompt(this.actualTokeniser, t ? this.outputConversation.slice(0, -1) : this.outputConversation, e), r = e?.maxLength ?? 1e3;
-		for (let t = 0; t < r && this.active; t++) {
-			let { output: i, probabilities: a, attention: o, loss: s, multinomialRand: c } = await this._generateToken(n, this.cache ? this.cache : void 0, {
+		}), n = !0, this.resetCache(!e?.noCache)) : (this.lastToken < 0 || t) && this.resetCache(!e?.noCache);
+		let r = this.lastToken >= 0 && this.cache ? ce([this.lastToken], [1, 1], "int32") : await this.tokenisePrompt(this.actualTokeniser, t ? n ? this.outputConversation.slice(0, -1) : this.outputConversation : void 0, e), i = e?.maxLength ?? 1e3;
+		for (let t = 0; t < i && this.active; t++) {
+			let { output: n, probabilities: a, attention: o, loss: s, multinomialRand: c } = await this._generateToken(r, this.cache ? this.cache : void 0, {
 				...e,
 				usePadding: !this.cache
 			});
-			if (this.lastMultinomialRand = c, s !== void 0 && (this.lastLoss = s), this.cache) n.dispose(), n = i;
+			if (this.lastMultinomialRand = c, s !== void 0 && (this.lastLoss = s), this.cache) r.dispose(), r = n;
 			else {
-				let e = n;
-				n = kt([n, i], 1), e.dispose();
+				let e = r;
+				r = kt([r, n], 1), e.dispose();
 			}
-			let l = await this.processResponse(this.actualTokeniser, i, e?.allowSpecial ?? !1, o, a);
-			if (this.cache || i.dispose(), l === null) {
+			let l = await this.processResponse(this.actualTokeniser, n, e?.allowSpecial ?? !1, o, a);
+			if (this.cache || n.dispose(), l === null) {
 				this.outputConversation[this.outputConversation.length - 1]._completed = !0;
 				break;
 			}
-			t === r - 1 && r > 1 && (this.outputConversation[this.outputConversation.length - 1]._completed = !0), this.outputConversation[this.outputConversation.length - 1].content += l;
+			t === i - 1 && i > 1 && (this.outputConversation[this.outputConversation.length - 1]._completed = !0), this.outputConversation[this.outputConversation.length - 1].content += l;
 		}
-		return n.dispose(), this.outputConversation;
+		return r.dispose(), this.outputConversation;
 	}
 	resetCache(e) {
 		this.cache && (this.cache.forEach((e) => {
@@ -13036,7 +13036,7 @@ var bS = class extends hc {
 	}
 	async startJob(e, t) {
 		this.initialise(e, t), this.active = !0, this.model.metaData.generationSettings = t, t?.maxLength !== 1 && this.emit("start");
-		let n = await this._generate(t);
+		let n = await this._generate(t, !!e);
 		if (this.active = !1, this.startTime !== null) {
 			let e = Date.now(), n = e - this.startTime;
 			this.startTime = null, this.model.metaData.actionLog = this.model.metaData.actionLog || [], this.model.metaData.actionLog.push({
@@ -13163,13 +13163,15 @@ var DS = class e extends hc {
 		}
 		if (!n) throw Error("Tokeniser must be provided when initializing Trainer with a model");
 		if (!t) throw Error("Model must be provided when initializing Trainer");
-		if (this.options = i || {
+		this.options = i || {
 			batchSize: 32,
 			sftMode: "full"
-		}, r === "sft") {
-			let e = new Ac(t, n, i, a);
+		};
+		let o = this.options.mixedPrecision && tc();
+		if (this.options.lossScaling = o ? t.lossScaling : 1, r === "sft") {
+			let e = new Ac(t, n, this.options, a);
 			this.trainer = e, e.loraName = i?.loraName;
-		} else this.trainer = new kc(t, n, i, a);
+		} else this.trainer = new kc(t, n, this.options, a);
 		this.trainingType = r || "pretraining", this.tokenizer = n;
 	}
 	get model() {

package/dist/main.js

@@ -1,4 +1,4 @@
-import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-PgJdpFTC.js";
+import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-BTmbwF1h.js";
 import c from "./tokeniser/CharTokeniser.js";
 import l from "./tokeniser/bpe.js";
 import { AdamWOptimizer as u } from "./training/AdamW.js";

package/dist/models/NanoGPTV1.js

@@ -1,2 +1,2 @@
-import { y as e } from "../main-PgJdpFTC.js";
+import { y as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/models/NanoGPTV2.js

@@ -1,2 +1,2 @@
-import { v as e } from "../main-PgJdpFTC.js";
+import { v as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/models/factory.js

@@ -1,2 +1,2 @@
-import { _ as e } from "../main-PgJdpFTC.js";
+import { _ as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/models/model.js

@@ -1,2 +1,2 @@
-import { b as e } from "../main-PgJdpFTC.js";
+import { b as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/ops/webgl/adamMoments.js

@@ -19,8 +19,8 @@ var t = class {
             float m = getMomentsAtOutCoords();
             ${n} coords = getOutputCoords();
 
-            // Add gradient clipping here
-            float g = clamp(getGradient(${i}), -1.0, 1.0);
+            // TODO: Add scaling
+            float g = getGradient(${i});
             int which = ${r};
 
             float beta = which == 0 ? beta1 : beta2;

package/dist/ops/webgl/matMul16.js

@@ -7,12 +7,16 @@ function d(e) {
 	let { A: d, B: f } = e.inputs, { transposeA: p, transposeB: m, scale: h, activation: g, scaleA: _, scaleB: v, forceOutputShape: y, perm: b } = e.attrs, x = !c(d), S = !c(f);
 	if (x && S) {
 		let e = _ === void 0 ? d : t(d, n(_)), c = v === void 0 ? f : t(f, n(v)), x;
-		if (h !== void 0) x = l(e, c, n(h), p, m);
-		else if (g === "gelu") x = u(e, c);
+		if (g === "gelu") x = u(e, c);
 		else if (g === "relu2") {
 			let t = i(e, c, p, m), n = o(t);
 			t.dispose(), x = a(n), n.dispose();
-		} else x = g === "relu" ? o(i(e, c, p, m)) : i(e, c, p, m);
+		} else if (g === "relu") {
+			if (x = o(i(e, c, p, m)), h !== void 0) {
+				let e = t(x, n(h));
+				x.dispose(), x = e;
+			}
+		} else x = h === void 0 ? i(e, c, p, m) : l(e, c, n(h), p, m);
 		if (b) if (y) {
 			let e = r(x, y);
 			x.dispose();

package/dist/ops/webgpu/matMul16.js

@@ -11,12 +11,16 @@ function h(e) {
 		let e = x === void 0 ? h : n(h, r(x)), t = S === void 0 ? g : n(g, r(S));
 		if (T) throw Error("Causal mask is not supported for unpacked MatMul16.");
 		let c;
-		if (y !== void 0) c = f(e, t, r(y), _, v);
-		else if (b === "gelu") c = p(e, t);
+		if (b === "gelu") c = p(e, t);
 		else if (b === "relu2") {
 			let n = a(e, t, _, v), r = s(n);
 			n.dispose(), c = o(r), r.dispose();
-		} else c = b === "relu" ? s(a(e, t, _, v)) : a(e, t, _, v);
+		} else if (b === "relu") {
+			if (c = s(a(e, t, _, v)), y !== void 0) {
+				let e = n(c, r(y));
+				c.dispose(), c = e;
+			}
+		} else c = y === void 0 ? a(e, t, _, v) : f(e, t, r(y), _, v);
 		if (w) if (C) {
 			let e = i(c, C);
 			c.dispose();

package/dist/ops/webgpu/normRMS.js

@@ -7,7 +7,7 @@ import s from "./normRMS32_program.js";
 import c from "./normRMS16_program.js";
 //#region lib/ops/webgpu/normRMS.ts
 function l(t) {
-	let { x: l, gamma: u } = t.inputs, d = t.backend, f = o(d), p = n(l), m = u ? n(u) : !1, h = p || m, g = !h || h ? l : r(l), _ = !h || !u ? u : r(u), v = _ ? [g, _] : [g], y = i(v, -1), b = h ? new c(f, y, !!u) : new s(f, y, !!u);
+	let { x: l, gamma: u } = t.inputs, d = t.backend, f = o(d), p = n(l), m = u ? n(u) : !1, h = p || m, g = !h || p ? l : r(l), _ = u && (!h || m ? u : r(u)), v = _ ? [g, _] : [g], y = i(v, -1), b = h ? new c(f, y, !!u) : new s(f, y, !!u);
 	if (_ && e(_.shape, [g.shape[g.shape.length - 1]], "Error in RMSNorm: "), l.shape.length !== 3) throw Error(`rmsNormGPU: input rank ${l.shape.length} not supported, only rank 3 is supported`);
 	if (y.inSize !== g.shape[g.shape.length - 1]) throw Error(`rmsNormGPU: reduction size ${y.inSize} does not match expected size ${g.shape[g.shape.length - 1]}`);
 	if (y.batchSize !== l.shape[0] * l.shape[1]) throw Error(`rmsNormGPU: batch size ${y.batchSize} does not match expected size ${l.shape[0] * l.shape[1]}`);

package/dist/training/BasicTrainer.js

@@ -39,7 +39,7 @@ var f = {
 		this.tokenizer = t, this.model = e, this.optimizerConfig = {
 			...p,
 			...n,
-			lossScaling: e.lossScaling
+			lossScaling: n?.lossScaling ?? e.lossScaling
 		};
 		let i = r || new o(this.optimizerConfig);
 		r && r.updateConfig(this.optimizerConfig), this.optimizer = i;

package/dist/training/validation.js

@@ -1,2 +1,2 @@
-import { o as e } from "../main-PgJdpFTC.js";
+import { o as e } from "../main-BTmbwF1h.js";
 export { e as createTrainValidationSplit };

package/dist/utilities/arrayClose.d.ts

@@ -1 +1,2 @@
 export declare function arraysClose(a: unknown, b: unknown): number;
+export declare function arraysClosePercentile(a: unknown, b: unknown, percentile?: number): number;

package/dist/utilities/arrayClose.js

@@ -12,5 +12,17 @@ function e(t, n) {
 		return r = Math.max(r, e), r;
 	} else return Infinity;
 }
+function t(e, t, n = .99) {
+	let r = [];
+	function i(e, t) {
+		if ((Array.isArray(e) || e instanceof Float32Array) && (Array.isArray(t) || t instanceof Float32Array)) {
+			if (e.length !== t.length) return !1;
+			for (let n = 0; n < e.length; n++) if (!i(e[n], t[n])) return !1;
+			return !0;
+		}
+		return typeof e == "number" && typeof t == "number" ? Number.isNaN(e) && Number.isNaN(t) ? (r.push(0), !0) : !Number.isFinite(e) || !Number.isFinite(t) ? e === t : (r.push(Math.abs(e - t)), !0) : !1;
+	}
+	return !i(e, t) || r.length === 0 ? Infinity : (r.sort((e, t) => e - t), r[Math.min(r.length - 1, Math.floor(n * r.length))]);
+}
 //#endregion
-export { e as arraysClose };
+export { e as arraysClose, t as arraysClosePercentile };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.20.5",
+    "version": "0.20.7",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",