@genai-fi/nanogpt 0.20.6 → 0.20.7

package/dist/Generator.js

@@ -1,2 +1,2 @@
-import { c as e, s as t } from "./main-C0YUnw98.js";
+import { c as e, s as t } from "./main-BTmbwF1h.js";
 export { t as default, e as isConversation };

package/dist/TeachableLLM.js

@@ -1,2 +1,2 @@
-import { i as e } from "./main-C0YUnw98.js";
+import { i as e } from "./main-BTmbwF1h.js";
 export { e as default };

package/dist/Trainer.js

@@ -1,2 +1,2 @@
-import { a as e } from "./main-C0YUnw98.js";
+import { a as e } from "./main-BTmbwF1h.js";
 export { e as default };

package/dist/checks/rope.js

@@ -19,11 +19,11 @@ async function i(i) {
 		useRope: !0
 	});
 	o.ensureRopeCache(120);
-	let s = t().runKernel("Rope", {
-		x: a,
-		sin: o.getSin(),
-		cos: o.getCos()
-	}, { pastLen: 20 });
+	let s = t().runKernel("Rope", { x: a }, {
+		pastLen: 20,
+		ropeCache: o,
+		negSin: !1
+	});
 	return Array.isArray(s) ? s.map((e) => e.array()) : s.array();
 }
 //#endregion

package/dist/loader/load.js

@@ -1,2 +1,2 @@
-import { d as e, u as t } from "../main-C0YUnw98.js";
+import { d as e, u as t } from "../main-BTmbwF1h.js";
 export { t as VERSION, e as loadModel };

package/dist/loader/loadHF.js

@@ -1,2 +1,2 @@
-import { f as e } from "../main-C0YUnw98.js";
+import { f as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/loader/loadTransformers.js

@@ -1,2 +1,2 @@
-import { h as e, m as t } from "../main-C0YUnw98.js";
+import { h as e, m as t } from "../main-BTmbwF1h.js";
 export { t as default, e as mapTransformersConfigToGPTConfig };

package/dist/loader/newZipLoad.js

@@ -1,2 +1,2 @@
-import { p as e } from "../main-C0YUnw98.js";
+import { p as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/loader/oldZipLoad.js

@@ -1,2 +1,2 @@
-import { g as e } from "../main-C0YUnw98.js";
+import { g as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/loader/save.js

@@ -1,2 +1,2 @@
-import { l as e } from "../main-C0YUnw98.js";
+import { l as e } from "../main-BTmbwF1h.js";
 export { e as saveModel };

package/dist/{main-C0YUnw98.js → main-BTmbwF1h.js}

@@ -13163,13 +13163,15 @@ var DS = class e extends hc {
 		}
 		if (!n) throw Error("Tokeniser must be provided when initializing Trainer with a model");
 		if (!t) throw Error("Model must be provided when initializing Trainer");
-		if (this.options = i || {
+		this.options = i || {
 			batchSize: 32,
 			sftMode: "full"
-		}, r === "sft") {
-			let e = new Ac(t, n, i, a);
+		};
+		let o = this.options.mixedPrecision && tc();
+		if (this.options.lossScaling = o ? t.lossScaling : 1, r === "sft") {
+			let e = new Ac(t, n, this.options, a);
 			this.trainer = e, e.loraName = i?.loraName;
-		} else this.trainer = new kc(t, n, i, a);
+		} else this.trainer = new kc(t, n, this.options, a);
 		this.trainingType = r || "pretraining", this.tokenizer = n;
 	}
 	get model() {

package/dist/main.js

@@ -1,4 +1,4 @@
-import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-C0YUnw98.js";
+import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-BTmbwF1h.js";
 import c from "./tokeniser/CharTokeniser.js";
 import l from "./tokeniser/bpe.js";
 import { AdamWOptimizer as u } from "./training/AdamW.js";

package/dist/models/NanoGPTV1.js

@@ -1,2 +1,2 @@
-import { y as e } from "../main-C0YUnw98.js";
+import { y as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/models/NanoGPTV2.js

@@ -1,2 +1,2 @@
-import { v as e } from "../main-C0YUnw98.js";
+import { v as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/models/factory.js

@@ -1,2 +1,2 @@
-import { _ as e } from "../main-C0YUnw98.js";
+import { _ as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/models/model.js

@@ -1,2 +1,2 @@
-import { b as e } from "../main-C0YUnw98.js";
+import { b as e } from "../main-BTmbwF1h.js";
 export { e as default };

package/dist/ops/webgl/adamMoments.js

@@ -19,8 +19,8 @@ var t = class {
             float m = getMomentsAtOutCoords();
             ${n} coords = getOutputCoords();
 
-            // Add gradient clipping here
-            float g = clamp(getGradient(${i}), -1.0, 1.0);
+            // TODO: Add scaling
+            float g = getGradient(${i});
             int which = ${r};
 
             float beta = which == 0 ? beta1 : beta2;

package/dist/ops/webgl/matMul16.js

@@ -7,12 +7,16 @@ function d(e) {
 	let { A: d, B: f } = e.inputs, { transposeA: p, transposeB: m, scale: h, activation: g, scaleA: _, scaleB: v, forceOutputShape: y, perm: b } = e.attrs, x = !c(d), S = !c(f);
 	if (x && S) {
 		let e = _ === void 0 ? d : t(d, n(_)), c = v === void 0 ? f : t(f, n(v)), x;
-		if (h !== void 0) x = l(e, c, n(h), p, m);
-		else if (g === "gelu") x = u(e, c);
+		if (g === "gelu") x = u(e, c);
 		else if (g === "relu2") {
 			let t = i(e, c, p, m), n = o(t);
 			t.dispose(), x = a(n), n.dispose();
-		} else x = g === "relu" ? o(i(e, c, p, m)) : i(e, c, p, m);
+		} else if (g === "relu") {
+			if (x = o(i(e, c, p, m)), h !== void 0) {
+				let e = t(x, n(h));
+				x.dispose(), x = e;
+			}
+		} else x = h === void 0 ? i(e, c, p, m) : l(e, c, n(h), p, m);
 		if (b) if (y) {
 			let e = r(x, y);
 			x.dispose();

package/dist/ops/webgpu/matMul16.js

@@ -11,12 +11,16 @@ function h(e) {
 		let e = x === void 0 ? h : n(h, r(x)), t = S === void 0 ? g : n(g, r(S));
 		if (T) throw Error("Causal mask is not supported for unpacked MatMul16.");
 		let c;
-		if (y !== void 0) c = f(e, t, r(y), _, v);
-		else if (b === "gelu") c = p(e, t);
+		if (b === "gelu") c = p(e, t);
 		else if (b === "relu2") {
 			let n = a(e, t, _, v), r = s(n);
 			n.dispose(), c = o(r), r.dispose();
-		} else c = b === "relu" ? s(a(e, t, _, v)) : a(e, t, _, v);
+		} else if (b === "relu") {
+			if (c = s(a(e, t, _, v)), y !== void 0) {
+				let e = n(c, r(y));
+				c.dispose(), c = e;
+			}
+		} else c = y === void 0 ? a(e, t, _, v) : f(e, t, r(y), _, v);
 		if (w) if (C) {
 			let e = i(c, C);
 			c.dispose();

package/dist/ops/webgpu/normRMS.js

@@ -7,7 +7,7 @@ import s from "./normRMS32_program.js";
 import c from "./normRMS16_program.js";
 //#region lib/ops/webgpu/normRMS.ts
 function l(t) {
-	let { x: l, gamma: u } = t.inputs, d = t.backend, f = o(d), p = n(l), m = u ? n(u) : !1, h = p || m, g = !h || h ? l : r(l), _ = !h || !u ? u : r(u), v = _ ? [g, _] : [g], y = i(v, -1), b = h ? new c(f, y, !!u) : new s(f, y, !!u);
+	let { x: l, gamma: u } = t.inputs, d = t.backend, f = o(d), p = n(l), m = u ? n(u) : !1, h = p || m, g = !h || p ? l : r(l), _ = u && (!h || m ? u : r(u)), v = _ ? [g, _] : [g], y = i(v, -1), b = h ? new c(f, y, !!u) : new s(f, y, !!u);
 	if (_ && e(_.shape, [g.shape[g.shape.length - 1]], "Error in RMSNorm: "), l.shape.length !== 3) throw Error(`rmsNormGPU: input rank ${l.shape.length} not supported, only rank 3 is supported`);
 	if (y.inSize !== g.shape[g.shape.length - 1]) throw Error(`rmsNormGPU: reduction size ${y.inSize} does not match expected size ${g.shape[g.shape.length - 1]}`);
 	if (y.batchSize !== l.shape[0] * l.shape[1]) throw Error(`rmsNormGPU: batch size ${y.batchSize} does not match expected size ${l.shape[0] * l.shape[1]}`);

package/dist/training/BasicTrainer.js

@@ -39,7 +39,7 @@ var f = {
 		this.tokenizer = t, this.model = e, this.optimizerConfig = {
 			...p,
 			...n,
-			lossScaling: e.lossScaling
+			lossScaling: n?.lossScaling ?? e.lossScaling
 		};
 		let i = r || new o(this.optimizerConfig);
 		r && r.updateConfig(this.optimizerConfig), this.optimizer = i;

package/dist/training/validation.js

@@ -1,2 +1,2 @@
-import { o as e } from "../main-C0YUnw98.js";
+import { o as e } from "../main-BTmbwF1h.js";
 export { e as createTrainValidationSplit };

package/dist/utilities/arrayClose.d.ts

@@ -1 +1,2 @@
 export declare function arraysClose(a: unknown, b: unknown): number;
+export declare function arraysClosePercentile(a: unknown, b: unknown, percentile?: number): number;

package/dist/utilities/arrayClose.js

@@ -12,5 +12,17 @@ function e(t, n) {
 		return r = Math.max(r, e), r;
 	} else return Infinity;
 }
+function t(e, t, n = .99) {
+	let r = [];
+	function i(e, t) {
+		if ((Array.isArray(e) || e instanceof Float32Array) && (Array.isArray(t) || t instanceof Float32Array)) {
+			if (e.length !== t.length) return !1;
+			for (let n = 0; n < e.length; n++) if (!i(e[n], t[n])) return !1;
+			return !0;
+		}
+		return typeof e == "number" && typeof t == "number" ? Number.isNaN(e) && Number.isNaN(t) ? (r.push(0), !0) : !Number.isFinite(e) || !Number.isFinite(t) ? e === t : (r.push(Math.abs(e - t)), !0) : !1;
+	}
+	return !i(e, t) || r.length === 0 ? Infinity : (r.sort((e, t) => e - t), r[Math.min(r.length - 1, Math.floor(n * r.length))]);
+}
 //#endregion
-export { e as arraysClose };
+export { e as arraysClose, t as arraysClosePercentile };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.20.6",
+    "version": "0.20.7",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",