@genai-fi/nanogpt 0.20.2 → 0.20.4

package/dist/Generator.js

@@ -1,2 +1,2 @@
-import { c as e, s as t } from "./main-D5CbfCiV.js";
+import { c as e, s as t } from "./main-BqRjvJ8B.js";
 export { t as default, e as isConversation };

package/dist/TeachableLLM.d.ts

@@ -44,7 +44,7 @@ export default class TeachableLLM {
     private setStatus;
     saveModel(options?: SaveOptions): Promise<Blob>;
     static loadModel(data: Blob | Buffer | string, options?: LoadModelOptions): TeachableLLM;
-    static create(tokeniserType: 'char' | 'bpe', config: GPTConfig): TeachableLLM;
+    static create(tokeniserType: 'char' | 'bpe' | ITokeniser, config: GPTConfig): TeachableLLM;
     getProfiler(): MemoryProfiler | undefined;
     get enableProfiler(): boolean;
     set enableProfiler(value: boolean);

package/dist/TeachableLLM.js

@@ -1,2 +1,2 @@
-import { i as e } from "./main-D5CbfCiV.js";
+import { i as e } from "./main-BqRjvJ8B.js";
 export { e as default };

package/dist/Trainer.js

@@ -1,2 +1,2 @@
-import { a as e } from "./main-D5CbfCiV.js";
+import { a as e } from "./main-BqRjvJ8B.js";
 export { e as default };

package/dist/loader/load.js

@@ -1,2 +1,2 @@
-import { d as e, u as t } from "../main-D5CbfCiV.js";
+import { d as e, u as t } from "../main-BqRjvJ8B.js";
 export { t as VERSION, e as loadModel };

package/dist/loader/loadHF.js

@@ -1,2 +1,2 @@
-import { f as e } from "../main-D5CbfCiV.js";
+import { f as e } from "../main-BqRjvJ8B.js";
 export { e as default };

package/dist/loader/loadTransformers.js

@@ -1,2 +1,2 @@
-import { h as e, m as t } from "../main-D5CbfCiV.js";
+import { h as e, m as t } from "../main-BqRjvJ8B.js";
 export { t as default, e as mapTransformersConfigToGPTConfig };

package/dist/loader/newZipLoad.js

@@ -1,2 +1,2 @@
-import { p as e } from "../main-D5CbfCiV.js";
+import { p as e } from "../main-BqRjvJ8B.js";
 export { e as default };

package/dist/loader/oldZipLoad.js

@@ -1,2 +1,2 @@
-import { g as e } from "../main-D5CbfCiV.js";
+import { g as e } from "../main-BqRjvJ8B.js";
 export { e as default };

package/dist/loader/save.js

@@ -1,2 +1,2 @@
-import { l as e } from "../main-D5CbfCiV.js";
+import { l as e } from "../main-BqRjvJ8B.js";
 export { e as saveModel };

package/dist/{main-D5CbfCiV.js → main-BqRjvJ8B.js}

@@ -13414,7 +13414,7 @@ var DS = class e extends hc {
 	}
 	static create(t, n) {
 		pc(n);
-		let r = n, i = t === "char" ? new gc(r.vocabSize) : new Sc(r.vocabSize), a = nS(r), o = new e(i, a);
+		let r = n, i = t === "char" ? new gc(r.vocabSize) : t === "bpe" ? new Sc(r.vocabSize) : t, a = nS(r), o = new e(i, a);
 		return o.setStatus("warmup"), xc(a).then((e) => {
 			o._memoryRequirements = e, o.tokeniser.trained ? (o.setStatus("ready"), o.ee.emit("loaded"), o.ee.emit("mode", o.mode)) : (o.setStatus("awaitingTokens"), o.ee.emit("loaded"), o.ee.emit("mode", o.mode), o.tokeniser.once("trainStatus", (e) => {
 				e === "trained" && o.setStatus("ready");

package/dist/main.js

@@ -1,4 +1,4 @@
-import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-D5CbfCiV.js";
+import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-BqRjvJ8B.js";
 import c from "./tokeniser/CharTokeniser.js";
 import l from "./tokeniser/bpe.js";
 import { AdamWOptimizer as u } from "./training/AdamW.js";

package/dist/models/NanoGPTV1.js

@@ -1,2 +1,2 @@
-import { y as e } from "../main-D5CbfCiV.js";
+import { y as e } from "../main-BqRjvJ8B.js";
 export { e as default };

package/dist/models/NanoGPTV2.js

@@ -1,2 +1,2 @@
-import { v as e } from "../main-D5CbfCiV.js";
+import { v as e } from "../main-BqRjvJ8B.js";
 export { e as default };

package/dist/models/factory.js

@@ -1,2 +1,2 @@
-import { _ as e } from "../main-D5CbfCiV.js";
+import { _ as e } from "../main-BqRjvJ8B.js";
 export { e as default };

package/dist/models/model.js

@@ -1,2 +1,2 @@
-import { b as e } from "../main-D5CbfCiV.js";
+import { b as e } from "../main-BqRjvJ8B.js";
 export { e as default };

package/dist/training/tasks/Task.js

@@ -11,10 +11,10 @@ function n(e, t, n, r, i, a) {
 			if (r.offset + n.length > o.length) {
 				let c = o.length - r.offset;
 				o.set(n.slice(0, c), r.offset);
-				let l = new Uint16Array(Math.floor(i * .1) + 100);
-				if (l.set(n.slice(c), 0), t.push(l), a && s && !Array.isArray(e)) {
+				let l = n.length - c, u = new Uint16Array(Math.max(Math.floor(i * .1) + 100, l));
+				if (u.set(n.slice(c), 0), t.push(u), a && s && !Array.isArray(e)) {
 					s.set(e.mask.slice(0, c).map((e) => +!!e), r.offset);
-					let t = new Uint8Array(l.length);
+					let t = new Uint8Array(u.length);
 					t.set(e.mask.slice(c).map((e) => +!!e), 0), a.push(t);
 				}
 				r.offset = n.length - c;

package/dist/training/validation.js

@@ -1,2 +1,2 @@
-import { o as e } from "../main-D5CbfCiV.js";
+import { o as e } from "../main-BqRjvJ8B.js";
 export { e as createTrainValidationSplit };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.20.2",
+    "version": "0.20.4",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",