@genai-fi/nanogpt 1.0.2 → 1.0.4

package/dist/data/textLoader.js

@@ -523,7 +523,7 @@ function d(e) {
 }
 async function f(e, t) {
 	let u = e.type === "" ? e instanceof File ? d(e.name) : "application/zip" : e.type;
-	if (u === "application/parquet") throw Error("Parquet loading is not currently supported in the browser. Please convert your data to JSONL format.");
+	if (u === "application/parquet") throw Error("unsupported_file_type");
 	if (u === "application/pdf") return new i(await n(e, t?.maxSize));
 	if (u === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") return new i(await r(e));
 	if (u === "application/json") {
@@ -538,7 +538,7 @@ async function f(e, t) {
 			role: "text",
 			content: JSON.stringify(e)
 		}]));
-		throw Error("Expected JSON array");
+		throw Error("bad_format");
 	}
 	if (u === "application/jsonl") return new o(e);
 	if (u === "application/zip") return new a(e);
@@ -550,7 +550,7 @@ async function f(e, t) {
 				skipEmptyLines: !0,
 				delimiter: ",",
 				complete: (n) => {
-					if (n.errors.length > 0) console.error(n.errors), r(/* @__PURE__ */ Error("Error parsing file"));
+					if (n.errors.length > 0) console.error(n.errors), r(/* @__PURE__ */ Error("bad_format"));
 					else {
 						let r = c(n.data[0], t?.column || "text");
 						e(new i((t?.hasHeader ?? l(n.data[0]) ? n.data.slice(1) : n.data).map((e) => [{
@@ -568,7 +568,7 @@ async function f(e, t) {
 		role: "text",
 		content: await e.text()
 	}]]);
-	throw Error(`Unsupported file type: ${u}`);
+	throw Error("unsupported_file_type");
 }
 //#endregion
 export { f as default };

package/dist/inference/Generator.d.ts

@@ -34,6 +34,7 @@ export default class Generator extends EE<'start' | 'stop' | 'tokens' | 'reset'>
     private startTime;
     private tokenCount;
     constructor(model: Model<ModelForwardAttributes>, tokeniser: ITokeniser);
+    private shouldTerminate;
     /** Generate logits and select a token. */
     private _generateToken;
     /** Generate multiple tokens in a loop and produce text */

package/dist/inference/Generator.js

@@ -40,6 +40,11 @@ var b = class extends e {
 	constructor(e, t) {
 		super(), this.model = e, this.tokeniser = t, this.actualTokeniser = t;
 	}
+	shouldTerminate(e, t) {
+		if (e) return !1;
+		let n = this.tokeniser.getSpecialTokenIndex("<|assistant_end|>");
+		return t === this.actualTokeniser.eosToken || t === n;
+	}
 	async _generateToken(e, t, n) {
 		let s = n?.temperature ?? 1, h = n?.topK, _ = n?.topP, v = n?.usePadding ?? !1, y = {
 			training: !1,
@@ -132,7 +137,7 @@ var b = class extends e {
 		S.dispose(), S = E;
 		let D = (await S.array())[0][0], O = this.actualTokeniser.decode([D]);
 		this.lastToken = D;
-		let k = !n?.allowSpecial && this.tokeniser.isSpecialToken(D), A = {
+		let k = this.shouldTerminate(n?.allowSpecial ?? !1, D), A = {
 			outputTensor: S,
 			token: D,
 			text: O,

package/dist/loader/loadZipMeta.js

@@ -7,7 +7,7 @@ async function e(e) {
 	if (t) try {
 		n = JSON.parse(t);
 	} catch (e) {
-		throw console.error(e), Error("Failed to parse meta.json in the zip archive", { cause: e });
+		throw console.error(e), Error("bad_format", { cause: e });
 	}
 	else console.warn("meta.json not found in the zip archive, using default metadata");
 	return n;

package/dist/loader/types.d.ts

@@ -36,6 +36,7 @@ export interface DatasetMetadata {
     id: string;
     name: string;
     conversational: boolean;
+    url?: string;
 }
 export interface ActionLogEntry {
     action: 'pretrain' | 'generate' | 'finetune';

package/dist/models/config.js

@@ -31,7 +31,7 @@ function n(e) {
 		if (e.windowSize !== void 0 && typeof e.windowSize != "string") throw Error("Invalid config: \"windowSize\" must be a string for GenAI_NanoGPT_v2.");
 		return;
 	}
-	throw Error("Invalid config: \"modelType\" must be \"GenAI_NanoGPT_v1\" or \"GenAI_NanoGPT_v2\".");
+	throw Error("invalid_model_type");
 }
 //#endregion
 export { e as defaultConfig, n as validateConfig };