@genai-fi/nanogpt 1.0.3 → 1.0.4

package/dist/inference/Generator.d.ts

@@ -34,6 +34,7 @@ export default class Generator extends EE<'start' | 'stop' | 'tokens' | 'reset'>
     private startTime;
     private tokenCount;
     constructor(model: Model<ModelForwardAttributes>, tokeniser: ITokeniser);
+    private shouldTerminate;
     /** Generate logits and select a token. */
     private _generateToken;
     /** Generate multiple tokens in a loop and produce text */

package/dist/inference/Generator.js

@@ -40,6 +40,11 @@ var b = class extends e {
 	constructor(e, t) {
 		super(), this.model = e, this.tokeniser = t, this.actualTokeniser = t;
 	}
+	shouldTerminate(e, t) {
+		if (e) return !1;
+		let n = this.tokeniser.getSpecialTokenIndex("<|assistant_end|>");
+		return t === this.actualTokeniser.eosToken || t === n;
+	}
 	async _generateToken(e, t, n) {
 		let s = n?.temperature ?? 1, h = n?.topK, _ = n?.topP, v = n?.usePadding ?? !1, y = {
 			training: !1,
@@ -132,7 +137,7 @@ var b = class extends e {
 		S.dispose(), S = E;
 		let D = (await S.array())[0][0], O = this.actualTokeniser.decode([D]);
 		this.lastToken = D;
-		let k = !n?.allowSpecial && this.tokeniser.isSpecialToken(D), A = {
+		let k = this.shouldTerminate(n?.allowSpecial ?? !1, D), A = {
 			outputTensor: S,
 			token: D,
 			text: O,

package/dist/loader/types.d.ts

@@ -36,6 +36,7 @@ export interface DatasetMetadata {
     id: string;
     name: string;
     conversational: boolean;
+    url?: string;
 }
 export interface ActionLogEntry {
     action: 'pretrain' | 'generate' | 'finetune';

package/dist/training/prepareData.js

@@ -8,13 +8,14 @@ async function a(a, o, s, c, l, u, d) {
 	if (d && f) throw Error("Cannot specify datasets when using LoRA fine-tuning");
 	if (!d && !f) throw Error("Must specify datasets for non-LoRA training");
 	if (d) {
-		let e = o.metaData.pretrainingData || [], t = [...e], n = !1;
-		for (let r of d) e.some((e) => e.id === r.id) || t.push({
-			id: r.id,
-			name: r.name,
-			conversational: r.conversational
-		}), r.conversational && (n = !0);
-		o.metaData.pretrainingData = t, n ? o.metaData.mode = "conversational" : o.metaData.mode !== "conversational" && (o.metaData.mode = "completion");
+		let e = !1;
+		for (let t of d) t.conversational && (e = !0);
+		o.metaData.pretrainingData = d.map((e) => ({
+			id: e.id,
+			name: e.name,
+			conversational: e.conversational,
+			url: e.url
+		})), e ? o.metaData.mode = "conversational" : o.metaData.mode !== "conversational" && (o.metaData.mode = "completion");
 	} else o.metaData.mode !== "conversational" && (o.metaData.mode = "completion");
 	let p = a.maskedLoss ?? a.method.type === "supervised", m, h = u;
 	if (Array.isArray(c)) if (c[0] instanceof Uint16Array) m = c;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "1.0.3",
+    "version": "1.0.4",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",