@genai-fi/nanogpt 1.1.4 → 1.2.1

package/dist/TeachableLLM.d.ts

@@ -9,6 +9,7 @@ import { ModelMode, TransformersMetadata } from './loader/types';
 import { default as Responses } from './api/responses';
 import { default as Training } from './api/training';
 import { GPUOptions } from './patches/webgpu_base';
+import { default as BeamAPI } from './api/beamer';
 type TeachableLLMStatus = 'warmup' | 'awaitingTokens' | 'ready' | 'training' | 'loading' | 'busy' | 'error';
 export default class TeachableLLM {
     static instances: Set<TeachableLLM>;
@@ -20,6 +21,7 @@ export default class TeachableLLM {
     private _memoryRequirements?;
     private _responses;
     private _training;
+    private _beaming;
     meta: TransformersMetadata;
     static selectBackend(backend: 'cpu' | 'webgl' | 'webgpu', options?: GPUOptions): Promise<void>;
     constructor(tokeniser?: ITokeniser, model?: Model<ModelForwardAttributes, GPTConfig>);
@@ -53,6 +55,7 @@ export default class TeachableLLM {
     getNumParams(): number;
     trainTokeniser(text: ConversationStream[]): Promise<number>;
     get responses(): Responses;
+    get beaming(): BeamAPI;
     get training(): Training;
     dispose(): void;
     on(event: 'status', listener: (status: TeachableLLMStatus) => void): void;

package/dist/TeachableLLM.js

@@ -11,8 +11,9 @@ import l from "./api/responses.js";
 import u from "./api/training.js";
 import { selectBackend as d } from "./backend.js";
 import { getBackendDevice as f } from "./patches/webgpu_base.js";
+import p from "./api/beamer.js";
 //#region lib/TeachableLLM.ts
-var p = class p {
+var m = class m {
 	static instances = /* @__PURE__ */ new Set();
 	ee = new e();
 	_config;
@@ -22,6 +23,7 @@ var p = class p {
 	_memoryRequirements;
 	_responses = null;
 	_training = null;
+	_beaming = null;
 	meta = {
 		version: 2,
 		application: "@genai-fi/nanogpt"
@@ -30,14 +32,14 @@ var p = class p {
 		if (await d(e, t), e === "webgpu") {
 			let e = f();
 			e && e.lost.then(() => {
-				console.warn("WebGPU device lost"), p.instances.forEach((e) => {
+				console.warn("WebGPU device lost"), m.instances.forEach((e) => {
 					e.setStatus("error"), e.ee.emit("lost");
 				});
 			});
 		}
 	}
 	constructor(e, t) {
-		this._config = t?.config, this._tokeniser = e, this._model = t, t?.metaData && (this.meta = t.metaData), p.instances.add(this);
+		this._config = t?.config, this._tokeniser = e, this._model = t, t?.metaData && (this.meta = t.metaData), m.instances.add(this);
 	}
 	get vocab() {
 		return this._tokeniser?.getVocab() || [];
@@ -124,7 +126,7 @@ var p = class p {
 		} : void 0);
 	}
 	static loadModel(e, t) {
-		let n = new p();
+		let n = new m();
 		return o(e, t).then(({ model: e, tokeniser: t, metaData: a, optimizer: o, log: s }) => {
 			r(e.config), n._model = e, n._tokeniser = t, n._config = e.config, a && (n.meta = a), n.setStatus("warmup"), i(e).then((t) => {
 				n._memoryRequirements = t, o && e.metaData.pretrainingSettings && e.metaData.pretrainingData && n.training.restore(e.metaData.pretrainingSettings, s || [], o, e.metaData.pretrainingData), n.setStatus("ready"), n.ee.emit("loaded"), n.ee.emit("mode", n.mode);
@@ -137,7 +139,7 @@ var p = class p {
 	}
 	static create(e, o) {
 		r(o);
-		let s = o, c = e === "char" ? new t(s.vocabSize) : e === "bpe" ? new n(s.vocabSize) : e, l = a(s), u = new p(c, l);
+		let s = o, c = e === "char" ? new t(s.vocabSize) : e === "bpe" ? new n(s.vocabSize) : e, l = a(s), u = new m(c, l);
 		return u.setStatus("warmup"), i(l).then((e) => {
 			u._memoryRequirements = e, u.tokeniser.trained ? (u.setStatus("ready"), u.ee.emit("loaded"), u.ee.emit("mode", u.mode)) : (u.setStatus("awaitingTokens"), u.ee.emit("loaded"), u.ee.emit("mode", u.mode), u.tokeniser.once("trainStatus", (e) => {
 				e === "trained" && u.setStatus("ready");
@@ -177,6 +179,17 @@ var p = class p {
 		}
 		return this._responses;
 	}
+	get beaming() {
+		if (!this._beaming) {
+			if (!this._model || !this._tokeniser) throw Error("model_or_tokeniser_not_initialized.");
+			this._beaming = new p(this._model, this._tokeniser), this._beaming.on("error", (e) => {
+				this.setStatus("error"), this.ee.emit("error", e);
+			}), this._beaming.on("status", (e) => {
+				e === "busy" ? this.setStatus("busy") : e === "ready" && this.setStatus("ready");
+			});
+		}
+		return this._beaming;
+	}
 	get training() {
 		if (!this._training) {
 			if (!this._model || !this._tokeniser) throw Error("model_or_tokeniser_not_initialized.");
@@ -193,7 +206,7 @@ var p = class p {
 		return this._training;
 	}
 	dispose() {
-		this._responses &&= (this._responses.dispose(), null), this._training &&= (this._training.dispose(), null), this._model?.dispose(), this.ee.removeAllListeners(), p.instances.delete(this);
+		this._responses &&= (this._responses.dispose(), null), this._training &&= (this._training.dispose(), null), this._model?.dispose(), this.ee.removeAllListeners(), m.instances.delete(this);
 	}
 	on(e, t) {
 		if (e === "loaded" && this.loaded) {
@@ -207,4 +220,4 @@ var p = class p {
 	}
 };
 //#endregion
-export { p as default };
+export { m as default };

package/dist/api/beamer.d.ts

@@ -0,0 +1,25 @@
+import { Conversation, ITokeniser } from '../../tokeniser/type';
+import { default as Model, ModelForwardAttributes } from '../../models/model';
+import { GPTConfig } from '../../models/config';
+import { BeamerOptions, IBeam } from '../../inference/types';
+interface BeamEvents {
+    error: (error: Error) => void;
+    status: (status: 'busy' | 'ready') => void;
+    progress: (id: string, beams: IBeam[]) => void;
+    done: (id: string, beams: IBeam[]) => void;
+}
+export default class BeamAPI {
+    private ee;
+    private _model;
+    private _tokeniser;
+    private _busyCount;
+    private _jobs;
+    private _queue;
+    constructor(model: Model<ModelForwardAttributes, GPTConfig>, tokeniser: ITokeniser);
+    on<E extends keyof BeamEvents>(event: E, listener: BeamEvents[E]): void;
+    off<E extends keyof BeamEvents>(event: E, listener: BeamEvents[E]): void;
+    cancel(id?: string): void;
+    private startJob;
+    create(conversation: Conversation[], options: BeamerOptions): string;
+}
+export {};

package/dist/api/beamer.js

@@ -0,0 +1,52 @@
+import { t as e } from "../eventemitter3-D_qV3Lof.js";
+import { t } from "../v4-BK7K-jy_.js";
+import n from "../inference/Beamer.js";
+//#region lib/api/beamer.ts
+var r = class {
+	ee;
+	_model;
+	_tokeniser;
+	_busyCount = 0;
+	_jobs = /* @__PURE__ */ new Map();
+	_queue = [];
+	constructor(t, n) {
+		this._model = t, this._tokeniser = n, this.ee = new e();
+	}
+	on(e, t) {
+		this.ee.on(e, t);
+	}
+	off(e, t) {
+		this.ee.off(e, t);
+	}
+	cancel(e) {
+		if (e) {
+			let t = this._jobs.get(e);
+			t && (t.canceled = !0, t.beamer.cancel());
+		} else for (let e of this._jobs.values()) e.canceled = !0, e.beamer.cancel();
+	}
+	startJob(e) {
+		this._busyCount++, this.ee.emit("status", "busy"), e.beamer.beam(e.conversation, e.options, (t) => {
+			this.ee.emit("progress", e.id, t);
+		}).then((t) => {
+			this._busyCount--, this._busyCount === 0 && this.ee.emit("status", "ready"), this.ee.emit("done", e.id, t), this._jobs.delete(e.id);
+			let n = this._queue.shift();
+			n && this.startJob(n);
+		}).catch((t) => {
+			this._busyCount--, this._busyCount === 0 && this.ee.emit("status", "ready"), this.ee.emit("error", t), this._jobs.delete(e.id);
+			let n = this._queue.shift();
+			n && this.startJob(n);
+		});
+	}
+	create(e, r) {
+		let i = t(), a = {
+			id: i,
+			beamer: new n(this._model, this._tokeniser),
+			canceled: !1,
+			conversation: e,
+			options: r
+		};
+		return this._jobs.set(i, a), this._busyCount > 0 ? (this._queue.push(a), i) : (this.startJob(a), i);
+	}
+};
+//#endregion
+export { r as default };

package/dist/inference/Beamer.d.ts

@@ -0,0 +1,18 @@
+import { Conversation, ITokeniser } from '../tokeniser/type';
+import { default as Model, ModelForwardAttributes } from '../models/model';
+import { BeamerOptions, IBeam } from './types';
+export default class Beamer {
+    private readonly model;
+    private readonly tokeniser;
+    private actualTokeniser;
+    private active;
+    constructor(model: Model<ModelForwardAttributes>, tokeniser: ITokeniser);
+    private shouldTerminate;
+    private initialise;
+    private tokeniseConversation;
+    private createInitialContext;
+    private appendTokenToContext;
+    private batchedNextProbabilities;
+    cancel(): void;
+    beam(conversation: Conversation[], options: BeamerOptions, onStep?: (beams: IBeam[]) => void): Promise<IBeam[]>;
+}

package/dist/inference/Beamer.js

@@ -0,0 +1,133 @@
+import e from "../tokeniser/CharTokeniser.js";
+import { L as t, V as n, Y as r, _n as i, _r as a, di as o, xt as s } from "../dist-Da20xy8E.js";
+import c from "../utilities/topP.js";
+import { CHARS as l, padArray as u } from "./utilities.js";
+//#region lib/inference/Beamer.ts
+function d(e, t) {
+	return e.map((e, t) => ({
+		token: t,
+		prob: e
+	})).filter((e) => e.prob > 0).sort((e, t) => t.prob - e.prob).slice(0, Math.max(1, t));
+}
+var f = class {
+	model;
+	tokeniser;
+	actualTokeniser;
+	active = !1;
+	constructor(e, t) {
+		this.model = e, this.tokeniser = t, this.actualTokeniser = t;
+	}
+	shouldTerminate(e, t) {
+		if (e) return !1;
+		let n = this.tokeniser.getSpecialTokenIndex("<|assistant_end|>");
+		return t === this.actualTokeniser.eosToken || t === n;
+	}
+	initialise(t) {
+		let n = this.tokeniser.trained ? this.tokeniser : new e(u(l, this.tokeniser.vocabSize));
+		this.actualTokeniser = n, t?.loraName ? this.model.attachLoRA(t.loraName) : this.model.hasLoRA() && this.model.detachLoRA();
+	}
+	async tokeniseConversation(e) {
+		let t = this.actualTokeniser.encodeConversation(e, !1);
+		for (; t.length > 0 && (t[t.length - 1] === this.actualTokeniser.eosToken || t[t.length - 1] === this.actualTokeniser.getSpecialTokenIndex("<|assistant_end|>"));) t.pop();
+		return t;
+	}
+	createInitialContext(e) {
+		let n = this.model.config.blockSize, r = e.length > n ? e.slice(-n) : e.slice();
+		return {
+			context: t(r.concat(Array(n - r.length).fill(0)), "int32"),
+			contextLength: r.length
+		};
+	}
+	appendTokenToContext(e, n, r) {
+		let c = this.model.config.blockSize;
+		return n < c ? {
+			context: o(() => {
+				let t = s([n], c).squeeze([0]).asType("int32"), a = i(1, "int32").sub(t), o = t.mul(i(r, "int32"));
+				return e.mul(a).add(o);
+			}),
+			contextLength: n + 1
+		} : {
+			context: o(() => a([e.slice([1], [c - 1]), t([r], "int32")], 0)),
+			contextLength: c
+		};
+	}
+	async batchedNextProbabilities(e, t, i) {
+		if (e.length === 0) return [];
+		let a = e[0], l = e.slice();
+		for (; l.length < t;) l.push(a);
+		let u = n(l.map((e) => e.context)), d = this.model.forward({
+			training: !1,
+			mixedPrecision: !0
+		}, u), f = l.map((e) => Math.max(0, e.contextLength - 1)), p = o(() => {
+			let e = s(f, this.model.config.blockSize).expandDims(2);
+			return r(d.mul(e).sum(1));
+		}), m = e.length, h = m < t ? p.slice([0, 0], [m, this.model.config.vocabSize]) : p, g = await h.array();
+		h !== p && h.dispose(), p.dispose(), d.dispose(), u.dispose();
+		let _ = i.topP ?? 1;
+		return g.map((e) => c(e, _));
+	}
+	cancel() {
+		this.active = !1;
+	}
+	async beam(e, t, n) {
+		if (!t || t.beams < 1 || t.maxBeamLength < 1) return [];
+		let r = Math.max(8, t.beams);
+		this.initialise(t), this.active = !0;
+		let i = await this.tokeniseConversation(e), a = this.createInitialContext(i), o = [{
+			tokens: [],
+			score: 1,
+			text: "",
+			terminated: !1,
+			context: a.context,
+			contextLength: a.contextLength
+		}], s = t.maxBeamLength, c = t.endOnWhiteSpace === !0 ? Math.max(s, t.maxLength ?? s + this.model.config.blockSize) : s, l = Date.now();
+		for (let e = 0; e < c && this.active; e++) {
+			let e = o.filter((e) => !e.terminated);
+			if (e.length === 0) break;
+			let i = await this.batchedNextProbabilities(e, r, t), a = [];
+			for (let n = 0; n < e.length; n++) {
+				let o = e[n], c = i[n], l = d(c, t.topK ? Math.min(t.topK, r) : Math.max(1, r));
+				for (let e of l) {
+					let n = this.actualTokeniser.decode([e.token]), r = this.shouldTerminate(t.allowSpecial ?? !1, e.token), i = o.tokens.length + 1 >= Math.max(2, s), c = t.endOnWhiteSpace === !0 && i && /\s/.test(n), l = c && o.tokens.length > 0 && /^\s/.test(n);
+					a.push({
+						parent: o,
+						token: e.token,
+						tokenText: n,
+						score: o.score * e.prob,
+						terminatedByToken: r,
+						terminatedByWhitespace: c,
+						dropLeadingWhitespaceTerminator: l
+					});
+				}
+			}
+			let c = o.filter((e) => e.terminated).slice();
+			a.sort((e, t) => t.score - e.score);
+			for (let e of a) {
+				if (c.length >= r) break;
+				let t = e.terminatedByToken || e.terminatedByWhitespace, n = !e.dropLeadingWhitespaceTerminator, i = n ? e.parent.tokens.concat(e.token) : e.parent.tokens.slice(), a = e.terminatedByToken || e.dropLeadingWhitespaceTerminator ? e.parent.text : e.parent.text + e.tokenText, o = n ? this.appendTokenToContext(e.parent.context, e.parent.contextLength, e.token) : {
+					context: e.parent.context.clone(),
+					contextLength: e.parent.contextLength
+				};
+				c.push({
+					tokens: i,
+					score: e.score,
+					text: a,
+					terminated: t,
+					context: o.context,
+					contextLength: o.contextLength
+				});
+			}
+			if (o.forEach((e) => e.context.dispose()), c.length === 0) break;
+			o = c, n && Date.now() - l >= 40 && (n(o.slice(0, t.beams)), l = Date.now());
+		}
+		this.active = !1;
+		let u = o.slice().sort((e, t) => t.score - e.score), f = [], p = /* @__PURE__ */ new Set();
+		for (let e of u) {
+			let n = e.text.trim();
+			if (p.has(n) || (p.add(n), f.push(e)), f.length >= t.beams) break;
+		}
+		return o.forEach((e) => e.context.dispose()), f;
+	}
+};
+//#endregion
+export { f as default };

package/dist/inference/Generator.js

@@ -1,28 +1,16 @@
 import { t as e } from "../eventemitter3-D_qV3Lof.js";
-import { SPECIALS as t } from "../tokeniser/BaseTokeniser.js";
-import n from "../tokeniser/CharTokeniser.js";
-import { A as r, Ct as i, I as a, Y as o, _r as s, an as c, di as l, oi as u, yt as d } from "../dist-Da20xy8E.js";
-import f from "../utilities/multinomialCPU.js";
-import p from "../utilities/topP.js";
-import { sparseSoftmaxCrossEntropy as m } from "../training/sparseCrossEntropy.js";
-import h from "./tokenisePrompt.js";
-import { getTokenConfidence as g } from "./utilities.js";
+import t from "../tokeniser/CharTokeniser.js";
+import { A as n, Ct as r, I as i, Y as a, _r as o, an as s, di as c, oi as l, yt as u } from "../dist-Da20xy8E.js";
+import d from "../utilities/multinomialCPU.js";
+import f from "../utilities/topP.js";
+import { sparseSoftmaxCrossEntropy as p } from "../training/sparseCrossEntropy.js";
+import m from "./tokenisePrompt.js";
+import { CHARS as h, getTokenConfidence as g, padArray as _ } from "./utilities.js";
 //#region lib/inference/Generator.ts
-function _(e) {
+function v(e) {
 	return Array.isArray(e);
 }
-var v = [
-	...t,
-	...Array.from({ length: 95 }, (e, t) => String.fromCharCode(t + 32)),
-	..."áéíóúüñ¿¡",
-	..."äöÄÖÅå",
-	..."αβγδεζηθικλμνξοπρστυφχψωΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩ",
-	..."абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ"
-];
-function y(e, t) {
-	return e.length === t ? e : e.length > t ? e.slice(0, t) : e.concat(Array(t - e.length).fill(""));
-}
-var b = class extends e {
+var y = class extends e {
 	model;
 	tokeniser;
 	active = !1;
@@ -45,56 +33,56 @@ var b = class extends e {
 		let n = this.tokeniser.getSpecialTokenIndex("<|assistant_end|>");
 		return t === this.actualTokeniser.eosToken || t === n;
 	}
-	async _generateToken(e, t, n) {
-		let s = n?.temperature ?? 1, h = n?.topK, _ = n?.topP, v = n?.usePadding ?? !1, y = {
+	async _generateToken(e, t, o) {
+		let m = o?.temperature ?? 1, h = o?.topK, _ = o?.topP, v = o?.usePadding ?? !1, y = {
 			training: !1,
-			attentionScores: n?.outputAttention ? { attentionOut: [] } : void 0,
+			attentionScores: o?.outputAttention ? { attentionOut: [] } : void 0,
 			cache: t,
-			outputEmbeddings: !!n?.outputHiddenStates
-		}, [b, x] = l(() => {
-			let t = e, r = t.shape[1], i = r <= this.model.config.blockSize ? t : t.slice([0, r - this.model.config.blockSize], [t.shape[0], this.model.config.blockSize]), o = v ? this.model.config.blockSize - i.shape[1] : 0, c = o > 0 ? d(i, [[0, 0], [0, o]]) : i, l = this.model.forward(y, c), f = l.shape[1] - 1 - o, p = l.slice([
+			outputEmbeddings: !!o?.outputHiddenStates
+		}, [b, x] = c(() => {
+			let t = e, n = t.shape[1], r = n <= this.model.config.blockSize ? t : t.slice([0, n - this.model.config.blockSize], [t.shape[0], this.model.config.blockSize]), a = v ? this.model.config.blockSize - r.shape[1] : 0, s = a > 0 ? u(r, [[0, 0], [0, a]]) : r, c = this.model.forward(y, s), d = c.shape[1] - 1 - a, f = c.slice([
 				0,
-				f,
+				d,
 				0
 			], [
-				l.shape[0],
+				c.shape[0],
 				1,
-				l.shape[2]
+				c.shape[2]
 			]), h;
-			if (n?.targets) {
-				let e = n.targets.shift();
+			if (o?.targets) {
+				let e = o.targets.shift();
 				if (e !== void 0) {
-					let t = a([[e]], [1, 1], "int32"), n = m(p, t);
+					let t = i([[e]], [1, 1], "int32"), n = p(f, t);
 					h = n.mean(), t.dispose(), n.dispose();
 				}
 			}
 			return y.attentionScores?.attentionOut && y.attentionScores.attentionOut.forEach((e, t) => {
-				e.shape[1] !== 1 && (y.attentionScores.attentionOut[t] = u(e.slice([
+				e.shape[1] !== 1 && (y.attentionScores.attentionOut[t] = l(e.slice([
 					0,
-					f,
+					d,
 					0
 				], [
 					e.shape[0],
 					1,
 					e.shape[2]
 				])), e.dispose());
-			}), l.dispose(), [p.div(s).squeeze([1]), h];
+			}), c.dispose(), [f.div(m).squeeze([1]), h];
 		}), S, C, w, T = Math.random();
 		if (_) {
-			let e = o(b), t = await e.array();
+			let e = a(b), t = await e.array();
 			e.dispose();
-			let r = p(t, _);
-			(n?.outputScores || n?.outputConfidence) && (C = t), S = f(r, T);
+			let n = f(t, _);
+			(o?.outputScores || o?.outputConfidence) && (C = t), S = d(n, T);
 		} else if (h) {
-			let { values: e, indices: t } = r(b, h), n = i(e, 1);
-			S = c(t, n, 1), e.dispose(), t.dispose(), n.dispose();
-		} else if (S = i(b, 1), n?.outputScores || n?.outputConfidence) {
-			let e = o(b);
+			let { values: e, indices: t } = n(b, h), i = r(e, 1);
+			S = s(t, i, 1), e.dispose(), t.dispose(), i.dispose();
+		} else if (S = r(b, 1), o?.outputScores || o?.outputConfidence) {
+			let e = a(b);
 			C = await e.array(), e.dispose();
 		}
 		if (y.embeddings) {
-			let e = (n?.outputHiddenStates === "all" ? y.embeddings : y.embeddings.filter((e) => e.name.startsWith("block_output_"))).map(async (e) => {
-				let t = e.tensor.shape[1], r = e.tensor.slice([
+			let e = (o?.outputHiddenStates === "all" ? y.embeddings : y.embeddings.filter((e) => e.name.startsWith("block_output_"))).map(async (e) => {
+				let t = e.tensor.shape[1], n = e.tensor.slice([
 					0,
 					t - 1,
 					0
@@ -104,28 +92,28 @@ var b = class extends e {
 					e.tensor.shape[2]
 				]);
 				e.tensor.dispose();
-				let i = r.squeeze([1]);
-				if (r.dispose(), n?.outputHiddenStates === "softmax") {
-					let t = this.model.project(i);
-					i.dispose();
-					let n = o(t, -1);
+				let r = n.squeeze([1]);
+				if (n.dispose(), o?.outputHiddenStates === "softmax") {
+					let t = this.model.project(r);
+					r.dispose();
+					let n = a(t, -1);
 					t.dispose();
-					let r = {
+					let i = {
 						name: e.name,
 						tensor: await n.array()
 					};
-					return n.dispose(), r;
-				} else if (n?.outputHiddenStates === "logits") {
-					let t = this.model.project(i);
-					i.dispose();
+					return n.dispose(), i;
+				} else if (o?.outputHiddenStates === "logits") {
+					let t = this.model.project(r);
+					r.dispose();
 					let n = {
 						name: e.name,
 						tensor: await t.array()
 					};
 					return t.dispose(), n;
 				} else {
-					let t = await i.array();
-					return i.dispose(), {
+					let t = await r.array();
+					return r.dispose(), {
 						name: e.name,
 						tensor: t
 					};
@@ -137,16 +125,16 @@ var b = class extends e {
 		S.dispose(), S = E;
 		let D = (await S.array())[0][0], O = this.actualTokeniser.decode([D]);
 		this.lastToken = D;
-		let k = this.shouldTerminate(n?.allowSpecial ?? !1, D), A = {
+		let k = this.shouldTerminate(o?.allowSpecial ?? !1, D), A = {
 			outputTensor: S,
 			token: D,
 			text: O,
-			confidence: n?.outputConfidence && C ? g(C[0]) : null,
-			score: n?.outputScore && C ? C[0][D] : null,
-			logits: n?.outputLogits ? (await b.array())[0] : null,
-			scores: n?.outputScores && C ? C[0] : null,
+			confidence: o?.outputConfidence && C ? g(C[0]) : null,
+			score: o?.outputScore && C ? C[0][D] : null,
+			logits: o?.outputLogits ? (await b.array())[0] : null,
+			scores: o?.outputScores && C ? C[0] : null,
 			hiddenStates: w ? w.map((e) => e.tensor[0]) : null,
-			attention: n?.outputAttention ? await Promise.all(y.attentionScores?.attentionOut?.map((e) => e.array()) ?? []) : null,
+			attention: o?.outputAttention ? await Promise.all(y.attentionScores?.attentionOut?.map((e) => e.array()) ?? []) : null,
 			loss: this.lastLoss,
 			multinomialRand: T,
 			terminated: k
@@ -155,7 +143,7 @@ var b = class extends e {
 			let e = await x.array();
 			x.dispose(), A.loss = e;
 		}
-		return this.rawOutput.push(A), (!n?.chunkSize || this.tokenCount++ % n.chunkSize === 0) && (this.emit("tokens", A), n?._onChunk && await n._onChunk(A)), A;
+		return this.rawOutput.push(A), (!o?.chunkSize || this.tokenCount++ % o.chunkSize === 0) && (this.emit("tokens", A), o?._onChunk && await o._onChunk(A)), A;
 	}
 	async _generate(e, t) {
 		let n = !1;
@@ -164,8 +152,8 @@ var b = class extends e {
 			content: "",
 			_timestamp: Date.now()
 		}), n = !0, this.resetCache(!e?.noCache)) : (this.lastToken < 0 || t) && this.resetCache(!e?.noCache);
-		let r = this.lastToken >= 0 && this.cache ? a([this.lastToken], [1, 1], "int32") : await h(this.actualTokeniser, this.model.config.blockSize, t ? n ? this.outputConversation.slice(0, -1) : this.outputConversation : void 0, e), i = e?.maxLength ?? 1e3;
-		for (let t = 0; t < i && this.active; t++) {
+		let r = this.lastToken >= 0 && this.cache ? i([this.lastToken], [1, 1], "int32") : await m(this.actualTokeniser, this.model.config.blockSize, t ? n ? this.outputConversation.slice(0, -1) : this.outputConversation : void 0, e), a = e?.maxLength ?? 1e3;
+		for (let t = 0; t < a && this.active; t++) {
 			let n = await this._generateToken(r, this.cache ? this.cache : void 0, {
 				...e,
 				usePadding: !this.cache
@@ -173,14 +161,14 @@ var b = class extends e {
 			if (this.cache) r.dispose(), r = n.outputTensor;
 			else {
 				let e = r;
-				r = s([r, n.outputTensor], 1), e.dispose();
+				r = o([r, n.outputTensor], 1), e.dispose();
 			}
-			let a = this.outputConversation[this.outputConversation.length - 1];
+			let i = this.outputConversation[this.outputConversation.length - 1];
 			if (this.cache || n.outputTensor.dispose(), n.terminated) {
-				a._completed = !0;
+				i._completed = !0;
 				break;
 			}
-			t === i - 1 && i > 1 && (a._completed = !0, n.terminated = !0), a.content += n.text, a._output ||= [], a._output.push(n);
+			t === a - 1 && a > 1 && (i._completed = !0, n.terminated = !0), i.content += n.text, i._output ||= [], i._output.push(n);
 		}
 		return r.dispose(), this.outputConversation;
 	}
@@ -195,8 +183,8 @@ var b = class extends e {
 	dispose() {
 		this.reset();
 	}
-	initialise(e, t) {
-		if (this.cache && t?.noCache && this.reset(), this.initialPrompt = e || null, this.lastToken === -1 ? this.outputConversation = (this.initialPrompt || []).slice() : e && e.length > this.outputConversation.length && (this.outputConversation = (this.initialPrompt || []).slice(), this.resetCache()), !this.cache && !t?.noCache && (this.model.config.modelType !== "GenAI_NanoGPT_v1" || this.model.config.useRope)) {
+	initialise(e, n) {
+		if (this.cache && n?.noCache && this.reset(), this.initialPrompt = e || null, this.lastToken === -1 ? this.outputConversation = (this.initialPrompt || []).slice() : e && e.length > this.outputConversation.length && (this.outputConversation = (this.initialPrompt || []).slice(), this.resetCache()), !this.cache && !n?.noCache && (this.model.config.modelType !== "GenAI_NanoGPT_v1" || this.model.config.useRope)) {
 			let e = Array(this.model.config.nLayer);
 			for (let t = 0; t < this.model.config.nLayer; t++) e[t] = {
 				k: void 0,
@@ -206,15 +194,15 @@ var b = class extends e {
 			};
 			this.cache = e, this.lastToken = -1;
 		}
-		let r = this.tokeniser.trained ? this.tokeniser : new n(y(v, this.tokeniser.vocabSize));
-		this.actualTokeniser = r, t?.loraName ? this.model.attachLoRA(t.loraName) : this.model.hasLoRA() && this.model.detachLoRA();
+		let r = this.tokeniser.trained ? this.tokeniser : new t(_(h, this.tokeniser.vocabSize));
+		this.actualTokeniser = r, n?.loraName ? this.model.attachLoRA(n.loraName) : this.model.hasLoRA() && this.model.detachLoRA();
 	}
 	async step(e, t) {
 		let n = {
 			...t,
 			maxLength: 1
 		};
-		return _(e) ? this.generate(e, n) : this.generate({
+		return v(e) ? this.generate(e, n) : this.generate({
 			...e,
 			...n
 		});
@@ -247,17 +235,7 @@ var b = class extends e {
 	async startJob(e, t) {
 		this.initialise(e, t), this.active = !0, this.model.metaData.generationSettings = t, t?.maxLength !== 1 && this.emit("start");
 		let n = await this._generate(t, !!e);
-		if (this.active = !1, this.startTime !== null) {
-			let e = Date.now(), n = e - this.startTime;
-			this.startTime = null, this.model.metaData.actionLog = this.model.metaData.actionLog || [], this.model.metaData.actionLog.push({
-				action: "generate",
-				timestamp: e,
-				duration: n,
-				tokensProcessed: this.rawOutput.length,
-				options: t || {}
-			});
-		}
-		return this.emit("stop"), n;
+		return this.active = !1, this.startTime !== null && (this.startTime = null), this.emit("stop"), n;
 	}
 	getQueueLength() {
 		return this.jobQueue.length;
@@ -273,4 +251,4 @@ var b = class extends e {
 	}
 };
 //#endregion
-export { b as default, _ as isConversation };
+export { y as default, v as isConversation };

package/dist/inference/types.d.ts

@@ -50,3 +50,13 @@ export interface IGeneratorResponse {
     id: string;
     done: boolean;
 }
+export interface BeamerOptions extends IGenerateOptions {
+    maxBeamLength: number;
+    beams: number;
+    endOnWhiteSpace?: boolean;
+}
+export interface IBeam {
+    tokens: number[];
+    score: number;
+    text: string;
+}

package/dist/inference/utilities.d.ts

@@ -7,3 +7,5 @@ import { IGeneratorOutput } from './types';
 export declare function getTokenConfidence(probabilities: number[]): number;
 export declare function getAttention(output: IGeneratorOutput, layer: number, head: number): number[] | null;
 export declare function getHiddenState(output: IGeneratorOutput, layer: number): number[] | null;
+export declare const CHARS: string[];
+export declare function padArray(arr: string[], length: number): string[];

package/dist/inference/utilities.js

@@ -1,5 +1,6 @@
+import { SPECIALS as e } from "../tokeniser/BaseTokeniser.js";
 //#region lib/inference/utilities.ts
-function e(e) {
+function t(e) {
 	if (!e.length) return 0;
 	let t = 0;
 	for (let n of e) n > 0 && (t -= n * Math.log(n));
@@ -8,13 +9,24 @@ function e(e) {
 	let r = t / n;
 	return Math.min(1, Math.max(0, 1 - r));
 }
-function t(e, t, n) {
+function n(e, t, n) {
 	if (!e.attention || !e.attention[t] || !e.attention[t][n]) return null;
 	let r = e.attention[t][n].length;
 	return e.attention[t][n][r - 1];
 }
-function n(e, t) {
+function r(e, t) {
 	return !e.hiddenStates || !e.hiddenStates[t] ? null : e.hiddenStates[t];
 }
+var i = [
+	...e,
+	...Array.from({ length: 95 }, (e, t) => String.fromCharCode(t + 32)),
+	..."áéíóúüñ¿¡",
+	..."äöÄÖÅå",
+	..."αβγδεζηθικλμνξοπρστυφχψωΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩ",
+	..."абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ"
+];
+function a(e, t) {
+	return e.length === t ? e : e.length > t ? e.slice(0, t) : e.concat(Array(t - e.length).fill(""));
+}
 //#endregion
-export { t as getAttention, n as getHiddenState, e as getTokenConfidence };
+export { i as CHARS, n as getAttention, r as getHiddenState, t as getTokenConfidence, a as padArray };

package/dist/loader/load.js

@@ -23,7 +23,9 @@ async function d(e, t) {
 	let n = e.file("config.json");
 	if (!n) return;
 	let r = await n.async("string"), i = JSON.parse(r);
-	if (i.loraName) {
+	if (i.loraConfig && Object.entries(i.loraConfig).forEach(([e, n]) => {
+		t.hasLoRA(e) || t.createLoRA(e, n);
+	}), i.loraName) {
 		if (t.hasLoRA()) throw Error("Model already has LoRA attached");
 		t.attachLoRA(i.loraName);
 	}

package/dist/main.d.ts

@@ -16,7 +16,7 @@ export type { DatasetMetadata, ModelMode } from './loader/types';
 export * as models from './models';
 export type { GPTConfig } from './models/config';
 export type { ModelForwardAttributes } from './models/model';
-export type { IGenerateOptions, IGeneratorResponse, IGeneratorOutput, GeneratorConversation } from './inference/types';
+export type { IGenerateOptions, IGeneratorResponse, IGeneratorOutput, GeneratorConversation, IBeam, BeamerOptions, } from './inference/types';
 export type { TrainingOptions, TrainingLogEntry } from './training/types';
 export type { ITrainingJob } from './api/training';
 export declare const training: {

package/dist/utilities/topP.d.ts

@@ -1 +1 @@
-export default function topP(probs: number[][], tP: number): number[];
+export default function topP(probs: number[][] | number[], tP: number): number[];

package/dist/utilities/topP.js

@@ -1,18 +1,18 @@
 //#region lib/utilities/topP.ts
 function e(e, t) {
-	let n = e[0].map((e, t) => ({
+	let n = Array.isArray(e[0]) ? e[0] : e, r = n.map((e, t) => ({
 		prob: e,
 		index: t
-	})).sort((e, t) => t.prob - e.prob), r = 0, i = Array(n.length).fill(0);
-	for (let e of n) if (r += e.prob, i[e.index] = e.prob, r >= t) break;
-	let a = i.reduce((e, t) => e + t, 0);
-	if (a === 0) {
-		let t = e[0], n = t.reduce((e, t) => e + t, 0);
-		if (n > 0) return t.map((e) => e / n);
-		let r = 1 / t.length;
-		return t.map(() => r);
+	})).sort((e, t) => t.prob - e.prob), i = 0, a = Array(r.length).fill(0);
+	for (let e of r) if (i += e.prob, a[e.index] = e.prob, i >= t) break;
+	let o = a.reduce((e, t) => e + t, 0);
+	if (o === 0) {
+		let e = n, t = e.reduce((e, t) => e + t, 0);
+		if (t > 0) return e.map((e) => e / t);
+		let r = 1 / e.length;
+		return e.map(() => r);
 	}
-	return i.map((e) => e / a);
+	return a.map((e) => e / o);
 }
 //#endregion
 export { e as default };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "1.1.4",
+    "version": "1.2.1",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",