@genai-fi/nanogpt 0.20.0 → 0.20.2

package/dist/layers/RMSNorm.js

@@ -0,0 +1,20 @@
+import { Ot as e, T as t, di as n } from "../dist-BewPQWjc.js";
+import r from "./BaseLayer.js";
+import { normRMS as i } from "../ops/normRMS.js";
+//#region lib/layers/RMSNorm.ts
+var a = class extends r {
+	GAMMA;
+	rmsConfig;
+	constructor(n, r, i = "", a) {
+		super(n, a), this.GAMMA = i, this.rmsConfig = r, (this.rmsConfig.useGamma ?? !0) && this.addVariable(this.GAMMA, t(e([n.nEmbed]), !0, this.GAMMA, "float32"));
+	}
+	forward(e, t) {
+		return n(() => {
+			this.startMemory();
+			let e = i(t, this.rmsConfig.useGamma ?? !0 ? this.getVariable(this.GAMMA) : void 0);
+			return this.endMemory("RMSNorm"), e;
+		});
+	}
+};
+//#endregion
+export { a as default };

package/dist/layers/RoPECache.d.ts

@@ -0,0 +1,18 @@
+import { Tensor } from '@tensorflow/tfjs-core';
+import { GPTConfig } from '../models/config';
+export default class RoPECache {
+    private readonly config;
+    readonly rotaryDim: number;
+    private ropeBase;
+    private ropeInvFreq;
+    private ropeCos;
+    private ropeSin;
+    private ropeNegSin;
+    private ropeCacheLen;
+    constructor(config: GPTConfig);
+    ensureRopeCache(needed: number): void;
+    getCos(): Tensor | null;
+    getSin(): Tensor | null;
+    getNegSin(): Tensor | null;
+    dispose(): void;
+}

package/dist/layers/RoPECache.js

@@ -0,0 +1,337 @@
+import { $n as e, $t as t, A as n, An as r, At as i, B as a, Bn as o, Br as s, Bt as ee, D as te, Dt as c, E as ne, Ei as l, Et as re, Fn as ie, Fr as ae, Ft as oe, G as se, Gn as ce, Gr as u, H as le, Hr as ue, Ht as de, Ir as d, It as f, J as p, Jt as m, K as h, Kn as g, Kr as _, Lr as v, Lt as y, Mn as b, Mr as x, Nn as S, Nr as C, Nt as w, O as T, Pn as E, Pr as D, Ps as O, Pt as k, Qn as A, Qt as j, R as M, Rr as N, Rt as P, St as F, Tt as I, U as L, Un as R, Ur as z, Ut as B, V, Vn as H, Vr as fe, Vt as pe, W as me, Wn as he, Wr as ge, Xn as _e, Xt as ve, Y as ye, Yr as U, Yt as be, Zt as xe, _n as W, _r as Se, _t as Ce, an as we, at as Te, bn as Ee, br as G, bt as De, cn as Oe, ct as ke, di as K, dn as Ae, dr as q, dt as J, en as je, et as Me, fn as Ne, fr as Pe, g as Fe, gn as Ie, gr as Le, gt as Re, h as ze, hn as Be, hr as Ve, ht as He, in as Ue, ir as We, it as Ge, jn as Ke, jt as qe, kn as Je, ln as Ye, lt as Xe, mn as Ze, mr as Qe, nn as $e, nt as et, oi as Y, on as tt, or as nt, ot as rt, pn as it, pr as at, q as ot, qr as st, rn as ct, rt as lt, sr as ut, st as dt, tn as ft, tt as X, un as pt, ut as Z, vn as Q, vt as mt, w as ht, wi as $, xt as gt, yn as _t, yr as vt, yt, z as bt, zn as xt, zr as St, zt as Ct } from "../dist-BewPQWjc.js";
+//#endregion
+//#region node_modules/@tensorflow/tfjs-core/dist/public/chained_ops/zeros_like.js
+l().prototype.abs = function() {
+	return this.throwIfDisposed(), z(this);
+}, l().prototype.acos = function() {
+	return this.throwIfDisposed(), ue(this);
+}, l().prototype.acosh = function() {
+	return this.throwIfDisposed(), fe(this);
+}, l().prototype.add = function(e) {
+	return this.throwIfDisposed(), st(this, e);
+}, l().prototype.all = function(e, t) {
+	return this.throwIfDisposed(), s(this, e, t);
+}, l().prototype.any = function(e, t) {
+	return this.throwIfDisposed(), St(this, e, t);
+}, l().prototype.argMax = function(e) {
+	return this.throwIfDisposed(), N(this, e);
+}, l().prototype.argMin = function(e) {
+	return this.throwIfDisposed(), v(this, e);
+}, l().prototype.asScalar = function() {
+	return this.throwIfDisposed(), O(this.size === 1, () => "The array must have only 1 element."), G(this, []);
+}, l().prototype.asType = function(e) {
+	return this.throwIfDisposed(), U(this, e);
+}, l().prototype.as1D = function() {
+	return this.throwIfDisposed(), G(this, [this.size]);
+}, l().prototype.as2D = function(e, t) {
+	return this.throwIfDisposed(), G(this, [e, t]);
+}, l().prototype.as3D = function(e, t, n) {
+	return this.throwIfDisposed(), G(this, [
+		e,
+		t,
+		n
+	]);
+}, l().prototype.as4D = function(e, t, n, r) {
+	return this.throwIfDisposed(), G(this, [
+		e,
+		t,
+		n,
+		r
+	]);
+}, l().prototype.as5D = function(e, t, n, r, i) {
+	return this.throwIfDisposed(), G(this, [
+		e,
+		t,
+		n,
+		r,
+		i
+	]);
+}, l().prototype.asin = function() {
+	return this.throwIfDisposed(), d(this);
+}, l().prototype.asinh = function() {
+	return this.throwIfDisposed(), ae(this);
+}, l().prototype.atan = function() {
+	return this.throwIfDisposed(), D(this);
+}, l().prototype.atan2 = function(e) {
+	return this.throwIfDisposed(), C(this, e);
+}, l().prototype.atanh = function() {
+	return this.throwIfDisposed(), x(this);
+}, l().prototype.avgPool = function(e, t, n, r) {
+	return this.throwIfDisposed(), vt(this, e, t, n, r);
+}, l().prototype.batchToSpaceND = function(e, t) {
+	return this.throwIfDisposed(), Pe(this, e, t);
+}, l().prototype.batchNorm = function(e, t, n, r, i) {
+	return this.throwIfDisposed(), q(this, e, t, n, r, i);
+}, l().prototype.broadcastTo = function(e) {
+	return this.throwIfDisposed(), ut(this, e);
+}, l().prototype.cast = function(e) {
+	return this.throwIfDisposed(), U(this, e);
+}, l().prototype.ceil = function() {
+	return this.throwIfDisposed(), nt(this);
+}, l().prototype.clipByValue = function(e, t) {
+	return this.throwIfDisposed(), We(this, e, t);
+}, l().prototype.concat = function(e, t) {
+	return this.throwIfDisposed(), e instanceof $ && (e = [e]), Se([this, ...e], t);
+}, l().prototype.conv1d = function(e, t, n, r, i, a) {
+	return this.throwIfDisposed(), A(this, e, t, n, r, i, a);
+}, l().prototype.conv2dTranspose = function(e, t, n, r, i) {
+	return this.throwIfDisposed(), _e(this, e, t, n, r, i);
+}, l().prototype.conv2d = function(t, n, r, i, a, o) {
+	return this.throwIfDisposed(), e(this, t, n, r, i, a, o);
+}, l().prototype.cos = function() {
+	return this.throwIfDisposed(), g(this);
+}, l().prototype.cosh = function() {
+	return this.throwIfDisposed(), ce(this);
+}, l().prototype.cumprod = function(e, t, n) {
+	return this.throwIfDisposed(), he(this, e, t, n);
+}, l().prototype.cumsum = function(e, t, n) {
+	return this.throwIfDisposed(), R(this, e, t, n);
+}, l().prototype.depthToSpace = function(e, t) {
+	return this.throwIfDisposed(), H(this, e, t);
+}, l().prototype.depthwiseConv2d = function(e, t, n, r, i, a) {
+	return this.throwIfDisposed(), o(this, e, t, n, r, i, a);
+}, l().prototype.dilation2d = function(e, t, n, r, i) {
+	return this.throwIfDisposed(), xt(this, e, t, n, r, i);
+}, l().prototype.divNoNan = function(e) {
+	return this.throwIfDisposed(), b(this, e);
+}, l().prototype.div = function(e) {
+	return this.throwIfDisposed(), u(this, e);
+}, l().prototype.dot = function(e) {
+	return this.throwIfDisposed(), Ke(this, e);
+}, l().prototype.elu = function() {
+	return this.throwIfDisposed(), r(this);
+}, l().prototype.equal = function(e) {
+	return this.throwIfDisposed(), ie(this, e);
+}, l().prototype.erf = function() {
+	return this.throwIfDisposed(), Je(this);
+}, l().prototype.euclideanNorm = function(e, t) {
+	return this.throwIfDisposed(), Ne(this, e, t);
+}, l().prototype.exp = function() {
+	return this.throwIfDisposed(), Ae(this);
+}, l().prototype.expandDims = function(e) {
+	return this.throwIfDisposed(), pt(this, e);
+}, l().prototype.expm1 = function() {
+	return this.throwIfDisposed(), Ye(this);
+}, l().prototype.fft = function() {
+	return this.throwIfDisposed(), p(this);
+}, l().prototype.flatten = function() {
+	return this.throwIfDisposed(), G(this, [this.size]);
+}, l().prototype.floor = function() {
+	return this.throwIfDisposed(), tt(this);
+}, l().prototype.floorDiv = function(e) {
+	return this.throwIfDisposed(), _(this, e);
+}, l().prototype.gather = function(e, t, n) {
+	return this.throwIfDisposed(), we(this, e, t, n);
+}, l().prototype.greaterEqual = function(e) {
+	return this.throwIfDisposed(), ct(this, e);
+}, l().prototype.greater = function(e) {
+	return this.throwIfDisposed(), Ue(this, e);
+}, l().prototype.ifft = function() {
+	return this.throwIfDisposed(), ot(this);
+}, l().prototype.irfft = function() {
+	return this.throwIfDisposed(), h(this);
+}, l().prototype.isFinite = function() {
+	return this.throwIfDisposed(), $e(this);
+}, l().prototype.isInf = function() {
+	return this.throwIfDisposed(), ft(this);
+}, l().prototype.isNaN = function() {
+	return this.throwIfDisposed(), je(this);
+}, l().prototype.leakyRelu = function(e) {
+	return this.throwIfDisposed(), t(this, e);
+}, l().prototype.lessEqual = function(e) {
+	return this.throwIfDisposed(), xe(this, e);
+}, l().prototype.less = function(e) {
+	return this.throwIfDisposed(), j(this, e);
+}, l().prototype.localResponseNormalization = function(e, t, n, r) {
+	return this.throwIfDisposed(), ve(this, e, t, n, r);
+}, l().prototype.logSigmoid = function() {
+	return this.throwIfDisposed(), pe(this);
+}, l().prototype.logSoftmax = function(e) {
+	return this.throwIfDisposed(), Ct(this, e);
+}, l().prototype.logSumExp = function(e, t) {
+	return this.throwIfDisposed(), P(this, e, t);
+}, l().prototype.log = function() {
+	return this.throwIfDisposed(), be(this);
+}, l().prototype.log1p = function() {
+	return this.throwIfDisposed(), m(this);
+}, l().prototype.logicalAnd = function(e) {
+	return this.throwIfDisposed(), y(this, e);
+}, l().prototype.logicalNot = function() {
+	return this.throwIfDisposed(), f(this);
+}, l().prototype.logicalOr = function(e) {
+	return this.throwIfDisposed(), oe(this, e);
+}, l().prototype.logicalXor = function(e) {
+	return this.throwIfDisposed(), k(this, e);
+}, l().prototype.matMul = function(e, t, n) {
+	return this.throwIfDisposed(), Le(this, e, t, n);
+}, l().prototype.maxPool = function(e, t, n, r) {
+	return this.throwIfDisposed(), w(this, e, t, n, r);
+}, l().prototype.max = function(e, t) {
+	return this.throwIfDisposed(), Ee(this, e, t);
+}, l().prototype.maximum = function(e) {
+	return this.throwIfDisposed(), qe(this, e);
+}, l().prototype.mean = function(e, t) {
+	return this.throwIfDisposed(), i(this, e, t);
+}, l().prototype.min = function(e, t) {
+	return this.throwIfDisposed(), _t(this, e, t);
+}, l().prototype.minimum = function(e) {
+	return this.throwIfDisposed(), c(this, e);
+}, l().prototype.mirrorPad = function(e, t) {
+	return this.throwIfDisposed(), re(this, e, t);
+}, l().prototype.mod = function(e) {
+	return this.throwIfDisposed(), I(this, e);
+}, l().prototype.mul = function(e) {
+	return this.throwIfDisposed(), ge(this, e);
+}, l().prototype.neg = function() {
+	return this.throwIfDisposed(), B(this);
+}, l().prototype.norm = function(e, t, n) {
+	return this.throwIfDisposed(), it(this, e, t, n);
+}, l().prototype.notEqual = function(e) {
+	return this.throwIfDisposed(), F(this, e);
+}, l().prototype.oneHot = function(e, t = 1, n = 0) {
+	return this.throwIfDisposed(), gt(this, e, t, n);
+}, l().prototype.onesLike = function() {
+	return this.throwIfDisposed(), De(this);
+}, l().prototype.pad = function(e, t) {
+	return this.throwIfDisposed(), yt(this, e, t);
+}, l().prototype.pool = function(e, t, n, r, i, a) {
+	return this.throwIfDisposed(), Ce(this, e, t, n, r, i, a);
+}, l().prototype.pow = function(e) {
+	return this.throwIfDisposed(), Q(this, e);
+}, l().prototype.prelu = function(e) {
+	return this.throwIfDisposed(), Re(this, e);
+}, l().prototype.prod = function(e, t) {
+	return this.throwIfDisposed(), He(this, e, t);
+}, l().prototype.reciprocal = function() {
+	return this.throwIfDisposed(), Z(this);
+}, l().prototype.relu = function() {
+	return this.throwIfDisposed(), Xe(this);
+}, l().prototype.relu6 = function() {
+	return this.throwIfDisposed(), ke(this);
+}, l().prototype.reshapeAs = function(e) {
+	return this.throwIfDisposed(), G(this, e.shape);
+}, l().prototype.reshape = function(e) {
+	return this.throwIfDisposed(), G(this, e);
+}, l().prototype.resizeBilinear = function(e, t, n) {
+	return this.throwIfDisposed(), Fe(this, e, t, n);
+}, l().prototype.resizeNearestNeighbor = function(e, t, n) {
+	return this.throwIfDisposed(), ze(this, e, t, n);
+}, l().prototype.reverse = function(e) {
+	return this.throwIfDisposed(), dt(this, e);
+}, l().prototype.rfft = function() {
+	return this.throwIfDisposed(), me(this);
+}, l().prototype.round = function() {
+	return this.throwIfDisposed(), rt(this);
+}, l().prototype.rsqrt = function() {
+	return this.throwIfDisposed(), Te(this);
+}, l().prototype.selu = function() {
+	return this.throwIfDisposed(), Ge(this);
+}, l().prototype.separableConv2d = function(e, t, n, r, i, a) {
+	return this.throwIfDisposed(), lt(this, e, t, n, r, i, a);
+}, l().prototype.sigmoid = function() {
+	return this.throwIfDisposed(), Ve(this);
+}, l().prototype.sign = function() {
+	return this.throwIfDisposed(), et(this);
+}, l().prototype.sin = function() {
+	return this.throwIfDisposed(), X(this);
+}, l().prototype.sinh = function() {
+	return this.throwIfDisposed(), Me(this);
+}, l().prototype.slice = function(e, t) {
+	return this.throwIfDisposed(), Qe(this, e, t);
+}, l().prototype.softmax = function(e) {
+	return this.throwIfDisposed(), ye(this, e);
+}, l().prototype.softplus = function() {
+	return this.throwIfDisposed(), de(this);
+}, l().prototype.spaceToBatchND = function(e, t) {
+	return this.throwIfDisposed(), mt(this, e, t);
+}, l().prototype.split = function(e, t) {
+	return this.throwIfDisposed(), se(this, e, t);
+}, l().prototype.sqrt = function() {
+	return this.throwIfDisposed(), Ie(this);
+}, l().prototype.square = function() {
+	return this.throwIfDisposed(), Be(this);
+}, l().prototype.squaredDifference = function(e) {
+	return this.throwIfDisposed(), L(this, e);
+}, l().prototype.squeeze = function(e) {
+	return this.throwIfDisposed(), le(this, e);
+}, l().prototype.stack = function(e, t) {
+	return this.throwIfDisposed(), V(e instanceof $ ? [this, e] : [this, ...e], t);
+}, l().prototype.step = function(e) {
+	return this.throwIfDisposed(), a(this, e);
+}, l().prototype.stridedSlice = function(e, t, n, r, i, a, o, s) {
+	return this.throwIfDisposed(), bt(this, e, t, n, r, i, a, o, s);
+}, l().prototype.sub = function(e) {
+	return this.throwIfDisposed(), ee(this, e);
+}, l().prototype.sum = function(e, t) {
+	return this.throwIfDisposed(), Ze(this, e, t);
+}, l().prototype.tan = function() {
+	return this.throwIfDisposed(), M(this);
+}, l().prototype.tanh = function() {
+	return this.throwIfDisposed(), at(this);
+}, l().prototype.tile = function(e) {
+	return this.throwIfDisposed(), Oe(this, e);
+}, l().prototype.toBool = function() {
+	return this.throwIfDisposed(), U(this, "bool");
+}, l().prototype.toFloat = function() {
+	return this.throwIfDisposed(), U(this, "float32");
+}, l().prototype.toInt = function() {
+	return this.throwIfDisposed(), U(this, "int32");
+}, l().prototype.topk = function(e, t) {
+	return this.throwIfDisposed(), n(this, e, t);
+}, l().prototype.transpose = function(e) {
+	return this.throwIfDisposed(), ht(this, e);
+}, l().prototype.unique = function(e) {
+	return this.throwIfDisposed(), T(this, e);
+}, l().prototype.unsortedSegmentSum = function(e, t) {
+	return this.throwIfDisposed(), te(this, e, t);
+}, l().prototype.unstack = function(e) {
+	return this.throwIfDisposed(), ne(this, e);
+}, l().prototype.where = function(e, t) {
+	return this.throwIfDisposed(), E(e, this, t);
+}, l().prototype.zerosLike = function() {
+	return this.throwIfDisposed(), S(this);
+};
+//#endregion
+//#region lib/layers/RoPECache.ts
+var wt = class {
+	config;
+	rotaryDim;
+	ropeBase;
+	ropeInvFreq;
+	ropeCos = null;
+	ropeSin = null;
+	ropeNegSin = null;
+	ropeCacheLen = 0;
+	constructor(e) {
+		this.config = e;
+		let t = this.config.nEmbed / this.config.nHead;
+		if (this.rotaryDim = t, this.rotaryDim % 2 != 0) throw Error("rotaryDim must be even");
+		this.ropeBase = 1e4;
+		let n = J(0, this.rotaryDim, 2, "float32"), r = u(n, W(this.rotaryDim, "float32")), i = Q(W(this.ropeBase, "float32"), r);
+		this.ropeInvFreq = Z(i), r.dispose(), i.dispose(), n.dispose(), (this.config.modelType === "GenAI_NanoGPT_v1" ? this.config.useRope ?? !0 : !0) === !1 ? (this.ropeCos = null, this.ropeSin = null, this.ropeNegSin = null, this.ropeCacheLen = 0) : K(() => {
+			this.ensureRopeCache(this.config.blockSize * 4);
+		});
+	}
+	ensureRopeCache(e) {
+		K(() => {
+			if (e <= this.ropeCacheLen) return;
+			this.ropeCos && this.ropeCos.dispose(), this.ropeSin && this.ropeSin.dispose();
+			let t = Math.max(e, this.ropeCacheLen + this.config.blockSize * 4), n = J(0, t, 1, "float32").expandDims(1).mul(this.ropeInvFreq.expandDims(0));
+			this.ropeCos = Y(g(n).expandDims(-1)), this.ropeSin = Y(X(n).expandDims(-1)), this.ropeNegSin = Y(this.ropeSin.neg()), this.ropeCacheLen = t;
+		});
+	}
+	getCos() {
+		return this.ropeCos;
+	}
+	getSin() {
+		return this.ropeSin;
+	}
+	getNegSin() {
+		return this.ropeNegSin;
+	}
+	dispose() {
+		this.ropeCos && this.ropeCos.dispose(), this.ropeSin && this.ropeSin.dispose(), this.ropeInvFreq.dispose();
+	}
+};
+//#endregion
+export { wt as default };

package/dist/layers/TiedEmbedding.d.ts

@@ -0,0 +1,13 @@
+import { Tensor } from '@tensorflow/tfjs-core';
+import { default as BaseLayer, ForwardAttributes } from './BaseLayer';
+import { GPTConfig } from '../../models/config';
+export default class TiedEmbeddingOutputLayer extends BaseLayer {
+    private vocabSize;
+    private embedDim;
+    private initializer;
+    private WEIGHTS;
+    constructor(config: GPTConfig, name: string, parent?: BaseLayer);
+    embed(inputs: Tensor): Tensor;
+    project(inputs: Tensor): Tensor;
+    forward(_: ForwardAttributes, x: Tensor): Tensor;
+}

package/dist/layers/TiedEmbedding.js

@@ -0,0 +1,32 @@
+import { T as e, an as t } from "../dist-BewPQWjc.js";
+import { n } from "../dist-VEU5mfO0.js";
+import { isPackedTensor as r } from "../utilities/packed.js";
+import { transpose16 as i } from "../ops/transpose16.js";
+import { t as a } from "../pack16-Ck-spx_F.js";
+import o from "./BaseLayer.js";
+import { dot16 as s } from "../ops/dot16.js";
+//#region lib/layers/TiedEmbedding.ts
+var c = class extends o {
+	vocabSize;
+	embedDim;
+	initializer;
+	WEIGHTS;
+	constructor(t, r, i) {
+		super(t, i), this.WEIGHTS = r, this.vocabSize = t.vocabSize, this.embedDim = t.nEmbed, this.initializer = n({
+			mean: 0,
+			stddev: .02
+		}), this.addVariable(this.WEIGHTS, e(this.initializer.apply([this.vocabSize, this.embedDim]), !0, r));
+	}
+	embed(e) {
+		return t(this.getVariable(this.WEIGHTS), e, 0);
+	}
+	project(e) {
+		let t = r(e) ? a(this.getVariable(this.WEIGHTS), void 0, 32) : this.getVariable(this.WEIGHTS), n = i(t);
+		return r(e) && t.dispose(), s(e, n);
+	}
+	forward(e, t) {
+		return this.project(t);
+	}
+};
+//#endregion
+export { c as default };

package/dist/layers/TransformerBlock.d.ts

@@ -0,0 +1,27 @@
+import { AttentionScores, CausalSelfAttentionConfig, KVCache } from './CausalSelfAttention';
+import { MLPConfig } from './MLP';
+import { RMSNormConfig } from './RMSNorm';
+import { default as BaseLayer, ForwardAttributes } from './BaseLayer';
+import { Tensor } from '@tensorflow/tfjs-core';
+import { GPTConfig } from '../../models/config';
+interface BlockAttributes extends ForwardAttributes {
+    pastKV?: KVCache;
+    seed?: number;
+    attentionScores?: AttentionScores;
+    ropePositionOffset?: number;
+}
+export type TransformerBlockConfig = MLPConfig & RMSNormConfig & CausalSelfAttentionConfig;
+export default class Block extends BaseLayer<BlockAttributes> {
+    private ln1;
+    private attn;
+    private ln2;
+    private mlp;
+    private index;
+    skipped: boolean;
+    private blockConfig;
+    constructor(index: number, config: GPTConfig, blockConfig: TransformerBlockConfig, parent?: BaseLayer);
+    private getMLPOutput;
+    forward(attrs: BlockAttributes, x: Tensor): Tensor;
+    dispose(): void;
+}
+export {};

package/dist/layers/TransformerBlock.js

@@ -0,0 +1,51 @@
+import { di as e, oi as t } from "../dist-BewPQWjc.js";
+import n from "./BaseLayer.js";
+import r from "./CausalSelfAttention.js";
+import i from "./MLP.js";
+import a from "./RMSNorm.js";
+import { add16 as o } from "../ops/add16.js";
+//#region lib/layers/TransformerBlock.ts
+var s = class extends n {
+	ln1;
+	attn;
+	ln2;
+	mlp;
+	index;
+	skipped = !1;
+	blockConfig;
+	constructor(e, t, n, o) {
+		super(t, o), this.index = e, this.blockConfig = n, this.ln1 = new a(t, this.blockConfig, `block_${this.index}_rms1`, this), this.attn = new r(this.index, t, this.blockConfig, this), this.ln2 = new a(t, this.blockConfig, `block_${this.index}_rms2`, this), this.mlp = new i(this.index, t, this.blockConfig, this);
+	}
+	getMLPOutput(e, n) {
+		let r = this.ln2.call({ training: n.training }, e), i = this.mlp.call(n, r);
+		n.outputEmbeddings ? (t(r), n.embeddings.push({
+			name: `block_ln2_${this.index}`,
+			tensor: r
+		})) : r.dispose();
+		let a = o(e, i);
+		return e.dispose(), n.outputEmbeddings ? (t(i), n.embeddings.push({
+			name: `block_mlp_out_${this.index}`,
+			tensor: i
+		})) : i.dispose(), a;
+	}
+	forward(n, r) {
+		return e(() => {
+			if (this.skipped) return r;
+			let e = this.ln1.call(n, r), i = this.attn.call(n, e);
+			n.outputEmbeddings ? (t(e), n.embeddings.push({
+				name: `block_ln1_${this.index}`,
+				tensor: e
+			})) : e.dispose();
+			let a = o(r, i);
+			return n.outputEmbeddings ? (t(i), n.embeddings.push({
+				name: `block_attn_out_${this.index}`,
+				tensor: i
+			})) : i.dispose(), this.getMLPOutput(a, n);
+		});
+	}
+	dispose() {
+		this.ln1.dispose(), this.attn.dispose(), this.ln2.dispose(), this.mlp.dispose();
+	}
+};
+//#endregion
+export { s as default };

package/dist/layers/WeightStore.d.ts

@@ -0,0 +1,20 @@
+import { Tensor, Variable } from '@tensorflow/tfjs-core';
+export default class WeightStore {
+    private _variables;
+    private touchedVariables;
+    onWeightRead?: (name: string, variable: Variable) => Tensor;
+    saveWeights(map: Map<string, Tensor[]>): void;
+    loadWeights(weights: Map<string, Tensor[]>, reference: boolean, trainable?: boolean): void;
+    addVariable(name: string, variable?: Variable): void;
+    deleteVariable(name: string): void;
+    get variables(): Variable[];
+    get variableNames(): string[];
+    get trainableVariables(): Variable[];
+    getRawVariable(name: string): Variable;
+    getVariable(name: string): Tensor;
+    setTrainable(names: string[]): void;
+    hasVariable(name: string): boolean;
+    setVariable(name: string, variable: Variable): void;
+    touchVariables(names: string[]): void;
+    dispose(): void;
+}

package/dist/layers/WeightStore.js

@@ -0,0 +1,69 @@
+import { a as e } from "../chunk-BPntVaq0.js";
+import { Jr as t, T as n } from "../dist-BewPQWjc.js";
+import { t as r } from "../picomatch-3tUnMMbd.js";
+//#region lib/layers/WeightStore.ts
+var i = /* @__PURE__ */ e(r(), 1), a = class {
+	_variables = /* @__PURE__ */ new Map();
+	touchedVariables = /* @__PURE__ */ new Set();
+	onWeightRead;
+	saveWeights(e) {
+		this._variables.forEach((n, r) => {
+			n && this.touchedVariables.has(r) && e.set(r, [t(n)]);
+		});
+	}
+	loadWeights(e, t, r = !0) {
+		e.forEach((e, i) => {
+			let a = e[0], o = this._variables.get(i);
+			o ? o.assign(a) : this._variables.set(i, n(a, r, i)), t ? this.touchedVariables.delete(i) : this.touchedVariables.add(i);
+		});
+	}
+	addVariable(e, t) {
+		this._variables.set(e, t || null);
+	}
+	deleteVariable(e) {
+		let t = this._variables.get(e);
+		t && t.dispose(), this._variables.delete(e), this.touchedVariables.delete(e);
+	}
+	get variables() {
+		return Array.from(this._variables.values()).filter((e) => e !== null);
+	}
+	get variableNames() {
+		return Array.from(this._variables.keys());
+	}
+	get trainableVariables() {
+		return Array.from(this._variables.values()).filter((e) => e !== null && e.trainable);
+	}
+	getRawVariable(e) {
+		let t = this._variables.get(e);
+		if (!t) throw Error(`Variable ${e} not found`);
+		return t;
+	}
+	getVariable(e) {
+		let t = this._variables.get(e);
+		if (!t) throw Error(`Variable ${e} not found`);
+		return this.onWeightRead ? this.onWeightRead(e, t) : t;
+	}
+	setTrainable(e) {
+		let t = (0, i.default)(e);
+		this._variables.forEach((e, n) => {
+			e && (e.trainable = t(n));
+		});
+	}
+	hasVariable(e) {
+		return !!this._variables.get(e);
+	}
+	setVariable(e, t) {
+		if (!this._variables.has(e)) throw Error(`Variable ${e} not found`);
+		this._variables.set(e, t);
+	}
+	touchVariables(e) {
+		for (let t of e) this._variables.get(t) && this.touchedVariables.add(t);
+	}
+	dispose() {
+		this._variables.forEach((e) => {
+			e?.dispose();
+		}), this._variables.clear();
+	}
+};
+//#endregion
+export { a as default };

package/dist/loader/load.d.ts

@@ -0,0 +1,6 @@
+import { LoadResult } from './types';
+export declare const VERSION = 2;
+export interface LoadModelOptions {
+    sourceURL?: string;
+}
+export declare function loadModel(data: Blob | Buffer | string, options?: LoadModelOptions): Promise<LoadResult>;

package/dist/loader/load.js

@@ -0,0 +1,2 @@
+import { d as e, u as t } from "../main-D5CbfCiV.js";
+export { t as VERSION, e as loadModel };

package/dist/loader/loadHF.d.ts

@@ -0,0 +1,8 @@
+import { ITokeniser } from '../../main';
+import { default as Model, ModelForwardAttributes } from '../../models/model';
+import { TransformersMetadata } from './types';
+export default function loadHuggingFace(name: string): Promise<{
+    model: Model<ModelForwardAttributes>;
+    tokeniser: ITokeniser;
+    metaData: TransformersMetadata;
+}>;

package/dist/loader/loadHF.js

@@ -0,0 +1,2 @@
+import { f as e } from "../main-D5CbfCiV.js";
+export { e as default };

package/dist/loader/loadTransformers.d.ts

@@ -0,0 +1,4 @@
+import { GPTConfig } from '../../models/config';
+import { LoadResult, TransformersConfig, TransformersMetadata, TransformersTokeniser } from './types';
+export declare function mapTransformersConfigToGPTConfig(config: TransformersConfig): GPTConfig;
+export default function loadTransformers(config: TransformersConfig, tokeniser: TransformersTokeniser, metadata: TransformersMetadata, weightData: ArrayBuffer): Promise<LoadResult>;

package/dist/loader/loadTransformers.js

@@ -0,0 +1,2 @@
+import { h as e, m as t } from "../main-D5CbfCiV.js";
+export { t as default, e as mapTransformersConfigToGPTConfig };

package/dist/loader/loadZipMeta.d.ts

@@ -0,0 +1,3 @@
+import { default as zip } from 'jszip';
+import { TransformersMetadata } from './types';
+export declare function loadZipMeta(zipFile: zip): Promise<TransformersMetadata>;

package/dist/loader/loadZipMeta.js

@@ -0,0 +1,16 @@
+//#region lib/loader/loadZipMeta.ts
+async function e(e) {
+	let t = await e.file("meta.json")?.async("string"), n = {
+		version: 0,
+		application: ""
+	};
+	if (t) try {
+		n = JSON.parse(t);
+	} catch (e) {
+		throw console.error(e), Error("Failed to parse meta.json in the zip archive");
+	}
+	else console.warn("meta.json not found in the zip archive, using default metadata");
+	return n;
+}
+//#endregion
+export { e as loadZipMeta };

package/dist/loader/newZipLoad.d.ts

@@ -0,0 +1,3 @@
+import { default as zip } from 'jszip';
+import { LoadResult, TransformersMetadata } from './types';
+export default function loadZipFile(zipFile: zip, metaData: TransformersMetadata): Promise<LoadResult>;

package/dist/loader/newZipLoad.js

@@ -0,0 +1,2 @@
+import { p as e } from "../main-D5CbfCiV.js";
+export { e as default };

package/dist/loader/oldZipLoad.d.ts

@@ -0,0 +1,9 @@
+import { default as zip } from 'jszip';
+import { ITokeniser } from '../../main';
+import { default as Model, ModelForwardAttributes } from '../../models/model';
+import { TransformersMetadata } from './types';
+export default function loadOldModel(zipFile: zip, metaData: TransformersMetadata): Promise<{
+    model: Model<ModelForwardAttributes>;
+    tokeniser: ITokeniser;
+    metaData: TransformersMetadata;
+}>;