@genai-fi/nanogpt 0.20.0 → 0.20.2

package/dist/main.d.ts

@@ -0,0 +1,50 @@
+import { default as PretrainingTask } from './training/tasks/PretrainingTask';
+import { default as StartSentenceTask } from './training/tasks/StartSentenceTask';
+import { default as ConversationTask } from './training/tasks/ConversationTask';
+import { pack16 } from './ops/pack16';
+import { unpack16 } from './ops/unpack16';
+import { default as CausalSelfAttention } from './layers/CausalSelfAttention';
+import { default as MLP } from './layers/MLP';
+import { default as TransformerBlock } from './layers/TransformerBlock';
+import { default as RoPECache } from './layers/RoPECache';
+export { default as NanoGPT } from './models/NanoGPTV1';
+export { default as TeachableLLM } from './TeachableLLM';
+export { default as CharTokeniser } from './tokeniser/CharTokeniser';
+export { default as BPETokeniser } from './tokeniser/bpe';
+export { default as waitForModel } from './utilities/waitForModel';
+export { default as generateDatasetID } from './utilities/datasetID';
+export { default as loadTextData } from './data/textLoader';
+export type { DatasetMetadata, ModelMode } from './loader/types';
+export { default as Generator, type IGenerator } from './Generator';
+export { default as Evaluator } from './training/Evaluator';
+export { default as Trainer } from './Trainer';
+export type { IGenerateOptions } from './Generator';
+export { type ModelForwardAttributes, default as Model } from './models/model';
+export type { ITokeniser, Conversation, Roles } from './tokeniser/type';
+export type { TrainingOptions, TrainingLogEntry } from './training/types';
+export type { GPTConfig } from './models/config';
+export { estimateParameterCount, estimateMemoryUsage, estimateTrainingMemoryUsage, estimateResources, validateConfig, } from './utilities/parameters';
+export { default as topP } from './utilities/topP';
+export { Task, tokensFromTasks } from './training/tasks/Task';
+export declare const tasks: {
+    PretrainingTask: typeof PretrainingTask;
+    StartSentenceTask: typeof StartSentenceTask;
+    ConversationTask: typeof ConversationTask;
+};
+declare const ops: {
+    pack16: typeof pack16;
+    unpack16: typeof unpack16;
+};
+export { ops };
+export { selectBackend } from './backend';
+export { default as performanceTest } from './utilities/performance';
+export declare const layers: {
+    CausalSelfAttention: typeof CausalSelfAttention;
+    MLP: typeof MLP;
+    TransformerBlock: typeof TransformerBlock;
+    RoPECache: typeof RoPECache;
+};
+export { AdamWOptimizer } from './training/AdamW';
+export { default as checks } from './checks';
+export type { TensorStatistics } from './checks/weights';
+export { sentenceEmbeddings, sentenceEmbeddingsTensor } from './utilities/sentences';

package/dist/main.js

@@ -0,0 +1,16 @@
+import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-D5CbfCiV.js";
+import c from "./tokeniser/CharTokeniser.js";
+import l from "./tokeniser/bpe.js";
+import { AdamWOptimizer as u } from "./training/AdamW.js";
+import d from "./utilities/topP.js";
+import f from "./training/Evaluator.js";
+import p from "./utilities/waitForModel.js";
+import m from "./utilities/datasetID.js";
+import h from "./data/textLoader.js";
+import { estimateMemoryUsage as g, estimateParameterCount as _, estimateResources as v, estimateTrainingMemoryUsage as y, validateConfig as b } from "./utilities/parameters.js";
+import { Task as x, tokensFromTasks as S } from "./training/tasks/Task.js";
+import { selectBackend as C } from "./backend.js";
+import w from "./utilities/performance.js";
+import T from "./checks/index.js";
+import { sentenceEmbeddings as E, sentenceEmbeddingsTensor as D } from "./utilities/sentences.js";
+export { u as AdamWOptimizer, l as BPETokeniser, c as CharTokeniser, f as Evaluator, a as Generator, t as Model, s as NanoGPT, x as Task, n as TeachableLLM, e as Trainer, T as checks, g as estimateMemoryUsage, _ as estimateParameterCount, v as estimateResources, y as estimateTrainingMemoryUsage, m as generateDatasetID, o as layers, h as loadTextData, r as ops, w as performanceTest, C as selectBackend, E as sentenceEmbeddings, D as sentenceEmbeddingsTensor, i as tasks, S as tokensFromTasks, d as topP, b as validateConfig, p as waitForModel };

package/dist/matMul16-BNfZSnNM.js

@@ -0,0 +1,81 @@
+import { Fi as e, ii as t } from "./dist-BewPQWjc.js";
+import { t as n } from "./gelu-Bf1HW1RY.js";
+import { isPackedTensor as r } from "./utilities/packed.js";
+import { transpose16 as i } from "./ops/transpose16.js";
+import { reshape16 as a } from "./ops/reshape16.js";
+import { mul16 as o } from "./ops/mul16.js";
+import "./ops/webgl/matMul16.js";
+import "./ops/cpu/matMul16.js";
+import { t as s } from "./pack16-Ck-spx_F.js";
+//#region lib/ops/grads/matMul16.ts
+var c = {
+	kernelName: "MatMul16",
+	inputsToSave: ["A", "B"],
+	outputsToSave: [],
+	gradFunc: (e, t, r) => {
+		let [s, c] = t;
+		if (Array.isArray(e)) throw Error("Expected dy to be a single Tensor");
+		let u = e, { transposeA: p, transposeB: m, scale: h, activation: g, originalShape: _, perm: v } = r;
+		if (v && _) {
+			let e = Array(v.length);
+			for (let t = 0; t < v.length; ++t) e[v[t]] = t;
+			let t = u;
+			u = i(u, e), t.dispose();
+		}
+		if (_) {
+			let e = u;
+			u = a(u, _), e.dispose();
+		}
+		if (g === "gelu") {
+			let e = u, t = l(s, c, p, m);
+			u = n(e, t), e.dispose(), t.dispose();
+		} else if (g === "relu2") {
+			let e = u, t = l(s, c, p, m, {
+				activation: "relu",
+				scale: 2
+			});
+			u = o(e, t), e.dispose(), t.dispose();
+		}
+		if (!p && !m) return {
+			A: () => h === void 0 ? l(u, c, !1, !0) : d(u, c, h, !1, !0),
+			B: () => h === void 0 ? l(s, u, !0, !1) : f(s, u, h, !0, !1)
+		};
+		if (!p && m) return {
+			A: () => h === void 0 ? l(u, c, !1, !1) : d(u, c, h, !1, !1),
+			B: () => h === void 0 ? l(s, u, !0, !1) : f(s, u, h, !0, !1)
+		};
+		if (p && !m) return {
+			A: () => h === void 0 ? l(c, u, !1, !0) : f(c, u, h, !1, !0),
+			B: () => h === void 0 ? l(s, u, !1, !1) : f(s, u, h, !1, !1)
+		};
+		throw Error("Gradient for transposeA=true and transposeB=true is not supported yet.");
+	}
+};
+e(c);
+//#endregion
+//#region lib/ops/matMul16.ts
+function l(e, n, i = !1, a = !1, o = {}) {
+	let c = r(e), l = r(n), u = c || l, d = !u || c ? e : s(e), f = !u || l ? n : s(n), p = t().runKernel("MatMul16", {
+		A: d,
+		B: f
+	}, {
+		transposeA: i,
+		transposeB: a,
+		...o
+	});
+	return u && !c && d.dispose(), u && !l && f.dispose(), p;
+}
+function u(e, t, n, r = !1, i = !1) {
+	return l(e, t, r, i, { scale: n });
+}
+function d(e, t, n, r = !1, i = !1) {
+	return l(e, t, r, i, { scaleA: n });
+}
+function f(e, t, n, r = !1, i = !1) {
+	return l(e, t, r, i, { scaleB: n });
+}
+function p(e, t, n = !1, r = !1) {
+	return l(e, t, n, r, { activation: "gelu" });
+}
+//#endregion
+export { u as a, f as i, p as n, c as o, d as r, l as t };

package/dist/matMulGelu-CPTntosE.js

@@ -0,0 +1,162 @@
+import { Ci as e, Ii as t, In as n, Ps as r, di as i, gr as a, ii as o, oc as s } from "./dist-BewPQWjc.js";
+import { a as c } from "./gpgpu_math-DvLcCH6u.js";
+import { t as l } from "./Reshape-CABOPB9d.js";
+//#region node_modules/@tensorflow/tfjs-backend-webgl/dist/mulmat_packed_gpu.js
+var u = class {
+	constructor(e, t, n, r = !1, i = !1, a = !1, o = null, s = !1, l = !1) {
+		this.variableNames = ["matrixA", "matrixB"], this.packedInputs = !0, this.packedOutput = !0, this.outputShape = n, this.enableShapeUniforms = c(this.outputShape.length);
+		let u = r ? e[1] : e[2], d = Math.ceil(u / 2), f = r ? "i * 2, rc.y" : "rc.y, i * 2", p = i ? "rc.z, i * 2" : "i * 2, rc.z", m = r ? ["a.xxyy", "a.zzww"] : ["a.xxzz", "a.yyww"], h = i ? ["b.xzxz", "b.ywyw"] : ["b.xyxy", "b.zwzw"], g = "", _ = "";
+		o && (g = s ? `vec4 activation(vec4 a) {
+          vec4 b = getPreluActivationWeightsAtOutCoords();
+          ${o}
+        }` : l ? `vec4 activation(vec4 a) {
+          vec4 b = getLeakyreluAlphaAtOutCoords();
+          ${o}
+        }` : `vec4 activation(vec4 x) {
+          ${o}
+        }`, _ = "result = activation(result);");
+		let v = a ? "result += getBiasAtOutCoords();" : "";
+		a && this.variableNames.push("bias"), s && this.variableNames.push("preluActivationWeights"), l && this.variableNames.push("leakyreluAlpha");
+		let y = "rc.x", b = "rc.x";
+		e[0] < t[0] ? y = `imod(rc.x, ${e[0]})` : t[0] < e[0] && (b = `imod(rc.x, ${t[0]})`), this.userCode = `
+      ${g}
+      // Don't use uniform for sharedDimensionPacked for performance.
+      const float sharedDimension = ${d}.0;
+
+      vec4 dot2x2ARowBCol(ivec3 rc) {
+        vec4 result = vec4(0);
+        int batchA = ${y};
+        int batchB = ${b};
+        for (int i = 0; i < ${d}; i++) {
+          vec4 a = getMatrixA(batchA, ${f});
+          vec4 b = getMatrixB(batchB, ${p});
+
+          // These swizzled products need to be separately added.
+          // See: https://github.com/tensorflow/tfjs/issues/1735
+          result += (${m[0]} * ${h[0]});
+          result += (${m[1]} * ${h[1]});
+        }
+        return result;
+      }
+
+      void main() {
+        ivec3 rc = getOutputCoords();
+        vec4 result = dot2x2ARowBCol(rc);
+
+        ${v}
+
+        ${_}
+
+        setOutput(result);
+      }
+    `;
+	}
+}, d = .7978845608028654, f = .044715, p = `
+    vec4 x3 = x * x * x;
+    vec4 inner = x + ${f} * x3;
+    inner = ${d} * inner;
+    inner = vec4(
+        abs(inner[0]) > 15.0 ? sign(inner[0]) : tanh(inner[0]),
+        abs(inner[1]) > 15.0 ? sign(inner[1]) : tanh(inner[1]),
+        abs(inner[2]) > 15.0 ? sign(inner[2]) : tanh(inner[2]),
+        abs(inner[3]) > 15.0 ? sign(inner[3]) : tanh(inner[3])
+    );
+    inner = 0.5 * (1.0 + inner);
+    vec4 result = x * inner;
+    return result;
+`, m = `
+    vec4 a2 = a * a;
+    vec4 a3 = a2 * a;
+    vec4 u  = ${d} * (a + ${f} * a3);
+    vec4 t = vec4(
+        abs(u[0]) > 15.0 ? sign(u[0]) : tanh(u[0]),
+        abs(u[1]) > 15.0 ? sign(u[1]) : tanh(u[1]),
+        abs(u[2]) > 15.0 ? sign(u[2]) : tanh(u[2]),
+        abs(u[3]) > 15.0 ? sign(u[3]) : tanh(u[3])
+    );
+    vec4 sech2 = 1.0 - t * t;
+    vec4 du_dx = ${d} * (1.0 + 3.0 * ${f} * a2);
+    vec4 dgelu = 0.5 * (1.0 + t) + 0.5 * a * sech2 * du_dx;
+    return dgelu * b;
+`, h = 1e3;
+function g({ a: t, b: i, transposeA: a, transposeB: o, backend: c, activationSnippet: d, multiplier: f }) {
+	let p = t.shape.length, m = i.shape.length, h = a ? t.shape[p - 2] : t.shape[p - 1], g = o ? i.shape[m - 1] : i.shape[m - 2], _ = a ? t.shape[p - 1] : t.shape[p - 2], v = o ? i.shape[m - 2] : i.shape[m - 1], y = t.shape.slice(0, -2), b = i.shape.slice(0, -2), x = s(y), S = s(b), C = n(t.shape.slice(0, -2), i.shape.slice(0, -2)).concat([_, v]);
+	r(h === g, () => `Error in matMul: inner shapes (${h}) and (${g}) of Tensors with shapes ${t.shape} and ${i.shape} and transposeA=${a} and transposeB=${o} must match.`);
+	let w = a ? [
+		x,
+		h,
+		_
+	] : [
+		x,
+		_,
+		h
+	], T = o ? [
+		S,
+		v,
+		g
+	] : [
+		S,
+		g,
+		v
+	], E = l({
+		inputs: { x: t },
+		backend: c,
+		attrs: { shape: w }
+	}), D = l({
+		inputs: { x: i },
+		backend: c,
+		attrs: { shape: T }
+	}), O = [E, D], k = Math.max(x, S), A = d, j = e(t.dtype, i.dtype), M = new u(w, T, [
+		k,
+		_,
+		v
+	], a, o, !1, A, !!f, !1), N = [E, D];
+	f && N.push(f);
+	let P = c.runWebGLProgram(M, N, j), F = l({
+		inputs: { x: P },
+		backend: c,
+		attrs: { shape: C }
+	});
+	O.push(P);
+	for (let e of O) c.disposeIntermediateTensorInfo(e);
+	return F;
+}
+function _(e) {
+	let { inputs: t, backend: n } = e, { x: r, kernel: i } = t;
+	if (r === void 0 || i === void 0) throw Error("BatchMatMul requires two input tensors.");
+	return g({
+		a: r,
+		b: i,
+		transposeA: !1,
+		transposeB: !1,
+		backend: n,
+		activationSnippet: p
+	});
+}
+t({
+	kernelName: "MatMulGelu",
+	backendName: "webgl",
+	kernelFunc: _
+});
+function v(e) {
+	let { dy: t, x: n, kernel: r } = e.inputs, s = e.backend;
+	return i(() => {
+		let e = o().makeTensorFromTensorInfo(g({
+			a: n,
+			b: r,
+			transposeA: !1,
+			transposeB: !1,
+			backend: s,
+			activationSnippet: m,
+			multiplier: t
+		}));
+		return [a(e, r, !1, !0), a(n, e, !0, !1)];
+	});
+}
+t({
+	kernelName: "MatMulGeluGrad",
+	backendName: "webgl",
+	kernelFunc: v
+});
+//#endregion
+export { u as i, g as n, _ as r, h as t };

package/dist/models/NanoGPTV1.d.ts

@@ -0,0 +1,16 @@
+import { GPTConfigV1 } from './config';
+import { Tensor } from '@tensorflow/tfjs-core';
+import { default as Model, ModelForwardAttributes } from './model';
+export default class NanoGPTV1 extends Model<ModelForwardAttributes, GPTConfigV1> {
+    private wte;
+    private wpe?;
+    private blocks;
+    private lnF;
+    private ropeCache?;
+    constructor(config?: Partial<GPTConfigV1>);
+    getClassName(): string;
+    private inputPhase;
+    forward(attrs: ModelForwardAttributes, idx: Tensor): Tensor;
+    project(embeddings: Tensor): Tensor;
+    dispose(): void;
+}

package/dist/models/NanoGPTV1.js

@@ -0,0 +1,2 @@
+import { y as e } from "../main-D5CbfCiV.js";
+export { e as default };

package/dist/models/NanoGPTV2.d.ts

@@ -0,0 +1,16 @@
+import { GPTConfigV2 } from './config';
+import { Tensor } from '@tensorflow/tfjs-core';
+import { default as Model, ModelForwardAttributes } from './model';
+export default class NanoGPTV2 extends Model<ModelForwardAttributes, GPTConfigV2> {
+    private wte;
+    private wpe?;
+    private blocks;
+    private lnF;
+    private ropeCache?;
+    constructor(config?: Partial<GPTConfigV2>);
+    getClassName(): string;
+    private inputPhase;
+    forward(attrs: ModelForwardAttributes, idx: Tensor): Tensor;
+    project(embeddings: Tensor): Tensor;
+    dispose(): void;
+}

package/dist/models/NanoGPTV2.js

@@ -0,0 +1,2 @@
+import { v as e } from "../main-D5CbfCiV.js";
+export { e as default };

package/dist/models/config.d.ts

@@ -0,0 +1,27 @@
+export interface LoRAConfig {
+    rank: number;
+    alpha: number;
+    variables: string[];
+}
+export interface GPTConfigBase {
+    modelType?: string;
+    vocabSize: number;
+    blockSize: number;
+    nLayer: number;
+    nHead: number;
+    nEmbed: number;
+    mlpFactor: number;
+    loraConfig?: Map<string, LoRAConfig>;
+    loraName?: string;
+}
+export interface GPTConfigV1 extends GPTConfigBase {
+    modelType: 'GenAI_NanoGPT_v1';
+    useRope: boolean;
+}
+export interface GPTConfigV2 extends GPTConfigBase {
+    modelType: 'GenAI_NanoGPT_v2';
+    windowSize?: string;
+}
+export type GPTConfig = GPTConfigV1 | GPTConfigV2;
+export declare const defaultConfig: GPTConfig;
+export declare function validateConfig(config: unknown): asserts config is GPTConfig;

package/dist/models/config.js

@@ -0,0 +1,37 @@
+//#region lib/models/config.ts
+var e = {
+	modelType: "GenAI_NanoGPT_v2",
+	vocabSize: 2e3,
+	blockSize: 128,
+	nLayer: 6,
+	nHead: 4,
+	nEmbed: 256,
+	mlpFactor: 4
+};
+function t(e, t) {
+	if (typeof e[t] != "number" || Number.isNaN(e[t])) throw Error(`Invalid config: "${t}" must be a number.`);
+}
+function n(e) {
+	let n = (e) => typeof e == "object" && !!e && !Array.isArray(e);
+	if (!n(e)) throw Error("Invalid config: expected an object.");
+	if (t(e, "vocabSize"), t(e, "blockSize"), t(e, "nLayer"), t(e, "nHead"), t(e, "nEmbed"), t(e, "mlpFactor"), e.loraConfig !== void 0) {
+		if (!n(e.loraConfig)) throw Error("Invalid config: \"loraConfig\" must be an object.");
+		let r = Object.values(e.loraConfig);
+		if (!r.every((e) => n(e))) throw Error("Invalid config: each entry in \"loraConfig\" must be an object.");
+		if (!r.every((e) => "rank" in e && "alpha" in e && "variables" in e)) throw Error("Invalid config: each LoRA config must have \"rank\", \"alpha\", and \"variables\" fields.");
+		r.forEach((e) => {
+			if (t(e, "rank"), t(e, "alpha"), !Array.isArray(e.variables) || !e.variables.every((e) => typeof e == "string")) throw Error("Invalid config: \"variables\" must be a string array.");
+		});
+	}
+	if (e.modelType === "GenAI_NanoGPT_v1") {
+		if (typeof e.useRope != "boolean") throw Error("Invalid config: \"useRope\" must be a boolean for GenAI_NanoGPT_v1.");
+		return;
+	}
+	if (e.modelType === "GenAI_NanoGPT_v2") {
+		if (e.windowSize !== void 0 && typeof e.windowSize != "string") throw Error("Invalid config: \"windowSize\" must be a string for GenAI_NanoGPT_v2.");
+		return;
+	}
+	throw Error("Invalid config: \"modelType\" must be \"GenAI_NanoGPT_v1\" or \"GenAI_NanoGPT_v2\".");
+}
+//#endregion
+export { e as defaultConfig, n as validateConfig };

package/dist/models/factory.d.ts

@@ -0,0 +1,3 @@
+import { GPTConfig } from '../../models/config';
+import { default as Model, ModelForwardAttributes } from './model';
+export default function createModelInstance(config: GPTConfig): Model<ModelForwardAttributes, GPTConfig>;

package/dist/models/factory.js

@@ -0,0 +1,2 @@
+import { _ as e } from "../main-D5CbfCiV.js";
+export { e as default };

package/dist/models/model.d.ts

@@ -0,0 +1,44 @@
+import { Tensor } from '@tensorflow/tfjs-core';
+import { ForwardAttributes, default as BaseLayer } from '../layers/BaseLayer';
+import { AttentionScores, KVCache } from '../layers/CausalSelfAttention';
+import { TransformersMetadata } from '../../loader/types';
+import { GPTConfig, LoRAConfig } from './config';
+import { default as LoRA } from '../../layers/LoRA';
+export interface ModelForwardAttributes extends ForwardAttributes {
+    cache?: KVCache[];
+    attentionScores?: AttentionScores;
+    seed?: number;
+    skipLogits?: boolean;
+    ropePositionOffset?: number;
+}
+export interface TrainingState {
+    steps: number;
+    learningRate: number;
+    batchSize: number;
+    loss: number;
+    tokensProcessed: number;
+    duration: number;
+}
+export default abstract class Model<T extends ModelForwardAttributes, C extends GPTConfig = GPTConfig> extends BaseLayer<T, C> {
+    lossScaling: number;
+    trainingState: TrainingState | null;
+    metaData: TransformersMetadata;
+    private loraLayer?;
+    private loraMap;
+    constructor(config: C);
+    createLoRA(name: string, loraConfig: LoRAConfig): void;
+    deleteLoRA(name: string): void;
+    renameLoRA(oldName: string, newName: string): void;
+    mergeLoRA(name: string): void;
+    attachLoRA(name: string): void;
+    detachLoRA(): void;
+    hasLoRA(name?: string): boolean;
+    listLoRAs(): string[];
+    get lora(): LoRA | null;
+    abstract getClassName(): string;
+    abstract forward(attrs: T, idx: Tensor): Tensor;
+    abstract project(embeddings: Tensor): Tensor;
+    abstract dispose(): void;
+    getNumParams(): number;
+    protected validateInput(idx: Tensor): void;
+}

package/dist/models/model.js

@@ -0,0 +1,2 @@
+import { b as e } from "../main-D5CbfCiV.js";
+export { e as default };

package/dist/ops/adamAdjust.d.ts

@@ -0,0 +1,2 @@
+import { Tensor } from '@tensorflow/tfjs-core';
+export declare function adamAdjust(moments: Tensor, value: Tensor, beta1: number, beta2: number, epsilon: number, learningRate: number, weightDecay?: number): Tensor;

package/dist/ops/adamAdjust.js

@@ -0,0 +1,18 @@
+import { ii as e } from "../dist-BewPQWjc.js";
+import "./cpu/adamAdjust.js";
+import "./webgl/adamAdjust.js";
+//#region lib/ops/adamAdjust.ts
+function t(t, n, r, i, a, o, s = 0) {
+	return e().runKernel("AdamAdjust", {
+		moments: t,
+		value: n
+	}, {
+		beta1: r,
+		beta2: i,
+		epsilon: a,
+		learningRate: o,
+		weightDecay: s
+	});
+}
+//#endregion
+export { t as adamAdjust };

package/dist/ops/adamMoments.d.ts

@@ -0,0 +1,2 @@
+import { Scalar, Tensor } from '@tensorflow/tfjs-core';
+export declare function adamMoments(moments: Tensor, gradient: Tensor, beta1: number, beta2: number, scaling: Scalar): Tensor;

package/dist/ops/adamMoments.js

@@ -0,0 +1,16 @@
+import { ii as e } from "../dist-BewPQWjc.js";
+import "./cpu/adamMoments.js";
+import "./webgl/adamMoments.js";
+//#region lib/ops/adamMoments.ts
+function t(t, n, r, i, a) {
+	return e().runKernel("AdamMoments", {
+		moments: t,
+		gradient: n,
+		scaling: a
+	}, {
+		beta1: r,
+		beta2: i
+	});
+}
+//#endregion
+export { t as adamMoments };

package/dist/ops/add16.d.ts

@@ -0,0 +1,2 @@
+import { Tensor } from '@tensorflow/tfjs-core';
+export declare function add16(a: Tensor, b: Tensor): Tensor;

package/dist/ops/add16.js

@@ -0,0 +1,12 @@
+import { ii as e, qr as t } from "../dist-BewPQWjc.js";
+import { isPackedTensor as n } from "../utilities/packed.js";
+import "./grads/add16.js";
+//#region lib/ops/add16.ts
+function r(r, i) {
+	return !n(r) && !n(i) ? t(r, i) : e().runKernel("Add16", {
+		a: r,
+		b: i
+	});
+}
+//#endregion
+export { r as add16 };

package/dist/ops/appendCache.d.ts

@@ -0,0 +1,2 @@
+import { Tensor } from '@tensorflow/tfjs-core';
+export declare function appendCache(item: Tensor, maxSize: number, pastLen: number, cache?: Tensor): Tensor;

package/dist/ops/appendCache.js

@@ -0,0 +1,25 @@
+import { _r as e, ii as t, kt as n } from "../dist-BewPQWjc.js";
+import { isPackedTensor as r } from "../utilities/packed.js";
+import "./cpu/appendCache.js";
+import "./webgl/appendCache.js";
+//#region lib/ops/appendCache.ts
+function i(i, a, o, s) {
+	if (!s) {
+		let t = i.shape[2], o = r(i);
+		return e([i, n([
+			i.shape[0],
+			i.shape[1],
+			a - t,
+			i.shape[3]
+		], o ? "int32" : i.dtype)], 2);
+	}
+	return t().runKernel("AppendCache", {
+		cache: s,
+		item: i
+	}, {
+		maxSize: a,
+		pastLen: o
+	});
+}
+//#endregion
+export { i as appendCache };

package/dist/ops/attentionMask.d.ts

@@ -0,0 +1,2 @@
+import { Tensor } from '@tensorflow/tfjs-core';
+export declare function attentionMask(q: Tensor, k: Tensor, divisor: number, pastLen?: number): Tensor;

package/dist/ops/attentionMask.js

@@ -0,0 +1,16 @@
+import { ii as e } from "../dist-BewPQWjc.js";
+import "./cpu/attentionMask.js";
+import "./webgl/attentionMask.js";
+import "./grads/attentionMask.js";
+//#region lib/ops/attentionMask.ts
+function t(t, n, r, i) {
+	return e().runKernel("AttentionMask", {
+		q: t,
+		k: n
+	}, {
+		divisor: r,
+		pastLen: i || 0
+	});
+}
+//#endregion
+export { t as attentionMask };

package/dist/ops/concat16.d.ts

@@ -0,0 +1,2 @@
+import { Rank, Tensor } from '@tensorflow/tfjs-core';
+export declare function concat16<R extends Rank = Rank>(x: Tensor<R>[], axis?: number): Tensor<R>;

package/dist/ops/concat16.js

@@ -0,0 +1,8 @@
+import { _r as e, ii as t } from "../dist-BewPQWjc.js";
+import { isPackedTensor as n } from "../utilities/packed.js";
+//#region lib/ops/concat16.ts
+function r(r, i) {
+	return n(r[0]) ? t().runKernel("Concat16", r, { axis: i ?? -1 }) : e(r, i);
+}
+//#endregion
+export { r as concat16 };

package/dist/ops/cpu/adamAdjust.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/ops/cpu/adamAdjust.js

@@ -0,0 +1,16 @@
+import { Gr as e, Ii as t, Wr as n, gn as r, qr as i } from "../../dist-BewPQWjc.js";
+//#region lib/ops/cpu/adamAdjust.ts
+function a(t) {
+	let { moments: a, value: o } = t.inputs, { beta1: s, beta2: c, epsilon: l, learningRate: u } = t.attrs, d = a.shape.length, f = Array(d).fill(0), p = a.shape.slice();
+	p[d - 1] = 1;
+	let m = f.slice();
+	m[d - 1] = 1;
+	let h = p.slice(), g = a.slice(f, p).squeeze([d - 1]), _ = a.slice(m, h).squeeze([d - 1]);
+	return i(n(e(e(g, s), i(r(e(_, c)), l ?? 1e-8)), -u), o);
+}
+t({
+	kernelName: "AdamAdjust",
+	backendName: "cpu",
+	kernelFunc: a
+});
+//#endregion

package/dist/ops/cpu/adamMoments.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/ops/cpu/adamMoments.js

@@ -0,0 +1,16 @@
+import { Ii as e, V as t } from "../../dist-BewPQWjc.js";
+//#region lib/ops/cpu/adamMoments.ts
+function n(e) {
+	let { moments: n, gradient: r } = e.inputs, { beta1: i, beta2: a } = e.attrs, o = n.shape.length, s = Array(o).fill(0), c = n.shape.slice();
+	c[o - 1] = 1;
+	let l = s.slice();
+	l[o - 1] = 1;
+	let u = c.slice(), d = n.slice(s, c).squeeze([o - 1]), f = n.slice(l, u).squeeze([o - 1]);
+	return t([d.mul(i).add(r.mul(1 - i)), f.mul(a).add(r.square().mul(1 - a))], -1);
+}
+e({
+	kernelName: "AdamMoments",
+	backendName: "cpu",
+	kernelFunc: n
+});
+//#endregion

package/dist/ops/cpu/appendCache.d.ts

@@ -0,0 +1 @@
+export {};