@genai-fi/nanogpt 0.19.0 → 0.20.0

package/dist/kernel_funcs_utils-pq0CK9co.js

@@ -1,306 +0,0 @@
-import { _ as B, U as G, aU as K, a7 as W, aH as z, aV as V, ab as N, aI as F, am as S } from "./index-CUXkjxiT.js";
-import { u as O, f as H } from "./gpgpu_math-BwvV12df.js";
-import { f as v } from "./backend_util-GaFarB78.js";
-function Y(t, e) {
-  return ["x", "y", "z", "w", "u", "v"].slice(0, e).map((s) => `${t}.${s}`);
-}
-function Z(t, e) {
-  return e === 1 ? [t] : Y(t, e);
-}
-function pe(t, e) {
-  if (t === 1)
-    return "rc";
-  let s = "";
-  for (let r = 0; r < t; r++)
-    s += e[r], r < t - 1 && (s += ",");
-  return s;
-}
-class q {
-  constructor(e, s) {
-    this.variableNames = ["A"], this.outputShape = e, this.enableShapeUniforms = O(this.outputShape.length), this.userCode = `
-      float unaryOperation(float x) {
-        ${s}
-      }
-
-      void main() {
-        float x = getAAtOutCoords();
-        float y = unaryOperation(x);
-
-        setOutput(y);
-      }
-    `;
-  }
-}
-const T = "if (isnan(x)) return x;", M = "return x;", de = "return abs(x);", j = "return (x >= 0.0) ? x : (exp(x) - 1.0);", J = T + `
-  return (x < 0.0) ? 0.0 : x;
-`, Q = T + `
-  return (x < 0.0) ? 0.0 : min(6.0, x);
-`, he = "return x;", X = "return 1.0 / (1.0 + exp(-1.0 * x));";
-const ee = "return x;", te = `
-  vec4 result;
-
-  result.r = (x.r >= 0.0) ? x.r : (exp(x.r) - 1.0);
-  result.g = (x.g >= 0.0) ? x.g : (exp(x.g) - 1.0);
-  result.b = (x.b >= 0.0) ? x.b : (exp(x.b) - 1.0);
-  result.a = (x.a >= 0.0) ? x.a : (exp(x.a) - 1.0);
-
-  return result;
-`, se = `
-  vec4 result = x * vec4(greaterThanEqual(x, vec4(0.0)));
-  bvec4 isNaN = isnan(x);
-
-  result.r = isNaN.r ? x.r : result.r;
-  result.g = isNaN.g ? x.g : result.g;
-  result.b = isNaN.b ? x.b : result.b;
-  result.a = isNaN.a ? x.a : result.a;
-
-  return result;
-`, ae = `
-  vec4 result = min(x, vec4(6.)) * vec4(greaterThanEqual(x, vec4(0.0)));
-  bvec4 isNaN = isnan(x);
-
-  result.r = isNaN.r ? x.r : result.r;
-  result.g = isNaN.g ? x.g : result.g;
-  result.b = isNaN.b ? x.b : result.b;
-  result.a = isNaN.a ? x.a : result.a;
-
-  return result;
-`, re = "return 1.0 / (1.0 + exp(-1.0 * x));";
-class ne {
-  constructor(e, s) {
-    this.variableNames = ["A"], this.packedInputs = !0, this.packedOutput = !0, this.outputShape = e, this.enableShapeUniforms = O(this.outputShape.length), this.userCode = `
-      vec4 unaryOperation(vec4 x) {
-        ${s}
-      }
-
-      void main() {
-        vec4 x = getAAtOutCoords();
-        vec4 y = unaryOperation(x);
-
-        setOutput(y);
-      }
-    `;
-  }
-}
-const fe = `
-  if (isnan(a)) return a;
-  if (isnan(b)) return b;
-`;
-class b {
-  constructor(e, s, r) {
-    this.variableNames = ["A", "B"], this.outputShape = B(s, r), this.enableShapeUniforms = O(this.outputShape.length), this.userCode = `
-      float binaryOperation(float a, float b) {
-        ${e}
-      }
-
-      void main() {
-        float a = getAAtOutCoords();
-        float b = getBAtOutCoords();
-        setOutput(binaryOperation(a, b));
-      }
-    `;
-  }
-}
-const xe = `
-  result.r = isNaN.r ? NAN : result.r;
-  result.g = isNaN.g ? NAN : result.g;
-  result.b = isNaN.b ? NAN : result.b;
-  result.a = isNaN.a ? NAN : result.a;
-`;
-class E {
-  constructor(e, s, r, u = !1) {
-    this.variableNames = ["A", "B"], this.supportsBroadcasting = !0, this.packedInputs = !0, this.packedOutput = !0, this.outputShape = B(s, r);
-    const n = this.outputShape.length;
-    this.enableShapeUniforms = O(n);
-    let o = "";
-    if (u)
-      if (n === 0 || G(this.outputShape) === 1)
-        o = `
-          result.y = 0.;
-          result.z = 0.;
-          result.w = 0.;
-        `;
-      else if (o = `
-          ${H(n)} coords = getOutputCoords();
-        `, n === 1)
-        this.enableShapeUniforms ? o += `
-            result.y = (coords + 1) >= outShape ? 0. : result.y;
-            result.z = 0.;
-            result.w = 0.;
-          ` : o += `
-            result.y = (coords + 1) >= ${this.outputShape[0]} ? 0. : result.y;
-            result.z = 0.;
-            result.w = 0.;
-          `;
-      else {
-        const a = Z("coords", n);
-        this.enableShapeUniforms ? o += `
-            bool nextRowOutOfBounds =
-              (${a[n - 2]} + 1) >= outShape[${n} - 2];
-            bool nextColOutOfBounds =
-              (${a[n - 1]} + 1) >= outShape[${n} - 1];
-            result.y = nextColOutOfBounds ? 0. : result.y;
-            result.z = nextRowOutOfBounds ? 0. : result.z;
-            result.w = nextColOutOfBounds || nextRowOutOfBounds ? 0. : result.w;
-          ` : o += `
-            bool nextRowOutOfBounds =
-              (${a[n - 2]} + 1) >= ${this.outputShape[n - 2]};
-            bool nextColOutOfBounds =
-              (${a[n - 1]} + 1) >= ${this.outputShape[n - 1]};
-            result.y = nextColOutOfBounds ? 0. : result.y;
-            result.z = nextRowOutOfBounds ? 0. : result.z;
-            result.w = nextColOutOfBounds || nextRowOutOfBounds ? 0. : result.w;
-          `;
-      }
-    this.userCode = `
-      vec4 binaryOperation(vec4 a, vec4 b) {
-        ${e}
-      }
-
-      void main() {
-        vec4 a = getAAtOutCoords();
-        vec4 b = getBAtOutCoords();
-
-        vec4 result = binaryOperation(a, b);
-        ${o}
-
-        setOutput(result);
-      }
-    `;
-  }
-}
-function P(t) {
-  const { inputs: e, backend: s } = t, { x: r } = e;
-  return s.incRef(r.dataId), { dataId: r.dataId, shape: r.shape, dtype: r.dtype };
-}
-const ge = {
-  kernelName: K,
-  backendName: "webgl",
-  kernelFunc: P
-};
-function L(t) {
-  const { inputs: e, backend: s } = t, { real: r, imag: u } = e, n = s.makeTensorInfo(r.shape, "complex64"), o = s.texData.get(n.dataId), i = P({ inputs: { x: r }, backend: s }), a = P({ inputs: { x: u }, backend: s });
-  return o.complexTensorInfos = { real: i, imag: a }, n;
-}
-const me = {
-  kernelName: W,
-  backendName: "webgl",
-  kernelFunc: L
-};
-const w = "return (a < 0.) ? b * a : a;", R = `
-  vec4 aLessThanZero = vec4(lessThan(a, vec4(0.)));
-  return (aLessThanZero * (b * a)) + ((vec4(1.0) - aLessThanZero) * a);
-`;
-function oe(t) {
-  const { inputs: e, backend: s, attrs: r } = t, { x: u } = e, { alpha: n } = r, o = s.makeTensorInfo([], "float32", V(n, "float32")), i = N().getBool("WEBGL_PACK_BINARY_OPERATIONS") ? new E(R, u.shape, o.shape) : new b(w, u.shape, o.shape), a = s.runWebGLProgram(i, [u, o], "float32");
-  return s.disposeIntermediateTensorInfo(o), a;
-}
-const be = {
-  kernelName: z,
-  backendName: "webgl",
-  kernelFunc: oe
-};
-const U = "return (a < 0.) ? b * a : a;", k = `
-  vec4 aLessThanZero = vec4(lessThan(a, vec4(0.)));
-  return (aLessThanZero * (b * a)) + ((vec4(1.0) - aLessThanZero) * a);
-`;
-function ue(t) {
-  const { inputs: e, backend: s } = t, { x: r, alpha: u } = e, n = N().getBool("WEBGL_PACK_BINARY_OPERATIONS") ? new E(k, r.shape, u.shape) : new b(U, r.shape, u.shape);
-  return s.runWebGLProgram(n, [r, u], "float32");
-}
-const Ne = {
-  kernelName: F,
-  backendName: "webgl",
-  kernelFunc: ue
-};
-const Oe = "if (isnan(x)) return x;";
-function ye({ opSnippet: t, packedOpSnippet: e, cpuKernelImpl: s, dtype: r }) {
-  return ({ inputs: u, backend: n }) => {
-    const { x: o } = u, i = n, a = r || o.dtype;
-    if (i.shouldExecuteOnCPU([o]) && s != null) {
-      const d = i.texData.get(o.dataId), y = s(d.values, a);
-      return i.makeTensorInfo(o.shape, a, y);
-    }
-    const c = N().getBool("WEBGL_PACK_UNARY_OPERATIONS") && e != null;
-    let l;
-    return c ? l = new ne(o.shape, e) : l = new q(o.shape, t), i.runWebGLProgram(l, [o], a);
-  };
-}
-function Ie({ opSnippet: t, packedOpSnippet: e, checkOutOfBounds: s = !1, supportsComplex: r = !1, cpuKernelImpl: u, dtype: n }) {
-  return ({ inputs: o, backend: i }) => {
-    const { a, b: c } = o, l = i;
-    if (r && a.dtype === "complex64") {
-      const h = l.texData.get(a.dataId), f = l.texData.get(c.dataId), [g, m] = [
-        [h.complexTensorInfos.real, f.complexTensorInfos.real],
-        [h.complexTensorInfos.imag, f.complexTensorInfos.imag]
-      ].map((C) => {
-        const [p, x] = C, $ = {
-          dataId: p.dataId,
-          dtype: p.dtype,
-          shape: a.shape
-        }, _ = {
-          dataId: x.dataId,
-          dtype: x.dtype,
-          shape: c.shape
-        }, D = new b(t, a.shape, c.shape);
-        return l.runWebGLProgram(D, [$, _], S(p.dtype, x.dtype));
-      }), A = L({ inputs: { real: g, imag: m }, backend: l });
-      return l.disposeIntermediateTensorInfo(g), l.disposeIntermediateTensorInfo(m), A;
-    }
-    const d = n || S(a.dtype, c.dtype);
-    if ((a.dtype === "string" || c.dtype === "string" || l.shouldExecuteOnCPU([a, c])) && u != null) {
-      const h = l.texData.get(a.dataId).values, f = l.texData.get(c.dataId).values, g = a.dtype === "string" ? (
-        // tslint:disable-next-line: no-any
-        v(h)
-      ) : h, m = a.dtype === "string" ? (
-        // tslint:disable-next-line: no-any
-        v(f)
-      ) : f, [A, C] = u(a.shape, c.shape, g, m, d), p = l.makeTensorInfo(C, d), x = l.texData.get(p.dataId);
-      return x.values = A, p;
-    }
-    const y = N().getBool("WEBGL_PACK_BINARY_OPERATIONS") && e != null;
-    let I;
-    return y ? I = new E(e, a.shape, c.shape, s) : I = new b(t, a.shape, c.shape), l.runWebGLProgram(I, [a, c], d);
-  };
-}
-function Ae(t, e = !1) {
-  if (t === "linear")
-    return e ? ee : M;
-  if (t === "relu")
-    return e ? se : J;
-  if (t === "elu")
-    return e ? te : j;
-  if (t === "relu6")
-    return e ? ae : Q;
-  if (t === "prelu")
-    return e ? k : U;
-  if (t === "leakyrelu")
-    return e ? R : w;
-  if (t === "sigmoid")
-    return e ? re : X;
-  throw new Error(`Activation ${t} has not been implemented for the WebGL backend.`);
-}
-export {
-  de as A,
-  E as B,
-  T as C,
-  ne as U,
-  Z as a,
-  Ie as b,
-  pe as c,
-  he as d,
-  q as e,
-  L as f,
-  Y as g,
-  b as h,
-  P as i,
-  fe as j,
-  xe as k,
-  Oe as l,
-  Ae as m,
-  me as n,
-  ge as o,
-  be as p,
-  Ne as q,
-  ye as u
-};

package/dist/layers/BaseLayer.d.ts

@@ -1,44 +0,0 @@
-import { GPTConfig } from '../models/config';
-import { default as MemoryProfiler } from '../utilities/profile';
-import { default as RoPECache } from './RoPECache';
-import { Tensor, Variable } from '@tensorflow/tfjs-core';
-import { default as WeightStore } from './WeightStore';
-export interface ForwardAttributes {
-    training: boolean;
-    checkpointing?: boolean;
-    mixedPrecision?: boolean;
-    ropeCache?: RoPECache;
-    outputEmbeddings?: boolean;
-    embeddings?: {
-        name: string;
-        tensor: Tensor;
-    }[];
-    dropout?: number;
-    layerDrop?: number;
-}
-export default abstract class BaseLayer<ATTR extends ForwardAttributes = ForwardAttributes, CONFIG extends GPTConfig = GPTConfig> {
-    readonly parent?: BaseLayer;
-    readonly config: CONFIG;
-    weightStore: WeightStore;
-    readonly children: BaseLayer[];
-    private profiler?;
-    private ownVariables;
-    constructor(config: CONFIG, parent?: BaseLayer);
-    getProfiler(): MemoryProfiler | undefined;
-    setProfiler(profiler: MemoryProfiler | null): void;
-    startMemory(): void;
-    endMemory(label: string): void;
-    addVariable(name: string, variable?: Variable): void;
-    addChildVariable(name: string): void;
-    get variables(): Variable[];
-    get trainableVariables(): Variable[];
-    getVariable(name: string): Tensor;
-    hasVariable(name: string): boolean;
-    setVariable(name: string, variable: Variable): void;
-    dispose(): void;
-    protected build(): void;
-    abstract forward(attrs: ATTR, ...x: Tensor[]): Tensor | Tensor[];
-    call(attrs: ATTR, ...x: Tensor[]): Tensor | Tensor[];
-    callCheckpoint(attrs: ATTR, ...x: Tensor[]): Tensor;
-    private checkpointingFn;
-}

package/dist/layers/BaseLayer.js

@@ -1,74 +0,0 @@
-import { a2 as p, h as s, a4 as g } from "../index-CUXkjxiT.js";
-import b from "./WeightStore.js";
-class T {
-  parent;
-  config;
-  weightStore;
-  children = [];
-  profiler;
-  ownVariables = /* @__PURE__ */ new Set();
-  constructor(t, e) {
-    this.config = t, this.parent = e, this.parent ? (this.parent.children.push(this), this.weightStore = this.parent.weightStore) : this.weightStore = new b();
-  }
-  getProfiler() {
-    return this.profiler;
-  }
-  setProfiler(t) {
-    this.profiler = t || void 0, this.children.forEach((e) => {
-      e.setProfiler(t);
-    });
-  }
-  startMemory() {
-    this.profiler?.startMemory();
-  }
-  endMemory(t) {
-    this.profiler?.endMemory(t);
-  }
-  addVariable(t, e) {
-    this.weightStore.addVariable(t, e), this.ownVariables.add(t), this.parent && this.parent.addChildVariable(t);
-  }
-  addChildVariable(t) {
-    this.ownVariables.add(t);
-  }
-  get variables() {
-    return this.weightStore.variables;
-  }
-  get trainableVariables() {
-    return this.weightStore.trainableVariables.filter((t) => this.ownVariables.has(t.name));
-  }
-  getVariable(t) {
-    return this.weightStore.getVariable(t);
-  }
-  hasVariable(t) {
-    return this.weightStore.hasVariable(t);
-  }
-  setVariable(t, e) {
-    this.weightStore.setVariable(t, e);
-  }
-  dispose() {
-    this.weightStore.dispose();
-  }
-  build() {
-  }
-  call(t, ...e) {
-    return this.build(), this.forward(t, ...e);
-  }
-  callCheckpoint(t, ...e) {
-    return this.build(), this.checkpointingFn(t, ...e);
-  }
-  checkpointingFn(t, ...e) {
-    const r = this.trainableVariables;
-    return p((...i) => {
-      const o = i[i.length - 1], a = i.slice(0, e.length), h = this.forward(t, ...a);
-      return o(a), { value: h, gradFunc: (n, l) => {
-        const c = s().state.activeTape;
-        s().state.activeTape = [];
-        const d = g((...u) => this.forward(t, ...u.slice(0, a.length)))([...l, ...r], n);
-        return s().state.activeTape = c, d;
-      } };
-    })(...e, ...r);
-  }
-}
-export {
-  T as default
-};

package/dist/layers/CausalSelfAttention.d.ts

@@ -1,39 +0,0 @@
-import { default as BaseLayer, ForwardAttributes } from './BaseLayer';
-import { Tensor } from '@tensorflow/tfjs-core';
-import { GPTConfig } from '../models/config';
-export interface KVCache {
-    k?: Tensor;
-    v?: Tensor;
-    length: number;
-    cumulativeLength: number;
-}
-export interface AttentionScores {
-    meanOfHeads?: boolean;
-    attentionOut?: Tensor[];
-}
-interface AttentionForwardAttributes extends ForwardAttributes {
-    attentionScores?: AttentionScores;
-    pastKV?: KVCache;
-    seed?: number;
-    ropePositionOffset?: number;
-}
-export interface CausalSelfAttentionConfig {
-    useQKNorm?: boolean;
-}
-export default class CausalSelfAttention extends BaseLayer<AttentionForwardAttributes> {
-    private readonly attentionConfig;
-    private divisor;
-    private index;
-    private units;
-    private projUnits;
-    private ATTN;
-    private PROJ;
-    constructor(index: number, config: GPTConfig, attentionConfig: CausalSelfAttentionConfig, parent?: BaseLayer);
-    protected build(): void;
-    private getAttentionScores;
-    private getQKV;
-    private getOutputProjection;
-    private updateCache;
-    forward(attr: AttentionForwardAttributes, x: Tensor): Tensor;
-}
-export {};

package/dist/layers/CausalSelfAttention.js

@@ -1,86 +0,0 @@
-import { attentionMask as R } from "../ops/attentionMask.js";
-import J from "./BaseLayer.js";
-import { r as v } from "../rope-DcrZM_e6.js";
-import { appendCache as A } from "../ops/appendCache.js";
-import { k as c, t as L } from "../index-CUXkjxiT.js";
-import { softmax16 as y } from "../ops/softmax16.js";
-import { b as M } from "../matMul16-BcVC_E62.js";
-import { p as K } from "../pack16-bqltoUlR.js";
-import { transpose16 as j } from "../ops/transpose16.js";
-import { dot16 as E } from "../ops/dot16.js";
-import { reshape16 as _ } from "../ops/reshape16.js";
-import { isPackedTensor as f } from "../utilities/packed.js";
-import { qkv as q } from "../ops/qkv.js";
-import { normRMS as O } from "../ops/normRMS.js";
-import { dropout16 as x } from "../ops/dropout16.js";
-import { v as P } from "../variable-Ck482e3n.js";
-import { r as S } from "../random_normal-IBRrha8a.js";
-class it extends J {
-  constructor(t, o, s, i) {
-    super(o, i), this.attentionConfig = s, this.index = t, this.units = o.nEmbed * 3, this.projUnits = o.nEmbed, this.ATTN = `block_${this.index}_cAttn`, this.PROJ = `block_${this.index}_cProj`, this.addVariable(this.ATTN), this.addVariable(this.PROJ), this.divisor = 1 / Math.sqrt(o.nEmbed / o.nHead);
-  }
-  divisor;
-  index;
-  units;
-  projUnits;
-  ATTN;
-  PROJ;
-  build() {
-    this.hasVariable(this.ATTN) === !1 && this.setVariable(
-      this.ATTN,
-      P(S([this.config.nEmbed, this.units], 0, 0.02), !0, this.ATTN)
-    ), this.hasVariable(this.PROJ) === !1 && this.setVariable(
-      this.PROJ,
-      P(S([this.projUnits, this.config.nEmbed], 0, 0.02), !0, this.PROJ)
-    );
-  }
-  getAttentionScores(t, o, s) {
-    const i = R(t, o, this.divisor, s), e = y(i);
-    return i.dispose(), e;
-  }
-  getQKV(t) {
-    const o = f(t) ? K(this.getVariable(this.ATTN)) : this.getVariable(this.ATTN), s = q(t, o, this.config.nHead);
-    return f(t) && o.dispose(), s;
-  }
-  getOutputProjection(t) {
-    const o = t.shape[0], s = t.shape[2], i = this.config.nEmbed, e = f(t), r = j(t, [0, 2, 1, 3]), n = _(r, [o, s, e ? i / 2 : i]);
-    r.dispose();
-    const p = e ? K(this.getVariable(this.PROJ)) : this.getVariable(this.PROJ), a = E(n, p);
-    return e && p.dispose(), n.dispose(), a;
-  }
-  updateCache(t, o, s) {
-    const i = this.config.blockSize, e = t.shape[2], r = s.length || 0, n = A(t, i, r, s.k);
-    t.dispose(), s.k && s.k.dispose();
-    const p = A(o, i, r, s.v);
-    o.dispose(), s.v && s.v.dispose();
-    const a = Math.min(r + e, i), h = s.cumulativeLength + e;
-    s.length = a, s.cumulativeLength = h, s.k = c(n), s.v = c(p);
-  }
-  forward(t, o) {
-    return L(() => {
-      this.startMemory();
-      const [s, i, e] = this.getQKV(o), r = t.pastKV ? t.pastKV.cumulativeLength : t.ropePositionOffset || 0, n = t.ropeCache, p = n ? v(s, n, r) : s, a = n ? v(i, n, r) : i, h = this.attentionConfig.useQKNorm ?? !1, m = h ? O(p) : p;
-      h && p.dispose();
-      const l = h ? O(a) : a;
-      h && a.dispose(), n && (s.dispose(), i.dispose());
-      const T = t.pastKV ? t.pastKV.length : 0;
-      t.pastKV && !t.training && this.updateCache(l, e, t.pastKV);
-      const u = t.pastKV?.k ? t.pastKV.k : l, V = t.pastKV?.v ? t.pastKV.v : e;
-      let d;
-      T > 0 ? d = this.getAttentionScores(m, u, T) : d = this.getAttentionScores(m, u), m.dispose(), t.pastKV || u.dispose();
-      const g = M(d, V), b = t.attentionScores !== void 0 && t.attentionScores.attentionOut !== void 0;
-      b || d.dispose(), t.pastKV || V.dispose();
-      const k = this.getOutputProjection(g);
-      if (g.dispose(), b && t.attentionScores && t.attentionScores.attentionOut !== void 0) {
-        const N = d.shape[1], C = d.shape[2];
-        t.attentionScores.attentionOut?.push(
-          c(d.slice([0, 0, 0, 0], [1, -1, -1, -1]).reshape([N, C, -1]))
-        );
-      }
-      return this.endMemory("CausalSelfAttention"), t.dropout && t.dropout > 0 ? x(k, t.dropout) : k;
-    });
-  }
-}
-export {
-  it as default
-};

package/dist/layers/LoRA.d.ts

@@ -1,14 +0,0 @@
-import { default as WeightStore } from './WeightStore';
-export default class LoRA {
-    private weightStore;
-    readonly alpha: number;
-    readonly rank: number;
-    readonly variables: Set<string>;
-    private scale;
-    readonly name: string;
-    constructor(name: string, weightStore: WeightStore, alpha: number, rank: number, variables: string[]);
-    attach(): void;
-    merge(): void;
-    detach(): void;
-    dispose(): void;
-}

package/dist/layers/LoRA.js

@@ -1,58 +0,0 @@
-import { a as m, t as n } from "../index-CUXkjxiT.js";
-import { p } from "../index-DmeWGGmS.js";
-import { v as g } from "../variable-Ck482e3n.js";
-import { r as S } from "../random_normal-IBRrha8a.js";
-import { z as _ } from "../zeros-DvZpK8s6.js";
-class B {
-  weightStore;
-  alpha;
-  rank;
-  variables;
-  scale;
-  name;
-  constructor(t, e, a, s, r) {
-    this.name = t, this.weightStore = e, this.alpha = a, this.rank = s;
-    const c = p(r), w = e.variableNames.filter(
-      (i) => c(i) && !i.endsWith("_loraA") && !i.endsWith("_loraB")
-    );
-    this.variables = new Set(w), this.scale = m(a / s), this.variables.forEach((i) => {
-      const o = this.weightStore.getRawVariable(i), [d, b] = o.shape, h = `${i}_${this.name}_loraA`, l = `${i}_${this.name}_loraB`;
-      if (o.shape.length !== 2) {
-        console.warn(
-          `LoRA currently only supports 2D weight matrices. Variable ${i} has shape ${o.shape}`
-        ), this.variables.delete(i);
-        return;
-      }
-      this.weightStore.hasVariable(h) || this.weightStore.hasVariable(l) || (this.weightStore.addVariable(
-        h,
-        g(S([d, this.rank], 0, 0.02), !0, h)
-      ), this.weightStore.addVariable(l, g(_([this.rank, b]), !0, l)));
-    });
-  }
-  attach() {
-    if (this.weightStore.onWeightRead)
-      throw new Error("LoRA cannot be applied to a WeightStore that already has a onWeightRead hook.");
-    this.weightStore.onWeightRead = (t, e) => this.variables.has(t) ? n(() => {
-      const a = this.weightStore.getRawVariable(`${t}_${this.name}_loraA`), s = this.weightStore.getRawVariable(`${t}_${this.name}_loraB`);
-      return e.add(a.matMul(s).mul(this.scale));
-    }) : e, this.weightStore.setTrainable([`*_${this.name}_loraA`, `*_${this.name}_loraB`]);
-  }
-  merge() {
-    this.variables.forEach((t) => {
-      const e = this.weightStore.getRawVariable(t), a = this.weightStore.getRawVariable(`${t}_${this.name}_loraA`), s = this.weightStore.getRawVariable(`${t}_${this.name}_loraB`), r = n(() => e.add(a.matMul(s).mul(this.scale)));
-      e.assign(r), r.dispose();
-    });
-  }
-  detach() {
-    this.weightStore.onWeightRead = void 0, this.weightStore.setTrainable(["*"]);
-  }
-  dispose() {
-    this.detach(), this.scale.dispose(), this.variables.forEach((t) => {
-      const e = `${t}_${this.name}_loraA`, a = `${t}_${this.name}_loraB`;
-      this.weightStore.getRawVariable(e).dispose(), this.weightStore.getRawVariable(a).dispose(), this.weightStore.deleteVariable(e), this.weightStore.deleteVariable(a);
-    }), this.variables.clear();
-  }
-}
-export {
-  B as default
-};

package/dist/layers/MLP.d.ts

@@ -1,17 +0,0 @@
-import { Tensor } from '@tensorflow/tfjs-core';
-import { default as BaseLayer, ForwardAttributes } from './BaseLayer';
-import { GPTConfig } from '../main';
-export interface MLPConfig {
-    activation?: 'gelu' | 'relu2';
-    hiddenFactor?: number;
-}
-export default class MLP extends BaseLayer {
-    private index;
-    private hiddenUnits;
-    private MLPHIDDEN;
-    private MLPOUT;
-    private mlpConfig;
-    constructor(index: number, config: GPTConfig, mlpConfig: MLPConfig, parent?: BaseLayer);
-    protected build(): void;
-    forward(attr: ForwardAttributes, x: Tensor): Tensor;
-}

package/dist/layers/MLP.js

@@ -1,44 +0,0 @@
-import { t as M } from "../index-CUXkjxiT.js";
-import f from "./BaseLayer.js";
-import { b as h } from "../matMul16-BcVC_E62.js";
-import { reshape16 as d } from "../ops/reshape16.js";
-import { dropout16 as L } from "../ops/dropout16.js";
-import { v as n } from "../variable-Ck482e3n.js";
-import { r as m } from "../random_normal-IBRrha8a.js";
-class N extends f {
-  index;
-  hiddenUnits;
-  MLPHIDDEN;
-  MLPOUT;
-  mlpConfig;
-  constructor(i, t, s, e) {
-    super(t, e), this.index = i, this.mlpConfig = s, this.hiddenUnits = (s.hiddenFactor ?? t.mlpFactor) * t.nEmbed, this.MLPHIDDEN = `block_${this.index}_mlpHidden`, this.MLPOUT = `block_${this.index}_mlpOut`, this.addVariable(this.MLPHIDDEN), this.addVariable(this.MLPOUT);
-  }
-  build() {
-    this.hasVariable(this.MLPHIDDEN) === !1 && this.setVariable(
-      this.MLPHIDDEN,
-      n(m([this.config.nEmbed, this.hiddenUnits], 0, 0.02), !0, this.MLPHIDDEN)
-    ), this.hasVariable(this.MLPOUT) === !1 && this.setVariable(
-      this.MLPOUT,
-      n(
-        m([this.hiddenUnits, this.config.nEmbed], 0, 0.02 / Math.sqrt(2 * this.config.nLayer)),
-        !0,
-        this.MLPOUT
-      )
-    );
-  }
-  forward(i, t) {
-    return M(() => {
-      this.startMemory();
-      const [s, e, r] = t.shape, l = d(t, [s * e, r]), a = h(l, this.getVariable(this.MLPHIDDEN), !1, !1, {
-        activation: this.mlpConfig.activation ?? "gelu"
-      }), p = h(a, this.getVariable(this.MLPOUT));
-      a.dispose();
-      const o = d(p, [s, e, r]);
-      return this.endMemory("MLP"), i.dropout && i.dropout > 0 ? L(o, i.dropout) : o;
-    });
-  }
-}
-export {
-  N as default
-};