@genai-fi/nanogpt 0.9.0 → 0.10.0

package/dist/kernel_funcs_utils-Dg_-E44D.js

@@ -0,0 +1,308 @@
+import { j as B } from "./index-ZyQhjEPo.js";
+import { u as N, f as G } from "./gpgpu_math-DDVJCn6-.js";
+import { s as K, I as W, e as O } from "./tensor-DdQUJZlz.js";
+import { I as z, h as V, W as F, X as Y, u as S } from "./tensor_util-DV-FP5Q3.js";
+import { f as v } from "./backend_util-D-rUb2ty.js";
+const he = `
+  if (isnan(a)) return a;
+  if (isnan(b)) return b;
+`;
+class b {
+  constructor(e, s, r) {
+    this.variableNames = ["A", "B"], this.outputShape = B(s, r), this.enableShapeUniforms = N(this.outputShape.length), this.userCode = `
+      float binaryOperation(float a, float b) {
+        ${e}
+      }
+
+      void main() {
+        float a = getAAtOutCoords();
+        float b = getBAtOutCoords();
+        setOutput(binaryOperation(a, b));
+      }
+    `;
+  }
+}
+function H(t, e) {
+  return ["x", "y", "z", "w", "u", "v"].slice(0, e).map((s) => `${t}.${s}`);
+}
+function Z(t, e) {
+  return e === 1 ? [t] : H(t, e);
+}
+function fe(t, e) {
+  if (t === 1)
+    return "rc";
+  let s = "";
+  for (let r = 0; r < t; r++)
+    s += e[r], r < t - 1 && (s += ",");
+  return s;
+}
+const xe = `
+  result.r = isNaN.r ? NAN : result.r;
+  result.g = isNaN.g ? NAN : result.g;
+  result.b = isNaN.b ? NAN : result.b;
+  result.a = isNaN.a ? NAN : result.a;
+`;
+class E {
+  constructor(e, s, r, u = !1) {
+    this.variableNames = ["A", "B"], this.supportsBroadcasting = !0, this.packedInputs = !0, this.packedOutput = !0, this.outputShape = B(s, r);
+    const n = this.outputShape.length;
+    this.enableShapeUniforms = N(n);
+    let o = "";
+    if (u)
+      if (n === 0 || K(this.outputShape) === 1)
+        o = `
+          result.y = 0.;
+          result.z = 0.;
+          result.w = 0.;
+        `;
+      else if (o = `
+          ${G(n)} coords = getOutputCoords();
+        `, n === 1)
+        this.enableShapeUniforms ? o += `
+            result.y = (coords + 1) >= outShape ? 0. : result.y;
+            result.z = 0.;
+            result.w = 0.;
+          ` : o += `
+            result.y = (coords + 1) >= ${this.outputShape[0]} ? 0. : result.y;
+            result.z = 0.;
+            result.w = 0.;
+          `;
+      else {
+        const a = Z("coords", n);
+        this.enableShapeUniforms ? o += `
+            bool nextRowOutOfBounds =
+              (${a[n - 2]} + 1) >= outShape[${n} - 2];
+            bool nextColOutOfBounds =
+              (${a[n - 1]} + 1) >= outShape[${n} - 1];
+            result.y = nextColOutOfBounds ? 0. : result.y;
+            result.z = nextRowOutOfBounds ? 0. : result.z;
+            result.w = nextColOutOfBounds || nextRowOutOfBounds ? 0. : result.w;
+          ` : o += `
+            bool nextRowOutOfBounds =
+              (${a[n - 2]} + 1) >= ${this.outputShape[n - 2]};
+            bool nextColOutOfBounds =
+              (${a[n - 1]} + 1) >= ${this.outputShape[n - 1]};
+            result.y = nextColOutOfBounds ? 0. : result.y;
+            result.z = nextRowOutOfBounds ? 0. : result.z;
+            result.w = nextColOutOfBounds || nextRowOutOfBounds ? 0. : result.w;
+          `;
+      }
+    this.userCode = `
+      vec4 binaryOperation(vec4 a, vec4 b) {
+        ${e}
+      }
+
+      void main() {
+        vec4 a = getAAtOutCoords();
+        vec4 b = getBAtOutCoords();
+
+        vec4 result = binaryOperation(a, b);
+        ${o}
+
+        setOutput(result);
+      }
+    `;
+  }
+}
+function P(t) {
+  const { inputs: e, backend: s } = t, { x: r } = e;
+  return s.incRef(r.dataId), { dataId: r.dataId, shape: r.shape, dtype: r.dtype };
+}
+const ge = {
+  kernelName: z,
+  backendName: "webgl",
+  kernelFunc: P
+};
+function T(t) {
+  const { inputs: e, backend: s } = t, { real: r, imag: u } = e, n = s.makeTensorInfo(r.shape, "complex64"), o = s.texData.get(n.dataId), i = P({ inputs: { x: r }, backend: s }), a = P({ inputs: { x: u }, backend: s });
+  return o.complexTensorInfos = { real: i, imag: a }, n;
+}
+const me = {
+  kernelName: V,
+  backendName: "webgl",
+  kernelFunc: T
+};
+const L = "return (a < 0.) ? b * a : a;", w = `
+  vec4 aLessThanZero = vec4(lessThan(a, vec4(0.)));
+  return (aLessThanZero * (b * a)) + ((vec4(1.0) - aLessThanZero) * a);
+`;
+function q(t) {
+  const { inputs: e, backend: s, attrs: r } = t, { x: u } = e, { alpha: n } = r, o = s.makeTensorInfo([], "float32", W(n, "float32")), i = O().getBool("WEBGL_PACK_BINARY_OPERATIONS") ? new E(w, u.shape, o.shape) : new b(L, u.shape, o.shape), a = s.runWebGLProgram(i, [u, o], "float32");
+  return s.disposeIntermediateTensorInfo(o), a;
+}
+const be = {
+  kernelName: F,
+  backendName: "webgl",
+  kernelFunc: q
+};
+const R = "return (a < 0.) ? b * a : a;", k = `
+  vec4 aLessThanZero = vec4(lessThan(a, vec4(0.)));
+  return (aLessThanZero * (b * a)) + ((vec4(1.0) - aLessThanZero) * a);
+`;
+function j(t) {
+  const { inputs: e, backend: s } = t, { x: r, alpha: u } = e, n = O().getBool("WEBGL_PACK_BINARY_OPERATIONS") ? new E(k, r.shape, u.shape) : new b(R, r.shape, u.shape);
+  return s.runWebGLProgram(n, [r, u], "float32");
+}
+const Ne = {
+  kernelName: Y,
+  backendName: "webgl",
+  kernelFunc: j
+};
+class M {
+  constructor(e, s) {
+    this.variableNames = ["A"], this.outputShape = e, this.enableShapeUniforms = N(this.outputShape.length), this.userCode = `
+      float unaryOperation(float x) {
+        ${s}
+      }
+
+      void main() {
+        float x = getAAtOutCoords();
+        float y = unaryOperation(x);
+
+        setOutput(y);
+      }
+    `;
+  }
+}
+const U = "if (isnan(x)) return x;", X = "return x;", Oe = "return abs(x);", J = "return (x >= 0.0) ? x : (exp(x) - 1.0);", Q = U + `
+  return (x < 0.0) ? 0.0 : x;
+`, ee = U + `
+  return (x < 0.0) ? 0.0 : min(6.0, x);
+`, ye = "return x;", te = "return 1.0 / (1.0 + exp(-1.0 * x));";
+const se = "return x;", ae = `
+  vec4 result;
+
+  result.r = (x.r >= 0.0) ? x.r : (exp(x.r) - 1.0);
+  result.g = (x.g >= 0.0) ? x.g : (exp(x.g) - 1.0);
+  result.b = (x.b >= 0.0) ? x.b : (exp(x.b) - 1.0);
+  result.a = (x.a >= 0.0) ? x.a : (exp(x.a) - 1.0);
+
+  return result;
+`, re = `
+  vec4 result = x * vec4(greaterThanEqual(x, vec4(0.0)));
+  bvec4 isNaN = isnan(x);
+
+  result.r = isNaN.r ? x.r : result.r;
+  result.g = isNaN.g ? x.g : result.g;
+  result.b = isNaN.b ? x.b : result.b;
+  result.a = isNaN.a ? x.a : result.a;
+
+  return result;
+`, ne = `
+  vec4 result = min(x, vec4(6.)) * vec4(greaterThanEqual(x, vec4(0.0)));
+  bvec4 isNaN = isnan(x);
+
+  result.r = isNaN.r ? x.r : result.r;
+  result.g = isNaN.g ? x.g : result.g;
+  result.b = isNaN.b ? x.b : result.b;
+  result.a = isNaN.a ? x.a : result.a;
+
+  return result;
+`, oe = "return 1.0 / (1.0 + exp(-1.0 * x));";
+class ue {
+  constructor(e, s) {
+    this.variableNames = ["A"], this.packedInputs = !0, this.packedOutput = !0, this.outputShape = e, this.enableShapeUniforms = N(this.outputShape.length), this.userCode = `
+      vec4 unaryOperation(vec4 x) {
+        ${s}
+      }
+
+      void main() {
+        vec4 x = getAAtOutCoords();
+        vec4 y = unaryOperation(x);
+
+        setOutput(y);
+      }
+    `;
+  }
+}
+const Ie = "if (isnan(x)) return x;";
+function Ae({ opSnippet: t, packedOpSnippet: e, cpuKernelImpl: s, dtype: r }) {
+  return ({ inputs: u, backend: n }) => {
+    const { x: o } = u, i = n, a = r || o.dtype;
+    if (i.shouldExecuteOnCPU([o]) && s != null) {
+      const d = i.texData.get(o.dataId), y = s(d.values, a);
+      return i.makeTensorInfo(o.shape, a, y);
+    }
+    const c = O().getBool("WEBGL_PACK_UNARY_OPERATIONS") && e != null;
+    let l;
+    return c ? l = new ue(o.shape, e) : l = new M(o.shape, t), i.runWebGLProgram(l, [o], a);
+  };
+}
+function Ce({ opSnippet: t, packedOpSnippet: e, checkOutOfBounds: s = !1, supportsComplex: r = !1, cpuKernelImpl: u, dtype: n }) {
+  return ({ inputs: o, backend: i }) => {
+    const { a, b: c } = o, l = i;
+    if (r && a.dtype === "complex64") {
+      const h = l.texData.get(a.dataId), f = l.texData.get(c.dataId), [g, m] = [
+        [h.complexTensorInfos.real, f.complexTensorInfos.real],
+        [h.complexTensorInfos.imag, f.complexTensorInfos.imag]
+      ].map((C) => {
+        const [p, x] = C, $ = {
+          dataId: p.dataId,
+          dtype: p.dtype,
+          shape: a.shape
+        }, _ = {
+          dataId: x.dataId,
+          dtype: x.dtype,
+          shape: c.shape
+        }, D = new b(t, a.shape, c.shape);
+        return l.runWebGLProgram(D, [$, _], S(p.dtype, x.dtype));
+      }), A = T({ inputs: { real: g, imag: m }, backend: l });
+      return l.disposeIntermediateTensorInfo(g), l.disposeIntermediateTensorInfo(m), A;
+    }
+    const d = n || S(a.dtype, c.dtype);
+    if ((a.dtype === "string" || c.dtype === "string" || l.shouldExecuteOnCPU([a, c])) && u != null) {
+      const h = l.texData.get(a.dataId).values, f = l.texData.get(c.dataId).values, g = a.dtype === "string" ? (
+        // tslint:disable-next-line: no-any
+        v(h)
+      ) : h, m = a.dtype === "string" ? (
+        // tslint:disable-next-line: no-any
+        v(f)
+      ) : f, [A, C] = u(a.shape, c.shape, g, m, d), p = l.makeTensorInfo(C, d), x = l.texData.get(p.dataId);
+      return x.values = A, p;
+    }
+    const y = O().getBool("WEBGL_PACK_BINARY_OPERATIONS") && e != null;
+    let I;
+    return y ? I = new E(e, a.shape, c.shape, s) : I = new b(t, a.shape, c.shape), l.runWebGLProgram(I, [a, c], d);
+  };
+}
+function Pe(t, e = !1) {
+  if (t === "linear")
+    return e ? se : X;
+  if (t === "relu")
+    return e ? re : Q;
+  if (t === "elu")
+    return e ? ae : J;
+  if (t === "relu6")
+    return e ? ne : ee;
+  if (t === "prelu")
+    return e ? k : R;
+  if (t === "leakyrelu")
+    return e ? w : L;
+  if (t === "sigmoid")
+    return e ? oe : te;
+  throw new Error(`Activation ${t} has not been implemented for the WebGL backend.`);
+}
+export {
+  Oe as A,
+  E as B,
+  U as C,
+  ue as U,
+  Z as a,
+  Ce as b,
+  fe as c,
+  ye as d,
+  M as e,
+  T as f,
+  H as g,
+  b as h,
+  P as i,
+  he as j,
+  xe as k,
+  Ie as l,
+  Pe as m,
+  me as n,
+  ge as o,
+  be as p,
+  Ne as q,
+  Ae as u
+};

package/dist/layers/BaseLayer.d.ts

@@ -5,6 +5,7 @@ import { Tensor, Variable } from '@tensorflow/tfjs-core';
 export interface ForwardAttributes {
     training: boolean;
     checkpointing?: boolean;
+    mixedPrecision?: boolean;
     ropeCache?: RoPECache;
     outputEmbeddings?: boolean;
     embeddings?: {

package/dist/layers/BaseLayer.js

@@ -1,5 +1,6 @@
-import { T as p, z as g, e as o, L as v } from "../index-BzFyqcy-.js";
-import { v as _ } from "../variable-Bm2OFwGI.js";
+import { w as p, e as o, F as g } from "../index-ZyQhjEPo.js";
+import { v } from "../variable-Bhn5bHYv.js";
+import { T as _ } from "../tensor-DdQUJZlz.js";
 class T {
   parent;
   config;
@@ -79,7 +80,7 @@ class T {
       const i = t.get(e)?.[0];
       if (!i)
         throw new Error(`Weights for ${e} not found`);
-      r ? r.assign(i) : this._variables.set(e, _(i, this._trainable));
+      r ? r.assign(i) : this._variables.set(e, v(i, this._trainable));
     }), this.children.forEach((r) => {
       r.loadWeights(t);
     });
@@ -97,7 +98,7 @@ class T {
   call(t, ...r) {
     this.build();
     const e = this.forward(t, ...r);
-    if (t.training && e instanceof p) {
+    if (t.training && e instanceof _) {
       const i = this.dropout(e);
       return i !== e && e.dispose(), i;
     } else
@@ -107,12 +108,12 @@ class T {
     return this.build(), this.checkpointingFn(t, ...r);
   }
   checkpointingFn(t, ...r) {
-    const e = this.trainableVariables, s = g((...a) => {
+    const e = this.trainableVariables, s = p((...a) => {
       const l = a[a.length - 1], n = a.slice(0, r.length), h = this.forward(t, ...n);
       return l(n), { value: h, gradFunc: (c, f) => {
         const u = o().state.activeTape;
         o().state.activeTape = [];
-        const b = v((...d) => this.forward(t, ...d.slice(0, n.length)))([...f, ...e], c);
+        const b = g((...d) => this.forward(t, ...d.slice(0, n.length)))([...f, ...e], c);
         return o().state.activeTape = u, b;
       } };
     })(...r, ...e);

package/dist/layers/CausalSelfAttention.d.ts

@@ -26,7 +26,6 @@ export default class CausalSelfAttention extends BaseLayer<AttentionForwardAttri
     constructor(index: number, config: GPTConfig, parent?: BaseLayer);
     protected build(): void;
     private getAttentionScores;
-    private getAttentionScoresWithPast;
     private getQKV;
     private getOutputProjection;
     private updateCache;

package/dist/layers/CausalSelfAttention.js

@@ -1,83 +1,84 @@
-import { attentionMask as g } from "../ops/attentionMask.js";
-import O from "./BaseLayer.js";
-import { qkv as _ } from "../ops/qkv.js";
-import { rope as v } from "../ops/rope.js";
-import { appendCache as V } from "../ops/appendCache.js";
-import { k as c, t as P } from "../index-BzFyqcy-.js";
-import { fusedSoftmax as b } from "../ops/fusedSoftmax.js";
-import { d as C } from "../random_width-CXVRloNK.js";
-import { v as k } from "../variable-Bm2OFwGI.js";
-import { r as T, d as L } from "../dropout-C-csYCLj.js";
-import { r as j } from "../reshape-CnIwVG1c.js";
-import { m as x } from "../mat_mul-DzjTFx-u.js";
-class W extends O {
+import { attentionMask as P } from "../ops/attentionMask.js";
+import _ from "./BaseLayer.js";
+import { r as k } from "../rope-B5UUMsPi.js";
+import { appendCache as b } from "../ops/appendCache.js";
+import { k as u, t as S } from "../index-ZyQhjEPo.js";
+import { softmax16 as C } from "../ops/softmax16.js";
+import { b as R } from "../matMul16--R5hOwDG.js";
+import { p as V } from "../pack16-CFUqumar.js";
+import { transpose16 as J } from "../ops/transpose16.js";
+import { dot16 as L } from "../ops/dot16.js";
+import { reshape16 as N } from "../ops/reshape16.js";
+import { isPackedTensor as l } from "../utilities/packed.js";
+import { qkv as j } from "../ops/qkv.js";
+import { v as T } from "../variable-Bhn5bHYv.js";
+import { r as v, d as y } from "../dropout-C1pM3f11.js";
+class X extends _ {
   divisor;
   index;
   units;
   projUnits;
   ATTN;
   PROJ;
-  constructor(t, i, s) {
-    super(i, s), this.index = t, this.units = i.nEmbed * 3, this.projUnits = i.nEmbed, this.ATTN = `block_${this.index}_cAttn`, this.PROJ = `block_${this.index}_cProj`, this.addVariable(this.ATTN), this.addVariable(this.PROJ), this.divisor = 1 / Math.sqrt(i.nEmbed / i.nHead);
+  constructor(t, o, s) {
+    super(o, s), this.index = t, this.units = o.nEmbed * 3, this.projUnits = o.nEmbed, this.ATTN = `block_${this.index}_cAttn`, this.PROJ = `block_${this.index}_cProj`, this.addVariable(this.ATTN), this.addVariable(this.PROJ), this.divisor = 1 / Math.sqrt(o.nEmbed / o.nHead);
   }
   build() {
     this.hasVariable(this.ATTN) === !1 && this.setVariable(
       this.ATTN,
-      k(
-        T([this.config.nEmbed, this.units], 0, 0.02),
+      T(
+        v([this.config.nEmbed, this.units], 0, 0.02),
         !0,
         `block_${this.index}_attn_cAttn_kernel`
       )
     ), this.hasVariable(this.PROJ) === !1 && this.setVariable(
       this.PROJ,
-      k(
-        T([this.projUnits, this.config.nEmbed], 0, 0.02),
+      T(
+        v([this.projUnits, this.config.nEmbed], 0, 0.02),
         !0,
         `block_${this.index}_attn_cProj_kernel`
       )
     );
   }
-  getAttentionScores(t, i, s, o) {
-    const e = g(t, i, this.divisor), n = b(e, s ? this.config.dropout : 0, o);
-    return e.dispose(), n;
-  }
-  // Attention with optional past. If pastLen > 0 and T_cur == 1, no mask needed.
-  getAttentionScoresWithPast(t, i, s) {
-    const o = g(t, i, this.divisor, s), e = b(o, 0, 0);
-    return o.dispose(), e;
+  getAttentionScores(t, o, s) {
+    const i = P(t, o, this.divisor, s), e = C(i);
+    return i.dispose(), e;
   }
   getQKV(t) {
-    return _(t, this.getVariable(this.ATTN), this.config.nHead);
+    const o = l(t) ? V(this.getVariable(this.ATTN)) : this.getVariable(this.ATTN), s = j(t, o, this.config.nHead);
+    return l(t) && o.dispose(), s;
   }
   getOutputProjection(t) {
-    const i = t.shape[0], s = t.shape[2], o = this.config.nEmbed, e = t.transpose([0, 2, 1, 3]), n = j(e, [i, s, o]), r = C(n, this.getVariable(this.PROJ));
-    return n.dispose(), e.dispose(), r;
+    const o = t.shape[0], s = t.shape[2], i = this.config.nEmbed, e = l(t), r = J(t, [0, 2, 1, 3]), n = N(r, [o, s, e ? i / 2 : i]);
+    r.dispose();
+    const a = e ? V(this.getVariable(this.PROJ)) : this.getVariable(this.PROJ), h = L(n, a);
+    return e && a.dispose(), n.dispose(), h;
   }
-  updateCache(t, i, s) {
-    const o = this.config.blockSize, e = t.shape[2], n = s.length || 0, r = V(t, o, n, s.k);
+  updateCache(t, o, s) {
+    const i = this.config.blockSize, e = t.shape[2], r = s.length || 0, n = b(t, i, r, s.k);
     t.dispose(), s.k && s.k.dispose();
-    const p = V(i, o, n, s.v);
-    i.dispose(), s.v && s.v.dispose();
-    const d = Math.min(n + e, o), h = s.cumulativeLength + e;
-    s.length = d, s.cumulativeLength = h, s.k = c(r), s.v = c(p);
+    const a = b(o, i, r, s.v);
+    o.dispose(), s.v && s.v.dispose();
+    const h = Math.min(r + e, i), d = s.cumulativeLength + e;
+    s.length = h, s.cumulativeLength = d, s.k = u(n), s.v = u(a);
   }
-  forward(t, i) {
-    return P(() => {
+  forward(t, o) {
+    return S(() => {
       this.startMemory();
-      const [s, o, e] = this.getQKV(i), n = t.pastKV ? t.pastKV.cumulativeLength : 0, r = t.ropeCache, p = r ? v(s, r, n) : s, d = r ? v(o, r, n) : o;
-      r && (s.dispose(), o.dispose());
-      const h = t.pastKV ? t.pastKV.length : 0;
-      t.pastKV && !t.training && this.updateCache(d, e, t.pastKV);
-      const u = t.pastKV?.k ? t.pastKV.k : d, l = t.pastKV?.v ? t.pastKV.v : e;
-      let a;
-      h > 0 ? a = this.getAttentionScoresWithPast(p, u, h) : a = this.getAttentionScores(p, u, t.training, t.seed || 0), p.dispose(), t.pastKV || u.dispose();
-      const m = x(a, l), f = t.attentionScores !== void 0 && t.attentionScores.attentionOut !== void 0;
-      f || a.dispose(), t.pastKV || l.dispose();
-      const A = this.getOutputProjection(m);
-      if (m.dispose(), f && t.attentionScores && t.attentionScores.attentionOut !== void 0) {
-        const K = a.shape[1], S = a.shape[2];
-        t.attentionScores.attentionOut?.push(
-          c(a.slice([0, 0, 0, 0], [1, -1, -1, -1]).reshape([K, S, -1]))
+      const [s, i, e] = this.getQKV(o), r = t.pastKV ? t.pastKV.cumulativeLength : 0, n = t.ropeCache, a = n ? k(s, n, r) : s, h = n ? k(i, n, r) : i;
+      n && (s.dispose(), i.dispose());
+      const d = t.pastKV ? t.pastKV.length : 0;
+      t.pastKV && !t.training && this.updateCache(h, e, t.pastKV);
+      const c = t.pastKV?.k ? t.pastKV.k : h, m = t.pastKV?.v ? t.pastKV.v : e;
+      let p;
+      d > 0 ? p = this.getAttentionScores(a, c, d) : p = this.getAttentionScores(a, c), a.dispose(), t.pastKV || c.dispose();
+      const f = R(p, m), g = t.attentionScores !== void 0 && t.attentionScores.attentionOut !== void 0;
+      g || p.dispose(), t.pastKV || m.dispose();
+      const A = this.getOutputProjection(f);
+      if (f.dispose(), g && t.attentionScores && t.attentionScores.attentionOut !== void 0) {
+        const O = p.shape[1], K = p.shape[2];
+        console.log("Outputting attention scores shape:", p.shape), t.attentionScores.attentionOut?.push(
+          u(p.slice([0, 0, 0, 0], [1, -1, -1, -1]).reshape([O, K, -1]))
         );
       }
       return this.endMemory("CausalSelfAttention"), A;
@@ -85,12 +86,12 @@ class W extends O {
   }
   dropout(t) {
     if (this.config.dropout > 0) {
-      const i = L(t, this.config.dropout);
-      return t.dispose(), i;
+      const o = y(t, this.config.dropout);
+      return t.dispose(), o;
     } else
       return t;
   }
 }
 export {
-  W as default
+  X as default
 };

package/dist/layers/MLP.js

@@ -1,11 +1,10 @@
-import { t as p } from "../index-BzFyqcy-.js";
+import { t as p } from "../index-ZyQhjEPo.js";
 import u from "./BaseLayer.js";
-import { matMulGelu as M } from "../ops/matMulGelu.js";
-import { v as a } from "../variable-Bm2OFwGI.js";
-import { r as d, d as c } from "../dropout-C-csYCLj.js";
-import { r as h } from "../reshape-CnIwVG1c.js";
-import { m as b } from "../mat_mul-DzjTFx-u.js";
-class H extends u {
+import { c, b as M } from "../matMul16--R5hOwDG.js";
+import { reshape16 as o } from "../ops/reshape16.js";
+import { v as d } from "../variable-Bhn5bHYv.js";
+import { r as h, d as b } from "../dropout-C1pM3f11.js";
+class O extends u {
   index;
   hiddenUnits;
   MLPHIDDEN;
@@ -16,15 +15,15 @@ class H extends u {
   build() {
     this.hasVariable(this.MLPHIDDEN) === !1 && this.setVariable(
       this.MLPHIDDEN,
-      a(
-        d([this.config.nEmbed, this.hiddenUnits], 0, 0.02),
+      d(
+        h([this.config.nEmbed, this.hiddenUnits], 0, 0.02),
         !0,
         `block_${this.index}_mlpHidden_kernel`
       )
     ), this.hasVariable(this.MLPOUT) === !1 && this.setVariable(
       this.MLPOUT,
-      a(
-        d([this.hiddenUnits, this.config.nEmbed], 0, 0.02 / Math.sqrt(2 * this.config.nLayer)),
+      d(
+        h([this.hiddenUnits, this.config.nEmbed], 0, 0.02 / Math.sqrt(2 * this.config.nLayer)),
         !0,
         `block_${this.index}_mlpOut_kernel`
       )
@@ -33,20 +32,20 @@ class H extends u {
   forward(i, t) {
     return p(() => {
       this.startMemory();
-      const [s, e, r] = t.shape, n = h(t, [s * e, r]), o = M(n, this.getVariable(this.MLPHIDDEN)), l = b(o, this.getVariable(this.MLPOUT));
-      o.dispose();
-      const m = h(l, [s, e, r]);
+      const [s, e, r] = t.shape, n = o(t, [s * e, r]), a = c(n, this.getVariable(this.MLPHIDDEN)), l = M(a, this.getVariable(this.MLPOUT));
+      a.dispose();
+      const m = o(l, [s, e, r]);
       return this.endMemory("MLP"), m;
     });
   }
   dropout(i) {
     if (this.config.dropout > 0) {
-      const t = c(i, this.config.dropout);
+      const t = b(i, this.config.dropout);
       return i.dispose(), t;
     }
     return i;
   }
 }
 export {
-  H as default
+  O as default
 };

package/dist/layers/PositionEmbedding.js

@@ -1,18 +1,9 @@
-import { t as c, a9 as u, b as i } from "../index-BzFyqcy-.js";
+import { t as c, I as u, b as i } from "../index-ZyQhjEPo.js";
 import f from "./BaseLayer.js";
-import { E as g, D as h } from "../random_width-CXVRloNK.js";
-import { r as b } from "../exports_initializers-B8iZMgQ0.js";
-import { m as l } from "../mod-Dobti4j4.js";
-import { r as w } from "../range-CWcz7xFA.js";
-/**
- * @license
- * Copyright 2018 Google LLC
- *
- * Use of this source code is governed by an MIT-style
- * license that can be found in the LICENSE file or at
- * https://opensource.org/licenses/MIT.
- * =============================================================================
- */
+import { E as g, D as h } from "../random_width-DY6Kk2Dl.js";
+import { r as b } from "../exports_initializers-xuidcwI4.js";
+import { m as l } from "../mod-Gt1rMB4n.js";
+import { r as w } from "../range-BMS52eQi.js";
 function E(t) {
   return new h(t);
 }

package/dist/layers/RMSNorm.js

@@ -1,8 +1,8 @@
-import { t as s } from "../index-BzFyqcy-.js";
+import { t as s } from "../index-ZyQhjEPo.js";
 import e from "./BaseLayer.js";
 import { normRMS as a } from "../ops/normRMS.js";
-import { v as i } from "../variable-Bm2OFwGI.js";
-import { o as m } from "../ones-tIJeHlq-.js";
+import { v as i } from "../variable-Bhn5bHYv.js";
+import { o as m } from "../ones-CAMiP4I2.js";
 class l extends e {
   GAMMA;
   constructor(r, t = "", o) {

package/dist/layers/RoPECache.d.ts

@@ -7,10 +7,12 @@ export default class RoPECache {
     private ropeInvFreq;
     private ropeCos;
     private ropeSin;
+    private ropeNegSin;
     private ropeCacheLen;
     constructor(config: GPTConfig);
     ensureRopeCache(needed: number): void;
     getCos(): Tensor | null;
     getSin(): Tensor | null;
+    getNegSin(): Tensor | null;
     dispose(): void;
 }