@genai-fi/nanogpt 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Generator.js +13 -9
- package/dist/NanoGPTModel.js +10 -10
- package/dist/{RealDiv-C4hOvYOZ.js → RealDiv-CVYNbZxu.js} +11 -11
- package/dist/{Reshape-BLijOA8h.js → Reshape-CEsEp0AI.js} +2 -2
- package/dist/Reshape-Do18N3gO.js +30 -0
- package/dist/TeachableLLM.js +9 -5
- package/dist/{TiedEmbedding-BLltddza.js → TiedEmbedding-ccLBFiZi.js} +4 -4
- package/dist/{axis_util-DaAl5MER.js → axis_util-5DTW2tFV.js} +1 -1
- package/dist/backend.js +2 -2
- package/dist/{backend_util-DWiwsi2N.js → backend_util-C9Ut8n0Q.js} +40 -40
- package/dist/{broadcast_to-C4v-j9yA.js → broadcast_to-Ba9h_8DO.js} +2 -2
- package/dist/{concat-CsHeR4zV.js → concat-CbXTetof.js} +1 -1
- package/dist/{dataset-JDyjG3QR.js → dataset-U3PrjwgU.js} +7 -7
- package/dist/{dropout-hpDwECTe.js → dropout-DPfPgWWe.js} +11 -11
- package/dist/{gather-D0_gPiBz.js → gather-Bbh8DHhM.js} +4 -4
- package/dist/{gelu-uyHP1x1f.js → gelu-BFwVnd1r.js} +1 -1
- package/dist/{gpgpu_math-DJm3ZTAf.js → gpgpu_math-DffelNS-.js} +2 -2
- package/dist/{index-BPPzKVdR.js → index-DYD_yPa-.js} +1083 -1106
- package/dist/{index-C0dhsYom.js → index-UdZhlibC.js} +126 -126
- package/dist/{kernel_funcs_utils-CwRTFqrc.js → kernel_funcs_utils-CXDy3EN7.js} +3 -3
- package/dist/layers/BaseLayer.js +2 -2
- package/dist/layers/CausalSelfAttention.js +8 -8
- package/dist/layers/MLP.js +5 -5
- package/dist/layers/RMSNorm.js +3 -3
- package/dist/layers/RoPECache.js +4 -4
- package/dist/layers/TiedEmbedding.js +5 -5
- package/dist/layers/TransformerBlock.js +1 -1
- package/dist/loader/loadTransformers.js +1 -1
- package/dist/loader/oldZipLoad.js +11 -7
- package/dist/{log_sum_exp-D086OgZJ.js → log_sum_exp-BnmCkHWl.js} +8 -8
- package/dist/main.d.ts +11 -0
- package/dist/main.js +44 -27
- package/dist/{mat_mul-1nwdPkQ_.js → mat_mul-dwmZz69e.js} +1 -1
- package/dist/{max-BQc2Aj-I.js → max-ByjEGoFx.js} +3 -3
- package/dist/{mulmat_packed_gpu-Gzf3I9UV.js → mulmat_packed_gpu-IGPBp6h9.js} +1 -1
- package/dist/{ones-D63HpSF_.js → ones-C8Mfln6-.js} +2 -2
- package/dist/ops/adamAdjust.d.ts +2 -0
- package/dist/ops/adamAdjust.js +9 -0
- package/dist/ops/adamMoments.d.ts +2 -0
- package/dist/ops/adamMoments.js +9 -0
- package/dist/ops/appendCache.js +3 -3
- package/dist/ops/attentionMask.js +1 -1
- package/dist/ops/cpu/adamAdjust.d.ts +1 -0
- package/dist/ops/cpu/adamAdjust.js +18 -0
- package/dist/ops/cpu/adamMoments.d.ts +1 -0
- package/dist/ops/cpu/adamMoments.js +16 -0
- package/dist/ops/cpu/appendCache.js +2 -2
- package/dist/ops/cpu/attentionMask.js +5 -5
- package/dist/ops/cpu/fusedSoftmax.js +2 -2
- package/dist/ops/cpu/gatherSub.js +3 -3
- package/dist/ops/cpu/gelu.js +1 -1
- package/dist/ops/cpu/matMulGelu.js +2 -2
- package/dist/ops/cpu/matMulMul.js +1 -1
- package/dist/ops/cpu/mulDropout.js +1 -1
- package/dist/ops/cpu/normRMS.js +1 -1
- package/dist/ops/cpu/qkv.js +3 -3
- package/dist/ops/cpu/rope.js +5 -5
- package/dist/ops/cpu/scatterSub.js +11 -11
- package/dist/ops/fusedSoftmax.js +1 -1
- package/dist/ops/gatherSub.js +1 -1
- package/dist/ops/gelu.js +2 -2
- package/dist/ops/grads/attentionMask.js +1 -1
- package/dist/ops/grads/fusedSoftmax.js +2 -2
- package/dist/ops/grads/gelu.js +2 -2
- package/dist/ops/grads/matMulGelu.js +1 -1
- package/dist/ops/grads/normRMS.js +1 -1
- package/dist/ops/grads/qkv.js +1 -1
- package/dist/ops/grads/rope.js +1 -1
- package/dist/ops/matMulGelu.js +1 -1
- package/dist/ops/matMulMul.js +1 -1
- package/dist/ops/mulDrop.js +1 -1
- package/dist/ops/normRMS.js +1 -1
- package/dist/ops/qkv.js +1 -1
- package/dist/ops/rope.js +4 -4
- package/dist/ops/scatterSub.js +1 -1
- package/dist/ops/webgl/adamAdjust.d.ts +1 -0
- package/dist/ops/webgl/adamAdjust.js +50 -0
- package/dist/ops/webgl/adamMoments.d.ts +1 -0
- package/dist/ops/webgl/adamMoments.js +38 -0
- package/dist/ops/webgl/appendCache.js +1 -1
- package/dist/ops/webgl/attentionMask.js +1 -1
- package/dist/ops/webgl/fusedSoftmax.js +4 -4
- package/dist/ops/webgl/gatherSub.js +8 -8
- package/dist/ops/webgl/gelu.js +2 -2
- package/dist/ops/webgl/log.js +3 -3
- package/dist/ops/webgl/matMulGelu.js +4 -4
- package/dist/ops/webgl/matMulMul.js +1 -1
- package/dist/ops/webgl/mulDropout.js +1 -1
- package/dist/ops/webgl/normRMS.js +2 -2
- package/dist/ops/webgl/qkv.js +1 -1
- package/dist/ops/webgl/rope.js +1 -1
- package/dist/ops/webgl/scatterSub.js +1 -1
- package/dist/ops/webgpu/adamAdjust.d.ts +1 -0
- package/dist/ops/webgpu/adamAdjust.js +52 -0
- package/dist/ops/webgpu/adamMoments.d.ts +1 -0
- package/dist/ops/webgpu/adamMoments.js +51 -0
- package/dist/ops/webgpu/appendCache.js +13 -12
- package/dist/ops/webgpu/attentionMask.js +11 -10
- package/dist/ops/webgpu/gatherSub.js +26 -11
- package/dist/ops/webgpu/gelu.js +7 -6
- package/dist/ops/webgpu/index.js +3 -0
- package/dist/ops/webgpu/normRMS.js +27 -101
- package/dist/ops/webgpu/normRMSGrad.d.ts +1 -0
- package/dist/ops/webgpu/normRMSGrad.js +128 -0
- package/dist/ops/webgpu/qkv.js +9 -8
- package/dist/ops/webgpu/rope.js +8 -7
- package/dist/ops/webgpu/scatterSub.js +8 -7
- package/dist/ops/webgpu/utils/reductions.d.ts +9 -0
- package/dist/ops/webgpu/utils/reductions.js +68 -0
- package/dist/{ops-CIQLNshk.js → ops-aRTXR2Sr.js} +195 -219
- package/dist/{random_width-DkYP8W8N.js → random_width-DbSpgl4o.js} +22 -21
- package/dist/{range-CYzpQY53.js → range-D9CZhVlR.js} +1 -1
- package/dist/{reciprocal-_A9yv27J.js → reciprocal-CGB48wZB.js} +1 -1
- package/dist/{register_all_kernels-guvSxp7M.js → register_all_kernels-DnbAyBXt.js} +30 -29
- package/dist/{reshape-BMUzc1UY.js → reshape-BR0eoLYN.js} +3 -3
- package/dist/{scatter_nd_util-IRBqKz_b.js → scatter_nd_util-OjyAxku2.js} +1 -1
- package/dist/{selu_util-Dt_iuXaq.js → selu_util-Ce6pu9IM.js} +41 -41
- package/dist/{shared-CDu9S76h.js → shared-Czipaeb6.js} +6 -6
- package/dist/{shared-BNa2q6jD.js → shared-DS5waSIY.js} +1 -1
- package/dist/{sin-Cocju-BY.js → sin-CiBxrDqX.js} +6 -6
- package/dist/slice-BHbDHObE.js +28 -0
- package/dist/{softmax-GPNK3o-U.js → softmax-JMEIUo2J.js} +3 -3
- package/dist/{split-CHzJjxDv.js → split-CRU0PjVV.js} +1 -1
- package/dist/{stack-Dpgg_1W1.js → stack-ikk2Y8_P.js} +1 -1
- package/dist/{sum-B8wEpKsg.js → sum-NLYbiDag.js} +3 -3
- package/dist/{tensor-RvZVNmg0.js → tensor-Do9PKbIE.js} +1 -1
- package/dist/{tensor2d-B_kyod7_.js → tensor2d-CWHxHpLh.js} +1 -1
- package/dist/training/Adam.d.ts +22 -0
- package/dist/training/Adam.js +93 -0
- package/dist/training/AdamExt.d.ts +1 -1
- package/dist/training/AdamExt.js +13 -12
- package/dist/training/DatasetBuilder.js +2 -2
- package/dist/training/FullTrainer.js +22 -22
- package/dist/training/Trainer.d.ts +1 -1
- package/dist/training/Trainer.js +32 -32
- package/dist/training/sparseCrossEntropy.d.ts +0 -4
- package/dist/training/sparseCrossEntropy.js +7 -7
- package/dist/utilities/arrayClose.d.ts +1 -0
- package/dist/utilities/arrayClose.js +11 -0
- package/dist/utilities/dummy.js +2 -2
- package/dist/utilities/generate.js +3 -3
- package/dist/utilities/multinomialCPU.js +2 -2
- package/dist/utilities/performance.d.ts +1 -1
- package/dist/utilities/performance.js +11 -11
- package/dist/utilities/profile.js +1 -1
- package/dist/utilities/safetensors.js +2 -2
- package/dist/utilities/weights.js +2 -2
- package/dist/{variable-DXEUOwew.js → variable-BTBkayv_.js} +1 -1
- package/dist/{webgpu_util-g13LvDIv.js → webgpu_program-WaoMq-WD.js} +138 -215
- package/dist/webgpu_util-DhSeP4b6.js +80 -0
- package/dist/{zeros-DCPCdFGq.js → zeros-DnPT2nD4.js} +4 -4
- package/package.json +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { z as c, c as f } from "./zeros-
|
|
1
|
+
import { F as n, H as t, j as m, E as i } from "./index-UdZhlibC.js";
|
|
2
|
+
import { z as c, c as f } from "./zeros-DnPT2nD4.js";
|
|
3
3
|
/**
|
|
4
4
|
* @license
|
|
5
5
|
* Copyright 2018 Google LLC. All Rights Reserved.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { e as i } from "../index-UdZhlibC.js";
|
|
2
|
+
import "./cpu/adamAdjust.js";
|
|
3
|
+
import "./webgl/adamAdjust.js";
|
|
4
|
+
function p(r, t, e, n, m, o) {
|
|
5
|
+
return i().runKernel("AdamAdjust", { moments: r, value: t }, { beta1: e, beta2: n, epsilon: m, learningRate: o });
|
|
6
|
+
}
|
|
7
|
+
export {
|
|
8
|
+
p as adamAdjust
|
|
9
|
+
};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { e as o } from "../index-UdZhlibC.js";
|
|
2
|
+
import "./cpu/adamMoments.js";
|
|
3
|
+
import "./webgl/adamMoments.js";
|
|
4
|
+
function p(e, n, r, m) {
|
|
5
|
+
return o().runKernel("AdamMoments", { moments: e, gradient: n }, { beta1: r, beta2: m });
|
|
6
|
+
}
|
|
7
|
+
export {
|
|
8
|
+
p as adamMoments
|
|
9
|
+
};
|
package/dist/ops/appendCache.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { e as a } from "../index-
|
|
1
|
+
import { e as a } from "../index-UdZhlibC.js";
|
|
2
2
|
import "./cpu/appendCache.js";
|
|
3
3
|
import "./webgl/appendCache.js";
|
|
4
|
-
import { c as s } from "../concat-
|
|
5
|
-
import { z as c } from "../zeros-
|
|
4
|
+
import { c as s } from "../concat-CbXTetof.js";
|
|
5
|
+
import { z as c } from "../zeros-DnPT2nD4.js";
|
|
6
6
|
function i(r, p, n, o) {
|
|
7
7
|
if (!o) {
|
|
8
8
|
const e = r.shape[2];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { f as k, n as t, o as i, a as q, q as z } from "../../index-UdZhlibC.js";
|
|
2
|
+
function A(a) {
|
|
3
|
+
const { moments: n, value: r } = a.inputs, { beta1: l, beta2: u, epsilon: m, learningRate: d } = a.attrs, e = n.shape.length, c = new Array(e).fill(0), s = n.shape.slice();
|
|
4
|
+
s[e - 1] = 1;
|
|
5
|
+
const o = c.slice();
|
|
6
|
+
o[e - 1] = 1;
|
|
7
|
+
const b = s.slice(), p = n.slice(c, s).squeeze([e - 1]), M = n.slice(o, b).squeeze([e - 1]), f = t(p, l), g = t(M, u);
|
|
8
|
+
return i(
|
|
9
|
+
q(t(f, i(z(g), m ?? 1e-8)), -d),
|
|
10
|
+
r
|
|
11
|
+
);
|
|
12
|
+
}
|
|
13
|
+
const C = {
|
|
14
|
+
kernelName: "AdamAdjust",
|
|
15
|
+
backendName: "cpu",
|
|
16
|
+
kernelFunc: A
|
|
17
|
+
};
|
|
18
|
+
k(C);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { f as p } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { s as f } from "../../stack-ikk2Y8_P.js";
|
|
3
|
+
function b(t) {
|
|
4
|
+
const { moments: n, gradient: c } = t.inputs, { beta1: o, beta2: m } = t.attrs, e = n.shape.length, a = new Array(e).fill(0), s = n.shape.slice();
|
|
5
|
+
s[e - 1] = 1;
|
|
6
|
+
const i = a.slice();
|
|
7
|
+
i[e - 1] = 1;
|
|
8
|
+
const r = s.slice(), l = n.slice(a, s).squeeze([e - 1]), u = n.slice(i, r).squeeze([e - 1]), M = l.mul(o).add(c.mul(1 - o)), d = u.mul(m).add(c.square().mul(1 - m));
|
|
9
|
+
return f([M, d], -1);
|
|
10
|
+
}
|
|
11
|
+
const g = {
|
|
12
|
+
kernelName: "AdamMoments",
|
|
13
|
+
backendName: "cpu",
|
|
14
|
+
kernelFunc: b
|
|
15
|
+
};
|
|
16
|
+
p(g);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { f as d } from "../../index-
|
|
2
|
-
import { c as h } from "../../concat-
|
|
1
|
+
import { f as d } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { c as h } from "../../concat-CbXTetof.js";
|
|
3
3
|
function u(p) {
|
|
4
4
|
const { cache: n, item: s } = p.inputs, { maxSize: i, pastLen: c } = p.attrs, t = n.shape[0], o = n.shape[1], a = n.shape[3], e = s.shape[2];
|
|
5
5
|
if (c + e <= i) {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { f as a, h as p, b as u } from "../../index-
|
|
2
|
-
import { l as N, w as b } from "../../ops-
|
|
3
|
-
import { o as A } from "../../ones-
|
|
4
|
-
import { z as I } from "../../zeros-
|
|
5
|
-
import { m as g } from "../../mat_mul-
|
|
1
|
+
import { f as a, h as p, b as u } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { l as N, w as b } from "../../ops-aRTXR2Sr.js";
|
|
3
|
+
import { o as A } from "../../ones-C8Mfln6-.js";
|
|
4
|
+
import { z as I } from "../../zeros-DnPT2nD4.js";
|
|
5
|
+
import { m as g } from "../../mat_mul-dwmZz69e.js";
|
|
6
6
|
function o(n) {
|
|
7
7
|
const { q: s, k: e } = n.inputs, { divisor: r } = n.attrs, c = s.shape[2], t = e.shape[2], m = N.bandPart(A([t, t]), -1, 0).cast("bool"), l = I([t, t]), i = p([t, t], Number.NEGATIVE_INFINITY), f = b(m, l, i), k = g(s, e, !1, !0).mul(u(r)), d = f.slice([0, 0], [c, t]).expandDims(0).expandDims(0);
|
|
8
8
|
return k.add(d);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { f as e } from "../../index-
|
|
2
|
-
import { s as f } from "../../softmax-
|
|
1
|
+
import { f as e } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { s as f } from "../../softmax-JMEIUo2J.js";
|
|
3
3
|
function n(t) {
|
|
4
4
|
const { inputs: s, attrs: a } = t, { logits: o } = s, { dim: i, dropoutRate: r } = a;
|
|
5
5
|
if (!o)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { r as f } from "../../range-
|
|
3
|
-
import { s as l } from "../../stack-
|
|
1
|
+
import { B as u, C as c, E as g, a4 as h, f as m, c as p } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { r as f } from "../../range-D9CZhVlR.js";
|
|
3
|
+
import { s as l } from "../../stack-ikk2Y8_P.js";
|
|
4
4
|
/**
|
|
5
5
|
* @license
|
|
6
6
|
* Copyright 2018 Google LLC. All Rights Reserved.
|
package/dist/ops/cpu/gelu.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { f as e, t as m } from "../../index-
|
|
2
|
-
import { g as d, d as M } from "../../gelu-
|
|
1
|
+
import { f as e, t as m } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { g as d, d as M } from "../../gelu-BFwVnd1r.js";
|
|
3
3
|
function c(t) {
|
|
4
4
|
const { inputs: u } = t, { x: n, kernel: l } = u, a = n, r = l;
|
|
5
5
|
return m(() => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { f as e, t as i } from "../../index-
|
|
1
|
+
import { f as e, t as i } from "../../index-UdZhlibC.js";
|
|
2
2
|
function n(t) {
|
|
3
3
|
const { inputs: r, attrs: l } = t, { transposeA: o, transposeB: s } = l, { x: u, kernel: a, y: c } = r, m = u, k = a, M = c;
|
|
4
4
|
return i(() => m.matMul(k, o, s).mul(M));
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { f as e, a as t } from "../../index-
|
|
1
|
+
import { f as e, a as t } from "../../index-UdZhlibC.js";
|
|
2
2
|
function n(o) {
|
|
3
3
|
const { inputs: r } = o, { a: l, b: u } = r;
|
|
4
4
|
return console.warn("Using fallback mulDrop implementation without dropout."), t(l, u);
|
package/dist/ops/cpu/normRMS.js
CHANGED
package/dist/ops/cpu/qkv.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { f as q } from "../../index-
|
|
2
|
-
import { r as o } from "../../reshape-
|
|
3
|
-
import { s as x } from "../../split-
|
|
1
|
+
import { f as q } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { r as o } from "../../reshape-BR0eoLYN.js";
|
|
3
|
+
import { s as x } from "../../split-CRU0PjVV.js";
|
|
4
4
|
function v(p) {
|
|
5
5
|
const { x: c, kernel: K } = p.inputs, { heads: n } = p.attrs, [s, e, t] = c.shape, a = o(c, [s * e, t]), i = a.dot(K);
|
|
6
6
|
a.dispose();
|
package/dist/ops/cpu/rope.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { f as S } from "../../index-
|
|
2
|
-
import { r as F } from "../../range-
|
|
3
|
-
import { g as I } from "../../gather-
|
|
4
|
-
import { s as E } from "../../stack-
|
|
5
|
-
import { c as T } from "../../concat-
|
|
1
|
+
import { f as S } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { r as F } from "../../range-D9CZhVlR.js";
|
|
3
|
+
import { g as I } from "../../gather-Bbh8DHhM.js";
|
|
4
|
+
import { s as E } from "../../stack-ikk2Y8_P.js";
|
|
5
|
+
import { c as T } from "../../concat-CbXTetof.js";
|
|
6
6
|
function U(t, c, p, o, r) {
|
|
7
7
|
const n = o.shape[3], s = p;
|
|
8
8
|
if (s > n) return o;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { v as D } from "../../scatter_nd_util-
|
|
3
|
-
import { r as k } from "../../range-
|
|
4
|
-
import { s as v } from "../../stack-
|
|
5
|
-
import { o as E } from "../../ones-
|
|
1
|
+
import { B as f, F as g, C as r, E as l, a3 as N, f as b, c as S, a as h } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { v as D } from "../../scatter_nd_util-OjyAxku2.js";
|
|
3
|
+
import { r as k } from "../../range-D9CZhVlR.js";
|
|
4
|
+
import { s as v } from "../../stack-ikk2Y8_P.js";
|
|
5
|
+
import { o as E } from "../../ones-C8Mfln6-.js";
|
|
6
6
|
/**
|
|
7
7
|
* @license
|
|
8
8
|
* Copyright 2018 Google LLC. All Rights Reserved.
|
|
@@ -26,14 +26,14 @@ function I(a, e, s) {
|
|
|
26
26
|
const c = { indices: n, updates: t }, o = { shape: s };
|
|
27
27
|
return l.runKernel(N, c, o);
|
|
28
28
|
}
|
|
29
|
-
const
|
|
30
|
-
function
|
|
31
|
-
const { logits: e, labels: s, dy: n } = a.inputs, t = s.shape[0], c = e.shape[1], o = k(0, t, 1, "int32"), i = v([o, s], 1), d = E([t]), u =
|
|
29
|
+
const C = /* @__PURE__ */ f({ scatterND_: I });
|
|
30
|
+
function K(a) {
|
|
31
|
+
const { logits: e, labels: s, dy: n } = a.inputs, t = s.shape[0], c = e.shape[1], o = k(0, t, 1, "int32"), i = v([o, s], 1), d = E([t]), u = C(i, d, [t, c]), p = S(e, u), m = n.reshape([t, 1]);
|
|
32
32
|
return h(p, m);
|
|
33
33
|
}
|
|
34
|
-
const
|
|
34
|
+
const L = {
|
|
35
35
|
kernelName: "EfficientScatterSub",
|
|
36
36
|
backendName: "cpu",
|
|
37
|
-
kernelFunc:
|
|
37
|
+
kernelFunc: K
|
|
38
38
|
};
|
|
39
|
-
b(
|
|
39
|
+
b(L);
|
package/dist/ops/fusedSoftmax.js
CHANGED
package/dist/ops/gatherSub.js
CHANGED
package/dist/ops/gelu.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import "../index-
|
|
1
|
+
import "../index-UdZhlibC.js";
|
|
2
2
|
import "./cpu/gelu.js";
|
|
3
3
|
import "./webgl/gelu.js";
|
|
4
|
-
import { d as e, g as i } from "../gelu-
|
|
4
|
+
import { d as e, g as i } from "../gelu-BFwVnd1r.js";
|
|
5
5
|
export {
|
|
6
6
|
e as dGelu,
|
|
7
7
|
i as gelu
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { i as f, a as i, c as l } from "../../index-
|
|
1
|
+
import { i as f, a as i, c as l } from "../../index-UdZhlibC.js";
|
|
2
2
|
import { mulDrop as g } from "../mulDrop.js";
|
|
3
|
-
import { s as T } from "../../sum-
|
|
3
|
+
import { s as T } from "../../sum-NLYbiDag.js";
|
|
4
4
|
const Y = {
|
|
5
5
|
kernelName: "FusedSoftmax",
|
|
6
6
|
outputsToSave: [!0],
|
package/dist/ops/grads/gelu.js
CHANGED
package/dist/ops/grads/qkv.js
CHANGED
package/dist/ops/grads/rope.js
CHANGED
package/dist/ops/matMulGelu.js
CHANGED
package/dist/ops/matMulMul.js
CHANGED
package/dist/ops/mulDrop.js
CHANGED
package/dist/ops/normRMS.js
CHANGED
package/dist/ops/qkv.js
CHANGED
package/dist/ops/rope.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { e as p } from "../index-
|
|
2
|
-
import "../random_width-
|
|
3
|
-
import "../register_all_kernels-
|
|
1
|
+
import { e as p } from "../index-UdZhlibC.js";
|
|
2
|
+
import "../random_width-DbSpgl4o.js";
|
|
3
|
+
import "../register_all_kernels-DnbAyBXt.js";
|
|
4
4
|
import "../index-Tf7vU29b.js";
|
|
5
|
-
import "../dataset-
|
|
5
|
+
import "../dataset-U3PrjwgU.js";
|
|
6
6
|
import "./cpu/rope.js";
|
|
7
7
|
import "./webgl/rope.js";
|
|
8
8
|
import "./grads/rope.js";
|
package/dist/ops/scatterSub.js
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { r as n } from "../../Reshape-CEsEp0AI.js";
|
|
2
|
+
import { f } from "../../index-UdZhlibC.js";
|
|
3
|
+
class v {
|
|
4
|
+
variableNames = ["moments", "value"];
|
|
5
|
+
outputShape;
|
|
6
|
+
userCode;
|
|
7
|
+
customUniforms = [
|
|
8
|
+
{ name: "invBeta1", type: "float" },
|
|
9
|
+
{ name: "invBeta2", type: "float" },
|
|
10
|
+
{ name: "learningRate", type: "float" },
|
|
11
|
+
{ name: "epsilon", type: "float" }
|
|
12
|
+
];
|
|
13
|
+
constructor(a) {
|
|
14
|
+
this.outputShape = a, this.userCode = `
|
|
15
|
+
void main() {
|
|
16
|
+
float v = getValueAtOutCoords();
|
|
17
|
+
int coords = getOutputCoords();
|
|
18
|
+
coords *= 2;
|
|
19
|
+
float m1 = getMoments(coords);
|
|
20
|
+
float m2 = getMoments(coords + 1);
|
|
21
|
+
|
|
22
|
+
float m1Hat = m1 * invBeta1;
|
|
23
|
+
float m2Hat = m2 * invBeta2;
|
|
24
|
+
|
|
25
|
+
float invSqrt = inversesqrt(max(m2Hat, 1e-30));
|
|
26
|
+
float invDenom = invSqrt / (1.0 + epsilon * invSqrt);
|
|
27
|
+
float adjustedValue = -learningRate * m1Hat * invDenom + v;
|
|
28
|
+
|
|
29
|
+
setOutput(adjustedValue);
|
|
30
|
+
}
|
|
31
|
+
`;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function h(t) {
|
|
35
|
+
const { moments: a, value: s } = t.inputs, { beta1: i, beta2: l, learningRate: u, epsilon: p } = t.attrs, e = t.backend, r = n({ inputs: { x: a }, backend: e, attrs: { shape: [-1] } }), o = n({ inputs: { x: s }, backend: e, attrs: { shape: [-1] } }), d = new v(o.shape), m = e.runWebGLProgram(d, [r, o], "float32", [
|
|
36
|
+
[1 / i],
|
|
37
|
+
[1 / l],
|
|
38
|
+
[u],
|
|
39
|
+
[p]
|
|
40
|
+
]);
|
|
41
|
+
e.disposeIntermediateTensorInfo(r), e.disposeIntermediateTensorInfo(o);
|
|
42
|
+
const c = n({ inputs: { x: m }, backend: e, attrs: { shape: s.shape } });
|
|
43
|
+
return e.disposeIntermediateTensorInfo(m), c;
|
|
44
|
+
}
|
|
45
|
+
const g = {
|
|
46
|
+
kernelName: "AdamAdjust",
|
|
47
|
+
backendName: "webgl",
|
|
48
|
+
kernelFunc: h
|
|
49
|
+
};
|
|
50
|
+
f(g);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { f as m } from "../../index-UdZhlibC.js";
|
|
2
|
+
class i {
|
|
3
|
+
variableNames = ["moments", "gradient"];
|
|
4
|
+
outputShape;
|
|
5
|
+
userCode;
|
|
6
|
+
customUniforms = [
|
|
7
|
+
{ name: "beta1", type: "float" },
|
|
8
|
+
{ name: "beta2", type: "float" }
|
|
9
|
+
];
|
|
10
|
+
constructor(e) {
|
|
11
|
+
this.outputShape = e;
|
|
12
|
+
const t = e.length, n = t === 1 ? "int" : `ivec${Math.min(t, 4)}`, a = t === 1 ? "coords" : `coords[${t - 1}]`, s = t === 1 ? "coords" : e.slice(0, -1).map((r, c) => `coords[${c}]`).join(", ");
|
|
13
|
+
this.userCode = `
|
|
14
|
+
void main() {
|
|
15
|
+
float m = getMomentsAtOutCoords();
|
|
16
|
+
${n} coords = getOutputCoords();
|
|
17
|
+
float g = getGradient(${s});
|
|
18
|
+
int which = ${a};
|
|
19
|
+
|
|
20
|
+
float beta = which == 0 ? beta1 : beta2;
|
|
21
|
+
float gg = which == 0 ? g : g * g;
|
|
22
|
+
|
|
23
|
+
float newM = m * beta + gg * (1.0 - beta);
|
|
24
|
+
setOutput(newM);
|
|
25
|
+
}
|
|
26
|
+
`;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function d(o) {
|
|
30
|
+
const { moments: e, gradient: t } = o.inputs, { beta1: n, beta2: a } = o.attrs, s = o.backend, r = new i(e.shape);
|
|
31
|
+
return s.runWebGLProgram(r, [e, t], "float32", [[n], [a]]);
|
|
32
|
+
}
|
|
33
|
+
const g = {
|
|
34
|
+
kernelName: "AdamMoments",
|
|
35
|
+
backendName: "webgl",
|
|
36
|
+
kernelFunc: d
|
|
37
|
+
};
|
|
38
|
+
m(g);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { m as b, s as I, r as k } from "../../RealDiv-
|
|
2
|
-
import { r as v } from "../../Reshape-
|
|
3
|
-
import { f as w, p as P } from "../../index-
|
|
4
|
-
import { e as S } from "../../axis_util-
|
|
1
|
+
import { m as b, s as I, r as k } from "../../RealDiv-CVYNbZxu.js";
|
|
2
|
+
import { r as v } from "../../Reshape-CEsEp0AI.js";
|
|
3
|
+
import { f as w, p as P } from "../../index-UdZhlibC.js";
|
|
4
|
+
import { e as S } from "../../axis_util-5DTW2tFV.js";
|
|
5
5
|
class T {
|
|
6
6
|
variableNames = ["logits", "maxLogits"];
|
|
7
7
|
outputShape;
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import { f as l } from "../../index-
|
|
1
|
+
import { f as l } from "../../index-UdZhlibC.js";
|
|
2
2
|
class u {
|
|
3
3
|
variableNames = ["labels", "logits", "values"];
|
|
4
4
|
outputShape;
|
|
5
5
|
userCode;
|
|
6
6
|
constructor(e) {
|
|
7
7
|
this.outputShape = [e], this.userCode = `
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
8
|
+
void main() {
|
|
9
|
+
int coords = getOutputCoords();
|
|
10
|
+
int index = int(getLabelsAtOutCoords());
|
|
11
|
+
float val = getValuesAtOutCoords();
|
|
12
|
+
float logit = getLogits(coords, index);
|
|
13
|
+
setOutput(val - logit);
|
|
14
|
+
}
|
|
15
15
|
`;
|
|
16
16
|
}
|
|
17
17
|
}
|
package/dist/ops/webgl/gelu.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { f as a } from "../../index-
|
|
2
|
-
import { u as s, C as x } from "../../kernel_funcs_utils-
|
|
1
|
+
import { f as a } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { u as s, C as x } from "../../kernel_funcs_utils-CXDy3EN7.js";
|
|
3
3
|
const t = 0.7978845608028654, r = 0.044715, c = x + `
|
|
4
4
|
float x3 = x * x * x;
|
|
5
5
|
float inner = x + ${r} * x3;
|
package/dist/ops/webgl/log.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { f as e,
|
|
2
|
-
import { u as s, l as N } from "../../kernel_funcs_utils-
|
|
3
|
-
import { y as l } from "../../shared-
|
|
1
|
+
import { f as e, af as r } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { u as s, l as N } from "../../kernel_funcs_utils-CXDy3EN7.js";
|
|
3
|
+
import { y as l } from "../../shared-DS5waSIY.js";
|
|
4
4
|
/**
|
|
5
5
|
* @license
|
|
6
6
|
* Copyright 2020 Google LLC. All Rights Reserved.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { f as _, t as R, e as C, j as A, k as N, l as H, u as O } from "../../index-
|
|
2
|
-
import { r as f } from "../../Reshape-
|
|
3
|
-
import { M as U } from "../../mulmat_packed_gpu-
|
|
4
|
-
import { m as E } from "../../mat_mul-
|
|
1
|
+
import { f as _, t as R, e as C, j as A, k as N, l as H, u as O } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { r as f } from "../../Reshape-CEsEp0AI.js";
|
|
3
|
+
import { M as U } from "../../mulmat_packed_gpu-IGPBp6h9.js";
|
|
4
|
+
import { m as E } from "../../mat_mul-dwmZz69e.js";
|
|
5
5
|
const M = 0.7978845608028654, x = 0.044715, j = `
|
|
6
6
|
vec4 x3 = x * x * x;
|
|
7
7
|
vec4 inner = x + ${x} * x3;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { f as p, e as G } from "../../index-
|
|
2
|
-
import { s as x } from "../../sum-
|
|
1
|
+
import { f as p, e as G } from "../../index-UdZhlibC.js";
|
|
2
|
+
import { s as x } from "../../sum-NLYbiDag.js";
|
|
3
3
|
class y {
|
|
4
4
|
variableNames = ["x", "meanSquare", "gamma"];
|
|
5
5
|
outputShape;
|
package/dist/ops/webgl/qkv.js
CHANGED
package/dist/ops/webgl/rope.js
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|