@genai-fi/nanogpt 0.6.3 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Generator.js +11 -11
- package/dist/NanoGPTModel.d.ts +2 -2
- package/dist/NanoGPTModel.js +104 -136
- package/dist/{RealDiv-BYViZwhN.js → RealDiv-C4hOvYOZ.js} +26 -25
- package/dist/{Reshape-t7Kcikjk.js → Reshape-BLijOA8h.js} +5 -5
- package/dist/TeachableLLM.js +5 -5
- package/dist/{TiedEmbedding-9WeDwvjO.js → TiedEmbedding-BLltddza.js} +4 -4
- package/dist/{axis_util-Bu4h7XWV.js → axis_util-DaAl5MER.js} +3 -3
- package/dist/backend.d.ts +1 -0
- package/dist/backend.js +7 -0
- package/dist/backend_util-DWiwsi2N.js +749 -0
- package/dist/{broadcast_to-DARN-DBD.js → broadcast_to-C4v-j9yA.js} +2 -2
- package/dist/{concat-5aPGqw3Z.js → concat-CsHeR4zV.js} +8 -8
- package/dist/{dataset-pgqp-YfL.js → dataset-JDyjG3QR.js} +3 -3
- package/dist/{dropout-Bciw46HT.js → dropout-hpDwECTe.js} +7 -7
- package/dist/{gather-DjyCjmOD.js → gather-D0_gPiBz.js} +4 -4
- package/dist/gelu-uyHP1x1f.js +26 -0
- package/dist/gpgpu_math-DJm3ZTAf.js +2371 -0
- package/dist/index-BPPzKVdR.js +12099 -0
- package/dist/{index-BAzbokzv.js → index-C0dhsYom.js} +405 -389
- package/dist/{kernel_funcs_utils-CUxJCg0g.js → kernel_funcs_utils-CwRTFqrc.js} +31 -30
- package/dist/layers/BaseLayer.js +2 -2
- package/dist/layers/CausalSelfAttention.js +6 -6
- package/dist/layers/MLP.js +5 -5
- package/dist/layers/RMSNorm.js +3 -3
- package/dist/layers/RoPECache.js +4 -4
- package/dist/layers/TiedEmbedding.js +5 -5
- package/dist/layers/TransformerBlock.js +1 -1
- package/dist/loader/loadTransformers.js +1 -1
- package/dist/loader/oldZipLoad.js +5 -5
- package/dist/{log_sum_exp-YEo2h3gb.js → log_sum_exp-D086OgZJ.js} +15 -15
- package/dist/main.d.ts +2 -0
- package/dist/main.js +9 -5
- package/dist/{mat_mul-7121rsJk.js → mat_mul-1nwdPkQ_.js} +4 -4
- package/dist/{max-DtlIuVeW.js → max-BQc2Aj-I.js} +4 -4
- package/dist/{mulmat_packed_gpu-D4nKF7Je.js → mulmat_packed_gpu-Gzf3I9UV.js} +1 -1
- package/dist/non_max_suppression_impl-CsEgBuMA.js +134 -0
- package/dist/{ones-BBlSRqn1.js → ones-D63HpSF_.js} +2 -2
- package/dist/ops/appendCache.js +3 -3
- package/dist/ops/attentionMask.js +1 -1
- package/dist/ops/cpu/appendCache.js +8 -8
- package/dist/ops/cpu/attentionMask.js +9 -9
- package/dist/ops/cpu/fusedSoftmax.js +17 -11
- package/dist/ops/cpu/gatherSub.js +7 -7
- package/dist/ops/cpu/gelu.js +13 -13
- package/dist/ops/cpu/matMulGelu.js +36 -24
- package/dist/ops/cpu/matMulMul.js +14 -8
- package/dist/ops/cpu/mulDropout.js +9 -3
- package/dist/ops/cpu/normRMS.js +5 -5
- package/dist/ops/cpu/qkv.js +3 -3
- package/dist/ops/cpu/rope.js +5 -5
- package/dist/ops/cpu/scatterSub.js +11 -11
- package/dist/ops/fusedSoftmax.js +1 -1
- package/dist/ops/gatherSub.js +1 -1
- package/dist/ops/gelu.js +2 -2
- package/dist/ops/grads/attentionMask.js +1 -1
- package/dist/ops/grads/fusedSoftmax.js +2 -2
- package/dist/ops/grads/gelu.js +3 -24
- package/dist/ops/grads/matMulGelu.js +5 -5
- package/dist/ops/grads/normRMS.js +6 -6
- package/dist/ops/grads/qkv.js +1 -1
- package/dist/ops/grads/rope.js +3 -3
- package/dist/ops/matMulGelu.js +1 -1
- package/dist/ops/matMulMul.js +1 -1
- package/dist/ops/mulDrop.js +1 -1
- package/dist/ops/normRMS.js +1 -1
- package/dist/ops/qkv.js +1 -1
- package/dist/ops/rope.js +4 -4
- package/dist/ops/scatterSub.js +1 -1
- package/dist/ops/webgl/appendCache.js +1 -1
- package/dist/ops/webgl/attentionMask.js +1 -1
- package/dist/ops/webgl/fusedSoftmax.js +4 -4
- package/dist/ops/webgl/gatherSub.js +1 -1
- package/dist/ops/webgl/gelu.js +2 -2
- package/dist/ops/webgl/log.js +5 -5
- package/dist/ops/webgl/matMulGelu.js +17 -17
- package/dist/ops/webgl/matMulMul.js +1 -1
- package/dist/ops/webgl/mulDropout.js +4 -4
- package/dist/ops/webgl/normRMS.js +2 -2
- package/dist/ops/webgl/qkv.js +1 -1
- package/dist/ops/webgl/rope.js +1 -1
- package/dist/ops/webgl/scatterSub.js +1 -1
- package/dist/ops/webgpu/appendCache.d.ts +1 -0
- package/dist/ops/webgpu/appendCache.js +56 -0
- package/dist/ops/webgpu/attentionMask.d.ts +1 -0
- package/dist/ops/webgpu/attentionMask.js +64 -0
- package/dist/ops/webgpu/gatherSub.d.ts +1 -0
- package/dist/ops/webgpu/gatherSub.js +37 -0
- package/dist/ops/webgpu/gelu.d.ts +14 -0
- package/dist/ops/webgpu/gelu.js +86 -0
- package/dist/ops/webgpu/index.d.ts +0 -0
- package/dist/ops/webgpu/index.js +8 -0
- package/dist/ops/webgpu/normRMS.d.ts +1 -0
- package/dist/ops/webgpu/normRMS.js +115 -0
- package/dist/ops/webgpu/qkv.d.ts +1 -0
- package/dist/ops/webgpu/qkv.js +56 -0
- package/dist/ops/webgpu/rope.d.ts +1 -0
- package/dist/ops/webgpu/rope.js +68 -0
- package/dist/ops/webgpu/scatterSub.d.ts +1 -0
- package/dist/ops/webgpu/scatterSub.js +37 -0
- package/dist/{ops-C0sQEcPw.js → ops-CIQLNshk.js} +452 -503
- package/dist/{random_width-DWzaOgrn.js → random_width-DkYP8W8N.js} +143 -144
- package/dist/{range-DYsrnfiy.js → range-CYzpQY53.js} +1 -1
- package/dist/{reciprocal-CJQeasVa.js → reciprocal-_A9yv27J.js} +1 -1
- package/dist/{register_all_kernels-BfFCQAqs.js → register_all_kernels-guvSxp7M.js} +202 -200
- package/dist/{reshape-krWGKraP.js → reshape-BMUzc1UY.js} +3 -3
- package/dist/{scatter_nd_util-93ln7Hut.js → scatter_nd_util-IRBqKz_b.js} +3 -3
- package/dist/{selu_util-sntGesxr.js → selu_util-Dt_iuXaq.js} +6 -6
- package/dist/shared-BNa2q6jD.js +69 -0
- package/dist/{shared-Ca6iDobD.js → shared-CDu9S76h.js} +541 -606
- package/dist/{sin-D_h-qCSx.js → sin-Cocju-BY.js} +6 -6
- package/dist/{softmax-fsdtf6JC.js → softmax-GPNK3o-U.js} +3 -3
- package/dist/{split-eiktj-6L.js → split-CHzJjxDv.js} +4 -4
- package/dist/{stack-dfEEz2OY.js → stack-Dpgg_1W1.js} +2 -2
- package/dist/{sum-BE_Irnim.js → sum-B8wEpKsg.js} +5 -5
- package/dist/{tensor-Xyi595sG.js → tensor-RvZVNmg0.js} +1 -1
- package/dist/{tensor2d-CPEkynbH.js → tensor2d-B_kyod7_.js} +1 -1
- package/dist/training/AdamExt.js +1 -1
- package/dist/training/DatasetBuilder.js +2 -2
- package/dist/training/Evaluator.js +1 -1
- package/dist/training/FullTrainer.js +20 -20
- package/dist/training/Trainer.d.ts +5 -6
- package/dist/training/Trainer.js +59 -60
- package/dist/training/sparseCrossEntropy.js +4 -4
- package/dist/utilities/dummy.js +19 -19
- package/dist/utilities/generate.js +15 -16
- package/dist/utilities/multinomialCPU.d.ts +2 -0
- package/dist/utilities/multinomialCPU.js +13 -0
- package/dist/utilities/performance.d.ts +2 -0
- package/dist/utilities/performance.js +16 -0
- package/dist/utilities/profile.d.ts +1 -0
- package/dist/utilities/profile.js +9 -6
- package/dist/utilities/safetensors.js +2 -2
- package/dist/utilities/weights.js +2 -2
- package/dist/{variable-wSS22xj5.js → variable-DXEUOwew.js} +1 -1
- package/dist/webgpu_util-g13LvDIv.js +625 -0
- package/dist/{zeros-YJDE7oRb.js → zeros-DCPCdFGq.js} +8 -8
- package/package.json +2 -1
- package/dist/gpgpu_math-CNslybmD.js +0 -3115
- package/dist/norm-CzltS9Fz.js +0 -86
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { c as h } from "../../concat-
|
|
1
|
+
import { f as d } from "../../index-C0dhsYom.js";
|
|
2
|
+
import { c as h } from "../../concat-CsHeR4zV.js";
|
|
3
3
|
function u(p) {
|
|
4
|
-
const { cache: n, item: s } = p.inputs, { maxSize:
|
|
5
|
-
if (c + e <=
|
|
6
|
-
const
|
|
7
|
-
return
|
|
4
|
+
const { cache: n, item: s } = p.inputs, { maxSize: i, pastLen: c } = p.attrs, t = n.shape[0], o = n.shape[1], a = n.shape[3], e = s.shape[2];
|
|
5
|
+
if (c + e <= i) {
|
|
6
|
+
const l = n.slice([0, 0, 0, 0], [t, o, c, a]), m = n.slice([0, 0, c + e, 0], [t, o, i - c - e, a]), r = e < e ? s.slice([0, 0, 0, 0], [t, o, e, a]) : s, k = h([l, r, m], 2);
|
|
7
|
+
return l.dispose(), m.dispose(), r !== s && r.dispose(), k;
|
|
8
8
|
}
|
|
9
|
-
const
|
|
10
|
-
return
|
|
9
|
+
const f = n.slice([0, 0, e, 0], [t, o, i - e, a]), C = h([f, s], 2);
|
|
10
|
+
return f.dispose(), C;
|
|
11
11
|
}
|
|
12
12
|
const w = {
|
|
13
13
|
kernelName: "AppendCache",
|
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { l as N, w as b } from "../../ops-
|
|
3
|
-
import { o as A } from "../../ones-
|
|
4
|
-
import { z as I } from "../../zeros-
|
|
5
|
-
import { m as g } from "../../mat_mul-
|
|
1
|
+
import { f as a, h as p, b as u } from "../../index-C0dhsYom.js";
|
|
2
|
+
import { l as N, w as b } from "../../ops-CIQLNshk.js";
|
|
3
|
+
import { o as A } from "../../ones-D63HpSF_.js";
|
|
4
|
+
import { z as I } from "../../zeros-DCPCdFGq.js";
|
|
5
|
+
import { m as g } from "../../mat_mul-1nwdPkQ_.js";
|
|
6
6
|
function o(n) {
|
|
7
7
|
const { q: s, k: e } = n.inputs, { divisor: r } = n.attrs, c = s.shape[2], t = e.shape[2], m = N.bandPart(A([t, t]), -1, 0).cast("bool"), l = I([t, t]), i = p([t, t], Number.NEGATIVE_INFINITY), f = b(m, l, i), k = g(s, e, !1, !0).mul(u(r)), d = f.slice([0, 0], [c, t]).expandDims(0).expandDims(0);
|
|
8
8
|
return k.add(d);
|
|
9
9
|
}
|
|
10
|
-
const
|
|
10
|
+
const h = {
|
|
11
11
|
kernelName: "AttentionMask",
|
|
12
12
|
backendName: "cpu",
|
|
13
13
|
kernelFunc: o
|
|
14
14
|
};
|
|
15
|
-
a(
|
|
16
|
-
const
|
|
15
|
+
a(h);
|
|
16
|
+
const w = {
|
|
17
17
|
kernelName: "AttentionMask",
|
|
18
18
|
backendName: "tensorflow",
|
|
19
19
|
kernelFunc: o
|
|
20
20
|
};
|
|
21
|
-
a(
|
|
21
|
+
a(w);
|
|
@@ -1,23 +1,29 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { s as f } from "../../softmax-
|
|
3
|
-
function
|
|
4
|
-
const { inputs: s, attrs:
|
|
1
|
+
import { f as e } from "../../index-C0dhsYom.js";
|
|
2
|
+
import { s as f } from "../../softmax-GPNK3o-U.js";
|
|
3
|
+
function n(t) {
|
|
4
|
+
const { inputs: s, attrs: a } = t, { logits: o } = s, { dim: i, dropoutRate: r } = a;
|
|
5
5
|
if (!o)
|
|
6
6
|
throw new Error("Error in softmax: input logits is null");
|
|
7
|
-
return
|
|
7
|
+
return r !== void 0 && r > 0 && console.warn("Dropout in fusedSoftmax not implemented for CPU backend, skipping dropout."), f(o, i);
|
|
8
8
|
}
|
|
9
9
|
const m = {
|
|
10
10
|
kernelName: "FusedSoftmax",
|
|
11
11
|
backendName: "cpu",
|
|
12
|
-
kernelFunc:
|
|
12
|
+
kernelFunc: n
|
|
13
13
|
};
|
|
14
|
-
|
|
15
|
-
const
|
|
14
|
+
e(m);
|
|
15
|
+
const u = {
|
|
16
16
|
kernelName: "FusedSoftmax",
|
|
17
17
|
backendName: "tensorflow",
|
|
18
|
-
kernelFunc:
|
|
18
|
+
kernelFunc: n
|
|
19
|
+
};
|
|
20
|
+
e(u);
|
|
21
|
+
const l = {
|
|
22
|
+
kernelName: "FusedSoftmax",
|
|
23
|
+
backendName: "webgpu",
|
|
24
|
+
kernelFunc: n
|
|
19
25
|
};
|
|
20
|
-
|
|
26
|
+
e(l);
|
|
21
27
|
export {
|
|
22
|
-
|
|
28
|
+
n as softmaxCPU
|
|
23
29
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { r as
|
|
3
|
-
import { s as
|
|
1
|
+
import { x as u, y as c, E as g, $ as h, f as m, c as p } from "../../index-C0dhsYom.js";
|
|
2
|
+
import { r as f } from "../../range-CYzpQY53.js";
|
|
3
|
+
import { s as l } from "../../stack-Dpgg_1W1.js";
|
|
4
4
|
/**
|
|
5
5
|
* @license
|
|
6
6
|
* Copyright 2018 Google LLC. All Rights Reserved.
|
|
@@ -17,14 +17,14 @@ import { s as N } from "../../stack-dfEEz2OY.js";
|
|
|
17
17
|
* limitations under the License.
|
|
18
18
|
* =============================================================================
|
|
19
19
|
*/
|
|
20
|
-
function
|
|
20
|
+
function N(e, t) {
|
|
21
21
|
const n = c(t, "indices", "gatherND", "int32"), s = { params: c(e, "x", "gatherND", "string_or_numeric"), indices: n };
|
|
22
22
|
return g.runKernel(h, s);
|
|
23
23
|
}
|
|
24
|
-
const
|
|
24
|
+
const b = /* @__PURE__ */ u({ gatherND_: N });
|
|
25
25
|
function d(e) {
|
|
26
|
-
const { values: t, labels: n, logits: r } = e.inputs, s = n.shape[0],
|
|
27
|
-
return p(t,
|
|
26
|
+
const { values: t, labels: n, logits: r } = e.inputs, s = n.shape[0], a = f(0, s, 1, "int32"), i = l([a, n], 1), o = b(r, i);
|
|
27
|
+
return p(t, o);
|
|
28
28
|
}
|
|
29
29
|
const k = {
|
|
30
30
|
kernelName: "EfficientGatherSub",
|
package/dist/ops/cpu/gelu.js
CHANGED
|
@@ -1,29 +1,29 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { f as t, t as d } from "../../index-C0dhsYom.js";
|
|
2
2
|
const o = 0.7978845608028654, c = 0.044715;
|
|
3
|
-
function m(
|
|
4
|
-
const { inputs:
|
|
3
|
+
function m(r) {
|
|
4
|
+
const { inputs: u } = r, { x: n } = u, e = n;
|
|
5
5
|
return d(() => {
|
|
6
|
-
const
|
|
7
|
-
return
|
|
6
|
+
const l = e.pow(3), s = e.add(l.mul(c)).mul(o).tanh().add(1).mul(0.5);
|
|
7
|
+
return e.mul(s);
|
|
8
8
|
});
|
|
9
9
|
}
|
|
10
|
-
const
|
|
10
|
+
const p = {
|
|
11
11
|
kernelName: "Gelu",
|
|
12
12
|
backendName: "cpu",
|
|
13
13
|
kernelFunc: m
|
|
14
14
|
};
|
|
15
|
-
t(
|
|
16
|
-
const
|
|
15
|
+
t(p);
|
|
16
|
+
const K = {
|
|
17
17
|
kernelName: "Gelu",
|
|
18
18
|
backendName: "tensorflow",
|
|
19
19
|
kernelFunc: m
|
|
20
20
|
};
|
|
21
|
-
t(
|
|
22
|
-
function i(
|
|
23
|
-
const { dy:
|
|
21
|
+
t(K);
|
|
22
|
+
function i(r) {
|
|
23
|
+
const { dy: u, x: n } = r.inputs;
|
|
24
24
|
return d(() => {
|
|
25
|
-
const
|
|
26
|
-
return
|
|
25
|
+
const e = n.square(), l = e.mul(n), a = n.add(l.mul(c)).mul(o).tanh(), f = a.square().neg().add(1), k = e.mul(3 * c).add(1), g = a.add(1).mul(0.5), G = n.mul(f).mul(o).mul(k).mul(0.5), N = g.add(G);
|
|
26
|
+
return u.mul(N);
|
|
27
27
|
});
|
|
28
28
|
}
|
|
29
29
|
const x = {
|
|
@@ -1,40 +1,52 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
function
|
|
4
|
-
const { inputs:
|
|
5
|
-
return
|
|
6
|
-
const
|
|
7
|
-
return
|
|
1
|
+
import { f as e, t as m } from "../../index-C0dhsYom.js";
|
|
2
|
+
import { g as d, d as M } from "../../gelu-uyHP1x1f.js";
|
|
3
|
+
function c(t) {
|
|
4
|
+
const { inputs: u } = t, { x: n, kernel: l } = u, a = n, r = l;
|
|
5
|
+
return m(() => {
|
|
6
|
+
const o = a.matMul(r);
|
|
7
|
+
return d(o);
|
|
8
8
|
});
|
|
9
9
|
}
|
|
10
|
-
const
|
|
10
|
+
const G = {
|
|
11
11
|
kernelName: "MatMulGelu",
|
|
12
12
|
backendName: "cpu",
|
|
13
|
-
kernelFunc:
|
|
13
|
+
kernelFunc: c
|
|
14
14
|
};
|
|
15
|
-
|
|
16
|
-
const
|
|
15
|
+
e(G);
|
|
16
|
+
const i = {
|
|
17
17
|
kernelName: "MatMulGelu",
|
|
18
18
|
backendName: "tensorflow",
|
|
19
|
-
kernelFunc:
|
|
19
|
+
kernelFunc: c
|
|
20
20
|
};
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
21
|
+
e(i);
|
|
22
|
+
const f = {
|
|
23
|
+
kernelName: "MatMulGelu",
|
|
24
|
+
backendName: "webgpu",
|
|
25
|
+
kernelFunc: c
|
|
26
|
+
};
|
|
27
|
+
e(f);
|
|
28
|
+
function s(t) {
|
|
29
|
+
const { dy: u, x: n, kernel: l } = t.inputs;
|
|
30
|
+
return m(() => {
|
|
31
|
+
const a = n.matMul(l), r = M(u, a), o = r.matMul(l.transpose()), k = n.transpose().matMul(r);
|
|
32
|
+
return [o, k];
|
|
27
33
|
});
|
|
28
34
|
}
|
|
29
|
-
const
|
|
35
|
+
const g = {
|
|
30
36
|
kernelName: "MatMulGeluGrad",
|
|
31
37
|
backendName: "cpu",
|
|
32
|
-
kernelFunc:
|
|
38
|
+
kernelFunc: s
|
|
33
39
|
};
|
|
34
|
-
|
|
35
|
-
const
|
|
40
|
+
e(g);
|
|
41
|
+
const p = {
|
|
36
42
|
kernelName: "MatMulGeluGrad",
|
|
37
43
|
backendName: "tensorflow",
|
|
38
|
-
kernelFunc:
|
|
44
|
+
kernelFunc: s
|
|
45
|
+
};
|
|
46
|
+
e(p);
|
|
47
|
+
const N = {
|
|
48
|
+
kernelName: "MatMulGeluGrad",
|
|
49
|
+
backendName: "webgpu",
|
|
50
|
+
kernelFunc: s
|
|
39
51
|
};
|
|
40
|
-
|
|
52
|
+
e(N);
|
|
@@ -1,17 +1,23 @@
|
|
|
1
|
-
import {
|
|
2
|
-
function
|
|
3
|
-
const { inputs: r, attrs:
|
|
4
|
-
return
|
|
1
|
+
import { f as e, t as i } from "../../index-C0dhsYom.js";
|
|
2
|
+
function n(t) {
|
|
3
|
+
const { inputs: r, attrs: l } = t, { transposeA: o, transposeB: s } = l, { x: u, kernel: a, y: c } = r, m = u, k = a, M = c;
|
|
4
|
+
return i(() => m.matMul(k, o, s).mul(M));
|
|
5
5
|
}
|
|
6
6
|
const f = {
|
|
7
7
|
kernelName: "MatMulMul",
|
|
8
8
|
backendName: "cpu",
|
|
9
|
-
kernelFunc:
|
|
9
|
+
kernelFunc: n
|
|
10
10
|
};
|
|
11
|
-
|
|
11
|
+
e(f);
|
|
12
12
|
const p = {
|
|
13
13
|
kernelName: "MatMulMul",
|
|
14
14
|
backendName: "tensorflow",
|
|
15
|
-
kernelFunc:
|
|
15
|
+
kernelFunc: n
|
|
16
16
|
};
|
|
17
|
-
|
|
17
|
+
e(p);
|
|
18
|
+
const g = {
|
|
19
|
+
kernelName: "MatMulMul",
|
|
20
|
+
backendName: "webgpu",
|
|
21
|
+
kernelFunc: n
|
|
22
|
+
};
|
|
23
|
+
e(g);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { f as e, a as t } from "../../index-C0dhsYom.js";
|
|
2
2
|
function n(o) {
|
|
3
|
-
const { inputs: r } = o, { a: l, b:
|
|
4
|
-
return console.warn("Using fallback mulDrop implementation without dropout."),
|
|
3
|
+
const { inputs: r } = o, { a: l, b: u } = r;
|
|
4
|
+
return console.warn("Using fallback mulDrop implementation without dropout."), t(l, u);
|
|
5
5
|
}
|
|
6
6
|
const a = {
|
|
7
7
|
kernelName: "MulDropout",
|
|
@@ -15,3 +15,9 @@ const c = {
|
|
|
15
15
|
kernelFunc: n
|
|
16
16
|
};
|
|
17
17
|
e(c);
|
|
18
|
+
const m = {
|
|
19
|
+
kernelName: "MulDropout",
|
|
20
|
+
backendName: "webgpu",
|
|
21
|
+
kernelFunc: n
|
|
22
|
+
};
|
|
23
|
+
e(m);
|
package/dist/ops/cpu/normRMS.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { f as o, t as d } from "../../index-C0dhsYom.js";
|
|
2
2
|
function i(t) {
|
|
3
3
|
const { inputs: e } = t, { x: n, gamma: s } = e, r = n, a = s;
|
|
4
4
|
return d(() => {
|
|
@@ -6,12 +6,12 @@ function i(t) {
|
|
|
6
6
|
return r.mul(u).mul(a);
|
|
7
7
|
});
|
|
8
8
|
}
|
|
9
|
-
const
|
|
9
|
+
const k = {
|
|
10
10
|
kernelName: "RMSNorm",
|
|
11
11
|
backendName: "cpu",
|
|
12
12
|
kernelFunc: i
|
|
13
13
|
};
|
|
14
|
-
o(
|
|
14
|
+
o(k);
|
|
15
15
|
const g = {
|
|
16
16
|
kernelName: "RMSNorm",
|
|
17
17
|
backendName: "tensorflow",
|
|
@@ -21,8 +21,8 @@ o(g);
|
|
|
21
21
|
function N(t) {
|
|
22
22
|
const { dy: e, x: n, gamma: s } = t.inputs;
|
|
23
23
|
return d(() => {
|
|
24
|
-
const r = n.shape[n.shape.length - 1], a = n.square().mean(-1, !0), m = a.add(1e-8).rsqrt(), u = n.mul(m), l = e.mul(u).sum([0, 1]), c = e.mul(s),
|
|
25
|
-
return [c.mul(m).sub(n.mul(
|
|
24
|
+
const r = n.shape[n.shape.length - 1], a = n.square().mean(-1, !0), m = a.add(1e-8).rsqrt(), u = n.mul(m), l = e.mul(u).sum([0, 1]), c = e.mul(s), f = c.mul(n).sum(-1, !0).div(r);
|
|
25
|
+
return [c.mul(m).sub(n.mul(f).mul(m).div(a.add(1e-8))), l];
|
|
26
26
|
});
|
|
27
27
|
}
|
|
28
28
|
const S = {
|
package/dist/ops/cpu/qkv.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { r as o } from "../../reshape-
|
|
3
|
-
import { s as x } from "../../split-
|
|
1
|
+
import { f as q } from "../../index-C0dhsYom.js";
|
|
2
|
+
import { r as o } from "../../reshape-BMUzc1UY.js";
|
|
3
|
+
import { s as x } from "../../split-CHzJjxDv.js";
|
|
4
4
|
function v(p) {
|
|
5
5
|
const { x: c, kernel: K } = p.inputs, { heads: n } = p.attrs, [s, e, t] = c.shape, a = o(c, [s * e, t]), i = a.dot(K);
|
|
6
6
|
a.dispose();
|
package/dist/ops/cpu/rope.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { r as F } from "../../range-
|
|
3
|
-
import { g as I } from "../../gather-
|
|
4
|
-
import { s as E } from "../../stack-
|
|
5
|
-
import { c as T } from "../../concat-
|
|
1
|
+
import { f as S } from "../../index-C0dhsYom.js";
|
|
2
|
+
import { r as F } from "../../range-CYzpQY53.js";
|
|
3
|
+
import { g as I } from "../../gather-D0_gPiBz.js";
|
|
4
|
+
import { s as E } from "../../stack-Dpgg_1W1.js";
|
|
5
|
+
import { c as T } from "../../concat-CsHeR4zV.js";
|
|
6
6
|
function U(t, c, p, o, r) {
|
|
7
7
|
const n = o.shape[3], s = p;
|
|
8
8
|
if (s > n) return o;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { v as D } from "../../scatter_nd_util-
|
|
3
|
-
import { r as k } from "../../range-
|
|
4
|
-
import { s as v } from "../../stack-
|
|
5
|
-
import { o as E } from "../../ones-
|
|
1
|
+
import { x as f, C as g, y as r, E as l, _ as N, f as b, c as S, a as h } from "../../index-C0dhsYom.js";
|
|
2
|
+
import { v as D } from "../../scatter_nd_util-IRBqKz_b.js";
|
|
3
|
+
import { r as k } from "../../range-CYzpQY53.js";
|
|
4
|
+
import { s as v } from "../../stack-Dpgg_1W1.js";
|
|
5
|
+
import { o as E } from "../../ones-D63HpSF_.js";
|
|
6
6
|
/**
|
|
7
7
|
* @license
|
|
8
8
|
* Copyright 2018 Google LLC. All Rights Reserved.
|
|
@@ -26,14 +26,14 @@ function I(a, e, s) {
|
|
|
26
26
|
const c = { indices: n, updates: t }, o = { shape: s };
|
|
27
27
|
return l.runKernel(N, c, o);
|
|
28
28
|
}
|
|
29
|
-
const
|
|
30
|
-
function
|
|
31
|
-
const { logits: e, labels: s, dy: n } = a.inputs, t = s.shape[0], c = e.shape[1], o = k(0, t, 1, "int32"), i = v([o, s], 1), d = E([t]), u =
|
|
29
|
+
const y = /* @__PURE__ */ f({ scatterND_: I });
|
|
30
|
+
function C(a) {
|
|
31
|
+
const { logits: e, labels: s, dy: n } = a.inputs, t = s.shape[0], c = e.shape[1], o = k(0, t, 1, "int32"), i = v([o, s], 1), d = E([t]), u = y(i, d, [t, c]), p = S(e, u), m = n.reshape([t, 1]);
|
|
32
32
|
return h(p, m);
|
|
33
33
|
}
|
|
34
|
-
const
|
|
34
|
+
const K = {
|
|
35
35
|
kernelName: "EfficientScatterSub",
|
|
36
36
|
backendName: "cpu",
|
|
37
|
-
kernelFunc:
|
|
37
|
+
kernelFunc: C
|
|
38
38
|
};
|
|
39
|
-
b(
|
|
39
|
+
b(K);
|
package/dist/ops/fusedSoftmax.js
CHANGED
package/dist/ops/gatherSub.js
CHANGED
package/dist/ops/gelu.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { i as f, a as i, c as l } from "../../index-C0dhsYom.js";
|
|
2
2
|
import { mulDrop as g } from "../mulDrop.js";
|
|
3
|
-
import { s as T } from "../../sum-
|
|
3
|
+
import { s as T } from "../../sum-B8wEpKsg.js";
|
|
4
4
|
const Y = {
|
|
5
5
|
kernelName: "FusedSoftmax",
|
|
6
6
|
outputsToSave: [!0],
|
package/dist/ops/grads/gelu.js
CHANGED
|
@@ -1,26 +1,5 @@
|
|
|
1
|
-
import
|
|
2
|
-
import "
|
|
3
|
-
import "../webgl/gelu.js";
|
|
4
|
-
const o = {
|
|
5
|
-
kernelName: "Gelu",
|
|
6
|
-
inputsToSave: ["x"],
|
|
7
|
-
outputsToSave: [],
|
|
8
|
-
gradFunc: (e, r) => {
|
|
9
|
-
const [u] = r;
|
|
10
|
-
return {
|
|
11
|
-
x: () => a(e, u)
|
|
12
|
-
};
|
|
13
|
-
}
|
|
14
|
-
};
|
|
15
|
-
t(o);
|
|
16
|
-
function g(e) {
|
|
17
|
-
return n().runKernel("Gelu", { x: e });
|
|
18
|
-
}
|
|
19
|
-
function a(e, r) {
|
|
20
|
-
return n().runKernel("GeluGrad", { dy: e, x: r });
|
|
21
|
-
}
|
|
1
|
+
import "../../index-C0dhsYom.js";
|
|
2
|
+
import { a as e } from "../../gelu-uyHP1x1f.js";
|
|
22
3
|
export {
|
|
23
|
-
|
|
24
|
-
g,
|
|
25
|
-
o as geluGradConfig
|
|
4
|
+
e as geluGradConfig
|
|
26
5
|
};
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import {
|
|
2
|
-
function
|
|
1
|
+
import { i as a, e as o } from "../../index-C0dhsYom.js";
|
|
2
|
+
function i(e, n, r) {
|
|
3
3
|
return o().runKernel("MatMulGeluGrad", { dy: e, x: n, kernel: r });
|
|
4
4
|
}
|
|
5
|
-
const
|
|
5
|
+
const s = {
|
|
6
6
|
kernelName: "MatMulGelu",
|
|
7
7
|
inputsToSave: ["x", "kernel"],
|
|
8
8
|
outputsToSave: [],
|
|
9
9
|
gradFunc: (e, n) => {
|
|
10
|
-
const [r, t] = n, [u, l] =
|
|
10
|
+
const [r, t] = n, [u, l] = i(e, r, t);
|
|
11
11
|
return {
|
|
12
12
|
x: () => u,
|
|
13
13
|
kernel: () => l
|
|
14
14
|
};
|
|
15
15
|
}
|
|
16
16
|
};
|
|
17
|
-
a(
|
|
17
|
+
a(s);
|
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
import {
|
|
2
|
-
function
|
|
1
|
+
import { i as t, e as g } from "../../index-C0dhsYom.js";
|
|
2
|
+
function i(r, a, n) {
|
|
3
3
|
return g().runKernel("RMSNormGrad", { dy: r, x: a, gamma: n });
|
|
4
4
|
}
|
|
5
|
-
const
|
|
5
|
+
const s = {
|
|
6
6
|
kernelName: "RMSNorm",
|
|
7
7
|
inputsToSave: ["x", "gamma"],
|
|
8
8
|
outputsToSave: [],
|
|
9
9
|
gradFunc: (r, a) => {
|
|
10
|
-
const [n, e] = a, [m, o] =
|
|
10
|
+
const [n, e] = a, [m, o] = i(r, n, e);
|
|
11
11
|
return {
|
|
12
12
|
x: () => m,
|
|
13
13
|
gamma: () => o
|
|
14
14
|
};
|
|
15
15
|
}
|
|
16
16
|
};
|
|
17
|
-
t(
|
|
17
|
+
t(s);
|
|
18
18
|
export {
|
|
19
|
-
|
|
19
|
+
s as normRMSGradConfig
|
|
20
20
|
};
|
package/dist/ops/grads/qkv.js
CHANGED
package/dist/ops/grads/rope.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { i, e as a } from "../../index-C0dhsYom.js";
|
|
2
2
|
function p(n, e, s, o) {
|
|
3
|
-
return
|
|
3
|
+
return a().runKernel("Rope", { x: n, sin: e, cos: s }, { pastLen: o });
|
|
4
4
|
}
|
|
5
5
|
const c = {
|
|
6
6
|
kernelName: "Rope",
|
|
@@ -11,4 +11,4 @@ const c = {
|
|
|
11
11
|
return t.dispose(), { x: () => r };
|
|
12
12
|
}
|
|
13
13
|
};
|
|
14
|
-
|
|
14
|
+
i(c);
|
package/dist/ops/matMulGelu.js
CHANGED
package/dist/ops/matMulMul.js
CHANGED
package/dist/ops/mulDrop.js
CHANGED
package/dist/ops/normRMS.js
CHANGED
package/dist/ops/qkv.js
CHANGED
package/dist/ops/rope.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { e as p } from "../index-
|
|
2
|
-
import "../random_width-
|
|
3
|
-
import "../register_all_kernels-
|
|
1
|
+
import { e as p } from "../index-C0dhsYom.js";
|
|
2
|
+
import "../random_width-DkYP8W8N.js";
|
|
3
|
+
import "../register_all_kernels-guvSxp7M.js";
|
|
4
4
|
import "../index-Tf7vU29b.js";
|
|
5
|
-
import "../dataset-
|
|
5
|
+
import "../dataset-JDyjG3QR.js";
|
|
6
6
|
import "./cpu/rope.js";
|
|
7
7
|
import "./webgl/rope.js";
|
|
8
8
|
import "./grads/rope.js";
|
package/dist/ops/scatterSub.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { m as b, s as I, r as k } from "../../RealDiv-
|
|
2
|
-
import { r as v } from "../../Reshape-
|
|
3
|
-
import {
|
|
4
|
-
import { e as S } from "../../axis_util-
|
|
1
|
+
import { m as b, s as I, r as k } from "../../RealDiv-C4hOvYOZ.js";
|
|
2
|
+
import { r as v } from "../../Reshape-BLijOA8h.js";
|
|
3
|
+
import { f as w, p as P } from "../../index-C0dhsYom.js";
|
|
4
|
+
import { e as S } from "../../axis_util-DaAl5MER.js";
|
|
5
5
|
class T {
|
|
6
6
|
variableNames = ["logits", "maxLogits"];
|
|
7
7
|
outputShape;
|