@genai-fi/nanogpt 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Generator.js +3 -3
- package/dist/NanoGPTModel.js +83 -70
- package/dist/TeachableLLM.js +1 -1
- package/dist/{random_width-CMHmdbSu.js → TiedEmbedding-CnJ1bx4q.js} +760 -719
- package/dist/{axis_util-DeydwOoC.js → axis_util-BgTGy5w8.js} +1 -1
- package/dist/{concat-DS_qH7MI.js → concat-CuRsVY-K.js} +1 -1
- package/dist/dropout-DfDdklfL.js +193 -0
- package/dist/{gather-BUmJIS8n.js → gather-ZYRWhmXR.js} +1 -1
- package/dist/gelu-CnCt17Lk.js +26 -0
- package/dist/{index-XjBAhiFO.js → index-C4JCoBvj.js} +61 -61
- package/dist/kernel_funcs_utils-CAd1h9X1.js +388 -0
- package/dist/layers/CausalSelfAttention.js +71 -70
- package/dist/layers/MLP.d.ts +3 -1
- package/dist/layers/MLP.js +93 -5
- package/dist/layers/RMSNorm.js +3 -3
- package/dist/layers/RoPECache.js +3 -3
- package/dist/layers/TiedEmbedding.js +6 -46
- package/dist/layers/TransformerBlock.js +2 -2
- package/dist/{log_sum_exp-DJPkVZZn.js → log_sum_exp-BswFnwOb.js} +5 -5
- package/dist/main.js +1 -1
- package/dist/{mat_mul-CKwFEV1Q.js → mat_mul-415y5Qn2.js} +1 -1
- package/dist/{max-DJvEiCAJ.js → max-CP_9O2Yd.js} +1 -1
- package/dist/{moments-CrWRPcR3.js → moments-CjeIaVdp.js} +3 -3
- package/dist/{norm-BzY929B_.js → norm-CZM380I3.js} +5 -5
- package/dist/{ones-BO01zpJG.js → ones-Bf3YR48P.js} +2 -2
- package/dist/ops/appendCache.js +1 -1
- package/dist/ops/attentionMask.d.ts +1 -1
- package/dist/ops/attentionMask.js +4 -4
- package/dist/ops/cpu/appendCache.js +2 -2
- package/dist/ops/cpu/attentionMask.js +13 -9
- package/dist/ops/cpu/fusedSoftmax.js +2 -2
- package/dist/ops/cpu/gatherSub.js +3 -3
- package/dist/ops/cpu/gelu.d.ts +1 -0
- package/dist/ops/cpu/gelu.js +40 -0
- package/dist/ops/cpu/mulDropout.js +1 -1
- package/dist/ops/cpu/qkv.js +3 -3
- package/dist/ops/cpu/rope.js +5 -5
- package/dist/ops/cpu/scatterSub.js +4 -4
- package/dist/ops/fusedSoftmax.js +1 -1
- package/dist/ops/gatherSub.js +1 -1
- package/dist/ops/gelu.d.ts +3 -0
- package/dist/ops/gelu.js +8 -0
- package/dist/ops/grads/attentionMask.js +1 -1
- package/dist/ops/grads/fusedSoftmax.js +2 -2
- package/dist/ops/grads/gelu.d.ts +2 -0
- package/dist/ops/grads/gelu.js +5 -0
- package/dist/ops/grads/qkv.js +1 -1
- package/dist/ops/grads/rope.js +1 -1
- package/dist/ops/mulDrop.js +1 -1
- package/dist/ops/node/sparseCrossEntropy.js +1 -1
- package/dist/ops/qkv.js +1 -1
- package/dist/ops/scatterSub.js +1 -1
- package/dist/ops/webgl/appendCache.js +1 -1
- package/dist/ops/webgl/attentionMask.js +19 -18
- package/dist/ops/webgl/fusedSoftmax.js +483 -782
- package/dist/ops/webgl/gatherSub.js +1 -1
- package/dist/ops/webgl/gelu.d.ts +2 -0
- package/dist/ops/webgl/gelu.js +50 -0
- package/dist/ops/webgl/mulDropout.js +1 -1
- package/dist/ops/webgl/qkv.js +1 -1
- package/dist/ops/webgl/rope.js +1 -1
- package/dist/ops/webgl/scatterSub.js +1 -1
- package/dist/{range-DQMNzBWs.js → range-9AzeApCc.js} +1 -1
- package/dist/{reshape-DFzh97Sc.js → reshape-Boe4DuIO.js} +1 -1
- package/dist/{sin-BYM-U4Ut.js → sin-KmhiDuMa.js} +1 -1
- package/dist/{slice_util-CnVNPQI-.js → slice_util-19zDNNSn.js} +2 -2
- package/dist/{softmax-4DOn6cPq.js → softmax-Cujsg4ay.js} +1 -1
- package/dist/{split-CkbeVdF8.js → split-DbcNm1-i.js} +1 -1
- package/dist/{stack-DaIMO5iX.js → stack-D1YjmgKN.js} +1 -1
- package/dist/{sum-C6u3xMi3.js → sum-R28pucR5.js} +1 -1
- package/dist/{tensor-Cu1fU7H7.js → tensor-BVeHdl7V.js} +1 -1
- package/dist/{tensor2d-D0CKdG6B.js → tensor2d-DqFGNs_K.js} +1 -1
- package/dist/{tfjs_backend-Bzl2SrRo.js → tfjs_backend-Cug-PH75.js} +826 -1015
- package/dist/training/AdamExt.js +1 -1
- package/dist/training/DatasetBuilder.js +3 -3
- package/dist/training/FullTrainer.js +1 -1
- package/dist/training/Trainer.js +5 -5
- package/dist/training/sparseCrossEntropy.js +4 -4
- package/dist/utilities/dummy.js +2 -2
- package/dist/utilities/generate.js +3 -3
- package/dist/utilities/load.js +1 -1
- package/dist/utilities/profile.js +1 -1
- package/dist/utilities/weights.js +2 -2
- package/dist/{variable-BS4AKqNU.js → variable-LJT9Ld63.js} +1 -1
- package/dist/{zeros-CmJFiC84.js → zeros-dnQxFgAD.js} +1 -1
- package/package.json +1 -1
- package/dist/MLP-KHhikThU.js +0 -83
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { r as a } from "../../index-C4JCoBvj.js";
|
|
2
|
+
import { u as s, C as x } from "../../kernel_funcs_utils-CAd1h9X1.js";
|
|
3
|
+
const t = 0.7978845608028654, r = 0.044715, c = x + `
|
|
4
|
+
float x3 = x * x * x;
|
|
5
|
+
float inner = x + ${r} * x3;
|
|
6
|
+
inner = ${t} * inner;
|
|
7
|
+
inner = tanh(inner);
|
|
8
|
+
inner = 0.5 * (1.0 + inner);
|
|
9
|
+
x = x * inner;
|
|
10
|
+
return x;
|
|
11
|
+
`, d = s({ opSnippet: c }), i = {
|
|
12
|
+
kernelName: "Gelu",
|
|
13
|
+
backendName: "webgl",
|
|
14
|
+
kernelFunc: d
|
|
15
|
+
};
|
|
16
|
+
a(i);
|
|
17
|
+
class f {
|
|
18
|
+
// Inputs: dy, x
|
|
19
|
+
variableNames = ["dy", "x"];
|
|
20
|
+
outputShape;
|
|
21
|
+
userCode;
|
|
22
|
+
constructor(n) {
|
|
23
|
+
this.outputShape = n, this.userCode = `
|
|
24
|
+
void main() {
|
|
25
|
+
float dy = getDyAtOutCoords();
|
|
26
|
+
float x = getXAtOutCoords();
|
|
27
|
+
float x2 = x * x;
|
|
28
|
+
float x3 = x2 * x;
|
|
29
|
+
float u = ${t} * (x + ${r} * x3);
|
|
30
|
+
float t = tanh(u);
|
|
31
|
+
float sech2 = 1.0 - t * t;
|
|
32
|
+
float du_dx = ${t} * (1.0 + 3.0 * ${r} * x2);
|
|
33
|
+
float dgelu = 0.5 * (1.0 + t) + 0.5 * x * sech2 * du_dx;
|
|
34
|
+
setOutput(dy * dgelu);
|
|
35
|
+
}`;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
function g(e) {
|
|
39
|
+
const { dy: n, x: o } = e.inputs, u = e.backend, l = new f(o.shape);
|
|
40
|
+
return u.runWebGLProgram(l, [n, o], "float32");
|
|
41
|
+
}
|
|
42
|
+
const p = {
|
|
43
|
+
kernelName: "GeluGrad",
|
|
44
|
+
backendName: "webgl",
|
|
45
|
+
kernelFunc: g
|
|
46
|
+
};
|
|
47
|
+
a(p);
|
|
48
|
+
export {
|
|
49
|
+
d as gelu
|
|
50
|
+
};
|
package/dist/ops/webgl/qkv.js
CHANGED
package/dist/ops/webgl/rope.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { o as u, h as p, k as g, w as m, E as w, a1 as x, j as i } from "./index-
|
|
2
|
-
import { r as y } from "./reshape-
|
|
1
|
+
import { o as u, h as p, k as g, w as m, E as w, a1 as x, j as i } from "./index-C4JCoBvj.js";
|
|
2
|
+
import { r as y } from "./reshape-Boe4DuIO.js";
|
|
3
3
|
/**
|
|
4
4
|
* @license
|
|
5
5
|
* Copyright 2020 Google LLC. All Rights Reserved.
|