@genai-fi/nanogpt 0.19.1 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/BaseTokeniser-DSg9zcYq.js +221 -0
- package/dist/DatasetBuilder-DgURD85T.js +712 -0
- package/dist/Generator.js +2 -11941
- package/dist/RealDiv-DBu0FQqT.js +362 -0
- package/dist/Reshape-CABOPB9d.js +94 -0
- package/dist/Reshape-DqO3r8BC.js +17 -0
- package/dist/TeachableLLM.d.ts +5 -5
- package/dist/TeachableLLM.js +2 -273
- package/dist/Trainer.js +2 -244
- package/dist/backend.js +12 -12
- package/dist/backend_util-Cg-roD1p.js +399 -0
- package/dist/binary_op_util-CrYk9LXL.js +103 -0
- package/dist/checks/appendCache.js +54 -21
- package/dist/checks/attentionMask.js +55 -36
- package/dist/checks/check.js +31 -19
- package/dist/checks/gelu.js +45 -17
- package/dist/checks/index.js +25 -25
- package/dist/checks/matMulGelu.js +83 -27
- package/dist/checks/normRMS.js +27 -15
- package/dist/checks/normRMSGrad.js +21 -11
- package/dist/checks/packUnpack.js +45 -17
- package/dist/checks/qkv.js +33 -33
- package/dist/checks/rope.js +29 -35
- package/dist/checks/weights.d.ts +1 -1
- package/dist/checks/weights.js +25 -29
- package/dist/chunk-BPntVaq0.js +23 -0
- package/dist/complex_util-CkazZsaH.js +60 -0
- package/dist/concat_util-CWDZCBlA.js +19 -0
- package/dist/data/docx.js +3044 -13
- package/dist/data/pdf.js +16 -13
- package/dist/data/textLoader.js +607 -112
- package/dist/dist-BewPQWjc.js +7572 -0
- package/dist/dist-DVmq73nz.js +8775 -0
- package/dist/dist-DXwIvKxl.js +896 -0
- package/dist/dist-VEU5mfO0.js +7545 -0
- package/dist/gelu-Bf1HW1RY.js +27 -0
- package/dist/gpgpu_math-DvLcCH6u.js +1612 -0
- package/dist/inference/types.js +0 -1
- package/dist/kernel_funcs_utils-HiXOOx3f.js +229 -0
- package/dist/layers/BaseLayer.d.ts +2 -2
- package/dist/layers/BaseLayer.js +75 -73
- package/dist/layers/CausalSelfAttention.d.ts +1 -1
- package/dist/layers/CausalSelfAttention.js +98 -85
- package/dist/layers/LoRA.js +47 -57
- package/dist/layers/MLP.d.ts +1 -1
- package/dist/layers/MLP.js +33 -43
- package/dist/layers/PositionEmbedding.d.ts +1 -1
- package/dist/layers/PositionEmbedding.js +26 -30
- package/dist/layers/RMSNorm.d.ts +1 -1
- package/dist/layers/RMSNorm.js +19 -21
- package/dist/layers/RoPECache.js +336 -49
- package/dist/layers/TiedEmbedding.d.ts +1 -1
- package/dist/layers/TiedEmbedding.js +30 -34
- package/dist/layers/TransformerBlock.d.ts +1 -1
- package/dist/layers/TransformerBlock.js +50 -39
- package/dist/layers/WeightStore.js +68 -75
- package/dist/loader/load.js +2 -68
- package/dist/loader/loadHF.d.ts +2 -2
- package/dist/loader/loadHF.js +2 -22
- package/dist/loader/loadTransformers.d.ts +1 -1
- package/dist/loader/loadTransformers.js +2 -44
- package/dist/loader/loadZipMeta.js +15 -15
- package/dist/loader/newZipLoad.js +2 -31
- package/dist/loader/oldZipLoad.d.ts +2 -2
- package/dist/loader/oldZipLoad.js +2 -80
- package/dist/loader/save.d.ts +5 -5
- package/dist/loader/save.js +2 -90
- package/dist/loader/types.d.ts +9 -8
- package/dist/loader/types.js +0 -1
- package/dist/main-CPjeMv0G.js +13500 -0
- package/dist/main.d.ts +1 -1
- package/dist/main.js +16 -109
- package/dist/matMul16-BNfZSnNM.js +81 -0
- package/dist/matMulGelu-CPTntosE.js +162 -0
- package/dist/models/NanoGPTV1.js +2 -99
- package/dist/models/NanoGPTV2.js +2 -90
- package/dist/models/config.js +34 -47
- package/dist/models/factory.d.ts +1 -1
- package/dist/models/factory.js +2 -16
- package/dist/models/model.d.ts +2 -2
- package/dist/models/model.js +2 -134
- package/dist/ops/adamAdjust.js +15 -6
- package/dist/ops/adamMoments.js +13 -6
- package/dist/ops/add16.js +9 -6
- package/dist/ops/appendCache.js +22 -19
- package/dist/ops/attentionMask.js +12 -6
- package/dist/ops/concat16.js +7 -8
- package/dist/ops/cpu/adamAdjust.js +15 -17
- package/dist/ops/cpu/adamMoments.js +15 -15
- package/dist/ops/cpu/appendCache.js +64 -22
- package/dist/ops/cpu/attentionMask.js +15 -21
- package/dist/ops/cpu/fusedSoftmax.js +21 -28
- package/dist/ops/cpu/gatherSub.js +11 -17
- package/dist/ops/cpu/gelu.js +34 -38
- package/dist/ops/cpu/matMul16.js +13 -14
- package/dist/ops/cpu/matMulGelu.js +39 -51
- package/dist/ops/cpu/matMulMul.js +19 -22
- package/dist/ops/cpu/mulDropout.js +19 -22
- package/dist/ops/cpu/normRMS.js +33 -37
- package/dist/ops/cpu/qkv.js +72 -40
- package/dist/ops/cpu/rope.js +79 -36
- package/dist/ops/cpu/scatterSub.js +11 -22
- package/dist/ops/dot16.js +28 -41
- package/dist/ops/dropout.js +10 -13
- package/dist/ops/dropout16.js +20 -23
- package/dist/ops/gatherSub.js +10 -6
- package/dist/ops/gelu.js +2 -8
- package/dist/ops/globalNorm.js +18 -12
- package/dist/ops/grads/add16.js +27 -26
- package/dist/ops/grads/attentionMask.js +26 -21
- package/dist/ops/grads/dropout16.js +0 -1
- package/dist/ops/grads/gelu.js +2 -5
- package/dist/ops/grads/matMul16.js +2 -9
- package/dist/ops/grads/matMulGelu.js +21 -16
- package/dist/ops/grads/mul16.js +0 -3
- package/dist/ops/grads/normRMS.js +34 -30
- package/dist/ops/grads/pack16.js +2 -6
- package/dist/ops/grads/qkv.js +44 -32
- package/dist/ops/grads/rope.js +2 -5
- package/dist/ops/grads/softmax16.js +21 -23
- package/dist/ops/grads/unpack16.js +2 -5
- package/dist/ops/grads/utils.js +9 -11
- package/dist/ops/matMul16.js +2 -13
- package/dist/ops/matMulGelu.js +16 -10
- package/dist/ops/matMulMul.js +13 -6
- package/dist/ops/mul16.js +42 -38
- package/dist/ops/mulDrop.js +12 -6
- package/dist/ops/normRMS.js +18 -15
- package/dist/ops/pack16.js +2 -5
- package/dist/ops/qkv.js +12 -6
- package/dist/ops/reshape16.js +31 -39
- package/dist/ops/rope.d.ts +1 -1
- package/dist/ops/rope.js +2 -7
- package/dist/ops/scatterSub.js +10 -6
- package/dist/ops/slice16.js +9 -7
- package/dist/ops/softmax16.js +7 -7
- package/dist/ops/sub16.js +9 -6
- package/dist/ops/sum16.js +12 -12
- package/dist/ops/transpose16.js +29 -37
- package/dist/ops/unpack16.js +2 -6
- package/dist/ops/webgl/adamAdjust.js +62 -29
- package/dist/ops/webgl/adamMoments.js +30 -26
- package/dist/ops/webgl/appendCache.js +30 -21
- package/dist/ops/webgl/attentionMask.js +43 -24
- package/dist/ops/webgl/dropout16.js +11 -10
- package/dist/ops/webgl/fusedSoftmax.js +69 -79
- package/dist/ops/webgl/gatherSub.js +27 -26
- package/dist/ops/webgl/gelu.js +32 -34
- package/dist/ops/webgl/log.js +14 -23
- package/dist/ops/webgl/matMul16.js +36 -44
- package/dist/ops/webgl/matMulGelu.js +2 -9
- package/dist/ops/webgl/matMulMul.js +23 -27
- package/dist/ops/webgl/mulDropout.js +31 -40
- package/dist/ops/webgl/normRMS.js +92 -71
- package/dist/ops/webgl/qkv.js +35 -27
- package/dist/ops/webgl/rope.js +37 -21
- package/dist/ops/webgl/scatterSub.js +27 -26
- package/dist/ops/webgpu/adamAdjust.js +59 -39
- package/dist/ops/webgpu/adamMoments.js +62 -46
- package/dist/ops/webgpu/add16.js +13 -12
- package/dist/ops/webgpu/appendCache.js +79 -54
- package/dist/ops/webgpu/attentionMask.js +41 -25
- package/dist/ops/webgpu/attentionMask32_program.js +34 -26
- package/dist/ops/webgpu/clipScale.js +44 -57
- package/dist/ops/webgpu/concat16.js +96 -111
- package/dist/ops/webgpu/dropout16.js +40 -32
- package/dist/ops/webgpu/gatherSub.js +43 -30
- package/dist/ops/webgpu/gelu.js +88 -82
- package/dist/ops/webgpu/index.js +16 -16
- package/dist/ops/webgpu/matMul16.js +69 -64
- package/dist/ops/webgpu/matMul16_program.js +152 -192
- package/dist/ops/webgpu/mul16.js +13 -12
- package/dist/ops/webgpu/norm2.js +45 -75
- package/dist/ops/webgpu/normRMS.js +25 -33
- package/dist/ops/webgpu/normRMS16_program.js +21 -18
- package/dist/ops/webgpu/normRMS32_program.js +21 -18
- package/dist/ops/webgpu/normRMSGrad.js +125 -184
- package/dist/ops/webgpu/pack16.js +20 -17
- package/dist/ops/webgpu/pack16_program.js +48 -47
- package/dist/ops/webgpu/qkv.js +63 -23
- package/dist/ops/webgpu/rope.js +85 -57
- package/dist/ops/webgpu/scatterSub.js +43 -30
- package/dist/ops/webgpu/slice16.js +66 -61
- package/dist/ops/webgpu/softmax16.js +17 -20
- package/dist/ops/webgpu/softmax16_program.js +34 -18
- package/dist/ops/webgpu/softmax16_subgroup_program.js +40 -45
- package/dist/ops/webgpu/softmax16grad.js +30 -36
- package/dist/ops/webgpu/sub16.js +13 -12
- package/dist/ops/webgpu/sum16.js +28 -37
- package/dist/ops/webgpu/transpose16.js +36 -33
- package/dist/ops/webgpu/transpose16_program.js +40 -39
- package/dist/ops/webgpu/transpose16_shared_program.js +53 -44
- package/dist/ops/webgpu/unpack16.js +49 -37
- package/dist/ops/webgpu/utils/binary_op.js +70 -68
- package/dist/ops/webgpu/utils/deviceInfo.d.ts +1 -1
- package/dist/ops/webgpu/utils/deviceInfo.js +10 -10
- package/dist/ops/webgpu/utils/reductions.js +136 -148
- package/dist/pack16-Ck-spx_F.js +39 -0
- package/dist/patches/webgpu_backend.d.ts +2 -2
- package/dist/patches/webgpu_backend.js +42 -55
- package/dist/patches/webgpu_base.js +21 -33
- package/dist/patches/webgpu_program.js +213 -320
- package/dist/pdf-UoDqCYzz.js +16726 -0
- package/dist/picomatch-3tUnMMbd.js +1063 -0
- package/dist/rope-CbeGlsV8.js +25 -0
- package/dist/selu_util-zkAx5doH.js +24 -0
- package/dist/shared-D1coEFea.js +1314 -0
- package/dist/shared-DOgWaqvL.js +5 -0
- package/dist/slice_util-Dgb3ANWI.js +208 -0
- package/dist/tfjs_backend-BjuQ5FqB.js +614 -0
- package/dist/tokeniser/BaseTokeniser.js +2 -124
- package/dist/tokeniser/CharTokeniser.js +91 -106
- package/dist/tokeniser/bpe.js +163 -166
- package/dist/tokeniser/messages.js +0 -1
- package/dist/tokeniser/type.js +0 -1
- package/dist/training/AdamW.js +127 -137
- package/dist/training/BasicTrainer.d.ts +1 -1
- package/dist/training/BasicTrainer.js +264 -264
- package/dist/training/DatasetBuilder.js +2 -86
- package/dist/training/Evaluator.d.ts +1 -1
- package/dist/training/Evaluator.js +47 -38
- package/dist/training/LRScheduler.js +37 -33
- package/dist/training/PreTrainer.d.ts +2 -2
- package/dist/training/PreTrainer.js +21 -19
- package/dist/training/SFTTrainer.d.ts +2 -2
- package/dist/training/SFTTrainer.js +23 -21
- package/dist/training/loss.js +17 -22
- package/dist/training/orthoGrad.js +9 -9
- package/dist/training/sparseCrossEntropy.js +45 -67
- package/dist/training/tasks/ConversationTask.d.ts +1 -1
- package/dist/training/tasks/ConversationTask.js +36 -38
- package/dist/training/tasks/PretrainingTask.d.ts +1 -1
- package/dist/training/tasks/PretrainingTask.js +41 -46
- package/dist/training/tasks/StartSentenceTask.d.ts +1 -1
- package/dist/training/tasks/StartSentenceTask.js +44 -48
- package/dist/training/tasks/Task.d.ts +1 -1
- package/dist/training/tasks/Task.js +53 -66
- package/dist/training/tasks/splitter.js +17 -20
- package/dist/training/types.d.ts +2 -2
- package/dist/training/types.js +0 -1
- package/dist/training/validation.d.ts +1 -1
- package/dist/training/validation.js +2 -84
- package/dist/utilities/arrayClose.js +15 -19
- package/dist/utilities/datasetID.js +17 -20
- package/dist/utilities/dummy.d.ts +1 -1
- package/dist/utilities/dummy.js +33 -40
- package/dist/utilities/multinomialCPU.js +8 -12
- package/dist/utilities/naming.js +0 -1
- package/dist/utilities/packed.js +10 -12
- package/dist/utilities/parameters.d.ts +1 -1
- package/dist/utilities/parameters.js +32 -51
- package/dist/utilities/performance.js +15 -15
- package/dist/utilities/profile.js +32 -37
- package/dist/utilities/safetensors.js +49 -79
- package/dist/utilities/sentences.d.ts +1 -1
- package/dist/utilities/sentences.js +29 -38
- package/dist/utilities/tokenParse.js +16 -20
- package/dist/utilities/topP.js +11 -12
- package/dist/utilities/waitForModel.d.ts +1 -1
- package/dist/utilities/waitForModel.js +11 -11
- package/dist/utilities/weights.js +37 -42
- package/dist/utilities/yielder.js +6 -6
- package/dist/webgpu-Dt7BMzWz.js +525 -0
- package/dist/webgpu_program-WOyIVMlZ.js +392 -0
- package/dist/webgpu_util-B_F3SShA.js +106 -0
- package/package.json +9 -10
- package/dist/RealDiv-CGwv0liw.js +0 -365
- package/dist/Reshape-BW__R4mZ.js +0 -79
- package/dist/Reshape-CPBkTIH2.js +0 -14
- package/dist/_commonjsHelpers-ByX85dGu.js +0 -33
- package/dist/axis_util-GTVlo58H.js +0 -55
- package/dist/backend_util-GaFarB78.js +0 -425
- package/dist/backend_webgpu-BqASlsbV.js +0 -545
- package/dist/binary_op_util-pKXltfxI.js +0 -192
- package/dist/broadcast_to-eS93CCN_.js +0 -28
- package/dist/clip_by_value-DDA7rrcT.js +0 -12
- package/dist/complex-DI35Q-gW.js +0 -11
- package/dist/complex_util-Yc1A_gV1.js +0 -55
- package/dist/concat-CAQpCret.js +0 -17
- package/dist/concat_util-D18dJ4fD.js +0 -22
- package/dist/data/parquet.d.ts +0 -2
- package/dist/data/parquet.js +0 -17
- package/dist/dataset-CGGp1z9P.js +0 -1124
- package/dist/dropout_util--NxWuYg2.js +0 -27
- package/dist/expand_dims-Bkd1YD5x.js +0 -11
- package/dist/exports_initializers-CYzKLjN7.js +0 -7
- package/dist/floor-BQtb-Azg.js +0 -9
- package/dist/gather-qIqEqaGn.js +0 -9
- package/dist/gelu-B220X1Go.js +0 -26
- package/dist/gpgpu_math-BwvV12df.js +0 -2022
- package/dist/index-CUXkjxiT.js +0 -3516
- package/dist/index-CieiGp4Y.js +0 -349
- package/dist/index-CjOWnMXP.js +0 -7308
- package/dist/index-Cp39cXWe.js +0 -1016
- package/dist/index-D5v913EJ.js +0 -4
- package/dist/index-DmeWGGmS.js +0 -1074
- package/dist/index-DvYrXKkX.js +0 -113
- package/dist/index-Ksja3su6.js +0 -151
- package/dist/index-xuotMAFm.js +0 -118
- package/dist/jszip.min-BZhlzntC.js +0 -2313
- package/dist/kernel_funcs_utils-pq0CK9co.js +0 -306
- package/dist/matMul16-BcVC_E62.js +0 -80
- package/dist/matMulGelu-JNLZqKQp.js +0 -163
- package/dist/mat_mul-DhG0Newp.js +0 -11
- package/dist/mod-CSdCpRjf.js +0 -11
- package/dist/non_max_suppression_impl-B2W7YjZB.js +0 -102
- package/dist/not_equal-hurPF26l.js +0 -64
- package/dist/ones-BytntneX.js +0 -14
- package/dist/ops-CsXeTq1P.js +0 -476
- package/dist/pack16-bqltoUlR.js +0 -39
- package/dist/papaparse.min-C0cScC2i.js +0 -418
- package/dist/parquet-Bqjmp2vo.js +0 -44231
- package/dist/pdf-NIhmP3sq.js +0 -19477
- package/dist/rand_util-CZ7yLoUm.js +0 -50
- package/dist/random_normal-IBRrha8a.js +0 -14
- package/dist/random_width-DN5ZtQkM.js +0 -9796
- package/dist/range-C-CjF-LI.js +0 -10
- package/dist/relu-J_X6MUzx.js +0 -9
- package/dist/reshape-BDOuCSNW.js +0 -9
- package/dist/resize_nearest_neighbor-BojqlfRe.js +0 -150
- package/dist/rope-DcrZM_e6.js +0 -24
- package/dist/scatter_nd_util-ByNJaL6I.js +0 -46
- package/dist/segment_util-Dasb2Zaf.js +0 -43
- package/dist/selu_util-BLhIqRkw.js +0 -44
- package/dist/shared-3agzAqQ_.js +0 -53
- package/dist/shared-CagdqkLh.js +0 -2143
- package/dist/slice-BzS11Qh0.js +0 -12
- package/dist/slice_util-CC35pLmT.js +0 -153
- package/dist/softmax-D4q1LJN7.js +0 -12
- package/dist/split-C2Sj255c.js +0 -9
- package/dist/squeeze-ho4wLUek.js +0 -10
- package/dist/stack-DudVrtmG.js +0 -11
- package/dist/step-BTxPtq1r.js +0 -261
- package/dist/sum-BpiwSWvg.js +0 -11
- package/dist/tensor-BWFldCso.js +0 -8
- package/dist/tensor1d-LMGMIUlr.js +0 -11
- package/dist/tensor2d-BnXMKScO.js +0 -14
- package/dist/tensor4d-C6UCG_u8.js +0 -14
- package/dist/tfjs_backend-BGnG-ppu.js +0 -654
- package/dist/tile-CFy-xTO6.js +0 -11
- package/dist/transpose-9kRxIXWR.js +0 -36
- package/dist/unsorted_segment_sum-DJvk5xnh.js +0 -277
- package/dist/variable-Ck482e3n.js +0 -7
- package/dist/webgpu_program-B4HmApL1.js +0 -525
- package/dist/webgpu_util-DYlGSwOJ.js +0 -64
- package/dist/zeros-DvZpK8s6.js +0 -13
- package/dist/zeros_like-CWjDdwr-.js +0 -721
|
@@ -1,39 +1,48 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
|
|
1
|
+
import { di as e } from "../dist-BewPQWjc.js";
|
|
2
|
+
import { calculateAccuracy as t, calculateLoss as n } from "./loss.js";
|
|
3
|
+
//#region lib/training/Evaluator.ts
|
|
4
|
+
var r = class {
|
|
5
|
+
model;
|
|
6
|
+
iterator;
|
|
7
|
+
xs;
|
|
8
|
+
ys;
|
|
9
|
+
masked = !1;
|
|
10
|
+
constructor(e, t, n) {
|
|
11
|
+
this.model = e, this.masked = !!n, this.iterator = t.iterator();
|
|
12
|
+
}
|
|
13
|
+
dispose() {
|
|
14
|
+
this.xs && this.xs.dispose(), this.ys && this.ys.dispose();
|
|
15
|
+
}
|
|
16
|
+
async calculateBatchLoss(r, i, a, o) {
|
|
17
|
+
let [s, c] = e(() => {
|
|
18
|
+
let e = this.model.forward({ training: !1 }, r), s = n(e, i, o, a), c = t(e, i);
|
|
19
|
+
return e.dispose(), [s, c];
|
|
20
|
+
}), l = await s.array(), u = await c.array(), d = l, f = u;
|
|
21
|
+
return c.dispose(), s.dispose(), Array.isArray(d) ? d.map((e) => ({
|
|
22
|
+
loss: e,
|
|
23
|
+
accuracy: f
|
|
24
|
+
})) : {
|
|
25
|
+
loss: d,
|
|
26
|
+
accuracy: f
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
async evaluate(e = 100) {
|
|
30
|
+
let t = 0, n = 0, r = 0;
|
|
31
|
+
if (this.iterator) {
|
|
32
|
+
let i = await this.iterator;
|
|
33
|
+
for (let a = 0; a < e; a++) {
|
|
34
|
+
let e = await i.next();
|
|
35
|
+
if (e.done) break;
|
|
36
|
+
let { xs: a, ys: o } = e.value, s = await this.calculateBatchLoss(a, o, !1, this.masked);
|
|
37
|
+
a.dispose(), o.dispose(), t += s.loss, n += s.accuracy, r++;
|
|
38
|
+
}
|
|
39
|
+
return {
|
|
40
|
+
loss: t / r,
|
|
41
|
+
accuracy: n / r
|
|
42
|
+
};
|
|
43
|
+
} else if (this.xs && this.ys) return this.calculateBatchLoss(this.xs, this.ys, !0, !0);
|
|
44
|
+
throw Error("No data available for evaluation");
|
|
45
|
+
}
|
|
39
46
|
};
|
|
47
|
+
//#endregion
|
|
48
|
+
export { r as default };
|
|
@@ -1,34 +1,38 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
1
|
+
//#region lib/training/LRScheduler.ts
|
|
2
|
+
var e = class {
|
|
3
|
+
learningRate;
|
|
4
|
+
config;
|
|
5
|
+
step = 0;
|
|
6
|
+
startLearningRate;
|
|
7
|
+
constructor(e, t) {
|
|
8
|
+
this.learningRate = e, this.config = t, this.startLearningRate = e, t.step !== void 0 && (this.step = t.step);
|
|
9
|
+
}
|
|
10
|
+
serializeConfig() {
|
|
11
|
+
return {
|
|
12
|
+
...this.config,
|
|
13
|
+
step: this.step
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
updateConfig(e, t) {
|
|
17
|
+
this.config = {
|
|
18
|
+
...this.config,
|
|
19
|
+
...e
|
|
20
|
+
}, t !== void 0 && (this.startLearningRate = t);
|
|
21
|
+
}
|
|
22
|
+
get lr() {
|
|
23
|
+
return this.learningRate;
|
|
24
|
+
}
|
|
25
|
+
getNextLR() {
|
|
26
|
+
let e = this.step;
|
|
27
|
+
if (this.config.warmupSteps > 0 && e < this.config.warmupSteps) {
|
|
28
|
+
let t = (e + 1) / this.config.warmupSteps, n = this.startLearningRate * t;
|
|
29
|
+
return this.learningRate = n, this.step++, n;
|
|
30
|
+
}
|
|
31
|
+
let t = this.config.epochSteps * this.config.decayEpochs;
|
|
32
|
+
if (e >= t || t <= this.config.warmupSteps) return this.learningRate = this.config.minLearningRate, this.step++, this.config.minLearningRate;
|
|
33
|
+
let n = (e - this.config.warmupSteps) / (t - this.config.warmupSteps), r = .5 * (1 + Math.cos(Math.PI * n)), i = this.config.minLearningRate + r * (this.startLearningRate - this.config.minLearningRate);
|
|
34
|
+
return this.learningRate = i, this.step++, i;
|
|
35
|
+
}
|
|
34
36
|
};
|
|
37
|
+
//#endregion
|
|
38
|
+
export { e as default };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { default as Model, ModelForwardAttributes } from '
|
|
1
|
+
import { default as Model, ModelForwardAttributes } from '../../models/model';
|
|
2
2
|
import { default as BasicTrainer } from './BasicTrainer';
|
|
3
|
-
import { ITokeniser } from '
|
|
3
|
+
import { ITokeniser } from '../../tokeniser/type';
|
|
4
4
|
import { DatasetBuilder } from './DatasetBuilder';
|
|
5
5
|
import { AdamWOptimizer } from './AdamW';
|
|
6
6
|
import { AdamWOptimizerConfig } from './types';
|
|
@@ -1,20 +1,22 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
1
|
+
import { t as e } from "../DatasetBuilder-DgURD85T.js";
|
|
2
|
+
import t from "./BasicTrainer.js";
|
|
3
|
+
//#region lib/training/PreTrainer.ts
|
|
4
|
+
var n = {
|
|
5
|
+
decayEpochs: 100,
|
|
6
|
+
epochSteps: 1e4,
|
|
7
|
+
warmupSteps: 1e3,
|
|
8
|
+
minLearningRate: 3e-5,
|
|
9
|
+
weightDecay: .1,
|
|
10
|
+
learningRate: 3e-4
|
|
11
|
+
}, r = class extends t {
|
|
12
|
+
tokenizer;
|
|
13
|
+
datasetBuilder;
|
|
14
|
+
constructor(t, r, i, a) {
|
|
15
|
+
super(t, r, {
|
|
16
|
+
...n,
|
|
17
|
+
...i
|
|
18
|
+
}, a), this.tokenizer = r, this.optimizerConfig.minLearningRate = i?.minLearningRate ?? this.optimizerConfig.learningRate / 20, this.updateOptimizer(), this.datasetBuilder = new e(r, t.config.blockSize);
|
|
19
|
+
}
|
|
20
20
|
};
|
|
21
|
+
//#endregion
|
|
22
|
+
export { r as default };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { default as Model, ModelForwardAttributes } from '
|
|
1
|
+
import { default as Model, ModelForwardAttributes } from '../../models/model';
|
|
2
2
|
import { default as BasicTrainer } from './BasicTrainer';
|
|
3
|
-
import { ITokeniser } from '
|
|
3
|
+
import { ITokeniser } from '../../tokeniser/type';
|
|
4
4
|
import { AdamWOptimizer } from './AdamW';
|
|
5
5
|
import { AdamWOptimizerConfig } from './types';
|
|
6
6
|
import { DatasetBuilder } from './DatasetBuilder';
|
|
@@ -1,22 +1,24 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
1
|
+
import { t as e } from "../DatasetBuilder-DgURD85T.js";
|
|
2
|
+
import t from "./BasicTrainer.js";
|
|
3
|
+
//#region lib/training/SFTTrainer.ts
|
|
4
|
+
var n = {
|
|
5
|
+
decayEpochs: 100,
|
|
6
|
+
epochSteps: 1e4,
|
|
7
|
+
warmupSteps: 100,
|
|
8
|
+
minLearningRate: 1e-5,
|
|
9
|
+
weightDecay: .1,
|
|
10
|
+
beta2: .95,
|
|
11
|
+
learningRate: 3e-4
|
|
12
|
+
}, r = class extends t {
|
|
13
|
+
tokenizer;
|
|
14
|
+
datasetBuilder;
|
|
15
|
+
loraName;
|
|
16
|
+
constructor(t, r, i, a) {
|
|
17
|
+
super(t, r, {
|
|
18
|
+
...n,
|
|
19
|
+
...i
|
|
20
|
+
}, a), this.tokenizer = r, this.optimizerConfig.minLearningRate = i?.minLearningRate ?? this.optimizerConfig.learningRate / 20, this.updateOptimizer(), this.datasetBuilder = new e(r, t.config.blockSize), this.maskedLoss = !0;
|
|
21
|
+
}
|
|
22
22
|
};
|
|
23
|
+
//#endregion
|
|
24
|
+
export { r as default };
|
package/dist/training/loss.js
CHANGED
|
@@ -1,24 +1,19 @@
|
|
|
1
|
-
import { createSoftmaxCrossEntropyWithGrad as
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
} catch (a) {
|
|
10
|
-
throw console.error("Error computing loss:", a), new Error(`Loss computation failed: ${a}`);
|
|
11
|
-
}
|
|
1
|
+
import { createSoftmaxCrossEntropyWithGrad as e } from "./sparseCrossEntropy.js";
|
|
2
|
+
//#region lib/training/loss.ts
|
|
3
|
+
function t(t, n, r, i, a) {
|
|
4
|
+
try {
|
|
5
|
+
return e(r, i, a && a > 0 ? a : void 0)(t, n);
|
|
6
|
+
} catch (e) {
|
|
7
|
+
throw console.error("Error computing loss:", e), Error(`Loss computation failed: ${e}`);
|
|
8
|
+
}
|
|
12
9
|
}
|
|
13
|
-
function
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
10
|
+
function n(e, t) {
|
|
11
|
+
try {
|
|
12
|
+
let n = e.argMax(-1), r = n.equal(t).cast("float32"), i = r.mean();
|
|
13
|
+
return n.dispose(), r.dispose(), i;
|
|
14
|
+
} catch (e) {
|
|
15
|
+
throw console.error("Error computing accuracy:", e), Error(`Accuracy computation failed: ${e}`);
|
|
16
|
+
}
|
|
20
17
|
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
i as calculateLoss
|
|
24
|
-
};
|
|
18
|
+
//#endregion
|
|
19
|
+
export { n as calculateAccuracy, t as calculateLoss };
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
1
|
+
import { di as e } from "../dist-BewPQWjc.js";
|
|
2
|
+
//#region lib/training/orthoGrad.ts
|
|
3
|
+
function t(t, n, r) {
|
|
4
|
+
return e(() => {
|
|
5
|
+
let e = t.reshape([-1]), i = n.reshape([-1]), a = e.mul(e).sum().add(r), o = e.mul(i).sum().div(a), s = i.sub(e.mul(o)), c = i.norm(), l = s.norm().add(r);
|
|
6
|
+
return s.mul(c.div(l)).reshape(n.shape);
|
|
7
|
+
});
|
|
7
8
|
}
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
};
|
|
9
|
+
//#endregion
|
|
10
|
+
export { t as orthogonalizeGradient };
|
|
@@ -1,69 +1,47 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
s = U(m(s), c);
|
|
23
|
-
}
|
|
24
|
-
else
|
|
25
|
-
g ? s = E(s.reshape(i), -1) : s = E(s);
|
|
26
|
-
return s;
|
|
27
|
-
});
|
|
1
|
+
import { At as e, Bt as t, Gr as n, Nn as r, Pn as i, Rt as a, St as o, Wr as s, Wt as c, Y as l, _n as u, bn as d, di as f, mn as p, qr as m } from "../dist-BewPQWjc.js";
|
|
2
|
+
import { gatherSub as h } from "../ops/gatherSub.js";
|
|
3
|
+
import { scatterSub as g } from "../ops/scatterSub.js";
|
|
4
|
+
//#region lib/training/sparseCrossEntropy.ts
|
|
5
|
+
function _(r, i, o, c, l, u = 0) {
|
|
6
|
+
return f(() => {
|
|
7
|
+
let f = r.shape[r.shape.length - 1], g = l || r.shape.slice(0, -1), _ = g.reduce((e, t) => e * t, 1), v = r.shape.length > 2 ? r.reshape([_, f]) : r, y = i.shape.length > 1 ? i.reshape([_]).cast("int32") : i.cast("int32"), b = t(v, d(v, -1, !0)), x = a(b, -1), S = h(x, y, b), C;
|
|
8
|
+
if (u > 0) {
|
|
9
|
+
let n = t(x, e(b, -1));
|
|
10
|
+
C = m(s(S, 1 - u), s(n, u));
|
|
11
|
+
} else C = S;
|
|
12
|
+
if (o) if (C = s(C, o), c) {
|
|
13
|
+
let e = p(o.reshape(g), -1);
|
|
14
|
+
C = n(p(C.reshape(g), -1), e);
|
|
15
|
+
} else {
|
|
16
|
+
let e = p(o);
|
|
17
|
+
C = n(p(C), e);
|
|
18
|
+
}
|
|
19
|
+
else C = c ? e(C.reshape(g), -1) : e(C);
|
|
20
|
+
return C;
|
|
21
|
+
});
|
|
28
22
|
}
|
|
29
|
-
function
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
if (a > 0) {
|
|
52
|
-
const V = a / f, W = G(w, V);
|
|
53
|
-
L = T(
|
|
54
|
-
W,
|
|
55
|
-
z,
|
|
56
|
-
x(M, 1 - a)
|
|
57
|
-
);
|
|
58
|
-
} else
|
|
59
|
-
L = T(w, z, M);
|
|
60
|
-
const P = q(S);
|
|
61
|
-
return [L.reshape(t.shape), P];
|
|
62
|
-
}) };
|
|
63
|
-
}
|
|
64
|
-
);
|
|
23
|
+
function v(e, n, a = 0) {
|
|
24
|
+
return c((c, d, m) => {
|
|
25
|
+
let h = c.shape[c.shape.length - 1], v = c.shape.slice(0, -1), y = v.reduce((e, t) => e * t, 1), b = c.reshape([y, h]), x = d.reshape([y]).cast("int32"), S, C = null;
|
|
26
|
+
if (e) {
|
|
27
|
+
let e = u(65535, "int32"), t = o(x, e);
|
|
28
|
+
C = t.cast("float32"), S = i(t, x, r(x)), e.dispose(), t.dispose();
|
|
29
|
+
} else S = x;
|
|
30
|
+
let w = _(b, S, C || void 0, n, v, a);
|
|
31
|
+
return m(C ? [
|
|
32
|
+
b,
|
|
33
|
+
S,
|
|
34
|
+
C
|
|
35
|
+
] : [b, S]), b.dispose(), x.dispose(), {
|
|
36
|
+
value: w,
|
|
37
|
+
gradFunc: (n, i) => f(() => {
|
|
38
|
+
let o = i[0], f = i[1], m = e ? i[2] : void 0, _ = l(o), v = m ? p(m) : u(o.shape[0], "float32"), y = n.div(v).broadcastTo([o.shape[0]]), b = m && e ? s(y, m) : y, x;
|
|
39
|
+
x = a > 0 ? g(t(_, a / h), f, s(b, 1 - a)) : g(_, f, b);
|
|
40
|
+
let S = r(d);
|
|
41
|
+
return [x.reshape(c.shape), S];
|
|
42
|
+
})
|
|
43
|
+
};
|
|
44
|
+
});
|
|
65
45
|
}
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
R as sparseSoftmaxCrossEntropy
|
|
69
|
-
};
|
|
46
|
+
//#endregion
|
|
47
|
+
export { v as createSoftmaxCrossEntropyWithGrad, _ as sparseSoftmaxCrossEntropy };
|
|
@@ -1,40 +1,38 @@
|
|
|
1
|
+
import { a as e } from "../../DatasetBuilder-DgURD85T.js";
|
|
1
2
|
import { Task as t } from "./Task.js";
|
|
2
|
-
|
|
3
|
-
class
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
return n.encodeConversation(this.rawConvo[0]).length * this.length;
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
export {
|
|
39
|
-
f as default
|
|
3
|
+
//#region lib/training/tasks/ConversationTask.ts
|
|
4
|
+
var n = class extends t {
|
|
5
|
+
rawConvo;
|
|
6
|
+
shuffledIndices = null;
|
|
7
|
+
index = 0;
|
|
8
|
+
get length() {
|
|
9
|
+
return this.rawConvo.length;
|
|
10
|
+
}
|
|
11
|
+
constructor(e) {
|
|
12
|
+
super(), this.rawConvo = e;
|
|
13
|
+
}
|
|
14
|
+
hasMoreConversations() {
|
|
15
|
+
return this.index < this.rawConvo.length;
|
|
16
|
+
}
|
|
17
|
+
nextConversation() {
|
|
18
|
+
if (this.index >= this.rawConvo.length) return null;
|
|
19
|
+
let e = this.rawConvo[this.shuffledIndices ? this.shuffledIndices[this.index] : this.index];
|
|
20
|
+
return this.index++, e;
|
|
21
|
+
}
|
|
22
|
+
nextTokens(e, t) {
|
|
23
|
+
let n = this.nextConversation();
|
|
24
|
+
return n ? e.encodeConversation(n, !1, t) : null;
|
|
25
|
+
}
|
|
26
|
+
shuffle() {
|
|
27
|
+
if (!this.shuffledIndices) {
|
|
28
|
+
this.shuffledIndices = new Uint32Array(this.rawConvo.length);
|
|
29
|
+
for (let e = 0; e < this.rawConvo.length; e++) this.shuffledIndices[e] = e;
|
|
30
|
+
}
|
|
31
|
+
e(this.shuffledIndices), this.index = 0;
|
|
32
|
+
}
|
|
33
|
+
async estimateTokens(e) {
|
|
34
|
+
return e.encodeConversation(this.rawConvo[0]).length * this.length;
|
|
35
|
+
}
|
|
40
36
|
};
|
|
37
|
+
//#endregion
|
|
38
|
+
export { n as default };
|
|
@@ -1,47 +1,42 @@
|
|
|
1
|
-
import { Task as
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
content: this.rawText[0]
|
|
41
|
-
}
|
|
42
|
-
]).length * this.length;
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
export {
|
|
46
|
-
h as default
|
|
1
|
+
import { Task as e } from "./Task.js";
|
|
2
|
+
//#region lib/training/tasks/PretrainingTask.ts
|
|
3
|
+
var t = class extends e {
|
|
4
|
+
rawText;
|
|
5
|
+
index = 0;
|
|
6
|
+
get length() {
|
|
7
|
+
return this.rawText.length;
|
|
8
|
+
}
|
|
9
|
+
constructor(e) {
|
|
10
|
+
super(), this.rawText = e;
|
|
11
|
+
}
|
|
12
|
+
hasMoreConversations() {
|
|
13
|
+
return this.index < this.rawText.length;
|
|
14
|
+
}
|
|
15
|
+
nextConversation() {
|
|
16
|
+
if (this.index >= this.rawText.length) return null;
|
|
17
|
+
let e = {
|
|
18
|
+
role: "assistant",
|
|
19
|
+
content: this.rawText[this.index]
|
|
20
|
+
};
|
|
21
|
+
return this.index++, [e];
|
|
22
|
+
}
|
|
23
|
+
nextTokens(e, t) {
|
|
24
|
+
if (this.index >= this.rawText.length) return null;
|
|
25
|
+
let n = e.encodeSequence(this.rawText[this.index]);
|
|
26
|
+
return this.index++, t ? {
|
|
27
|
+
tokens: n,
|
|
28
|
+
mask: Array(n.length).fill(!0)
|
|
29
|
+
} : n;
|
|
30
|
+
}
|
|
31
|
+
shuffle() {
|
|
32
|
+
this.index = 0;
|
|
33
|
+
}
|
|
34
|
+
async estimateTokens(e) {
|
|
35
|
+
return e.encodeConversation([{
|
|
36
|
+
role: "assistant",
|
|
37
|
+
content: this.rawText[0]
|
|
38
|
+
}]).length * this.length;
|
|
39
|
+
}
|
|
47
40
|
};
|
|
41
|
+
//#endregion
|
|
42
|
+
export { t as default };
|