@genai-fi/nanogpt 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -10
- package/dist/Generator.d.ts +0 -82
- package/dist/Generator.js +0 -11941
- package/dist/RealDiv-CGwv0liw.js +0 -365
- package/dist/Reshape-BW__R4mZ.js +0 -79
- package/dist/Reshape-CPBkTIH2.js +0 -14
- package/dist/TeachableLLM.d.ts +0 -70
- package/dist/TeachableLLM.js +0 -273
- package/dist/Trainer.d.ts +0 -43
- package/dist/Trainer.js +0 -244
- package/dist/_commonjsHelpers-ByX85dGu.js +0 -33
- package/dist/axis_util-GTVlo58H.js +0 -55
- package/dist/backend.d.ts +0 -2
- package/dist/backend.js +0 -13
- package/dist/backend_util-GaFarB78.js +0 -425
- package/dist/backend_webgpu-BqASlsbV.js +0 -545
- package/dist/binary_op_util-pKXltfxI.js +0 -192
- package/dist/broadcast_to-eS93CCN_.js +0 -28
- package/dist/checks/appendCache.d.ts +0 -1
- package/dist/checks/appendCache.js +0 -22
- package/dist/checks/attentionMask.d.ts +0 -1
- package/dist/checks/attentionMask.js +0 -37
- package/dist/checks/check.d.ts +0 -9
- package/dist/checks/check.js +0 -20
- package/dist/checks/gelu.d.ts +0 -1
- package/dist/checks/gelu.js +0 -18
- package/dist/checks/index.d.ts +0 -26
- package/dist/checks/index.js +0 -28
- package/dist/checks/matMulGelu.d.ts +0 -1
- package/dist/checks/matMulGelu.js +0 -28
- package/dist/checks/normRMS.d.ts +0 -1
- package/dist/checks/normRMS.js +0 -16
- package/dist/checks/normRMSGrad.d.ts +0 -1
- package/dist/checks/normRMSGrad.js +0 -12
- package/dist/checks/packUnpack.d.ts +0 -1
- package/dist/checks/packUnpack.js +0 -18
- package/dist/checks/qkv.d.ts +0 -1
- package/dist/checks/qkv.js +0 -34
- package/dist/checks/rope.d.ts +0 -1
- package/dist/checks/rope.js +0 -36
- package/dist/checks/weights.d.ts +0 -14
- package/dist/checks/weights.js +0 -31
- package/dist/clip_by_value-DDA7rrcT.js +0 -12
- package/dist/complex-DI35Q-gW.js +0 -11
- package/dist/complex_util-Yc1A_gV1.js +0 -55
- package/dist/concat-CAQpCret.js +0 -17
- package/dist/concat_util-D18dJ4fD.js +0 -22
- package/dist/data/docx.d.ts +0 -2
- package/dist/data/docx.js +0 -15
- package/dist/data/parquet.d.ts +0 -2
- package/dist/data/parquet.js +0 -17
- package/dist/data/pdf.d.ts +0 -2
- package/dist/data/pdf.js +0 -14
- package/dist/data/textLoader.d.ts +0 -7
- package/dist/data/textLoader.js +0 -108
- package/dist/dataset-CGGp1z9P.js +0 -1124
- package/dist/dropout_util--NxWuYg2.js +0 -27
- package/dist/expand_dims-Bkd1YD5x.js +0 -11
- package/dist/exports_initializers-CYzKLjN7.js +0 -7
- package/dist/floor-BQtb-Azg.js +0 -9
- package/dist/gather-qIqEqaGn.js +0 -9
- package/dist/gelu-B220X1Go.js +0 -26
- package/dist/gpgpu_math-BwvV12df.js +0 -2022
- package/dist/index-CUXkjxiT.js +0 -3516
- package/dist/index-CieiGp4Y.js +0 -349
- package/dist/index-CjOWnMXP.js +0 -7308
- package/dist/index-Cp39cXWe.js +0 -1016
- package/dist/index-D5v913EJ.js +0 -4
- package/dist/index-DmeWGGmS.js +0 -1074
- package/dist/index-DvYrXKkX.js +0 -113
- package/dist/index-Ksja3su6.js +0 -151
- package/dist/index-xuotMAFm.js +0 -118
- package/dist/inference/types.d.ts +0 -16
- package/dist/inference/types.js +0 -1
- package/dist/jszip.min-BZhlzntC.js +0 -2313
- package/dist/kernel_funcs_utils-pq0CK9co.js +0 -306
- package/dist/layers/BaseLayer.d.ts +0 -44
- package/dist/layers/BaseLayer.js +0 -74
- package/dist/layers/CausalSelfAttention.d.ts +0 -39
- package/dist/layers/CausalSelfAttention.js +0 -86
- package/dist/layers/LoRA.d.ts +0 -14
- package/dist/layers/LoRA.js +0 -58
- package/dist/layers/MLP.d.ts +0 -17
- package/dist/layers/MLP.js +0 -44
- package/dist/layers/PositionEmbedding.d.ts +0 -8
- package/dist/layers/PositionEmbedding.js +0 -31
- package/dist/layers/RMSNorm.d.ts +0 -12
- package/dist/layers/RMSNorm.js +0 -22
- package/dist/layers/RoPECache.d.ts +0 -18
- package/dist/layers/RoPECache.js +0 -50
- package/dist/layers/TiedEmbedding.d.ts +0 -13
- package/dist/layers/TiedEmbedding.js +0 -36
- package/dist/layers/TransformerBlock.d.ts +0 -27
- package/dist/layers/TransformerBlock.js +0 -40
- package/dist/layers/WeightStore.d.ts +0 -20
- package/dist/layers/WeightStore.js +0 -76
- package/dist/loader/load.d.ts +0 -6
- package/dist/loader/load.js +0 -68
- package/dist/loader/loadHF.d.ts +0 -8
- package/dist/loader/loadHF.js +0 -22
- package/dist/loader/loadTransformers.d.ts +0 -4
- package/dist/loader/loadTransformers.js +0 -44
- package/dist/loader/loadZipMeta.d.ts +0 -3
- package/dist/loader/loadZipMeta.js +0 -16
- package/dist/loader/newZipLoad.d.ts +0 -3
- package/dist/loader/newZipLoad.js +0 -31
- package/dist/loader/oldZipLoad.d.ts +0 -9
- package/dist/loader/oldZipLoad.js +0 -80
- package/dist/loader/save.d.ts +0 -16
- package/dist/loader/save.js +0 -90
- package/dist/loader/types.d.ts +0 -67
- package/dist/loader/types.js +0 -1
- package/dist/main.d.ts +0 -50
- package/dist/main.js +0 -109
- package/dist/matMul16-BcVC_E62.js +0 -80
- package/dist/matMulGelu-JNLZqKQp.js +0 -163
- package/dist/mat_mul-DhG0Newp.js +0 -11
- package/dist/mod-CSdCpRjf.js +0 -11
- package/dist/models/NanoGPTV1.d.ts +0 -16
- package/dist/models/NanoGPTV1.js +0 -99
- package/dist/models/NanoGPTV2.d.ts +0 -16
- package/dist/models/NanoGPTV2.js +0 -90
- package/dist/models/config.d.ts +0 -27
- package/dist/models/config.js +0 -50
- package/dist/models/factory.d.ts +0 -3
- package/dist/models/factory.js +0 -16
- package/dist/models/model.d.ts +0 -44
- package/dist/models/model.js +0 -134
- package/dist/non_max_suppression_impl-B2W7YjZB.js +0 -102
- package/dist/not_equal-hurPF26l.js +0 -64
- package/dist/ones-BytntneX.js +0 -14
- package/dist/ops/adamAdjust.d.ts +0 -2
- package/dist/ops/adamAdjust.js +0 -9
- package/dist/ops/adamMoments.d.ts +0 -2
- package/dist/ops/adamMoments.js +0 -9
- package/dist/ops/add16.d.ts +0 -2
- package/dist/ops/add16.js +0 -9
- package/dist/ops/appendCache.d.ts +0 -2
- package/dist/ops/appendCache.js +0 -22
- package/dist/ops/attentionMask.d.ts +0 -2
- package/dist/ops/attentionMask.js +0 -10
- package/dist/ops/concat16.d.ts +0 -2
- package/dist/ops/concat16.js +0 -9
- package/dist/ops/cpu/adamAdjust.d.ts +0 -1
- package/dist/ops/cpu/adamAdjust.js +0 -18
- package/dist/ops/cpu/adamMoments.d.ts +0 -1
- package/dist/ops/cpu/adamMoments.js +0 -16
- package/dist/ops/cpu/appendCache.d.ts +0 -1
- package/dist/ops/cpu/appendCache.js +0 -23
- package/dist/ops/cpu/attentionMask.d.ts +0 -1
- package/dist/ops/cpu/attentionMask.js +0 -22
- package/dist/ops/cpu/fusedSoftmax.d.ts +0 -9
- package/dist/ops/cpu/fusedSoftmax.js +0 -29
- package/dist/ops/cpu/gatherSub.d.ts +0 -1
- package/dist/ops/cpu/gatherSub.js +0 -18
- package/dist/ops/cpu/gelu.d.ts +0 -1
- package/dist/ops/cpu/gelu.js +0 -40
- package/dist/ops/cpu/matMul16.d.ts +0 -1
- package/dist/ops/cpu/matMul16.js +0 -15
- package/dist/ops/cpu/matMulGelu.d.ts +0 -1
- package/dist/ops/cpu/matMulGelu.js +0 -53
- package/dist/ops/cpu/matMulMul.d.ts +0 -1
- package/dist/ops/cpu/matMulMul.js +0 -23
- package/dist/ops/cpu/mulDropout.d.ts +0 -1
- package/dist/ops/cpu/mulDropout.js +0 -23
- package/dist/ops/cpu/normRMS.d.ts +0 -1
- package/dist/ops/cpu/normRMS.js +0 -39
- package/dist/ops/cpu/qkv.d.ts +0 -5
- package/dist/ops/cpu/qkv.js +0 -41
- package/dist/ops/cpu/rope.d.ts +0 -6
- package/dist/ops/cpu/rope.js +0 -38
- package/dist/ops/cpu/scatterSub.d.ts +0 -1
- package/dist/ops/cpu/scatterSub.js +0 -23
- package/dist/ops/dot16.d.ts +0 -2
- package/dist/ops/dot16.js +0 -42
- package/dist/ops/dropout.d.ts +0 -2
- package/dist/ops/dropout.js +0 -14
- package/dist/ops/dropout16.d.ts +0 -2
- package/dist/ops/dropout16.js +0 -25
- package/dist/ops/gatherSub.d.ts +0 -2
- package/dist/ops/gatherSub.js +0 -9
- package/dist/ops/gelu.d.ts +0 -3
- package/dist/ops/gelu.js +0 -8
- package/dist/ops/globalNorm.d.ts +0 -2
- package/dist/ops/globalNorm.js +0 -13
- package/dist/ops/grads/add16.d.ts +0 -1
- package/dist/ops/grads/add16.js +0 -26
- package/dist/ops/grads/attentionMask.d.ts +0 -1
- package/dist/ops/grads/attentionMask.js +0 -21
- package/dist/ops/grads/dropout16.d.ts +0 -1
- package/dist/ops/grads/dropout16.js +0 -2
- package/dist/ops/grads/gelu.d.ts +0 -2
- package/dist/ops/grads/gelu.js +0 -5
- package/dist/ops/grads/matMul16.d.ts +0 -2
- package/dist/ops/grads/matMul16.js +0 -9
- package/dist/ops/grads/matMulGelu.d.ts +0 -1
- package/dist/ops/grads/matMulGelu.js +0 -17
- package/dist/ops/grads/mul16.d.ts +0 -1
- package/dist/ops/grads/mul16.js +0 -4
- package/dist/ops/grads/normRMS.d.ts +0 -3
- package/dist/ops/grads/normRMS.js +0 -33
- package/dist/ops/grads/pack16.d.ts +0 -2
- package/dist/ops/grads/pack16.js +0 -6
- package/dist/ops/grads/qkv.d.ts +0 -3
- package/dist/ops/grads/qkv.js +0 -34
- package/dist/ops/grads/rope.d.ts +0 -2
- package/dist/ops/grads/rope.js +0 -5
- package/dist/ops/grads/softmax16.d.ts +0 -2
- package/dist/ops/grads/softmax16.js +0 -25
- package/dist/ops/grads/unpack16.d.ts +0 -2
- package/dist/ops/grads/unpack16.js +0 -5
- package/dist/ops/grads/utils.d.ts +0 -4
- package/dist/ops/grads/utils.js +0 -14
- package/dist/ops/log.d.ts +0 -0
- package/dist/ops/log.js +0 -1
- package/dist/ops/matMul16.d.ts +0 -15
- package/dist/ops/matMul16.js +0 -13
- package/dist/ops/matMulGelu.d.ts +0 -3
- package/dist/ops/matMulGelu.js +0 -14
- package/dist/ops/matMulMul.d.ts +0 -2
- package/dist/ops/matMulMul.js +0 -9
- package/dist/ops/mul16.d.ts +0 -2
- package/dist/ops/mul16.js +0 -39
- package/dist/ops/mulDrop.d.ts +0 -2
- package/dist/ops/mulDrop.js +0 -9
- package/dist/ops/normRMS.d.ts +0 -2
- package/dist/ops/normRMS.js +0 -19
- package/dist/ops/pack16.d.ts +0 -2
- package/dist/ops/pack16.js +0 -5
- package/dist/ops/qkv.d.ts +0 -2
- package/dist/ops/qkv.js +0 -10
- package/dist/ops/reshape16.d.ts +0 -2
- package/dist/ops/reshape16.js +0 -41
- package/dist/ops/rope.d.ts +0 -3
- package/dist/ops/rope.js +0 -7
- package/dist/ops/scatterSub.d.ts +0 -2
- package/dist/ops/scatterSub.js +0 -9
- package/dist/ops/slice16.d.ts +0 -2
- package/dist/ops/slice16.js +0 -9
- package/dist/ops/softmax16.d.ts +0 -2
- package/dist/ops/softmax16.js +0 -9
- package/dist/ops/sub16.d.ts +0 -2
- package/dist/ops/sub16.js +0 -8
- package/dist/ops/sum16.d.ts +0 -2
- package/dist/ops/sum16.js +0 -13
- package/dist/ops/transpose16.d.ts +0 -3
- package/dist/ops/transpose16.js +0 -40
- package/dist/ops/unpack16.d.ts +0 -2
- package/dist/ops/unpack16.js +0 -6
- package/dist/ops/webgl/adamAdjust.d.ts +0 -1
- package/dist/ops/webgl/adamAdjust.js +0 -49
- package/dist/ops/webgl/adamMoments.d.ts +0 -1
- package/dist/ops/webgl/adamMoments.js +0 -40
- package/dist/ops/webgl/appendCache.d.ts +0 -1
- package/dist/ops/webgl/appendCache.js +0 -44
- package/dist/ops/webgl/attentionMask.d.ts +0 -1
- package/dist/ops/webgl/attentionMask.js +0 -45
- package/dist/ops/webgl/dropout16.d.ts +0 -1
- package/dist/ops/webgl/dropout16.js +0 -11
- package/dist/ops/webgl/fusedSoftmax.d.ts +0 -11
- package/dist/ops/webgl/fusedSoftmax.js +0 -80
- package/dist/ops/webgl/gatherSub.d.ts +0 -1
- package/dist/ops/webgl/gatherSub.js +0 -27
- package/dist/ops/webgl/gelu.d.ts +0 -2
- package/dist/ops/webgl/gelu.js +0 -50
- package/dist/ops/webgl/log.d.ts +0 -17
- package/dist/ops/webgl/log.js +0 -23
- package/dist/ops/webgl/matMul16.d.ts +0 -1
- package/dist/ops/webgl/matMul16.js +0 -45
- package/dist/ops/webgl/matMulGelu.d.ts +0 -21
- package/dist/ops/webgl/matMulGelu.js +0 -9
- package/dist/ops/webgl/matMulMul.d.ts +0 -14
- package/dist/ops/webgl/matMulMul.js +0 -28
- package/dist/ops/webgl/mulDropout.d.ts +0 -1
- package/dist/ops/webgl/mulDropout.js +0 -41
- package/dist/ops/webgl/normRMS.d.ts +0 -1
- package/dist/ops/webgl/normRMS.js +0 -93
- package/dist/ops/webgl/qkv.d.ts +0 -1
- package/dist/ops/webgl/qkv.js +0 -46
- package/dist/ops/webgl/rope.d.ts +0 -1
- package/dist/ops/webgl/rope.js +0 -56
- package/dist/ops/webgl/scatterSub.d.ts +0 -1
- package/dist/ops/webgl/scatterSub.js +0 -27
- package/dist/ops/webgpu/adamAdjust.d.ts +0 -1
- package/dist/ops/webgpu/adamAdjust.js +0 -57
- package/dist/ops/webgpu/adamMoments.d.ts +0 -1
- package/dist/ops/webgpu/adamMoments.js +0 -60
- package/dist/ops/webgpu/add16.d.ts +0 -1
- package/dist/ops/webgpu/add16.js +0 -13
- package/dist/ops/webgpu/appendCache.d.ts +0 -1
- package/dist/ops/webgpu/appendCache.js +0 -105
- package/dist/ops/webgpu/attentionMask.d.ts +0 -1
- package/dist/ops/webgpu/attentionMask.js +0 -26
- package/dist/ops/webgpu/attentionMask32_program.d.ts +0 -19
- package/dist/ops/webgpu/attentionMask32_program.js +0 -54
- package/dist/ops/webgpu/clipScale.d.ts +0 -1
- package/dist/ops/webgpu/clipScale.js +0 -58
- package/dist/ops/webgpu/concat16.d.ts +0 -19
- package/dist/ops/webgpu/concat16.js +0 -126
- package/dist/ops/webgpu/dropout16.d.ts +0 -1
- package/dist/ops/webgpu/dropout16.js +0 -51
- package/dist/ops/webgpu/gatherSub.d.ts +0 -1
- package/dist/ops/webgpu/gatherSub.js +0 -39
- package/dist/ops/webgpu/gelu.d.ts +0 -14
- package/dist/ops/webgpu/gelu.js +0 -141
- package/dist/ops/webgpu/index.d.ts +0 -0
- package/dist/ops/webgpu/index.js +0 -26
- package/dist/ops/webgpu/matMul16.d.ts +0 -1
- package/dist/ops/webgpu/matMul16.js +0 -65
- package/dist/ops/webgpu/matMul16_program.d.ts +0 -42
- package/dist/ops/webgpu/matMul16_program.js +0 -343
- package/dist/ops/webgpu/mul16.d.ts +0 -1
- package/dist/ops/webgpu/mul16.js +0 -13
- package/dist/ops/webgpu/norm2.d.ts +0 -1
- package/dist/ops/webgpu/norm2.js +0 -76
- package/dist/ops/webgpu/normRMS.d.ts +0 -1
- package/dist/ops/webgpu/normRMS.js +0 -34
- package/dist/ops/webgpu/normRMS16_program.d.ts +0 -10
- package/dist/ops/webgpu/normRMS16_program.js +0 -25
- package/dist/ops/webgpu/normRMS32_program.d.ts +0 -10
- package/dist/ops/webgpu/normRMS32_program.js +0 -25
- package/dist/ops/webgpu/normRMSGrad.d.ts +0 -1
- package/dist/ops/webgpu/normRMSGrad.js +0 -284
- package/dist/ops/webgpu/pack16.d.ts +0 -1
- package/dist/ops/webgpu/pack16.js +0 -18
- package/dist/ops/webgpu/pack16_program.d.ts +0 -19
- package/dist/ops/webgpu/pack16_program.js +0 -92
- package/dist/ops/webgpu/qkv.d.ts +0 -1
- package/dist/ops/webgpu/qkv.js +0 -24
- package/dist/ops/webgpu/rope.d.ts +0 -1
- package/dist/ops/webgpu/rope.js +0 -135
- package/dist/ops/webgpu/scatterSub.d.ts +0 -1
- package/dist/ops/webgpu/scatterSub.js +0 -40
- package/dist/ops/webgpu/slice16.d.ts +0 -7
- package/dist/ops/webgpu/slice16.js +0 -69
- package/dist/ops/webgpu/softmax16.d.ts +0 -17
- package/dist/ops/webgpu/softmax16.js +0 -21
- package/dist/ops/webgpu/softmax16_program.d.ts +0 -13
- package/dist/ops/webgpu/softmax16_program.js +0 -73
- package/dist/ops/webgpu/softmax16_subgroup_program.d.ts +0 -17
- package/dist/ops/webgpu/softmax16_subgroup_program.js +0 -75
- package/dist/ops/webgpu/softmax16grad.d.ts +0 -1
- package/dist/ops/webgpu/softmax16grad.js +0 -37
- package/dist/ops/webgpu/sub16.d.ts +0 -1
- package/dist/ops/webgpu/sub16.js +0 -13
- package/dist/ops/webgpu/sum16.d.ts +0 -1
- package/dist/ops/webgpu/sum16.js +0 -38
- package/dist/ops/webgpu/transpose16.d.ts +0 -1
- package/dist/ops/webgpu/transpose16.js +0 -34
- package/dist/ops/webgpu/transpose16_program.d.ts +0 -16
- package/dist/ops/webgpu/transpose16_program.js +0 -50
- package/dist/ops/webgpu/transpose16_shared_program.d.ts +0 -15
- package/dist/ops/webgpu/transpose16_shared_program.js +0 -70
- package/dist/ops/webgpu/unpack16.d.ts +0 -1
- package/dist/ops/webgpu/unpack16.js +0 -48
- package/dist/ops/webgpu/utils/binary_op.d.ts +0 -35
- package/dist/ops/webgpu/utils/binary_op.js +0 -139
- package/dist/ops/webgpu/utils/deviceInfo.d.ts +0 -7
- package/dist/ops/webgpu/utils/deviceInfo.js +0 -11
- package/dist/ops/webgpu/utils/reductions.d.ts +0 -43
- package/dist/ops/webgpu/utils/reductions.js +0 -275
- package/dist/ops-CsXeTq1P.js +0 -476
- package/dist/pack16-bqltoUlR.js +0 -39
- package/dist/papaparse.min-C0cScC2i.js +0 -418
- package/dist/parquet-Bqjmp2vo.js +0 -44231
- package/dist/patches/webgpu_backend.d.ts +0 -18
- package/dist/patches/webgpu_backend.js +0 -56
- package/dist/patches/webgpu_base.d.ts +0 -21
- package/dist/patches/webgpu_base.js +0 -34
- package/dist/patches/webgpu_program.d.ts +0 -36
- package/dist/patches/webgpu_program.js +0 -400
- package/dist/pdf-NIhmP3sq.js +0 -19477
- package/dist/rand_util-CZ7yLoUm.js +0 -50
- package/dist/random_normal-IBRrha8a.js +0 -14
- package/dist/random_width-DN5ZtQkM.js +0 -9796
- package/dist/range-C-CjF-LI.js +0 -10
- package/dist/relu-J_X6MUzx.js +0 -9
- package/dist/reshape-BDOuCSNW.js +0 -9
- package/dist/resize_nearest_neighbor-BojqlfRe.js +0 -150
- package/dist/rope-DcrZM_e6.js +0 -24
- package/dist/scatter_nd_util-ByNJaL6I.js +0 -46
- package/dist/segment_util-Dasb2Zaf.js +0 -43
- package/dist/selu_util-BLhIqRkw.js +0 -44
- package/dist/shared-3agzAqQ_.js +0 -53
- package/dist/shared-CagdqkLh.js +0 -2143
- package/dist/slice-BzS11Qh0.js +0 -12
- package/dist/slice_util-CC35pLmT.js +0 -153
- package/dist/softmax-D4q1LJN7.js +0 -12
- package/dist/split-C2Sj255c.js +0 -9
- package/dist/squeeze-ho4wLUek.js +0 -10
- package/dist/stack-DudVrtmG.js +0 -11
- package/dist/step-BTxPtq1r.js +0 -261
- package/dist/sum-BpiwSWvg.js +0 -11
- package/dist/tensor-BWFldCso.js +0 -8
- package/dist/tensor1d-LMGMIUlr.js +0 -11
- package/dist/tensor2d-BnXMKScO.js +0 -14
- package/dist/tensor4d-C6UCG_u8.js +0 -14
- package/dist/tfjs_backend-BGnG-ppu.js +0 -654
- package/dist/tile-CFy-xTO6.js +0 -11
- package/dist/tokeniser/BaseTokeniser.d.ts +0 -33
- package/dist/tokeniser/BaseTokeniser.js +0 -124
- package/dist/tokeniser/CharTokeniser.d.ts +0 -24
- package/dist/tokeniser/CharTokeniser.js +0 -107
- package/dist/tokeniser/bpe.d.ts +0 -28
- package/dist/tokeniser/bpe.js +0 -173
- package/dist/tokeniser/messages.d.ts +0 -61
- package/dist/tokeniser/messages.js +0 -1
- package/dist/tokeniser/type.d.ts +0 -34
- package/dist/tokeniser/type.js +0 -1
- package/dist/training/AdamW.d.ts +0 -36
- package/dist/training/AdamW.js +0 -138
- package/dist/training/BasicTrainer.d.ts +0 -63
- package/dist/training/BasicTrainer.js +0 -265
- package/dist/training/DatasetBuilder.d.ts +0 -26
- package/dist/training/DatasetBuilder.js +0 -86
- package/dist/training/Evaluator.d.ts +0 -19
- package/dist/training/Evaluator.js +0 -39
- package/dist/training/LRScheduler.d.ts +0 -12
- package/dist/training/LRScheduler.js +0 -34
- package/dist/training/PreTrainer.d.ts +0 -11
- package/dist/training/PreTrainer.js +0 -20
- package/dist/training/SFTTrainer.d.ts +0 -12
- package/dist/training/SFTTrainer.js +0 -22
- package/dist/training/loss.d.ts +0 -3
- package/dist/training/loss.js +0 -24
- package/dist/training/orthoGrad.d.ts +0 -2
- package/dist/training/orthoGrad.js +0 -10
- package/dist/training/sparseCrossEntropy.d.ts +0 -7
- package/dist/training/sparseCrossEntropy.js +0 -69
- package/dist/training/tasks/ConversationTask.d.ts +0 -18
- package/dist/training/tasks/ConversationTask.js +0 -40
- package/dist/training/tasks/PretrainingTask.d.ts +0 -17
- package/dist/training/tasks/PretrainingTask.js +0 -47
- package/dist/training/tasks/StartSentenceTask.d.ts +0 -18
- package/dist/training/tasks/StartSentenceTask.js +0 -49
- package/dist/training/tasks/Task.d.ts +0 -22
- package/dist/training/tasks/Task.js +0 -68
- package/dist/training/tasks/splitter.d.ts +0 -5
- package/dist/training/tasks/splitter.js +0 -21
- package/dist/training/types.d.ts +0 -78
- package/dist/training/types.js +0 -1
- package/dist/training/validation.d.ts +0 -17
- package/dist/training/validation.js +0 -84
- package/dist/transpose-9kRxIXWR.js +0 -36
- package/dist/unsorted_segment_sum-DJvk5xnh.js +0 -277
- package/dist/utilities/arrayClose.d.ts +0 -1
- package/dist/utilities/arrayClose.js +0 -20
- package/dist/utilities/datasetID.d.ts +0 -2
- package/dist/utilities/datasetID.js +0 -21
- package/dist/utilities/dummy.d.ts +0 -9
- package/dist/utilities/dummy.js +0 -43
- package/dist/utilities/multinomialCPU.d.ts +0 -2
- package/dist/utilities/multinomialCPU.js +0 -13
- package/dist/utilities/naming.d.ts +0 -4
- package/dist/utilities/naming.js +0 -1
- package/dist/utilities/packed.d.ts +0 -4
- package/dist/utilities/packed.js +0 -15
- package/dist/utilities/parameters.d.ts +0 -11
- package/dist/utilities/parameters.js +0 -57
- package/dist/utilities/performance.d.ts +0 -2
- package/dist/utilities/performance.js +0 -16
- package/dist/utilities/profile.d.ts +0 -17
- package/dist/utilities/profile.js +0 -38
- package/dist/utilities/safetensors.d.ts +0 -3
- package/dist/utilities/safetensors.js +0 -83
- package/dist/utilities/sentences.d.ts +0 -5
- package/dist/utilities/sentences.js +0 -41
- package/dist/utilities/tokenParse.d.ts +0 -1
- package/dist/utilities/tokenParse.js +0 -21
- package/dist/utilities/topP.d.ts +0 -1
- package/dist/utilities/topP.js +0 -13
- package/dist/utilities/waitForModel.d.ts +0 -2
- package/dist/utilities/waitForModel.js +0 -12
- package/dist/utilities/weights.d.ts +0 -12
- package/dist/utilities/weights.js +0 -45
- package/dist/utilities/yielder.d.ts +0 -1
- package/dist/utilities/yielder.js +0 -7
- package/dist/variable-Ck482e3n.js +0 -7
- package/dist/webgpu_program-B4HmApL1.js +0 -525
- package/dist/webgpu_util-DYlGSwOJ.js +0 -64
- package/dist/zeros-DvZpK8s6.js +0 -13
- package/dist/zeros_like-CWjDdwr-.js +0 -721
package/dist/mod-CSdCpRjf.js
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { o as r, q as s, B as m, E as c, M as d } from "./index-CUXkjxiT.js";
|
|
2
|
-
function p(t, e) {
|
|
3
|
-
let o = s(t, "a", "mod"), a = s(e, "b", "mod");
|
|
4
|
-
[o, a] = m(o, a);
|
|
5
|
-
const n = { a: o, b: a };
|
|
6
|
-
return c.runKernel(d, n);
|
|
7
|
-
}
|
|
8
|
-
const b = /* @__PURE__ */ r({ mod_: p });
|
|
9
|
-
export {
|
|
10
|
-
b as m
|
|
11
|
-
};
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { GPTConfigV1 } from './config';
|
|
2
|
-
import { Tensor } from '@tensorflow/tfjs-core';
|
|
3
|
-
import { default as Model, ModelForwardAttributes } from './model';
|
|
4
|
-
export default class NanoGPTV1 extends Model<ModelForwardAttributes, GPTConfigV1> {
|
|
5
|
-
private wte;
|
|
6
|
-
private wpe?;
|
|
7
|
-
private blocks;
|
|
8
|
-
private lnF;
|
|
9
|
-
private ropeCache?;
|
|
10
|
-
constructor(config?: Partial<GPTConfigV1>);
|
|
11
|
-
getClassName(): string;
|
|
12
|
-
private inputPhase;
|
|
13
|
-
forward(attrs: ModelForwardAttributes, idx: Tensor): Tensor;
|
|
14
|
-
project(embeddings: Tensor): Tensor;
|
|
15
|
-
dispose(): void;
|
|
16
|
-
}
|
package/dist/models/NanoGPTV1.js
DELETED
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
import u from "../layers/TransformerBlock.js";
|
|
2
|
-
import f from "../layers/TiedEmbedding.js";
|
|
3
|
-
import g from "../layers/RoPECache.js";
|
|
4
|
-
import b from "../layers/RMSNorm.js";
|
|
5
|
-
import { t as r, k as d } from "../index-CUXkjxiT.js";
|
|
6
|
-
import k from "./model.js";
|
|
7
|
-
import w from "../layers/PositionEmbedding.js";
|
|
8
|
-
import { packingSupported as E } from "../utilities/packed.js";
|
|
9
|
-
import { p as _, u as C } from "../pack16-bqltoUlR.js";
|
|
10
|
-
const P = {
|
|
11
|
-
modelType: "GenAI_NanoGPT_v1",
|
|
12
|
-
vocabSize: 2e3,
|
|
13
|
-
blockSize: 128,
|
|
14
|
-
// Maximum sequence length
|
|
15
|
-
nLayer: 6,
|
|
16
|
-
// Number of transformer layers
|
|
17
|
-
nHead: 4,
|
|
18
|
-
// Number of attention heads
|
|
19
|
-
nEmbed: 256,
|
|
20
|
-
// Embedding dimension
|
|
21
|
-
mlpFactor: 4,
|
|
22
|
-
useRope: !0
|
|
23
|
-
};
|
|
24
|
-
class S extends k {
|
|
25
|
-
wte;
|
|
26
|
-
// Token embeddings
|
|
27
|
-
wpe;
|
|
28
|
-
// Position embeddings
|
|
29
|
-
// private drop: layers.Layer; // Dropout
|
|
30
|
-
blocks;
|
|
31
|
-
lnF;
|
|
32
|
-
// Final layer norm
|
|
33
|
-
ropeCache;
|
|
34
|
-
constructor(e = {}) {
|
|
35
|
-
super({ ...P, ...e });
|
|
36
|
-
const t = {
|
|
37
|
-
activation: "gelu",
|
|
38
|
-
hiddenFactor: this.config.mlpFactor,
|
|
39
|
-
useGamma: !0,
|
|
40
|
-
useQKNorm: !1
|
|
41
|
-
};
|
|
42
|
-
this.wte = new f(this.config, "token_embedding", this), this.config.useRope === !1 ? this.wpe = new w(this.config, "positional_embedding", this) : this.ropeCache = new g(this.config), this.blocks = [];
|
|
43
|
-
for (let o = 0; o < this.config.nLayer; o++)
|
|
44
|
-
this.blocks.push(new u(o, this.config, t, this));
|
|
45
|
-
this.lnF = new b(this.config, t, "final_rms_norm", this);
|
|
46
|
-
}
|
|
47
|
-
getClassName() {
|
|
48
|
-
return "GenAI_NanoGPT_v1";
|
|
49
|
-
}
|
|
50
|
-
inputPhase(e, t) {
|
|
51
|
-
return r(() => {
|
|
52
|
-
const o = this.wte.embed(e);
|
|
53
|
-
if (this.config.useRope === !1) {
|
|
54
|
-
const i = this.wpe.call(t, o);
|
|
55
|
-
if (Array.isArray(i))
|
|
56
|
-
throw new Error("PositionEmbedding output should not be an array");
|
|
57
|
-
return i;
|
|
58
|
-
}
|
|
59
|
-
return o;
|
|
60
|
-
});
|
|
61
|
-
}
|
|
62
|
-
forward(e, t) {
|
|
63
|
-
return this.validateInput(t), e.ropeCache = this.ropeCache, e.outputEmbeddings && (e.embeddings = []), r(() => {
|
|
64
|
-
this.startMemory();
|
|
65
|
-
let o = this.inputPhase(t, e);
|
|
66
|
-
if (e.cache && e.cache.length !== this.blocks.length)
|
|
67
|
-
throw console.error("Cache", e.cache), new Error(
|
|
68
|
-
`Cache length ${e.cache.length} does not match number of blocks ${this.blocks.length}`
|
|
69
|
-
);
|
|
70
|
-
const i = e.mixedPrecision === !0 && E();
|
|
71
|
-
let s = i ? _(o) : o;
|
|
72
|
-
i && o !== s && o.dispose();
|
|
73
|
-
for (let n = 0; n < this.blocks.length; n++) {
|
|
74
|
-
const p = this.blocks[n], a = Math.random() * 1e9, l = {
|
|
75
|
-
...e,
|
|
76
|
-
seed: a,
|
|
77
|
-
pastKV: e.cache ? e.cache[n] : void 0,
|
|
78
|
-
mixedPrecision: i
|
|
79
|
-
}, m = e.checkpointing && e.training ? p.callCheckpoint(l, s) : p.call(l, s);
|
|
80
|
-
e.outputEmbeddings ? (d(s), e.embeddings.push({ name: `block_output_${n}`, tensor: s })) : s.dispose(), s = m;
|
|
81
|
-
}
|
|
82
|
-
if (o = this.lnF.call({ ...e, mixedPrecision: i }, s), s.dispose(), e.skipLogits)
|
|
83
|
-
return this.endMemory("Forward"), o;
|
|
84
|
-
const c = this.wte.project(o);
|
|
85
|
-
e.outputEmbeddings ? (d(o), e.embeddings.push({ name: "final_norm_output", tensor: o })) : o.dispose();
|
|
86
|
-
const h = i ? C(c) : c;
|
|
87
|
-
return i && c !== h && c.dispose(), h;
|
|
88
|
-
});
|
|
89
|
-
}
|
|
90
|
-
project(e) {
|
|
91
|
-
return r(() => this.wte.project(e));
|
|
92
|
-
}
|
|
93
|
-
dispose() {
|
|
94
|
-
this.weightStore.dispose(), this.wte.dispose(), this.wpe && this.wpe.dispose(), this.blocks.forEach((e) => e.dispose()), this.lnF.dispose();
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
export {
|
|
98
|
-
S as default
|
|
99
|
-
};
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { GPTConfigV2 } from './config';
|
|
2
|
-
import { Tensor } from '@tensorflow/tfjs-core';
|
|
3
|
-
import { default as Model, ModelForwardAttributes } from './model';
|
|
4
|
-
export default class NanoGPTV2 extends Model<ModelForwardAttributes, GPTConfigV2> {
|
|
5
|
-
private wte;
|
|
6
|
-
private wpe?;
|
|
7
|
-
private blocks;
|
|
8
|
-
private lnF;
|
|
9
|
-
private ropeCache?;
|
|
10
|
-
constructor(config?: Partial<GPTConfigV2>);
|
|
11
|
-
getClassName(): string;
|
|
12
|
-
private inputPhase;
|
|
13
|
-
forward(attrs: ModelForwardAttributes, idx: Tensor): Tensor;
|
|
14
|
-
project(embeddings: Tensor): Tensor;
|
|
15
|
-
dispose(): void;
|
|
16
|
-
}
|
package/dist/models/NanoGPTV2.js
DELETED
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
import u from "../layers/TransformerBlock.js";
|
|
2
|
-
import f from "../layers/TiedEmbedding.js";
|
|
3
|
-
import g from "../layers/RoPECache.js";
|
|
4
|
-
import b from "../layers/RMSNorm.js";
|
|
5
|
-
import { t as r, k as d } from "../index-CUXkjxiT.js";
|
|
6
|
-
import k from "./model.js";
|
|
7
|
-
import { packingSupported as w } from "../utilities/packed.js";
|
|
8
|
-
import { p as C, u as _ } from "../pack16-bqltoUlR.js";
|
|
9
|
-
const E = {
|
|
10
|
-
modelType: "GenAI_NanoGPT_v2",
|
|
11
|
-
vocabSize: 2e3,
|
|
12
|
-
blockSize: 128,
|
|
13
|
-
// Maximum sequence length
|
|
14
|
-
nLayer: 6,
|
|
15
|
-
// Number of transformer layers
|
|
16
|
-
nHead: 4,
|
|
17
|
-
// Number of attention heads
|
|
18
|
-
nEmbed: 256,
|
|
19
|
-
// Embedding dimension
|
|
20
|
-
mlpFactor: 4
|
|
21
|
-
};
|
|
22
|
-
class S extends k {
|
|
23
|
-
wte;
|
|
24
|
-
// Token embeddings
|
|
25
|
-
wpe;
|
|
26
|
-
// Position embeddings
|
|
27
|
-
// private drop: layers.Layer; // Dropout
|
|
28
|
-
blocks;
|
|
29
|
-
lnF;
|
|
30
|
-
// Final layer norm
|
|
31
|
-
ropeCache;
|
|
32
|
-
constructor(e = {}) {
|
|
33
|
-
super({ ...E, ...e });
|
|
34
|
-
const s = {
|
|
35
|
-
activation: "relu2",
|
|
36
|
-
hiddenFactor: this.config.mlpFactor,
|
|
37
|
-
useGamma: !1,
|
|
38
|
-
useQKNorm: !0
|
|
39
|
-
};
|
|
40
|
-
this.wte = new f(this.config, "token_embedding", this), this.ropeCache = new g(this.config), this.blocks = [];
|
|
41
|
-
for (let o = 0; o < this.config.nLayer; o++)
|
|
42
|
-
this.blocks.push(new u(o, this.config, s, this));
|
|
43
|
-
this.lnF = new b(this.config, s, "final_rms_norm", this);
|
|
44
|
-
}
|
|
45
|
-
getClassName() {
|
|
46
|
-
return "GenAI_NanoGPT_v2";
|
|
47
|
-
}
|
|
48
|
-
inputPhase(e) {
|
|
49
|
-
return r(() => this.wte.embed(e));
|
|
50
|
-
}
|
|
51
|
-
forward(e, s) {
|
|
52
|
-
return this.validateInput(s), e.ropeCache = this.ropeCache, e.outputEmbeddings && (e.embeddings = []), r(() => {
|
|
53
|
-
this.startMemory();
|
|
54
|
-
let o = this.inputPhase(s);
|
|
55
|
-
if (e.cache && e.cache.length !== this.blocks.length)
|
|
56
|
-
throw console.error("Cache", e.cache), new Error(
|
|
57
|
-
`Cache length ${e.cache.length} does not match number of blocks ${this.blocks.length}`
|
|
58
|
-
);
|
|
59
|
-
const t = e.mixedPrecision === !0 && w();
|
|
60
|
-
let i = t ? C(o) : o;
|
|
61
|
-
t && o !== i && o.dispose();
|
|
62
|
-
for (let n = 0; n < this.blocks.length; n++) {
|
|
63
|
-
if (e.layerDrop && Math.random() < e.layerDrop * (n / this.blocks.length))
|
|
64
|
-
continue;
|
|
65
|
-
const p = this.blocks[n], m = Math.random() * 1e9, l = {
|
|
66
|
-
...e,
|
|
67
|
-
seed: m,
|
|
68
|
-
pastKV: e.cache ? e.cache[n] : void 0,
|
|
69
|
-
mixedPrecision: t
|
|
70
|
-
}, a = e.checkpointing && e.training ? p.callCheckpoint(l, i) : p.call(l, i);
|
|
71
|
-
e.outputEmbeddings ? (d(i), e.embeddings.push({ name: `block_output_${n}`, tensor: i })) : i.dispose(), i = a;
|
|
72
|
-
}
|
|
73
|
-
if (o = this.lnF.call({ ...e, mixedPrecision: t }, i), i.dispose(), e.skipLogits)
|
|
74
|
-
return this.endMemory("Forward"), o;
|
|
75
|
-
const c = this.wte.project(o);
|
|
76
|
-
e.outputEmbeddings ? (d(o), e.embeddings.push({ name: "final_norm_output", tensor: o })) : o.dispose();
|
|
77
|
-
const h = t ? _(c) : c;
|
|
78
|
-
return t && c !== h && c.dispose(), h;
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
project(e) {
|
|
82
|
-
return r(() => this.wte.project(e));
|
|
83
|
-
}
|
|
84
|
-
dispose() {
|
|
85
|
-
this.weightStore.dispose(), this.wte.dispose(), this.wpe && this.wpe.dispose(), this.blocks.forEach((e) => e.dispose()), this.lnF.dispose();
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
export {
|
|
89
|
-
S as default
|
|
90
|
-
};
|
package/dist/models/config.d.ts
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
export interface LoRAConfig {
|
|
2
|
-
rank: number;
|
|
3
|
-
alpha: number;
|
|
4
|
-
variables: string[];
|
|
5
|
-
}
|
|
6
|
-
export interface GPTConfigBase {
|
|
7
|
-
modelType?: string;
|
|
8
|
-
vocabSize: number;
|
|
9
|
-
blockSize: number;
|
|
10
|
-
nLayer: number;
|
|
11
|
-
nHead: number;
|
|
12
|
-
nEmbed: number;
|
|
13
|
-
mlpFactor: number;
|
|
14
|
-
loraConfig?: Map<string, LoRAConfig>;
|
|
15
|
-
loraName?: string;
|
|
16
|
-
}
|
|
17
|
-
export interface GPTConfigV1 extends GPTConfigBase {
|
|
18
|
-
modelType: 'GenAI_NanoGPT_v1';
|
|
19
|
-
useRope: boolean;
|
|
20
|
-
}
|
|
21
|
-
export interface GPTConfigV2 extends GPTConfigBase {
|
|
22
|
-
modelType: 'GenAI_NanoGPT_v2';
|
|
23
|
-
windowSize?: string;
|
|
24
|
-
}
|
|
25
|
-
export type GPTConfig = GPTConfigV1 | GPTConfigV2;
|
|
26
|
-
export declare const defaultConfig: GPTConfig;
|
|
27
|
-
export declare function validateConfig(config: unknown): asserts config is GPTConfig;
|
package/dist/models/config.js
DELETED
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
const t = {
|
|
2
|
-
modelType: "GenAI_NanoGPT_v2",
|
|
3
|
-
vocabSize: 2e3,
|
|
4
|
-
blockSize: 128,
|
|
5
|
-
// Maximum sequence length
|
|
6
|
-
nLayer: 6,
|
|
7
|
-
// Number of transformer layers
|
|
8
|
-
nHead: 4,
|
|
9
|
-
// Number of attention heads
|
|
10
|
-
nEmbed: 256,
|
|
11
|
-
// Embedding dimension
|
|
12
|
-
mlpFactor: 4
|
|
13
|
-
};
|
|
14
|
-
function n(e, a) {
|
|
15
|
-
if (typeof e[a] != "number" || Number.isNaN(e[a]))
|
|
16
|
-
throw new Error(`Invalid config: "${a}" must be a number.`);
|
|
17
|
-
}
|
|
18
|
-
function l(e) {
|
|
19
|
-
const a = (o) => typeof o == "object" && o !== null && !Array.isArray(o);
|
|
20
|
-
if (!a(e))
|
|
21
|
-
throw new Error("Invalid config: expected an object.");
|
|
22
|
-
if (n(e, "vocabSize"), n(e, "blockSize"), n(e, "nLayer"), n(e, "nHead"), n(e, "nEmbed"), n(e, "mlpFactor"), e.loraConfig !== void 0) {
|
|
23
|
-
if (!a(e.loraConfig))
|
|
24
|
-
throw new Error('Invalid config: "loraConfig" must be an object.');
|
|
25
|
-
const o = Object.values(e.loraConfig);
|
|
26
|
-
if (!o.every((r) => a(r)))
|
|
27
|
-
throw new Error('Invalid config: each entry in "loraConfig" must be an object.');
|
|
28
|
-
if (!o.every((r) => "rank" in r && "alpha" in r && "variables" in r))
|
|
29
|
-
throw new Error('Invalid config: each LoRA config must have "rank", "alpha", and "variables" fields.');
|
|
30
|
-
o.forEach((r) => {
|
|
31
|
-
if (n(r, "rank"), n(r, "alpha"), !Array.isArray(r.variables) || !r.variables.every((i) => typeof i == "string"))
|
|
32
|
-
throw new Error('Invalid config: "variables" must be a string array.');
|
|
33
|
-
});
|
|
34
|
-
}
|
|
35
|
-
if (e.modelType === "GenAI_NanoGPT_v1") {
|
|
36
|
-
if (typeof e.useRope != "boolean")
|
|
37
|
-
throw new Error('Invalid config: "useRope" must be a boolean for GenAI_NanoGPT_v1.');
|
|
38
|
-
return;
|
|
39
|
-
}
|
|
40
|
-
if (e.modelType === "GenAI_NanoGPT_v2") {
|
|
41
|
-
if (e.windowSize !== void 0 && typeof e.windowSize != "string")
|
|
42
|
-
throw new Error('Invalid config: "windowSize" must be a string for GenAI_NanoGPT_v2.');
|
|
43
|
-
return;
|
|
44
|
-
}
|
|
45
|
-
throw new Error('Invalid config: "modelType" must be "GenAI_NanoGPT_v1" or "GenAI_NanoGPT_v2".');
|
|
46
|
-
}
|
|
47
|
-
export {
|
|
48
|
-
t as defaultConfig,
|
|
49
|
-
l as validateConfig
|
|
50
|
-
};
|
package/dist/models/factory.d.ts
DELETED
package/dist/models/factory.js
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import o from "./NanoGPTV1.js";
|
|
2
|
-
import r from "./NanoGPTV2.js";
|
|
3
|
-
function p(e) {
|
|
4
|
-
const n = e.modelType || "GenAI_NanoGPT_v1";
|
|
5
|
-
switch (n) {
|
|
6
|
-
case "GenAI_NanoGPT_v1":
|
|
7
|
-
return new o(e);
|
|
8
|
-
case "GenAI_NanoGPT_v2":
|
|
9
|
-
return new r(e);
|
|
10
|
-
default:
|
|
11
|
-
throw new Error(`Unsupported model type: ${n}`);
|
|
12
|
-
}
|
|
13
|
-
}
|
|
14
|
-
export {
|
|
15
|
-
p as default
|
|
16
|
-
};
|
package/dist/models/model.d.ts
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
import { Tensor } from '@tensorflow/tfjs-core';
|
|
2
|
-
import { ForwardAttributes, default as BaseLayer } from '../layers/BaseLayer';
|
|
3
|
-
import { AttentionScores, KVCache } from '../layers/CausalSelfAttention';
|
|
4
|
-
import { TransformersMetadata } from '../loader/types';
|
|
5
|
-
import { GPTConfig, LoRAConfig } from './config';
|
|
6
|
-
import { default as LoRA } from '../layers/LoRA';
|
|
7
|
-
export interface ModelForwardAttributes extends ForwardAttributes {
|
|
8
|
-
cache?: KVCache[];
|
|
9
|
-
attentionScores?: AttentionScores;
|
|
10
|
-
seed?: number;
|
|
11
|
-
skipLogits?: boolean;
|
|
12
|
-
ropePositionOffset?: number;
|
|
13
|
-
}
|
|
14
|
-
export interface TrainingState {
|
|
15
|
-
steps: number;
|
|
16
|
-
learningRate: number;
|
|
17
|
-
batchSize: number;
|
|
18
|
-
loss: number;
|
|
19
|
-
tokensProcessed: number;
|
|
20
|
-
duration: number;
|
|
21
|
-
}
|
|
22
|
-
export default abstract class Model<T extends ModelForwardAttributes, C extends GPTConfig = GPTConfig> extends BaseLayer<T, C> {
|
|
23
|
-
lossScaling: number;
|
|
24
|
-
trainingState: TrainingState | null;
|
|
25
|
-
metaData: TransformersMetadata;
|
|
26
|
-
private loraLayer?;
|
|
27
|
-
private loraMap;
|
|
28
|
-
constructor(config: C);
|
|
29
|
-
createLoRA(name: string, loraConfig: LoRAConfig): void;
|
|
30
|
-
deleteLoRA(name: string): void;
|
|
31
|
-
renameLoRA(oldName: string, newName: string): void;
|
|
32
|
-
mergeLoRA(name: string): void;
|
|
33
|
-
attachLoRA(name: string): void;
|
|
34
|
-
detachLoRA(): void;
|
|
35
|
-
hasLoRA(name?: string): boolean;
|
|
36
|
-
listLoRAs(): string[];
|
|
37
|
-
get lora(): LoRA | null;
|
|
38
|
-
abstract getClassName(): string;
|
|
39
|
-
abstract forward(attrs: T, idx: Tensor): Tensor;
|
|
40
|
-
abstract project(embeddings: Tensor): Tensor;
|
|
41
|
-
abstract dispose(): void;
|
|
42
|
-
getNumParams(): number;
|
|
43
|
-
protected validateInput(idx: Tensor): void;
|
|
44
|
-
}
|
package/dist/models/model.js
DELETED
|
@@ -1,134 +0,0 @@
|
|
|
1
|
-
import i from "../layers/BaseLayer.js";
|
|
2
|
-
import "../index-CUXkjxiT.js";
|
|
3
|
-
import "../random_width-DN5ZtQkM.js";
|
|
4
|
-
import "../zeros_like-CWjDdwr-.js";
|
|
5
|
-
import "../Generator.js";
|
|
6
|
-
import "../index-Cp39cXWe.js";
|
|
7
|
-
import "../dataset-CGGp1z9P.js";
|
|
8
|
-
import "../ops/cpu/attentionMask.js";
|
|
9
|
-
import "../ops/webgl/attentionMask.js";
|
|
10
|
-
import "../ops/grads/attentionMask.js";
|
|
11
|
-
import "../ops/cpu/rope.js";
|
|
12
|
-
import "../ops/webgl/rope.js";
|
|
13
|
-
import "../rope-DcrZM_e6.js";
|
|
14
|
-
import "../ops/cpu/appendCache.js";
|
|
15
|
-
import "../ops/webgl/appendCache.js";
|
|
16
|
-
import "../ops/grads/softmax16.js";
|
|
17
|
-
import "../matMul16-BcVC_E62.js";
|
|
18
|
-
import "../ops/webgl/matMul16.js";
|
|
19
|
-
import "../ops/cpu/matMul16.js";
|
|
20
|
-
import "../pack16-bqltoUlR.js";
|
|
21
|
-
import "../ops/transpose16.js";
|
|
22
|
-
import "../ops/reshape16.js";
|
|
23
|
-
import "../ops/cpu/qkv.js";
|
|
24
|
-
import "../ops/webgl/qkv.js";
|
|
25
|
-
import "../ops/grads/qkv.js";
|
|
26
|
-
import "../ops/cpu/normRMS.js";
|
|
27
|
-
import "../ops/webgl/normRMS.js";
|
|
28
|
-
import "../ops/grads/normRMS.js";
|
|
29
|
-
import "../ops/dropout16.js";
|
|
30
|
-
import "../ops/webgl/dropout16.js";
|
|
31
|
-
import "../ops/grads/add16.js";
|
|
32
|
-
import "../jszip.min-BZhlzntC.js";
|
|
33
|
-
import "../index-DvYrXKkX.js";
|
|
34
|
-
import "../ops/cpu/adamAdjust.js";
|
|
35
|
-
import "../ops/webgl/adamAdjust.js";
|
|
36
|
-
import "../ops/cpu/adamMoments.js";
|
|
37
|
-
import "../ops/webgl/adamMoments.js";
|
|
38
|
-
import "../ops/cpu/gatherSub.js";
|
|
39
|
-
import "../ops/webgl/gatherSub.js";
|
|
40
|
-
import "../ops/cpu/scatterSub.js";
|
|
41
|
-
import "../ops/webgl/scatterSub.js";
|
|
42
|
-
import "../papaparse.min-C0cScC2i.js";
|
|
43
|
-
import { estimateParameterCount as a } from "../utilities/parameters.js";
|
|
44
|
-
import "../ops/cpu/matMulGelu.js";
|
|
45
|
-
import "../matMulGelu-JNLZqKQp.js";
|
|
46
|
-
import "../ops/grads/matMulGelu.js";
|
|
47
|
-
import "../ops/cpu/gelu.js";
|
|
48
|
-
import "../ops/webgl/gelu.js";
|
|
49
|
-
import "../gelu-B220X1Go.js";
|
|
50
|
-
import "../ops/webgl/log.js";
|
|
51
|
-
import "../checks/normRMS.js";
|
|
52
|
-
import "../checks/normRMSGrad.js";
|
|
53
|
-
import e from "../layers/LoRA.js";
|
|
54
|
-
class st extends i {
|
|
55
|
-
lossScaling = 128;
|
|
56
|
-
trainingState = null;
|
|
57
|
-
metaData = { version: 2, application: "@genai-fi/nanogpt" };
|
|
58
|
-
loraLayer;
|
|
59
|
-
loraMap = /* @__PURE__ */ new Map();
|
|
60
|
-
constructor(t) {
|
|
61
|
-
super(t), t.loraConfig && t.loraConfig.forEach((r, o) => {
|
|
62
|
-
this.createLoRA(o, r);
|
|
63
|
-
});
|
|
64
|
-
}
|
|
65
|
-
createLoRA(t, r) {
|
|
66
|
-
if (this.loraMap.has(t))
|
|
67
|
-
return;
|
|
68
|
-
const o = new e(t, this.weightStore, r.alpha, r.rank, r.variables);
|
|
69
|
-
this.loraMap.set(t, o), this.config.loraConfig = this.config.loraConfig || /* @__PURE__ */ new Map(), this.config.loraConfig.set(t, r);
|
|
70
|
-
}
|
|
71
|
-
deleteLoRA(t) {
|
|
72
|
-
const r = this.loraMap.get(t);
|
|
73
|
-
if (!r)
|
|
74
|
-
throw new Error(`No LoRA with name ${t} exists.`);
|
|
75
|
-
this.loraLayer === r && this.detachLoRA(), r.dispose(), this.loraMap.delete(t), this.config.loraConfig && this.config.loraConfig.delete(t);
|
|
76
|
-
}
|
|
77
|
-
renameLoRA(t, r) {
|
|
78
|
-
if (!this.loraMap.has(t))
|
|
79
|
-
throw new Error(`No LoRA with name ${t} exists.`);
|
|
80
|
-
if (this.loraMap.has(r))
|
|
81
|
-
throw new Error(`LoRA with name ${r} already exists.`);
|
|
82
|
-
const o = this.loraMap.get(t);
|
|
83
|
-
this.loraMap.set(r, o), this.loraMap.delete(t), this.config.loraConfig && (this.config.loraConfig.delete(t), this.config.loraConfig.set(r, {
|
|
84
|
-
rank: o.rank,
|
|
85
|
-
alpha: o.alpha,
|
|
86
|
-
variables: Array.from(o.variables)
|
|
87
|
-
}));
|
|
88
|
-
}
|
|
89
|
-
mergeLoRA(t) {
|
|
90
|
-
const r = this.loraMap.get(t);
|
|
91
|
-
if (!r)
|
|
92
|
-
throw new Error(`No LoRA with name ${t} exists.`);
|
|
93
|
-
r.merge(), this.deleteLoRA(t);
|
|
94
|
-
}
|
|
95
|
-
attachLoRA(t) {
|
|
96
|
-
if (this.loraLayer) {
|
|
97
|
-
if (this.loraLayer.name === t)
|
|
98
|
-
return;
|
|
99
|
-
this.detachLoRA();
|
|
100
|
-
}
|
|
101
|
-
const r = this.loraMap.get(t);
|
|
102
|
-
if (!r)
|
|
103
|
-
throw new Error(`No LoRA with name ${t} exists.`);
|
|
104
|
-
r.attach(), this.loraLayer = r, this.config.loraName = t;
|
|
105
|
-
}
|
|
106
|
-
detachLoRA() {
|
|
107
|
-
if (!this.loraLayer)
|
|
108
|
-
throw new Error("No LoRA layer is attached to this model.");
|
|
109
|
-
this.loraLayer.detach(), this.loraLayer = void 0, this.config.loraName = void 0;
|
|
110
|
-
}
|
|
111
|
-
hasLoRA(t) {
|
|
112
|
-
return t ? this.loraMap.has(t) : !!this.loraLayer;
|
|
113
|
-
}
|
|
114
|
-
listLoRAs() {
|
|
115
|
-
return Array.from(this.loraMap.keys());
|
|
116
|
-
}
|
|
117
|
-
get lora() {
|
|
118
|
-
return this.loraLayer || null;
|
|
119
|
-
}
|
|
120
|
-
getNumParams() {
|
|
121
|
-
return a(this.config);
|
|
122
|
-
}
|
|
123
|
-
validateInput(t) {
|
|
124
|
-
if (t.shape.length !== 2)
|
|
125
|
-
throw new Error(`Invalid input shape: expected [batch_size, sequence_length], got ${t.shape}`);
|
|
126
|
-
if (t.shape[1] > this.config.blockSize)
|
|
127
|
-
throw new Error(`Input sequence length ${t.shape[1]} isn't block size ${this.config.blockSize}`);
|
|
128
|
-
if (t.dtype !== "int32")
|
|
129
|
-
throw new Error(`Input tensor must be of type int32, got ${t.dtype}`);
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
export {
|
|
133
|
-
st as default
|
|
134
|
-
};
|
|
@@ -1,102 +0,0 @@
|
|
|
1
|
-
function J(e, t, o) {
|
|
2
|
-
const n = k(e, t, o), s = n < 0 ? -(n + 1) : n;
|
|
3
|
-
e.splice(s, 0, t);
|
|
4
|
-
}
|
|
5
|
-
function k(e, t, o) {
|
|
6
|
-
return A(e, t, o || v);
|
|
7
|
-
}
|
|
8
|
-
function v(e, t) {
|
|
9
|
-
return e > t ? 1 : e < t ? -1 : 0;
|
|
10
|
-
}
|
|
11
|
-
function A(e, t, o) {
|
|
12
|
-
let n = 0, s = e.length, r = 0, u = !1;
|
|
13
|
-
for (; n < s; ) {
|
|
14
|
-
r = n + (s - n >>> 1);
|
|
15
|
-
const l = o(t, e[r]);
|
|
16
|
-
l > 0 ? n = r + 1 : (s = r, u = !l);
|
|
17
|
-
}
|
|
18
|
-
return u ? n : -n - 1;
|
|
19
|
-
}
|
|
20
|
-
function O(e, t, o, n, s) {
|
|
21
|
-
return y(
|
|
22
|
-
e,
|
|
23
|
-
t,
|
|
24
|
-
o,
|
|
25
|
-
n,
|
|
26
|
-
s,
|
|
27
|
-
0
|
|
28
|
-
/* softNmsSigma */
|
|
29
|
-
);
|
|
30
|
-
}
|
|
31
|
-
function P(e, t, o, n, s, r) {
|
|
32
|
-
return y(
|
|
33
|
-
e,
|
|
34
|
-
t,
|
|
35
|
-
o,
|
|
36
|
-
n,
|
|
37
|
-
s,
|
|
38
|
-
0,
|
|
39
|
-
!1,
|
|
40
|
-
r,
|
|
41
|
-
!0
|
|
42
|
-
/* returnValidOutputs */
|
|
43
|
-
);
|
|
44
|
-
}
|
|
45
|
-
function X(e, t, o, n, s, r) {
|
|
46
|
-
return y(
|
|
47
|
-
e,
|
|
48
|
-
t,
|
|
49
|
-
o,
|
|
50
|
-
n,
|
|
51
|
-
s,
|
|
52
|
-
r,
|
|
53
|
-
!0
|
|
54
|
-
/* returnScoresTensor */
|
|
55
|
-
);
|
|
56
|
-
}
|
|
57
|
-
function y(e, t, o, n, s, r, u = !1, l = !1, h = !1) {
|
|
58
|
-
const c = [];
|
|
59
|
-
for (let i = 0; i < t.length; i++)
|
|
60
|
-
t[i] > s && c.push({ score: t[i], boxIndex: i, suppressBeginIndex: 0 });
|
|
61
|
-
c.sort(C);
|
|
62
|
-
const d = r > 0 ? -0.5 / r : 0, a = [], f = [];
|
|
63
|
-
for (; a.length < o && c.length > 0; ) {
|
|
64
|
-
const i = c.pop(), { score: M, boxIndex: I, suppressBeginIndex: g } = i;
|
|
65
|
-
if (M < s)
|
|
66
|
-
break;
|
|
67
|
-
let S = !1;
|
|
68
|
-
for (let b = a.length - 1; b >= g; --b) {
|
|
69
|
-
const w = B(e, I, a[b]);
|
|
70
|
-
if (w >= n) {
|
|
71
|
-
S = !0;
|
|
72
|
-
break;
|
|
73
|
-
}
|
|
74
|
-
if (i.score = i.score * V(n, d, w), i.score <= s)
|
|
75
|
-
break;
|
|
76
|
-
}
|
|
77
|
-
i.suppressBeginIndex = a.length, S || (i.score === M ? (a.push(I), f.push(i.score)) : i.score > s && J(c, i, C));
|
|
78
|
-
}
|
|
79
|
-
const x = a.length, p = o - x;
|
|
80
|
-
l && p > 0 && (a.push(...new Array(p).fill(0)), f.push(...new Array(p).fill(0)));
|
|
81
|
-
const m = { selectedIndices: a };
|
|
82
|
-
return u && (m.selectedScores = f), h && (m.validOutputs = x), m;
|
|
83
|
-
}
|
|
84
|
-
function B(e, t, o) {
|
|
85
|
-
const n = e.subarray(t * 4, t * 4 + 4), s = e.subarray(o * 4, o * 4 + 4), r = Math.min(n[0], n[2]), u = Math.min(n[1], n[3]), l = Math.max(n[0], n[2]), h = Math.max(n[1], n[3]), c = Math.min(s[0], s[2]), d = Math.min(s[1], s[3]), a = Math.max(s[0], s[2]), f = Math.max(s[1], s[3]), x = (l - r) * (h - u), p = (a - c) * (f - d);
|
|
86
|
-
if (x <= 0 || p <= 0)
|
|
87
|
-
return 0;
|
|
88
|
-
const m = Math.max(r, c), i = Math.max(u, d), M = Math.min(l, a), I = Math.min(h, f), g = Math.max(M - m, 0) * Math.max(I - i, 0);
|
|
89
|
-
return g / (x + p - g);
|
|
90
|
-
}
|
|
91
|
-
function V(e, t, o) {
|
|
92
|
-
const n = Math.exp(t * o * o);
|
|
93
|
-
return o <= e ? n : 0;
|
|
94
|
-
}
|
|
95
|
-
function C(e, t) {
|
|
96
|
-
return e.score - t.score || e.score === t.score && t.boxIndex - e.boxIndex;
|
|
97
|
-
}
|
|
98
|
-
export {
|
|
99
|
-
X as a,
|
|
100
|
-
P as b,
|
|
101
|
-
O as n
|
|
102
|
-
};
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
import { o as c, q as e, _ as h, E as u, aL as b, aM as E, aN as _, aO as g, ah as S, a1 as w, j as K, aP as q, B as M, aQ as T } from "./index-CUXkjxiT.js";
|
|
2
|
-
import { b as d } from "./broadcast_to-eS93CCN_.js";
|
|
3
|
-
import { e as N } from "./axis_util-GTVlo58H.js";
|
|
4
|
-
import { r as f } from "./reshape-BDOuCSNW.js";
|
|
5
|
-
import { s as A } from "./sum-BpiwSWvg.js";
|
|
6
|
-
function B(s, o, n) {
|
|
7
|
-
const a = e(o, "a", "where"), t = e(n, "b", "where"), r = e(s, "condition", "where", "bool"), p = h(h(r.shape, a.shape), t.shape), l = d(r, p), m = d(a, p), x = d(t, p), i = {
|
|
8
|
-
condition: l,
|
|
9
|
-
t: m,
|
|
10
|
-
e: x
|
|
11
|
-
};
|
|
12
|
-
return u.runKernel(b, i);
|
|
13
|
-
}
|
|
14
|
-
const H = /* @__PURE__ */ c({ where_: B });
|
|
15
|
-
function G(s, o = null, n = !1) {
|
|
16
|
-
const t = { x: e(s, "x", "max") }, r = { reductionIndices: o, keepDims: n };
|
|
17
|
-
return u.runKernel(E, t, r);
|
|
18
|
-
}
|
|
19
|
-
const I = /* @__PURE__ */ c({ max_: G });
|
|
20
|
-
function L(s) {
|
|
21
|
-
const n = { x: e(s, "x", "exp") };
|
|
22
|
-
return u.runKernel(_, n);
|
|
23
|
-
}
|
|
24
|
-
const P = /* @__PURE__ */ c({ exp_: L });
|
|
25
|
-
function j(s) {
|
|
26
|
-
const n = { x: e(s, "x", "log", "float32") };
|
|
27
|
-
return u.runKernel(g, n);
|
|
28
|
-
}
|
|
29
|
-
const k = /* @__PURE__ */ c({ log_: j });
|
|
30
|
-
function v(s, o = null, n = !1) {
|
|
31
|
-
const a = e(s, "x", "logSumExp"), t = S(o, a.shape), r = I(
|
|
32
|
-
a,
|
|
33
|
-
t,
|
|
34
|
-
!0
|
|
35
|
-
/* keepDims */
|
|
36
|
-
), p = w(a, r), l = P(p), m = A(l, t), x = k(m), i = K(f(r, x.shape), x);
|
|
37
|
-
if (n) {
|
|
38
|
-
const $ = N(i.shape, t);
|
|
39
|
-
return f(i, $);
|
|
40
|
-
}
|
|
41
|
-
return i;
|
|
42
|
-
}
|
|
43
|
-
const J = /* @__PURE__ */ c({ logSumExp_: v });
|
|
44
|
-
function y(s, o = null, n = !1) {
|
|
45
|
-
const t = { x: e(s, "x", "mean") }, r = { axis: o, keepDims: n };
|
|
46
|
-
return u.runKernel(q, t, r);
|
|
47
|
-
}
|
|
48
|
-
const R = /* @__PURE__ */ c({ mean_: y });
|
|
49
|
-
function C(s, o) {
|
|
50
|
-
let n = e(s, "a", "notEqual", "string_or_numeric"), a = e(o, "b", "notEqual", "string_or_numeric");
|
|
51
|
-
[n, a] = M(n, a), h(n.shape, a.shape);
|
|
52
|
-
const t = { a: n, b: a };
|
|
53
|
-
return u.runKernel(T, t);
|
|
54
|
-
}
|
|
55
|
-
const U = /* @__PURE__ */ c({ notEqual_: C });
|
|
56
|
-
export {
|
|
57
|
-
R as a,
|
|
58
|
-
k as b,
|
|
59
|
-
P as e,
|
|
60
|
-
J as l,
|
|
61
|
-
I as m,
|
|
62
|
-
U as n,
|
|
63
|
-
H as w
|
|
64
|
-
};
|