@genai-fi/nanogpt 0.19.1 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/dist/BaseTokeniser-DSg9zcYq.js +221 -0
  2. package/dist/DatasetBuilder-DgURD85T.js +712 -0
  3. package/dist/Generator.js +2 -11941
  4. package/dist/RealDiv-DBu0FQqT.js +362 -0
  5. package/dist/Reshape-CABOPB9d.js +94 -0
  6. package/dist/Reshape-DqO3r8BC.js +17 -0
  7. package/dist/TeachableLLM.d.ts +5 -5
  8. package/dist/TeachableLLM.js +2 -273
  9. package/dist/Trainer.js +2 -244
  10. package/dist/backend.js +12 -12
  11. package/dist/backend_util-Cg-roD1p.js +399 -0
  12. package/dist/binary_op_util-CrYk9LXL.js +103 -0
  13. package/dist/checks/appendCache.js +54 -21
  14. package/dist/checks/attentionMask.js +55 -36
  15. package/dist/checks/check.js +31 -19
  16. package/dist/checks/gelu.js +45 -17
  17. package/dist/checks/index.js +25 -25
  18. package/dist/checks/matMulGelu.js +83 -27
  19. package/dist/checks/normRMS.js +27 -15
  20. package/dist/checks/normRMSGrad.js +21 -11
  21. package/dist/checks/packUnpack.js +45 -17
  22. package/dist/checks/qkv.js +33 -33
  23. package/dist/checks/rope.js +29 -35
  24. package/dist/checks/weights.d.ts +1 -1
  25. package/dist/checks/weights.js +25 -29
  26. package/dist/chunk-BPntVaq0.js +23 -0
  27. package/dist/complex_util-CkazZsaH.js +60 -0
  28. package/dist/concat_util-CWDZCBlA.js +19 -0
  29. package/dist/data/docx.js +3044 -13
  30. package/dist/data/pdf.js +16 -13
  31. package/dist/data/textLoader.js +607 -112
  32. package/dist/dist-BewPQWjc.js +7572 -0
  33. package/dist/dist-DVmq73nz.js +8775 -0
  34. package/dist/dist-DXwIvKxl.js +896 -0
  35. package/dist/dist-VEU5mfO0.js +7545 -0
  36. package/dist/gelu-Bf1HW1RY.js +27 -0
  37. package/dist/gpgpu_math-DvLcCH6u.js +1612 -0
  38. package/dist/inference/types.js +0 -1
  39. package/dist/kernel_funcs_utils-HiXOOx3f.js +229 -0
  40. package/dist/layers/BaseLayer.d.ts +2 -2
  41. package/dist/layers/BaseLayer.js +75 -73
  42. package/dist/layers/CausalSelfAttention.d.ts +1 -1
  43. package/dist/layers/CausalSelfAttention.js +98 -85
  44. package/dist/layers/LoRA.js +47 -57
  45. package/dist/layers/MLP.d.ts +1 -1
  46. package/dist/layers/MLP.js +33 -43
  47. package/dist/layers/PositionEmbedding.d.ts +1 -1
  48. package/dist/layers/PositionEmbedding.js +26 -30
  49. package/dist/layers/RMSNorm.d.ts +1 -1
  50. package/dist/layers/RMSNorm.js +19 -21
  51. package/dist/layers/RoPECache.js +336 -49
  52. package/dist/layers/TiedEmbedding.d.ts +1 -1
  53. package/dist/layers/TiedEmbedding.js +30 -34
  54. package/dist/layers/TransformerBlock.d.ts +1 -1
  55. package/dist/layers/TransformerBlock.js +50 -39
  56. package/dist/layers/WeightStore.js +68 -75
  57. package/dist/loader/load.js +2 -68
  58. package/dist/loader/loadHF.d.ts +2 -2
  59. package/dist/loader/loadHF.js +2 -22
  60. package/dist/loader/loadTransformers.d.ts +1 -1
  61. package/dist/loader/loadTransformers.js +2 -44
  62. package/dist/loader/loadZipMeta.js +15 -15
  63. package/dist/loader/newZipLoad.js +2 -31
  64. package/dist/loader/oldZipLoad.d.ts +2 -2
  65. package/dist/loader/oldZipLoad.js +2 -80
  66. package/dist/loader/save.d.ts +5 -5
  67. package/dist/loader/save.js +2 -90
  68. package/dist/loader/types.d.ts +9 -8
  69. package/dist/loader/types.js +0 -1
  70. package/dist/main-CPjeMv0G.js +13500 -0
  71. package/dist/main.d.ts +1 -1
  72. package/dist/main.js +16 -109
  73. package/dist/matMul16-BNfZSnNM.js +81 -0
  74. package/dist/matMulGelu-CPTntosE.js +162 -0
  75. package/dist/models/NanoGPTV1.js +2 -99
  76. package/dist/models/NanoGPTV2.js +2 -90
  77. package/dist/models/config.js +34 -47
  78. package/dist/models/factory.d.ts +1 -1
  79. package/dist/models/factory.js +2 -16
  80. package/dist/models/model.d.ts +2 -2
  81. package/dist/models/model.js +2 -134
  82. package/dist/ops/adamAdjust.js +15 -6
  83. package/dist/ops/adamMoments.js +13 -6
  84. package/dist/ops/add16.js +9 -6
  85. package/dist/ops/appendCache.js +22 -19
  86. package/dist/ops/attentionMask.js +12 -6
  87. package/dist/ops/concat16.js +7 -8
  88. package/dist/ops/cpu/adamAdjust.js +15 -17
  89. package/dist/ops/cpu/adamMoments.js +15 -15
  90. package/dist/ops/cpu/appendCache.js +64 -22
  91. package/dist/ops/cpu/attentionMask.js +15 -21
  92. package/dist/ops/cpu/fusedSoftmax.js +21 -28
  93. package/dist/ops/cpu/gatherSub.js +11 -17
  94. package/dist/ops/cpu/gelu.js +34 -38
  95. package/dist/ops/cpu/matMul16.js +13 -14
  96. package/dist/ops/cpu/matMulGelu.js +39 -51
  97. package/dist/ops/cpu/matMulMul.js +19 -22
  98. package/dist/ops/cpu/mulDropout.js +19 -22
  99. package/dist/ops/cpu/normRMS.js +33 -37
  100. package/dist/ops/cpu/qkv.js +72 -40
  101. package/dist/ops/cpu/rope.js +79 -36
  102. package/dist/ops/cpu/scatterSub.js +11 -22
  103. package/dist/ops/dot16.js +28 -41
  104. package/dist/ops/dropout.js +10 -13
  105. package/dist/ops/dropout16.js +20 -23
  106. package/dist/ops/gatherSub.js +10 -6
  107. package/dist/ops/gelu.js +2 -8
  108. package/dist/ops/globalNorm.js +18 -12
  109. package/dist/ops/grads/add16.js +27 -26
  110. package/dist/ops/grads/attentionMask.js +26 -21
  111. package/dist/ops/grads/dropout16.js +0 -1
  112. package/dist/ops/grads/gelu.js +2 -5
  113. package/dist/ops/grads/matMul16.js +2 -9
  114. package/dist/ops/grads/matMulGelu.js +21 -16
  115. package/dist/ops/grads/mul16.js +0 -3
  116. package/dist/ops/grads/normRMS.js +34 -30
  117. package/dist/ops/grads/pack16.js +2 -6
  118. package/dist/ops/grads/qkv.js +44 -32
  119. package/dist/ops/grads/rope.js +2 -5
  120. package/dist/ops/grads/softmax16.js +21 -23
  121. package/dist/ops/grads/unpack16.js +2 -5
  122. package/dist/ops/grads/utils.js +9 -11
  123. package/dist/ops/matMul16.js +2 -13
  124. package/dist/ops/matMulGelu.js +16 -10
  125. package/dist/ops/matMulMul.js +13 -6
  126. package/dist/ops/mul16.js +42 -38
  127. package/dist/ops/mulDrop.js +12 -6
  128. package/dist/ops/normRMS.js +18 -15
  129. package/dist/ops/pack16.js +2 -5
  130. package/dist/ops/qkv.js +12 -6
  131. package/dist/ops/reshape16.js +31 -39
  132. package/dist/ops/rope.d.ts +1 -1
  133. package/dist/ops/rope.js +2 -7
  134. package/dist/ops/scatterSub.js +10 -6
  135. package/dist/ops/slice16.js +9 -7
  136. package/dist/ops/softmax16.js +7 -7
  137. package/dist/ops/sub16.js +9 -6
  138. package/dist/ops/sum16.js +12 -12
  139. package/dist/ops/transpose16.js +29 -37
  140. package/dist/ops/unpack16.js +2 -6
  141. package/dist/ops/webgl/adamAdjust.js +62 -29
  142. package/dist/ops/webgl/adamMoments.js +30 -26
  143. package/dist/ops/webgl/appendCache.js +30 -21
  144. package/dist/ops/webgl/attentionMask.js +43 -24
  145. package/dist/ops/webgl/dropout16.js +11 -10
  146. package/dist/ops/webgl/fusedSoftmax.js +69 -79
  147. package/dist/ops/webgl/gatherSub.js +27 -26
  148. package/dist/ops/webgl/gelu.js +32 -34
  149. package/dist/ops/webgl/log.js +14 -23
  150. package/dist/ops/webgl/matMul16.js +36 -44
  151. package/dist/ops/webgl/matMulGelu.js +2 -9
  152. package/dist/ops/webgl/matMulMul.js +23 -27
  153. package/dist/ops/webgl/mulDropout.js +31 -40
  154. package/dist/ops/webgl/normRMS.js +92 -71
  155. package/dist/ops/webgl/qkv.js +35 -27
  156. package/dist/ops/webgl/rope.js +37 -21
  157. package/dist/ops/webgl/scatterSub.js +27 -26
  158. package/dist/ops/webgpu/adamAdjust.js +59 -39
  159. package/dist/ops/webgpu/adamMoments.js +62 -46
  160. package/dist/ops/webgpu/add16.js +13 -12
  161. package/dist/ops/webgpu/appendCache.js +79 -54
  162. package/dist/ops/webgpu/attentionMask.js +41 -25
  163. package/dist/ops/webgpu/attentionMask32_program.js +34 -26
  164. package/dist/ops/webgpu/clipScale.js +44 -57
  165. package/dist/ops/webgpu/concat16.js +96 -111
  166. package/dist/ops/webgpu/dropout16.js +40 -32
  167. package/dist/ops/webgpu/gatherSub.js +43 -30
  168. package/dist/ops/webgpu/gelu.js +88 -82
  169. package/dist/ops/webgpu/index.js +16 -16
  170. package/dist/ops/webgpu/matMul16.js +69 -64
  171. package/dist/ops/webgpu/matMul16_program.js +152 -192
  172. package/dist/ops/webgpu/mul16.js +13 -12
  173. package/dist/ops/webgpu/norm2.js +45 -75
  174. package/dist/ops/webgpu/normRMS.js +25 -33
  175. package/dist/ops/webgpu/normRMS16_program.js +21 -18
  176. package/dist/ops/webgpu/normRMS32_program.js +21 -18
  177. package/dist/ops/webgpu/normRMSGrad.js +125 -184
  178. package/dist/ops/webgpu/pack16.js +20 -17
  179. package/dist/ops/webgpu/pack16_program.js +48 -47
  180. package/dist/ops/webgpu/qkv.js +63 -23
  181. package/dist/ops/webgpu/rope.js +85 -57
  182. package/dist/ops/webgpu/scatterSub.js +43 -30
  183. package/dist/ops/webgpu/slice16.js +66 -61
  184. package/dist/ops/webgpu/softmax16.js +17 -20
  185. package/dist/ops/webgpu/softmax16_program.js +34 -18
  186. package/dist/ops/webgpu/softmax16_subgroup_program.js +40 -45
  187. package/dist/ops/webgpu/softmax16grad.js +30 -36
  188. package/dist/ops/webgpu/sub16.js +13 -12
  189. package/dist/ops/webgpu/sum16.js +28 -37
  190. package/dist/ops/webgpu/transpose16.js +36 -33
  191. package/dist/ops/webgpu/transpose16_program.js +40 -39
  192. package/dist/ops/webgpu/transpose16_shared_program.js +53 -44
  193. package/dist/ops/webgpu/unpack16.js +49 -37
  194. package/dist/ops/webgpu/utils/binary_op.js +70 -68
  195. package/dist/ops/webgpu/utils/deviceInfo.d.ts +1 -1
  196. package/dist/ops/webgpu/utils/deviceInfo.js +10 -10
  197. package/dist/ops/webgpu/utils/reductions.js +136 -148
  198. package/dist/pack16-Ck-spx_F.js +39 -0
  199. package/dist/patches/webgpu_backend.d.ts +2 -2
  200. package/dist/patches/webgpu_backend.js +42 -55
  201. package/dist/patches/webgpu_base.js +21 -33
  202. package/dist/patches/webgpu_program.js +213 -320
  203. package/dist/pdf-UoDqCYzz.js +16726 -0
  204. package/dist/picomatch-3tUnMMbd.js +1063 -0
  205. package/dist/rope-CbeGlsV8.js +25 -0
  206. package/dist/selu_util-zkAx5doH.js +24 -0
  207. package/dist/shared-D1coEFea.js +1314 -0
  208. package/dist/shared-DOgWaqvL.js +5 -0
  209. package/dist/slice_util-Dgb3ANWI.js +208 -0
  210. package/dist/tfjs_backend-BjuQ5FqB.js +614 -0
  211. package/dist/tokeniser/BaseTokeniser.js +2 -124
  212. package/dist/tokeniser/CharTokeniser.js +91 -106
  213. package/dist/tokeniser/bpe.js +163 -166
  214. package/dist/tokeniser/messages.js +0 -1
  215. package/dist/tokeniser/type.js +0 -1
  216. package/dist/training/AdamW.js +127 -137
  217. package/dist/training/BasicTrainer.d.ts +1 -1
  218. package/dist/training/BasicTrainer.js +264 -264
  219. package/dist/training/DatasetBuilder.js +2 -86
  220. package/dist/training/Evaluator.d.ts +1 -1
  221. package/dist/training/Evaluator.js +47 -38
  222. package/dist/training/LRScheduler.js +37 -33
  223. package/dist/training/PreTrainer.d.ts +2 -2
  224. package/dist/training/PreTrainer.js +21 -19
  225. package/dist/training/SFTTrainer.d.ts +2 -2
  226. package/dist/training/SFTTrainer.js +23 -21
  227. package/dist/training/loss.js +17 -22
  228. package/dist/training/orthoGrad.js +9 -9
  229. package/dist/training/sparseCrossEntropy.js +45 -67
  230. package/dist/training/tasks/ConversationTask.d.ts +1 -1
  231. package/dist/training/tasks/ConversationTask.js +36 -38
  232. package/dist/training/tasks/PretrainingTask.d.ts +1 -1
  233. package/dist/training/tasks/PretrainingTask.js +41 -46
  234. package/dist/training/tasks/StartSentenceTask.d.ts +1 -1
  235. package/dist/training/tasks/StartSentenceTask.js +44 -48
  236. package/dist/training/tasks/Task.d.ts +1 -1
  237. package/dist/training/tasks/Task.js +53 -66
  238. package/dist/training/tasks/splitter.js +17 -20
  239. package/dist/training/types.d.ts +2 -2
  240. package/dist/training/types.js +0 -1
  241. package/dist/training/validation.d.ts +1 -1
  242. package/dist/training/validation.js +2 -84
  243. package/dist/utilities/arrayClose.js +15 -19
  244. package/dist/utilities/datasetID.js +17 -20
  245. package/dist/utilities/dummy.d.ts +1 -1
  246. package/dist/utilities/dummy.js +33 -40
  247. package/dist/utilities/multinomialCPU.js +8 -12
  248. package/dist/utilities/naming.js +0 -1
  249. package/dist/utilities/packed.js +10 -12
  250. package/dist/utilities/parameters.d.ts +1 -1
  251. package/dist/utilities/parameters.js +32 -51
  252. package/dist/utilities/performance.js +15 -15
  253. package/dist/utilities/profile.js +32 -37
  254. package/dist/utilities/safetensors.js +49 -79
  255. package/dist/utilities/sentences.d.ts +1 -1
  256. package/dist/utilities/sentences.js +29 -38
  257. package/dist/utilities/tokenParse.js +16 -20
  258. package/dist/utilities/topP.js +11 -12
  259. package/dist/utilities/waitForModel.d.ts +1 -1
  260. package/dist/utilities/waitForModel.js +11 -11
  261. package/dist/utilities/weights.js +37 -42
  262. package/dist/utilities/yielder.js +6 -6
  263. package/dist/webgpu-Dt7BMzWz.js +525 -0
  264. package/dist/webgpu_program-WOyIVMlZ.js +392 -0
  265. package/dist/webgpu_util-B_F3SShA.js +106 -0
  266. package/package.json +9 -10
  267. package/dist/RealDiv-CGwv0liw.js +0 -365
  268. package/dist/Reshape-BW__R4mZ.js +0 -79
  269. package/dist/Reshape-CPBkTIH2.js +0 -14
  270. package/dist/_commonjsHelpers-ByX85dGu.js +0 -33
  271. package/dist/axis_util-GTVlo58H.js +0 -55
  272. package/dist/backend_util-GaFarB78.js +0 -425
  273. package/dist/backend_webgpu-BqASlsbV.js +0 -545
  274. package/dist/binary_op_util-pKXltfxI.js +0 -192
  275. package/dist/broadcast_to-eS93CCN_.js +0 -28
  276. package/dist/clip_by_value-DDA7rrcT.js +0 -12
  277. package/dist/complex-DI35Q-gW.js +0 -11
  278. package/dist/complex_util-Yc1A_gV1.js +0 -55
  279. package/dist/concat-CAQpCret.js +0 -17
  280. package/dist/concat_util-D18dJ4fD.js +0 -22
  281. package/dist/data/parquet.d.ts +0 -2
  282. package/dist/data/parquet.js +0 -17
  283. package/dist/dataset-CGGp1z9P.js +0 -1124
  284. package/dist/dropout_util--NxWuYg2.js +0 -27
  285. package/dist/expand_dims-Bkd1YD5x.js +0 -11
  286. package/dist/exports_initializers-CYzKLjN7.js +0 -7
  287. package/dist/floor-BQtb-Azg.js +0 -9
  288. package/dist/gather-qIqEqaGn.js +0 -9
  289. package/dist/gelu-B220X1Go.js +0 -26
  290. package/dist/gpgpu_math-BwvV12df.js +0 -2022
  291. package/dist/index-CUXkjxiT.js +0 -3516
  292. package/dist/index-CieiGp4Y.js +0 -349
  293. package/dist/index-CjOWnMXP.js +0 -7308
  294. package/dist/index-Cp39cXWe.js +0 -1016
  295. package/dist/index-D5v913EJ.js +0 -4
  296. package/dist/index-DmeWGGmS.js +0 -1074
  297. package/dist/index-DvYrXKkX.js +0 -113
  298. package/dist/index-Ksja3su6.js +0 -151
  299. package/dist/index-xuotMAFm.js +0 -118
  300. package/dist/jszip.min-BZhlzntC.js +0 -2313
  301. package/dist/kernel_funcs_utils-pq0CK9co.js +0 -306
  302. package/dist/matMul16-BcVC_E62.js +0 -80
  303. package/dist/matMulGelu-JNLZqKQp.js +0 -163
  304. package/dist/mat_mul-DhG0Newp.js +0 -11
  305. package/dist/mod-CSdCpRjf.js +0 -11
  306. package/dist/non_max_suppression_impl-B2W7YjZB.js +0 -102
  307. package/dist/not_equal-hurPF26l.js +0 -64
  308. package/dist/ones-BytntneX.js +0 -14
  309. package/dist/ops-CsXeTq1P.js +0 -476
  310. package/dist/pack16-bqltoUlR.js +0 -39
  311. package/dist/papaparse.min-C0cScC2i.js +0 -418
  312. package/dist/parquet-Bqjmp2vo.js +0 -44231
  313. package/dist/pdf-NIhmP3sq.js +0 -19477
  314. package/dist/rand_util-CZ7yLoUm.js +0 -50
  315. package/dist/random_normal-IBRrha8a.js +0 -14
  316. package/dist/random_width-DN5ZtQkM.js +0 -9796
  317. package/dist/range-C-CjF-LI.js +0 -10
  318. package/dist/relu-J_X6MUzx.js +0 -9
  319. package/dist/reshape-BDOuCSNW.js +0 -9
  320. package/dist/resize_nearest_neighbor-BojqlfRe.js +0 -150
  321. package/dist/rope-DcrZM_e6.js +0 -24
  322. package/dist/scatter_nd_util-ByNJaL6I.js +0 -46
  323. package/dist/segment_util-Dasb2Zaf.js +0 -43
  324. package/dist/selu_util-BLhIqRkw.js +0 -44
  325. package/dist/shared-3agzAqQ_.js +0 -53
  326. package/dist/shared-CagdqkLh.js +0 -2143
  327. package/dist/slice-BzS11Qh0.js +0 -12
  328. package/dist/slice_util-CC35pLmT.js +0 -153
  329. package/dist/softmax-D4q1LJN7.js +0 -12
  330. package/dist/split-C2Sj255c.js +0 -9
  331. package/dist/squeeze-ho4wLUek.js +0 -10
  332. package/dist/stack-DudVrtmG.js +0 -11
  333. package/dist/step-BTxPtq1r.js +0 -261
  334. package/dist/sum-BpiwSWvg.js +0 -11
  335. package/dist/tensor-BWFldCso.js +0 -8
  336. package/dist/tensor1d-LMGMIUlr.js +0 -11
  337. package/dist/tensor2d-BnXMKScO.js +0 -14
  338. package/dist/tensor4d-C6UCG_u8.js +0 -14
  339. package/dist/tfjs_backend-BGnG-ppu.js +0 -654
  340. package/dist/tile-CFy-xTO6.js +0 -11
  341. package/dist/transpose-9kRxIXWR.js +0 -36
  342. package/dist/unsorted_segment_sum-DJvk5xnh.js +0 -277
  343. package/dist/variable-Ck482e3n.js +0 -7
  344. package/dist/webgpu_program-B4HmApL1.js +0 -525
  345. package/dist/webgpu_util-DYlGSwOJ.js +0 -64
  346. package/dist/zeros-DvZpK8s6.js +0 -13
  347. package/dist/zeros_like-CWjDdwr-.js +0 -721
@@ -1,39 +1,48 @@
1
- import { t as d } from "../index-CUXkjxiT.js";
2
- import { calculateLoss as f, calculateAccuracy as p } from "./loss.js";
3
- class b {
4
- constructor(o, t, a) {
5
- this.model = o, this.masked = !!a, this.iterator = t.iterator();
6
- }
7
- iterator;
8
- xs;
9
- ys;
10
- masked = !1;
11
- dispose() {
12
- this.xs && this.xs.dispose(), this.ys && this.ys.dispose();
13
- }
14
- async calculateBatchLoss(o, t, a, r) {
15
- const [l, e] = d(() => {
16
- const s = this.model.forward({ training: !1 }, o), h = f(s, t, r, a), y = p(s, t);
17
- return s.dispose(), [h, y];
18
- }), u = await l.array(), n = await e.array(), c = u, i = n;
19
- return e.dispose(), l.dispose(), Array.isArray(c) ? c.map((s) => ({ loss: s, accuracy: i })) : { loss: c, accuracy: i };
20
- }
21
- async evaluate(o = 100) {
22
- let t = 0, a = 0, r = 0;
23
- if (this.iterator) {
24
- const l = await this.iterator;
25
- for (let e = 0; e < o; e++) {
26
- const u = await l.next();
27
- if (u.done) break;
28
- const n = u.value, { xs: c, ys: i } = n, s = await this.calculateBatchLoss(c, i, !1, this.masked);
29
- c.dispose(), i.dispose(), t += s.loss, a += s.accuracy, r++;
30
- }
31
- return { loss: t / r, accuracy: a / r };
32
- } else if (this.xs && this.ys)
33
- return this.calculateBatchLoss(this.xs, this.ys, !0, !0);
34
- throw new Error("No data available for evaluation");
35
- }
36
- }
37
- export {
38
- b as default
1
+ import { di as e } from "../dist-BewPQWjc.js";
2
+ import { calculateAccuracy as t, calculateLoss as n } from "./loss.js";
3
+ //#region lib/training/Evaluator.ts
4
+ var r = class {
5
+ model;
6
+ iterator;
7
+ xs;
8
+ ys;
9
+ masked = !1;
10
+ constructor(e, t, n) {
11
+ this.model = e, this.masked = !!n, this.iterator = t.iterator();
12
+ }
13
+ dispose() {
14
+ this.xs && this.xs.dispose(), this.ys && this.ys.dispose();
15
+ }
16
+ async calculateBatchLoss(r, i, a, o) {
17
+ let [s, c] = e(() => {
18
+ let e = this.model.forward({ training: !1 }, r), s = n(e, i, o, a), c = t(e, i);
19
+ return e.dispose(), [s, c];
20
+ }), l = await s.array(), u = await c.array(), d = l, f = u;
21
+ return c.dispose(), s.dispose(), Array.isArray(d) ? d.map((e) => ({
22
+ loss: e,
23
+ accuracy: f
24
+ })) : {
25
+ loss: d,
26
+ accuracy: f
27
+ };
28
+ }
29
+ async evaluate(e = 100) {
30
+ let t = 0, n = 0, r = 0;
31
+ if (this.iterator) {
32
+ let i = await this.iterator;
33
+ for (let a = 0; a < e; a++) {
34
+ let e = await i.next();
35
+ if (e.done) break;
36
+ let { xs: a, ys: o } = e.value, s = await this.calculateBatchLoss(a, o, !1, this.masked);
37
+ a.dispose(), o.dispose(), t += s.loss, n += s.accuracy, r++;
38
+ }
39
+ return {
40
+ loss: t / r,
41
+ accuracy: n / r
42
+ };
43
+ } else if (this.xs && this.ys) return this.calculateBatchLoss(this.xs, this.ys, !0, !0);
44
+ throw Error("No data available for evaluation");
45
+ }
39
46
  };
47
+ //#endregion
48
+ export { r as default };
@@ -1,34 +1,38 @@
1
- class o {
2
- constructor(i, t) {
3
- this.learningRate = i, this.config = t, this.startLearningRate = i, t.step !== void 0 && (this.step = t.step);
4
- }
5
- step = 0;
6
- startLearningRate;
7
- serializeConfig() {
8
- return {
9
- ...this.config,
10
- step: this.step
11
- };
12
- }
13
- updateConfig(i, t) {
14
- this.config = { ...this.config, ...i }, t !== void 0 && (this.startLearningRate = t);
15
- }
16
- get lr() {
17
- return this.learningRate;
18
- }
19
- getNextLR() {
20
- const i = this.step;
21
- if (this.config.warmupSteps > 0 && i < this.config.warmupSteps) {
22
- const r = (i + 1) / this.config.warmupSteps, e = this.startLearningRate * r;
23
- return this.learningRate = e, this.step++, e;
24
- }
25
- const t = this.config.epochSteps * this.config.decayEpochs;
26
- if (i >= t || t <= this.config.warmupSteps)
27
- return this.learningRate = this.config.minLearningRate, this.step++, this.config.minLearningRate;
28
- const n = (i - this.config.warmupSteps) / (t - this.config.warmupSteps), a = 0.5 * (1 + Math.cos(Math.PI * n)), s = this.config.minLearningRate + a * (this.startLearningRate - this.config.minLearningRate);
29
- return this.learningRate = s, this.step++, s;
30
- }
31
- }
32
- export {
33
- o as default
1
+ //#region lib/training/LRScheduler.ts
2
+ var e = class {
3
+ learningRate;
4
+ config;
5
+ step = 0;
6
+ startLearningRate;
7
+ constructor(e, t) {
8
+ this.learningRate = e, this.config = t, this.startLearningRate = e, t.step !== void 0 && (this.step = t.step);
9
+ }
10
+ serializeConfig() {
11
+ return {
12
+ ...this.config,
13
+ step: this.step
14
+ };
15
+ }
16
+ updateConfig(e, t) {
17
+ this.config = {
18
+ ...this.config,
19
+ ...e
20
+ }, t !== void 0 && (this.startLearningRate = t);
21
+ }
22
+ get lr() {
23
+ return this.learningRate;
24
+ }
25
+ getNextLR() {
26
+ let e = this.step;
27
+ if (this.config.warmupSteps > 0 && e < this.config.warmupSteps) {
28
+ let t = (e + 1) / this.config.warmupSteps, n = this.startLearningRate * t;
29
+ return this.learningRate = n, this.step++, n;
30
+ }
31
+ let t = this.config.epochSteps * this.config.decayEpochs;
32
+ if (e >= t || t <= this.config.warmupSteps) return this.learningRate = this.config.minLearningRate, this.step++, this.config.minLearningRate;
33
+ let n = (e - this.config.warmupSteps) / (t - this.config.warmupSteps), r = .5 * (1 + Math.cos(Math.PI * n)), i = this.config.minLearningRate + r * (this.startLearningRate - this.config.minLearningRate);
34
+ return this.learningRate = i, this.step++, i;
35
+ }
34
36
  };
37
+ //#endregion
38
+ export { e as default };
@@ -1,6 +1,6 @@
1
- import { default as Model, ModelForwardAttributes } from '../models/model';
1
+ import { default as Model, ModelForwardAttributes } from '../../models/model';
2
2
  import { default as BasicTrainer } from './BasicTrainer';
3
- import { ITokeniser } from '../tokeniser/type';
3
+ import { ITokeniser } from '../../tokeniser/type';
4
4
  import { DatasetBuilder } from './DatasetBuilder';
5
5
  import { AdamWOptimizer } from './AdamW';
6
6
  import { AdamWOptimizerConfig } from './types';
@@ -1,20 +1,22 @@
1
- import r from "./BasicTrainer.js";
2
- import { DatasetBuilder as n } from "./DatasetBuilder.js";
3
- const s = {
4
- decayEpochs: 100,
5
- epochSteps: 1e4,
6
- warmupSteps: 1e3,
7
- minLearningRate: 3e-5,
8
- weightDecay: 0.1,
9
- learningRate: 3e-4
10
- // clipNorm: 1.0,
11
- };
12
- class m extends r {
13
- constructor(i, e, t, a) {
14
- super(i, e, { ...s, ...t }, a), this.tokenizer = e, this.optimizerConfig.minLearningRate = t?.minLearningRate ?? this.optimizerConfig.learningRate / 20, this.updateOptimizer(), this.datasetBuilder = new n(e, i.config.blockSize);
15
- }
16
- datasetBuilder;
17
- }
18
- export {
19
- m as default
1
+ import { t as e } from "../DatasetBuilder-DgURD85T.js";
2
+ import t from "./BasicTrainer.js";
3
+ //#region lib/training/PreTrainer.ts
4
+ var n = {
5
+ decayEpochs: 100,
6
+ epochSteps: 1e4,
7
+ warmupSteps: 1e3,
8
+ minLearningRate: 3e-5,
9
+ weightDecay: .1,
10
+ learningRate: 3e-4
11
+ }, r = class extends t {
12
+ tokenizer;
13
+ datasetBuilder;
14
+ constructor(t, r, i, a) {
15
+ super(t, r, {
16
+ ...n,
17
+ ...i
18
+ }, a), this.tokenizer = r, this.optimizerConfig.minLearningRate = i?.minLearningRate ?? this.optimizerConfig.learningRate / 20, this.updateOptimizer(), this.datasetBuilder = new e(r, t.config.blockSize);
19
+ }
20
20
  };
21
+ //#endregion
22
+ export { r as default };
@@ -1,6 +1,6 @@
1
- import { default as Model, ModelForwardAttributes } from '../models/model';
1
+ import { default as Model, ModelForwardAttributes } from '../../models/model';
2
2
  import { default as BasicTrainer } from './BasicTrainer';
3
- import { ITokeniser } from '../tokeniser/type';
3
+ import { ITokeniser } from '../../tokeniser/type';
4
4
  import { AdamWOptimizer } from './AdamW';
5
5
  import { AdamWOptimizerConfig } from './types';
6
6
  import { DatasetBuilder } from './DatasetBuilder';
@@ -1,22 +1,24 @@
1
- import r from "./BasicTrainer.js";
2
- import { DatasetBuilder as s } from "./DatasetBuilder.js";
3
- const n = {
4
- decayEpochs: 100,
5
- epochSteps: 1e4,
6
- warmupSteps: 100,
7
- minLearningRate: 1e-5,
8
- weightDecay: 0.1,
9
- beta2: 0.95,
10
- learningRate: 3e-4
11
- // clipNorm: 1.0,
12
- };
13
- class c extends r {
14
- constructor(t, e, i, a) {
15
- super(t, e, { ...n, ...i }, a), this.tokenizer = e, this.optimizerConfig.minLearningRate = i?.minLearningRate ?? this.optimizerConfig.learningRate / 20, this.updateOptimizer(), this.datasetBuilder = new s(e, t.config.blockSize), this.maskedLoss = !0;
16
- }
17
- datasetBuilder;
18
- loraName;
19
- }
20
- export {
21
- c as default
1
+ import { t as e } from "../DatasetBuilder-DgURD85T.js";
2
+ import t from "./BasicTrainer.js";
3
+ //#region lib/training/SFTTrainer.ts
4
+ var n = {
5
+ decayEpochs: 100,
6
+ epochSteps: 1e4,
7
+ warmupSteps: 100,
8
+ minLearningRate: 1e-5,
9
+ weightDecay: .1,
10
+ beta2: .95,
11
+ learningRate: 3e-4
12
+ }, r = class extends t {
13
+ tokenizer;
14
+ datasetBuilder;
15
+ loraName;
16
+ constructor(t, r, i, a) {
17
+ super(t, r, {
18
+ ...n,
19
+ ...i
20
+ }, a), this.tokenizer = r, this.optimizerConfig.minLearningRate = i?.minLearningRate ?? this.optimizerConfig.learningRate / 20, this.updateOptimizer(), this.datasetBuilder = new e(r, t.config.blockSize), this.maskedLoss = !0;
21
+ }
22
22
  };
23
+ //#endregion
24
+ export { r as default };
@@ -1,24 +1,19 @@
1
- import { createSoftmaxCrossEntropyWithGrad as n } from "./sparseCrossEntropy.js";
2
- function i(t, s, r, c, o) {
3
- try {
4
- return n(
5
- r,
6
- c,
7
- o && o > 0 ? o : void 0
8
- )(t, s);
9
- } catch (a) {
10
- throw console.error("Error computing loss:", a), new Error(`Loss computation failed: ${a}`);
11
- }
1
+ import { createSoftmaxCrossEntropyWithGrad as e } from "./sparseCrossEntropy.js";
2
+ //#region lib/training/loss.ts
3
+ function t(t, n, r, i, a) {
4
+ try {
5
+ return e(r, i, a && a > 0 ? a : void 0)(t, n);
6
+ } catch (e) {
7
+ throw console.error("Error computing loss:", e), Error(`Loss computation failed: ${e}`);
8
+ }
12
9
  }
13
- function l(t, s) {
14
- try {
15
- const r = t.argMax(-1), c = r.equal(s).cast("float32"), o = c.mean();
16
- return r.dispose(), c.dispose(), o;
17
- } catch (r) {
18
- throw console.error("Error computing accuracy:", r), new Error(`Accuracy computation failed: ${r}`);
19
- }
10
+ function n(e, t) {
11
+ try {
12
+ let n = e.argMax(-1), r = n.equal(t).cast("float32"), i = r.mean();
13
+ return n.dispose(), r.dispose(), i;
14
+ } catch (e) {
15
+ throw console.error("Error computing accuracy:", e), Error(`Accuracy computation failed: ${e}`);
16
+ }
20
17
  }
21
- export {
22
- l as calculateAccuracy,
23
- i as calculateLoss
24
- };
18
+ //#endregion
19
+ export { n as calculateAccuracy, t as calculateLoss };
@@ -1,10 +1,10 @@
1
- import { t as d } from "../index-CUXkjxiT.js";
2
- function l(m, t, s) {
3
- return d(() => {
4
- const r = m.reshape([-1]), o = t.reshape([-1]), e = r.mul(r).sum().add(s), c = r.mul(o).sum().div(e), n = o.sub(r.mul(c)), a = o.norm(), u = n.norm().add(s);
5
- return n.mul(a.div(u)).reshape(t.shape);
6
- });
1
+ import { di as e } from "../dist-BewPQWjc.js";
2
+ //#region lib/training/orthoGrad.ts
3
+ function t(t, n, r) {
4
+ return e(() => {
5
+ let e = t.reshape([-1]), i = n.reshape([-1]), a = e.mul(e).sum().add(r), o = e.mul(i).sum().div(a), s = i.sub(e.mul(o)), c = i.norm(), l = s.norm().add(r);
6
+ return s.mul(c.div(l)).reshape(n.shape);
7
+ });
7
8
  }
8
- export {
9
- l as orthogonalizeGradient
10
- };
9
+ //#endregion
10
+ export { t as orthogonalizeGradient };
@@ -1,69 +1,47 @@
1
- import { gatherSub as $ } from "../ops/gatherSub.js";
2
- import { scatterSub as T } from "../ops/scatterSub.js";
3
- import { t as F, a1 as G, j as A, m as x, i as U, a2 as D, a as j, a3 as q } from "../index-CUXkjxiT.js";
4
- import { m as J, l as K, a as E, n as N, w as O } from "../not_equal-hurPF26l.js";
5
- import { s as m } from "../sum-BpiwSWvg.js";
6
- import { s as Q } from "../softmax-D4q1LJN7.js";
7
- function R(e, l, a, g, v, t = 0) {
8
- return F(() => {
9
- const S = e.shape[e.shape.length - 1], i = v || e.shape.slice(0, -1), f = i.reduce((c, d) => c * d, 1), h = e.shape.length > 2 ? e.reshape([f, S]) : e, y = l.shape.length > 1 ? l.reshape([f]).cast("int32") : l.cast("int32"), u = J(h, -1, !0), o = G(h, u), n = K(o, -1), p = $(n, y, o);
10
- let s;
11
- if (t > 0) {
12
- const c = G(n, E(o, -1));
13
- s = A(x(p, 1 - t), x(c, t));
14
- } else
15
- s = p;
16
- if (a)
17
- if (s = x(s, a), g) {
18
- const c = m(a.reshape(i), -1);
19
- s = U(m(s.reshape(i), -1), c);
20
- } else {
21
- const c = m(a);
22
- s = U(m(s), c);
23
- }
24
- else
25
- g ? s = E(s.reshape(i), -1) : s = E(s);
26
- return s;
27
- });
1
+ import { At as e, Bt as t, Gr as n, Nn as r, Pn as i, Rt as a, St as o, Wr as s, Wt as c, Y as l, _n as u, bn as d, di as f, mn as p, qr as m } from "../dist-BewPQWjc.js";
2
+ import { gatherSub as h } from "../ops/gatherSub.js";
3
+ import { scatterSub as g } from "../ops/scatterSub.js";
4
+ //#region lib/training/sparseCrossEntropy.ts
5
+ function _(r, i, o, c, l, u = 0) {
6
+ return f(() => {
7
+ let f = r.shape[r.shape.length - 1], g = l || r.shape.slice(0, -1), _ = g.reduce((e, t) => e * t, 1), v = r.shape.length > 2 ? r.reshape([_, f]) : r, y = i.shape.length > 1 ? i.reshape([_]).cast("int32") : i.cast("int32"), b = t(v, d(v, -1, !0)), x = a(b, -1), S = h(x, y, b), C;
8
+ if (u > 0) {
9
+ let n = t(x, e(b, -1));
10
+ C = m(s(S, 1 - u), s(n, u));
11
+ } else C = S;
12
+ if (o) if (C = s(C, o), c) {
13
+ let e = p(o.reshape(g), -1);
14
+ C = n(p(C.reshape(g), -1), e);
15
+ } else {
16
+ let e = p(o);
17
+ C = n(p(C), e);
18
+ }
19
+ else C = c ? e(C.reshape(g), -1) : e(C);
20
+ return C;
21
+ });
28
22
  }
29
- function as(e, l, a = 0) {
30
- return D(
31
- // @ts-expect-error Invalid params
32
- (t, S, i) => {
33
- const f = t.shape[t.shape.length - 1], h = t.shape.slice(0, -1), y = h.reduce((d, r) => d * r, 1), u = t.reshape([y, f]), o = S.reshape([y]).cast("int32");
34
- let n, p = null;
35
- if (e) {
36
- const d = j(65535, "int32"), r = N(o, d);
37
- p = r.cast("float32"), n = O(r, o, q(o)), d.dispose(), r.dispose();
38
- } else
39
- n = o;
40
- const s = R(
41
- u,
42
- n,
43
- p || void 0,
44
- l,
45
- h,
46
- a
47
- );
48
- return i(p ? [u, n, p] : [u, n]), u.dispose(), o.dispose(), { value: s, gradFunc: (d, r) => F(() => {
49
- const C = r[0], z = r[1], b = e ? r[2] : void 0, w = Q(C), H = b ? m(b) : j(C.shape[0], "float32"), I = d.div(H).broadcastTo([C.shape[0]]), M = b && e ? x(I, b) : I;
50
- let L;
51
- if (a > 0) {
52
- const V = a / f, W = G(w, V);
53
- L = T(
54
- W,
55
- z,
56
- x(M, 1 - a)
57
- );
58
- } else
59
- L = T(w, z, M);
60
- const P = q(S);
61
- return [L.reshape(t.shape), P];
62
- }) };
63
- }
64
- );
23
+ function v(e, n, a = 0) {
24
+ return c((c, d, m) => {
25
+ let h = c.shape[c.shape.length - 1], v = c.shape.slice(0, -1), y = v.reduce((e, t) => e * t, 1), b = c.reshape([y, h]), x = d.reshape([y]).cast("int32"), S, C = null;
26
+ if (e) {
27
+ let e = u(65535, "int32"), t = o(x, e);
28
+ C = t.cast("float32"), S = i(t, x, r(x)), e.dispose(), t.dispose();
29
+ } else S = x;
30
+ let w = _(b, S, C || void 0, n, v, a);
31
+ return m(C ? [
32
+ b,
33
+ S,
34
+ C
35
+ ] : [b, S]), b.dispose(), x.dispose(), {
36
+ value: w,
37
+ gradFunc: (n, i) => f(() => {
38
+ let o = i[0], f = i[1], m = e ? i[2] : void 0, _ = l(o), v = m ? p(m) : u(o.shape[0], "float32"), y = n.div(v).broadcastTo([o.shape[0]]), b = m && e ? s(y, m) : y, x;
39
+ x = a > 0 ? g(t(_, a / h), f, s(b, 1 - a)) : g(_, f, b);
40
+ let S = r(d);
41
+ return [x.reshape(c.shape), S];
42
+ })
43
+ };
44
+ });
65
45
  }
66
- export {
67
- as as createSoftmaxCrossEntropyWithGrad,
68
- R as sparseSoftmaxCrossEntropy
69
- };
46
+ //#endregion
47
+ export { v as createSoftmaxCrossEntropyWithGrad, _ as sparseSoftmaxCrossEntropy };
@@ -1,4 +1,4 @@
1
- import { Conversation, ITokeniser } from '../../main';
1
+ import { Conversation, ITokeniser } from '../../../main';
2
2
  import { Task } from './Task';
3
3
  export default class ConversationTask extends Task {
4
4
  private rawConvo;
@@ -1,40 +1,38 @@
1
+ import { a as e } from "../../DatasetBuilder-DgURD85T.js";
1
2
  import { Task as t } from "./Task.js";
2
- import { shuffle as i } from "../DatasetBuilder.js";
3
- class f extends t {
4
- rawConvo;
5
- shuffledIndices = null;
6
- index = 0;
7
- get length() {
8
- return this.rawConvo.length;
9
- }
10
- constructor(n) {
11
- super(), this.rawConvo = n;
12
- }
13
- hasMoreConversations() {
14
- return this.index < this.rawConvo.length;
15
- }
16
- nextConversation() {
17
- if (this.index >= this.rawConvo.length)
18
- return null;
19
- const n = this.rawConvo[this.shuffledIndices ? this.shuffledIndices[this.index] : this.index];
20
- return this.index++, n;
21
- }
22
- nextTokens(n, s) {
23
- const e = this.nextConversation();
24
- return e ? n.encodeConversation(e, !1, s) : null;
25
- }
26
- shuffle() {
27
- if (!this.shuffledIndices) {
28
- this.shuffledIndices = new Uint32Array(this.rawConvo.length);
29
- for (let n = 0; n < this.rawConvo.length; n++)
30
- this.shuffledIndices[n] = n;
31
- }
32
- i(this.shuffledIndices), this.index = 0;
33
- }
34
- async estimateTokens(n) {
35
- return n.encodeConversation(this.rawConvo[0]).length * this.length;
36
- }
37
- }
38
- export {
39
- f as default
3
+ //#region lib/training/tasks/ConversationTask.ts
4
+ var n = class extends t {
5
+ rawConvo;
6
+ shuffledIndices = null;
7
+ index = 0;
8
+ get length() {
9
+ return this.rawConvo.length;
10
+ }
11
+ constructor(e) {
12
+ super(), this.rawConvo = e;
13
+ }
14
+ hasMoreConversations() {
15
+ return this.index < this.rawConvo.length;
16
+ }
17
+ nextConversation() {
18
+ if (this.index >= this.rawConvo.length) return null;
19
+ let e = this.rawConvo[this.shuffledIndices ? this.shuffledIndices[this.index] : this.index];
20
+ return this.index++, e;
21
+ }
22
+ nextTokens(e, t) {
23
+ let n = this.nextConversation();
24
+ return n ? e.encodeConversation(n, !1, t) : null;
25
+ }
26
+ shuffle() {
27
+ if (!this.shuffledIndices) {
28
+ this.shuffledIndices = new Uint32Array(this.rawConvo.length);
29
+ for (let e = 0; e < this.rawConvo.length; e++) this.shuffledIndices[e] = e;
30
+ }
31
+ e(this.shuffledIndices), this.index = 0;
32
+ }
33
+ async estimateTokens(e) {
34
+ return e.encodeConversation(this.rawConvo[0]).length * this.length;
35
+ }
40
36
  };
37
+ //#endregion
38
+ export { n as default };
@@ -1,4 +1,4 @@
1
- import { Conversation, ITokeniser } from '../../main';
1
+ import { Conversation, ITokeniser } from '../../../main';
2
2
  import { Task } from './Task';
3
3
  export default class PretrainingTask extends Task {
4
4
  private rawText;
@@ -1,47 +1,42 @@
1
- import { Task as r } from "./Task.js";
2
- class h extends r {
3
- rawText;
4
- index = 0;
5
- get length() {
6
- return this.rawText.length;
7
- }
8
- constructor(t) {
9
- super(), this.rawText = t;
10
- }
11
- hasMoreConversations() {
12
- return this.index < this.rawText.length;
13
- }
14
- nextConversation() {
15
- if (this.index >= this.rawText.length)
16
- return null;
17
- const t = {
18
- role: "assistant",
19
- content: this.rawText[this.index]
20
- };
21
- return this.index++, [t];
22
- }
23
- nextTokens(t, n) {
24
- if (this.index >= this.rawText.length)
25
- return null;
26
- const e = t.encodeSequence(this.rawText[this.index]);
27
- if (this.index++, n) {
28
- const s = new Array(e.length).fill(!0);
29
- return { tokens: e, mask: s };
30
- }
31
- return e;
32
- }
33
- shuffle() {
34
- this.index = 0;
35
- }
36
- async estimateTokens(t) {
37
- return t.encodeConversation([
38
- {
39
- role: "assistant",
40
- content: this.rawText[0]
41
- }
42
- ]).length * this.length;
43
- }
44
- }
45
- export {
46
- h as default
1
+ import { Task as e } from "./Task.js";
2
+ //#region lib/training/tasks/PretrainingTask.ts
3
+ var t = class extends e {
4
+ rawText;
5
+ index = 0;
6
+ get length() {
7
+ return this.rawText.length;
8
+ }
9
+ constructor(e) {
10
+ super(), this.rawText = e;
11
+ }
12
+ hasMoreConversations() {
13
+ return this.index < this.rawText.length;
14
+ }
15
+ nextConversation() {
16
+ if (this.index >= this.rawText.length) return null;
17
+ let e = {
18
+ role: "assistant",
19
+ content: this.rawText[this.index]
20
+ };
21
+ return this.index++, [e];
22
+ }
23
+ nextTokens(e, t) {
24
+ if (this.index >= this.rawText.length) return null;
25
+ let n = e.encodeSequence(this.rawText[this.index]);
26
+ return this.index++, t ? {
27
+ tokens: n,
28
+ mask: Array(n.length).fill(!0)
29
+ } : n;
30
+ }
31
+ shuffle() {
32
+ this.index = 0;
33
+ }
34
+ async estimateTokens(e) {
35
+ return e.encodeConversation([{
36
+ role: "assistant",
37
+ content: this.rawText[0]
38
+ }]).length * this.length;
39
+ }
47
40
  };
41
+ //#endregion
42
+ export { t as default };