@genai-fi/nanogpt 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (482) hide show
  1. package/package.json +9 -10
  2. package/dist/Generator.d.ts +0 -82
  3. package/dist/Generator.js +0 -11941
  4. package/dist/RealDiv-CGwv0liw.js +0 -365
  5. package/dist/Reshape-BW__R4mZ.js +0 -79
  6. package/dist/Reshape-CPBkTIH2.js +0 -14
  7. package/dist/TeachableLLM.d.ts +0 -70
  8. package/dist/TeachableLLM.js +0 -273
  9. package/dist/Trainer.d.ts +0 -43
  10. package/dist/Trainer.js +0 -244
  11. package/dist/_commonjsHelpers-ByX85dGu.js +0 -33
  12. package/dist/axis_util-GTVlo58H.js +0 -55
  13. package/dist/backend.d.ts +0 -2
  14. package/dist/backend.js +0 -13
  15. package/dist/backend_util-GaFarB78.js +0 -425
  16. package/dist/backend_webgpu-BqASlsbV.js +0 -545
  17. package/dist/binary_op_util-pKXltfxI.js +0 -192
  18. package/dist/broadcast_to-eS93CCN_.js +0 -28
  19. package/dist/checks/appendCache.d.ts +0 -1
  20. package/dist/checks/appendCache.js +0 -22
  21. package/dist/checks/attentionMask.d.ts +0 -1
  22. package/dist/checks/attentionMask.js +0 -37
  23. package/dist/checks/check.d.ts +0 -9
  24. package/dist/checks/check.js +0 -20
  25. package/dist/checks/gelu.d.ts +0 -1
  26. package/dist/checks/gelu.js +0 -18
  27. package/dist/checks/index.d.ts +0 -26
  28. package/dist/checks/index.js +0 -28
  29. package/dist/checks/matMulGelu.d.ts +0 -1
  30. package/dist/checks/matMulGelu.js +0 -28
  31. package/dist/checks/normRMS.d.ts +0 -1
  32. package/dist/checks/normRMS.js +0 -16
  33. package/dist/checks/normRMSGrad.d.ts +0 -1
  34. package/dist/checks/normRMSGrad.js +0 -12
  35. package/dist/checks/packUnpack.d.ts +0 -1
  36. package/dist/checks/packUnpack.js +0 -18
  37. package/dist/checks/qkv.d.ts +0 -1
  38. package/dist/checks/qkv.js +0 -34
  39. package/dist/checks/rope.d.ts +0 -1
  40. package/dist/checks/rope.js +0 -36
  41. package/dist/checks/weights.d.ts +0 -14
  42. package/dist/checks/weights.js +0 -31
  43. package/dist/clip_by_value-DDA7rrcT.js +0 -12
  44. package/dist/complex-DI35Q-gW.js +0 -11
  45. package/dist/complex_util-Yc1A_gV1.js +0 -55
  46. package/dist/concat-CAQpCret.js +0 -17
  47. package/dist/concat_util-D18dJ4fD.js +0 -22
  48. package/dist/data/docx.d.ts +0 -2
  49. package/dist/data/docx.js +0 -15
  50. package/dist/data/parquet.d.ts +0 -2
  51. package/dist/data/parquet.js +0 -17
  52. package/dist/data/pdf.d.ts +0 -2
  53. package/dist/data/pdf.js +0 -14
  54. package/dist/data/textLoader.d.ts +0 -7
  55. package/dist/data/textLoader.js +0 -108
  56. package/dist/dataset-CGGp1z9P.js +0 -1124
  57. package/dist/dropout_util--NxWuYg2.js +0 -27
  58. package/dist/expand_dims-Bkd1YD5x.js +0 -11
  59. package/dist/exports_initializers-CYzKLjN7.js +0 -7
  60. package/dist/floor-BQtb-Azg.js +0 -9
  61. package/dist/gather-qIqEqaGn.js +0 -9
  62. package/dist/gelu-B220X1Go.js +0 -26
  63. package/dist/gpgpu_math-BwvV12df.js +0 -2022
  64. package/dist/index-CUXkjxiT.js +0 -3516
  65. package/dist/index-CieiGp4Y.js +0 -349
  66. package/dist/index-CjOWnMXP.js +0 -7308
  67. package/dist/index-Cp39cXWe.js +0 -1016
  68. package/dist/index-D5v913EJ.js +0 -4
  69. package/dist/index-DmeWGGmS.js +0 -1074
  70. package/dist/index-DvYrXKkX.js +0 -113
  71. package/dist/index-Ksja3su6.js +0 -151
  72. package/dist/index-xuotMAFm.js +0 -118
  73. package/dist/inference/types.d.ts +0 -16
  74. package/dist/inference/types.js +0 -1
  75. package/dist/jszip.min-BZhlzntC.js +0 -2313
  76. package/dist/kernel_funcs_utils-pq0CK9co.js +0 -306
  77. package/dist/layers/BaseLayer.d.ts +0 -44
  78. package/dist/layers/BaseLayer.js +0 -74
  79. package/dist/layers/CausalSelfAttention.d.ts +0 -39
  80. package/dist/layers/CausalSelfAttention.js +0 -86
  81. package/dist/layers/LoRA.d.ts +0 -14
  82. package/dist/layers/LoRA.js +0 -58
  83. package/dist/layers/MLP.d.ts +0 -17
  84. package/dist/layers/MLP.js +0 -44
  85. package/dist/layers/PositionEmbedding.d.ts +0 -8
  86. package/dist/layers/PositionEmbedding.js +0 -31
  87. package/dist/layers/RMSNorm.d.ts +0 -12
  88. package/dist/layers/RMSNorm.js +0 -22
  89. package/dist/layers/RoPECache.d.ts +0 -18
  90. package/dist/layers/RoPECache.js +0 -50
  91. package/dist/layers/TiedEmbedding.d.ts +0 -13
  92. package/dist/layers/TiedEmbedding.js +0 -36
  93. package/dist/layers/TransformerBlock.d.ts +0 -27
  94. package/dist/layers/TransformerBlock.js +0 -40
  95. package/dist/layers/WeightStore.d.ts +0 -20
  96. package/dist/layers/WeightStore.js +0 -76
  97. package/dist/loader/load.d.ts +0 -6
  98. package/dist/loader/load.js +0 -68
  99. package/dist/loader/loadHF.d.ts +0 -8
  100. package/dist/loader/loadHF.js +0 -22
  101. package/dist/loader/loadTransformers.d.ts +0 -4
  102. package/dist/loader/loadTransformers.js +0 -44
  103. package/dist/loader/loadZipMeta.d.ts +0 -3
  104. package/dist/loader/loadZipMeta.js +0 -16
  105. package/dist/loader/newZipLoad.d.ts +0 -3
  106. package/dist/loader/newZipLoad.js +0 -31
  107. package/dist/loader/oldZipLoad.d.ts +0 -9
  108. package/dist/loader/oldZipLoad.js +0 -80
  109. package/dist/loader/save.d.ts +0 -16
  110. package/dist/loader/save.js +0 -90
  111. package/dist/loader/types.d.ts +0 -67
  112. package/dist/loader/types.js +0 -1
  113. package/dist/main.d.ts +0 -50
  114. package/dist/main.js +0 -109
  115. package/dist/matMul16-BcVC_E62.js +0 -80
  116. package/dist/matMulGelu-JNLZqKQp.js +0 -163
  117. package/dist/mat_mul-DhG0Newp.js +0 -11
  118. package/dist/mod-CSdCpRjf.js +0 -11
  119. package/dist/models/NanoGPTV1.d.ts +0 -16
  120. package/dist/models/NanoGPTV1.js +0 -99
  121. package/dist/models/NanoGPTV2.d.ts +0 -16
  122. package/dist/models/NanoGPTV2.js +0 -90
  123. package/dist/models/config.d.ts +0 -27
  124. package/dist/models/config.js +0 -50
  125. package/dist/models/factory.d.ts +0 -3
  126. package/dist/models/factory.js +0 -16
  127. package/dist/models/model.d.ts +0 -44
  128. package/dist/models/model.js +0 -134
  129. package/dist/non_max_suppression_impl-B2W7YjZB.js +0 -102
  130. package/dist/not_equal-hurPF26l.js +0 -64
  131. package/dist/ones-BytntneX.js +0 -14
  132. package/dist/ops/adamAdjust.d.ts +0 -2
  133. package/dist/ops/adamAdjust.js +0 -9
  134. package/dist/ops/adamMoments.d.ts +0 -2
  135. package/dist/ops/adamMoments.js +0 -9
  136. package/dist/ops/add16.d.ts +0 -2
  137. package/dist/ops/add16.js +0 -9
  138. package/dist/ops/appendCache.d.ts +0 -2
  139. package/dist/ops/appendCache.js +0 -22
  140. package/dist/ops/attentionMask.d.ts +0 -2
  141. package/dist/ops/attentionMask.js +0 -10
  142. package/dist/ops/concat16.d.ts +0 -2
  143. package/dist/ops/concat16.js +0 -9
  144. package/dist/ops/cpu/adamAdjust.d.ts +0 -1
  145. package/dist/ops/cpu/adamAdjust.js +0 -18
  146. package/dist/ops/cpu/adamMoments.d.ts +0 -1
  147. package/dist/ops/cpu/adamMoments.js +0 -16
  148. package/dist/ops/cpu/appendCache.d.ts +0 -1
  149. package/dist/ops/cpu/appendCache.js +0 -23
  150. package/dist/ops/cpu/attentionMask.d.ts +0 -1
  151. package/dist/ops/cpu/attentionMask.js +0 -22
  152. package/dist/ops/cpu/fusedSoftmax.d.ts +0 -9
  153. package/dist/ops/cpu/fusedSoftmax.js +0 -29
  154. package/dist/ops/cpu/gatherSub.d.ts +0 -1
  155. package/dist/ops/cpu/gatherSub.js +0 -18
  156. package/dist/ops/cpu/gelu.d.ts +0 -1
  157. package/dist/ops/cpu/gelu.js +0 -40
  158. package/dist/ops/cpu/matMul16.d.ts +0 -1
  159. package/dist/ops/cpu/matMul16.js +0 -15
  160. package/dist/ops/cpu/matMulGelu.d.ts +0 -1
  161. package/dist/ops/cpu/matMulGelu.js +0 -53
  162. package/dist/ops/cpu/matMulMul.d.ts +0 -1
  163. package/dist/ops/cpu/matMulMul.js +0 -23
  164. package/dist/ops/cpu/mulDropout.d.ts +0 -1
  165. package/dist/ops/cpu/mulDropout.js +0 -23
  166. package/dist/ops/cpu/normRMS.d.ts +0 -1
  167. package/dist/ops/cpu/normRMS.js +0 -39
  168. package/dist/ops/cpu/qkv.d.ts +0 -5
  169. package/dist/ops/cpu/qkv.js +0 -41
  170. package/dist/ops/cpu/rope.d.ts +0 -6
  171. package/dist/ops/cpu/rope.js +0 -38
  172. package/dist/ops/cpu/scatterSub.d.ts +0 -1
  173. package/dist/ops/cpu/scatterSub.js +0 -23
  174. package/dist/ops/dot16.d.ts +0 -2
  175. package/dist/ops/dot16.js +0 -42
  176. package/dist/ops/dropout.d.ts +0 -2
  177. package/dist/ops/dropout.js +0 -14
  178. package/dist/ops/dropout16.d.ts +0 -2
  179. package/dist/ops/dropout16.js +0 -25
  180. package/dist/ops/gatherSub.d.ts +0 -2
  181. package/dist/ops/gatherSub.js +0 -9
  182. package/dist/ops/gelu.d.ts +0 -3
  183. package/dist/ops/gelu.js +0 -8
  184. package/dist/ops/globalNorm.d.ts +0 -2
  185. package/dist/ops/globalNorm.js +0 -13
  186. package/dist/ops/grads/add16.d.ts +0 -1
  187. package/dist/ops/grads/add16.js +0 -26
  188. package/dist/ops/grads/attentionMask.d.ts +0 -1
  189. package/dist/ops/grads/attentionMask.js +0 -21
  190. package/dist/ops/grads/dropout16.d.ts +0 -1
  191. package/dist/ops/grads/dropout16.js +0 -2
  192. package/dist/ops/grads/gelu.d.ts +0 -2
  193. package/dist/ops/grads/gelu.js +0 -5
  194. package/dist/ops/grads/matMul16.d.ts +0 -2
  195. package/dist/ops/grads/matMul16.js +0 -9
  196. package/dist/ops/grads/matMulGelu.d.ts +0 -1
  197. package/dist/ops/grads/matMulGelu.js +0 -17
  198. package/dist/ops/grads/mul16.d.ts +0 -1
  199. package/dist/ops/grads/mul16.js +0 -4
  200. package/dist/ops/grads/normRMS.d.ts +0 -3
  201. package/dist/ops/grads/normRMS.js +0 -33
  202. package/dist/ops/grads/pack16.d.ts +0 -2
  203. package/dist/ops/grads/pack16.js +0 -6
  204. package/dist/ops/grads/qkv.d.ts +0 -3
  205. package/dist/ops/grads/qkv.js +0 -34
  206. package/dist/ops/grads/rope.d.ts +0 -2
  207. package/dist/ops/grads/rope.js +0 -5
  208. package/dist/ops/grads/softmax16.d.ts +0 -2
  209. package/dist/ops/grads/softmax16.js +0 -25
  210. package/dist/ops/grads/unpack16.d.ts +0 -2
  211. package/dist/ops/grads/unpack16.js +0 -5
  212. package/dist/ops/grads/utils.d.ts +0 -4
  213. package/dist/ops/grads/utils.js +0 -14
  214. package/dist/ops/log.d.ts +0 -0
  215. package/dist/ops/log.js +0 -1
  216. package/dist/ops/matMul16.d.ts +0 -15
  217. package/dist/ops/matMul16.js +0 -13
  218. package/dist/ops/matMulGelu.d.ts +0 -3
  219. package/dist/ops/matMulGelu.js +0 -14
  220. package/dist/ops/matMulMul.d.ts +0 -2
  221. package/dist/ops/matMulMul.js +0 -9
  222. package/dist/ops/mul16.d.ts +0 -2
  223. package/dist/ops/mul16.js +0 -39
  224. package/dist/ops/mulDrop.d.ts +0 -2
  225. package/dist/ops/mulDrop.js +0 -9
  226. package/dist/ops/normRMS.d.ts +0 -2
  227. package/dist/ops/normRMS.js +0 -19
  228. package/dist/ops/pack16.d.ts +0 -2
  229. package/dist/ops/pack16.js +0 -5
  230. package/dist/ops/qkv.d.ts +0 -2
  231. package/dist/ops/qkv.js +0 -10
  232. package/dist/ops/reshape16.d.ts +0 -2
  233. package/dist/ops/reshape16.js +0 -41
  234. package/dist/ops/rope.d.ts +0 -3
  235. package/dist/ops/rope.js +0 -7
  236. package/dist/ops/scatterSub.d.ts +0 -2
  237. package/dist/ops/scatterSub.js +0 -9
  238. package/dist/ops/slice16.d.ts +0 -2
  239. package/dist/ops/slice16.js +0 -9
  240. package/dist/ops/softmax16.d.ts +0 -2
  241. package/dist/ops/softmax16.js +0 -9
  242. package/dist/ops/sub16.d.ts +0 -2
  243. package/dist/ops/sub16.js +0 -8
  244. package/dist/ops/sum16.d.ts +0 -2
  245. package/dist/ops/sum16.js +0 -13
  246. package/dist/ops/transpose16.d.ts +0 -3
  247. package/dist/ops/transpose16.js +0 -40
  248. package/dist/ops/unpack16.d.ts +0 -2
  249. package/dist/ops/unpack16.js +0 -6
  250. package/dist/ops/webgl/adamAdjust.d.ts +0 -1
  251. package/dist/ops/webgl/adamAdjust.js +0 -49
  252. package/dist/ops/webgl/adamMoments.d.ts +0 -1
  253. package/dist/ops/webgl/adamMoments.js +0 -40
  254. package/dist/ops/webgl/appendCache.d.ts +0 -1
  255. package/dist/ops/webgl/appendCache.js +0 -44
  256. package/dist/ops/webgl/attentionMask.d.ts +0 -1
  257. package/dist/ops/webgl/attentionMask.js +0 -45
  258. package/dist/ops/webgl/dropout16.d.ts +0 -1
  259. package/dist/ops/webgl/dropout16.js +0 -11
  260. package/dist/ops/webgl/fusedSoftmax.d.ts +0 -11
  261. package/dist/ops/webgl/fusedSoftmax.js +0 -80
  262. package/dist/ops/webgl/gatherSub.d.ts +0 -1
  263. package/dist/ops/webgl/gatherSub.js +0 -27
  264. package/dist/ops/webgl/gelu.d.ts +0 -2
  265. package/dist/ops/webgl/gelu.js +0 -50
  266. package/dist/ops/webgl/log.d.ts +0 -17
  267. package/dist/ops/webgl/log.js +0 -23
  268. package/dist/ops/webgl/matMul16.d.ts +0 -1
  269. package/dist/ops/webgl/matMul16.js +0 -45
  270. package/dist/ops/webgl/matMulGelu.d.ts +0 -21
  271. package/dist/ops/webgl/matMulGelu.js +0 -9
  272. package/dist/ops/webgl/matMulMul.d.ts +0 -14
  273. package/dist/ops/webgl/matMulMul.js +0 -28
  274. package/dist/ops/webgl/mulDropout.d.ts +0 -1
  275. package/dist/ops/webgl/mulDropout.js +0 -41
  276. package/dist/ops/webgl/normRMS.d.ts +0 -1
  277. package/dist/ops/webgl/normRMS.js +0 -93
  278. package/dist/ops/webgl/qkv.d.ts +0 -1
  279. package/dist/ops/webgl/qkv.js +0 -46
  280. package/dist/ops/webgl/rope.d.ts +0 -1
  281. package/dist/ops/webgl/rope.js +0 -56
  282. package/dist/ops/webgl/scatterSub.d.ts +0 -1
  283. package/dist/ops/webgl/scatterSub.js +0 -27
  284. package/dist/ops/webgpu/adamAdjust.d.ts +0 -1
  285. package/dist/ops/webgpu/adamAdjust.js +0 -57
  286. package/dist/ops/webgpu/adamMoments.d.ts +0 -1
  287. package/dist/ops/webgpu/adamMoments.js +0 -60
  288. package/dist/ops/webgpu/add16.d.ts +0 -1
  289. package/dist/ops/webgpu/add16.js +0 -13
  290. package/dist/ops/webgpu/appendCache.d.ts +0 -1
  291. package/dist/ops/webgpu/appendCache.js +0 -105
  292. package/dist/ops/webgpu/attentionMask.d.ts +0 -1
  293. package/dist/ops/webgpu/attentionMask.js +0 -26
  294. package/dist/ops/webgpu/attentionMask32_program.d.ts +0 -19
  295. package/dist/ops/webgpu/attentionMask32_program.js +0 -54
  296. package/dist/ops/webgpu/clipScale.d.ts +0 -1
  297. package/dist/ops/webgpu/clipScale.js +0 -58
  298. package/dist/ops/webgpu/concat16.d.ts +0 -19
  299. package/dist/ops/webgpu/concat16.js +0 -126
  300. package/dist/ops/webgpu/dropout16.d.ts +0 -1
  301. package/dist/ops/webgpu/dropout16.js +0 -51
  302. package/dist/ops/webgpu/gatherSub.d.ts +0 -1
  303. package/dist/ops/webgpu/gatherSub.js +0 -39
  304. package/dist/ops/webgpu/gelu.d.ts +0 -14
  305. package/dist/ops/webgpu/gelu.js +0 -141
  306. package/dist/ops/webgpu/index.d.ts +0 -0
  307. package/dist/ops/webgpu/index.js +0 -26
  308. package/dist/ops/webgpu/matMul16.d.ts +0 -1
  309. package/dist/ops/webgpu/matMul16.js +0 -65
  310. package/dist/ops/webgpu/matMul16_program.d.ts +0 -42
  311. package/dist/ops/webgpu/matMul16_program.js +0 -343
  312. package/dist/ops/webgpu/mul16.d.ts +0 -1
  313. package/dist/ops/webgpu/mul16.js +0 -13
  314. package/dist/ops/webgpu/norm2.d.ts +0 -1
  315. package/dist/ops/webgpu/norm2.js +0 -76
  316. package/dist/ops/webgpu/normRMS.d.ts +0 -1
  317. package/dist/ops/webgpu/normRMS.js +0 -34
  318. package/dist/ops/webgpu/normRMS16_program.d.ts +0 -10
  319. package/dist/ops/webgpu/normRMS16_program.js +0 -25
  320. package/dist/ops/webgpu/normRMS32_program.d.ts +0 -10
  321. package/dist/ops/webgpu/normRMS32_program.js +0 -25
  322. package/dist/ops/webgpu/normRMSGrad.d.ts +0 -1
  323. package/dist/ops/webgpu/normRMSGrad.js +0 -284
  324. package/dist/ops/webgpu/pack16.d.ts +0 -1
  325. package/dist/ops/webgpu/pack16.js +0 -18
  326. package/dist/ops/webgpu/pack16_program.d.ts +0 -19
  327. package/dist/ops/webgpu/pack16_program.js +0 -92
  328. package/dist/ops/webgpu/qkv.d.ts +0 -1
  329. package/dist/ops/webgpu/qkv.js +0 -24
  330. package/dist/ops/webgpu/rope.d.ts +0 -1
  331. package/dist/ops/webgpu/rope.js +0 -135
  332. package/dist/ops/webgpu/scatterSub.d.ts +0 -1
  333. package/dist/ops/webgpu/scatterSub.js +0 -40
  334. package/dist/ops/webgpu/slice16.d.ts +0 -7
  335. package/dist/ops/webgpu/slice16.js +0 -69
  336. package/dist/ops/webgpu/softmax16.d.ts +0 -17
  337. package/dist/ops/webgpu/softmax16.js +0 -21
  338. package/dist/ops/webgpu/softmax16_program.d.ts +0 -13
  339. package/dist/ops/webgpu/softmax16_program.js +0 -73
  340. package/dist/ops/webgpu/softmax16_subgroup_program.d.ts +0 -17
  341. package/dist/ops/webgpu/softmax16_subgroup_program.js +0 -75
  342. package/dist/ops/webgpu/softmax16grad.d.ts +0 -1
  343. package/dist/ops/webgpu/softmax16grad.js +0 -37
  344. package/dist/ops/webgpu/sub16.d.ts +0 -1
  345. package/dist/ops/webgpu/sub16.js +0 -13
  346. package/dist/ops/webgpu/sum16.d.ts +0 -1
  347. package/dist/ops/webgpu/sum16.js +0 -38
  348. package/dist/ops/webgpu/transpose16.d.ts +0 -1
  349. package/dist/ops/webgpu/transpose16.js +0 -34
  350. package/dist/ops/webgpu/transpose16_program.d.ts +0 -16
  351. package/dist/ops/webgpu/transpose16_program.js +0 -50
  352. package/dist/ops/webgpu/transpose16_shared_program.d.ts +0 -15
  353. package/dist/ops/webgpu/transpose16_shared_program.js +0 -70
  354. package/dist/ops/webgpu/unpack16.d.ts +0 -1
  355. package/dist/ops/webgpu/unpack16.js +0 -48
  356. package/dist/ops/webgpu/utils/binary_op.d.ts +0 -35
  357. package/dist/ops/webgpu/utils/binary_op.js +0 -139
  358. package/dist/ops/webgpu/utils/deviceInfo.d.ts +0 -7
  359. package/dist/ops/webgpu/utils/deviceInfo.js +0 -11
  360. package/dist/ops/webgpu/utils/reductions.d.ts +0 -43
  361. package/dist/ops/webgpu/utils/reductions.js +0 -275
  362. package/dist/ops-CsXeTq1P.js +0 -476
  363. package/dist/pack16-bqltoUlR.js +0 -39
  364. package/dist/papaparse.min-C0cScC2i.js +0 -418
  365. package/dist/parquet-Bqjmp2vo.js +0 -44231
  366. package/dist/patches/webgpu_backend.d.ts +0 -18
  367. package/dist/patches/webgpu_backend.js +0 -56
  368. package/dist/patches/webgpu_base.d.ts +0 -21
  369. package/dist/patches/webgpu_base.js +0 -34
  370. package/dist/patches/webgpu_program.d.ts +0 -36
  371. package/dist/patches/webgpu_program.js +0 -400
  372. package/dist/pdf-NIhmP3sq.js +0 -19477
  373. package/dist/rand_util-CZ7yLoUm.js +0 -50
  374. package/dist/random_normal-IBRrha8a.js +0 -14
  375. package/dist/random_width-DN5ZtQkM.js +0 -9796
  376. package/dist/range-C-CjF-LI.js +0 -10
  377. package/dist/relu-J_X6MUzx.js +0 -9
  378. package/dist/reshape-BDOuCSNW.js +0 -9
  379. package/dist/resize_nearest_neighbor-BojqlfRe.js +0 -150
  380. package/dist/rope-DcrZM_e6.js +0 -24
  381. package/dist/scatter_nd_util-ByNJaL6I.js +0 -46
  382. package/dist/segment_util-Dasb2Zaf.js +0 -43
  383. package/dist/selu_util-BLhIqRkw.js +0 -44
  384. package/dist/shared-3agzAqQ_.js +0 -53
  385. package/dist/shared-CagdqkLh.js +0 -2143
  386. package/dist/slice-BzS11Qh0.js +0 -12
  387. package/dist/slice_util-CC35pLmT.js +0 -153
  388. package/dist/softmax-D4q1LJN7.js +0 -12
  389. package/dist/split-C2Sj255c.js +0 -9
  390. package/dist/squeeze-ho4wLUek.js +0 -10
  391. package/dist/stack-DudVrtmG.js +0 -11
  392. package/dist/step-BTxPtq1r.js +0 -261
  393. package/dist/sum-BpiwSWvg.js +0 -11
  394. package/dist/tensor-BWFldCso.js +0 -8
  395. package/dist/tensor1d-LMGMIUlr.js +0 -11
  396. package/dist/tensor2d-BnXMKScO.js +0 -14
  397. package/dist/tensor4d-C6UCG_u8.js +0 -14
  398. package/dist/tfjs_backend-BGnG-ppu.js +0 -654
  399. package/dist/tile-CFy-xTO6.js +0 -11
  400. package/dist/tokeniser/BaseTokeniser.d.ts +0 -33
  401. package/dist/tokeniser/BaseTokeniser.js +0 -124
  402. package/dist/tokeniser/CharTokeniser.d.ts +0 -24
  403. package/dist/tokeniser/CharTokeniser.js +0 -107
  404. package/dist/tokeniser/bpe.d.ts +0 -28
  405. package/dist/tokeniser/bpe.js +0 -173
  406. package/dist/tokeniser/messages.d.ts +0 -61
  407. package/dist/tokeniser/messages.js +0 -1
  408. package/dist/tokeniser/type.d.ts +0 -34
  409. package/dist/tokeniser/type.js +0 -1
  410. package/dist/training/AdamW.d.ts +0 -36
  411. package/dist/training/AdamW.js +0 -138
  412. package/dist/training/BasicTrainer.d.ts +0 -63
  413. package/dist/training/BasicTrainer.js +0 -265
  414. package/dist/training/DatasetBuilder.d.ts +0 -26
  415. package/dist/training/DatasetBuilder.js +0 -86
  416. package/dist/training/Evaluator.d.ts +0 -19
  417. package/dist/training/Evaluator.js +0 -39
  418. package/dist/training/LRScheduler.d.ts +0 -12
  419. package/dist/training/LRScheduler.js +0 -34
  420. package/dist/training/PreTrainer.d.ts +0 -11
  421. package/dist/training/PreTrainer.js +0 -20
  422. package/dist/training/SFTTrainer.d.ts +0 -12
  423. package/dist/training/SFTTrainer.js +0 -22
  424. package/dist/training/loss.d.ts +0 -3
  425. package/dist/training/loss.js +0 -24
  426. package/dist/training/orthoGrad.d.ts +0 -2
  427. package/dist/training/orthoGrad.js +0 -10
  428. package/dist/training/sparseCrossEntropy.d.ts +0 -7
  429. package/dist/training/sparseCrossEntropy.js +0 -69
  430. package/dist/training/tasks/ConversationTask.d.ts +0 -18
  431. package/dist/training/tasks/ConversationTask.js +0 -40
  432. package/dist/training/tasks/PretrainingTask.d.ts +0 -17
  433. package/dist/training/tasks/PretrainingTask.js +0 -47
  434. package/dist/training/tasks/StartSentenceTask.d.ts +0 -18
  435. package/dist/training/tasks/StartSentenceTask.js +0 -49
  436. package/dist/training/tasks/Task.d.ts +0 -22
  437. package/dist/training/tasks/Task.js +0 -68
  438. package/dist/training/tasks/splitter.d.ts +0 -5
  439. package/dist/training/tasks/splitter.js +0 -21
  440. package/dist/training/types.d.ts +0 -78
  441. package/dist/training/types.js +0 -1
  442. package/dist/training/validation.d.ts +0 -17
  443. package/dist/training/validation.js +0 -84
  444. package/dist/transpose-9kRxIXWR.js +0 -36
  445. package/dist/unsorted_segment_sum-DJvk5xnh.js +0 -277
  446. package/dist/utilities/arrayClose.d.ts +0 -1
  447. package/dist/utilities/arrayClose.js +0 -20
  448. package/dist/utilities/datasetID.d.ts +0 -2
  449. package/dist/utilities/datasetID.js +0 -21
  450. package/dist/utilities/dummy.d.ts +0 -9
  451. package/dist/utilities/dummy.js +0 -43
  452. package/dist/utilities/multinomialCPU.d.ts +0 -2
  453. package/dist/utilities/multinomialCPU.js +0 -13
  454. package/dist/utilities/naming.d.ts +0 -4
  455. package/dist/utilities/naming.js +0 -1
  456. package/dist/utilities/packed.d.ts +0 -4
  457. package/dist/utilities/packed.js +0 -15
  458. package/dist/utilities/parameters.d.ts +0 -11
  459. package/dist/utilities/parameters.js +0 -57
  460. package/dist/utilities/performance.d.ts +0 -2
  461. package/dist/utilities/performance.js +0 -16
  462. package/dist/utilities/profile.d.ts +0 -17
  463. package/dist/utilities/profile.js +0 -38
  464. package/dist/utilities/safetensors.d.ts +0 -3
  465. package/dist/utilities/safetensors.js +0 -83
  466. package/dist/utilities/sentences.d.ts +0 -5
  467. package/dist/utilities/sentences.js +0 -41
  468. package/dist/utilities/tokenParse.d.ts +0 -1
  469. package/dist/utilities/tokenParse.js +0 -21
  470. package/dist/utilities/topP.d.ts +0 -1
  471. package/dist/utilities/topP.js +0 -13
  472. package/dist/utilities/waitForModel.d.ts +0 -2
  473. package/dist/utilities/waitForModel.js +0 -12
  474. package/dist/utilities/weights.d.ts +0 -12
  475. package/dist/utilities/weights.js +0 -45
  476. package/dist/utilities/yielder.d.ts +0 -1
  477. package/dist/utilities/yielder.js +0 -7
  478. package/dist/variable-Ck482e3n.js +0 -7
  479. package/dist/webgpu_program-B4HmApL1.js +0 -525
  480. package/dist/webgpu_util-DYlGSwOJ.js +0 -64
  481. package/dist/zeros-DvZpK8s6.js +0 -13
  482. package/dist/zeros_like-CWjDdwr-.js +0 -721
@@ -1,273 +0,0 @@
1
- import { validateConfig as _ } from "./models/config.js";
2
- import { saveModel as p } from "./loader/save.js";
3
- import { loadModel as u } from "./loader/load.js";
4
- import f from "./Generator.js";
5
- import h from "./Trainer.js";
6
- import { E as c } from "./index-DvYrXKkX.js";
7
- import { dummyPassTrainAsync as d } from "./utilities/dummy.js";
8
- import "./index-CUXkjxiT.js";
9
- import "./random_width-DN5ZtQkM.js";
10
- import "./zeros_like-CWjDdwr-.js";
11
- import "./index-Cp39cXWe.js";
12
- import "./dataset-CGGp1z9P.js";
13
- import "./ops/cpu/attentionMask.js";
14
- import "./ops/webgl/attentionMask.js";
15
- import "./ops/grads/attentionMask.js";
16
- import "./ops/cpu/rope.js";
17
- import "./ops/webgl/rope.js";
18
- import "./rope-DcrZM_e6.js";
19
- import "./ops/cpu/appendCache.js";
20
- import "./ops/webgl/appendCache.js";
21
- import "./ops/grads/softmax16.js";
22
- import "./matMul16-BcVC_E62.js";
23
- import "./ops/webgl/matMul16.js";
24
- import "./ops/cpu/matMul16.js";
25
- import "./pack16-bqltoUlR.js";
26
- import "./ops/transpose16.js";
27
- import "./ops/reshape16.js";
28
- import "./ops/cpu/qkv.js";
29
- import "./ops/webgl/qkv.js";
30
- import "./ops/grads/qkv.js";
31
- import "./ops/cpu/normRMS.js";
32
- import "./ops/webgl/normRMS.js";
33
- import "./ops/grads/normRMS.js";
34
- import "./ops/dropout16.js";
35
- import "./ops/webgl/dropout16.js";
36
- import "./ops/grads/add16.js";
37
- import g from "./tokeniser/CharTokeniser.js";
38
- import w from "./tokeniser/bpe.js";
39
- import "./papaparse.min-C0cScC2i.js";
40
- import "./jszip.min-BZhlzntC.js";
41
- import "./ops/cpu/gatherSub.js";
42
- import "./ops/webgl/gatherSub.js";
43
- import "./ops/cpu/scatterSub.js";
44
- import "./ops/webgl/scatterSub.js";
45
- import "./ops/cpu/matMulGelu.js";
46
- import "./matMulGelu-JNLZqKQp.js";
47
- import "./ops/grads/matMulGelu.js";
48
- import "./ops/cpu/gelu.js";
49
- import "./ops/webgl/gelu.js";
50
- import "./gelu-B220X1Go.js";
51
- import "./ops/webgl/log.js";
52
- import "./ops/cpu/adamMoments.js";
53
- import "./ops/webgl/adamMoments.js";
54
- import "./ops/cpu/adamAdjust.js";
55
- import "./ops/webgl/adamAdjust.js";
56
- import "./checks/normRMS.js";
57
- import "./checks/normRMSGrad.js";
58
- import k from "./utilities/profile.js";
59
- import y from "./models/factory.js";
60
- class l {
61
- ee = new c();
62
- _config;
63
- _model;
64
- _tokeniser;
65
- _status = "loading";
66
- _memoryRequirements;
67
- meta = {
68
- version: 2,
69
- application: "@genai-fi/nanogpt"
70
- };
71
- _trainer = null;
72
- constructor(t, i) {
73
- this._config = i?.config, this._tokeniser = t, this._model = i, i?.metaData && (this.meta = i.metaData);
74
- }
75
- get currentTrainer() {
76
- return this._trainer;
77
- }
78
- get vocab() {
79
- return this._tokeniser?.getVocab() || [];
80
- }
81
- get phase() {
82
- return this._model?.metaData?.phase ?? "untrained";
83
- }
84
- set phase(t) {
85
- if (!this._model)
86
- throw new Error("model_not_initialized.");
87
- this._model.metaData.phase = t, this.ee.emit("phase", t);
88
- }
89
- /** Model is fully loaded */
90
- get loaded() {
91
- return !!this._model && !!this._tokeniser && !!this._config;
92
- }
93
- get config() {
94
- if (!this._config)
95
- throw new Error("configuration_not_initialized.");
96
- return this._config;
97
- }
98
- get model() {
99
- if (!this._model)
100
- throw new Error("model_not_initialized.");
101
- return this._model;
102
- }
103
- get tokeniser() {
104
- if (!this._tokeniser)
105
- throw new Error("tokeniser_not_initialized.");
106
- return this._tokeniser;
107
- }
108
- get status() {
109
- return this._status;
110
- }
111
- /** Model is both ready and not busy */
112
- get ready() {
113
- return this._status === "ready" && !!this._model && !!this._tokeniser;
114
- }
115
- get busy() {
116
- return this._status === "busy" || this._status === "training";
117
- }
118
- createLoRA(t, i) {
119
- if (!this._model)
120
- throw new Error("model_not_initialized.");
121
- this._model.createLoRA(t, i), this.ee.emit("changeLoRA");
122
- }
123
- deleteLoRA(t) {
124
- if (!this._model)
125
- throw new Error("model_not_initialized.");
126
- this._model.deleteLoRA(t), this.ee.emit("changeLoRA");
127
- }
128
- renameLoRA(t, i) {
129
- if (!this._model)
130
- throw new Error("model_not_initialized.");
131
- this._model.renameLoRA(t, i), this.ee.emit("changeLoRA");
132
- }
133
- attachLoRA(t) {
134
- if (!this._model)
135
- throw new Error("model_not_initialized.");
136
- this.model.lora?.name !== t && (this._trainer && (this._trainer.dispose(), this._trainer = null), this._model.attachLoRA(t), this.ee.emit("changeLoRA"));
137
- }
138
- detachLoRA() {
139
- if (!this._model)
140
- throw new Error("model_not_initialized.");
141
- this._model.detachLoRA(), this.ee.emit("changeLoRA");
142
- }
143
- hasLoRA(t) {
144
- if (!this._model)
145
- throw new Error("model_not_initialized.");
146
- return this._model.hasLoRA(t);
147
- }
148
- listLoRAs() {
149
- if (!this._model)
150
- throw new Error("model_not_initialized.");
151
- return this._model.listLoRAs();
152
- }
153
- estimateTrainingMemoryUsage(t) {
154
- const i = this._memoryRequirements ?? { perBatch: 0, gradients: 0 }, e = i.perBatch * t, o = i.gradients;
155
- return e * 0.66 + o * 4;
156
- }
157
- setStatus(t) {
158
- this._status !== t && (this._status = t, this.ee.emit("status", t));
159
- }
160
- saveModel(t) {
161
- if (!this._model || !this._tokeniser)
162
- throw new Error("model_or_tokeniser_not_initialized.");
163
- return p(
164
- this._model,
165
- this._tokeniser,
166
- {
167
- ...t,
168
- name: t?.name || this.meta.name
169
- },
170
- t?.includeOptimizer && this._trainer?.trainingType === "pretraining" ? { optimizer: this._trainer.optimizer, trainingLog: this._trainer.log } : void 0
171
- );
172
- }
173
- static loadModel(t, i) {
174
- const e = new l();
175
- return u(t, i).then(({ model: o, tokeniser: s, metaData: r, optimizer: n, log: a }) => {
176
- _(o.config), e._model = o, e._tokeniser = s, e._config = o.config, r && (e.meta = r), e.setStatus("warmup"), d(o).then((m) => {
177
- e._memoryRequirements = m, n && (e._trainer = new h(
178
- o,
179
- s,
180
- "pretraining",
181
- o.metaData.pretrainingSettings,
182
- n
183
- ), a && (e._trainer.log = a)), e.setStatus("ready"), e.ee.emit("loaded"), e.ee.emit("phase", e.phase);
184
- }).catch((m) => {
185
- e.setStatus("error"), e.ee.emit("error", m), console.error("Error during warmup:", m);
186
- });
187
- }).catch((o) => {
188
- e.setStatus("error"), e.ee.emit("error", o), console.error("Error loading model:", o);
189
- }), e;
190
- }
191
- static create(t, i) {
192
- _(i);
193
- const e = i, o = t === "char" ? new g(e.vocabSize) : new w(e.vocabSize), s = y(e), r = new l(o, s);
194
- return r.setStatus("warmup"), d(s).then((n) => {
195
- r._memoryRequirements = n, r.tokeniser.trained ? (r.setStatus("ready"), r.ee.emit("loaded"), r.ee.emit("phase", r.phase)) : (r.setStatus("awaitingTokens"), r.ee.emit("loaded"), r.ee.emit("phase", r.phase), r.tokeniser.once("trainStatus", (a) => {
196
- a === "trained" && r.setStatus("ready");
197
- }));
198
- }).catch((n) => {
199
- r.setStatus("error"), r.ee.emit("error", n), console.error("Error during warmup:", n);
200
- }), r;
201
- }
202
- getProfiler() {
203
- return this._model?.getProfiler();
204
- }
205
- get enableProfiler() {
206
- return !!this._model?.getProfiler();
207
- }
208
- set enableProfiler(t) {
209
- if (t) {
210
- if (!this._config)
211
- return;
212
- this.model.getProfiler() || this.model.setProfiler(new k());
213
- } else
214
- this.model.getProfiler() && this.model.setProfiler(null);
215
- }
216
- getNumParams() {
217
- return this._model ? this._model.getNumParams() : 0;
218
- }
219
- trainer(t, i) {
220
- if (!this._model || !this._tokeniser)
221
- throw new Error("model_or_tokeniser_not_initialized.");
222
- this._trainer && t && this._trainer.trainingType !== t && (this._trainer.dispose(), this._trainer = null);
223
- const e = this._trainer === null ? new h(this._model, this._tokeniser, t, i) : new h(this._trainer, i);
224
- return e.on("start", () => {
225
- this.setStatus("training"), this.phase = t === "sft" ? "finetuned" : "pretrained";
226
- }), e.on("stop", () => this.setStatus("ready")), e.on("log", async (o) => {
227
- const s = this.ee.listeners("trainStep");
228
- for (const r of s)
229
- await r(o);
230
- }), this._trainer && this._trainer !== e && this._trainer.removeAllListeners(), this._trainer = e, e;
231
- }
232
- async train(t, i, e) {
233
- const o = this.trainer(e, i);
234
- await o.prepare(t), await o.train();
235
- }
236
- async trainTokeniser(t) {
237
- if (!this._tokeniser)
238
- throw new Error("tokeniser_not_initialized.");
239
- const i = await this._tokeniser.train(t);
240
- return this._status === "awaitingTokens" && this.setStatus("ready"), i;
241
- }
242
- generator() {
243
- if (!this._model || !this._tokeniser)
244
- throw new Error("model_or_tokeniser_not_initialized.");
245
- const t = new f(this._model, this._tokeniser);
246
- return t.on("start", () => {
247
- this.status === "ready" && this.setStatus("busy");
248
- }), t.on("stop", () => {
249
- this.status === "busy" && this.setStatus("ready");
250
- }), t;
251
- }
252
- generateText(t, i) {
253
- return Array.isArray(t) ? this.generator().generate(t, i) : this.generator().generate([], i);
254
- }
255
- dispose() {
256
- this._trainer && (this._trainer.dispose(), this._trainer = null), this._model?.dispose(), this.ee.removeAllListeners();
257
- }
258
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
259
- on(t, i) {
260
- if (t === "loaded" && this.loaded) {
261
- setTimeout(() => i(), 0);
262
- return;
263
- }
264
- this.ee.on(t, i);
265
- }
266
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
267
- off(t, i) {
268
- this.ee.off(t, i);
269
- }
270
- }
271
- export {
272
- l as default
273
- };
package/dist/Trainer.d.ts DELETED
@@ -1,43 +0,0 @@
1
- import { ITokeniser } from './tokeniser/type';
2
- import { default as EE } from 'eventemitter3';
3
- import { default as Model, ModelForwardAttributes } from './models/model';
4
- import { Task } from './training/tasks/Task';
5
- import { TrainingOptions, TrainingLogEntry } from './training/types';
6
- import { AdamWOptimizer } from './training/AdamW';
7
- import { DatasetMetadata } from './loader/types';
8
- interface TrainingProgress {
9
- lastLog: TrainingLogEntry;
10
- progress: number;
11
- remaining: number;
12
- }
13
- export type TrainingType = 'pretraining' | 'sft';
14
- export default class Trainer extends EE<'start' | 'stop' | 'log'> {
15
- private trainer;
16
- readonly trainingType: TrainingType;
17
- private hasTrained;
18
- private trainDataset?;
19
- private validationDataset?;
20
- private totalTokens;
21
- private tokensProcessed;
22
- log: TrainingLogEntry[];
23
- private progress;
24
- options: TrainingOptions;
25
- protected tokenizer: ITokeniser;
26
- constructor(model: Model<ModelForwardAttributes>, tokeniser: ITokeniser, trainingType?: TrainingType, options?: TrainingOptions, optimizer?: AdamWOptimizer);
27
- constructor(trainer: Trainer, options?: TrainingOptions);
28
- get model(): Model<ModelForwardAttributes>;
29
- get optimizer(): AdamWOptimizer;
30
- stop(): void;
31
- reset(): void;
32
- dispose(): void;
33
- getTotalTokens(): number;
34
- setOptions(options: TrainingOptions): void;
35
- prepare(tasks?: Task[] | Uint16Array, datasets?: DatasetMetadata[]): Promise<void>;
36
- private configureModel;
37
- train(): Promise<void>;
38
- step(options?: TrainingOptions): Promise<void>;
39
- getLog(): TrainingLogEntry[];
40
- getProgress(): TrainingProgress | null;
41
- isPrepared(): boolean;
42
- }
43
- export {};
package/dist/Trainer.js DELETED
@@ -1,244 +0,0 @@
1
- import { E as w } from "./index-DvYrXKkX.js";
2
- import c from "./training/PreTrainer.js";
3
- import { createTrainValidationSplit as f } from "./training/validation.js";
4
- import g from "./training/SFTTrainer.js";
5
- const s = [];
6
- for (let n = 0; n < 256; ++n)
7
- s.push((n + 256).toString(16).slice(1));
8
- function T(n, t = 0) {
9
- return (s[n[t + 0]] + s[n[t + 1]] + s[n[t + 2]] + s[n[t + 3]] + "-" + s[n[t + 4]] + s[n[t + 5]] + "-" + s[n[t + 6]] + s[n[t + 7]] + "-" + s[n[t + 8]] + s[n[t + 9]] + "-" + s[n[t + 10]] + s[n[t + 11]] + s[n[t + 12]] + s[n[t + 13]] + s[n[t + 14]] + s[n[t + 15]]).toLowerCase();
10
- }
11
- const p = new Uint8Array(16);
12
- function k() {
13
- return crypto.getRandomValues(p);
14
- }
15
- function m(n, t, e) {
16
- return crypto.randomUUID ? crypto.randomUUID() : L(n);
17
- }
18
- function L(n, t, e) {
19
- n = n || {};
20
- const i = n.random ?? n.rng?.() ?? k();
21
- if (i.length < 16)
22
- throw new Error("Random bytes length must be >= 16");
23
- return i[6] = i[6] & 15 | 64, i[8] = i[8] & 63 | 128, T(i);
24
- }
25
- class u extends w {
26
- trainer;
27
- trainingType = "pretraining";
28
- hasTrained = !1;
29
- trainDataset;
30
- validationDataset;
31
- totalTokens = 0;
32
- tokensProcessed = 0;
33
- log = [];
34
- progress = null;
35
- options = {
36
- batchSize: 32,
37
- sftMode: "full",
38
- logInterval: 10
39
- };
40
- tokenizer;
41
- constructor(t, e, i, a, o) {
42
- if (super(), t instanceof u) {
43
- const r = e || t.options, h = t.options;
44
- let l = !1;
45
- if (t.trainingType === "sft" && r.sftMode !== h.sftMode && (l = !0), t.trainer instanceof g && t.trainer.loraName && r.loraName !== t.trainer.loraName && (l = !0), i !== void 0 && i !== t.trainingType && (l = !0), l) {
46
- if (t.trainingType === "sft") {
47
- const d = new g(t.model, t.tokenizer, r);
48
- this.trainer = d, d.loraName = r.loraName;
49
- } else
50
- this.trainer = new c(t.model, t.tokenizer, r);
51
- this.trainingType = i || t.trainingType, this.options = r, this.tokenizer = t.tokenizer;
52
- } else
53
- this.trainer = t.trainer, this.trainingType = i || t.trainingType, this.options = r, this.trainer.updateOptimizer(this.options), this.log = t.log, this.progress = t.progress, this.totalTokens = t.totalTokens, this.tokenizer = t.tokenizer, r.batchSize === h.batchSize && (this.trainDataset = t.trainDataset, this.validationDataset = t.validationDataset);
54
- return;
55
- }
56
- if (!e)
57
- throw new Error("Tokeniser must be provided when initializing Trainer with a model");
58
- if (!t)
59
- throw new Error("Model must be provided when initializing Trainer");
60
- if (this.options = a || {
61
- batchSize: 32,
62
- sftMode: "full"
63
- }, i === "sft") {
64
- const r = new g(t, e, a, o);
65
- this.trainer = r, r.loraName = a?.loraName;
66
- } else
67
- this.trainer = new c(t, e, a, o);
68
- this.trainingType = i || "pretraining", this.tokenizer = e;
69
- }
70
- get model() {
71
- return this.trainer.model;
72
- }
73
- get optimizer() {
74
- return this.trainer.optimizer;
75
- }
76
- stop() {
77
- this.trainer.stop();
78
- }
79
- reset() {
80
- this.hasTrained = !1, this.log = [], this.trainer.reset();
81
- }
82
- dispose() {
83
- this.trainer.dispose(), this.removeAllListeners();
84
- }
85
- getTotalTokens() {
86
- return this.totalTokens;
87
- }
88
- setOptions(t) {
89
- const e = new Set(
90
- Object.keys(t).filter(
91
- (i) => t[i] !== this.options[i]
92
- )
93
- );
94
- if (this.trainer.isRunning) {
95
- if (e.has("batchSize"))
96
- throw new Error("Cannot change batch size during training");
97
- if (e.has("sftMode"))
98
- throw new Error("Cannot change SFT mode during training");
99
- if (e.has("loraConfig"))
100
- throw new Error("Cannot change LoRA configuration during training");
101
- if (e.has("validationSplit"))
102
- throw new Error("Cannot change validation split during training");
103
- if (e.has("trainableWeights"))
104
- throw new Error("Cannot change trainable weights during training");
105
- if (e.has("mixedPrecision"))
106
- throw new Error("Cannot change mixed precision setting during training");
107
- if (e.has("gradientCheckpointing"))
108
- throw new Error("Cannot change gradient checkpointing setting during training");
109
- }
110
- this.options = {
111
- ...this.options,
112
- ...t
113
- }, this.trainer.updateOptimizer(this.options), e.has("metrics") && this.trainer.setMetrics(t.metrics || []);
114
- }
115
- async prepare(t = [], e) {
116
- const i = this.options;
117
- if (e && (this.model.metaData.pretrainingData = e.map((a) => ({
118
- id: a.id,
119
- name: a.name
120
- }))), this.trainingType === "pretraining" && this.trainer instanceof c) {
121
- const { trainDataset: a, validationDataset: o, size: r } = await f(
122
- t,
123
- this.trainer.tokenizer,
124
- this.trainer.datasetBuilder,
125
- i?.batchSize || 32,
126
- i?.validationSplit || 0.1,
127
- i?.maskedLoss ?? !1
128
- ), h = r * (1 - (i?.validationSplit || 0));
129
- this.trainDataset = a, this.validationDataset = o, this.totalTokens = h, this.options.epochSteps = Math.ceil(
130
- this.totalTokens / ((i?.batchSize || 32) * this.model.config.blockSize)
131
- ), this.trainer.updateOptimizer(this.options);
132
- } else if (this.trainingType === "sft" && this.trainer instanceof g) {
133
- if (t instanceof Uint16Array)
134
- throw new Error("SFT training requires Task[] input");
135
- const { trainDataset: a, validationDataset: o, size: r } = await f(
136
- t,
137
- this.trainer.tokenizer,
138
- this.trainer.datasetBuilder,
139
- i?.batchSize || 32,
140
- i?.validationSplit || 0.1,
141
- i?.maskedLoss ?? !0
142
- ), h = r * (1 - (i?.validationSplit || 0));
143
- this.trainDataset = a, this.validationDataset = o, this.totalTokens = h, this.options.epochSteps = Math.ceil(
144
- this.totalTokens / ((i?.batchSize || 32) * this.model.config.blockSize)
145
- ), this.trainer.updateOptimizer(this.options);
146
- }
147
- }
148
- configureModel(t) {
149
- const e = t?.sftMode || "full";
150
- if (this.trainingType === "pretraining" && (this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA(), this.trainer.model.weightStore.setTrainable(["*"])), this.trainingType === "sft") {
151
- if (e === "lora") {
152
- const i = this.trainer.model;
153
- if (console.log("Configuring model for LoRA fine-tuning with options:", t), t?.loraName)
154
- if (i.hasLoRA(t.loraName)) {
155
- if (i.attachLoRA(t.loraName), t.loraConfig) {
156
- const a = i.lora;
157
- (a.alpha !== t.loraConfig.alpha || a.rank !== t.loraConfig.rank) && (i.detachLoRA(), i.deleteLoRA(t.loraName), i.createLoRA(t.loraName, t.loraConfig), i.attachLoRA(t.loraName), console.warn("Resetting LoRA with new configuration."));
158
- }
159
- } else if (t.loraConfig)
160
- i.createLoRA(t.loraName, t.loraConfig), i.attachLoRA(t.loraName);
161
- else
162
- throw new Error(
163
- `LoRA configuration must be provided to create LoRA with name ${t.loraName}`
164
- );
165
- else if (t?.loraConfig)
166
- if (i.hasLoRA()) {
167
- const a = i.lora;
168
- if (a.alpha !== t.loraConfig.alpha || a.rank !== t.loraConfig.rank) {
169
- i.detachLoRA();
170
- const o = t.loraName || m();
171
- i.createLoRA(o, t.loraConfig), i.attachLoRA(o);
172
- }
173
- } else {
174
- const a = t.loraName || m();
175
- i.createLoRA(a, t.loraConfig), i.attachLoRA(a);
176
- }
177
- else if (!i.hasLoRA()) throw new Error("LoRA configuration must be provided for lora SFT mode");
178
- } else
179
- this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA();
180
- e === "last-layer" ? this.trainer.model.weightStore.setTrainable([
181
- `block_${this.trainer.model.config.nLayer - 1}_*`,
182
- "token_embedding"
183
- ]) : e === "full" && this.trainer.model.weightStore.setTrainable(["*"]);
184
- }
185
- t?.trainableWeights && this.trainer.model.weightStore.setTrainable(t.trainableWeights);
186
- }
187
- async train() {
188
- const t = this.options;
189
- if (!this.trainDataset)
190
- throw new Error("Dataset not prepared");
191
- this.hasTrained || this.trainer.setLearningRate(t?.learningRate || 1e-3), this.hasTrained = !0, this.emit("start"), this.model.metaData.pretrainingSettings = t;
192
- const e = Date.now();
193
- this.log.length > 0 && this.trainer.resumeFromLog(this.log[this.log.length - 1]), this.trainer.setGradientCheckpointing(t?.gradientCheckpointing || !1), this.trainer.setMixedPrecision(t?.mixedPrecision || !1), this.trainer.setLabelSmoothing(t?.labelSmoothing || 0), this.trainer.setDropout(t?.dropout || 0), this.trainer.setLayerDrop(t?.layerDrop || 0), this.configureModel(t), await this.trainer.trainOnDataset(
194
- this.trainDataset,
195
- {
196
- ...t,
197
- onStep: async (a) => {
198
- this.log.push(a), this.progress = {
199
- lastLog: a,
200
- progress: a.totalTokens / this.totalTokens,
201
- remaining: Math.max(0, (this.totalTokens - a.totalTokens) / a.totalTokens * a.duration)
202
- }, this.tokensProcessed = a.totalTokens;
203
- const o = this.listeners("log");
204
- for (const r of o)
205
- await r(a, this.progress);
206
- }
207
- },
208
- this.validationDataset
209
- ), this.model.metaData.actionLog = this.model.metaData.actionLog || [];
210
- const i = Date.now();
211
- this.model.metaData.actionLog.push({
212
- action: "pretrain",
213
- timestamp: i,
214
- duration: i - e,
215
- tokensProcessed: this.tokensProcessed,
216
- options: t
217
- }), this.emit("stop");
218
- }
219
- async step(t) {
220
- if (!this.trainDataset)
221
- throw new Error("Dataset not prepared");
222
- this.hasTrained || this.trainer.setLearningRate(t?.learningRate || 1e-3), this.hasTrained = !0, this.emit("start");
223
- const { log: e } = await this.trainer.stepDataset(this.trainDataset, t || {}, this.validationDataset), i = this.listeners("log");
224
- for (const a of i)
225
- await a(e, {
226
- lastLog: e,
227
- progress: e.totalTokens / this.totalTokens,
228
- remaining: Math.max(0, (this.totalTokens - e.totalTokens) / e.totalTokens * e.duration)
229
- });
230
- this.emit("stop");
231
- }
232
- getLog() {
233
- return this.log;
234
- }
235
- getProgress() {
236
- return this.progress;
237
- }
238
- isPrepared() {
239
- return this.trainDataset !== void 0 && this.validationDataset !== void 0;
240
- }
241
- }
242
- export {
243
- u as default
244
- };
@@ -1,33 +0,0 @@
1
- var u = typeof globalThis < "u" ? globalThis : typeof window < "u" ? window : typeof global < "u" ? global : typeof self < "u" ? self : {};
2
- function a(e) {
3
- return e && e.__esModule && Object.prototype.hasOwnProperty.call(e, "default") ? e.default : e;
4
- }
5
- function f(e) {
6
- if (Object.prototype.hasOwnProperty.call(e, "__esModule")) return e;
7
- var n = e.default;
8
- if (typeof n == "function") {
9
- var t = function r() {
10
- var o = !1;
11
- try {
12
- o = this instanceof r;
13
- } catch {
14
- }
15
- return o ? Reflect.construct(n, arguments, this.constructor) : n.apply(this, arguments);
16
- };
17
- t.prototype = n.prototype;
18
- } else t = {};
19
- return Object.defineProperty(t, "__esModule", { value: !0 }), Object.keys(e).forEach(function(r) {
20
- var o = Object.getOwnPropertyDescriptor(e, r);
21
- Object.defineProperty(t, r, o.get ? o : {
22
- enumerable: !0,
23
- get: function() {
24
- return e[r];
25
- }
26
- });
27
- }), t;
28
- }
29
- export {
30
- f as a,
31
- u as c,
32
- a as g
33
- };
@@ -1,55 +0,0 @@
1
- import { x as c } from "./index-CUXkjxiT.js";
2
- function i(e, n) {
3
- for (let t = 0; t < e.length; ++t)
4
- if (e[e.length - t - 1] !== n - 1 - t)
5
- return !1;
6
- return !0;
7
- }
8
- function a(e, n, t) {
9
- const r = e.length + n.length, s = [];
10
- let o = 0, f = 0;
11
- for (let u = 0; u < r; u++)
12
- t.indexOf(u) === -1 ? s.push(e[o++]) : s.push(n[f++]);
13
- return s;
14
- }
15
- function l(e, n) {
16
- const t = [], r = e.length;
17
- for (let o = 0; o < r; o++)
18
- n.indexOf(o) === -1 && t.push(e[o]);
19
- const s = n.map((o) => e[o]);
20
- return [t, s];
21
- }
22
- function h(e, n) {
23
- const t = n.map((r) => 1);
24
- return a(e, t, n);
25
- }
26
- function m(e, n, t) {
27
- c(i(n, t), () => `${e} supports only inner-most axes for now. Got axes ${n} and rank-${t} input.`);
28
- }
29
- function d(e, n) {
30
- if (i(e, n))
31
- return null;
32
- const t = [];
33
- for (let r = 0; r < n; ++r)
34
- e.indexOf(r) === -1 && t.push(r);
35
- return e.forEach((r) => t.push(r)), t;
36
- }
37
- function g(e) {
38
- return e.map((n, t) => [t, n]).sort((n, t) => n[1] - t[1]).map((n) => n[0]);
39
- }
40
- function x(e, n) {
41
- const t = [];
42
- for (let r = n - e; r < n; ++r)
43
- t.push(r);
44
- return t;
45
- }
46
- export {
47
- d as a,
48
- x as b,
49
- l as c,
50
- m as d,
51
- h as e,
52
- i as f,
53
- g,
54
- a as h
55
- };
package/dist/backend.d.ts DELETED
@@ -1,2 +0,0 @@
1
- import { GPUOptions } from './patches/webgpu_base';
2
- export declare function selectBackend(backendName: 'cpu' | 'webgl' | 'webgpu', options?: GPUOptions): Promise<void>;
package/dist/backend.js DELETED
@@ -1,13 +0,0 @@
1
- import { g as o, s as e, r as s } from "./index-CUXkjxiT.js";
2
- async function c(t, a) {
3
- if (o() !== t) {
4
- if (t === "webgpu") {
5
- const { registerWebGPUBackend: i } = await import("./patches/webgpu_base.js");
6
- i(a), await import("./index-CjOWnMXP.js"), await import("./ops/webgpu/index.js");
7
- }
8
- await e(t), await s(), console.log(`Backend set to ${t}`);
9
- }
10
- }
11
- export {
12
- c as selectBackend
13
- };