elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,3936 @@
1
+ #define _CRT_SECURE_NO_DEPRECATE // Disables "unsafe" warnings on Windows
2
+ #define _USE_MATH_DEFINES // For M_PI on MSVC
3
+
4
+ #include "ggml-backend-impl.h"
5
+ #include "ggml-backend.h"
6
+ #include "traits.h"
7
+ #include "iqp.h"
8
+ #include "ggml-cpu-impl.h"
9
+ #include "ggml-impl.h"
10
+ #include "quants.h"
11
+ #include "ggml-threading.h"
12
+ #include "unary-ops.h"
13
+ #include "binary-ops.h"
14
+ #include "vec.h"
15
+ #include "ops.h"
16
+ #include "ggml.h"
17
+ #include "common.h"
18
+
19
+ #if defined(_MSC_VER) || defined(__MINGW32__)
20
+ #include <malloc.h> // using malloc.h with MSC/MINGW
21
+ #elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
22
+ #include <alloca.h>
23
+ #endif
24
+
25
+ #include <assert.h>
26
+ #include <errno.h>
27
+ #include <time.h>
28
+ #include <math.h>
29
+ #include <stdlib.h>
30
+ #include <string.h>
31
+ #include <stdint.h>
32
+ #include <inttypes.h>
33
+ #include <stdio.h>
34
+ #include <float.h>
35
+ #include <limits.h>
36
+ #include <stdarg.h>
37
+ #include <signal.h>
38
+ #if defined(__gnu_linux__)
39
+ #include <syscall.h>
40
+ #endif
41
+
42
+ #ifdef GGML_USE_OPENMP
43
+ #include <omp.h>
44
+ #endif
45
+
46
+ #if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_MATMUL_INT8)
47
+ #undef GGML_USE_LLAMAFILE
48
+ #endif
49
+
50
+ #ifdef GGML_USE_LLAMAFILE
51
+ #include "llamafile/sgemm.h"
52
+ #endif
53
+
54
+ #ifdef GGML_USE_CPU_RISCV64_SPACEMIT
55
+ # include "spacemit/ime.h"
56
+ #endif
57
+
58
+ // Note: once we move threading into a separate C++ file
59
+ // will use std::hardware_destructive_interference_size instead of hardcoding it here
60
+ // and we'll use C++ attribute syntax.
61
+ #define GGML_CACHE_LINE 64
62
+
63
+ #if defined(__clang__) || defined(__GNUC__)
64
+ #define GGML_CACHE_ALIGN __attribute__((aligned(GGML_CACHE_LINE)))
65
+ #endif
66
+
67
+ #if defined(__has_feature)
68
+ #if __has_feature(thread_sanitizer)
69
+ #define GGML_TSAN_ENABLED 1
70
+ #endif
71
+ #else // __has_feature
72
+ #if defined(__SANITIZE_THREAD__)
73
+ #define GGML_TSAN_ENABLED 1
74
+ #endif
75
+ #endif // __has_feature
76
+
77
+ #define UNUSED GGML_UNUSED
78
+ #define SWAP(x, y, T) do { T SWAP = x; (x) = y; (y) = SWAP; } while (0)
79
+
80
+ // precomputed f32 table for f16 (256 KB) (simd-mappings.h)
81
+ float ggml_table_f32_f16[1 << 16];
82
+
83
+ // precomputed f32 table for e8m0 half (1 KB) (simd-mappings.h)
84
+ float ggml_table_f32_e8m0_half[1 << 8];
85
+
86
+ // precomputed f32 table for ue4m3 (1 KB) (simd-mappings.h)
87
+ float ggml_table_f32_ue4m3[1 << 8];
88
+
89
+ #if defined(__ARM_ARCH)
90
+ struct ggml_arm_arch_features_type {
91
+ int sve_cnt;
92
+ } ggml_arm_arch_features = { 0 };
93
+ #endif
94
+
95
+ #if defined(__riscv)
96
+ struct ggml_riscv_arch_features_type {
97
+ int rvv_vlen;
98
+ } ggml_riscv_arch_features = { 0 };
99
+ #endif
100
+
101
+ #if defined(_WIN32)
102
+
103
+ #define WIN32_LEAN_AND_MEAN
104
+ #ifndef NOMINMAX
105
+ #define NOMINMAX
106
+ #endif
107
+ #include <windows.h>
108
+
109
+ #if defined(_MSC_VER) && !defined(__clang__)
110
+ #define GGML_CACHE_ALIGN __declspec(align(GGML_CACHE_LINE))
111
+
112
+ typedef volatile LONG atomic_int;
113
+ typedef atomic_int atomic_bool;
114
+ typedef atomic_int atomic_flag;
115
+
116
+ #define ATOMIC_FLAG_INIT 0
117
+
118
+ typedef enum {
119
+ memory_order_relaxed,
120
+ memory_order_consume,
121
+ memory_order_acquire,
122
+ memory_order_release,
123
+ memory_order_acq_rel,
124
+ memory_order_seq_cst
125
+ } memory_order;
126
+
127
+ static void atomic_store(atomic_int * ptr, LONG val) {
128
+ InterlockedExchange(ptr, val);
129
+ }
130
+ static void atomic_store_explicit(atomic_int * ptr, LONG val, memory_order mo) {
131
+ // TODO: add support for explicit memory order
132
+ InterlockedExchange(ptr, val);
133
+ }
134
+ static LONG atomic_load(atomic_int * ptr) {
135
+ return InterlockedCompareExchange(ptr, 0, 0);
136
+ }
137
+ static LONG atomic_load_explicit(atomic_int * ptr, memory_order mo) {
138
+ // TODO: add support for explicit memory order
139
+ return InterlockedCompareExchange(ptr, 0, 0);
140
+ }
141
+ static LONG atomic_fetch_add(atomic_int * ptr, LONG inc) {
142
+ return InterlockedExchangeAdd(ptr, inc);
143
+ }
144
+ static LONG atomic_fetch_add_explicit(atomic_int * ptr, LONG inc, memory_order mo) {
145
+ // TODO: add support for explicit memory order
146
+ return InterlockedExchangeAdd(ptr, inc);
147
+ }
148
+ static atomic_bool atomic_flag_test_and_set(atomic_flag * ptr) {
149
+ return InterlockedExchange(ptr, 1);
150
+ }
151
+ static void atomic_flag_clear(atomic_flag * ptr) {
152
+ InterlockedExchange(ptr, 0);
153
+ }
154
+ static void atomic_thread_fence(memory_order mo) {
155
+ MemoryBarrier();
156
+ }
157
+ #else // clang
158
+ #include <stdatomic.h>
159
+ #endif
160
+
161
+ typedef HANDLE pthread_t;
162
+
163
+ typedef DWORD thread_ret_t;
164
+ static int pthread_create(pthread_t * out, void * unused, thread_ret_t(*func)(void *), void * arg) {
165
+ (void) unused;
166
+ HANDLE handle = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) func, arg, 0, NULL);
167
+ if (handle == NULL)
168
+ {
169
+ return EAGAIN;
170
+ }
171
+
172
+ *out = handle;
173
+ return 0;
174
+ }
175
+
176
+ static int pthread_join(pthread_t thread, void * unused) {
177
+ (void) unused;
178
+ int ret = (int) WaitForSingleObject(thread, INFINITE);
179
+ CloseHandle(thread);
180
+ return ret;
181
+ }
182
+
183
+ static int sched_yield (void) {
184
+ Sleep (0);
185
+ return 0;
186
+ }
187
+ #else
188
+
189
+ #include <pthread.h>
190
+ #include <stdatomic.h>
191
+ #include <sched.h>
192
+ #if defined(__FreeBSD__)
193
+ #include <pthread_np.h>
194
+ #endif
195
+
196
+ typedef void * thread_ret_t;
197
+
198
+ #include <sys/types.h>
199
+ #include <sys/stat.h>
200
+ #include <unistd.h>
201
+
202
+ #endif
203
+
204
+ typedef pthread_t ggml_thread_t;
205
+
206
+ #define GGML_THREADPOOL_N_THREADS_MASK (0xffffU)
207
+ #define GGML_THREADPOOL_N_THREADS_BITS (16)
208
+
209
+ #if defined(__APPLE__)
210
+ #include <unistd.h>
211
+ #include <mach/mach.h>
212
+ #include <TargetConditionals.h>
213
+ #endif
214
+
215
+ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
216
+ [GGML_TYPE_F32] = {
217
+ .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp32,
218
+ .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f32,
219
+ .vec_dot_type = GGML_TYPE_F32,
220
+ .nrows = 1,
221
+ },
222
+ [GGML_TYPE_F16] = {
223
+ .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp16,
224
+ .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16,
225
+ .vec_dot_type = GGML_TYPE_F16,
226
+ .nrows = 1,
227
+ },
228
+ [GGML_TYPE_Q1_0] = {
229
+ .from_float = quantize_row_q1_0,
230
+ .vec_dot = ggml_vec_dot_q1_0_q8_0,
231
+ .vec_dot_type = GGML_TYPE_Q8_0,
232
+ .nrows = 1,
233
+ },
234
+ [GGML_TYPE_Q2_0] = {
235
+ .from_float = quantize_row_q2_0,
236
+ .vec_dot = ggml_vec_dot_q2_0_q8_0,
237
+ .vec_dot_type = GGML_TYPE_Q8_0,
238
+ .nrows = 1,
239
+ },
240
+ [GGML_TYPE_Q4_0] = {
241
+ .from_float = quantize_row_q4_0,
242
+ .vec_dot = ggml_vec_dot_q4_0_q8_0,
243
+ .vec_dot_type = GGML_TYPE_Q8_0,
244
+ #if defined (__ARM_FEATURE_MATMUL_INT8)
245
+ .nrows = 2,
246
+ #else
247
+ .nrows = 1,
248
+ #endif
249
+ },
250
+ [GGML_TYPE_Q4_1] = {
251
+ .from_float = quantize_row_q4_1,
252
+ .vec_dot = ggml_vec_dot_q4_1_q8_1,
253
+ .vec_dot_type = GGML_TYPE_Q8_1,
254
+ #if defined (__ARM_FEATURE_MATMUL_INT8)
255
+ .nrows = 2,
256
+ #else
257
+ .nrows = 1,
258
+ #endif
259
+ },
260
+ [GGML_TYPE_Q5_0] = {
261
+ .from_float = quantize_row_q5_0,
262
+ .vec_dot = ggml_vec_dot_q5_0_q8_0,
263
+ .vec_dot_type = GGML_TYPE_Q8_0,
264
+ .nrows = 1,
265
+ },
266
+ [GGML_TYPE_Q5_1] = {
267
+ .from_float = quantize_row_q5_1,
268
+ .vec_dot = ggml_vec_dot_q5_1_q8_1,
269
+ .vec_dot_type = GGML_TYPE_Q8_1,
270
+ .nrows = 1,
271
+ },
272
+ [GGML_TYPE_Q8_0] = {
273
+ .from_float = quantize_row_q8_0,
274
+ .vec_dot = ggml_vec_dot_q8_0_q8_0,
275
+ .vec_dot_type = GGML_TYPE_Q8_0,
276
+ #if defined (__ARM_FEATURE_MATMUL_INT8)
277
+ .nrows = 2,
278
+ #else
279
+ .nrows = 1,
280
+ #endif
281
+ },
282
+ [GGML_TYPE_Q8_1] = {
283
+ .from_float = quantize_row_q8_1,
284
+ .vec_dot_type = GGML_TYPE_Q8_1,
285
+ .nrows = 1,
286
+ },
287
+ [GGML_TYPE_MXFP4] = {
288
+ .from_float = quantize_row_mxfp4,
289
+ .vec_dot = ggml_vec_dot_mxfp4_q8_0,
290
+ .vec_dot_type = GGML_TYPE_Q8_0,
291
+ .nrows = 1,
292
+ },
293
+ [GGML_TYPE_NVFP4] = {
294
+ .from_float = quantize_row_nvfp4,
295
+ .vec_dot = ggml_vec_dot_nvfp4_q8_0,
296
+ .vec_dot_type = GGML_TYPE_Q8_0,
297
+ .nrows = 1,
298
+ },
299
+ [GGML_TYPE_Q2_K] = {
300
+ .from_float = quantize_row_q2_K,
301
+ .vec_dot = ggml_vec_dot_q2_K_q8_K,
302
+ .vec_dot_type = GGML_TYPE_Q8_K,
303
+ .nrows = 1,
304
+ },
305
+ [GGML_TYPE_Q3_K] = {
306
+ .from_float = quantize_row_q3_K,
307
+ .vec_dot = ggml_vec_dot_q3_K_q8_K,
308
+ .vec_dot_type = GGML_TYPE_Q8_K,
309
+ .nrows = 1,
310
+ },
311
+ [GGML_TYPE_Q4_K] = {
312
+ .from_float = quantize_row_q4_K,
313
+ .vec_dot = ggml_vec_dot_q4_K_q8_K,
314
+ .vec_dot_type = GGML_TYPE_Q8_K,
315
+ #if defined (__ARM_FEATURE_MATMUL_INT8)
316
+ .nrows = 2,
317
+ #else
318
+ .nrows = 1,
319
+ #endif
320
+ },
321
+ [GGML_TYPE_Q5_K] = {
322
+ .from_float = quantize_row_q5_K,
323
+ .vec_dot = ggml_vec_dot_q5_K_q8_K,
324
+ .vec_dot_type = GGML_TYPE_Q8_K,
325
+ .nrows = 1,
326
+ },
327
+ [GGML_TYPE_Q6_K] = {
328
+ .from_float = quantize_row_q6_K,
329
+ .vec_dot = ggml_vec_dot_q6_K_q8_K,
330
+ .vec_dot_type = GGML_TYPE_Q8_K,
331
+ #if defined (__ARM_FEATURE_MATMUL_INT8)
332
+ .nrows = 2,
333
+ #else
334
+ .nrows = 1,
335
+ #endif
336
+ },
337
+ [GGML_TYPE_IQ2_XXS] = {
338
+ .from_float = NULL,
339
+ .vec_dot = ggml_vec_dot_iq2_xxs_q8_K,
340
+ .vec_dot_type = GGML_TYPE_Q8_K,
341
+ .nrows = 1,
342
+ },
343
+ [GGML_TYPE_IQ2_XS] = {
344
+ .from_float = NULL,
345
+ .vec_dot = ggml_vec_dot_iq2_xs_q8_K,
346
+ .vec_dot_type = GGML_TYPE_Q8_K,
347
+ .nrows = 1,
348
+ },
349
+ [GGML_TYPE_IQ3_XXS] = {
350
+ // NOTE: from_float for iq3 and iq2_s was removed because these quants require initialization in ggml_quantize_init
351
+ //.from_float = quantize_row_iq3_xxs,
352
+ .vec_dot = ggml_vec_dot_iq3_xxs_q8_K,
353
+ .vec_dot_type = GGML_TYPE_Q8_K,
354
+ .nrows = 1,
355
+ },
356
+ [GGML_TYPE_IQ3_S] = {
357
+ //.from_float = quantize_row_iq3_s,
358
+ .vec_dot = ggml_vec_dot_iq3_s_q8_K,
359
+ .vec_dot_type = GGML_TYPE_Q8_K,
360
+ .nrows = 1,
361
+ },
362
+ [GGML_TYPE_IQ2_S] = {
363
+ //.from_float = quantize_row_iq2_s,
364
+ .vec_dot = ggml_vec_dot_iq2_s_q8_K,
365
+ .vec_dot_type = GGML_TYPE_Q8_K,
366
+ .nrows = 1,
367
+ },
368
+ [GGML_TYPE_IQ1_S] = {
369
+ .from_float = NULL,
370
+ .vec_dot = ggml_vec_dot_iq1_s_q8_K,
371
+ .vec_dot_type = GGML_TYPE_Q8_K,
372
+ .nrows = 1,
373
+ },
374
+ [GGML_TYPE_IQ1_M] = {
375
+ .from_float = NULL,
376
+ .vec_dot = ggml_vec_dot_iq1_m_q8_K,
377
+ .vec_dot_type = GGML_TYPE_Q8_K,
378
+ .nrows = 1,
379
+ },
380
+ [GGML_TYPE_IQ4_NL] = {
381
+ .from_float = quantize_row_iq4_nl,
382
+ .vec_dot = ggml_vec_dot_iq4_nl_q8_0,
383
+ .vec_dot_type = GGML_TYPE_Q8_0,
384
+ .nrows = 1,
385
+ },
386
+ [GGML_TYPE_IQ4_XS] = {
387
+ .from_float = quantize_row_iq4_xs,
388
+ .vec_dot = ggml_vec_dot_iq4_xs_q8_K,
389
+ .vec_dot_type = GGML_TYPE_Q8_K,
390
+ .nrows = 1,
391
+ },
392
+ [GGML_TYPE_Q8_K] = {
393
+ .from_float = quantize_row_q8_K,
394
+ },
395
+ [GGML_TYPE_BF16] = {
396
+ .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_bf16,
397
+ .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_bf16,
398
+ .vec_dot_type = GGML_TYPE_BF16,
399
+ .nrows = 1,
400
+ },
401
+ [GGML_TYPE_TQ1_0] = {
402
+ .from_float = quantize_row_tq1_0,
403
+ .vec_dot = ggml_vec_dot_tq1_0_q8_K,
404
+ .vec_dot_type = GGML_TYPE_Q8_K,
405
+ .nrows = 1,
406
+ },
407
+ [GGML_TYPE_TQ2_0] = {
408
+ .from_float = quantize_row_tq2_0,
409
+ .vec_dot = ggml_vec_dot_tq2_0_q8_K,
410
+ .vec_dot_type = GGML_TYPE_Q8_K,
411
+ .nrows = 1,
412
+ },
413
+ [GGML_TYPE_I32] = {
414
+ .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_i32,
415
+ },
416
+ };
417
+
418
+ const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) {
419
+ return &type_traits_cpu[type];
420
+ }
421
+
422
+ //
423
+ // Threading defs
424
+ //
425
+
426
+ typedef pthread_t ggml_thread_t;
427
+
428
+ #if defined(_WIN32)
429
+
430
+ typedef CONDITION_VARIABLE ggml_cond_t;
431
+ typedef SRWLOCK ggml_mutex_t;
432
+
433
+ #define ggml_mutex_init(m) InitializeSRWLock(m)
434
+ #define ggml_mutex_destroy(m)
435
+ #define ggml_mutex_lock(m) AcquireSRWLockExclusive(m)
436
+ #define ggml_mutex_unlock(m) ReleaseSRWLockExclusive(m)
437
+ #define ggml_mutex_lock_shared(m) AcquireSRWLockShared(m)
438
+ #define ggml_mutex_unlock_shared(m) ReleaseSRWLockShared(m)
439
+
440
+ #define ggml_cond_init(c) InitializeConditionVariable(c)
441
+ #define ggml_cond_destroy(c)
442
+ #define ggml_cond_wait(c, m) SleepConditionVariableSRW(c, m, INFINITE, CONDITION_VARIABLE_LOCKMODE_SHARED)
443
+ #define ggml_cond_broadcast(c) WakeAllConditionVariable(c)
444
+
445
+ #define ggml_thread_create pthread_create
446
+ #define ggml_thread_join pthread_join
447
+
448
+ #else
449
+
450
+ typedef pthread_cond_t ggml_cond_t;
451
+ typedef pthread_mutex_t ggml_mutex_t;
452
+
453
+ #define ggml_mutex_init(m) pthread_mutex_init(m, NULL)
454
+ #define ggml_mutex_destroy(m) pthread_mutex_destroy(m)
455
+ #define ggml_mutex_lock(m) pthread_mutex_lock(m)
456
+ #define ggml_mutex_unlock(m) pthread_mutex_unlock(m)
457
+ #define ggml_mutex_lock_shared(m) pthread_mutex_lock(m)
458
+ #define ggml_mutex_unlock_shared(m) pthread_mutex_unlock(m)
459
+
460
+ #define ggml_lock_init(x) UNUSED(x)
461
+ #define ggml_lock_destroy(x) UNUSED(x)
462
+ #if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64))
463
+ #define ggml_lock_lock(x) _mm_pause()
464
+ #else
465
+ #define ggml_lock_lock(x) UNUSED(x)
466
+ #endif
467
+ #define ggml_lock_unlock(x) UNUSED(x)
468
+
469
+ #define GGML_LOCK_INITIALIZER 0
470
+ #define ggml_cond_init(c) pthread_cond_init(c, NULL)
471
+ #define ggml_cond_destroy(c) pthread_cond_destroy(c)
472
+ #define ggml_cond_wait(c, m) pthread_cond_wait(c, m)
473
+ #define ggml_cond_broadcast(c) pthread_cond_broadcast(c)
474
+
475
+ #define ggml_thread_create pthread_create
476
+ #define ggml_thread_join pthread_join
477
+
478
+ #endif
479
+
480
+ // Threadpool def
481
+ struct ggml_threadpool {
482
+ ggml_mutex_t mutex; // mutex for cond.var
483
+ ggml_cond_t cond; // cond.var for waiting for new work
484
+
485
+ struct ggml_cgraph * cgraph;
486
+ struct ggml_cplan * cplan;
487
+
488
+ // synchronization primitives
489
+ atomic_int n_graph; // updated when there is work to be done (i.e each graph) holds graph and active thread counts.
490
+ atomic_int GGML_CACHE_ALIGN n_barrier;
491
+ atomic_int GGML_CACHE_ALIGN n_barrier_passed;
492
+ atomic_int GGML_CACHE_ALIGN current_chunk; // currently processing chunk during Mat_Mul, shared between all the threads.
493
+
494
+ // these are atomic as an annotation for thread-sanitizer
495
+ atomic_bool stop; // Used for stopping the threadpool altogether
496
+ atomic_bool pause; // Used for pausing the threadpool or individual threads
497
+ atomic_int abort; // Used for aborting processing of a graph
498
+
499
+ struct ggml_compute_state * workers; // per thread state
500
+ int n_threads; // Number of threads in the pool
501
+ int32_t prio; // Scheduling priority
502
+ uint32_t poll; // Polling level (0 - no polling)
503
+
504
+ enum ggml_status ec;
505
+ };
506
+
507
+ // Per-thread state
508
+ struct ggml_compute_state {
509
+ #ifndef GGML_USE_OPENMP
510
+ ggml_thread_t thrd;
511
+ int last_graph;
512
+ bool pending;
513
+ #endif
514
+ bool cpumask[GGML_MAX_N_THREADS];
515
+ struct ggml_threadpool * threadpool;
516
+ int ith;
517
+ };
518
+
519
+ // Helpers for polling loops
520
+ #if defined(__aarch64__) && ( defined(__clang__) || defined(__GNUC__) )
521
+ static inline void ggml_thread_cpu_relax(void) {
522
+ __asm__ volatile("yield" ::: "memory");
523
+ }
524
+ #elif defined(__x86_64__)
525
+ static inline void ggml_thread_cpu_relax(void) {
526
+ _mm_pause();
527
+ }
528
+ #elif defined(__riscv)
529
+ static inline void ggml_thread_cpu_relax(void) {
530
+ #ifdef __riscv_zihintpause
531
+ __asm__ __volatile__ ("pause");
532
+ #else
533
+ /* Encoding of the pause instruction */
534
+ __asm__ __volatile__ (".4byte 0x100000F");
535
+ #endif
536
+ }
537
+ #else
538
+ static inline void ggml_thread_cpu_relax(void) {;}
539
+ #endif
540
+
541
+ //
542
+ // NUMA support
543
+ //
544
+
545
+ #define GGML_NUMA_MAX_NODES 8
546
+ #define GGML_NUMA_MAX_CPUS 512
547
+
548
+ struct ggml_numa_node {
549
+ uint32_t cpus[GGML_NUMA_MAX_CPUS]; // hardware threads on this node
550
+ uint32_t n_cpus;
551
+ };
552
+
553
+ struct ggml_numa_nodes {
554
+ enum ggml_numa_strategy numa_strategy;
555
+ struct ggml_numa_node nodes[GGML_NUMA_MAX_NODES];
556
+ uint32_t n_nodes;
557
+ uint32_t total_cpus; // hardware threads on system
558
+ uint32_t current_node; // node on which main process is execting
559
+ #if defined(__gnu_linux__)
560
+ cpu_set_t cpuset; // cpuset from numactl
561
+ #else
562
+ uint32_t cpuset; // no NUMA support outside of Linux at this time. Use a portable datatype
563
+ #endif
564
+ };
565
+
566
+ //
567
+ // ggml state
568
+ //
569
+
570
+ struct ggml_state {
571
+ struct ggml_numa_nodes numa;
572
+ };
573
+
574
+ static struct ggml_state g_state = {0};
575
+
576
+ void ggml_barrier(struct ggml_threadpool * tp) {
577
+ int n_threads = atomic_load_explicit(&tp->n_graph, memory_order_relaxed) & GGML_THREADPOOL_N_THREADS_MASK;
578
+ if (n_threads == 1) {
579
+ return;
580
+ }
581
+
582
+ #ifdef GGML_USE_OPENMP
583
+ #pragma omp barrier
584
+ #else
585
+ int n_passed = atomic_load_explicit(&tp->n_barrier_passed, memory_order_relaxed);
586
+
587
+ // enter barrier (full seq-cst fence)
588
+ int n_barrier = atomic_fetch_add_explicit(&tp->n_barrier, 1, memory_order_seq_cst);
589
+
590
+ if (n_barrier == (n_threads - 1)) {
591
+ // last thread
592
+ atomic_store_explicit(&tp->n_barrier, 0, memory_order_relaxed);
593
+
594
+ // exit barrier (full seq-cst fence)
595
+ atomic_fetch_add_explicit(&tp->n_barrier_passed, 1, memory_order_seq_cst);
596
+ return;
597
+ }
598
+
599
+ // wait for other threads
600
+ while (atomic_load_explicit(&tp->n_barrier_passed, memory_order_relaxed) == n_passed) {
601
+ ggml_thread_cpu_relax();
602
+ }
603
+
604
+ // exit barrier (full seq-cst fence)
605
+ // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead
606
+ #ifdef GGML_TSAN_ENABLED
607
+ atomic_fetch_add_explicit(&tp->n_barrier_passed, 0, memory_order_seq_cst);
608
+ #else
609
+ atomic_thread_fence(memory_order_seq_cst);
610
+ #endif
611
+ #endif
612
+ }
613
+
614
+ void ggml_threadpool_chunk_set(struct ggml_threadpool * tp, int value) {
615
+ atomic_store_explicit(&tp->current_chunk, value, memory_order_relaxed);
616
+ }
617
+
618
+ int ggml_threadpool_chunk_add(struct ggml_threadpool * tp, int value) {
619
+ return atomic_fetch_add_explicit(&tp->current_chunk, value, memory_order_relaxed);
620
+ }
621
+
622
+ #if defined(__gnu_linux__)
623
+ static cpu_set_t ggml_get_numa_affinity(void) {
624
+ cpu_set_t cpuset;
625
+ pthread_t thread;
626
+ thread = pthread_self();
627
+ CPU_ZERO(&cpuset);
628
+ pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
629
+ return cpuset;
630
+ }
631
+ #else
632
+ static uint32_t ggml_get_numa_affinity(void) {
633
+ return 0; // no NUMA support
634
+ }
635
+ #endif
636
+
637
+ void ggml_numa_init(enum ggml_numa_strategy numa_flag) {
638
+ if (g_state.numa.n_nodes > 0) {
639
+ fprintf(stderr, "ggml_numa_init: NUMA already initialized\n");
640
+
641
+ return;
642
+ }
643
+
644
+ #if defined(__gnu_linux__)
645
+ struct stat st;
646
+ char path[256];
647
+ int rv;
648
+
649
+ // set numa scheme
650
+ g_state.numa.numa_strategy = numa_flag;
651
+
652
+ GGML_PRINT_DEBUG("numa strategy %u\n",g_state.numa.numa_strategy);
653
+
654
+ g_state.numa.cpuset = ggml_get_numa_affinity();
655
+
656
+ // enumerate nodes
657
+ while (g_state.numa.n_nodes < GGML_NUMA_MAX_NODES) {
658
+ rv = snprintf(path, sizeof(path), "/sys/devices/system/node/node%u", g_state.numa.n_nodes);
659
+ GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));
660
+ if (stat(path, &st) != 0) { break; }
661
+ ++g_state.numa.n_nodes;
662
+ }
663
+
664
+ // enumerate CPUs
665
+ while (g_state.numa.total_cpus < GGML_NUMA_MAX_CPUS) {
666
+ rv = snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%u", g_state.numa.total_cpus);
667
+ GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));
668
+ if (stat(path, &st) != 0) { break; }
669
+ ++g_state.numa.total_cpus;
670
+ }
671
+
672
+ GGML_PRINT_DEBUG("found %u numa nodes, %u CPUs\n", g_state.numa.n_nodes, g_state.numa.total_cpus);
673
+
674
+ // figure out which node we're on
675
+ uint current_cpu;
676
+ int getcpu_ret = 0;
677
+ #if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 33) || defined(__COSMOPOLITAN__)
678
+ getcpu_ret = getcpu(&current_cpu, &g_state.numa.current_node);
679
+ #else
680
+ // old glibc doesn't have a wrapper for this call. Fall back on direct syscall
681
+ # if !defined(SYS_getcpu) && defined(SYS_get_cpu)
682
+ # define SYS_getcpu SYS_get_cpu // some older glibc versions use this name
683
+ # endif
684
+ getcpu_ret = syscall(SYS_getcpu, &current_cpu, &g_state.numa.current_node);
685
+ #endif
686
+
687
+ if (g_state.numa.n_nodes < 1 || g_state.numa.total_cpus < 1 || getcpu_ret != 0) {
688
+ g_state.numa.n_nodes = 0;
689
+ return;
690
+ }
691
+
692
+ GGML_PRINT_DEBUG("found our process on numa node %u, CPU %u\n", g_state.numa.current_node, current_cpu);
693
+
694
+ for (uint32_t n = 0; n < g_state.numa.n_nodes; ++n) {
695
+ struct ggml_numa_node * node = &g_state.numa.nodes[n];
696
+ GGML_PRINT_DEBUG("CPUs on node %u:", n);
697
+ node->n_cpus = 0;
698
+ for (uint32_t c = 0; c < g_state.numa.total_cpus; ++c) {
699
+ rv = snprintf(path, sizeof(path), "/sys/devices/system/node/node%u/cpu%u", n, c);
700
+ GGML_ASSERT(rv > 0 && (unsigned)rv < sizeof(path));
701
+ if (stat(path, &st) == 0) {
702
+ node->cpus[node->n_cpus++] = c;
703
+ GGML_PRINT_DEBUG(" %u", c);
704
+ }
705
+ }
706
+ GGML_PRINT_DEBUG("\n");
707
+ }
708
+
709
+ if (ggml_is_numa()) {
710
+ FILE *fptr = fopen("/proc/sys/kernel/numa_balancing", "r");
711
+ if (fptr != NULL) {
712
+ char buf[42];
713
+ if (fgets(buf, sizeof(buf), fptr) && strncmp(buf, "0\n", sizeof(buf)) != 0) {
714
+ GGML_LOG_WARN("/proc/sys/kernel/numa_balancing is enabled, this has been observed to impair performance\n");
715
+ }
716
+ fclose(fptr);
717
+ }
718
+ }
719
+ #else
720
+ UNUSED(numa_flag);
721
+ // TODO
722
+ #endif
723
+ }
724
+
725
+ bool ggml_is_numa(void) {
726
+ return g_state.numa.n_nodes > 1;
727
+ }
728
+
729
+ #if defined(__ARM_ARCH)
730
+ #if defined(__aarch64__) && defined(__ARM_FEATURE_SVE)
731
+ #include <arm_sve.h>
732
+ static void ggml_init_arm_arch_features(void) {
733
+ ggml_arm_arch_features.sve_cnt = svcntb();
734
+ }
735
+ #else
736
+ static void ggml_init_arm_arch_features(void) {}
737
+ #endif
738
+ #endif // __ARM_ARCH
739
+
740
+ #if defined(__riscv) && defined(__riscv_v_intrinsic)
741
+ #include <riscv_vector.h>
742
+ static void ggml_init_riscv_arch_features(void) {
743
+ ggml_riscv_arch_features.rvv_vlen = __riscv_vlenb();
744
+ }
745
+ #else
746
+ static void ggml_init_riscv_arch_features(void) {}
747
+ #endif
748
+
749
+ struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value) {
750
+ GGML_ASSERT(!ggml_get_no_alloc(ctx));
751
+
752
+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);
753
+
754
+ ggml_set_i32(result, value);
755
+
756
+ return result;
757
+ }
758
+
759
+ struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value) {
760
+ GGML_ASSERT(!ggml_get_no_alloc(ctx));
761
+
762
+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
763
+
764
+ ggml_set_f32(result, value);
765
+
766
+ return result;
767
+ }
768
+
769
+ struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value) {
770
+ const int n = ggml_nrows(tensor);
771
+ const int nc = tensor->ne[0];
772
+ const size_t n1 = tensor->nb[1];
773
+
774
+ char * const data = tensor->data;
775
+
776
+ switch (tensor->type) {
777
+ case GGML_TYPE_I8:
778
+ {
779
+ assert(tensor->nb[0] == sizeof(int8_t));
780
+ for (int i = 0; i < n; i++) {
781
+ ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);
782
+ }
783
+ } break;
784
+ case GGML_TYPE_I16:
785
+ {
786
+ assert(tensor->nb[0] == sizeof(int16_t));
787
+ for (int i = 0; i < n; i++) {
788
+ ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);
789
+ }
790
+ } break;
791
+ case GGML_TYPE_I32:
792
+ {
793
+ assert(tensor->nb[0] == sizeof(int32_t));
794
+ for (int i = 0; i < n; i++) {
795
+ ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);
796
+ }
797
+ } break;
798
+ case GGML_TYPE_F16:
799
+ {
800
+ assert(tensor->nb[0] == sizeof(ggml_fp16_t));
801
+ for (int i = 0; i < n; i++) {
802
+ ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value));
803
+ }
804
+ } break;
805
+ case GGML_TYPE_BF16:
806
+ {
807
+ assert(tensor->nb[0] == sizeof(ggml_fp16_t));
808
+ for (int i = 0; i < n; i++) {
809
+ ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));
810
+ }
811
+ } break;
812
+ case GGML_TYPE_F32:
813
+ {
814
+ assert(tensor->nb[0] == sizeof(float));
815
+ for (int i = 0; i < n; i++) {
816
+ ggml_vec_set_f32(nc, (float *)(data + i*n1), value);
817
+ }
818
+ } break;
819
+ default:
820
+ {
821
+ GGML_ABORT("fatal error");
822
+ }
823
+ }
824
+
825
+ return tensor;
826
+ }
827
+
828
+ struct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) {
829
+ const int n = ggml_nrows(tensor);
830
+ const int nc = tensor->ne[0];
831
+ const size_t n1 = tensor->nb[1];
832
+
833
+ char * const data = tensor->data;
834
+
835
+ switch (tensor->type) {
836
+ case GGML_TYPE_I8:
837
+ {
838
+ assert(tensor->nb[0] == sizeof(int8_t));
839
+ for (int i = 0; i < n; i++) {
840
+ ggml_vec_set_i8(nc, (int8_t *)(data + i*n1), value);
841
+ }
842
+ } break;
843
+ case GGML_TYPE_I16:
844
+ {
845
+ assert(tensor->nb[0] == sizeof(int16_t));
846
+ for (int i = 0; i < n; i++) {
847
+ ggml_vec_set_i16(nc, (int16_t *)(data + i*n1), value);
848
+ }
849
+ } break;
850
+ case GGML_TYPE_I32:
851
+ {
852
+ assert(tensor->nb[0] == sizeof(int32_t));
853
+ for (int i = 0; i < n; i++) {
854
+ ggml_vec_set_i32(nc, (int32_t *)(data + i*n1), value);
855
+ }
856
+ } break;
857
+ case GGML_TYPE_F16:
858
+ {
859
+ assert(tensor->nb[0] == sizeof(ggml_fp16_t));
860
+ for (int i = 0; i < n; i++) {
861
+ ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_CPU_FP32_TO_FP16(value));
862
+ }
863
+ } break;
864
+ case GGML_TYPE_BF16:
865
+ {
866
+ assert(tensor->nb[0] == sizeof(ggml_bf16_t));
867
+ for (int i = 0; i < n; i++) {
868
+ ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));
869
+ }
870
+ } break;
871
+ case GGML_TYPE_F32:
872
+ {
873
+ assert(tensor->nb[0] == sizeof(float));
874
+ for (int i = 0; i < n; i++) {
875
+ ggml_vec_set_f32(nc, (float *)(data + i*n1), value);
876
+ }
877
+ } break;
878
+ default:
879
+ {
880
+ GGML_ABORT("fatal error");
881
+ }
882
+ }
883
+
884
+ return tensor;
885
+ }
886
+
887
+ int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i) {
888
+ if (!ggml_is_contiguous(tensor)) {
889
+ int64_t id[4] = { 0, 0, 0, 0 };
890
+ ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
891
+ return ggml_get_i32_nd(tensor, id[0], id[1], id[2], id[3]);
892
+ }
893
+ switch (tensor->type) {
894
+ case GGML_TYPE_I8:
895
+ {
896
+ GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));
897
+ return ((int8_t *)(tensor->data))[i];
898
+ }
899
+ case GGML_TYPE_I16:
900
+ {
901
+ GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));
902
+ return ((int16_t *)(tensor->data))[i];
903
+ }
904
+ case GGML_TYPE_I32:
905
+ {
906
+ GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));
907
+ return ((int32_t *)(tensor->data))[i];
908
+ }
909
+ case GGML_TYPE_F16:
910
+ {
911
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
912
+ return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);
913
+ }
914
+ case GGML_TYPE_BF16:
915
+ {
916
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));
917
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);
918
+ }
919
+ case GGML_TYPE_F32:
920
+ {
921
+ GGML_ASSERT(tensor->nb[0] == sizeof(float));
922
+ return ((float *)(tensor->data))[i];
923
+ }
924
+ default:
925
+ {
926
+ GGML_ABORT("fatal error");
927
+ }
928
+ }
929
+ }
930
+
931
+ void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value) {
932
+ if (!ggml_is_contiguous(tensor)) {
933
+ int64_t id[4] = { 0, 0, 0, 0 };
934
+ ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
935
+ ggml_set_i32_nd(tensor, id[0], id[1], id[2], id[3], value);
936
+ return;
937
+ }
938
+ switch (tensor->type) {
939
+ case GGML_TYPE_I8:
940
+ {
941
+ GGML_ASSERT(tensor->nb[0] == sizeof(int8_t));
942
+ ((int8_t *)(tensor->data))[i] = value;
943
+ } break;
944
+ case GGML_TYPE_I16:
945
+ {
946
+ GGML_ASSERT(tensor->nb[0] == sizeof(int16_t));
947
+ ((int16_t *)(tensor->data))[i] = value;
948
+ } break;
949
+ case GGML_TYPE_I32:
950
+ {
951
+ GGML_ASSERT(tensor->nb[0] == sizeof(int32_t));
952
+ ((int32_t *)(tensor->data))[i] = value;
953
+ } break;
954
+ case GGML_TYPE_F16:
955
+ {
956
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
957
+ ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value);
958
+ } break;
959
+ case GGML_TYPE_BF16:
960
+ {
961
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));
962
+ ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);
963
+ } break;
964
+ case GGML_TYPE_F32:
965
+ {
966
+ GGML_ASSERT(tensor->nb[0] == sizeof(float));
967
+ ((float *)(tensor->data))[i] = value;
968
+ } break;
969
+ default:
970
+ {
971
+ GGML_ABORT("fatal error");
972
+ }
973
+ }
974
+ }
975
+
976
+ int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {
977
+ void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
978
+ switch (tensor->type) {
979
+ case GGML_TYPE_I8:
980
+ return ((int8_t *) data)[0];
981
+ case GGML_TYPE_I16:
982
+ return ((int16_t *) data)[0];
983
+ case GGML_TYPE_I32:
984
+ return ((int32_t *) data)[0];
985
+ case GGML_TYPE_F16:
986
+ return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);
987
+ case GGML_TYPE_BF16:
988
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);
989
+ case GGML_TYPE_F32:
990
+ return ((float *) data)[0];
991
+ default:
992
+ GGML_ABORT("fatal error");
993
+ }
994
+ }
995
+
996
+ void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value) {
997
+ void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
998
+ switch (tensor->type) {
999
+ case GGML_TYPE_I8:
1000
+ {
1001
+ ((int8_t *)(data))[0] = value;
1002
+ } break;
1003
+ case GGML_TYPE_I16:
1004
+ {
1005
+ ((int16_t *)(data))[0] = value;
1006
+ } break;
1007
+ case GGML_TYPE_I32:
1008
+ {
1009
+ ((int32_t *)(data))[0] = value;
1010
+ } break;
1011
+ case GGML_TYPE_F16:
1012
+ {
1013
+ ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value);
1014
+ } break;
1015
+ case GGML_TYPE_BF16:
1016
+ {
1017
+ ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);
1018
+ } break;
1019
+ case GGML_TYPE_F32:
1020
+ {
1021
+ ((float *)(data))[0] = value;
1022
+ } break;
1023
+ default:
1024
+ {
1025
+ GGML_ABORT("fatal error");
1026
+ }
1027
+ }
1028
+ }
1029
+
1030
+ float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i) {
1031
+ if (!ggml_is_contiguous(tensor)) {
1032
+ int64_t id[4] = { 0, 0, 0, 0 };
1033
+ ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
1034
+ return ggml_get_f32_nd(tensor, id[0], id[1], id[2], id[3]);
1035
+ }
1036
+ switch (tensor->type) {
1037
+ case GGML_TYPE_I8:
1038
+ {
1039
+ return ((int8_t *)(tensor->data))[i];
1040
+ }
1041
+ case GGML_TYPE_I16:
1042
+ {
1043
+ return ((int16_t *)(tensor->data))[i];
1044
+ }
1045
+ case GGML_TYPE_I32:
1046
+ {
1047
+ return ((int32_t *)(tensor->data))[i];
1048
+ }
1049
+ case GGML_TYPE_F16:
1050
+ {
1051
+ return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);
1052
+ }
1053
+ case GGML_TYPE_BF16:
1054
+ {
1055
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);
1056
+ }
1057
+ case GGML_TYPE_F32:
1058
+ {
1059
+ return ((float *)(tensor->data))[i];
1060
+ }
1061
+ default:
1062
+ {
1063
+ GGML_ABORT("fatal error");
1064
+ }
1065
+ }
1066
+ }
1067
+
1068
+ void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value) {
1069
+ if (!ggml_is_contiguous(tensor)) {
1070
+ int64_t id[4] = { 0, 0, 0, 0 };
1071
+ ggml_unravel_index(tensor, i, &id[0], &id[1], &id[2], &id[3]);
1072
+ ggml_set_f32_nd(tensor, id[0], id[1], id[2], id[3], value);
1073
+ return;
1074
+ }
1075
+ switch (tensor->type) {
1076
+ case GGML_TYPE_I8:
1077
+ {
1078
+ ((int8_t *)(tensor->data))[i] = value;
1079
+ } break;
1080
+ case GGML_TYPE_I16:
1081
+ {
1082
+ ((int16_t *)(tensor->data))[i] = value;
1083
+ } break;
1084
+ case GGML_TYPE_I32:
1085
+ {
1086
+ ((int32_t *)(tensor->data))[i] = value;
1087
+ } break;
1088
+ case GGML_TYPE_F16:
1089
+ {
1090
+ ((ggml_fp16_t *)(tensor->data))[i] = GGML_CPU_FP32_TO_FP16(value);
1091
+ } break;
1092
+ case GGML_TYPE_BF16:
1093
+ {
1094
+ ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);
1095
+ } break;
1096
+ case GGML_TYPE_F32:
1097
+ {
1098
+ ((float *)(tensor->data))[i] = value;
1099
+ } break;
1100
+ default:
1101
+ {
1102
+ GGML_ABORT("fatal error");
1103
+ }
1104
+ }
1105
+ }
1106
+
1107
+ float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3) {
1108
+ void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
1109
+ switch (tensor->type) {
1110
+ case GGML_TYPE_I8:
1111
+ return ((int8_t *) data)[0];
1112
+ case GGML_TYPE_I16:
1113
+ return ((int16_t *) data)[0];
1114
+ case GGML_TYPE_I32:
1115
+ return ((int32_t *) data)[0];
1116
+ case GGML_TYPE_F16:
1117
+ return GGML_CPU_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);
1118
+ case GGML_TYPE_BF16:
1119
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);
1120
+ case GGML_TYPE_F32:
1121
+ return ((float *) data)[0];
1122
+ default:
1123
+ GGML_ABORT("fatal error");
1124
+ }
1125
+ }
1126
+
1127
+ void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value) {
1128
+ void * data = (char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3];
1129
+ switch (tensor->type) {
1130
+ case GGML_TYPE_I8:
1131
+ {
1132
+ ((int8_t *)(data))[0] = value;
1133
+ } break;
1134
+ case GGML_TYPE_I16:
1135
+ {
1136
+ ((int16_t *)(data))[0] = value;
1137
+ } break;
1138
+ case GGML_TYPE_I32:
1139
+ {
1140
+ ((int32_t *)(data))[0] = value;
1141
+ } break;
1142
+ case GGML_TYPE_F16:
1143
+ {
1144
+ ((ggml_fp16_t *)(data))[0] = GGML_CPU_FP32_TO_FP16(value);
1145
+ } break;
1146
+ case GGML_TYPE_BF16:
1147
+ {
1148
+ ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);
1149
+ } break;
1150
+ case GGML_TYPE_F32:
1151
+ {
1152
+ ((float *)(data))[0] = value;
1153
+ } break;
1154
+ default:
1155
+ {
1156
+ GGML_ABORT("fatal error");
1157
+ }
1158
+ }
1159
+ }
1160
+
1161
+ ////////////////////////////////////////////////////////////////////////////////
1162
+
1163
+ // ggml_compute_forward_mul_mat
1164
+
1165
+ static void ggml_compute_forward_mul_mat_one_chunk(
1166
+ const struct ggml_compute_params * params,
1167
+ struct ggml_tensor * dst,
1168
+ const enum ggml_type type,
1169
+ const int64_t num_rows_per_vec_dot,
1170
+ const int64_t ir0_start,
1171
+ const int64_t ir0_end,
1172
+ const int64_t ir1_start,
1173
+ const int64_t ir1_end) {
1174
+
1175
+ const struct ggml_tensor * src0 = dst->src[0];
1176
+ const struct ggml_tensor * src1 = dst->src[1];
1177
+
1178
+ GGML_TENSOR_BINARY_OP_LOCALS
1179
+
1180
+ const bool src1_cont = ggml_is_contiguous(src1);
1181
+
1182
+ ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;
1183
+ enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
1184
+
1185
+ // broadcast factors
1186
+ const int64_t r2 = ne12 / ne02;
1187
+ const int64_t r3 = ne13 / ne03;
1188
+
1189
+ //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end);
1190
+
1191
+ // threads with no work simply yield (not sure if it helps)
1192
+ if (ir0_start >= ir0_end || ir1_start >= ir1_end) {
1193
+ return;
1194
+ }
1195
+
1196
+ const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
1197
+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);
1198
+
1199
+ assert(ne12 % ne02 == 0);
1200
+ assert(ne13 % ne03 == 0);
1201
+
1202
+ // block-tiling attempt
1203
+ const int64_t blck_0 = 16;
1204
+ const int64_t blck_1 = 16;
1205
+
1206
+ const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11;
1207
+
1208
+ // attempt to reduce false-sharing (does not seem to make a difference)
1209
+ // 16 * 2, accounting for mmla kernels
1210
+ float tmp[32];
1211
+
1212
+ for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {
1213
+ for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {
1214
+ for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) {
1215
+ const int64_t i13 = (ir1 / (ne12 * ne1));
1216
+ const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1;
1217
+ const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1);
1218
+
1219
+ // broadcast src0 into src1
1220
+ const int64_t i03 = i13 / r3;
1221
+ const int64_t i02 = i12 / r2;
1222
+
1223
+ const int64_t i1 = i11;
1224
+ const int64_t i2 = i12;
1225
+ const int64_t i3 = i13;
1226
+
1227
+ const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03);
1228
+
1229
+ // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides
1230
+ // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using
1231
+ // the original src1 data pointer, so we should index using the indices directly
1232
+ // TODO: this is a bit of a hack, we should probably have a better way to handle this
1233
+ const char * src1_col = (const char*)wdata +
1234
+ (src1_cont || src1->type != vec_dot_type
1235
+ ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size
1236
+ : (i11 * nb11 + i12 * nb12 + i13 * nb13));
1237
+ float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3));
1238
+
1239
+ //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) {
1240
+ // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col);
1241
+ //}
1242
+
1243
+ for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) {
1244
+ vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot);
1245
+ }
1246
+
1247
+ for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) {
1248
+ memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float));
1249
+ }
1250
+ }
1251
+ }
1252
+ }
1253
+ }
1254
+
1255
+ void ggml_compute_forward_mul_mat(
1256
+ const struct ggml_compute_params * params,
1257
+ struct ggml_tensor * dst) {
1258
+
1259
+ const struct ggml_tensor * src0 = dst->src[0];
1260
+ const struct ggml_tensor * src1 = dst->src[1];
1261
+
1262
+ const int32_t hint = ggml_get_op_params_i32(dst, 1);
1263
+ if (hint == GGML_HINT_SRC0_IS_HADAMARD && !params->use_ref) {
1264
+ ggml_compute_forward_fwht(params, dst);
1265
+ return;
1266
+ }
1267
+
1268
+ GGML_TENSOR_BINARY_OP_LOCALS
1269
+
1270
+ const int ith = params->ith;
1271
+ const int nth = params->nth;
1272
+
1273
+ enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;
1274
+ ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
1275
+ int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows;
1276
+
1277
+ GGML_ASSERT(ne0 == ne01);
1278
+ GGML_ASSERT(ne1 == ne11);
1279
+ GGML_ASSERT(ne2 == ne12);
1280
+ GGML_ASSERT(ne3 == ne13);
1281
+
1282
+ // we don't support permuted src0 or src1
1283
+ GGML_ASSERT(nb00 == ggml_type_size(src0->type));
1284
+ GGML_ASSERT(nb10 == ggml_type_size(src1->type));
1285
+
1286
+ // dst cannot be transposed or permuted
1287
+ GGML_ASSERT(nb0 == sizeof(float));
1288
+ GGML_ASSERT(nb0 <= nb1);
1289
+ GGML_ASSERT(nb1 <= nb2);
1290
+ GGML_ASSERT(nb2 <= nb3);
1291
+
1292
+ // nb01 >= nb00 - src0 is not transposed
1293
+ // compute by src0 rows
1294
+
1295
+ // TODO: extract to "extra_op"
1296
+ #if GGML_USE_LLAMAFILE
1297
+ // broadcast factors
1298
+ const int64_t r2 = ne12 / ne02;
1299
+ const int64_t r3 = ne13 / ne03;
1300
+
1301
+ const bool src1_cont = ggml_is_contiguous(src1);
1302
+
1303
+ if (src1_cont) {
1304
+ for (int64_t i13 = 0; i13 < ne13; i13++)
1305
+ for (int64_t i12 = 0; i12 < ne12; i12++)
1306
+ if (!llamafile_sgemm(params,
1307
+ ne01, ne11, ne00/ggml_blck_size(src0->type),
1308
+ (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,
1309
+ nb01/ggml_type_size(src0->type),
1310
+ (const char *)src1->data + i12*nb12 + i13*nb13,
1311
+ nb11/ggml_type_size(src1->type),
1312
+ (char *)dst->data + i12*nb2 + i13*nb3,
1313
+ nb1/ggml_type_size(dst->type),
1314
+ src0->type,
1315
+ src1->type,
1316
+ dst->type))
1317
+ goto UseGgmlGemm1;
1318
+ return;
1319
+ }
1320
+ UseGgmlGemm1:;
1321
+ #endif
1322
+
1323
+ if (src1->type != vec_dot_type) {
1324
+ char * wdata = params->wdata;
1325
+
1326
+ const size_t nbw0 = ggml_type_size(vec_dot_type);
1327
+ const size_t nbw1 = ggml_row_size(vec_dot_type, ne10);
1328
+ const size_t nbw2 = nbw1*ne11;
1329
+ const size_t nbw3 = nbw2*ne12;
1330
+
1331
+ assert(params->wsize >= ne13*nbw3);
1332
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
1333
+
1334
+ #if 0
1335
+ for (int64_t i13 = 0; i13 < ne13; ++i13) {
1336
+ for (int64_t i12 = 0; i12 < ne12; ++i12) {
1337
+ for (int64_t i11 = ith; i11 < ne11; i11 += nth) {
1338
+ from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),
1339
+ (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),
1340
+ ne10);
1341
+ }
1342
+ }
1343
+ }
1344
+ #else
1345
+ for (int64_t i13 = 0; i13 < ne13; ++i13) {
1346
+ for (int64_t i12 = 0; i12 < ne12; ++i12) {
1347
+ for (int64_t i11 = 0; i11 < ne11; ++i11) {
1348
+ size_t bs = ggml_blck_size(vec_dot_type);
1349
+ int64_t ne10_block_start = (ith * ne10/bs) / nth;
1350
+ int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth;
1351
+ from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),
1352
+ (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),
1353
+ (ne10_block_end - ne10_block_start) * bs);
1354
+ }
1355
+ }
1356
+ }
1357
+ #endif
1358
+ }
1359
+
1360
+ if (ith == 0) {
1361
+ // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.
1362
+ atomic_store_explicit(&params->threadpool->current_chunk, nth, memory_order_relaxed);
1363
+ }
1364
+
1365
+ ggml_barrier(params->threadpool);
1366
+
1367
+ // IQ panel gemm (see iqp.h) - must come after the barrier above, it consumes the q8_K rows
1368
+ // of src1 from the work buffer
1369
+ if (ggml_cpu_iqp_supports_mul_mat(dst) && !params->use_ref) {
1370
+ ggml_compute_forward_mul_mat_iqp(params, dst);
1371
+ return;
1372
+ }
1373
+
1374
+ #if GGML_USE_LLAMAFILE
1375
+ if (src1->type != vec_dot_type) {
1376
+ const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
1377
+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);
1378
+
1379
+ for (int64_t i13 = 0; i13 < ne13; i13++)
1380
+ for (int64_t i12 = 0; i12 < ne12; i12++)
1381
+ if (!llamafile_sgemm(params,
1382
+ ne01, ne11, ne00/ggml_blck_size(src0->type),
1383
+ (const char *)src0->data + i12/r2*nb02 + i13/r3*nb03,
1384
+ nb01/ggml_type_size(src0->type),
1385
+ (const char *)wdata + (i12*ne11 + i13*ne12*ne11)*row_size,
1386
+ row_size/ggml_type_size(vec_dot_type),
1387
+ (char *)dst->data + i12*nb2 + i13*nb3,
1388
+ nb1/ggml_type_size(dst->type),
1389
+ src0->type,
1390
+ vec_dot_type,
1391
+ dst->type))
1392
+ goto UseGgmlGemm2;
1393
+ return;
1394
+ }
1395
+ UseGgmlGemm2:;
1396
+ #endif
1397
+
1398
+ // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers)
1399
+ const int64_t nr0 = ne0;
1400
+
1401
+ // This is the size of the rest of the dimensions of the result
1402
+ const int64_t nr1 = ne1 * ne2 * ne3;
1403
+
1404
+ // Now select a reasonable chunk size.
1405
+ int chunk_size = 16;
1406
+
1407
+ // We need to step up the size if it's small
1408
+ if (nr0 == 1 || nr1 == 1) {
1409
+ chunk_size = 64;
1410
+ }
1411
+
1412
+ // distribute the work across the inner or outer loop based on which one is larger
1413
+ // The number of chunks in the 0/1 dim.
1414
+ // CEIL(nr0/chunk_size)
1415
+ int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;
1416
+ int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;
1417
+
1418
+ // If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread.
1419
+ // Also, chunking by thread was measured to have perform better on NUMA systems. See https://github.com/ggml-org/llama.cpp/pull/6915
1420
+ // In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that.
1421
+ if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) {
1422
+ // distribute the thread work across the inner or outer loop based on which one is larger
1423
+ nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows
1424
+ nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows
1425
+ }
1426
+
1427
+ // The number of elements in each chunk
1428
+ const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;
1429
+ const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;
1430
+
1431
+ // The first chunk comes from our thread_id, the rest will get auto-assigned.
1432
+ int current_chunk = ith;
1433
+
1434
+ while (current_chunk < nchunk0 * nchunk1) {
1435
+ const int64_t ith0 = current_chunk % nchunk0;
1436
+ const int64_t ith1 = current_chunk / nchunk0;
1437
+
1438
+ const int64_t ir0_start = dr0 * ith0;
1439
+ const int64_t ir0_end = MIN(ir0_start + dr0, nr0);
1440
+
1441
+ const int64_t ir1_start = dr1 * ith1;
1442
+ const int64_t ir1_end = MIN(ir1_start + dr1, nr1);
1443
+
1444
+ // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols
1445
+ int64_t num_rows_per_vec_dot = vec_dot_num_rows;
1446
+
1447
+ // these checks are needed to avoid crossing dim1 boundaries
1448
+ // can be optimized, but the logic would become more complicated, so keeping it like this for simplicity
1449
+ if ((nr0 % 2 != 0) || (ne11 % 2 != 0) || ((ir0_end - ir0_start) % 2 != 0) || ((ir1_end - ir1_start) % 2 != 0)) {
1450
+ num_rows_per_vec_dot = 1;
1451
+ }
1452
+ ggml_compute_forward_mul_mat_one_chunk(params, dst, src0->type, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end);
1453
+
1454
+ if (nth >= nchunk0 * nchunk1) {
1455
+ break;
1456
+ }
1457
+
1458
+ current_chunk = atomic_fetch_add_explicit(&params->threadpool->current_chunk, 1, memory_order_relaxed);
1459
+ }
1460
+ }
1461
+
1462
+ // ggml_compute_forward_mul_mat_id
1463
+
1464
+ #define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id)*ids->ne[0]*ids->ne[1] + (i1)]
1465
+
1466
+ struct mmid_row_mapping {
1467
+ int32_t i1;
1468
+ int32_t i2;
1469
+ };
1470
+
1471
+ static void ggml_compute_forward_mul_mat_id_one_chunk(
1472
+ struct ggml_tensor * dst,
1473
+ const struct ggml_tensor * src0,
1474
+ const struct ggml_tensor * src1,
1475
+ const struct ggml_tensor * ids,
1476
+ const int64_t cur_a,
1477
+ const int64_t ir0_start,
1478
+ const int64_t ir0_end,
1479
+ const int64_t ir1_start,
1480
+ const int64_t ir1_end,
1481
+ const char * src0_cur,
1482
+ const struct mmid_row_mapping * matrix_rows,
1483
+ const size_t row_size,
1484
+ const bool src1_cont,
1485
+ const void * wdata) {
1486
+
1487
+ GGML_TENSOR_BINARY_OP_LOCALS
1488
+
1489
+ const enum ggml_type type = src0->type;
1490
+
1491
+ ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;
1492
+ enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
1493
+
1494
+ const int64_t blck_0 = 16;
1495
+ const int64_t blck_1 = 16;
1496
+
1497
+ float tmp[16];
1498
+
1499
+ for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {
1500
+ for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {
1501
+ for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ++ir1) {
1502
+ const int64_t _i12 = ir1; // logical row index for this expert
1503
+
1504
+ struct mmid_row_mapping row_mapping = MMID_MATRIX_ROW(cur_a, _i12);
1505
+ const int id = row_mapping.i1; // selected expert index
1506
+
1507
+ const int64_t i11 = id % ne11;
1508
+ const int64_t i12 = row_mapping.i2; // row index in src1
1509
+
1510
+ const int64_t i1 = id; // selected expert index
1511
+ const int64_t i2 = i12; // row
1512
+
1513
+ // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides
1514
+ // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using
1515
+ // the original src1 data pointer, so we should index using the indices directly
1516
+ // TODO: this is a bit of a hack, we should probably have a better way to handle this
1517
+ const char * src1_col = (const char *) wdata +
1518
+ (src1_cont || src1->type != vec_dot_type
1519
+ ? (i11 + i12*ne11)*row_size
1520
+ : (i11*nb11 + i12*nb12));
1521
+
1522
+ float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2));
1523
+
1524
+ for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) {
1525
+ vec_dot(ne00, &tmp[ir0 - iir0], 0, src0_cur + ir0*nb01, 0, src1_col, 0, 1);
1526
+ }
1527
+
1528
+ memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir0_end) - iir0)*sizeof(float));
1529
+ }
1530
+ }
1531
+ }
1532
+ }
1533
+
1534
+ static void * incr_ptr_aligned(void ** p, size_t size, size_t align) {
1535
+
1536
+ void * ptr = *p;
1537
+ ptr = (void *) GGML_PAD((uintptr_t) ptr, align);
1538
+ *p = (void *) ((char *) ptr + size);
1539
+ return ptr;
1540
+ }
1541
+
1542
+ static void ggml_compute_forward_mul_mat_id(
1543
+ const struct ggml_compute_params * params,
1544
+ struct ggml_tensor * dst) {
1545
+
1546
+ const struct ggml_tensor * src0 = dst->src[0];
1547
+ const struct ggml_tensor * src1 = dst->src[1];
1548
+ const struct ggml_tensor * ids = dst->src[2];
1549
+
1550
+ GGML_TENSOR_BINARY_OP_LOCALS
1551
+
1552
+ const int ith = params->ith;
1553
+ const int nth = params->nth;
1554
+
1555
+ const enum ggml_type type = src0->type;
1556
+
1557
+ const bool src1_cont = ggml_is_contiguous(src1);
1558
+
1559
+ enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
1560
+ ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
1561
+
1562
+ // we don't support permuted src0 or src1
1563
+ GGML_ASSERT(nb00 == ggml_type_size(type));
1564
+ GGML_ASSERT(nb10 == ggml_type_size(src1->type));
1565
+
1566
+ // dst cannot be transposed or permuted
1567
+ GGML_ASSERT(nb0 == sizeof(float));
1568
+ GGML_ASSERT(nb0 <= nb1);
1569
+ GGML_ASSERT(nb1 <= nb2);
1570
+ GGML_ASSERT(nb2 <= nb3);
1571
+
1572
+ // row groups
1573
+ const int n_ids = ids->ne[0]; // n_expert_used
1574
+ const int n_as = ne02; // n_expert
1575
+
1576
+ void * wdata_cur = params->wdata;
1577
+
1578
+ if (src1->type != vec_dot_type) {
1579
+ incr_ptr_aligned(&wdata_cur, ggml_row_size(vec_dot_type, ggml_nelements(src1)), sizeof(int64_t));
1580
+ }
1581
+
1582
+ int64_t * matrix_row_counts = // [n_as]
1583
+ incr_ptr_aligned(&wdata_cur, n_as*sizeof(int64_t), sizeof(int64_t));
1584
+
1585
+ struct mmid_row_mapping * matrix_rows = // [n_as][ids->ne[0]*ids->ne[1]]
1586
+ incr_ptr_aligned(&wdata_cur, n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping), sizeof(int64_t));
1587
+
1588
+ char (*atomic_current_chunk)[CACHE_LINE_SIZE] = // [n_as]
1589
+ incr_ptr_aligned(&wdata_cur, CACHE_LINE_SIZE * n_as, CACHE_LINE_SIZE);
1590
+
1591
+ // IQ panel gemm (see iqp.h); per expert eligibility is decided below, but the work buffer is
1592
+ // reserved for the whole node (ggml_graph_plan sizes it without params, use_ref only skips the dispatch)
1593
+ const bool iqp = ggml_cpu_iqp_supports_mul_mat_id(dst) && !params->use_ref;
1594
+
1595
+ char * iqp_panels = NULL;
1596
+
1597
+ if (iqp) {
1598
+ iqp_panels = incr_ptr_aligned(&wdata_cur, nth * ggml_cpu_iqp_scratch_size(dst), 64);
1599
+ }
1600
+
1601
+ GGML_ASSERT(params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata));
1602
+
1603
+ if (src1->type != vec_dot_type) {
1604
+ char * wdata = params->wdata;
1605
+
1606
+ const size_t nbw0 = ggml_type_size(vec_dot_type);
1607
+ const size_t nbw1 = ggml_row_size(vec_dot_type, ne10);
1608
+ const size_t nbw2 = nbw1*ne11;
1609
+ const size_t nbw3 = nbw2*ne12;
1610
+
1611
+ assert(params->wsize >= ne13*nbw3);
1612
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
1613
+
1614
+ #if 0
1615
+ for (int64_t i13 = 0; i13 < ne13; ++i13) {
1616
+ for (int64_t i12 = ith; i12 < ne12; i12 += nth) {
1617
+ for (int64_t i11 = 0; i11 < ne11; ++i11) {
1618
+ from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11),
1619
+ (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1),
1620
+ ne10);
1621
+ }
1622
+ }
1623
+ }
1624
+ #else
1625
+ for (int64_t i13 = 0; i13 < ne13; ++i13) {
1626
+ for (int64_t i12 = 0; i12 < ne12; ++i12) {
1627
+ for (int64_t i11 = 0; i11 < ne11; ++i11) {
1628
+ size_t bs = ggml_blck_size(vec_dot_type);
1629
+ int64_t ne10_block_start = (ith * ne10/bs) / nth;
1630
+ int64_t ne10_block_end = ((ith + 1) * ne10/bs) / nth;
1631
+ from_float((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11 + ne10_block_start*bs*nb10),
1632
+ (void *) (wdata + i13*nbw3 + i12*nbw2 + i11*nbw1 + ne10_block_start*nbw0),
1633
+ (ne10_block_end - ne10_block_start) * bs);
1634
+ }
1635
+ }
1636
+ }
1637
+ #endif
1638
+ }
1639
+
1640
+ if (ith == 0) {
1641
+ // initialize matrix_row_counts
1642
+ memset(matrix_row_counts, 0, n_as*sizeof(int64_t));
1643
+
1644
+ // group rows by src0 matrix
1645
+ for (int64_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
1646
+ for (int id = 0; id < n_ids; ++id) {
1647
+ const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1*ids->nb[1] + id*ids->nb[0]);
1648
+
1649
+ assert(i02 >= 0 && i02 < n_as);
1650
+
1651
+ MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = (struct mmid_row_mapping) {id, iid1};
1652
+ matrix_row_counts[i02] += 1;
1653
+ }
1654
+ }
1655
+ }
1656
+
1657
+ // reset current_chunk
1658
+ for (int cur_a = ith; cur_a < n_as; cur_a += nth) {
1659
+ atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a);
1660
+ *current_chunk_ctr = nth;
1661
+ }
1662
+
1663
+ ggml_barrier(params->threadpool);
1664
+
1665
+ for (int cur_a = 0; cur_a < n_as; ++cur_a) {
1666
+ const int64_t cne1 = matrix_row_counts[cur_a];
1667
+
1668
+ if (cne1 == 0) {
1669
+ continue;
1670
+ }
1671
+
1672
+ if (iqp && ggml_cpu_iqp_mul_mat_id_min_batch(cne1)) {
1673
+ ggml_compute_forward_mul_mat_id_iqp(params, dst, cur_a, cne1, (const int32_t *) &MMID_MATRIX_ROW(cur_a, 0),
1674
+ iqp_panels);
1675
+
1676
+ continue;
1677
+ }
1678
+
1679
+ const char * src0_cur = (const char *) src0->data + cur_a * nb02;
1680
+ const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
1681
+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);
1682
+
1683
+ const int64_t nr0 = ne01;
1684
+ const int64_t nr1 = cne1;
1685
+
1686
+ int chunk_size = 16;
1687
+ if (nr0 == 1 || nr1 == 1) {
1688
+ chunk_size = 64;
1689
+ }
1690
+
1691
+ // disable for NUMA
1692
+ const bool disable_chunking = ggml_is_numa();
1693
+
1694
+ int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size;
1695
+ int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size;
1696
+
1697
+ if (nchunk0 * nchunk1 < nth * 4 || disable_chunking) {
1698
+ nchunk0 = nr0 > nr1 ? nth : 1;
1699
+ nchunk1 = nr0 > nr1 ? 1 : nth;
1700
+ }
1701
+
1702
+ const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0;
1703
+ const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1;
1704
+
1705
+ int current_chunk = ith;
1706
+
1707
+ atomic_int * current_chunk_ctr = (atomic_int *)(atomic_current_chunk + cur_a);
1708
+
1709
+ while (current_chunk < nchunk0 * nchunk1) {
1710
+ const int64_t ith0 = current_chunk % nchunk0;
1711
+ const int64_t ith1 = current_chunk / nchunk0;
1712
+
1713
+ const int64_t ir0_start = dr0 * ith0;
1714
+ const int64_t ir0_end = MIN(ir0_start + dr0, nr0);
1715
+
1716
+ const int64_t ir1_start = dr1 * ith1;
1717
+ const int64_t ir1_end = MIN(ir1_start + dr1, nr1);
1718
+
1719
+ ggml_compute_forward_mul_mat_id_one_chunk(
1720
+ dst, src0, src1, ids, cur_a,
1721
+ ir0_start, ir0_end, ir1_start, ir1_end,
1722
+ src0_cur, matrix_rows, row_size, src1_cont, wdata
1723
+ );
1724
+
1725
+ if (nth >= nchunk0 * nchunk1) {
1726
+ break;
1727
+ }
1728
+
1729
+ current_chunk = atomic_fetch_add_explicit(current_chunk_ctr, 1, memory_order_relaxed);
1730
+ }
1731
+ }
1732
+ }
1733
+
1734
+ /////////////////////////////////
1735
+
1736
+ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) {
1737
+ GGML_ASSERT(params);
1738
+
1739
+ if (tensor->op == GGML_OP_NONE || ggml_is_empty(tensor)) {
1740
+ return;
1741
+ }
1742
+
1743
+ // extra_buffer op?
1744
+ if (ggml_cpu_extra_compute_forward(params, tensor)) {
1745
+ return;
1746
+ }
1747
+
1748
+ switch (tensor->op) {
1749
+ case GGML_OP_DUP:
1750
+ {
1751
+ ggml_compute_forward_dup(params, tensor);
1752
+ } break;
1753
+ case GGML_OP_ADD:
1754
+ {
1755
+ ggml_compute_forward_add(params, tensor);
1756
+ } break;
1757
+ case GGML_OP_ADD_ID:
1758
+ {
1759
+ ggml_compute_forward_add_id(params, tensor);
1760
+ } break;
1761
+ case GGML_OP_ADD1:
1762
+ {
1763
+ ggml_compute_forward_add1(params, tensor);
1764
+ } break;
1765
+ case GGML_OP_ACC:
1766
+ {
1767
+ ggml_compute_forward_acc(params, tensor);
1768
+ } break;
1769
+ case GGML_OP_SUB:
1770
+ {
1771
+ ggml_compute_forward_sub(params, tensor);
1772
+ } break;
1773
+ case GGML_OP_MUL:
1774
+ {
1775
+ ggml_compute_forward_mul(params, tensor);
1776
+ } break;
1777
+ case GGML_OP_DIV:
1778
+ {
1779
+ ggml_compute_forward_div(params, tensor);
1780
+ } break;
1781
+ case GGML_OP_SQR:
1782
+ {
1783
+ ggml_compute_forward_sqr(params, tensor);
1784
+ } break;
1785
+ case GGML_OP_SQRT:
1786
+ {
1787
+ ggml_compute_forward_sqrt(params, tensor);
1788
+ } break;
1789
+ case GGML_OP_LOG:
1790
+ {
1791
+ ggml_compute_forward_log(params, tensor);
1792
+ } break;
1793
+ case GGML_OP_SIN:
1794
+ {
1795
+ ggml_compute_forward_sin(params, tensor);
1796
+ } break;
1797
+ case GGML_OP_COS:
1798
+ {
1799
+ ggml_compute_forward_cos(params, tensor);
1800
+ } break;
1801
+ case GGML_OP_SUM:
1802
+ {
1803
+ ggml_compute_forward_sum(params, tensor);
1804
+ } break;
1805
+ case GGML_OP_SUM_ROWS:
1806
+ {
1807
+ ggml_compute_forward_sum_rows(params, tensor);
1808
+ } break;
1809
+ case GGML_OP_CUMSUM:
1810
+ {
1811
+ ggml_compute_forward_cumsum(params, tensor);
1812
+ } break;
1813
+ case GGML_OP_MEAN:
1814
+ {
1815
+ ggml_compute_forward_mean(params, tensor);
1816
+ } break;
1817
+ case GGML_OP_ARGMAX:
1818
+ {
1819
+ ggml_compute_forward_argmax(params, tensor);
1820
+ } break;
1821
+ case GGML_OP_COUNT_EQUAL:
1822
+ {
1823
+ ggml_compute_forward_count_equal(params, tensor);
1824
+ } break;
1825
+ case GGML_OP_REPEAT:
1826
+ {
1827
+ ggml_compute_forward_repeat(params, tensor);
1828
+ } break;
1829
+ case GGML_OP_REPEAT_BACK:
1830
+ {
1831
+ ggml_compute_forward_repeat_back(params, tensor);
1832
+ } break;
1833
+ case GGML_OP_CONCAT:
1834
+ {
1835
+ ggml_compute_forward_concat(params, tensor);
1836
+ } break;
1837
+ case GGML_OP_SILU_BACK:
1838
+ {
1839
+ ggml_compute_forward_silu_back(params, tensor);
1840
+ } break;
1841
+ case GGML_OP_NORM:
1842
+ {
1843
+ ggml_compute_forward_norm(params, tensor);
1844
+ } break;
1845
+ case GGML_OP_RMS_NORM:
1846
+ {
1847
+ ggml_compute_forward_rms_norm(params, tensor);
1848
+ } break;
1849
+ case GGML_OP_RMS_NORM_BACK:
1850
+ {
1851
+ ggml_compute_forward_rms_norm_back(params, tensor);
1852
+ } break;
1853
+ case GGML_OP_GROUP_NORM:
1854
+ {
1855
+ ggml_compute_forward_group_norm(params, tensor);
1856
+ } break;
1857
+ case GGML_OP_L2_NORM:
1858
+ {
1859
+ ggml_compute_forward_l2_norm(params, tensor);
1860
+ } break;
1861
+ case GGML_OP_MUL_MAT:
1862
+ {
1863
+ ggml_compute_forward_mul_mat(params, tensor);
1864
+ } break;
1865
+ case GGML_OP_MUL_MAT_ID:
1866
+ {
1867
+ ggml_compute_forward_mul_mat_id(params, tensor);
1868
+ } break;
1869
+ case GGML_OP_OUT_PROD:
1870
+ {
1871
+ ggml_compute_forward_out_prod(params, tensor);
1872
+ } break;
1873
+ case GGML_OP_SCALE:
1874
+ {
1875
+ ggml_compute_forward_scale(params, tensor);
1876
+ } break;
1877
+ case GGML_OP_SET:
1878
+ {
1879
+ ggml_compute_forward_set(params, tensor);
1880
+ } break;
1881
+ case GGML_OP_CPY:
1882
+ {
1883
+ ggml_compute_forward_cpy(params, tensor);
1884
+ } break;
1885
+ case GGML_OP_CONT:
1886
+ {
1887
+ ggml_compute_forward_cont(params, tensor);
1888
+ } break;
1889
+ case GGML_OP_GET_ROWS:
1890
+ {
1891
+ ggml_compute_forward_get_rows(params, tensor);
1892
+ } break;
1893
+ case GGML_OP_GET_ROWS_BACK:
1894
+ {
1895
+ ggml_compute_forward_get_rows_back(params, tensor);
1896
+ } break;
1897
+ case GGML_OP_SET_ROWS:
1898
+ {
1899
+ ggml_compute_forward_set_rows(params, tensor);
1900
+ } break;
1901
+ case GGML_OP_DIAG:
1902
+ {
1903
+ ggml_compute_forward_diag(params, tensor);
1904
+ } break;
1905
+ case GGML_OP_DIAG_MASK_INF:
1906
+ {
1907
+ ggml_compute_forward_diag_mask_inf(params, tensor);
1908
+ } break;
1909
+ case GGML_OP_DIAG_MASK_ZERO:
1910
+ {
1911
+ ggml_compute_forward_diag_mask_zero(params, tensor);
1912
+ } break;
1913
+ case GGML_OP_SOFT_MAX:
1914
+ {
1915
+ ggml_compute_forward_soft_max(params, tensor);
1916
+ } break;
1917
+ case GGML_OP_SOFT_MAX_BACK:
1918
+ {
1919
+ ggml_compute_forward_soft_max_ext_back(params, tensor);
1920
+ } break;
1921
+ case GGML_OP_ROPE:
1922
+ {
1923
+ ggml_compute_forward_rope(params, tensor);
1924
+ } break;
1925
+ case GGML_OP_ROPE_BACK:
1926
+ {
1927
+ ggml_compute_forward_rope_back(params, tensor);
1928
+ } break;
1929
+ case GGML_OP_CLAMP:
1930
+ {
1931
+ ggml_compute_forward_clamp(params, tensor);
1932
+ } break;
1933
+ case GGML_OP_CONV_TRANSPOSE_1D:
1934
+ {
1935
+ ggml_compute_forward_conv_transpose_1d(params, tensor);
1936
+ } break;
1937
+ case GGML_OP_IM2COL:
1938
+ {
1939
+ ggml_compute_forward_im2col(params, tensor);
1940
+ } break;
1941
+ case GGML_OP_IM2COL_BACK:
1942
+ {
1943
+ ggml_compute_forward_im2col_back_f32(params, tensor);
1944
+ } break;
1945
+ case GGML_OP_IM2COL_3D:
1946
+ {
1947
+ ggml_compute_forward_im2col_3d(params, tensor);
1948
+ } break;
1949
+ case GGML_OP_COL2IM_1D:
1950
+ {
1951
+ ggml_compute_forward_col2im_1d(params, tensor);
1952
+ } break;
1953
+ case GGML_OP_CONV_2D:
1954
+ {
1955
+ ggml_compute_forward_conv_2d(params, tensor);
1956
+ } break;
1957
+ case GGML_OP_CONV_3D:
1958
+ {
1959
+ ggml_compute_forward_conv_3d(params, tensor);
1960
+ } break;
1961
+ case GGML_OP_CONV_2D_DW:
1962
+ {
1963
+ ggml_compute_forward_conv_2d_dw(params, tensor);
1964
+ } break;
1965
+ case GGML_OP_CONV_TRANSPOSE_2D:
1966
+ {
1967
+ ggml_compute_forward_conv_transpose_2d(params, tensor);
1968
+ } break;
1969
+ case GGML_OP_POOL_1D:
1970
+ {
1971
+ ggml_compute_forward_pool_1d(params, tensor);
1972
+ } break;
1973
+ case GGML_OP_POOL_2D:
1974
+ {
1975
+ ggml_compute_forward_pool_2d(params, tensor);
1976
+ } break;
1977
+ case GGML_OP_POOL_2D_BACK:
1978
+ {
1979
+ ggml_compute_forward_pool_2d_back(params, tensor);
1980
+ } break;
1981
+ case GGML_OP_UPSCALE:
1982
+ {
1983
+ ggml_compute_forward_upscale(params, tensor);
1984
+ } break;
1985
+ case GGML_OP_PAD:
1986
+ {
1987
+ ggml_compute_forward_pad(params, tensor);
1988
+ } break;
1989
+ case GGML_OP_PAD_REFLECT_1D:
1990
+ {
1991
+ ggml_compute_forward_pad_reflect_1d(params, tensor);
1992
+ } break;
1993
+ case GGML_OP_ROLL:
1994
+ {
1995
+ ggml_compute_forward_roll(params, tensor);
1996
+ } break;
1997
+ case GGML_OP_ARANGE:
1998
+ {
1999
+ ggml_compute_forward_arange(params, tensor);
2000
+ } break;
2001
+ case GGML_OP_TIMESTEP_EMBEDDING:
2002
+ {
2003
+ ggml_compute_forward_timestep_embedding(params, tensor);
2004
+ } break;
2005
+ case GGML_OP_ARGSORT:
2006
+ {
2007
+ ggml_compute_forward_argsort(params, tensor);
2008
+ } break;
2009
+ case GGML_OP_TOP_K:
2010
+ {
2011
+ ggml_compute_forward_top_k(params, tensor);
2012
+ } break;
2013
+ case GGML_OP_LEAKY_RELU:
2014
+ {
2015
+ ggml_compute_forward_leaky_relu(params, tensor);
2016
+ } break;
2017
+ case GGML_OP_TRI:
2018
+ {
2019
+ ggml_compute_forward_tri(params, tensor);
2020
+ } break;
2021
+ case GGML_OP_FILL:
2022
+ {
2023
+ ggml_compute_forward_fill(params, tensor);
2024
+ } break;
2025
+ case GGML_OP_FLASH_ATTN_EXT:
2026
+ {
2027
+ ggml_compute_forward_flash_attn_ext(params, tensor);
2028
+ } break;
2029
+ case GGML_OP_FLASH_ATTN_BACK:
2030
+ {
2031
+ int32_t t = ggml_get_op_params_i32(tensor, 0);
2032
+ GGML_ASSERT(t == 0 || t == 1);
2033
+ bool masked = t != 0;
2034
+ ggml_compute_forward_flash_attn_back(params, masked, tensor);
2035
+ } break;
2036
+ case GGML_OP_SSM_CONV:
2037
+ {
2038
+ ggml_compute_forward_ssm_conv(params, tensor);
2039
+ } break;
2040
+ case GGML_OP_SSM_SCAN:
2041
+ {
2042
+ ggml_compute_forward_ssm_scan(params, tensor);
2043
+ } break;
2044
+ case GGML_OP_WIN_PART:
2045
+ {
2046
+ ggml_compute_forward_win_part(params, tensor);
2047
+ } break;
2048
+ case GGML_OP_WIN_UNPART:
2049
+ {
2050
+ ggml_compute_forward_win_unpart(params, tensor);
2051
+ } break;
2052
+ case GGML_OP_UNARY:
2053
+ {
2054
+ ggml_compute_forward_unary(params, tensor);
2055
+ } break;
2056
+ case GGML_OP_GLU:
2057
+ {
2058
+ ggml_compute_forward_glu(params, tensor);
2059
+ } break;
2060
+ case GGML_OP_GET_REL_POS:
2061
+ {
2062
+ ggml_compute_forward_get_rel_pos(params, tensor);
2063
+ } break;
2064
+ case GGML_OP_ADD_REL_POS:
2065
+ {
2066
+ ggml_compute_forward_add_rel_pos(params, tensor);
2067
+ } break;
2068
+ case GGML_OP_RWKV_WKV6:
2069
+ {
2070
+ ggml_compute_forward_rwkv_wkv6(params, tensor);
2071
+ } break;
2072
+ case GGML_OP_GATED_LINEAR_ATTN:
2073
+ {
2074
+ ggml_compute_forward_gla(params, tensor);
2075
+ } break;
2076
+ case GGML_OP_RWKV_WKV7:
2077
+ {
2078
+ ggml_compute_forward_rwkv_wkv7(params, tensor);
2079
+ } break;
2080
+ case GGML_OP_SOLVE_TRI:
2081
+ {
2082
+ ggml_compute_forward_solve_tri(params, tensor);
2083
+ } break;
2084
+ case GGML_OP_GATED_DELTA_NET:
2085
+ {
2086
+ ggml_compute_forward_gated_delta_net(params, tensor);
2087
+ } break;
2088
+ case GGML_OP_LIGHTNING_INDEXER:
2089
+ {
2090
+ ggml_compute_forward_lightning_indexer(params, tensor);
2091
+ } break;
2092
+ case GGML_OP_DSV4_HC_COMB:
2093
+ {
2094
+ ggml_compute_forward_dsv4_hc_comb(params, tensor);
2095
+ } break;
2096
+ case GGML_OP_DSV4_HC_PRE:
2097
+ {
2098
+ ggml_compute_forward_dsv4_hc_pre(params, tensor);
2099
+ } break;
2100
+ case GGML_OP_DSV4_HC_POST:
2101
+ {
2102
+ ggml_compute_forward_dsv4_hc_post(params, tensor);
2103
+ } break;
2104
+ case GGML_OP_MAP_CUSTOM1:
2105
+ {
2106
+ ggml_compute_forward_map_custom1(params, tensor);
2107
+ }
2108
+ break;
2109
+ case GGML_OP_MAP_CUSTOM2:
2110
+ {
2111
+ ggml_compute_forward_map_custom2(params, tensor);
2112
+ }
2113
+ break;
2114
+ case GGML_OP_MAP_CUSTOM3:
2115
+ {
2116
+ ggml_compute_forward_map_custom3(params, tensor);
2117
+ }
2118
+ break;
2119
+ case GGML_OP_CUSTOM:
2120
+ {
2121
+ ggml_compute_forward_custom(params, tensor);
2122
+ }
2123
+ break;
2124
+ case GGML_OP_CROSS_ENTROPY_LOSS:
2125
+ {
2126
+ ggml_compute_forward_cross_entropy_loss(params, tensor);
2127
+ }
2128
+ break;
2129
+ case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
2130
+ {
2131
+ ggml_compute_forward_cross_entropy_loss_back(params, tensor);
2132
+ }
2133
+ break;
2134
+ case GGML_OP_OPT_STEP_ADAMW:
2135
+ {
2136
+ ggml_compute_forward_opt_step_adamw(params, tensor);
2137
+ }
2138
+ break;
2139
+ case GGML_OP_OPT_STEP_SGD:
2140
+ {
2141
+ ggml_compute_forward_opt_step_sgd(params, tensor);
2142
+ }
2143
+ break;
2144
+ case GGML_OP_NONE:
2145
+ {
2146
+ // nop
2147
+ } break;
2148
+ case GGML_OP_RESHAPE:
2149
+ {
2150
+ // nop
2151
+ } break;
2152
+ case GGML_OP_PERMUTE:
2153
+ {
2154
+ // nop
2155
+ } break;
2156
+ case GGML_OP_VIEW:
2157
+ {
2158
+ // nop
2159
+ } break;
2160
+ case GGML_OP_TRANSPOSE:
2161
+ {
2162
+ // nop
2163
+ } break;
2164
+ case GGML_OP_COUNT:
2165
+ {
2166
+ GGML_ABORT("fatal error");
2167
+ }
2168
+ }
2169
+ }
2170
+
2171
+ // Android's libc implementation "bionic" does not support setting affinity
2172
+ #if defined(__gnu_linux__)
2173
+ static void set_numa_thread_affinity(int thread_n) {
2174
+ if (!ggml_is_numa()) {
2175
+ return;
2176
+ }
2177
+
2178
+ int node_num;
2179
+ int rv;
2180
+ size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus);
2181
+
2182
+ switch(g_state.numa.numa_strategy) {
2183
+ case GGML_NUMA_STRATEGY_DISTRIBUTE:
2184
+ // run thread on node_num thread_n / (threads per node)
2185
+ node_num = thread_n % g_state.numa.n_nodes;
2186
+ break;
2187
+ case GGML_NUMA_STRATEGY_ISOLATE:
2188
+ // run thread on current_node
2189
+ node_num = g_state.numa.current_node;
2190
+ break;
2191
+ case GGML_NUMA_STRATEGY_NUMACTL:
2192
+ // use the cpuset that numactl gave us
2193
+ rv = pthread_setaffinity_np(pthread_self(), setsize, &g_state.numa.cpuset);
2194
+ if (rv) {
2195
+ fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n",strerror(rv));
2196
+ }
2197
+ return;
2198
+ default:
2199
+ return;
2200
+ }
2201
+
2202
+ struct ggml_numa_node * node = &g_state.numa.nodes[node_num];
2203
+
2204
+ cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus);
2205
+ CPU_ZERO_S(setsize, cpus);
2206
+ for (size_t i = 0; i < node->n_cpus; ++i) {
2207
+ CPU_SET_S(node->cpus[i], setsize, cpus);
2208
+ }
2209
+
2210
+ rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);
2211
+ if (rv) {
2212
+ fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv));
2213
+ }
2214
+
2215
+ CPU_FREE(cpus);
2216
+ }
2217
+
2218
+ static void clear_numa_thread_affinity(void) {
2219
+ if (!ggml_is_numa()) {
2220
+ return;
2221
+ }
2222
+
2223
+ size_t setsize = CPU_ALLOC_SIZE(g_state.numa.total_cpus);
2224
+
2225
+ cpu_set_t * cpus = CPU_ALLOC(g_state.numa.total_cpus);
2226
+ CPU_ZERO_S(setsize, cpus);
2227
+ for (unsigned i = 0; i < g_state.numa.total_cpus; ++i) {
2228
+ CPU_SET_S(i, setsize, cpus);
2229
+ }
2230
+
2231
+ int rv = pthread_setaffinity_np(pthread_self(), setsize, cpus);
2232
+ if (rv) {
2233
+ fprintf(stderr, "warning: pthread_setaffinity_np() failed: %s\n", strerror(rv));
2234
+ }
2235
+
2236
+ CPU_FREE(cpus);
2237
+ }
2238
+ #else
2239
+ // TODO: Windows etc.
2240
+ // (the linux implementation may also work on BSD, someone should test)
2241
+ static void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n); }
2242
+ static void clear_numa_thread_affinity(void) {}
2243
+ #endif
2244
+
2245
+ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
2246
+ int n_tasks = 0;
2247
+
2248
+ if (ggml_is_empty(node)) {
2249
+ // no need to multi-thread a no-op
2250
+ n_tasks = 1;
2251
+ return n_tasks;
2252
+ }
2253
+
2254
+ switch (node->op) {
2255
+ case GGML_OP_CPY:
2256
+ case GGML_OP_DUP:
2257
+ case GGML_OP_CONT:
2258
+ case GGML_OP_ADD:
2259
+ case GGML_OP_ADD_ID:
2260
+ case GGML_OP_ADD1:
2261
+ case GGML_OP_ACC:
2262
+ case GGML_OP_CUMSUM:
2263
+ case GGML_OP_TRI:
2264
+ case GGML_OP_FILL:
2265
+ {
2266
+ n_tasks = n_threads;
2267
+ } break;
2268
+ case GGML_OP_SUB:
2269
+ case GGML_OP_SQR:
2270
+ case GGML_OP_SQRT:
2271
+ case GGML_OP_LOG:
2272
+ case GGML_OP_SIN:
2273
+ case GGML_OP_COS:
2274
+ case GGML_OP_SUM:
2275
+ case GGML_OP_SUM_ROWS:
2276
+ case GGML_OP_MEAN:
2277
+ case GGML_OP_ARGMAX:
2278
+ {
2279
+ n_tasks = 1;
2280
+ } break;
2281
+ case GGML_OP_COUNT_EQUAL:
2282
+ case GGML_OP_SOLVE_TRI:
2283
+ case GGML_OP_GATED_DELTA_NET:
2284
+ case GGML_OP_DSV4_HC_COMB:
2285
+ case GGML_OP_DSV4_HC_PRE:
2286
+ case GGML_OP_DSV4_HC_POST:
2287
+ {
2288
+ n_tasks = n_threads;
2289
+ } break;
2290
+ case GGML_OP_REPEAT:
2291
+ case GGML_OP_REPEAT_BACK:
2292
+ case GGML_OP_LEAKY_RELU:
2293
+ {
2294
+ n_tasks = 1;
2295
+ } break;
2296
+ case GGML_OP_UNARY:
2297
+ switch (ggml_get_unary_op(node)) {
2298
+ case GGML_UNARY_OP_ABS:
2299
+ case GGML_UNARY_OP_SGN:
2300
+ case GGML_UNARY_OP_NEG:
2301
+ case GGML_UNARY_OP_STEP:
2302
+ case GGML_UNARY_OP_TANH:
2303
+ case GGML_UNARY_OP_ELU:
2304
+ case GGML_UNARY_OP_RELU:
2305
+ case GGML_UNARY_OP_SIGMOID:
2306
+ case GGML_UNARY_OP_HARDSWISH:
2307
+ case GGML_UNARY_OP_HARDSIGMOID:
2308
+ case GGML_UNARY_OP_EXP:
2309
+ case GGML_UNARY_OP_SOFTPLUS:
2310
+ case GGML_UNARY_OP_EXPM1:
2311
+ case GGML_UNARY_OP_FLOOR:
2312
+ case GGML_UNARY_OP_CEIL:
2313
+ case GGML_UNARY_OP_ROUND:
2314
+ case GGML_UNARY_OP_TRUNC:
2315
+ {
2316
+ n_tasks = 1;
2317
+ } break;
2318
+
2319
+ case GGML_UNARY_OP_GELU:
2320
+ case GGML_UNARY_OP_GELU_ERF:
2321
+ case GGML_UNARY_OP_GELU_QUICK:
2322
+ case GGML_UNARY_OP_SILU:
2323
+ case GGML_UNARY_OP_XIELU:
2324
+ {
2325
+ n_tasks = n_threads;
2326
+ } break;
2327
+ default:
2328
+ GGML_ABORT("fatal error");
2329
+ }
2330
+ break;
2331
+ case GGML_OP_GLU:
2332
+ switch (ggml_get_glu_op(node)) {
2333
+ case GGML_GLU_OP_REGLU:
2334
+ case GGML_GLU_OP_GEGLU:
2335
+ case GGML_GLU_OP_SWIGLU:
2336
+ case GGML_GLU_OP_SWIGLU_OAI:
2337
+ case GGML_GLU_OP_GEGLU_ERF:
2338
+ case GGML_GLU_OP_GEGLU_QUICK:
2339
+ case GGML_GLU_OP_SWIGLU_CLAMP:
2340
+ {
2341
+ n_tasks = n_threads;
2342
+ } break;
2343
+ default:
2344
+ GGML_ABORT("fatal error");
2345
+ }
2346
+ break;
2347
+ case GGML_OP_SILU_BACK:
2348
+ case GGML_OP_MUL:
2349
+ case GGML_OP_DIV:
2350
+ case GGML_OP_NORM:
2351
+ case GGML_OP_RMS_NORM:
2352
+ case GGML_OP_RMS_NORM_BACK:
2353
+ case GGML_OP_L2_NORM:
2354
+ case GGML_OP_GROUP_NORM:
2355
+ case GGML_OP_CONCAT:
2356
+ case GGML_OP_MUL_MAT:
2357
+ case GGML_OP_MUL_MAT_ID:
2358
+ case GGML_OP_OUT_PROD:
2359
+ {
2360
+ n_tasks = n_threads;
2361
+ } break;
2362
+ case GGML_OP_GET_ROWS:
2363
+ case GGML_OP_SET_ROWS:
2364
+ {
2365
+ // FIXME: get_rows can use additional threads, but the cost of launching additional threads
2366
+ // decreases performance with GPU offloading
2367
+ //n_tasks = n_threads;
2368
+ n_tasks = 1;
2369
+ } break;
2370
+ case GGML_OP_SCALE:
2371
+ case GGML_OP_SET:
2372
+ case GGML_OP_RESHAPE:
2373
+ case GGML_OP_VIEW:
2374
+ case GGML_OP_PERMUTE:
2375
+ case GGML_OP_TRANSPOSE:
2376
+ case GGML_OP_GET_ROWS_BACK:
2377
+ case GGML_OP_DIAG:
2378
+ {
2379
+ n_tasks = 1;
2380
+ } break;
2381
+ case GGML_OP_DIAG_MASK_ZERO:
2382
+ case GGML_OP_DIAG_MASK_INF:
2383
+ case GGML_OP_SOFT_MAX_BACK:
2384
+ case GGML_OP_ROPE:
2385
+ case GGML_OP_ROPE_BACK:
2386
+ case GGML_OP_ADD_REL_POS:
2387
+ {
2388
+ n_tasks = n_threads;
2389
+ } break;
2390
+ case GGML_OP_CLAMP:
2391
+ {
2392
+ n_tasks = 1; //TODO
2393
+ } break;
2394
+ case GGML_OP_SOFT_MAX:
2395
+ {
2396
+ n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));
2397
+ } break;
2398
+ case GGML_OP_IM2COL:
2399
+ case GGML_OP_IM2COL_BACK:
2400
+ case GGML_OP_IM2COL_3D:
2401
+ case GGML_OP_CONV_2D:
2402
+ case GGML_OP_CONV_3D:
2403
+ case GGML_OP_CONV_2D_DW:
2404
+ case GGML_OP_COL2IM_1D:
2405
+ case GGML_OP_CONV_TRANSPOSE_1D:
2406
+ case GGML_OP_CONV_TRANSPOSE_2D:
2407
+ {
2408
+ n_tasks = n_threads;
2409
+ } break;
2410
+ case GGML_OP_POOL_1D:
2411
+ case GGML_OP_POOL_2D:
2412
+ case GGML_OP_POOL_2D_BACK:
2413
+ {
2414
+ n_tasks = 1;
2415
+ } break;
2416
+ case GGML_OP_UPSCALE:
2417
+ case GGML_OP_PAD:
2418
+ case GGML_OP_PAD_REFLECT_1D:
2419
+ case GGML_OP_ROLL:
2420
+ case GGML_OP_ARANGE:
2421
+ case GGML_OP_TIMESTEP_EMBEDDING:
2422
+ case GGML_OP_ARGSORT:
2423
+ case GGML_OP_TOP_K:
2424
+ case GGML_OP_FLASH_ATTN_EXT:
2425
+ case GGML_OP_FLASH_ATTN_BACK:
2426
+ case GGML_OP_SSM_CONV:
2427
+ case GGML_OP_SSM_SCAN:
2428
+ case GGML_OP_LIGHTNING_INDEXER:
2429
+ {
2430
+ n_tasks = n_threads;
2431
+ } break;
2432
+ case GGML_OP_RWKV_WKV6:
2433
+ case GGML_OP_GATED_LINEAR_ATTN:
2434
+ case GGML_OP_RWKV_WKV7:
2435
+ {
2436
+ const int64_t n_heads = node->src[1]->ne[1];
2437
+ n_tasks = MIN(n_threads, n_heads);
2438
+ } break;
2439
+ case GGML_OP_WIN_PART:
2440
+ case GGML_OP_WIN_UNPART:
2441
+ case GGML_OP_GET_REL_POS:
2442
+ {
2443
+ n_tasks = 1;
2444
+ } break;
2445
+ case GGML_OP_MAP_CUSTOM1:
2446
+ {
2447
+ struct ggml_map_custom1_op_params p;
2448
+ memcpy(&p, node->op_params, sizeof(p));
2449
+ if (p.n_tasks == GGML_N_TASKS_MAX) {
2450
+ n_tasks = n_threads;
2451
+ } else {
2452
+ n_tasks = MIN(p.n_tasks, n_threads);
2453
+ }
2454
+ } break;
2455
+ case GGML_OP_MAP_CUSTOM2:
2456
+ {
2457
+ struct ggml_map_custom2_op_params p;
2458
+ memcpy(&p, node->op_params, sizeof(p));
2459
+ if (p.n_tasks == GGML_N_TASKS_MAX) {
2460
+ n_tasks = n_threads;
2461
+ } else {
2462
+ n_tasks = MIN(p.n_tasks, n_threads);
2463
+ }
2464
+ } break;
2465
+ case GGML_OP_MAP_CUSTOM3:
2466
+ {
2467
+ struct ggml_map_custom3_op_params p;
2468
+ memcpy(&p, node->op_params, sizeof(p));
2469
+ if (p.n_tasks == GGML_N_TASKS_MAX) {
2470
+ n_tasks = n_threads;
2471
+ } else {
2472
+ n_tasks = MIN(p.n_tasks, n_threads);
2473
+ }
2474
+ } break;
2475
+ case GGML_OP_CUSTOM:
2476
+ {
2477
+ struct ggml_custom_op_params p;
2478
+ memcpy(&p, node->op_params, sizeof(p));
2479
+ if (p.n_tasks == GGML_N_TASKS_MAX) {
2480
+ n_tasks = n_threads;
2481
+ } else {
2482
+ n_tasks = MIN(p.n_tasks, n_threads);
2483
+ }
2484
+ } break;
2485
+ case GGML_OP_CROSS_ENTROPY_LOSS:
2486
+ case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
2487
+ case GGML_OP_OPT_STEP_ADAMW:
2488
+ case GGML_OP_OPT_STEP_SGD:
2489
+ {
2490
+ n_tasks = n_threads;
2491
+ } break;
2492
+ case GGML_OP_NONE:
2493
+ {
2494
+ n_tasks = 1;
2495
+ } break;
2496
+ case GGML_OP_COUNT:
2497
+ {
2498
+ GGML_ABORT("fatal error");
2499
+ }
2500
+ default:
2501
+ {
2502
+ fprintf(stderr, "%s: op not implemented: ", __func__);
2503
+ if (node->op < GGML_OP_COUNT) {
2504
+ fprintf(stderr, "%s\n", ggml_op_name(node->op));
2505
+ } else {
2506
+ fprintf(stderr, "%d\n", node->op);
2507
+ }
2508
+ GGML_ABORT("fatal error");
2509
+ }
2510
+ }
2511
+
2512
+ assert(n_tasks > 0);
2513
+
2514
+ return n_tasks;
2515
+ }
2516
+
2517
+ static thread_ret_t ggml_graph_compute_secondary_thread(void* data);
2518
+
2519
+ #if defined(_WIN32)
2520
+ #include "windows.h"
2521
+
2522
+ // TODO: support > 64 CPUs
2523
+ static bool ggml_thread_apply_affinity(bool * mask) {
2524
+ HANDLE h = GetCurrentThread();
2525
+ uint64_t bitmask = 0ULL;
2526
+
2527
+ assert(GGML_MAX_N_THREADS >= 64);
2528
+
2529
+ for (int32_t i = 0; i < 8; i++) {
2530
+ int32_t idx = i * 8;
2531
+ uint8_t val = 0;
2532
+ val |= mask[idx + 0] << 0;
2533
+ val |= mask[idx + 1] << 1;
2534
+ val |= mask[idx + 2] << 2;
2535
+ val |= mask[idx + 3] << 3;
2536
+ val |= mask[idx + 4] << 4;
2537
+ val |= mask[idx + 5] << 5;
2538
+ val |= mask[idx + 6] << 6;
2539
+ val |= mask[idx + 7] << 7;
2540
+ bitmask |= (uint64_t)val << idx;
2541
+ }
2542
+
2543
+ for (int32_t i = 64; i < GGML_MAX_N_THREADS; i++) {
2544
+ if (mask[i]) {
2545
+ fprintf(stderr, "warn: setting thread-affinity for > 64 CPUs isn't supported on windows!\n");
2546
+ break;
2547
+ }
2548
+ }
2549
+
2550
+ DWORD_PTR m = (DWORD_PTR)bitmask;
2551
+
2552
+ m = SetThreadAffinityMask(h, m);
2553
+
2554
+ return m != 0;
2555
+ }
2556
+
2557
+ static bool ggml_thread_apply_priority(int32_t prio) {
2558
+ // Note that on Windows the Process Priority Class must be updated in order to set Thread priority.
2559
+ // This is up to the applications.
2560
+ DWORD p = THREAD_PRIORITY_NORMAL;
2561
+ switch (prio) {
2562
+ case GGML_SCHED_PRIO_LOW: p = THREAD_PRIORITY_BELOW_NORMAL; break;
2563
+ case GGML_SCHED_PRIO_NORMAL: p = THREAD_PRIORITY_NORMAL; break;
2564
+ case GGML_SCHED_PRIO_MEDIUM: p = THREAD_PRIORITY_ABOVE_NORMAL; break;
2565
+ case GGML_SCHED_PRIO_HIGH: p = THREAD_PRIORITY_HIGHEST; break;
2566
+ case GGML_SCHED_PRIO_REALTIME: p = THREAD_PRIORITY_TIME_CRITICAL; break;
2567
+ }
2568
+
2569
+ if (prio != GGML_SCHED_PRIO_LOW) {
2570
+ // Tell Windows that this thread should not be throttled (needs its own CPU core).
2571
+ // Newer Windows 11 versions aggressively park (offline) CPU cores and often place
2572
+ // all our threads onto the first 4 cores which results in terrible performance with
2573
+ // n_threads > 4
2574
+ #if _WIN32_WINNT >= 0x0602
2575
+ THREAD_POWER_THROTTLING_STATE t;
2576
+ ZeroMemory(&t, sizeof(t));
2577
+ t.Version = THREAD_POWER_THROTTLING_CURRENT_VERSION;
2578
+ t.ControlMask = THREAD_POWER_THROTTLING_EXECUTION_SPEED;
2579
+ t.StateMask = 0;
2580
+
2581
+ if (!SetThreadInformation(GetCurrentThread(), ThreadPowerThrottling, &t, sizeof(t))) {
2582
+ GGML_LOG_DEBUG("failed to disable thread power throttling %d : (%d)\n", prio, (int) GetLastError());
2583
+ return false;
2584
+ }
2585
+ #endif
2586
+ }
2587
+
2588
+ if (prio == GGML_SCHED_PRIO_NORMAL) {
2589
+ // Keep inherited policy/priority
2590
+ return true;
2591
+ }
2592
+
2593
+ if (!SetThreadPriority(GetCurrentThread(), p)) {
2594
+ fprintf(stderr, "warn: failed to set thread priority %d : (%d)\n", prio, (int) GetLastError());
2595
+ return false;
2596
+ }
2597
+
2598
+ return true;
2599
+ }
2600
+
2601
+ #elif defined(__APPLE__)
2602
+ #include <sys/types.h>
2603
+ #include <sys/resource.h>
2604
+
2605
+ static bool ggml_thread_apply_affinity(const bool * mask) {
2606
+ // Not supported on Apple platforms
2607
+ UNUSED(mask);
2608
+ return true;
2609
+ }
2610
+
2611
+ static bool ggml_thread_apply_priority(int32_t prio) {
2612
+ struct sched_param p;
2613
+ int32_t policy = SCHED_OTHER;
2614
+ switch (prio) {
2615
+ // TODO: there seems to be no way to set lower prio on Apple platforms
2616
+ case GGML_SCHED_PRIO_LOW: policy = SCHED_OTHER; p.sched_priority = 0; break;
2617
+ case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER; p.sched_priority = 0; break;
2618
+ case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO; p.sched_priority = 40; break;
2619
+ case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO; p.sched_priority = 80; break;
2620
+ case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO; p.sched_priority = 90; break;
2621
+ }
2622
+
2623
+ if (prio == GGML_SCHED_PRIO_NORMAL) {
2624
+ // Keep inherited policy/priority
2625
+ return true;
2626
+ }
2627
+
2628
+ int32_t err = pthread_setschedparam(pthread_self(), policy, &p);
2629
+ if (err != 0) {
2630
+ fprintf(stderr, "warn: failed to set thread priority %d : %s (%d)\n", prio, strerror(err), err);
2631
+ return false;
2632
+ }
2633
+
2634
+ return true;
2635
+ }
2636
+
2637
+ #elif defined(__linux__)
2638
+ // TODO: this may not work on BSD, to be verified
2639
+
2640
+ static bool ggml_thread_apply_affinity(const bool * mask) {
2641
+ cpu_set_t cpuset;
2642
+ int err;
2643
+
2644
+ CPU_ZERO(&cpuset);
2645
+
2646
+ for (uint32_t i = 0; i < GGML_MAX_N_THREADS; i++) {
2647
+ if (mask[i]) {
2648
+ GGML_PRINT_DEBUG("Thread %lx: adding %d to cpuset\n", pthread_self(), i);
2649
+ CPU_SET(i, &cpuset);
2650
+ }
2651
+ }
2652
+
2653
+ #ifdef __ANDROID__
2654
+ err = sched_setaffinity(0, sizeof(cpuset), &cpuset);
2655
+ if (err < 0) {
2656
+ err = errno;
2657
+ }
2658
+ #else
2659
+ err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
2660
+ #endif
2661
+ if (err != 0) {
2662
+ fprintf(stderr, "warn: failed to set affinity mask 0x%llx : %s (%d)\n", (unsigned long long)mask, strerror(err), err);
2663
+ return false;
2664
+ }
2665
+
2666
+ return true;
2667
+ }
2668
+
2669
+ static bool ggml_thread_apply_priority(int32_t prio) {
2670
+ struct sched_param p;
2671
+ int32_t policy = SCHED_OTHER;
2672
+ switch (prio) {
2673
+ case GGML_SCHED_PRIO_LOW: policy = SCHED_BATCH; p.sched_priority = 0; break;
2674
+ case GGML_SCHED_PRIO_NORMAL: policy = SCHED_OTHER; p.sched_priority = 0; break;
2675
+ case GGML_SCHED_PRIO_MEDIUM: policy = SCHED_FIFO; p.sched_priority = 40; break;
2676
+ case GGML_SCHED_PRIO_HIGH: policy = SCHED_FIFO; p.sched_priority = 80; break;
2677
+ case GGML_SCHED_PRIO_REALTIME: policy = SCHED_FIFO; p.sched_priority = 90; break;
2678
+ }
2679
+
2680
+ if (prio == GGML_SCHED_PRIO_NORMAL) {
2681
+ // Keep inherited policy/priority
2682
+ return true;
2683
+ }
2684
+
2685
+ int32_t err = pthread_setschedparam(pthread_self(), policy, &p);
2686
+ if (err != 0) {
2687
+ fprintf(stderr, "warn: failed to set thread priority %d : %s (%d)\n", prio, strerror(err), err);
2688
+ return false;
2689
+ }
2690
+
2691
+ return true;
2692
+ }
2693
+
2694
+ #else // unsupported platforms
2695
+
2696
+ static bool ggml_thread_apply_affinity(const bool * mask) {
2697
+ UNUSED(mask);
2698
+ return true;
2699
+ }
2700
+
2701
+ static bool ggml_thread_apply_priority(int32_t prio) {
2702
+ UNUSED(prio);
2703
+ return true;
2704
+ }
2705
+
2706
+ #endif
2707
+
2708
+ static bool ggml_thread_cpumask_is_valid(const bool * mask) {
2709
+ for (int i = 0; i < GGML_MAX_N_THREADS; i++) {
2710
+ if (mask[i]) { return true; }
2711
+ }
2712
+ return false;
2713
+ }
2714
+
2715
+ static void ggml_thread_cpumask_next(const bool * global_mask, bool * local_mask, bool strict, int32_t* iter) {
2716
+ if (!strict) {
2717
+ memcpy(local_mask, global_mask, GGML_MAX_N_THREADS);
2718
+ return;
2719
+ } else {
2720
+ memset(local_mask, 0, GGML_MAX_N_THREADS);
2721
+ int32_t base_idx = *iter;
2722
+ for (int32_t i = 0; i < GGML_MAX_N_THREADS; i++) {
2723
+ int32_t idx = base_idx + i;
2724
+ if (idx >= GGML_MAX_N_THREADS) {
2725
+ // Just a cheaper modulo
2726
+ idx -= GGML_MAX_N_THREADS;
2727
+ }
2728
+ if (global_mask[idx]) {
2729
+ local_mask[idx] = 1;
2730
+ *iter = idx + 1;
2731
+ return;
2732
+ }
2733
+ }
2734
+ }
2735
+ }
2736
+
2737
+ void ggml_threadpool_free(struct ggml_threadpool* threadpool) {
2738
+ if (!threadpool) return;
2739
+
2740
+ const int n_threads = threadpool->n_threads;
2741
+
2742
+ #ifndef GGML_USE_OPENMP
2743
+ struct ggml_compute_state* workers = threadpool->workers;
2744
+
2745
+ ggml_mutex_lock(&threadpool->mutex);
2746
+
2747
+ threadpool->stop = true;
2748
+ threadpool->pause = false;
2749
+
2750
+ ggml_cond_broadcast(&threadpool->cond);
2751
+ ggml_mutex_unlock(&threadpool->mutex);
2752
+
2753
+ for (int j = 1; j < n_threads; j++) {
2754
+ int32_t rc = ggml_thread_join(workers[j].thrd, NULL);
2755
+ GGML_ASSERT(rc == GGML_EXIT_SUCCESS || rc == GGML_EXIT_ABORTED);
2756
+ UNUSED(rc);
2757
+ }
2758
+
2759
+ ggml_mutex_destroy(&threadpool->mutex);
2760
+ ggml_cond_destroy(&threadpool->cond);
2761
+ #endif // GGML_USE_OPENMP
2762
+
2763
+ const size_t workers_size = sizeof(struct ggml_compute_state) * n_threads;
2764
+ ggml_aligned_free(threadpool->workers, workers_size);
2765
+ ggml_aligned_free(threadpool, sizeof(struct ggml_threadpool));
2766
+ }
2767
+
2768
+ #ifndef GGML_USE_OPENMP
2769
+ // pause/resume must be called under mutex
2770
+ static void ggml_threadpool_pause_locked(struct ggml_threadpool * threadpool) {
2771
+ GGML_PRINT_DEBUG("Pausing threadpool\n");
2772
+ threadpool->pause = true;
2773
+ ggml_cond_broadcast(&threadpool->cond);
2774
+ }
2775
+
2776
+ static void ggml_threadpool_resume_locked(struct ggml_threadpool * threadpool) {
2777
+ GGML_PRINT_DEBUG("Resuming threadpool\n");
2778
+ threadpool->pause = false;
2779
+ ggml_cond_broadcast(&threadpool->cond);
2780
+ }
2781
+ #endif
2782
+
2783
+ void ggml_threadpool_pause(struct ggml_threadpool * threadpool) {
2784
+ #ifndef GGML_USE_OPENMP
2785
+ ggml_mutex_lock(&threadpool->mutex);
2786
+ if (!threadpool->pause) {
2787
+ ggml_threadpool_pause_locked(threadpool);
2788
+ }
2789
+ ggml_mutex_unlock(&threadpool->mutex);
2790
+ #else
2791
+ UNUSED(threadpool);
2792
+ #endif
2793
+ }
2794
+
2795
+ void ggml_threadpool_resume(struct ggml_threadpool * threadpool) {
2796
+ #ifndef GGML_USE_OPENMP
2797
+ ggml_mutex_lock(&threadpool->mutex);
2798
+ if (threadpool->pause) {
2799
+ ggml_threadpool_resume_locked(threadpool);
2800
+ }
2801
+ ggml_mutex_unlock(&threadpool->mutex);
2802
+ #else
2803
+ UNUSED(threadpool);
2804
+ #endif
2805
+ }
2806
+
2807
+ struct ggml_cplan ggml_graph_plan(
2808
+ const struct ggml_cgraph * cgraph,
2809
+ int n_threads,
2810
+ struct ggml_threadpool * threadpool) {
2811
+
2812
+ if (threadpool == NULL) {
2813
+ //GGML_PRINT_DEBUG("Threadpool is not specified. Will create a disposable threadpool : n_threads %d\n", n_threads);
2814
+ }
2815
+ if (n_threads <= 0) {
2816
+ n_threads = threadpool ? threadpool->n_threads : GGML_DEFAULT_N_THREADS;
2817
+ }
2818
+
2819
+ #if defined(__EMSCRIPTEN__) && !defined(__EMSCRIPTEN_PTHREADS__)
2820
+ // Emscripten without pthreads support can only use a single thread
2821
+ n_threads = 1;
2822
+ #endif
2823
+
2824
+ #if defined(__wasi__)
2825
+ // WASI doesn't support parallelism yet
2826
+ n_threads = 1;
2827
+ #endif
2828
+
2829
+ size_t work_size = 0;
2830
+
2831
+ struct ggml_cplan cplan;
2832
+ memset(&cplan, 0, sizeof(struct ggml_cplan));
2833
+
2834
+ int max_tasks = 1;
2835
+
2836
+ // thread scheduling for the different operations + work buffer size estimation
2837
+ for (int i = 0; i < cgraph->n_nodes; i++) {
2838
+ struct ggml_tensor * node = cgraph->nodes[i];
2839
+
2840
+ const int n_tasks = ggml_get_n_tasks(node, n_threads);
2841
+
2842
+ max_tasks = MAX(max_tasks, n_tasks);
2843
+
2844
+ size_t cur = 0;
2845
+
2846
+ if (!ggml_cpu_extra_work_size(n_threads, node, &cur)) {
2847
+ switch (node->op) {
2848
+ case GGML_OP_CPY:
2849
+ case GGML_OP_DUP:
2850
+ {
2851
+ if (ggml_is_quantized(node->type) ||
2852
+ // F16 -> BF16 and BF16 -> F16 copies go through intermediate F32
2853
+ (node->src[0]->type == GGML_TYPE_F16 && node->src[1] && node->src[1]->type == GGML_TYPE_BF16) ||
2854
+ (node->src[0]->type == GGML_TYPE_BF16 && node->src[1] && node->src[1]->type == GGML_TYPE_F16) ||
2855
+ // conversion between F32 and I32
2856
+ (node->src[0]->type == GGML_TYPE_F32 && node->src[1] && node->src[1]->type == GGML_TYPE_I32) ||
2857
+ (node->src[0]->type == GGML_TYPE_I32 && node->src[1] && node->src[1]->type == GGML_TYPE_F32)) {
2858
+ cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;
2859
+ }
2860
+ } break;
2861
+ case GGML_OP_ADD:
2862
+ case GGML_OP_ADD_ID:
2863
+ case GGML_OP_ADD1:
2864
+ {
2865
+ if (ggml_is_quantized(node->src[0]->type)) {
2866
+ cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
2867
+ }
2868
+ } break;
2869
+ case GGML_OP_ACC:
2870
+ {
2871
+ if (ggml_is_quantized(node->src[0]->type)) {
2872
+ cur = ggml_type_size(GGML_TYPE_F32) * node->src[1]->ne[0] * n_tasks;
2873
+ }
2874
+ } break;
2875
+ case GGML_OP_COUNT_EQUAL:
2876
+ {
2877
+ cur = ggml_type_size(node->type)*n_tasks;
2878
+ } break;
2879
+ case GGML_OP_MUL_MAT:
2880
+ {
2881
+ const enum ggml_type vec_dot_type = type_traits_cpu[node->src[0]->type].vec_dot_type;
2882
+
2883
+ if (node->src[1]->type != vec_dot_type) {
2884
+ cur = ggml_row_size(vec_dot_type, ggml_nelements(node->src[1]));
2885
+ }
2886
+
2887
+ // the IQ panel path needs one scratch panel per thread past the q8_K rows
2888
+ if (ggml_cpu_iqp_supports_mul_mat(node)) {
2889
+ cur = GGML_PAD(cur, 64) + n_tasks * ggml_cpu_iqp_scratch_size(node);
2890
+ }
2891
+ } break;
2892
+ case GGML_OP_MUL_MAT_ID:
2893
+ {
2894
+ cur = 0;
2895
+ const struct ggml_tensor * src0 = node->src[0];
2896
+ const struct ggml_tensor * src1 = node->src[1];
2897
+ const struct ggml_tensor * ids = node->src[2];
2898
+ const enum ggml_type vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;
2899
+ const int n_as = src0->ne[2];
2900
+ // src1
2901
+ if (src1->type != vec_dot_type) {
2902
+ cur += ggml_row_size(vec_dot_type, ggml_nelements(src1)) + sizeof(int64_t);
2903
+ }
2904
+ // matrix_row_counts
2905
+ cur += n_as * sizeof(int64_t) + sizeof(int64_t);
2906
+ // matrix_rows
2907
+ cur += n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping) + sizeof(int64_t);
2908
+ // atomic_current_chunk
2909
+ cur += CACHE_LINE_SIZE*n_as + CACHE_LINE_SIZE;
2910
+ // the IQ panel path needs one scratch panel per thread on top of that
2911
+ if (ggml_cpu_iqp_supports_mul_mat_id(node)) {
2912
+ cur += n_tasks * ggml_cpu_iqp_scratch_size(node) + 64;
2913
+ }
2914
+ } break;
2915
+ case GGML_OP_OUT_PROD:
2916
+ {
2917
+ if (ggml_is_quantized(node->src[0]->type) ||
2918
+ node->src[0]->type == GGML_TYPE_F16) {
2919
+ cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
2920
+ }
2921
+ } break;
2922
+ case GGML_OP_SET_ROWS:
2923
+ {
2924
+ if (node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) {
2925
+ cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
2926
+ }
2927
+ } break;
2928
+ case GGML_OP_SOFT_MAX:
2929
+ case GGML_OP_ROPE:
2930
+ case GGML_OP_ROPE_BACK:
2931
+ {
2932
+ cur = ggml_type_size(GGML_TYPE_F32) * node->ne[0] * n_tasks;
2933
+ } break;
2934
+ case GGML_OP_CONV_TRANSPOSE_1D:
2935
+ {
2936
+ GGML_ASSERT(node->src[0]->ne[3] == 1);
2937
+ GGML_ASSERT(node->src[1]->ne[2] == 1);
2938
+ GGML_ASSERT(node->src[1]->ne[3] == 1);
2939
+
2940
+ const int64_t ne00 = node->src[0]->ne[0]; // K
2941
+ const int64_t ne01 = node->src[0]->ne[1]; // Cout
2942
+ const int64_t ne02 = node->src[0]->ne[2]; // Cin
2943
+ const int64_t ne10 = node->src[1]->ne[0]; // L
2944
+ const int64_t ne11 = node->src[1]->ne[1]; // Cin
2945
+
2946
+ if ((node->src[0]->type == GGML_TYPE_F16 ||
2947
+ node->src[0]->type == GGML_TYPE_BF16) &&
2948
+ node->src[1]->type == GGML_TYPE_F32) {
2949
+ cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02;
2950
+ cur += sizeof(ggml_fp16_t)*ne10*ne11;
2951
+ } else if (node->src[0]->type == GGML_TYPE_F32 &&
2952
+ node->src[1]->type == GGML_TYPE_F32) {
2953
+ cur += sizeof(float)*ne00*ne01*ne02;
2954
+ cur += sizeof(float)*ne10*ne11;
2955
+ } else {
2956
+ GGML_ABORT("fatal error");
2957
+ }
2958
+ } break;
2959
+ case GGML_OP_CONV_2D:
2960
+ case GGML_OP_CONV_3D:
2961
+ {
2962
+ cur = GGML_IM2COL_WORK_SIZE;
2963
+ } break;
2964
+ case GGML_OP_CONV_TRANSPOSE_2D:
2965
+ {
2966
+ const int64_t ne00 = node->src[0]->ne[0]; // W
2967
+ const int64_t ne01 = node->src[0]->ne[1]; // H
2968
+ const int64_t ne02 = node->src[0]->ne[2]; // Channels Out
2969
+ const int64_t ne03 = node->src[0]->ne[3]; // Channels In
2970
+
2971
+ const int64_t ne10 = node->src[1]->ne[0]; // W
2972
+ const int64_t ne11 = node->src[1]->ne[1]; // H
2973
+ const int64_t ne12 = node->src[1]->ne[2]; // Channels In
2974
+ const int64_t ne13 = node->src[1]->ne[3]; // Batch
2975
+
2976
+ GGML_ASSERT(node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32);
2977
+ GGML_ASSERT(node->src[1]->type == GGML_TYPE_F32);
2978
+
2979
+ cur += ggml_type_size(node->src[0]->type) * ne00 * ne01 * ne02 * ne03;
2980
+ cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12 * ne13;
2981
+
2982
+ } break;
2983
+ case GGML_OP_TOP_K:
2984
+ {
2985
+ cur += sizeof(int32_t)*node->src[0]->ne[0]*n_tasks;
2986
+ } break;
2987
+ case GGML_OP_FLASH_ATTN_EXT:
2988
+ {
2989
+ const int64_t neq2 = node->src[0]->ne[2]; // number of query heads
2990
+ const int64_t DK = node->src[1]->ne[0];
2991
+ const int64_t DV = node->src[2]->ne[0];
2992
+
2993
+ // Tiled flash attention scratch (tile sizes defined in common.h)
2994
+ // Per-thread: Q_q + KQ + mask + VKQ32 + V32 + K_f32 + padding
2995
+ size_t prefill = sizeof(float)*(GGML_FA_TILE_Q*DK + 2*GGML_FA_TILE_Q*GGML_FA_TILE_KV + GGML_FA_TILE_Q*DV + GGML_FA_TILE_KV*DV + GGML_FA_TILE_KV*DK)*n_tasks;
2996
+
2997
+ // Decode path: n_kv_chunks = n_tasks (one chunk per thread)
2998
+ // Per-thread: VKQ accmulator (DV), partial M, partial S + intra-thread scratch for V, Q and VKQ
2999
+ size_t n_chunks = n_tasks;
3000
+ size_t decode = sizeof(float)*(neq2*n_chunks*(2+DV) + n_tasks*(DK + 2*DV));
3001
+
3002
+ cur += MAX(prefill, decode);
3003
+ } break;
3004
+ case GGML_OP_FLASH_ATTN_BACK:
3005
+ {
3006
+ const int64_t D = node->src[0]->ne[0];
3007
+ const int64_t ne11 = ggml_up(node->src[1]->ne[1], GGML_SOFT_MAX_UNROLL);
3008
+ const int64_t mxDn = MAX(D, ne11) * 2; // *2 because of S and SM in ggml_compute_forward_flash_attn_back
3009
+ if (node->src[1]->type == GGML_TYPE_F32) {
3010
+ cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
3011
+ cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
3012
+ } else if (node->src[1]->type == GGML_TYPE_F16) {
3013
+ cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
3014
+ cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
3015
+ } else if (node->src[1]->type == GGML_TYPE_BF16) {
3016
+ cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
3017
+ cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
3018
+ }
3019
+ } break;
3020
+
3021
+ case GGML_OP_CROSS_ENTROPY_LOSS:
3022
+ {
3023
+ cur = ggml_type_size(node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks);
3024
+ } break;
3025
+ case GGML_OP_GATED_DELTA_NET:
3026
+ {
3027
+ const int64_t S_v = node->src[2]->ne[0];
3028
+ const int64_t K = ggml_get_op_params_i32(node, 0);
3029
+ const int64_t per_thread = S_v + (K > 1 ? S_v * S_v : 0);
3030
+ cur = per_thread * sizeof(float) * n_tasks;
3031
+ } break;
3032
+ case GGML_OP_COUNT:
3033
+ {
3034
+ GGML_ABORT("fatal error");
3035
+ }
3036
+ case GGML_OP_LIGHTNING_INDEXER:
3037
+ {
3038
+ // temp buffer for dequantizing lightning indexer keys
3039
+ const int64_t ne10 = node->src[1]->ne[0];
3040
+ cur += sizeof(float)*ne10*n_tasks;
3041
+ } break;
3042
+ default:
3043
+ break;
3044
+ }
3045
+ }
3046
+
3047
+ work_size = MAX(work_size, cur);
3048
+ }
3049
+
3050
+ if (work_size > 0) {
3051
+ work_size += CACHE_LINE_SIZE*(n_threads);
3052
+ }
3053
+
3054
+ cplan.threadpool = threadpool;
3055
+ cplan.n_threads = MIN(max_tasks, n_threads);
3056
+ cplan.work_size = work_size;
3057
+ cplan.work_data = NULL;
3058
+
3059
+ return cplan;
3060
+ }
3061
+
3062
+
3063
+ // Try to fuse the current node with subsequent nodes for better performance.
3064
+ // Returns the number of nodes skipped by fusion (>=1), or 0 if no fusion was applied.
3065
+ static bool ggml_cpu_disable_fusion = false; // initialized once in ggml_cpu_init(), read-only afterwards
3066
+
3067
+ static int ggml_cpu_try_fuse_ops(
3068
+ const struct ggml_cgraph * cgraph,
3069
+ const int node_n,
3070
+ const struct ggml_compute_params * params,
3071
+ const struct ggml_cplan * cplan) {
3072
+
3073
+ if (ggml_cpu_disable_fusion || cplan->use_ref) {
3074
+ return 0;
3075
+ }
3076
+
3077
+ struct ggml_tensor * node = cgraph->nodes[node_n];
3078
+
3079
+ if (node->op == GGML_OP_RMS_NORM) {
3080
+ // RMS_NORM + MUL fusion
3081
+ const enum ggml_op fuse_ops[] = { GGML_OP_RMS_NORM, GGML_OP_MUL };
3082
+ if (ggml_can_fuse(cgraph, node_n, fuse_ops, 2)) {
3083
+ struct ggml_tensor * mul_node = cgraph->nodes[node_n + 1];
3084
+ const struct ggml_tensor * mul_w = (mul_node->src[0] == node)
3085
+ ? mul_node->src[1] : mul_node->src[0];
3086
+ if (node->src[0]->type == GGML_TYPE_F32 &&
3087
+ mul_node->type == GGML_TYPE_F32 &&
3088
+ mul_w->type == GGML_TYPE_F32 &&
3089
+ mul_w->ne[0] == node->ne[0] &&
3090
+ mul_w->nb[0] == sizeof(float)) {
3091
+
3092
+ ggml_compute_forward_rms_norm_mul_fused(params, node, mul_node);
3093
+ return 1;
3094
+ }
3095
+ }
3096
+ }
3097
+
3098
+ return 0;
3099
+ }
3100
+
3101
+ static thread_ret_t ggml_graph_compute_thread(void * data) {
3102
+ struct ggml_compute_state * state = (struct ggml_compute_state *) data;
3103
+ struct ggml_threadpool * tp = state->threadpool;
3104
+
3105
+ const struct ggml_cgraph * cgraph = tp->cgraph;
3106
+ const struct ggml_cplan * cplan = tp->cplan;
3107
+
3108
+ #ifdef GGML_USE_CPU_RISCV64_SPACEMIT
3109
+ ggml_backend_cpu_riscv64_spacemit_set_numa_thread_affinity(state->ith);
3110
+ #else
3111
+ set_numa_thread_affinity(state->ith);
3112
+ #endif
3113
+
3114
+ struct ggml_compute_params params = {
3115
+ /*.ith =*/ state->ith,
3116
+ /*.nth =*/ atomic_load_explicit(&tp->n_graph, memory_order_relaxed) & GGML_THREADPOOL_N_THREADS_MASK,
3117
+ /*.wsize =*/ cplan->work_size,
3118
+ /*.wdata =*/ cplan->work_data,
3119
+ /*.threadpool =*/ tp,
3120
+ /*.use_ref =*/ cplan->use_ref,
3121
+ };
3122
+
3123
+ #ifdef GGML_USE_OPENMP
3124
+ GGML_PRINT_DEBUG("thread #%d compute-start cplan %p\n", state->ith, (const void *)cplan);
3125
+ #else
3126
+ GGML_PRINT_DEBUG("thread #%d compute-start cplan %p last-graph %d\n", state->ith, (const void *)cplan, state->last_graph);
3127
+ #endif
3128
+
3129
+ for (int node_n = 0; node_n < cgraph->n_nodes && atomic_load_explicit(&tp->abort, memory_order_relaxed) != node_n; node_n++) {
3130
+ struct ggml_tensor * node = cgraph->nodes[node_n];
3131
+
3132
+ if (ggml_op_is_empty(node->op)) {
3133
+ // skip NOPs
3134
+ continue;
3135
+ }
3136
+
3137
+ if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {
3138
+ continue;
3139
+ }
3140
+
3141
+ // TODO: move fused-op detection into ggml_graph_plan so fusion decisions are made once at planning time
3142
+ // Try fused ops, fall back to normal compute
3143
+ const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, &params, cplan);
3144
+ if (n_fused > 0) {
3145
+ node_n += n_fused;
3146
+ } else {
3147
+ ggml_compute_forward(&params, node);
3148
+ }
3149
+
3150
+ if (state->ith == 0 && cplan->abort_callback &&
3151
+ cplan->abort_callback(cplan->abort_callback_data)) {
3152
+ atomic_store_explicit(&tp->abort, node_n + 1, memory_order_relaxed);
3153
+ tp->ec = GGML_STATUS_ABORTED;
3154
+ }
3155
+
3156
+ if (node_n + 1 < cgraph->n_nodes) {
3157
+ ggml_barrier(state->threadpool);
3158
+ }
3159
+ }
3160
+
3161
+ #ifdef GGML_USE_OPENMP
3162
+ GGML_PRINT_DEBUG("thread #%d compute-done cplan %p\n", state->ith, (const void *)cplan);
3163
+ #else
3164
+ GGML_PRINT_DEBUG("thread #%d compute-done cplan %p last-graph %d\n", state->ith, (const void *)cplan, state->last_graph);
3165
+ #endif
3166
+
3167
+ ggml_barrier(state->threadpool);
3168
+
3169
+ #ifdef GGML_USE_CPU_RISCV64_SPACEMIT
3170
+ ggml_backend_cpu_riscv64_spacemit_clear_numa_thread_affinity_threaded(state->ith);
3171
+ #endif
3172
+
3173
+ return 0;
3174
+ }
3175
+
3176
+ #ifndef GGML_USE_OPENMP
3177
+
3178
+ // check if thread is ready to proceed (exit from polling or sleeping)
3179
+ // returns true if loops should exit, sets state->pending to indicate new work
3180
+ static inline bool ggml_graph_compute_thread_ready(struct ggml_compute_state * state) {
3181
+ struct ggml_threadpool * threadpool = state->threadpool;
3182
+
3183
+ if (state->pending || threadpool->stop || threadpool->pause) { return true; }
3184
+
3185
+ // check for new graph/work
3186
+ int n_graph = atomic_load_explicit(&threadpool->n_graph, memory_order_relaxed);
3187
+ int n_threads = n_graph & GGML_THREADPOOL_N_THREADS_MASK;
3188
+ if (n_graph != state->last_graph) {
3189
+ state->pending = (state->ith < n_threads);
3190
+ state->last_graph = n_graph;
3191
+ return true;
3192
+ }
3193
+
3194
+ return false;
3195
+ }
3196
+
3197
+ // sync thread state after polling
3198
+ static inline void ggml_graph_compute_thread_sync(struct ggml_compute_state * state) {
3199
+ // TSAN doesn't support standalone fence yet, we use a dummy read-modify-write instead
3200
+ #ifdef GGML_TSAN_ENABLED
3201
+ atomic_fetch_add_explicit(&state->threadpool->n_graph, 0, memory_order_seq_cst);
3202
+ #else
3203
+ atomic_thread_fence(memory_order_seq_cst);
3204
+ #endif
3205
+ UNUSED(state);
3206
+ }
3207
+
3208
+ static inline bool ggml_graph_compute_poll_for_work(struct ggml_compute_state * state) {
3209
+ struct ggml_threadpool * threadpool = state->threadpool;
3210
+
3211
+ // This seems to make 0 ... 100 a decent range for polling level across modern processors.
3212
+ // Perhaps, we can adjust it dynamically based on load and things.
3213
+ const uint64_t n_rounds = 1024UL * 128 * threadpool->poll;
3214
+
3215
+ for (uint64_t i=0; !ggml_graph_compute_thread_ready(state) && i < n_rounds; i++) {
3216
+ // No new work. Keep polling.
3217
+ ggml_thread_cpu_relax();
3218
+ }
3219
+
3220
+ return state->pending;
3221
+ }
3222
+
3223
+ static inline bool ggml_graph_compute_check_for_work(struct ggml_compute_state * state) {
3224
+ struct ggml_threadpool * threadpool = state->threadpool;
3225
+
3226
+ if (ggml_graph_compute_poll_for_work(state)) {
3227
+ ggml_graph_compute_thread_sync(state);
3228
+ return state->pending;
3229
+ }
3230
+
3231
+ ggml_mutex_lock_shared(&threadpool->mutex);
3232
+ while (!ggml_graph_compute_thread_ready(state)) {
3233
+ // No new work. Wait for the signal.
3234
+ GGML_PRINT_DEBUG("thread #%d waiting for work (sleeping)\n", state->ith);
3235
+ ggml_cond_wait(&threadpool->cond, &threadpool->mutex);
3236
+ }
3237
+ ggml_mutex_unlock_shared(&threadpool->mutex);
3238
+
3239
+ return state->pending;
3240
+ }
3241
+
3242
+ static thread_ret_t ggml_graph_compute_secondary_thread(void* data) {
3243
+ struct ggml_compute_state * state = (struct ggml_compute_state *) data;
3244
+ struct ggml_threadpool * threadpool = state->threadpool;
3245
+
3246
+ ggml_thread_apply_priority(threadpool->prio);
3247
+ if (ggml_thread_cpumask_is_valid(state->cpumask)) {
3248
+ ggml_thread_apply_affinity(state->cpumask);
3249
+ }
3250
+
3251
+ while (true) {
3252
+ // Check if we need to sleep
3253
+ while (threadpool->pause) {
3254
+ GGML_PRINT_DEBUG("thread #%d inside pause loop\n", state->ith);
3255
+ ggml_mutex_lock_shared(&threadpool->mutex);
3256
+ if (threadpool->pause) {
3257
+ ggml_cond_wait(&threadpool->cond, &threadpool->mutex);
3258
+ }
3259
+ GGML_PRINT_DEBUG("thread #%d resuming after wait\n", state->ith);
3260
+ ggml_mutex_unlock_shared(&threadpool->mutex);
3261
+ }
3262
+
3263
+ // This needs to be checked for after the cond_wait
3264
+ if (threadpool->stop) break;
3265
+
3266
+ // Check if there is new work
3267
+ // The main thread is the only one that can dispatch new work
3268
+
3269
+ ggml_graph_compute_check_for_work(state);
3270
+ if (state->pending) {
3271
+ state->pending = false;
3272
+ ggml_graph_compute_thread(state);
3273
+ }
3274
+ }
3275
+
3276
+ return (thread_ret_t) 0;
3277
+ }
3278
+
3279
+ // Start processing new graph
3280
+ static void ggml_graph_compute_kickoff(struct ggml_threadpool * threadpool, int n_threads)
3281
+ {
3282
+ // Always take the mutex here because the worker threads are doing hybrid poll/wait
3283
+
3284
+ ggml_mutex_lock(&threadpool->mutex);
3285
+
3286
+ // Update the number of active threads and the graph count
3287
+ int n_graph = atomic_load_explicit(&threadpool->n_graph, memory_order_relaxed) >> GGML_THREADPOOL_N_THREADS_BITS;
3288
+ n_graph = ((n_graph + 1) << GGML_THREADPOOL_N_THREADS_BITS) | (n_threads & GGML_THREADPOOL_N_THREADS_MASK);
3289
+
3290
+ GGML_PRINT_DEBUG("compute-kickoff: n_threads %d n_graph %d\n", n_threads, n_graph);
3291
+
3292
+ // Indicate the graph is ready to be processed
3293
+ // We need the full seq-cst fence here because of the polling threads (used in thread_sync)
3294
+ atomic_store_explicit(&threadpool->n_graph, n_graph, memory_order_seq_cst);
3295
+
3296
+ if (threadpool->pause) {
3297
+ // Update main thread prio and affinity to match the threadpool settings
3298
+ ggml_thread_apply_priority(threadpool->prio);
3299
+ if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {
3300
+ ggml_thread_apply_affinity(threadpool->workers[0].cpumask);
3301
+ }
3302
+
3303
+ // resume does cond broadcast
3304
+ ggml_threadpool_resume_locked(threadpool);
3305
+ } else {
3306
+ ggml_cond_broadcast(&threadpool->cond);
3307
+ }
3308
+
3309
+ ggml_mutex_unlock(&threadpool->mutex);
3310
+ }
3311
+
3312
+ #endif // GGML_USE_OPENMP
3313
+
3314
+ static struct ggml_threadpool * ggml_threadpool_new_impl(
3315
+ struct ggml_threadpool_params * tpp,
3316
+ struct ggml_cgraph * cgraph,
3317
+ struct ggml_cplan * cplan) {
3318
+
3319
+ struct ggml_threadpool * threadpool =
3320
+ ggml_aligned_malloc(sizeof(struct ggml_threadpool));
3321
+ {
3322
+ threadpool->cgraph = cgraph;
3323
+ threadpool->cplan = cplan;
3324
+ threadpool->n_graph = 0;
3325
+ threadpool->n_barrier = 0;
3326
+ threadpool->n_barrier_passed = 0;
3327
+ threadpool->current_chunk = 0;
3328
+ threadpool->stop = false;
3329
+ threadpool->pause = tpp->paused;
3330
+ threadpool->abort = -1;
3331
+ threadpool->workers = NULL;
3332
+ threadpool->n_threads = tpp->n_threads;
3333
+ threadpool->poll = tpp->poll;
3334
+ threadpool->prio = tpp->prio;
3335
+ threadpool->ec = GGML_STATUS_SUCCESS;
3336
+ }
3337
+
3338
+ // Allocate and init workers state
3339
+ const size_t workers_size = sizeof(struct ggml_compute_state) * tpp->n_threads;
3340
+ struct ggml_compute_state * workers = ggml_aligned_malloc(workers_size);
3341
+
3342
+ memset(workers, 0, workers_size);
3343
+ for (int j = 0; j < tpp->n_threads; j++) {
3344
+ workers[j].threadpool = threadpool;
3345
+ workers[j].ith = j;
3346
+ }
3347
+
3348
+ threadpool->workers = workers;
3349
+
3350
+ #ifdef GGML_USE_OPENMP
3351
+ int32_t cpumask_iter = 0;
3352
+
3353
+ // Compute CPU masks for each thread
3354
+ for (int j = 0; j < tpp->n_threads; j++) {
3355
+ ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter);
3356
+ }
3357
+ #else // GGML_USE_OPENMP
3358
+ ggml_mutex_init(&threadpool->mutex);
3359
+ ggml_cond_init(&threadpool->cond);
3360
+
3361
+ // Spin the threads for all workers, and update CPU placements.
3362
+ // Place the main thread last (towards the higher numbered CPU cores).
3363
+
3364
+ int32_t cpumask_iter = 0;
3365
+
3366
+ for (int j = 1; j < tpp->n_threads; j++) {
3367
+ ggml_thread_cpumask_next(tpp->cpumask, workers[j].cpumask, tpp->strict_cpu, &cpumask_iter);
3368
+
3369
+ int32_t rc = ggml_thread_create(&workers[j].thrd, NULL, ggml_graph_compute_secondary_thread, &workers[j]);
3370
+ GGML_ASSERT(rc == 0);
3371
+ }
3372
+
3373
+ ggml_thread_cpumask_next(tpp->cpumask, workers[0].cpumask, tpp->strict_cpu, &cpumask_iter);
3374
+
3375
+ if (!threadpool->pause) {
3376
+ // Update main thread prio and affinity at the start, otherwise we'll do it in resume
3377
+ ggml_thread_apply_priority(threadpool->prio);
3378
+ if (ggml_thread_cpumask_is_valid(threadpool->workers[0].cpumask)) {
3379
+ ggml_thread_apply_affinity(threadpool->workers[0].cpumask);
3380
+ }
3381
+ }
3382
+ #endif // GGML_USE_OPENMP
3383
+
3384
+ return threadpool;
3385
+ }
3386
+
3387
+ struct ggml_threadpool * ggml_threadpool_new(struct ggml_threadpool_params * tpp) {
3388
+ return ggml_threadpool_new_impl(tpp, NULL, NULL);
3389
+ }
3390
+
3391
+ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan) {
3392
+ ggml_cpu_init();
3393
+
3394
+ GGML_ASSERT(cplan);
3395
+ GGML_ASSERT(cplan->n_threads > 0);
3396
+ GGML_ASSERT(cplan->work_size == 0 || cplan->work_data != NULL);
3397
+
3398
+ int n_threads = cplan->n_threads;
3399
+ struct ggml_threadpool * threadpool = cplan->threadpool;
3400
+
3401
+ bool disposable_threadpool = false;
3402
+
3403
+ if (threadpool == NULL) {
3404
+ //GGML_PRINT_DEBUG("Threadpool is not specified. Will create a disposable threadpool : n_threads %d\n", n_threads);
3405
+ disposable_threadpool = true;
3406
+
3407
+ struct ggml_threadpool_params ttp = ggml_threadpool_params_default(n_threads);
3408
+ threadpool = ggml_threadpool_new_impl(&ttp, cgraph, cplan);
3409
+ } else {
3410
+ // Reset some of the parameters that need resetting
3411
+ // No worker threads should be accessing the parameters below at this stage
3412
+ threadpool->cgraph = cgraph;
3413
+ threadpool->cplan = cplan;
3414
+ threadpool->current_chunk = 0;
3415
+ threadpool->abort = -1;
3416
+ threadpool->ec = GGML_STATUS_SUCCESS;
3417
+ }
3418
+
3419
+ #ifdef GGML_USE_OPENMP
3420
+ if (n_threads > 1) {
3421
+ #pragma omp parallel num_threads(n_threads)
3422
+ {
3423
+ #pragma omp single
3424
+ {
3425
+ // update the number of threads from the actual number of threads that we got from OpenMP
3426
+ n_threads = omp_get_num_threads();
3427
+ atomic_store_explicit(&threadpool->n_graph, n_threads, memory_order_relaxed);
3428
+ }
3429
+
3430
+ // Apply thread CPU mask and priority
3431
+ int ith = omp_get_thread_num();
3432
+
3433
+ ggml_thread_apply_priority(threadpool->prio);
3434
+ if (ggml_thread_cpumask_is_valid(threadpool->workers[ith].cpumask)) {
3435
+ ggml_thread_apply_affinity(threadpool->workers[ith].cpumask);
3436
+ }
3437
+ ggml_graph_compute_thread(&threadpool->workers[ith]);
3438
+ }
3439
+ } else {
3440
+ atomic_store_explicit(&threadpool->n_graph, 1, memory_order_relaxed);
3441
+ ggml_graph_compute_thread(&threadpool->workers[0]);
3442
+ }
3443
+ #else
3444
+ if (n_threads > threadpool->n_threads) {
3445
+ GGML_LOG_WARN("cplan requested more threads (%d) than available (%d)\n", n_threads, threadpool->n_threads);
3446
+ n_threads = threadpool->n_threads;
3447
+ }
3448
+
3449
+ // Kick all threads to start the new graph
3450
+ ggml_graph_compute_kickoff(threadpool, n_threads);
3451
+
3452
+ // This is a work thread too
3453
+ ggml_graph_compute_thread(&threadpool->workers[0]);
3454
+ #endif
3455
+
3456
+ // don't leave affinity set on the main thread
3457
+ clear_numa_thread_affinity();
3458
+
3459
+ enum ggml_status ret = threadpool->ec;
3460
+
3461
+ if (disposable_threadpool) {
3462
+ ggml_threadpool_free(threadpool);
3463
+ }
3464
+
3465
+ return ret;
3466
+ }
3467
+
3468
+ enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads) {
3469
+ struct ggml_cplan cplan = ggml_graph_plan(cgraph, n_threads, NULL);
3470
+
3471
+ cplan.work_data = (uint8_t *)ggml_new_buffer(ctx, cplan.work_size);
3472
+
3473
+ return ggml_graph_compute(cgraph, &cplan);
3474
+ }
3475
+
3476
+ void ggml_cpu_fp32_to_fp32(const float * x, float * y, int64_t n) {
3477
+ memcpy(y, x, n * sizeof(float));
3478
+ }
3479
+
3480
+ void ggml_cpu_fp32_to_fp16(const float * x, ggml_fp16_t * y, int64_t n) {
3481
+ int64_t i = 0;
3482
+ #if defined(__F16C__)
3483
+ #if defined(__AVX512F__)
3484
+ for (; i + 15 < n; i += 16) {
3485
+ __m512 x_vec = _mm512_loadu_ps(x + i);
3486
+ __m256i y_vec = _mm512_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
3487
+ _mm256_storeu_si256((__m256i *)(y + i), y_vec);
3488
+ }
3489
+ #endif
3490
+ for (; i + 7 < n; i += 8) {
3491
+ __m256 x_vec = _mm256_loadu_ps(x + i);
3492
+ __m128i y_vec = _mm256_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
3493
+ _mm_storeu_si128((__m128i *)(y + i), y_vec);
3494
+ }
3495
+ for (; i + 3 < n; i += 4) {
3496
+ __m128 x_vec = _mm_loadu_ps(x + i);
3497
+ __m128i y_vec = _mm_cvtps_ph(x_vec, _MM_FROUND_TO_NEAREST_INT);
3498
+ _mm_storel_epi64((__m128i *)(y + i), y_vec);
3499
+ }
3500
+ #elif defined(__riscv_zvfh)
3501
+ for (int vl; i < n; i += vl) {
3502
+ vl = __riscv_vsetvl_e32m2(n - i);
3503
+ vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl);
3504
+ vfloat16m1_t vy = __riscv_vfncvt_f_f_w_f16m1(vx, vl);
3505
+ __riscv_vse16_v_f16m1((_Float16 *)&y[i], vy, vl);
3506
+ }
3507
+ #endif
3508
+ for (; i < n; ++i) {
3509
+ y[i] = GGML_CPU_FP32_TO_FP16(x[i]);
3510
+ }
3511
+ }
3512
+
3513
+ void ggml_cpu_fp16_to_fp32(const ggml_fp16_t * x, float * y, int64_t n) {
3514
+ int64_t i = 0;
3515
+ #if defined(__F16C__)
3516
+ #if defined(__AVX512F__)
3517
+ for (; i + 15 < n; i += 16) {
3518
+ __m256i x_vec = _mm256_loadu_si256((const __m256i *)(x + i));
3519
+ __m512 y_vec = _mm512_cvtph_ps(x_vec);
3520
+ _mm512_storeu_ps(y + i, y_vec);
3521
+ }
3522
+ #endif
3523
+ for (; i + 7 < n; i += 8) {
3524
+ __m128i x_vec = _mm_loadu_si128((const __m128i *)(x + i));
3525
+ __m256 y_vec = _mm256_cvtph_ps(x_vec);
3526
+ _mm256_storeu_ps(y + i, y_vec);
3527
+ }
3528
+ for (; i + 3 < n; i += 4) {
3529
+ __m128i x_vec = _mm_loadl_epi64((const __m128i *)(x + i));
3530
+ __m128 y_vec = _mm_cvtph_ps(x_vec);
3531
+ _mm_storeu_ps(y + i, y_vec);
3532
+ }
3533
+
3534
+ #elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfhmin)
3535
+ // calculate step size
3536
+ const int epr = __riscv_vsetvlmax_e16m2();
3537
+ const int step = epr * 2;
3538
+ const int np = (n & ~(step - 1));
3539
+
3540
+ // unroll by 2
3541
+ for (; i < np; i += step) {
3542
+ vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16*)x + i, epr);
3543
+ vfloat32m4_t ay0 = __riscv_vfwcvt_f_f_v_f32m4(ax0, epr);
3544
+ __riscv_vse32_v_f32m4(y + i, ay0, epr);
3545
+
3546
+ vfloat16m2_t ax1 = __riscv_vle16_v_f16m2((const _Float16*)x + i + epr, epr);
3547
+ vfloat32m4_t ay1 = __riscv_vfwcvt_f_f_v_f32m4(ax1, epr);
3548
+ __riscv_vse32_v_f32m4(y + i + epr, ay1, epr);
3549
+ }
3550
+
3551
+ // leftovers
3552
+ int vl;
3553
+ for (i = np; i < n; i += vl) {
3554
+ vl = __riscv_vsetvl_e16m2(n - i);
3555
+ vfloat16m2_t ax0 = __riscv_vle16_v_f16m2((const _Float16*)x + i, vl);
3556
+ vfloat32m4_t ay0 = __riscv_vfwcvt_f_f_v_f32m4(ax0, vl);
3557
+ __riscv_vse32_v_f32m4(y + i, ay0, vl);
3558
+ }
3559
+
3560
+ #endif
3561
+
3562
+ for (; i < n; ++i) {
3563
+ y[i] = GGML_CPU_FP16_TO_FP32(x[i]);
3564
+ }
3565
+ }
3566
+
3567
+ void ggml_cpu_fp32_to_bf16(const float * x, ggml_bf16_t * y, int64_t n) {
3568
+ int64_t i = 0;
3569
+ for (; i < n; ++i) {
3570
+ y[i] = GGML_FP32_TO_BF16(x[i]);
3571
+ }
3572
+ }
3573
+
3574
+ void ggml_cpu_fp32_to_i32(const float * x, int32_t * y, int64_t n) {
3575
+ int64_t i = 0;
3576
+ for (; i < n; ++i) {
3577
+ y[i] = x[i];
3578
+ }
3579
+ }
3580
+
3581
+ void ggml_cpu_bf16_to_fp32(const ggml_bf16_t * x, float * y, int64_t n) {
3582
+ int64_t i = 0;
3583
+ #if defined(__AVX2__)
3584
+ #if defined(__AVX512F__)
3585
+ for (; i + 15 < n; i += 16) {
3586
+ _mm512_storeu_ps(y + i,
3587
+ _mm512_castsi512_ps(
3588
+ _mm512_slli_epi32(
3589
+ _mm512_cvtepu16_epi32(
3590
+ _mm256_loadu_si256(
3591
+ (const __m256i *)(x + i))),
3592
+ 16)));
3593
+ }
3594
+ #endif
3595
+ for (; i + 7 < n; i += 8) {
3596
+ _mm256_storeu_ps(y + i,
3597
+ _mm256_castsi256_ps(
3598
+ _mm256_slli_epi32(
3599
+ _mm256_cvtepu16_epi32(
3600
+ _mm_loadu_si128(
3601
+ (const __m128i *)(x + i))),
3602
+ 16)));
3603
+ }
3604
+ #elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfmin)
3605
+ // calculate step size
3606
+ const int epr = __riscv_vsetvlmax_e16m2();
3607
+ const int step = epr * 2;
3608
+ const int np = (n & ~(step - 1));
3609
+
3610
+ // unroll by 2
3611
+ for (; i < np; i += step) {
3612
+ vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16*)x + i, epr);
3613
+ vfloat32m4_t ay0 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax0, epr);
3614
+ __riscv_vse32_v_f32m4(y + i, ay0, epr);
3615
+
3616
+ vbfloat16m2_t ax1 = __riscv_vle16_v_bf16m2((const __bf16*)x + i + epr, epr);
3617
+ vfloat32m4_t ay1 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax1, epr);
3618
+ __riscv_vse32_v_f32m4(y + i + epr, ay1, epr);
3619
+ }
3620
+
3621
+ // leftovers
3622
+ int vl;
3623
+ for (i = np; i < n; i += vl) {
3624
+ vl = __riscv_vsetvl_e16m2(n - i);
3625
+ vbfloat16m2_t ax0 = __riscv_vle16_v_bf16m2((const __bf16*)x + i, vl);
3626
+ vfloat32m4_t ay0 = __riscv_vfwcvtbf16_f_f_v_f32m4(ax0, vl);
3627
+ __riscv_vse32_v_f32m4(y + i, ay0, vl);
3628
+ }
3629
+ #endif
3630
+ for (; i < n; i++) {
3631
+ y[i] = GGML_BF16_TO_FP32(x[i]);
3632
+ }
3633
+ }
3634
+
3635
+ int ggml_cpu_has_avx(void) {
3636
+ #if defined(__AVX__)
3637
+ return 1;
3638
+ #else
3639
+ return 0;
3640
+ #endif
3641
+ }
3642
+
3643
+ int ggml_cpu_has_avx_vnni(void) {
3644
+ #if defined(__AVXVNNI__)
3645
+ return 1;
3646
+ #else
3647
+ return 0;
3648
+ #endif
3649
+ }
3650
+
3651
+ int ggml_cpu_has_avx2(void) {
3652
+ #if defined(__AVX2__)
3653
+ return 1;
3654
+ #else
3655
+ return 0;
3656
+ #endif
3657
+ }
3658
+
3659
+ int ggml_cpu_has_avx512(void) {
3660
+ #if defined(__AVX512F__)
3661
+ return 1;
3662
+ #else
3663
+ return 0;
3664
+ #endif
3665
+ }
3666
+
3667
+ int ggml_cpu_has_avx512_vbmi(void) {
3668
+ #if defined(__AVX512VBMI__)
3669
+ return 1;
3670
+ #else
3671
+ return 0;
3672
+ #endif
3673
+ }
3674
+
3675
+ int ggml_cpu_has_avx512_vnni(void) {
3676
+ #if defined(__AVX512VNNI__)
3677
+ return 1;
3678
+ #else
3679
+ return 0;
3680
+ #endif
3681
+ }
3682
+
3683
+ int ggml_cpu_has_avx512_bf16(void) {
3684
+ #if defined(__AVX512BF16__)
3685
+ return 1;
3686
+ #else
3687
+ return 0;
3688
+ #endif
3689
+ }
3690
+
3691
+ int ggml_cpu_has_amx_int8(void) {
3692
+ #if defined(__AMX_INT8__)
3693
+ return 1;
3694
+ #else
3695
+ return 0;
3696
+ #endif
3697
+ }
3698
+
3699
+ int ggml_cpu_has_bmi2(void) {
3700
+ #if defined(__BMI2__)
3701
+ return 1;
3702
+ #else
3703
+ return 0;
3704
+ #endif
3705
+ }
3706
+
3707
+ int ggml_cpu_has_fma(void) {
3708
+ #if defined(__FMA__)
3709
+ return 1;
3710
+ #else
3711
+ return 0;
3712
+ #endif
3713
+ }
3714
+
3715
+ int ggml_cpu_has_arm_fma(void) {
3716
+ #if defined(__ARM_FEATURE_FMA)
3717
+ return 1;
3718
+ #else
3719
+ return 0;
3720
+ #endif
3721
+ }
3722
+
3723
+ int ggml_cpu_has_riscv_v(void) {
3724
+ #if defined(__riscv_v_intrinsic)
3725
+ return 1;
3726
+ #else
3727
+ return 0;
3728
+ #endif
3729
+ }
3730
+
3731
+ int ggml_cpu_get_rvv_vlen(void) {
3732
+ #if defined(__riscv) && defined(__riscv_v_intrinsic)
3733
+ return ggml_riscv_arch_features.rvv_vlen;
3734
+ #else
3735
+ return 0;
3736
+ #endif
3737
+ }
3738
+
3739
+ int ggml_cpu_has_f16c(void) {
3740
+ #if defined(__F16C__)
3741
+ return 1;
3742
+ #else
3743
+ return 0;
3744
+ #endif
3745
+ }
3746
+
3747
+ int ggml_cpu_has_fp16_va(void) {
3748
+ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
3749
+ return 1;
3750
+ #else
3751
+ return 0;
3752
+ #endif
3753
+ }
3754
+
3755
+ int ggml_cpu_has_wasm_simd(void) {
3756
+ #if defined(__wasm_simd128__)
3757
+ return 1;
3758
+ #else
3759
+ return 0;
3760
+ #endif
3761
+ }
3762
+
3763
+ int ggml_cpu_has_llamafile(void) {
3764
+ #if defined(GGML_USE_LLAMAFILE)
3765
+ return 1;
3766
+ #else
3767
+ return 0;
3768
+ #endif
3769
+ }
3770
+
3771
+ int ggml_cpu_has_sse3(void) {
3772
+ #if defined(__SSE3__)
3773
+ return 1;
3774
+ #else
3775
+ return 0;
3776
+ #endif
3777
+ }
3778
+
3779
+ int ggml_cpu_has_ssse3(void) {
3780
+ #if defined(__SSSE3__)
3781
+ return 1;
3782
+ #else
3783
+ return 0;
3784
+ #endif
3785
+ }
3786
+
3787
+ int ggml_cpu_has_vsx(void) {
3788
+ #if defined(__POWER9_VECTOR__)
3789
+ return 1;
3790
+ #else
3791
+ return 0;
3792
+ #endif
3793
+ }
3794
+
3795
+ int ggml_cpu_has_vxe(void) {
3796
+ #if defined(__VXE__) || defined(__VXE2__)
3797
+ return 1;
3798
+ #else
3799
+ return 0;
3800
+ #endif
3801
+ }
3802
+
3803
+ int ggml_cpu_has_neon(void) {
3804
+ #if defined(__ARM_ARCH) && defined(__ARM_NEON)
3805
+ return 1;
3806
+ #else
3807
+ return 0;
3808
+ #endif
3809
+ }
3810
+
3811
+ int ggml_cpu_has_dotprod(void) {
3812
+ #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_DOTPROD)
3813
+ return 1;
3814
+ #else
3815
+ return 0;
3816
+ #endif
3817
+ }
3818
+
3819
+ int ggml_cpu_has_sve(void) {
3820
+ #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)
3821
+ return 1;
3822
+ #else
3823
+ return 0;
3824
+ #endif
3825
+ }
3826
+
3827
+ int ggml_cpu_has_matmul_int8(void) {
3828
+ #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_MATMUL_INT8)
3829
+ return 1;
3830
+ #else
3831
+ return 0;
3832
+ #endif
3833
+ }
3834
+
3835
+ int ggml_cpu_get_sve_cnt(void) {
3836
+ #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SVE)
3837
+ return ggml_arm_arch_features.sve_cnt;
3838
+ #else
3839
+ return 0;
3840
+ #endif
3841
+ }
3842
+
3843
+ int ggml_cpu_has_sme(void) {
3844
+ #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SME)
3845
+ return 1;
3846
+ #else
3847
+ return 0;
3848
+ #endif
3849
+ }
3850
+
3851
+ int ggml_cpu_has_sme2(void) {
3852
+ #if defined(__ARM_ARCH) && defined(__ARM_FEATURE_SME2)
3853
+ return 1;
3854
+ #else
3855
+ return 0;
3856
+ #endif
3857
+ }
3858
+
3859
+ void ggml_cpu_init(void) {
3860
+ // needed to initialize ggml_time
3861
+ {
3862
+ struct ggml_init_params params = { 0, NULL, false };
3863
+ struct ggml_context * ctx = ggml_init(params);
3864
+ ggml_free(ctx);
3865
+ }
3866
+
3867
+ ggml_critical_section_start();
3868
+
3869
+ static bool is_first_call = true;
3870
+
3871
+ if (is_first_call) {
3872
+ // initialize GELU, Quick GELU, SILU and EXP F32 tables
3873
+ {
3874
+ const uint64_t t_start = ggml_time_us(); UNUSED(t_start);
3875
+
3876
+ for (int i = 0; i < (1 << 16); ++i) {
3877
+ union {
3878
+ uint16_t u16;
3879
+ ggml_fp16_t fp16;
3880
+ } u = {i};
3881
+ float f = GGML_COMPUTE_FP16_TO_FP32(u.fp16);
3882
+ ggml_table_f32_f16[i] = f;
3883
+ ggml_table_gelu_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_f32(f));
3884
+ ggml_table_gelu_quick_f16[i] = GGML_CPU_FP32_TO_FP16(ggml_gelu_quick_f32(f));
3885
+ }
3886
+
3887
+ // initialize E8M0 half table (256 entries)
3888
+ for (int i = 0; i < (1 << 8); ++i) {
3889
+ ggml_table_f32_e8m0_half[i] = GGML_E8M0_TO_FP32_HALF(i);
3890
+ }
3891
+
3892
+ // initialize UE4M3 table (256 entries)
3893
+ for (int i = 0; i < (1 << 8); ++i) {
3894
+ ggml_table_f32_ue4m3[i] = ggml_ue4m3_to_fp32(i);
3895
+ }
3896
+
3897
+ const uint64_t t_end = ggml_time_us(); UNUSED(t_end);
3898
+
3899
+ GGML_PRINT_DEBUG("%s: GELU, Quick GELU, SILU and EXP tables initialized in %f ms\n", __func__, (t_end - t_start)/1000.0);
3900
+
3901
+ #ifdef GGML_USE_OPENMP
3902
+ //if (!getenv("OMP_WAIT_POLICY")) {
3903
+ // // set the wait policy to active, so that OpenMP threads don't sleep
3904
+ // setenv("OMP_WAIT_POLICY", "active", 0)
3905
+ //}
3906
+
3907
+ if (!getenv("KMP_BLOCKTIME")) {
3908
+ // set the time to wait before sleeping a thread
3909
+ // this is less aggressive than setting the wait policy to active, but should achieve similar results in most cases
3910
+ #ifdef _WIN32
3911
+ _putenv_s("KMP_BLOCKTIME", "200"); // 200ms
3912
+ #else
3913
+ setenv("KMP_BLOCKTIME", "200", 0); // 200ms
3914
+ #endif
3915
+ }
3916
+ #endif
3917
+ }
3918
+
3919
+ #if defined(__ARM_ARCH)
3920
+ ggml_init_arm_arch_features();
3921
+ #endif
3922
+
3923
+ #if defined(__riscv)
3924
+ ggml_init_riscv_arch_features();
3925
+ #endif
3926
+
3927
+ {
3928
+ const char * env = getenv("GGML_CPU_DISABLE_FUSION");
3929
+ ggml_cpu_disable_fusion = (env != NULL && atoi(env) == 1);
3930
+ }
3931
+
3932
+ is_first_call = false;
3933
+ }
3934
+
3935
+ ggml_critical_section_end();
3936
+ }