elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,2473 @@
1
+ #import "ggml-metal-device.h"
2
+
3
+ #import "ggml-impl.h"
4
+ #import "ggml-backend-impl.h"
5
+ #import "ggml-metal-impl.h"
6
+ #import "ggml-metal-common.h"
7
+
8
+ #include <Foundation/Foundation.h>
9
+
10
+ #include <Metal/Metal.h>
11
+
12
+ #include <stdatomic.h>
13
+
14
+ #ifndef TARGET_OS_VISION
15
+ #define TARGET_OS_VISION 0
16
+ #endif
17
+
18
+ // create residency sets only on macOS >= 15.0
19
+ #if !TARGET_CPU_X86_64 && TARGET_OS_OSX && __MAC_OS_X_VERSION_MAX_ALLOWED >= 150000 || \
20
+ TARGET_OS_IOS && __IPHONE_OS_VERSION_MAX_ALLOWED >= 180000 || \
21
+ TARGET_OS_TV && __TV_OS_VERSION_MAX_ALLOWED >= 180000 || \
22
+ TARGET_OS_VISION && __VISION_OS_VERSION_MAX_ALLOWED >= 200000
23
+ #define GGML_METAL_HAS_RESIDENCY_SETS 1
24
+ #endif
25
+
26
+ // overload of MTLGPUFamilyMetalX (not available in some environments)
27
+ static const NSInteger MTLGPUFamilyMetal3_GGML = 5001;
28
+ static const NSInteger MTLGPUFamilyMetal4_GGML = 5002;
29
+
30
+ // MTLLanguageVersion4_0 is not present in older SDKs
31
+ static const NSUInteger MTLLanguageVersion4_0_GGML = 4 << 16;
32
+
33
+ #if !GGML_METAL_EMBED_LIBRARY
34
+ // Here to assist with NSBundle Path Hack
35
+ @interface GGMLMetalClass : NSObject
36
+ @end
37
+ @implementation GGMLMetalClass
38
+ @end
39
+ #endif
40
+
41
+ //
42
+ // MTLFunctionConstantValues wrapper
43
+ //
44
+
45
+ struct ggml_metal_cv {
46
+ MTLFunctionConstantValues * obj;
47
+ };
48
+
49
+ ggml_metal_cv_t ggml_metal_cv_init(void) {
50
+ ggml_metal_cv_t res = calloc(1, sizeof(struct ggml_metal_cv));
51
+
52
+ res->obj = [[MTLFunctionConstantValues alloc] init];
53
+
54
+ return res;
55
+ }
56
+
57
+ void ggml_metal_cv_free(ggml_metal_cv_t cv) {
58
+ [cv->obj release];
59
+ free(cv);
60
+ }
61
+
62
+ void ggml_metal_cv_set_int16(ggml_metal_cv_t cv, int16_t value, int32_t idx) {
63
+ [cv->obj setConstantValue:&value type:MTLDataTypeShort atIndex:idx];
64
+ }
65
+
66
+ void ggml_metal_cv_set_int32(ggml_metal_cv_t cv, int32_t value, int32_t idx) {
67
+ [cv->obj setConstantValue:&value type:MTLDataTypeInt atIndex:idx];
68
+ }
69
+
70
+ void ggml_metal_cv_set_bool(ggml_metal_cv_t cv, bool value, int32_t idx) {
71
+ [cv->obj setConstantValue:&value type:MTLDataTypeBool atIndex:idx];
72
+ }
73
+
74
+ //
75
+ // MTLComputePipelineState wrapper
76
+ //
77
+
78
+ struct ggml_metal_pipeline {
79
+ id<MTLComputePipelineState> obj;
80
+ };
81
+
82
+ ggml_metal_pipeline_t ggml_metal_pipeline_init(void) {
83
+ ggml_metal_pipeline_t res = calloc(1, sizeof(struct ggml_metal_pipeline));
84
+
85
+ *res = (struct ggml_metal_pipeline) {
86
+ /*.obj =*/ nil,
87
+ };
88
+
89
+ return res;
90
+ }
91
+
92
+ void ggml_metal_pipeline_free(ggml_metal_pipeline_t pipeline) {
93
+ [pipeline->obj release];
94
+
95
+ free(pipeline);
96
+ }
97
+
98
+ int ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_with_params pipeline) {
99
+ return pipeline.pipeline->obj.maxTotalThreadsPerThreadgroup;
100
+ }
101
+
102
+ //
103
+ // MTLLibrary collection (one library per op-source, compiled separately)
104
+ //
105
+
106
+ // Single source of truth for the per-kind metal libraries. The order here
107
+ // defines the enum values and every per-kind table below, so adding a library
108
+ // is a one-line change here (plus adding its source to CMakeLists.txt).
109
+ // X(suffix, name): name is both the kernels/<name>.metal basename and the
110
+ // ggml_metallib_<name>_{start,end} embed-symbol stem.
111
+ #define GGML_METAL_LIBS \
112
+ X(FA, fa) \
113
+ X(MUL_MV, mul_mv) \
114
+ X(MUL_MM, mul_mm) \
115
+ X(QUANTIZE, quantize) \
116
+ X(SOFTMAX, softmax) \
117
+ X(NORM, norm) \
118
+ X(UNARY, unary) \
119
+ X(BINBCAST, binbcast) \
120
+ X(REDUCE, reduce) \
121
+ X(TRI, tri) \
122
+ X(SSM, ssm) \
123
+ X(WKV, wkv) \
124
+ X(GATED_DELTA_NET, gated_delta_net)\
125
+ X(SOLVE_TRI, solve_tri) \
126
+ X(ROPE, rope) \
127
+ X(CONV, conv) \
128
+ X(UPSCALE, upscale) \
129
+ X(ARGSORT, argsort) \
130
+ X(POOL, pool) \
131
+ X(MISC, misc)
132
+
133
+ enum ggml_metal_lib_kind {
134
+ #define X(e, s) GGML_METAL_LIB_##e,
135
+ GGML_METAL_LIBS
136
+ #undef X
137
+ GGML_METAL_LIB_COUNT,
138
+ };
139
+
140
+ static const char * const k_lib_names[GGML_METAL_LIB_COUNT] = {
141
+ #define X(e, s) [GGML_METAL_LIB_##e] = #s,
142
+ GGML_METAL_LIBS
143
+ #undef X
144
+ };
145
+
146
+ struct ggml_metal_library {
147
+ // Per-kind compiled libraries. When single_library is true, the whole library
148
+ // (e.g. a pre-compiled default.metallib or a from-source build) lives at
149
+ // objs[0] and the remaining slots are nil.
150
+ id<MTLLibrary> objs[GGML_METAL_LIB_COUNT];
151
+ bool single_library; // true: combined library at objs[0]; false: per-kind libs in objs[*]
152
+
153
+ // Routing table: kernel function name -> objs[] index, populated from each
154
+ // compiled library's -[MTLLibrary functionNames]. The actual compiled
155
+ // libraries are the single source of truth for which library owns a kernel,
156
+ // so adding kernels later requires no manual routing maintenance.
157
+ // nil in single_library mode (everything resolves to objs[0]).
158
+ NSMutableDictionary<NSString *, NSNumber *> * fn_to_lib;
159
+
160
+ // kernels from a second metallib, resolved ahead of the combined library
161
+ NSSet<NSString *> * override_fns;
162
+
163
+ ggml_metal_device_t dev;
164
+ ggml_metal_pipelines_t pipelines; // cache of compiled pipelines
165
+
166
+ NSLock * lock;
167
+ };
168
+
169
+ // Build the fn_to_lib routing table by querying each compiled library's public
170
+ // function names. Call once after all per-kind libraries have been compiled.
171
+ static void ggml_metal_library_build_index(ggml_metal_library_t lib) {
172
+ @autoreleasepool {
173
+ NSMutableDictionary<NSString *, NSNumber *> * index = [[NSMutableDictionary alloc] init];
174
+ for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
175
+ for (NSString * fname in [lib->objs[kind] functionNames]) {
176
+ index[fname] = @(kind);
177
+ }
178
+ }
179
+ lib->fn_to_lib = index;
180
+ }
181
+ }
182
+
183
+ // note: defined below, after struct ggml_metal_device
184
+ static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev);
185
+
186
+ // the tensor API headers are exposed to the shader compiler only at Metal language version 4.0
187
+ static void ggml_metal_compile_options_set_lang(MTLCompileOptions * options, bool has_tensor) {
188
+ if (!has_tensor) {
189
+ return;
190
+ }
191
+
192
+ options.languageVersion = (MTLLanguageVersion) MTLLanguageVersion4_0_GGML;
193
+ }
194
+
195
+ // Parse a `#include "name"` line. Returns the quoted name in *include_name on
196
+ // success. Whitespace-tolerant; ignores `#include <...>` (system headers).
197
+ static bool ggml_metal_library_parse_quoted_include(NSString * line, NSString ** include_name) {
198
+ NSScanner * scanner = [NSScanner scannerWithString:line];
199
+ scanner.charactersToBeSkipped = [NSCharacterSet whitespaceCharacterSet];
200
+
201
+ if (![scanner scanString:@"#" intoString:NULL] ||
202
+ ![scanner scanString:@"include" intoString:NULL] ||
203
+ ![scanner scanString:@"\"" intoString:NULL]) {
204
+ return false;
205
+ }
206
+
207
+ NSString * name = nil;
208
+ if (![scanner scanUpToString:@"\"" intoString:&name]) {
209
+ return false;
210
+ }
211
+
212
+ if (include_name) {
213
+ *include_name = name;
214
+ }
215
+ return true;
216
+ }
217
+
218
+ // Recursively inline `#include "name"` directives. System includes (<...>),
219
+ // `#if/#else/#endif`, and other preprocessor lines are passed through to the
220
+ // Metal compiler unchanged. `#pragma once` is dropped since `seen` already
221
+ // guards against double-inclusion.
222
+ static bool ggml_metal_library_flatten_file(NSMutableString * dst, NSString * path,
223
+ NSArray<NSString *> * search_paths,
224
+ NSMutableSet<NSString *> * seen, NSError ** error) {
225
+ NSString * key = [path stringByStandardizingPath];
226
+ if ([seen containsObject:key]) {
227
+ return true;
228
+ }
229
+ [seen addObject:key];
230
+
231
+ NSString * src = [NSString stringWithContentsOfFile:path encoding:NSUTF8StringEncoding error:error];
232
+ if (!src) {
233
+ return false;
234
+ }
235
+
236
+ NSFileManager * fm = [NSFileManager defaultManager];
237
+ for (NSString * line in [src componentsSeparatedByString:@"\n"]) {
238
+ NSString * trimmed = [line stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
239
+ if ([trimmed isEqualToString:@"#pragma once"]) {
240
+ continue;
241
+ }
242
+
243
+ NSString * include_name = nil;
244
+ if (ggml_metal_library_parse_quoted_include(line, &include_name)) {
245
+ NSString * resolved = nil;
246
+ for (NSString * dir in search_paths) {
247
+ NSString * candidate = [dir stringByAppendingPathComponent:include_name];
248
+ if ([fm isReadableFileAtPath:candidate]) {
249
+ resolved = candidate;
250
+ break;
251
+ }
252
+ }
253
+ if (!resolved) {
254
+ if (error) {
255
+ NSString * msg = [NSString stringWithFormat:@"could not resolve include \"%@\" from '%@'", include_name, path];
256
+ *error = [NSError errorWithDomain:@"ggml-metal-source-flatten" code:1
257
+ userInfo:@{NSLocalizedDescriptionKey: msg}];
258
+ }
259
+ return false;
260
+ }
261
+ if (!ggml_metal_library_flatten_file(dst, resolved, search_paths, seen, error)) {
262
+ return false;
263
+ }
264
+ continue;
265
+ }
266
+
267
+ [dst appendString:line];
268
+ [dst appendString:@"\n"];
269
+ }
270
+
271
+ return true;
272
+ }
273
+
274
+ static NSString * ggml_metal_library_flatten_source(NSString * path_source, NSError ** error) {
275
+ // Search paths cover both runtime layout (build/bin/kernels + build/bin)
276
+ // and source-tree layout (ggml/src/ggml-metal/kernels + ggml/src/ggml-metal + ggml/src).
277
+ NSString * path_kernels = [path_source stringByDeletingLastPathComponent];
278
+ NSString * path_base = [path_kernels stringByDeletingLastPathComponent];
279
+ NSArray<NSString *> * search_paths = @[
280
+ path_kernels,
281
+ path_base,
282
+ [path_base stringByDeletingLastPathComponent],
283
+ ];
284
+
285
+ NSMutableString * src = [[NSMutableString alloc] init];
286
+ NSMutableSet<NSString *> * seen = [NSMutableSet set];
287
+
288
+ if (!ggml_metal_library_flatten_file(src, path_source, search_paths, seen, error)) {
289
+ [src release];
290
+ return nil;
291
+ }
292
+ return src;
293
+ }
294
+
295
+ // Compile all per-kind libraries in parallel. `source_for_kind` returns the MSL
296
+ // source for a kind (the helper takes ownership and releases it), or nil with
297
+ // *err set on failure. On success the objs[] slots are populated and the routing
298
+ // index is built; on any failure every error is logged and false is returned
299
+ // (the caller is responsible for freeing `res`).
300
+ static bool ggml_metal_library_compile_all(
301
+ ggml_metal_library_t res,
302
+ id<MTLDevice> device,
303
+ NSDictionary * prep,
304
+ NSString * (^source_for_kind)(int kind, NSError ** err),
305
+ const char * origin) {
306
+ const int64_t t_start = ggml_time_us();
307
+
308
+ int64_t * t_per_lib = calloc(GGML_METAL_LIB_COUNT, sizeof(int64_t));
309
+ NSError ** err_per_lib = calloc(GGML_METAL_LIB_COUNT, sizeof(NSError *));
310
+ __block atomic_bool any_failure = false;
311
+
312
+ dispatch_group_t group = dispatch_group_create();
313
+ dispatch_queue_t queue = dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0);
314
+
315
+ for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
316
+ dispatch_group_async(group, queue, ^{
317
+
318
+ const int64_t t0 = ggml_time_us();
319
+
320
+ NSError * error = nil;
321
+
322
+ NSString * src = source_for_kind(kind, &error);
323
+ if (!src) {
324
+ err_per_lib[kind] = [error retain];
325
+ atomic_store(&any_failure, true);
326
+ return;
327
+ }
328
+
329
+ id<MTLLibrary> lib = nil;
330
+
331
+ @autoreleasepool {
332
+ MTLCompileOptions * options = [MTLCompileOptions new];
333
+ options.preprocessorMacros = prep;
334
+ ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(res->dev)->has_tensor);
335
+
336
+ lib = [device newLibraryWithSource:src options:options error:&error];
337
+
338
+ [options release];
339
+
340
+ // retain the error before the autorelease pool drains it
341
+ if (!lib) {
342
+ err_per_lib[kind] = [error retain];
343
+ }
344
+ }
345
+
346
+ [src release];
347
+
348
+ t_per_lib[kind] = ggml_time_us() - t0;
349
+
350
+ if (!lib) {
351
+ atomic_store(&any_failure, true);
352
+ return;
353
+ }
354
+
355
+ res->objs[kind] = lib;
356
+ });
357
+ }
358
+ dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
359
+ dispatch_release(group);
360
+
361
+ const bool ok = !atomic_load(&any_failure);
362
+
363
+ if (ok) {
364
+ const int64_t t_total = ggml_time_us() - t_start;
365
+ int64_t t_max = 0;
366
+ for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
367
+ GGML_LOG_DEBUG("%s: compiled '%s' library in %.3f sec\n",
368
+ __func__, k_lib_names[kind], t_per_lib[kind] / 1e6);
369
+ if (t_per_lib[kind] > t_max) t_max = t_per_lib[kind];
370
+ }
371
+ GGML_LOG_INFO("%s: loaded %d libraries from %s in %.3f sec (max single = %.3f sec)\n",
372
+ __func__, GGML_METAL_LIB_COUNT, origin, t_total / 1e6, t_max / 1e6);
373
+
374
+ ggml_metal_library_build_index(res);
375
+ } else {
376
+ for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
377
+ if (err_per_lib[kind]) {
378
+ GGML_LOG_ERROR("%s: failed to build '%s' library: %s\n", __func__,
379
+ k_lib_names[kind], [[err_per_lib[kind] description] UTF8String]);
380
+ [err_per_lib[kind] release];
381
+ }
382
+ }
383
+ }
384
+
385
+ free(err_per_lib);
386
+ free(t_per_lib);
387
+
388
+ return ok;
389
+ }
390
+
391
+ // look for <name>.metallib as a bundle resource, then next to the running binary
392
+ static NSString * ggml_metal_find_metallib(NSBundle * bundle, NSString * name) {
393
+ NSError * error = nil;
394
+
395
+ NSString * path_lib = [bundle pathForResource:name ofType:@"metallib"];
396
+ if (path_lib == nil) {
397
+ // Try to find the resource in the directory where the current binary located.
398
+ NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0];
399
+ NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent];
400
+
401
+ NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, [name stringByAppendingPathExtension:@"metallib"]]];
402
+ if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
403
+ GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]);
404
+
405
+ NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error];
406
+ if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) {
407
+ // Optionally, if this is a symlink, try to resolve it.
408
+ path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error];
409
+ if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) {
410
+ // It is a relative path, adding the binary directory as directory prefix.
411
+ path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]];
412
+ }
413
+ if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
414
+ // Link to the resource could not be resolved.
415
+ path_lib_default = nil;
416
+ } else {
417
+ GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]);
418
+ }
419
+ }
420
+ } else {
421
+ // The resource couldn't be found in the binary's directory.
422
+ path_lib_default = nil;
423
+ }
424
+
425
+ path_lib = path_lib_default;
426
+ }
427
+
428
+ return path_lib;
429
+ }
430
+
431
+ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
432
+ id<MTLDevice> device = ggml_metal_device_get_obj(dev);
433
+
434
+ ggml_metal_library_t res = calloc(1, sizeof(struct ggml_metal_library));
435
+ res->dev = dev;
436
+ res->pipelines = ggml_metal_pipelines_init();
437
+ res->lock = [NSLock new];
438
+
439
+ // shared MTLCompileOptions preprocessor macros (matches the build-time defines)
440
+ NSMutableDictionary * prep = [NSMutableDictionary dictionary];
441
+ if (ggml_metal_device_get_props(dev)->has_bfloat) {
442
+ [prep setObject:@"1" forKey:@"GGML_METAL_HAS_BF16"];
443
+ }
444
+ if (ggml_metal_device_get_props(dev)->has_tensor) {
445
+ [prep setObject:@"1" forKey:@"GGML_METAL_HAS_TENSOR"];
446
+ }
447
+ #if GGML_METAL_EMBED_LIBRARY
448
+ [prep setObject:@"1" forKey:@"GGML_METAL_EMBED_LIBRARY"];
449
+ #endif
450
+
451
+ #if GGML_METAL_EMBED_LIBRARY
452
+ GGML_LOG_INFO("%s: using embedded metal library\n", __func__);
453
+
454
+ // start/end symbols emitted by CMake (see CMakeLists.txt), one pair per kind
455
+ #define X(e, s) extern const char ggml_metallib_##s##_start[]; extern const char ggml_metallib_##s##_end[];
456
+ GGML_METAL_LIBS
457
+ #undef X
458
+
459
+ static const char * const lib_start[GGML_METAL_LIB_COUNT] = {
460
+ #define X(e, s) [GGML_METAL_LIB_##e] = ggml_metallib_##s##_start,
461
+ GGML_METAL_LIBS
462
+ #undef X
463
+ };
464
+ static const char * const lib_end[GGML_METAL_LIB_COUNT] = {
465
+ #define X(e, s) [GGML_METAL_LIB_##e] = ggml_metallib_##s##_end,
466
+ GGML_METAL_LIBS
467
+ #undef X
468
+ };
469
+
470
+ const bool ok = ggml_metal_library_compile_all(res, device, prep,
471
+ ^NSString * (int kind, NSError ** err) {
472
+ (void) err;
473
+ return [[NSString alloc] initWithBytes:lib_start[kind]
474
+ length:(lib_end[kind] - lib_start[kind])
475
+ encoding:NSUTF8StringEncoding];
476
+ }, "embedded data");
477
+
478
+ if (!ok) {
479
+ ggml_metal_library_free(res);
480
+ return NULL;
481
+ }
482
+
483
+ return res;
484
+ #else
485
+ #ifdef SWIFT_PACKAGE
486
+ NSBundle * bundle = SWIFTPM_MODULE_BUNDLE;
487
+ #else
488
+ NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]];
489
+ #endif
490
+
491
+ const int64_t t_start = ggml_time_us();
492
+
493
+ NSError * error = nil;
494
+ NSString * path_lib = ggml_metal_find_metallib(bundle, @"default");
495
+
496
+ if (path_lib != nil) {
497
+ // pre-compiled library found: a single combined default.metallib
498
+ NSURL * libURL = [NSURL fileURLWithPath:path_lib];
499
+ GGML_LOG_INFO("%s: loading '%s'\n", __func__, [path_lib UTF8String]);
500
+
501
+ res->objs[0] = [device newLibraryWithURL:libURL error:&error];
502
+ res->single_library = true;
503
+ if (!res->objs[0]) {
504
+ GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
505
+ ggml_metal_library_free(res);
506
+ return NULL;
507
+ }
508
+
509
+ // the tensor API kernels are built into a separate metallib
510
+ if (ggml_metal_device_get_props(dev)->has_tensor) {
511
+ NSString * path_mm = ggml_metal_find_metallib(bundle, @"ggml-tensor");
512
+
513
+ id<MTLLibrary> lib_mm = nil;
514
+ if (path_mm != nil) {
515
+ lib_mm = [device newLibraryWithURL:[NSURL fileURLWithPath:path_mm] error:&error];
516
+ if (!lib_mm && error) {
517
+ GGML_LOG_ERROR("%s: %s\n", __func__, [[error description] UTF8String]);
518
+ }
519
+ }
520
+
521
+ if (lib_mm) {
522
+ GGML_LOG_INFO("%s: loaded '%s'\n", __func__, [path_mm UTF8String]);
523
+
524
+ res->objs[GGML_METAL_LIB_MUL_MM] = [lib_mm retain];
525
+ res->override_fns = [[NSSet setWithArray:[lib_mm functionNames]] retain];
526
+ } else {
527
+ GGML_LOG_INFO("%s: ggml-tensor.metallib not found - disabling the tensor API\n", __func__);
528
+
529
+ ggml_metal_device_disable_tensor(dev);
530
+ }
531
+ }
532
+
533
+ GGML_LOG_INFO("%s: loaded in %.3f sec\n", __func__, (ggml_time_us() - t_start) / 1e6);
534
+ return res;
535
+ }
536
+
537
+ // no pre-compiled metallib: fall back to compiling each kernel source separately
538
+ GGML_LOG_INFO("%s: default.metallib not found, loading kernel sources\n", __func__);
539
+
540
+ NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"];
541
+ if (path_resource) {
542
+ GGML_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, [path_resource UTF8String]);
543
+ }
544
+
545
+ // resolve each kind's source path up front (file lookup/logging stays on the calling thread)
546
+ NSString ** path_per_kind = calloc(GGML_METAL_LIB_COUNT, sizeof(NSString *));
547
+ for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
548
+ NSString * rel = [NSString stringWithFormat:@"kernels/%s.metal", k_lib_names[kind]];
549
+
550
+ NSString * path_source = nil;
551
+ if (path_resource) {
552
+ path_source = [path_resource stringByAppendingPathComponent:rel];
553
+ } else {
554
+ NSString * stem = [NSString stringWithFormat:@"kernels/%s", k_lib_names[kind]];
555
+ path_source = [bundle pathForResource:stem ofType:@"metal"];
556
+ }
557
+
558
+ if (path_source == nil || ![[NSFileManager defaultManager] isReadableFileAtPath:path_source]) {
559
+ GGML_LOG_WARN("%s: could not locate %s in bundle, falling back to cwd\n", __func__, [rel UTF8String]);
560
+ path_source = rel;
561
+ }
562
+
563
+ GGML_LOG_DEBUG("%s: loading '%s'\n", __func__, [path_source UTF8String]);
564
+
565
+ path_per_kind[kind] = [path_source retain];
566
+ }
567
+
568
+ const bool ok = ggml_metal_library_compile_all(res, device, prep,
569
+ ^NSString * (int kind, NSError ** err) {
570
+ return ggml_metal_library_flatten_source(path_per_kind[kind], err);
571
+ }, "source");
572
+
573
+ for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
574
+ [path_per_kind[kind] release];
575
+ }
576
+ free(path_per_kind);
577
+
578
+ if (!ok) {
579
+ ggml_metal_library_free(res);
580
+ return NULL;
581
+ }
582
+
583
+ return res;
584
+ #endif
585
+ }
586
+
587
+ ggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev, const char * source, bool verbose) {
588
+ if (source == NULL) {
589
+ GGML_LOG_ERROR("%s: source is NULL\n", __func__);
590
+ return NULL;
591
+ }
592
+
593
+ id<MTLDevice> device = ggml_metal_device_get_obj(dev);
594
+ id<MTLLibrary> library = nil;
595
+ NSError * error = nil;
596
+
597
+ const int64_t t_start = ggml_time_us();
598
+
599
+ NSString * src = [[NSString alloc] initWithBytes:source
600
+ length:strlen(source)
601
+ encoding:NSUTF8StringEncoding];
602
+ if (!src) {
603
+ GGML_LOG_ERROR("%s: failed to create NSString from source\n", __func__);
604
+ return NULL;
605
+ }
606
+
607
+ @autoreleasepool {
608
+ NSMutableDictionary * prep = [NSMutableDictionary dictionary];
609
+
610
+ MTLCompileOptions * options = [MTLCompileOptions new];
611
+ options.preprocessorMacros = prep;
612
+ ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(dev)->has_tensor);
613
+
614
+ library = [device newLibraryWithSource:src options:options error:&error];
615
+ if (error) {
616
+ if (verbose) {
617
+ GGML_LOG_ERROR("%s: error compiling source: %s\n", __func__, [[error description] UTF8String]);
618
+ } else {
619
+ GGML_LOG_ERROR("%s: error compiling source\n", __func__);
620
+ }
621
+ library = nil;
622
+ }
623
+
624
+ [options release];
625
+ }
626
+
627
+ [src release];
628
+
629
+ if (!library) {
630
+ if (verbose) {
631
+ GGML_LOG_ERROR("%s: failed to create Metal library from source\n", __func__);
632
+ }
633
+
634
+ return NULL;
635
+ }
636
+
637
+ if (verbose) {
638
+ GGML_LOG_INFO("%s: compiled in %.3f sec\n", __func__, (ggml_time_us() - t_start) / 1e6);
639
+ }
640
+
641
+ ggml_metal_library_t res = calloc(1, sizeof(struct ggml_metal_library));
642
+ if (!res) {
643
+ GGML_LOG_ERROR("%s: calloc failed\n", __func__);
644
+ return NULL;
645
+ }
646
+
647
+ res->objs[0] = library;
648
+ res->single_library = true;
649
+ res->dev = dev;
650
+ res->pipelines = ggml_metal_pipelines_init();
651
+ res->lock = [NSLock new];
652
+
653
+ return res;
654
+ }
655
+
656
+ void ggml_metal_library_free(ggml_metal_library_t lib) {
657
+ if (!lib) {
658
+ return;
659
+ }
660
+
661
+ for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
662
+ if (lib->objs[kind]) {
663
+ [lib->objs[kind] release];
664
+ }
665
+ }
666
+
667
+ if (lib->fn_to_lib) {
668
+ [lib->fn_to_lib release];
669
+ }
670
+
671
+ if (lib->override_fns) {
672
+ [lib->override_fns release];
673
+ }
674
+
675
+ ggml_metal_pipelines_free(lib->pipelines);
676
+
677
+ [lib->lock release];
678
+
679
+ free(lib);
680
+ }
681
+
682
+ ggml_metal_device_t ggml_metal_library_get_device(ggml_metal_library_t lib) {
683
+ return lib->dev;
684
+ }
685
+
686
+ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline(ggml_metal_library_t lib, const char * name) {
687
+ [lib->lock lock];
688
+
689
+ struct ggml_metal_pipeline_with_params res = {
690
+ /*.pipeline =*/ nil,
691
+ /*.nsg =*/ 0,
692
+ /*.nr0 =*/ 0,
693
+ /*.nr1 =*/ 0,
694
+ /*.smem =*/ 0,
695
+ /*.c4 =*/ false,
696
+ /*.cnt =*/ false,
697
+ };
698
+
699
+ res.pipeline = ggml_metal_pipelines_get(lib->pipelines, name);
700
+
701
+ [lib->lock unlock];
702
+
703
+ return res;
704
+ }
705
+
706
+ struct ggml_metal_pipeline_with_params ggml_metal_library_compile_pipeline(ggml_metal_library_t lib, const char * base, const char * name, ggml_metal_cv_t cv) {
707
+ struct ggml_metal_pipeline_with_params res = {
708
+ /*.pipeline =*/ nil,
709
+ /*.nsg =*/ 0,
710
+ /*.nr0 =*/ 0,
711
+ /*.nr1 =*/ 0,
712
+ /*.smem =*/ 0,
713
+ /*.c4 =*/ false,
714
+ /*.cnt =*/ false,
715
+ };
716
+
717
+ [lib->lock lock];
718
+
719
+ res.pipeline = ggml_metal_pipelines_get(lib->pipelines, name);
720
+ if (res.pipeline) {
721
+ [lib->lock unlock];
722
+
723
+ return res;
724
+ }
725
+
726
+ @autoreleasepool {
727
+ NSError * error = nil;
728
+
729
+ NSString * base_func = [NSString stringWithUTF8String:base];
730
+
731
+ GGML_LOG_DEBUG("%s: compiling pipeline: base = '%s', name = '%s'\n", __func__, base, name);
732
+
733
+ // route to the library that actually defines this kernel; fn_to_lib is
734
+ // built from -[MTLLibrary functionNames] so it's always in sync
735
+ int lib_idx = 0;
736
+ if (lib->override_fns && [lib->override_fns containsObject:base_func]) {
737
+ lib_idx = GGML_METAL_LIB_MUL_MM;
738
+ } else if (!lib->single_library) {
739
+ NSNumber * idx = lib->fn_to_lib[base_func];
740
+ if (!idx) {
741
+ [lib->lock unlock];
742
+
743
+ GGML_LOG_ERROR("%s: kernel not found in any metal library: base = '%s', name = '%s'\n", __func__, base, name);
744
+
745
+ return res;
746
+ }
747
+ lib_idx = [idx intValue];
748
+ }
749
+
750
+ id<MTLLibrary> mtl_lib = lib->objs[lib_idx];
751
+
752
+ id<MTLFunction> mtl_function;
753
+ if (!cv) {
754
+ mtl_function = [mtl_lib newFunctionWithName:base_func];
755
+ } else {
756
+ mtl_function = [mtl_lib newFunctionWithName:base_func constantValues:cv->obj error:&error];
757
+ }
758
+ if (!mtl_function) {
759
+ [lib->lock unlock];
760
+
761
+ GGML_LOG_ERROR("%s: failed to compile pipeline: base = '%s', name = '%s'\n", __func__, base, name);
762
+ if (error) {
763
+ GGML_LOG_ERROR("%s: %s\n", __func__, [[error description] UTF8String]);
764
+ }
765
+
766
+ return res;
767
+ }
768
+
769
+ id<MTLDevice> device = ggml_metal_device_get_obj(lib->dev);
770
+ id<MTLComputePipelineState> obj = [device newComputePipelineStateWithFunction:mtl_function error:&error];
771
+
772
+ [mtl_function release];
773
+
774
+ if (!obj) {
775
+ [lib->lock unlock];
776
+
777
+ GGML_LOG_ERROR("%s: failed to create pipeline state: base = '%s', name = '%s'\n", __func__, base, name);
778
+ if (error) {
779
+ GGML_LOG_ERROR("%s: %s\n", __func__, [[error description] UTF8String]);
780
+ }
781
+
782
+ return res;
783
+ }
784
+
785
+ GGML_LOG_DEBUG("%s: loaded %-40s %16p | th_max = %4d | th_width = %4d\n", __func__, name,
786
+ (void *) obj,
787
+ (int) obj.maxTotalThreadsPerThreadgroup,
788
+ (int) obj.threadExecutionWidth);
789
+
790
+ if (obj.maxTotalThreadsPerThreadgroup == 0 || obj.threadExecutionWidth == 0) {
791
+ [obj release];
792
+
793
+ [lib->lock unlock];
794
+
795
+ GGML_LOG_ERROR("%s: incompatible pipeline %s\n", __func__, name);
796
+
797
+ return res;
798
+ }
799
+
800
+ res.pipeline = ggml_metal_pipeline_init();
801
+ res.pipeline->obj = obj;
802
+
803
+ ggml_metal_pipelines_add(lib->pipelines, name, res.pipeline);
804
+ }
805
+
806
+ [lib->lock unlock];
807
+
808
+ return res;
809
+ }
810
+
811
+ //
812
+ // MTLComputeCommandEncoder wrapper
813
+ //
814
+
815
+ struct ggml_metal_encoder {
816
+ id<MTLComputeCommandEncoder> obj;
817
+ };
818
+
819
+ ggml_metal_encoder_t ggml_metal_encoder_init(ggml_metal_cmd_buf_t cmd_buf_raw, bool concurrent) {
820
+ ggml_metal_encoder_t res = calloc(1, sizeof(struct ggml_metal_encoder));
821
+
822
+ id<MTLCommandBuffer> cmd_buf = (id<MTLCommandBuffer>) cmd_buf_raw;
823
+
824
+ if (concurrent) {
825
+ res->obj = [cmd_buf computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent];
826
+ } else {
827
+ res->obj = [cmd_buf computeCommandEncoder];
828
+ }
829
+
830
+ [res->obj retain];
831
+
832
+ return res;
833
+ }
834
+
835
+ void ggml_metal_encoder_free(ggml_metal_encoder_t encoder) {
836
+ [encoder->obj release];
837
+ free(encoder);
838
+ }
839
+
840
+ void ggml_metal_encoder_debug_group_push(ggml_metal_encoder_t encoder, const char * name) {
841
+ @autoreleasepool {
842
+ [encoder->obj pushDebugGroup:[NSString stringWithCString:name encoding:NSUTF8StringEncoding]];
843
+ }
844
+ }
845
+
846
+ void ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder) {
847
+ [encoder->obj popDebugGroup];
848
+ }
849
+
850
+ void ggml_metal_encoder_set_pipeline(ggml_metal_encoder_t encoder, struct ggml_metal_pipeline_with_params pipeline) {
851
+ if (!pipeline.pipeline) {
852
+ GGML_ABORT("%s: nil Metal pipeline (missing kernel; see compile_pipeline log above)\n", __func__);
853
+ }
854
+
855
+ [encoder->obj setComputePipelineState:pipeline.pipeline->obj];
856
+ }
857
+
858
+ void ggml_metal_encoder_set_bytes(ggml_metal_encoder_t encoder, void * data, size_t size, int idx) {
859
+ [encoder->obj setBytes:data length:size atIndex:idx];
860
+ }
861
+
862
+ void ggml_metal_encoder_set_buffer(ggml_metal_encoder_t encoder, struct ggml_metal_buffer_id buffer, int idx) {
863
+ [encoder->obj setBuffer:buffer.metal offset:buffer.offs atIndex:idx];
864
+ }
865
+
866
+ void ggml_metal_encoder_set_threadgroup_memory_size(ggml_metal_encoder_t encoder, size_t size, int idx) {
867
+ // ref: https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setthreadgroupmemorylength(_:index:)
868
+ GGML_ASSERT(size % 16 == 0);
869
+
870
+ [encoder->obj setThreadgroupMemoryLength:size atIndex:idx];
871
+ }
872
+
873
+ void ggml_metal_encoder_dispatch_threadgroups(ggml_metal_encoder_t encoder, int tg0, int tg1, int tg2, int tptg0, int tptg1, int tptg2) {
874
+ [encoder->obj dispatchThreadgroups:MTLSizeMake(tg0, tg1, tg2) threadsPerThreadgroup:MTLSizeMake(tptg0, tptg1, tptg2)];
875
+ }
876
+
877
+ void ggml_metal_encoder_memory_barrier(ggml_metal_encoder_t encoder) {
878
+ [encoder->obj memoryBarrierWithScope:MTLBarrierScopeBuffers];
879
+ }
880
+
881
+ void ggml_metal_encoder_end_encoding(ggml_metal_encoder_t encoder) {
882
+ [encoder->obj endEncoding];
883
+ }
884
+
885
+ struct ggml_metal_device {
886
+ id<MTLDevice> mtl_device;
887
+
888
+ // a single global queue shared by all Metal backends
889
+ // technically not needed for devices with unified memory, but enables discrete GPUs support
890
+ // ref: https://github.com/ggml-org/llama.cpp/pull/15906
891
+ id<MTLCommandQueue> mtl_queue;
892
+
893
+ ggml_metal_rsets_t rsets;
894
+
895
+ ggml_metal_library_t library;
896
+
897
+ struct ggml_metal_device_props props;
898
+
899
+ // virtual address for GPU memory allocations
900
+ atomic_uintptr_t addr_virt;
901
+ };
902
+
903
+ //
904
+ // MTLResidenceSet wrapper
905
+ //
906
+
907
+ struct ggml_metal_rsets {
908
+ NSLock * lock;
909
+
910
+ NSMutableArray * data;
911
+
912
+ // number of seconds since the last graph computation
913
+ // keep the residency sets wired for that amount of time to avoid being collected by the OS
914
+ int keep_alive_s;
915
+ int loops_per_s;
916
+ int time_per_loop_ms;
917
+
918
+ // background heartbeat thread to keep the residency sets alive
919
+ atomic_bool d_stop;
920
+ atomic_int d_loop;
921
+
922
+ dispatch_group_t d_group;
923
+ };
924
+
925
+ #if defined(GGML_METAL_HAS_RESIDENCY_SETS)
926
+ static void ggml_metal_dummy_work(ggml_metal_device_t dev) {
927
+ if (dev->mtl_queue == nil) {
928
+ return;
929
+ }
930
+
931
+ @autoreleasepool {
932
+ // perform a minimal dummy operation on the GPU
933
+ id<MTLBuffer> buf = [dev->mtl_device newBufferWithLength:1 options:MTLResourceStorageModePrivate];
934
+ id<MTLCommandBuffer> cmd_buf = [dev->mtl_queue commandBuffer];
935
+
936
+ {
937
+ id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
938
+
939
+ [encoder fillBuffer:buf range:NSMakeRange(0, 1) value:0];
940
+
941
+ [encoder endEncoding];
942
+ }
943
+
944
+ [cmd_buf commit];
945
+ [buf release];
946
+ }
947
+ }
948
+ #endif
949
+
950
+ ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev) {
951
+ ggml_metal_rsets_t res = calloc(1, sizeof(struct ggml_metal_rsets));
952
+
953
+ res->lock = [[NSLock alloc] init];
954
+ res->data = [[NSMutableArray alloc] init];
955
+
956
+ // by default keep the memory wired for 3 minutes
957
+ res->keep_alive_s = 3*60;
958
+
959
+ const char * GGML_METAL_RESIDENCY_KEEP_ALIVE_S = getenv("GGML_METAL_RESIDENCY_KEEP_ALIVE_S");
960
+ if (GGML_METAL_RESIDENCY_KEEP_ALIVE_S) {
961
+ res->keep_alive_s = atoi(GGML_METAL_RESIDENCY_KEEP_ALIVE_S);
962
+ }
963
+
964
+ if (res->keep_alive_s <= 0) {
965
+ res->keep_alive_s = 3*60;
966
+ }
967
+
968
+ res->time_per_loop_ms = 5;
969
+ res->loops_per_s = 1000/res->time_per_loop_ms;
970
+
971
+ GGML_LOG_INFO("%s: creating a residency set collection (keep_alive = %d s)\n", __func__, res->keep_alive_s);
972
+
973
+ atomic_store_explicit(&res->d_stop, false, memory_order_relaxed);
974
+ atomic_store_explicit(&res->d_loop, res->loops_per_s*res->keep_alive_s, memory_order_relaxed);
975
+
976
+ res->d_group = dispatch_group_create();
977
+
978
+ // start a background thread that periodically requests residency for all the currently active sets in the collection
979
+ // the requests stop after a certain amount of time (keep_alive_s) of inactivity
980
+ dispatch_queue_t d_queue = dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0);
981
+ dispatch_group_async(res->d_group, d_queue, ^{
982
+ #if defined(GGML_METAL_HAS_RESIDENCY_SETS)
983
+ if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {
984
+ while (!atomic_load_explicit(&res->d_stop, memory_order_relaxed)) {
985
+ if (atomic_load_explicit(&res->d_loop, memory_order_relaxed) > 0) {
986
+ [res->lock lock];
987
+
988
+ for (int i = 0; i < (int) res->data.count; ++i) {
989
+ [res->data[i] requestResidency];
990
+ }
991
+
992
+ atomic_fetch_sub_explicit(&res->d_loop, 1, memory_order_relaxed);
993
+
994
+ [res->lock unlock];
995
+ }
996
+
997
+ usleep(res->time_per_loop_ms * 1000);
998
+ }
999
+ }
1000
+ #endif
1001
+ });
1002
+
1003
+ #if defined(GGML_METAL_HAS_RESIDENCY_SETS)
1004
+ if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {
1005
+ // workaround for residency set memory not being released if no GPU operation occurs
1006
+ // https://developer.apple.com/forums/thread/839089
1007
+ // https://github.com/ggml-org/llama.cpp/issues/25937
1008
+ ggml_metal_dummy_work(dev);
1009
+ }
1010
+ #endif
1011
+
1012
+ return res;
1013
+ }
1014
+
1015
+ void ggml_metal_rsets_free(ggml_metal_rsets_t rsets) {
1016
+ if (rsets == NULL) {
1017
+ return;
1018
+ }
1019
+
1020
+ // note: if you hit this assert, most likely you haven't deallocated all Metal resources before exiting
1021
+ GGML_ASSERT([rsets->data count] == 0);
1022
+
1023
+ atomic_store_explicit(&rsets->d_stop, true, memory_order_relaxed);
1024
+
1025
+ dispatch_group_wait(rsets->d_group, DISPATCH_TIME_FOREVER);
1026
+ dispatch_release(rsets->d_group);
1027
+
1028
+ [rsets->data release];
1029
+ [rsets->lock release];
1030
+
1031
+ free(rsets);
1032
+ }
1033
+
1034
+ static const struct {
1035
+ const char * name;
1036
+ const char * token;
1037
+ enum ggml_metal_device_id id;
1038
+ } k_metal_devices[] = {
1039
+ #define DEV(name, id) { name, #id, id }
1040
+ DEV("M1", GGML_METAL_DEVICE_M1),
1041
+ DEV("M1 Pro", GGML_METAL_DEVICE_M1_PRO),
1042
+ DEV("M1 Max", GGML_METAL_DEVICE_M1_MAX),
1043
+ DEV("M1 Ultra", GGML_METAL_DEVICE_M1_ULTRA),
1044
+ DEV("M2", GGML_METAL_DEVICE_M2),
1045
+ DEV("M2 Pro", GGML_METAL_DEVICE_M2_PRO),
1046
+ DEV("M2 Max", GGML_METAL_DEVICE_M2_MAX),
1047
+ DEV("M2 Ultra", GGML_METAL_DEVICE_M2_ULTRA),
1048
+ DEV("M3", GGML_METAL_DEVICE_M3),
1049
+ DEV("M3 Pro", GGML_METAL_DEVICE_M3_PRO),
1050
+ DEV("M3 Max", GGML_METAL_DEVICE_M3_MAX),
1051
+ DEV("M3 Ultra", GGML_METAL_DEVICE_M3_ULTRA),
1052
+ DEV("M4", GGML_METAL_DEVICE_M4),
1053
+ DEV("M4 Pro", GGML_METAL_DEVICE_M4_PRO),
1054
+ DEV("M4 Max", GGML_METAL_DEVICE_M4_MAX),
1055
+ DEV("M5", GGML_METAL_DEVICE_M5),
1056
+ DEV("M5 Pro", GGML_METAL_DEVICE_M5_PRO),
1057
+ DEV("M5 Max", GGML_METAL_DEVICE_M5_MAX),
1058
+ DEV("M5 Ultra", GGML_METAL_DEVICE_M5_ULTRA),
1059
+ DEV("A18 Pro", GGML_METAL_DEVICE_A18_PRO),
1060
+ #undef DEV
1061
+ };
1062
+
1063
+ static enum ggml_metal_device_id ggml_metal_device_id_parse(const char * name) {
1064
+ if (!name) {
1065
+ return GGML_METAL_DEVICE_GENERIC;
1066
+ }
1067
+
1068
+ static const char prefix[] = "Apple ";
1069
+ if (strncmp(name, prefix, sizeof(prefix) - 1) != 0) {
1070
+ return GGML_METAL_DEVICE_GENERIC;
1071
+ }
1072
+ const char * suffix = name + sizeof(prefix) - 1;
1073
+
1074
+ for (size_t i = 0; i < sizeof(k_metal_devices)/sizeof(k_metal_devices[0]); ++i) {
1075
+ if (strcmp(suffix, k_metal_devices[i].name) == 0) {
1076
+ return k_metal_devices[i].id;
1077
+ }
1078
+ }
1079
+ return GGML_METAL_DEVICE_GENERIC;
1080
+ }
1081
+
1082
+ const char * ggml_metal_device_id_token(enum ggml_metal_device_id id) {
1083
+ for (size_t i = 0; i < sizeof(k_metal_devices)/sizeof(k_metal_devices[0]); ++i) {
1084
+ if (k_metal_devices[i].id == id) {
1085
+ return k_metal_devices[i].token;
1086
+ }
1087
+ }
1088
+ return "GGML_METAL_DEVICE_GENERIC";
1089
+ }
1090
+
1091
+ ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {
1092
+ ggml_metal_device_t dev = calloc(1, sizeof(struct ggml_metal_device));
1093
+
1094
+ assert(dev != NULL);
1095
+
1096
+ @autoreleasepool {
1097
+ if (dev->mtl_device == nil) {
1098
+ dev->mtl_device = MTLCreateSystemDefaultDevice();
1099
+
1100
+ if (dev->mtl_device) {
1101
+ dev->mtl_queue = [dev->mtl_device newCommandQueue];
1102
+ if (dev->mtl_queue == nil) {
1103
+ GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);
1104
+ }
1105
+
1106
+ dev->addr_virt = 0x000000400ULL;
1107
+
1108
+ dev->props.device = device;
1109
+
1110
+ // the Metal backend uses the system default device as the single physical device;
1111
+ // additional (virtual) devices are emulated on top of it via GGML_METAL_DEVICES
1112
+ dev->props.device_phys = 0;
1113
+ dev->props.device_virt = device;
1114
+
1115
+ dev->props.has_simdgroup_reduction = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
1116
+ dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
1117
+
1118
+ dev->props.has_simdgroup_mm = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
1119
+ dev->props.has_unified_memory = dev->mtl_device.hasUnifiedMemory;
1120
+
1121
+ dev->props.has_bfloat = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
1122
+ dev->props.has_bfloat |= [dev->mtl_device supportsFamily:MTLGPUFamilyApple6];
1123
+ if (getenv("GGML_METAL_BF16_DISABLE") != NULL) {
1124
+ dev->props.has_bfloat = false;
1125
+ }
1126
+
1127
+ dev->props.has_tensor = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal4_GGML];
1128
+ if (getenv("GGML_METAL_TENSOR_DISABLE") != NULL) {
1129
+ dev->props.has_tensor = false;
1130
+ }
1131
+
1132
+ // note: disable the tensor API by default for old chips because with the current implementation it is not useful
1133
+ // - M2 Ultra: ~5% slower
1134
+ // - M4, M4 Max: no significant difference
1135
+ //
1136
+ // TODO: try to update the tensor API kernels to at least match the simdgroup performance
1137
+ if (getenv("GGML_METAL_TENSOR_ENABLE") == NULL &&
1138
+ ![[dev->mtl_device name] containsString:@"M5"] &&
1139
+ ![[dev->mtl_device name] containsString:@"M6"] &&
1140
+ ![[dev->mtl_device name] containsString:@"A19"] &&
1141
+ ![[dev->mtl_device name] containsString:@"A20"]) {
1142
+ GGML_LOG_INFO("%s: tensor API disabled for pre-M5 and pre-A19 devices\n", __func__);
1143
+ dev->props.has_tensor = false;
1144
+ }
1145
+
1146
+ // double-check that the tensor API compiles
1147
+ if (dev->props.has_tensor) {
1148
+ const char * src_tensor_f16 = "\n"
1149
+ "#include <metal_stdlib> \n"
1150
+ "#include <metal_tensor> \n"
1151
+ "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
1152
+ " \n"
1153
+ "using namespace metal; \n"
1154
+ "using namespace mpp::tensor_ops; \n"
1155
+ " \n"
1156
+ "kernel void dummy_kernel( \n"
1157
+ " tensor<device half, dextents<int32_t, 2>> A [[buffer(0)]], \n"
1158
+ " tensor<device half, dextents<int32_t, 2>> B [[buffer(1)]], \n"
1159
+ " device float * C [[buffer(2)]], \n"
1160
+ " uint2 tgid [[threadgroup_position_in_grid]]) \n"
1161
+ "{ \n"
1162
+ " auto tA = A.slice(0, (int)tgid.y); \n"
1163
+ " auto tB = B.slice((int)tgid.x, 0); \n"
1164
+ " \n"
1165
+ " matmul2d< \n"
1166
+ " matmul2d_descriptor(16, 16, dynamic_extent), \n"
1167
+ " execution_simdgroups<4>> mm; \n"
1168
+ " \n"
1169
+ " auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
1170
+ " \n"
1171
+ " auto sA = tA.slice(0, 0); \n"
1172
+ " auto sB = tB.slice(0, 0); \n"
1173
+ " mm.run(sB, sA, cT); \n"
1174
+ " \n"
1175
+ " auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
1176
+ " \n"
1177
+ " cT.store(tC); \n"
1178
+ "}";
1179
+
1180
+ GGML_LOG_INFO("%s: testing tensor API for f16 support\n", __func__);
1181
+ ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_f16, false);
1182
+ if (lib == NULL) {
1183
+ GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);
1184
+ dev->props.has_tensor = false;
1185
+ } else {
1186
+ struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
1187
+ if (!ppl.pipeline) {
1188
+ GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);
1189
+ dev->props.has_tensor = false;
1190
+ }
1191
+
1192
+ ggml_metal_library_free(lib);
1193
+ }
1194
+ }
1195
+
1196
+ // try to compile a dummy kernel to determine if the tensor API is supported for bfloat
1197
+ if (dev->props.has_tensor && dev->props.has_bfloat) {
1198
+ const char * src_tensor_bf16 = "\n"
1199
+ "#include <metal_stdlib> \n"
1200
+ "#include <metal_tensor> \n"
1201
+ "#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
1202
+ " \n"
1203
+ "using namespace metal; \n"
1204
+ "using namespace mpp::tensor_ops; \n"
1205
+ " \n"
1206
+ "kernel void dummy_kernel( \n"
1207
+ " tensor<device bfloat, dextents<int32_t, 2>> A [[buffer(0)]], \n"
1208
+ " tensor<device bfloat, dextents<int32_t, 2>> B [[buffer(1)]], \n"
1209
+ " device float * C [[buffer(2)]], \n"
1210
+ " uint2 tgid [[threadgroup_position_in_grid]]) \n"
1211
+ "{ \n"
1212
+ " auto tA = A.slice(0, (int)tgid.y); \n"
1213
+ " auto tB = B.slice((int)tgid.x, 0); \n"
1214
+ " \n"
1215
+ " matmul2d< \n"
1216
+ " matmul2d_descriptor(16, 16, dynamic_extent), \n"
1217
+ " execution_simdgroups<4>> mm; \n"
1218
+ " \n"
1219
+ " auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
1220
+ " \n"
1221
+ " auto sA = tA.slice(0, 0); \n"
1222
+ " auto sB = tB.slice(0, 0); \n"
1223
+ " mm.run(sB, sA, cT); \n"
1224
+ " \n"
1225
+ " auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
1226
+ " \n"
1227
+ " cT.store(tC); \n"
1228
+ "}";
1229
+
1230
+ GGML_LOG_INFO("%s: testing tensor API for bfloat support\n", __func__);
1231
+ ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_bf16, false);
1232
+ if (lib == NULL) {
1233
+ GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);
1234
+ dev->props.has_bfloat = false;
1235
+ } else {
1236
+ struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
1237
+ if (!ppl.pipeline) {
1238
+ GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);
1239
+ dev->props.has_bfloat = false;
1240
+ }
1241
+
1242
+ ggml_metal_library_free(lib);
1243
+ }
1244
+ }
1245
+
1246
+ dev->props.use_residency_sets = true;
1247
+ #if defined(GGML_METAL_HAS_RESIDENCY_SETS)
1248
+ dev->props.use_residency_sets = getenv("GGML_METAL_NO_RESIDENCY") == nil;
1249
+ #endif
1250
+
1251
+ dev->props.use_shared_buffers = dev->props.has_unified_memory;
1252
+ #if TARGET_OS_OSX
1253
+ // In case of eGPU, shared memory may be preferable.
1254
+ dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal;
1255
+ #endif
1256
+ if (getenv("GGML_METAL_SHARED_BUFFERS_DISABLE") != NULL) {
1257
+ dev->props.use_shared_buffers = false;
1258
+ }
1259
+ if (getenv("GGML_METAL_SHARED_BUFFERS_ENABLE") != NULL) {
1260
+ dev->props.use_shared_buffers = true;
1261
+ }
1262
+
1263
+ dev->props.supports_gpu_family_apple7 = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
1264
+
1265
+ dev->props.device_id = ggml_metal_device_id_parse([[dev->mtl_device name] UTF8String]);
1266
+
1267
+ dev->props.op_offload_min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32;
1268
+
1269
+ dev->props.max_buffer_size = dev->mtl_device.maxBufferLength;
1270
+ dev->props.max_theadgroup_memory_size = dev->mtl_device.maxThreadgroupMemoryLength;
1271
+ if (@available(macOS 10.12, iOS 16.0, *)) {
1272
+ dev->props.max_working_set_size = dev->mtl_device.recommendedMaxWorkingSetSize;
1273
+ } else {
1274
+ dev->props.max_working_set_size = dev->mtl_device.maxBufferLength;
1275
+ }
1276
+
1277
+ snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);
1278
+ const char * gpu_name = [[dev->mtl_device name] UTF8String];
1279
+ if (n_devices > 1) {
1280
+ snprintf(dev->props.desc, sizeof(dev->props.desc), "%s (dev p%d/v%d)",
1281
+ gpu_name, dev->props.device_phys, dev->props.device_virt);
1282
+ } else {
1283
+ snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", gpu_name);
1284
+ }
1285
+
1286
+ dev->library = ggml_metal_library_init(dev);
1287
+ if (!dev->library) {
1288
+ GGML_LOG_ERROR("%s: error: failed to create library\n", __func__);
1289
+ }
1290
+
1291
+ if (dev->props.use_residency_sets) {
1292
+ dev->rsets = ggml_metal_rsets_init(dev);
1293
+ } else {
1294
+ dev->rsets = nil;
1295
+ }
1296
+
1297
+ // print MTL GPU family:
1298
+ GGML_LOG_INFO("%s: GPU name: %s (%s)\n", __func__, dev->props.name, dev->props.desc);
1299
+
1300
+ // determine max supported GPU family
1301
+ // https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf
1302
+ // https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
1303
+ {
1304
+ for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
1305
+ if ([dev->mtl_device supportsFamily:i]) {
1306
+ dev->props.gpu_family = i - (int) MTLGPUFamilyApple1 + 1;
1307
+ GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, dev->props.gpu_family, i);
1308
+ break;
1309
+ }
1310
+ }
1311
+
1312
+ for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {
1313
+ if ([dev->mtl_device supportsFamily:i]) {
1314
+ GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);
1315
+ break;
1316
+ }
1317
+ }
1318
+
1319
+ for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) {
1320
+ if ([dev->mtl_device supportsFamily:i]) {
1321
+ GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3_GGML + 3, i);
1322
+ break;
1323
+ }
1324
+ }
1325
+ }
1326
+
1327
+ GGML_LOG_INFO("%s: simdgroup reduction = %s\n", __func__, dev->props.has_simdgroup_reduction ? "true" : "false");
1328
+ GGML_LOG_INFO("%s: simdgroup matrix mul. = %s\n", __func__, dev->props.has_simdgroup_mm ? "true" : "false");
1329
+ GGML_LOG_INFO("%s: has unified memory = %s\n", __func__, dev->props.has_unified_memory ? "true" : "false");
1330
+ GGML_LOG_INFO("%s: has bfloat = %s\n", __func__, dev->props.has_bfloat ? "true" : "false");
1331
+ GGML_LOG_INFO("%s: has tensor = %s\n", __func__, dev->props.has_tensor ? "true" : "false");
1332
+ GGML_LOG_INFO("%s: use residency sets = %s\n", __func__, dev->props.use_residency_sets ? "true" : "false");
1333
+ GGML_LOG_INFO("%s: use shared buffers = %s\n", __func__, dev->props.use_shared_buffers ? "true" : "false");
1334
+
1335
+ #if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
1336
+ if (@available(macOS 10.12, iOS 16.0, *)) {
1337
+ GGML_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, dev->props.max_working_set_size / 1e6);
1338
+ }
1339
+ #endif
1340
+ }
1341
+ }
1342
+ }
1343
+
1344
+ return dev;
1345
+ }
1346
+
1347
+ void ggml_metal_device_free(ggml_metal_device_t dev) {
1348
+ assert(dev != NULL);
1349
+
1350
+ @autoreleasepool {
1351
+ ggml_metal_rsets_free(dev->rsets);
1352
+
1353
+ ggml_metal_library_free(dev->library);
1354
+ dev->library = NULL;
1355
+
1356
+ if (dev->mtl_queue) {
1357
+ [dev->mtl_queue release];
1358
+ dev->mtl_queue = nil;
1359
+ }
1360
+
1361
+ if (dev->mtl_device) {
1362
+ [dev->mtl_device release];
1363
+ dev->mtl_device = nil;
1364
+ }
1365
+ }
1366
+
1367
+ free(dev);
1368
+ }
1369
+
1370
+ void * ggml_metal_device_get_obj(ggml_metal_device_t dev) {
1371
+ return dev->mtl_device;
1372
+ }
1373
+
1374
+ void * ggml_metal_device_get_queue(ggml_metal_device_t dev) {
1375
+ return dev->mtl_queue;
1376
+ }
1377
+
1378
+ ggml_metal_library_t ggml_metal_device_get_library(ggml_metal_device_t dev) {
1379
+ return dev->library;
1380
+ }
1381
+
1382
+ void ggml_metal_device_rsets_add(ggml_metal_device_t dev, ggml_metal_rset_t rset) {
1383
+ if (rset == nil) {
1384
+ return;
1385
+ }
1386
+
1387
+ GGML_ASSERT(dev->rsets);
1388
+
1389
+ [dev->rsets->lock lock];
1390
+
1391
+ [dev->rsets->data addObject:rset];
1392
+
1393
+ [dev->rsets->lock unlock];
1394
+ }
1395
+
1396
+ void ggml_metal_device_rsets_rm(ggml_metal_device_t dev, ggml_metal_rset_t rset) {
1397
+ if (rset == nil) {
1398
+ return;
1399
+ }
1400
+
1401
+ GGML_ASSERT(dev->rsets);
1402
+
1403
+ [dev->rsets->lock lock];
1404
+
1405
+ [dev->rsets->data removeObject:rset];
1406
+
1407
+ [dev->rsets->lock unlock];
1408
+ }
1409
+
1410
+ void ggml_metal_device_rsets_keep_alive(ggml_metal_device_t dev) {
1411
+ if (dev->rsets == NULL) {
1412
+ return;
1413
+ }
1414
+
1415
+ atomic_store_explicit(&dev->rsets->d_loop, dev->rsets->loops_per_s*dev->rsets->keep_alive_s, memory_order_relaxed);
1416
+ }
1417
+
1418
+ struct ggml_metal_event {
1419
+ void * obj; // id<MTLSharedEvent>
1420
+
1421
+ atomic_int value;
1422
+ };
1423
+
1424
+ void ggml_metal_event_encode_signal(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf_raw) {
1425
+ id<MTLSharedEvent> event = (id<MTLSharedEvent>)ev->obj;
1426
+
1427
+ id<MTLCommandBuffer> cmd_buf = (id<MTLCommandBuffer>) cmd_buf_raw;
1428
+
1429
+ [cmd_buf encodeSignalEvent:event value:atomic_fetch_add_explicit(&ev->value, 1, memory_order_relaxed) + 1];
1430
+ }
1431
+
1432
+ void ggml_metal_event_encode_wait(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf_raw) {
1433
+ id<MTLSharedEvent> event = (id<MTLSharedEvent>)ev->obj;
1434
+
1435
+ id<MTLCommandBuffer> cmd_buf = (id<MTLCommandBuffer>) cmd_buf_raw;
1436
+
1437
+ [cmd_buf encodeWaitForEvent:event value:atomic_load_explicit(&ev->value, memory_order_relaxed)];
1438
+ }
1439
+
1440
+ ggml_metal_event_t ggml_metal_device_event_init(ggml_metal_device_t dev) {
1441
+ id<MTLSharedEvent> event = [dev->mtl_device newSharedEvent];
1442
+
1443
+ ggml_metal_event_t ev = calloc(1, sizeof(struct ggml_metal_event));
1444
+
1445
+ ev->obj = (__bridge void *)event;
1446
+ ev->value = 0;
1447
+
1448
+ return ev;
1449
+ }
1450
+
1451
+ void ggml_metal_device_event_free(ggml_metal_device_t dev, ggml_metal_event_t ev) {
1452
+ @autoreleasepool {
1453
+ id<MTLSharedEvent> event = ev->obj;
1454
+ [event release];
1455
+
1456
+ free(ev);
1457
+
1458
+ GGML_UNUSED(dev);
1459
+ }
1460
+ }
1461
+
1462
+ void ggml_metal_device_event_synchronize(ggml_metal_device_t dev, ggml_metal_event_t ev) {
1463
+ id<MTLSharedEvent> event = ev->obj;
1464
+ const bool res = [event waitUntilSignaledValue:atomic_load_explicit(&ev->value, memory_order_relaxed) timeoutMS:60000];
1465
+ if (!res) {
1466
+ GGML_ABORT("%s: failed to wait for event\n", __func__);
1467
+ }
1468
+
1469
+ GGML_UNUSED(dev);
1470
+ }
1471
+
1472
+ void ggml_metal_device_get_memory(ggml_metal_device_t dev, size_t * free, size_t * total) {
1473
+ if (@available(macOS 10.12, iOS 16.0, *)) {
1474
+ *total = dev->mtl_device.recommendedMaxWorkingSetSize;
1475
+ size_t cur = dev->mtl_device.currentAllocatedSize;
1476
+ // it's possible to allocate more than `recommendedMaxWorkingSetSize`
1477
+ *free = *total > cur ? *total - cur : 0;
1478
+ } else {
1479
+ *free = 0;
1480
+ *total = 0;
1481
+ }
1482
+ }
1483
+
1484
+ static bool ggml_metal_supports_mul_mat_op(
1485
+ bool has_simdgroup_reduction,
1486
+ const struct ggml_tensor * op,
1487
+ bool src0_f16_has_mv,
1488
+ bool mm_path) {
1489
+ if (!has_simdgroup_reduction || op->src[0]->type == GGML_TYPE_NVFP4) {
1490
+ return false;
1491
+ }
1492
+
1493
+ if (op->src[1]->type != GGML_TYPE_F16) {
1494
+ return true;
1495
+ }
1496
+
1497
+ if (op->src[0]->type == GGML_TYPE_BF16) {
1498
+ return false;
1499
+ }
1500
+
1501
+ if (src0_f16_has_mv && op->src[0]->type == GGML_TYPE_F16) {
1502
+ return true;
1503
+ }
1504
+
1505
+ return mm_path;
1506
+ }
1507
+
1508
+ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tensor * op) {
1509
+ const bool has_simdgroup_mm = dev->props.has_simdgroup_mm;
1510
+ const bool has_simdgroup_reduction = dev->props.has_simdgroup_reduction;
1511
+ const bool has_bfloat = dev->props.has_bfloat;
1512
+
1513
+ if (!has_bfloat) {
1514
+ if (op->type == GGML_TYPE_BF16) {
1515
+ return false;
1516
+ }
1517
+
1518
+ for (size_t i = 0, n = 3; i < n; ++i) {
1519
+ if (op->src[i] != NULL && op->src[i]->type == GGML_TYPE_BF16) {
1520
+ return false;
1521
+ }
1522
+ }
1523
+ }
1524
+
1525
+ switch (op->op) {
1526
+ case GGML_OP_SCALE:
1527
+ case GGML_OP_FILL:
1528
+ case GGML_OP_CLAMP:
1529
+ case GGML_OP_SQR:
1530
+ case GGML_OP_SQRT:
1531
+ case GGML_OP_SIN:
1532
+ case GGML_OP_COS:
1533
+ case GGML_OP_LOG:
1534
+ return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
1535
+ case GGML_OP_UNARY:
1536
+ switch (ggml_get_unary_op(op)) {
1537
+ case GGML_UNARY_OP_TANH:
1538
+ case GGML_UNARY_OP_RELU:
1539
+ case GGML_UNARY_OP_SIGMOID:
1540
+ case GGML_UNARY_OP_GELU:
1541
+ case GGML_UNARY_OP_GELU_ERF:
1542
+ case GGML_UNARY_OP_GELU_QUICK:
1543
+ case GGML_UNARY_OP_SILU:
1544
+ case GGML_UNARY_OP_ELU:
1545
+ case GGML_UNARY_OP_NEG:
1546
+ case GGML_UNARY_OP_ABS:
1547
+ case GGML_UNARY_OP_SGN:
1548
+ case GGML_UNARY_OP_STEP:
1549
+ case GGML_UNARY_OP_HARDSWISH:
1550
+ case GGML_UNARY_OP_HARDSIGMOID:
1551
+ case GGML_UNARY_OP_EXP:
1552
+ case GGML_UNARY_OP_SOFTPLUS:
1553
+ case GGML_UNARY_OP_EXPM1:
1554
+ case GGML_UNARY_OP_FLOOR:
1555
+ case GGML_UNARY_OP_CEIL:
1556
+ case GGML_UNARY_OP_ROUND:
1557
+ case GGML_UNARY_OP_TRUNC:
1558
+ case GGML_UNARY_OP_XIELU:
1559
+ return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
1560
+ default:
1561
+ return false;
1562
+ }
1563
+ case GGML_OP_SILU_BACK:
1564
+ return (op->src[0]->type == GGML_TYPE_F32) &&
1565
+ (op->src[1]->type == GGML_TYPE_F32) &&
1566
+ (op->type == GGML_TYPE_F32) &&
1567
+ ggml_is_contiguous(op->src[0]) &&
1568
+ ggml_is_contiguous(op->src[1]) &&
1569
+ ggml_is_contiguous(op) &&
1570
+ ggml_are_same_shape(op->src[0], op->src[1]);
1571
+ case GGML_OP_GLU:
1572
+ switch (ggml_get_glu_op(op)) {
1573
+ case GGML_GLU_OP_REGLU:
1574
+ case GGML_GLU_OP_GEGLU:
1575
+ case GGML_GLU_OP_SWIGLU:
1576
+ case GGML_GLU_OP_SWIGLU_OAI:
1577
+ case GGML_GLU_OP_GEGLU_ERF:
1578
+ case GGML_GLU_OP_GEGLU_QUICK:
1579
+ case GGML_GLU_OP_SWIGLU_CLAMP:
1580
+ return ggml_is_contiguous_1(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
1581
+ default:
1582
+ return false;
1583
+ }
1584
+ case GGML_OP_NONE:
1585
+ case GGML_OP_RESHAPE:
1586
+ case GGML_OP_VIEW:
1587
+ case GGML_OP_TRANSPOSE:
1588
+ case GGML_OP_PERMUTE:
1589
+ return true;
1590
+ case GGML_OP_CONCAT:
1591
+ {
1592
+ const enum ggml_type src0_type = op->src[0]->type;
1593
+ const enum ggml_type src1_type = op->src[1]->type;
1594
+ if (src0_type != src1_type || src0_type != op->type) {
1595
+ return false;
1596
+ }
1597
+ switch (src0_type) {
1598
+ case GGML_TYPE_F32:
1599
+ case GGML_TYPE_F16:
1600
+ case GGML_TYPE_I8:
1601
+ case GGML_TYPE_I16:
1602
+ case GGML_TYPE_I32:
1603
+ case GGML_TYPE_I64:
1604
+ return true;
1605
+ case GGML_TYPE_BF16:
1606
+ return has_bfloat;
1607
+ case GGML_TYPE_Q4_0:
1608
+ case GGML_TYPE_Q4_1:
1609
+ case GGML_TYPE_Q5_0:
1610
+ case GGML_TYPE_Q5_1:
1611
+ case GGML_TYPE_Q8_0:
1612
+ return true;
1613
+ default:
1614
+ return false;
1615
+ }
1616
+ }
1617
+ case GGML_OP_ADD:
1618
+ case GGML_OP_SUB:
1619
+ case GGML_OP_MUL:
1620
+ case GGML_OP_DIV:
1621
+ case GGML_OP_ADD_ID:
1622
+ return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && (op->src[0]->type == op->src[1]->type);
1623
+ case GGML_OP_ACC:
1624
+ return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && op->src[0]->type == GGML_TYPE_F32;
1625
+ case GGML_OP_REPEAT:
1626
+ case GGML_OP_CONV_TRANSPOSE_1D:
1627
+ return true;
1628
+ case GGML_OP_CONV_TRANSPOSE_2D:
1629
+ return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) &&
1630
+ (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) &&
1631
+ op->src[1]->type == GGML_TYPE_F32 &&
1632
+ op->type == GGML_TYPE_F32;
1633
+ case GGML_OP_COL2IM_1D:
1634
+ return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) &&
1635
+ op->type == op->src[0]->type &&
1636
+ ggml_is_contiguous(op->src[0]) &&
1637
+ ggml_is_contiguous(op);
1638
+ case GGML_OP_CONV_3D:
1639
+ return ggml_is_contiguous(op->src[0]) &&
1640
+ ggml_is_contiguous(op->src[1]) &&
1641
+ (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) &&
1642
+ op->src[1]->type == GGML_TYPE_F32;
1643
+ case GGML_OP_SUM:
1644
+ return has_simdgroup_reduction && ggml_is_contiguous(op->src[0]);
1645
+ case GGML_OP_TRI:
1646
+ return ggml_is_contiguous_rows(op->src[0]);
1647
+ case GGML_OP_SUM_ROWS:
1648
+ case GGML_OP_CUMSUM:
1649
+ case GGML_OP_MEAN:
1650
+ case GGML_OP_SOFT_MAX:
1651
+ case GGML_OP_GROUP_NORM:
1652
+ case GGML_OP_L2_NORM:
1653
+ return has_simdgroup_reduction && ggml_is_contiguous_rows(op->src[0]);
1654
+ case GGML_OP_COUNT_EQUAL:
1655
+ return has_simdgroup_reduction &&
1656
+ op->src[0]->type == GGML_TYPE_I32 &&
1657
+ op->src[1]->type == GGML_TYPE_I32 &&
1658
+ op->type == GGML_TYPE_I64;
1659
+ case GGML_OP_ARGMAX:
1660
+ return has_simdgroup_reduction;
1661
+ case GGML_OP_NORM:
1662
+ case GGML_OP_RMS_NORM:
1663
+ return has_simdgroup_reduction && (ggml_is_contiguous_rows(op->src[0]));
1664
+ case GGML_OP_ROPE:
1665
+ case GGML_OP_ROPE_BACK:
1666
+ return true;
1667
+ case GGML_OP_IM2COL:
1668
+ return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32);
1669
+ case GGML_OP_CONV_2D:
1670
+ return ggml_is_contiguous(op->src[0]) &&
1671
+ op->src[1]->type == GGML_TYPE_F32 &&
1672
+ op->type == GGML_TYPE_F32 &&
1673
+ (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32);
1674
+ case GGML_OP_CONV_2D_DW:
1675
+ return op->src[1]->type == GGML_TYPE_F32 &&
1676
+ op->type == GGML_TYPE_F32 &&
1677
+ (op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32);
1678
+ case GGML_OP_UPSCALE:
1679
+ return op->src[0]->type == GGML_TYPE_F32;
1680
+ case GGML_OP_POOL_1D:
1681
+ return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;
1682
+ case GGML_OP_POOL_2D:
1683
+ return op->src[0]->type == GGML_TYPE_F32;
1684
+ case GGML_OP_PAD:
1685
+ // TODO: add circular padding support for metal, see https://github.com/ggml-org/llama.cpp/pull/16985
1686
+ if (ggml_get_op_params_i32(op, 8) != 0) {
1687
+ return false;
1688
+ }
1689
+
1690
+ return (ggml_get_op_params_i32(op, 0) == 0) && (ggml_get_op_params_i32(op, 2) == 0) &&
1691
+ (ggml_get_op_params_i32(op, 4) == 0) && (ggml_get_op_params_i32(op, 6) == 0);
1692
+ case GGML_OP_PAD_REFLECT_1D:
1693
+ case GGML_OP_TIMESTEP_EMBEDDING:
1694
+ return op->src[0]->type == GGML_TYPE_F32;
1695
+ case GGML_OP_LEAKY_RELU:
1696
+ return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
1697
+ case GGML_OP_ARGSORT:
1698
+ case GGML_OP_TOP_K:
1699
+ case GGML_OP_ARANGE:
1700
+ return true;
1701
+ case GGML_OP_ROLL:
1702
+ return ggml_is_contiguous(op->src[0]);
1703
+ case GGML_OP_FLASH_ATTN_EXT:
1704
+ // for new head sizes, add checks here
1705
+ if (op->src[0]->ne[0] != 32 &&
1706
+ op->src[0]->ne[0] != 40 &&
1707
+ op->src[0]->ne[0] != 48 &&
1708
+ op->src[0]->ne[0] != 64 &&
1709
+ op->src[0]->ne[0] != 72 &&
1710
+ op->src[0]->ne[0] != 80 &&
1711
+ op->src[0]->ne[0] != 96 &&
1712
+ op->src[0]->ne[0] != 112 &&
1713
+ op->src[0]->ne[0] != 128 &&
1714
+ op->src[0]->ne[0] != 192 &&
1715
+ op->src[0]->ne[0] != 256 &&
1716
+ op->src[0]->ne[0] != 320 &&
1717
+ op->src[0]->ne[0] != 512 &&
1718
+ op->src[0]->ne[0] != 576) {
1719
+ return false;
1720
+ }
1721
+ if (op->src[1]->type != op->src[2]->type) {
1722
+ return false;
1723
+ }
1724
+ switch (op->src[1]->type) {
1725
+ case GGML_TYPE_F32:
1726
+ case GGML_TYPE_F16:
1727
+ case GGML_TYPE_Q8_0:
1728
+ case GGML_TYPE_Q4_0:
1729
+ case GGML_TYPE_Q4_1:
1730
+ case GGML_TYPE_Q5_0:
1731
+ case GGML_TYPE_Q5_1:
1732
+ break;
1733
+ case GGML_TYPE_BF16:
1734
+ if (!has_bfloat) {
1735
+ return false;
1736
+ }
1737
+ break;
1738
+ default:
1739
+ return false;
1740
+ }
1741
+ return has_simdgroup_mm; // TODO: over-restricted for vec-kernels
1742
+ case GGML_OP_LIGHTNING_INDEXER:
1743
+ if (op->src[0]->ne[0] != OP_LIGHTNING_INDEXER_DK ||
1744
+ op->src[0]->ne[1] != OP_LIGHTNING_INDEXER_NH) {
1745
+ return false;
1746
+ }
1747
+ if (!has_simdgroup_mm ||
1748
+ op->src[0]->type != GGML_TYPE_F32 ||
1749
+ op->src[2]->type != GGML_TYPE_F32 ||
1750
+ op->src[3]->type != GGML_TYPE_F16 ||
1751
+ op->type != GGML_TYPE_F32 ||
1752
+ !ggml_is_contiguous_rows(op->src[0]) ||
1753
+ !ggml_is_contiguous_rows(op->src[1]) ||
1754
+ !ggml_is_contiguous_rows(op->src[2]) ||
1755
+ !ggml_is_contiguous_rows(op->src[3])) {
1756
+ return false;
1757
+ }
1758
+ switch (op->src[1]->type) {
1759
+ case GGML_TYPE_F32:
1760
+ case GGML_TYPE_F16:
1761
+ case GGML_TYPE_Q4_0:
1762
+ case GGML_TYPE_Q4_1:
1763
+ case GGML_TYPE_Q5_0:
1764
+ case GGML_TYPE_Q5_1:
1765
+ case GGML_TYPE_Q8_0:
1766
+ return true;
1767
+ case GGML_TYPE_BF16:
1768
+ return has_bfloat;
1769
+ default:
1770
+ return false;
1771
+ }
1772
+ case GGML_OP_DSV4_HC_COMB:
1773
+ return has_simdgroup_reduction &&
1774
+ op->src[0]->type == GGML_TYPE_F32 &&
1775
+ op->src[1]->type == GGML_TYPE_F32 &&
1776
+ op->src[2]->type == GGML_TYPE_F32 &&
1777
+ op->type == GGML_TYPE_F32 &&
1778
+ op->src[0]->ne[0] == 24 &&
1779
+ op->src[1]->ne[0] >= 3 &&
1780
+ op->src[2]->ne[0] == 24 &&
1781
+ ggml_is_contiguous_rows(op->src[0]) &&
1782
+ ggml_is_contiguous_rows(op->src[1]) &&
1783
+ ggml_is_contiguous_rows(op->src[2]);
1784
+ case GGML_OP_DSV4_HC_PRE:
1785
+ return has_simdgroup_reduction &&
1786
+ op->src[0]->type == GGML_TYPE_F32 &&
1787
+ op->src[1]->type == GGML_TYPE_F32 &&
1788
+ op->type == GGML_TYPE_F32 &&
1789
+ op->src[0]->ne[1] == 4 &&
1790
+ op->src[1]->ne[0] == 4 &&
1791
+ ggml_is_contiguous_rows(op->src[0]) &&
1792
+ ggml_is_contiguous_rows(op->src[1]);
1793
+ case GGML_OP_DSV4_HC_POST:
1794
+ return has_simdgroup_reduction &&
1795
+ op->src[0]->type == GGML_TYPE_F32 &&
1796
+ op->src[1]->type == GGML_TYPE_F32 &&
1797
+ op->src[2]->type == GGML_TYPE_F32 &&
1798
+ op->src[3]->type == GGML_TYPE_F32 &&
1799
+ op->type == GGML_TYPE_F32 &&
1800
+ op->src[1]->ne[1] == 4 &&
1801
+ op->src[2]->ne[0] == 4 &&
1802
+ op->src[3]->ne[0] == 4 &&
1803
+ op->src[3]->ne[1] == 4 &&
1804
+ ggml_is_contiguous_rows(op->src[0]) &&
1805
+ ggml_is_contiguous_rows(op->src[1]) &&
1806
+ ggml_is_contiguous_rows(op->src[2]) &&
1807
+ ggml_is_contiguous_rows(op->src[3]);
1808
+ case GGML_OP_SSM_SCAN:
1809
+ return has_simdgroup_reduction;
1810
+ case GGML_OP_SSM_CONV:
1811
+ return has_simdgroup_reduction;
1812
+ case GGML_OP_RWKV_WKV6:
1813
+ case GGML_OP_RWKV_WKV7:
1814
+ return true;
1815
+ case GGML_OP_GATED_DELTA_NET:
1816
+ return has_simdgroup_reduction && op->src[2]->ne[0] % 32 == 0;
1817
+ case GGML_OP_SOLVE_TRI:
1818
+ return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32;
1819
+ case GGML_OP_MUL_MAT:
1820
+ return ggml_metal_supports_mul_mat_op(
1821
+ has_simdgroup_reduction, op, true,
1822
+ ggml_metal_op_mul_mat_use_mm(op, has_simdgroup_mm));
1823
+ case GGML_OP_MUL_MAT_ID:
1824
+ return ggml_metal_supports_mul_mat_op(
1825
+ has_simdgroup_reduction, op, false,
1826
+ ggml_metal_op_mul_mat_id_use_mm(op, has_simdgroup_mm));
1827
+ case GGML_OP_SET:
1828
+ case GGML_OP_CPY:
1829
+ case GGML_OP_DUP:
1830
+ case GGML_OP_CONT:
1831
+ {
1832
+ switch (op->src[0]->type) {
1833
+ case GGML_TYPE_F32:
1834
+ switch (op->type) {
1835
+ case GGML_TYPE_F32:
1836
+ case GGML_TYPE_F16:
1837
+ case GGML_TYPE_BF16:
1838
+ case GGML_TYPE_Q8_0:
1839
+ case GGML_TYPE_Q1_0:
1840
+ case GGML_TYPE_Q2_0:
1841
+ case GGML_TYPE_Q4_0:
1842
+ case GGML_TYPE_Q4_1:
1843
+ case GGML_TYPE_Q5_0:
1844
+ case GGML_TYPE_Q5_1:
1845
+ case GGML_TYPE_IQ4_NL:
1846
+ case GGML_TYPE_TQ2_0:
1847
+ case GGML_TYPE_I32:
1848
+ return true;
1849
+ default:
1850
+ return false;
1851
+ }
1852
+ case GGML_TYPE_F16:
1853
+ switch (op->type) {
1854
+ case GGML_TYPE_F32:
1855
+ case GGML_TYPE_F16:
1856
+ return true;
1857
+ default:
1858
+ return false;
1859
+ }
1860
+ case GGML_TYPE_BF16:
1861
+ switch (op->type) {
1862
+ case GGML_TYPE_F32:
1863
+ case GGML_TYPE_BF16:
1864
+ return true;
1865
+ default:
1866
+ return false;
1867
+ }
1868
+ case GGML_TYPE_Q1_0:
1869
+ case GGML_TYPE_Q2_0:
1870
+ case GGML_TYPE_Q4_0:
1871
+ case GGML_TYPE_Q4_1:
1872
+ case GGML_TYPE_Q5_0:
1873
+ case GGML_TYPE_Q5_1:
1874
+ case GGML_TYPE_Q8_0:
1875
+ case GGML_TYPE_TQ2_0:
1876
+ switch (op->type) {
1877
+ case GGML_TYPE_F32:
1878
+ case GGML_TYPE_F16:
1879
+ return true;
1880
+ default:
1881
+ return false;
1882
+ }
1883
+ case GGML_TYPE_I32:
1884
+ return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_I32;
1885
+ default:
1886
+ return false;
1887
+ };
1888
+ }
1889
+ case GGML_OP_GET_ROWS:
1890
+ return op->src[0]->type != GGML_TYPE_NVFP4;
1891
+ case GGML_OP_SET_ROWS:
1892
+ {
1893
+ if (op->src[0]->type == GGML_TYPE_F16) {
1894
+ return op->type == GGML_TYPE_F16;
1895
+ }
1896
+
1897
+ if (op->src[0]->type != GGML_TYPE_F32) {
1898
+ return false;
1899
+ }
1900
+
1901
+ switch (op->type) {
1902
+ case GGML_TYPE_F32:
1903
+ case GGML_TYPE_F16:
1904
+ case GGML_TYPE_BF16:
1905
+ case GGML_TYPE_Q8_0:
1906
+ case GGML_TYPE_Q4_0:
1907
+ case GGML_TYPE_Q4_1:
1908
+ case GGML_TYPE_Q5_0:
1909
+ case GGML_TYPE_Q5_1:
1910
+ case GGML_TYPE_IQ4_NL:
1911
+ case GGML_TYPE_TQ2_0:
1912
+ return true;
1913
+ default:
1914
+ return false;
1915
+ };
1916
+ }
1917
+ case GGML_OP_DIAG:
1918
+ return true;
1919
+ case GGML_OP_OPT_STEP_ADAMW:
1920
+ case GGML_OP_OPT_STEP_SGD:
1921
+ return has_simdgroup_reduction;
1922
+ default:
1923
+ return false;
1924
+ }
1925
+ }
1926
+
1927
+ const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev) {
1928
+ return &dev->props;
1929
+ }
1930
+
1931
+ static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) {
1932
+ dev->props.has_tensor = false;
1933
+ }
1934
+
1935
+ //
1936
+ // device buffers
1937
+ //
1938
+
1939
+ // max memory buffers that can be mapped to the device
1940
+ #define GGML_METAL_MAX_BUFFERS 64
1941
+
1942
+ struct ggml_metal_buffer_wrapper {
1943
+ void * data;
1944
+ size_t size;
1945
+
1946
+ id<MTLBuffer> metal;
1947
+ };
1948
+
1949
+ struct ggml_metal_buffer {
1950
+ void * all_data;
1951
+ size_t all_size;
1952
+
1953
+ // if false, the Metal buffer data is allocated in private GPU memory and is not shared with the host
1954
+ bool is_shared;
1955
+ bool owned;
1956
+
1957
+ // multiple buffers are used only to avoid the maximum buffer size limitation when using mmap
1958
+ int n_buffers;
1959
+ struct ggml_metal_buffer_wrapper buffers[GGML_METAL_MAX_BUFFERS];
1960
+
1961
+ bool use_residency_sets;
1962
+
1963
+ // optional MTLResidencySet
1964
+ // note: cannot use explicitly "id<MTLResidencySet>" here because it is not available on certain OSes
1965
+ id rset;
1966
+
1967
+ // pointers to global device
1968
+ ggml_metal_device_t dev;
1969
+ };
1970
+
1971
+ static void ggml_metal_log_allocated_size(id<MTLDevice> device, size_t size_aligned) {
1972
+ #ifndef GGML_METAL_NDEBUG
1973
+ #if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
1974
+ if (@available(macOS 10.12, iOS 16.0, *)) {
1975
+ GGML_LOG_DEBUG("%s: allocated buffer, size = %8.2f MiB, (%8.2f / %8.2f)\n",
1976
+ __func__,
1977
+ size_aligned / 1024.0 / 1024.0,
1978
+ device.currentAllocatedSize / 1024.0 / 1024.0,
1979
+ device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
1980
+
1981
+ if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {
1982
+ GGML_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);
1983
+ }
1984
+ } else {
1985
+ GGML_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, (%8.2f)\n",
1986
+ __func__,
1987
+ size_aligned / 1024.0 / 1024.0,
1988
+ device.currentAllocatedSize / 1024.0 / 1024.0);
1989
+ }
1990
+ #endif
1991
+ #endif
1992
+ GGML_UNUSED(device);
1993
+ GGML_UNUSED(size_aligned);
1994
+ }
1995
+
1996
+ // rset init
1997
+ static bool ggml_metal_buffer_rset_init(ggml_metal_buffer_t buf) {
1998
+ buf->rset = nil;
1999
+
2000
+ if (!buf->use_residency_sets) {
2001
+ return true;
2002
+ }
2003
+
2004
+ #if defined(GGML_METAL_HAS_RESIDENCY_SETS)
2005
+ if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {
2006
+ MTLResidencySetDescriptor * desc = [[MTLResidencySetDescriptor alloc] init];
2007
+ desc.label = @"ggml_metal";
2008
+ desc.initialCapacity = buf->n_buffers;
2009
+
2010
+ NSError * error;
2011
+ buf->rset = [buf->dev->mtl_device newResidencySetWithDescriptor:desc error:&error];
2012
+ if (error) {
2013
+ GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
2014
+ [desc release];
2015
+ return false;
2016
+ }
2017
+
2018
+ [desc release];
2019
+
2020
+ for (int i = 0; i < buf->n_buffers; i++) {
2021
+ [buf->rset addAllocation:buf->buffers[i].metal];
2022
+ }
2023
+
2024
+ [buf->rset commit];
2025
+ [buf->rset requestResidency];
2026
+
2027
+ return true;
2028
+ }
2029
+ #endif
2030
+
2031
+ return true;
2032
+ }
2033
+
2034
+ // rset free
2035
+ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) {
2036
+ #if defined(GGML_METAL_HAS_RESIDENCY_SETS)
2037
+ if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {
2038
+ if (buf->rset) {
2039
+ [buf->rset endResidency];
2040
+ [buf->rset removeAllAllocations];
2041
+ [buf->rset commit];
2042
+ [buf->rset release];
2043
+ }
2044
+ }
2045
+ #else
2046
+ GGML_UNUSED(buf);
2047
+ #endif
2048
+ }
2049
+
2050
+ static void * ggml_metal_host_malloc(size_t n) {
2051
+ void * data = NULL;
2052
+
2053
+ #if TARGET_OS_OSX
2054
+ kern_return_t err = vm_allocate((vm_map_t) mach_task_self(), (void *) &data, n, VM_FLAGS_ANYWHERE);
2055
+ if (err != KERN_SUCCESS) {
2056
+ GGML_LOG_ERROR("%s: error: vm_allocate failed\n", __func__);
2057
+ return NULL;
2058
+ }
2059
+ #else
2060
+ const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);
2061
+ if (result != 0) {
2062
+ GGML_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);
2063
+ return NULL;
2064
+ }
2065
+ #endif
2066
+
2067
+ return data;
2068
+ }
2069
+
2070
+ ggml_metal_buffer_t ggml_metal_buffer_init(ggml_metal_device_t dev, size_t size, bool shared) {
2071
+ ggml_metal_buffer_t res = calloc(1, sizeof(struct ggml_metal_buffer));
2072
+
2073
+ res->dev = dev;
2074
+
2075
+ const size_t size_page = sysconf(_SC_PAGESIZE);
2076
+
2077
+ size_t size_aligned = size;
2078
+ if ((size_aligned % size_page) != 0) {
2079
+ size_aligned += (size_page - (size_aligned % size_page));
2080
+ }
2081
+
2082
+ const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);
2083
+
2084
+ shared = shared && props_dev->use_shared_buffers;
2085
+
2086
+ // allocate shared buffer if the device supports it and it is required by the buffer type
2087
+ if (shared) {
2088
+ res->all_data = ggml_metal_host_malloc(size_aligned);
2089
+ res->is_shared = true;
2090
+ } else {
2091
+ // use virtual address
2092
+ res->all_data = (void *) atomic_fetch_add_explicit(&dev->addr_virt, size_aligned, memory_order_relaxed);
2093
+ res->is_shared = false;
2094
+ }
2095
+ res->all_size = size_aligned;
2096
+
2097
+ res->owned = true;
2098
+
2099
+ res->n_buffers = 1;
2100
+
2101
+ if (res->all_data != NULL) {
2102
+ res->buffers[0].size = size;
2103
+ res->buffers[0].metal = nil;
2104
+
2105
+ if (size_aligned > 0) {
2106
+ if (props_dev->use_shared_buffers && shared) {
2107
+ res->buffers[0].metal = [res->dev->mtl_device newBufferWithBytesNoCopy:res->all_data
2108
+ length:size_aligned
2109
+ options:MTLResourceStorageModeShared
2110
+ deallocator:nil];
2111
+ } else {
2112
+ res->buffers[0].metal = [res->dev->mtl_device newBufferWithLength:size_aligned options:MTLResourceStorageModePrivate];
2113
+ }
2114
+ }
2115
+
2116
+ res->buffers[0].data = res->all_data;
2117
+ }
2118
+
2119
+ if (size_aligned > 0 && (res->all_data == NULL || res->buffers[0].metal == nil)) {
2120
+ GGML_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);
2121
+ free(res);
2122
+ return NULL;
2123
+ }
2124
+
2125
+ res->use_residency_sets = props_dev->use_residency_sets;
2126
+
2127
+ if (!ggml_metal_buffer_rset_init(res)) {
2128
+ GGML_LOG_ERROR("%s: error: failed to initialize residency set\n", __func__);
2129
+ free(res);
2130
+ return NULL;
2131
+ }
2132
+
2133
+ ggml_metal_device_rsets_add(dev, res->rset);
2134
+
2135
+ //ggml_metal_log_allocated_size(device, size_aligned);
2136
+
2137
+ return res;
2138
+ }
2139
+
2140
+ ggml_metal_buffer_t ggml_metal_buffer_map(ggml_metal_device_t dev, void * ptr, size_t size, size_t max_tensor_size) {
2141
+ ggml_metal_buffer_t res = calloc(1, sizeof(struct ggml_metal_buffer));
2142
+
2143
+ res->dev = dev;
2144
+
2145
+ res->all_data = ptr;
2146
+ res->all_size = size;
2147
+
2148
+ res->is_shared = true;
2149
+ res->owned = false;
2150
+
2151
+ res->n_buffers = 0;
2152
+
2153
+ const size_t size_page = sysconf(_SC_PAGESIZE);
2154
+
2155
+ // page-align the data ptr
2156
+ {
2157
+ const uintptr_t offs = (uintptr_t) ptr % size_page;
2158
+ ptr = (void *) ((char *) ptr - offs);
2159
+ size += offs;
2160
+ }
2161
+
2162
+ size_t size_aligned = size;
2163
+ if ((size_aligned % size_page) != 0) {
2164
+ size_aligned += (size_page - (size_aligned % size_page));
2165
+ }
2166
+
2167
+ const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);
2168
+
2169
+ // the buffer fits into the max buffer size allowed by the device
2170
+ if (size_aligned <= props_dev->max_buffer_size) {
2171
+ res->buffers[res->n_buffers].data = ptr;
2172
+ res->buffers[res->n_buffers].size = size;
2173
+ res->buffers[res->n_buffers].metal = nil;
2174
+
2175
+ if (size_aligned > 0) {
2176
+ res->buffers[res->n_buffers].metal = [res->dev->mtl_device newBufferWithBytesNoCopy:ptr length:size_aligned options:MTLResourceStorageModeShared deallocator:nil];
2177
+
2178
+ if (res->buffers[res->n_buffers].metal == nil) {
2179
+ GGML_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);
2180
+ free(res);
2181
+ return NULL;
2182
+ }
2183
+ }
2184
+
2185
+ ggml_metal_log_allocated_size(res->dev->mtl_device, size_aligned);
2186
+
2187
+ ++res->n_buffers;
2188
+ } else {
2189
+ // this overlap between the views will guarantee that the tensor with the maximum size will fully fit into
2190
+ // one of the views
2191
+ const size_t size_ovlp = ((max_tensor_size + size_page - 1) / size_page + 1) * size_page; // round-up 2 pages just in case
2192
+ const size_t size_step = props_dev->max_buffer_size - size_ovlp;
2193
+ const size_t size_view = props_dev->max_buffer_size;
2194
+
2195
+ for (size_t i = 0; i < size; i += size_step) {
2196
+ const size_t size_step_aligned = (i + size_view <= size) ? size_view : (size_aligned - i);
2197
+
2198
+ res->buffers[res->n_buffers].data = (void *) ((uint8_t *) ptr + i);
2199
+ res->buffers[res->n_buffers].size = size_step_aligned;
2200
+ res->buffers[res->n_buffers].metal = nil;
2201
+
2202
+ if (size_step_aligned > 0) {
2203
+ res->buffers[res->n_buffers].metal = [res->dev->mtl_device newBufferWithBytesNoCopy:(void *) ((uint8_t *) ptr + i) length:size_step_aligned options:MTLResourceStorageModeShared deallocator:nil];
2204
+
2205
+ if (res->buffers[res->n_buffers].metal == nil) {
2206
+ GGML_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_step_aligned / 1024.0 / 1024.0);
2207
+ free(res);
2208
+ return NULL;
2209
+ }
2210
+ }
2211
+
2212
+ ggml_metal_log_allocated_size(res->dev->mtl_device, size_step_aligned);
2213
+
2214
+ if (i + size_step < size) {
2215
+ GGML_LOG_INFO("\n");
2216
+ }
2217
+
2218
+ ++res->n_buffers;
2219
+ }
2220
+ }
2221
+
2222
+ res->use_residency_sets = props_dev->use_residency_sets;
2223
+
2224
+ if (!ggml_metal_buffer_rset_init(res)) {
2225
+ GGML_LOG_ERROR("%s: error: failed to initialize residency set\n", __func__);
2226
+ free(res);
2227
+ return NULL;
2228
+ }
2229
+
2230
+ ggml_metal_device_rsets_add(dev, res->rset);
2231
+
2232
+ return res;
2233
+ }
2234
+
2235
+ void ggml_metal_buffer_free(ggml_metal_buffer_t buf) {
2236
+ @autoreleasepool {
2237
+ ggml_metal_device_rsets_rm(buf->dev, buf->rset);
2238
+
2239
+ for (int i = 0; i < buf->n_buffers; i++) {
2240
+ [buf->buffers[i].metal release];
2241
+ }
2242
+
2243
+ ggml_metal_buffer_rset_free(buf);
2244
+ }
2245
+
2246
+ if (buf->is_shared && buf->owned) {
2247
+ #if TARGET_OS_OSX
2248
+ vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)buf->all_data, buf->all_size);
2249
+ #else
2250
+ free(buf->all_data);
2251
+ #endif
2252
+ }
2253
+
2254
+ free(buf);
2255
+ }
2256
+
2257
+ void * ggml_metal_buffer_get_base(ggml_metal_buffer_t buf) {
2258
+ return buf->all_data;
2259
+ }
2260
+
2261
+ bool ggml_metal_buffer_is_shared(ggml_metal_buffer_t buf) {
2262
+ return buf->is_shared;
2263
+ }
2264
+
2265
+ void ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
2266
+ if (buf->is_shared) {
2267
+ memset((char *) tensor->data + offset, value, size);
2268
+ return;
2269
+ }
2270
+
2271
+ @autoreleasepool {
2272
+ // dst
2273
+ struct ggml_metal_buffer_id bid_dst = ggml_metal_buffer_get_id(buf, tensor);
2274
+ bid_dst.offs += offset;
2275
+
2276
+ id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];
2277
+
2278
+ {
2279
+ id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
2280
+
2281
+ [encoder fillBuffer:bid_dst.metal
2282
+ range:NSMakeRange(bid_dst.offs, bid_dst.offs + size)
2283
+ value:value];
2284
+
2285
+ [encoder endEncoding];
2286
+ }
2287
+
2288
+ [cmd_buf commit];
2289
+ [cmd_buf waitUntilCompleted];
2290
+ }
2291
+ }
2292
+
2293
+ void ggml_metal_buffer_set_tensor(ggml_metal_buffer_t buf, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
2294
+ if (buf->is_shared) {
2295
+ memcpy((char *) tensor->data + offset, data, size);
2296
+ return;
2297
+ }
2298
+
2299
+ @autoreleasepool {
2300
+ // src
2301
+ void * data_ptr = (void *)(uintptr_t) data; // "const cast" the src data
2302
+ id<MTLBuffer> buf_src = [buf->dev->mtl_device newBufferWithBytesNoCopy:data_ptr
2303
+ length:size
2304
+ options:MTLResourceStorageModeShared
2305
+ deallocator:nil];
2306
+
2307
+ GGML_ASSERT(buf_src);
2308
+
2309
+ // dst
2310
+ struct ggml_metal_buffer_id bid_dst = ggml_metal_buffer_get_id(buf, tensor);
2311
+ bid_dst.offs += offset;
2312
+
2313
+ // note: for experimentation purposes, here we use a semaphore to wait for the copy to complete
2314
+ // this is alternative to waitUntilCompleted, which should be faster, but don't seem to make much difference
2315
+ dispatch_semaphore_t completion_semaphore = dispatch_semaphore_create(0);
2316
+
2317
+ id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];
2318
+
2319
+ {
2320
+ id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
2321
+
2322
+ [encoder copyFromBuffer:buf_src
2323
+ sourceOffset:0
2324
+ toBuffer:bid_dst.metal
2325
+ destinationOffset:bid_dst.offs
2326
+ size:size];
2327
+
2328
+ [encoder endEncoding];
2329
+ }
2330
+
2331
+ [cmd_buf addCompletedHandler:^(id<MTLCommandBuffer> cb) {
2332
+ // TODO: can check for errors here
2333
+ GGML_UNUSED(cb);
2334
+
2335
+ dispatch_semaphore_signal(completion_semaphore);
2336
+ }];
2337
+
2338
+ [cmd_buf commit];
2339
+
2340
+ dispatch_semaphore_wait(completion_semaphore, DISPATCH_TIME_FOREVER);
2341
+ dispatch_release(completion_semaphore);
2342
+
2343
+ //[cmd_buf waitUntilCompleted];
2344
+ }
2345
+ }
2346
+
2347
+ void ggml_metal_buffer_get_tensor(ggml_metal_buffer_t buf, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
2348
+ if (buf->is_shared) {
2349
+ memcpy(data, (const char *) tensor->data + offset, size);
2350
+ return;
2351
+ }
2352
+
2353
+ @autoreleasepool {
2354
+ // src
2355
+ struct ggml_metal_buffer_id bid_src = ggml_metal_buffer_get_id(buf, tensor);
2356
+ bid_src.offs += offset;
2357
+
2358
+ // dst
2359
+ id<MTLBuffer> buf_dst = [buf->dev->mtl_device newBufferWithBytesNoCopy:data
2360
+ length:size
2361
+ options:MTLResourceStorageModeShared
2362
+ deallocator:nil];
2363
+
2364
+ GGML_ASSERT(buf_dst);
2365
+
2366
+ id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];
2367
+
2368
+ {
2369
+ id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
2370
+
2371
+ [encoder copyFromBuffer:bid_src.metal
2372
+ sourceOffset:bid_src.offs
2373
+ toBuffer:buf_dst
2374
+ destinationOffset:0
2375
+ size:size];
2376
+
2377
+ [encoder endEncoding];
2378
+ }
2379
+
2380
+ [cmd_buf commit];
2381
+ [cmd_buf waitUntilCompleted];
2382
+ }
2383
+ }
2384
+
2385
+ bool ggml_metal_buffer_cpy_tensor(ggml_metal_buffer_t buf_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {
2386
+ ggml_metal_buffer_t buf_src = (ggml_metal_buffer_t)src->buffer->context;
2387
+
2388
+ const size_t size = ggml_nbytes(src);
2389
+
2390
+ // if both buffers are shared, we can use memcpy directly
2391
+ if (buf_dst->is_shared && buf_src->is_shared) {
2392
+ memcpy(dst->data, src->data, size);
2393
+ return true;
2394
+ }
2395
+
2396
+ // for private buffers, we need to use Metal blit commands
2397
+ @autoreleasepool {
2398
+ struct ggml_metal_buffer_id bid_src = ggml_metal_buffer_get_id(buf_src, src);
2399
+ struct ggml_metal_buffer_id bid_dst = ggml_metal_buffer_get_id(buf_dst, dst);
2400
+
2401
+ if (bid_src.metal == nil || bid_dst.metal == nil) {
2402
+ return false;
2403
+ }
2404
+
2405
+ id<MTLCommandBuffer> cmd_buf = [buf_dst->dev->mtl_queue commandBufferWithUnretainedReferences];
2406
+
2407
+ {
2408
+ id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
2409
+
2410
+ [encoder copyFromBuffer:bid_src.metal
2411
+ sourceOffset:bid_src.offs
2412
+ toBuffer:bid_dst.metal
2413
+ destinationOffset:bid_dst.offs
2414
+ size:size];
2415
+
2416
+ [encoder endEncoding];
2417
+ }
2418
+
2419
+ [cmd_buf commit];
2420
+ [cmd_buf waitUntilCompleted];
2421
+ }
2422
+
2423
+ return true;
2424
+ }
2425
+
2426
+ void ggml_metal_buffer_clear(ggml_metal_buffer_t buf, uint8_t value) {
2427
+ if (buf->is_shared) {
2428
+ memset(buf->all_data, value, buf->all_size);
2429
+ return;
2430
+ }
2431
+
2432
+ @autoreleasepool {
2433
+ id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];
2434
+
2435
+ {
2436
+ id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
2437
+
2438
+ [encoder fillBuffer:buf->buffers[0].metal
2439
+ range:NSMakeRange(0, buf->buffers[0].size)
2440
+ value:value];
2441
+
2442
+ [encoder endEncoding];
2443
+ }
2444
+
2445
+ [cmd_buf commit];
2446
+ [cmd_buf waitUntilCompleted];
2447
+ }
2448
+ }
2449
+
2450
+ struct ggml_metal_buffer_id ggml_metal_buffer_get_id(ggml_metal_buffer_t buf, const struct ggml_tensor * t) {
2451
+ struct ggml_metal_buffer_id res = { nil, 0 };
2452
+
2453
+ const int64_t tsize = ggml_nbytes(t);
2454
+
2455
+ // find the view that contains the tensor fully
2456
+ for (int i = 0; i < buf->n_buffers; ++i) {
2457
+ const int64_t ioffs = (int64_t) t->data - (int64_t) buf->buffers[i].data;
2458
+
2459
+ //GGML_LOG_INFO("ioffs = %10ld, tsize = %10ld, sum = %10ld, buf->buffers[%d].size = %10ld\n", ioffs, tsize, ioffs + tsize, i, buf->buffers[i].size);
2460
+ if (ioffs >= 0 && ioffs + tsize <= (int64_t) buf->buffers[i].size) {
2461
+ res.metal = buf->buffers[i].metal;
2462
+ res.offs = (size_t) ioffs;
2463
+
2464
+ //GGML_LOG_INFO("%s: tensor '%16s', offs = %8ld\n", __func__, t->name, *offs);
2465
+
2466
+ return res;
2467
+ }
2468
+ }
2469
+
2470
+ GGML_LOG_ERROR("%s: error: tensor '%s' buffer is nil\n", __func__, t->name);
2471
+
2472
+ return res;
2473
+ }