elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,3225 @@
1
+ #include "common.h"
2
+ #include "dequantize.h"
3
+ // Q1_0 dot product: dot = d * (2 * Σ(yl[i] where bit=1) - sumy)
4
+ inline float block_q_n_dot_y(device const block_q1_0 * qb_curr, float sumy, thread float * yl, int il) {
5
+ device const uint8_t * qs = qb_curr->qs + il / 8;
6
+ const uint8_t b0 = qs[0];
7
+ const uint8_t b1 = qs[1];
8
+
9
+ float acc = 0.0f;
10
+
11
+ acc += select(0.0f, yl[ 0], bool(b0 & 0x01));
12
+ acc += select(0.0f, yl[ 1], bool(b0 & 0x02));
13
+ acc += select(0.0f, yl[ 2], bool(b0 & 0x04));
14
+ acc += select(0.0f, yl[ 3], bool(b0 & 0x08));
15
+ acc += select(0.0f, yl[ 4], bool(b0 & 0x10));
16
+ acc += select(0.0f, yl[ 5], bool(b0 & 0x20));
17
+ acc += select(0.0f, yl[ 6], bool(b0 & 0x40));
18
+ acc += select(0.0f, yl[ 7], bool(b0 & 0x80));
19
+
20
+ acc += select(0.0f, yl[ 8], bool(b1 & 0x01));
21
+ acc += select(0.0f, yl[ 9], bool(b1 & 0x02));
22
+ acc += select(0.0f, yl[10], bool(b1 & 0x04));
23
+ acc += select(0.0f, yl[11], bool(b1 & 0x08));
24
+ acc += select(0.0f, yl[12], bool(b1 & 0x10));
25
+ acc += select(0.0f, yl[13], bool(b1 & 0x20));
26
+ acc += select(0.0f, yl[14], bool(b1 & 0x40));
27
+ acc += select(0.0f, yl[15], bool(b1 & 0x80));
28
+
29
+ return qb_curr->d * (2.0f * acc - sumy);
30
+ }
31
+
32
+ // Q2_0 dot: d * (sum_lo(y) + 2*sum_hi(y) - sumy) via per-bit conditional adds
33
+ inline float block_q_n_dot_y(device const block_q2_0 * qb_curr, float sumy, thread float * yl, int il) {
34
+ device const uint8_t * qs = qb_curr->qs + (il / 4);
35
+ const uint8_t b0 = qs[0];
36
+ const uint8_t b1 = qs[1];
37
+ const uint8_t b2 = qs[2];
38
+ const uint8_t b3 = qs[3];
39
+
40
+ // Accumulate where low bit is set (bits 0,2,4,6 of each byte)
41
+ float acc_lo = 0.0f;
42
+ acc_lo += select(0.0f, yl[ 0], bool(b0 & 0x01));
43
+ acc_lo += select(0.0f, yl[ 1], bool(b0 & 0x04));
44
+ acc_lo += select(0.0f, yl[ 2], bool(b0 & 0x10));
45
+ acc_lo += select(0.0f, yl[ 3], bool(b0 & 0x40));
46
+ acc_lo += select(0.0f, yl[ 4], bool(b1 & 0x01));
47
+ acc_lo += select(0.0f, yl[ 5], bool(b1 & 0x04));
48
+ acc_lo += select(0.0f, yl[ 6], bool(b1 & 0x10));
49
+ acc_lo += select(0.0f, yl[ 7], bool(b1 & 0x40));
50
+ acc_lo += select(0.0f, yl[ 8], bool(b2 & 0x01));
51
+ acc_lo += select(0.0f, yl[ 9], bool(b2 & 0x04));
52
+ acc_lo += select(0.0f, yl[10], bool(b2 & 0x10));
53
+ acc_lo += select(0.0f, yl[11], bool(b2 & 0x40));
54
+ acc_lo += select(0.0f, yl[12], bool(b3 & 0x01));
55
+ acc_lo += select(0.0f, yl[13], bool(b3 & 0x04));
56
+ acc_lo += select(0.0f, yl[14], bool(b3 & 0x10));
57
+ acc_lo += select(0.0f, yl[15], bool(b3 & 0x40));
58
+
59
+ // Accumulate where high bit is set (bits 1,3,5,7 of each byte)
60
+ float acc_hi = 0.0f;
61
+ acc_hi += select(0.0f, yl[ 0], bool(b0 & 0x02));
62
+ acc_hi += select(0.0f, yl[ 1], bool(b0 & 0x08));
63
+ acc_hi += select(0.0f, yl[ 2], bool(b0 & 0x20));
64
+ acc_hi += select(0.0f, yl[ 3], bool(b0 & 0x80));
65
+ acc_hi += select(0.0f, yl[ 4], bool(b1 & 0x02));
66
+ acc_hi += select(0.0f, yl[ 5], bool(b1 & 0x08));
67
+ acc_hi += select(0.0f, yl[ 6], bool(b1 & 0x20));
68
+ acc_hi += select(0.0f, yl[ 7], bool(b1 & 0x80));
69
+ acc_hi += select(0.0f, yl[ 8], bool(b2 & 0x02));
70
+ acc_hi += select(0.0f, yl[ 9], bool(b2 & 0x08));
71
+ acc_hi += select(0.0f, yl[10], bool(b2 & 0x20));
72
+ acc_hi += select(0.0f, yl[11], bool(b2 & 0x80));
73
+ acc_hi += select(0.0f, yl[12], bool(b3 & 0x02));
74
+ acc_hi += select(0.0f, yl[13], bool(b3 & 0x08));
75
+ acc_hi += select(0.0f, yl[14], bool(b3 & 0x20));
76
+ acc_hi += select(0.0f, yl[15], bool(b3 & 0x80));
77
+
78
+ return qb_curr->d * (acc_lo + 2.0f * acc_hi - sumy);
79
+ }
80
+
81
+ // function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i])
82
+ // il indicates where the q4 quants begin (0 or QK4_0/4)
83
+ // we assume that the yl's have been multiplied with the appropriate scale factor
84
+ // that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)
85
+ inline float block_q_n_dot_y(device const block_q4_0 * qb_curr, float sumy, thread float * yl, int il) {
86
+ float d = qb_curr->d;
87
+
88
+ float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
89
+
90
+ device const uint16_t * qs = ((device const uint16_t *) qb_curr + 1 + il/2);
91
+
92
+ for (int i = 0; i < 8; i += 2) {
93
+ acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F);
94
+ acc[1] += yl[i + 1] * (qs[i / 2] & 0x0F00);
95
+ acc[2] += yl[i + 8] * (qs[i / 2] & 0x00F0);
96
+ acc[3] += yl[i + 9] * (qs[i / 2] & 0xF000);
97
+ }
98
+
99
+ return d * (sumy * -8.f + acc[0] + acc[1] + acc[2] + acc[3]);
100
+ }
101
+
102
+ // function for calculate inner product between half a q4_1 block and 16 floats (yl), sumy is SUM(yl[i])
103
+ // il indicates where the q4 quants begin (0 or QK4_0/4)
104
+ // we assume that the yl's have been multiplied with the appropriate scale factor
105
+ // that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)
106
+ inline float block_q_n_dot_y(device const block_q4_1 * qb_curr, float sumy, thread float * yl, int il) {
107
+ float d = qb_curr->d;
108
+ float m = qb_curr->m;
109
+
110
+ float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
111
+
112
+ device const uint16_t * qs = ((device const uint16_t *) qb_curr + 2 + il/2);
113
+
114
+ for (int i = 0; i < 8; i+=2) {
115
+ acc[0] += yl[i + 0] * (qs[i / 2] & 0x000F);
116
+ acc[1] += yl[i + 1] * (qs[i / 2] & 0x0F00);
117
+ acc[2] += yl[i + 8] * (qs[i / 2] & 0x00F0);
118
+ acc[3] += yl[i + 9] * (qs[i / 2] & 0xF000);
119
+ }
120
+
121
+ return d * (acc[0] + acc[1] + acc[2] + acc[3]) + sumy * m;
122
+ }
123
+
124
+ // function for calculate inner product between half a q5_0 block and 16 floats (yl), sumy is SUM(yl[i])
125
+ // il indicates where the q5 quants begin (0 or QK5_0/4)
126
+ // we assume that the yl's have been multiplied with the appropriate scale factor
127
+ // that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)
128
+ inline float block_q_n_dot_y(device const block_q5_0 * qb_curr, float sumy, thread float * yl, int il) {
129
+ float d = qb_curr->d;
130
+
131
+ float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
132
+
133
+ device const uint16_t * qs = ((device const uint16_t *)qb_curr + 3 + il/2);
134
+ const uint32_t qh = *((device const uint32_t *)qb_curr->qh);
135
+
136
+ for (int i = 0; i < 8; i+=2) {
137
+ acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010));
138
+ acc[1] += yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000));
139
+ acc[2] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100));
140
+ acc[3] += yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000));
141
+ }
142
+
143
+ return d * (sumy * -16.f + acc[0] + acc[1] + acc[2] + acc[3]);
144
+ }
145
+
146
+ // function for calculate inner product between half a q5_1 block and 16 floats (yl), sumy is SUM(yl[i])
147
+ // il indicates where the q5 quants begin (0 or QK5_1/4)
148
+ // we assume that the yl's have been multiplied with the appropriate scale factor
149
+ // that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)
150
+ inline float block_q_n_dot_y(device const block_q5_1 * qb_curr, float sumy, thread float * yl, int il) {
151
+ float d = qb_curr->d;
152
+ float m = qb_curr->m;
153
+
154
+ float acc[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
155
+
156
+ device const uint16_t * qs = ((device const uint16_t *)qb_curr + 4 + il/2);
157
+ const uint32_t qh = *((device const uint32_t *)qb_curr->qh);
158
+
159
+ for (int i = 0; i < 8; i+=2) {
160
+ acc[0] += yl[i + 0] * ((qs[i / 2] & 0x000F) | ((qh >> (i+0+il ) << 4 ) & 0x00010));
161
+ acc[1] += yl[i + 1] * ((qs[i / 2] & 0x0F00) | ((qh >> (i+1+il ) << 12) & 0x01000));
162
+ acc[2] += yl[i + 8] * ((qs[i / 2] & 0x00F0) | ((qh >> (i+0+il+QK5_0/2) << 8 ) & 0x00100));
163
+ acc[3] += yl[i + 9] * ((qs[i / 2] & 0xF000) | ((qh >> (i+1+il+QK5_0/2) << 16) & 0x10000));
164
+ }
165
+
166
+ return d * (acc[0] + acc[1] + acc[2] + acc[3]) + sumy * m;
167
+ }
168
+
169
+ template<short NR0>
170
+ static inline void helper_mv_reduce_and_write(
171
+ device float * dst_f32,
172
+ float sumf[NR0],
173
+ const int r0,
174
+ const int ne01,
175
+ ushort tiisg,
176
+ ushort sgitg,
177
+ threadgroup char * shmem) {
178
+ constexpr short NW = N_SIMDWIDTH;
179
+
180
+ threadgroup float * shmem_f32[NR0];
181
+
182
+ for (short row = 0; row < NR0; ++row) {
183
+ shmem_f32[row] = (threadgroup float *) shmem + NW*row;
184
+
185
+ if (sgitg == 0) {
186
+ shmem_f32[row][tiisg] = 0.0f;
187
+ }
188
+
189
+ sumf[row] = simd_sum(sumf[row]);
190
+ }
191
+
192
+ threadgroup_barrier(mem_flags::mem_threadgroup);
193
+
194
+ for (short row = 0; row < NR0; ++row) {
195
+ if (tiisg == 0) {
196
+ shmem_f32[row][sgitg] = sumf[row];
197
+ }
198
+ }
199
+
200
+ threadgroup_barrier(mem_flags::mem_threadgroup);
201
+
202
+ for (short row = 0; row < NR0 && r0 + row < ne01; ++row) {
203
+ float tot = simd_sum(shmem_f32[row][tiisg]);
204
+
205
+ if (tiisg == 0 && sgitg == 0) {
206
+ dst_f32[r0 + row] = tot;
207
+ }
208
+ }
209
+ }
210
+
211
+ constant short FC_mul_mv_nsg [[function_constant(FC_MUL_MV + 0)]];
212
+ constant short FC_mul_mv_nxpsg [[function_constant(FC_MUL_MV + 1)]];
213
+ constant short FC_mul_mv_ne12 [[function_constant(FC_MUL_MV + 2)]];
214
+ constant short FC_mul_mv_r2 [[function_constant(FC_MUL_MV + 3)]];
215
+ constant short FC_mul_mv_r3 [[function_constant(FC_MUL_MV + 4)]];
216
+
217
+ template<typename block_q_type, short NR0, typename args_t>
218
+ void mul_vec_q_n_f32_impl(
219
+ args_t args,
220
+ device const char * src0,
221
+ device const char * src1,
222
+ device char * dst,
223
+ threadgroup char * shmem,
224
+ uint3 tgpig,
225
+ ushort tiisg,
226
+ ushort sgitg) {
227
+ const short NSG = FC_mul_mv_nsg;
228
+
229
+ constexpr short NW = N_SIMDWIDTH;
230
+ constexpr short NQ = 16;
231
+
232
+ const int nb = args.ne00/QK4_0;
233
+
234
+ const int r0 = (tgpig.x*NSG + sgitg)*NR0;
235
+ //const int r0 = tgpig.x*NR0;
236
+ const int r1 = tgpig.y;
237
+ const int im = tgpig.z;
238
+
239
+ const uint i12 = im%FC_mul_mv_ne12;
240
+ const uint i13 = im/FC_mul_mv_ne12;
241
+
242
+ //const uint64_t offset0 = r0*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
243
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
244
+
245
+ //device const block_q_type * x = (device const block_q_type *) (src0 + offset0);
246
+ device const float * y = (device const float *) (src1 + offset1);
247
+
248
+ // pointers to src0 rows
249
+ device const block_q_type * ax[NR0];
250
+ FOR_UNROLL (int row = 0; row < NR0; ++row) {
251
+ const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
252
+
253
+ ax[row] = (device const block_q_type *) ((device char *) src0 + offset0);
254
+ }
255
+
256
+ float sumf[NR0] = {0.f};
257
+
258
+ const short ix = (tiisg/(NW/NQ));
259
+ const short il = (tiisg%(NW/NQ))*8;
260
+
261
+ //const int ib0 = sgitg*NQ + ix;
262
+ const int ib0 = ix;
263
+
264
+ float yl[16]; // src1 vector cache
265
+
266
+ //device const float * yb = y + ix*QK4_0 + il;
267
+ device const float * yb = y + ib0*QK4_0 + il;
268
+
269
+ // each thread in a SIMD group deals with half a block.
270
+ //for (int ib = ib0; ib < nb; ib += NSG*NQ) {
271
+ for (int ib = ib0; ib < nb; ib += NQ) {
272
+ float sumy[2] = { 0.f, 0.f };
273
+
274
+ FOR_UNROLL (short i = 0; i < 8; i += 2) {
275
+ sumy[0] += yb[i + 0] + yb[i + 1];
276
+ yl[i + 0] = yb[i + 0];
277
+ yl[i + 1] = yb[i + 1]/256.f;
278
+
279
+ sumy[1] += yb[i + 16] + yb[i + 17];
280
+ yl[i + 8] = yb[i + 16]/16.f;
281
+ yl[i + 9] = yb[i + 17]/4096.f;
282
+ }
283
+
284
+ FOR_UNROLL (short row = 0; row < NR0; row++) {
285
+ sumf[row] += block_q_n_dot_y(ax[row] + ib, sumy[0] + sumy[1], yl, il);
286
+ }
287
+
288
+ yb += QK4_0 * 16;
289
+ //yb += NSG*NQ*QK4_0;
290
+ }
291
+
292
+ device float * dst_f32 = (device float *) dst + im*args.ne0*args.ne1 + r1*args.ne0;
293
+
294
+ //helper_mv_reduce_and_write<NR0>(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem);
295
+
296
+ for (int row = 0; row < NR0; ++row) {
297
+ const float tot = simd_sum(sumf[row]);
298
+
299
+ if (tiisg == 0 && r0 + row < args.ne01) {
300
+ dst_f32[r0 + row] = tot;
301
+ }
302
+ }
303
+ }
304
+
305
+ template<int nr0, typename args_t>
306
+ void kernel_mul_mv_q1_0_f32_impl(
307
+ args_t args,
308
+ device const char * src0,
309
+ device const char * src1,
310
+ device char * dst,
311
+ threadgroup char * shmem,
312
+ uint3 tgpig,
313
+ ushort tiisg,
314
+ ushort sgitg) {
315
+ const short NSG = FC_mul_mv_nsg;
316
+
317
+ const int nb = args.ne00/QK1_0;
318
+
319
+ const int r0 = tgpig.x;
320
+ const int r1 = tgpig.y;
321
+ const int im = tgpig.z;
322
+
323
+ const int first_row = (r0 * NSG + sgitg) * nr0;
324
+
325
+ const uint i12 = im%FC_mul_mv_ne12;
326
+ const uint i13 = im/FC_mul_mv_ne12;
327
+
328
+ const uint64_t offset1 = r1*args.nb11 + (i12)*args.nb12 + (i13)*args.nb13;
329
+
330
+ device const float * y = (device const float *) (src1 + offset1);
331
+
332
+ device const block_q1_0 * ax[nr0];
333
+ for (int row = 0; row < nr0; ++row) {
334
+ const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
335
+ ax[row] = (device const block_q1_0 *) ((device char *) src0 + offset0);
336
+ }
337
+
338
+ float yl[16];
339
+ float sumf[nr0] = {0.f};
340
+
341
+ const short ix = (tiisg/8);
342
+ const short il = (tiisg%8)*16;
343
+
344
+ device const float * yb = y + ix*QK1_0 + il;
345
+
346
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) {
347
+ float sumy = 0.f;
348
+
349
+ FOR_UNROLL (short i = 0; i < 16; i++) {
350
+ yl[i] = yb[i];
351
+ sumy += yb[i];
352
+ }
353
+
354
+ FOR_UNROLL (short row = 0; row < nr0; row++) {
355
+ sumf[row] += block_q_n_dot_y(ax[row] + ib, sumy, yl, il);
356
+ }
357
+
358
+ yb += QK1_0 * (N_SIMDWIDTH/8);
359
+ }
360
+
361
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
362
+
363
+ for (int row = 0; row < nr0; ++row) {
364
+ const float tot = simd_sum(sumf[row]);
365
+
366
+ if (tiisg == 0 && first_row + row < args.ne01) {
367
+ dst_f32[first_row + row] = tot;
368
+ }
369
+ }
370
+ }
371
+
372
+ [[host_name("kernel_mul_mv_q1_0_f32")]]
373
+ kernel void kernel_mul_mv_q1_0_f32(
374
+ constant ggml_metal_kargs_mul_mv & args,
375
+ device const char * src0,
376
+ device const char * src1,
377
+ device char * dst,
378
+ uint3 tgpig[[threadgroup_position_in_grid]],
379
+ ushort tiisg[[thread_index_in_simdgroup]],
380
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
381
+ kernel_mul_mv_q1_0_f32_impl<N_R0_Q1_0, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
382
+ }
383
+
384
+ template<int nr0, typename args_t>
385
+ void kernel_mul_mv_q2_0_f32_impl(
386
+ args_t args,
387
+ device const char * src0,
388
+ device const char * src1,
389
+ device char * dst,
390
+ threadgroup char * shmem,
391
+ uint3 tgpig,
392
+ ushort tiisg,
393
+ ushort sgitg) {
394
+ const short NSG = FC_mul_mv_nsg;
395
+
396
+ const int nb = args.ne00/QK2_0;
397
+
398
+ const int r0 = tgpig.x;
399
+ const int r1 = tgpig.y;
400
+ const int im = tgpig.z;
401
+
402
+ const int first_row = (r0 * NSG + sgitg) * nr0;
403
+
404
+ const uint i12 = im%FC_mul_mv_ne12;
405
+ const uint i13 = im/FC_mul_mv_ne12;
406
+
407
+ const uint64_t offset1 = r1*args.nb11 + (i12)*args.nb12 + (i13)*args.nb13;
408
+
409
+ device const float * y = (device const float *) (src1 + offset1);
410
+
411
+ device const block_q2_0 * ax[nr0];
412
+ for (int row = 0; row < nr0; ++row) {
413
+ const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
414
+ ax[row] = (device const block_q2_0 *) ((device char *) src0 + offset0);
415
+ }
416
+
417
+ float yl[16];
418
+ float sumf[nr0] = {0.f};
419
+
420
+ // group 64: 4 sub-blocks of 16 weights per Q2_0 block
421
+ const short ix = (tiisg/4);
422
+ const short il = (tiisg%4)*16;
423
+
424
+ device const float * yb = y + ix*QK2_0 + il;
425
+
426
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/4) {
427
+ float sumy = 0.f;
428
+
429
+ FOR_UNROLL (short i = 0; i < 16; i++) {
430
+ yl[i] = yb[i];
431
+ sumy += yb[i];
432
+ }
433
+
434
+ FOR_UNROLL (short row = 0; row < nr0; row++) {
435
+ sumf[row] += block_q_n_dot_y(ax[row] + ib, sumy, yl, il);
436
+ }
437
+
438
+ yb += QK2_0 * (N_SIMDWIDTH/4);
439
+ }
440
+
441
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
442
+
443
+ for (int row = 0; row < nr0; ++row) {
444
+ const float tot = simd_sum(sumf[row]);
445
+
446
+ if (tiisg == 0 && first_row + row < args.ne01) {
447
+ dst_f32[first_row + row] = tot;
448
+ }
449
+ }
450
+ }
451
+
452
+ [[host_name("kernel_mul_mv_q2_0_f32")]]
453
+ kernel void kernel_mul_mv_q2_0_f32(
454
+ constant ggml_metal_kargs_mul_mv & args,
455
+ device const char * src0,
456
+ device const char * src1,
457
+ device char * dst,
458
+ uint3 tgpig[[threadgroup_position_in_grid]],
459
+ ushort tiisg[[thread_index_in_simdgroup]],
460
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
461
+ kernel_mul_mv_q2_0_f32_impl<N_R0_Q2_0, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
462
+ }
463
+
464
+ kernel void kernel_mul_mv_q4_0_f32(
465
+ constant ggml_metal_kargs_mul_mv & args,
466
+ device const char * src0,
467
+ device const char * src1,
468
+ device char * dst,
469
+ threadgroup char * shmem [[threadgroup(0)]],
470
+ uint3 tgpig[[threadgroup_position_in_grid]],
471
+ ushort tiisg[[thread_index_in_simdgroup]],
472
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
473
+ mul_vec_q_n_f32_impl<block_q4_0, N_R0_Q4_0, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
474
+ }
475
+
476
+ kernel void kernel_mul_mv_q4_1_f32(
477
+ constant ggml_metal_kargs_mul_mv & args,
478
+ device const char * src0,
479
+ device const char * src1,
480
+ device char * dst,
481
+ threadgroup char * shmem [[threadgroup(0)]],
482
+ uint3 tgpig[[threadgroup_position_in_grid]],
483
+ ushort tiisg[[thread_index_in_simdgroup]],
484
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
485
+ mul_vec_q_n_f32_impl<block_q4_1, N_R0_Q4_1, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
486
+ }
487
+
488
+ kernel void kernel_mul_mv_q5_0_f32(
489
+ constant ggml_metal_kargs_mul_mv & args,
490
+ device const char * src0,
491
+ device const char * src1,
492
+ device char * dst,
493
+ threadgroup char * shmem [[threadgroup(0)]],
494
+ uint3 tgpig[[threadgroup_position_in_grid]],
495
+ ushort tiisg[[thread_index_in_simdgroup]],
496
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
497
+ mul_vec_q_n_f32_impl<block_q5_0, N_R0_Q5_0, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
498
+ }
499
+
500
+ kernel void kernel_mul_mv_q5_1_f32(
501
+ constant ggml_metal_kargs_mul_mv & args,
502
+ device const char * src0,
503
+ device const char * src1,
504
+ device char * dst,
505
+ threadgroup char * shmem [[threadgroup(0)]],
506
+ uint3 tgpig[[threadgroup_position_in_grid]],
507
+ ushort tiisg[[thread_index_in_simdgroup]],
508
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
509
+ mul_vec_q_n_f32_impl<block_q5_1, N_R0_Q5_1, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
510
+ }
511
+
512
+ template<short NR0, typename args_t>
513
+ void kernel_mul_mv_q8_0_f32_impl(
514
+ args_t args,
515
+ device const char * src0,
516
+ device const char * src1,
517
+ device char * dst,
518
+ threadgroup char * shmem,
519
+ uint3 tgpig,
520
+ ushort tiisg,
521
+ ushort sgitg) {
522
+ const short NSG = FC_mul_mv_nsg;
523
+
524
+ constexpr short NW = N_SIMDWIDTH;
525
+ constexpr short NQ = 8;
526
+
527
+ const int nb = args.ne00/QK8_0;
528
+
529
+ const int r0 = tgpig.x*NR0;
530
+ const int r1 = tgpig.y;
531
+ const int im = tgpig.z;
532
+
533
+ const uint i12 = im%FC_mul_mv_ne12;
534
+ const uint i13 = im/FC_mul_mv_ne12;
535
+
536
+ //const uint64_t offset0 = r0*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
537
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
538
+
539
+ //device const block_q8_0 * x = (device const block_q8_0 *) (src0 + offset0);
540
+ device const float * y = (device const float *) (src1 + offset1);
541
+
542
+ // pointers to src0 rows
543
+ device const block_q8_0 * ax[NR0];
544
+ FOR_UNROLL (short row = 0; row < NR0; ++row) {
545
+ const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
546
+
547
+ ax[row] = (device const block_q8_0 *) ((device char *) src0 + offset0);
548
+ }
549
+
550
+ float sumf[NR0] = { 0.f };
551
+
552
+ const short ix = tiisg/(NW/NQ);
553
+ const short il = tiisg%(NW/NQ);
554
+
555
+ const int ib0 = sgitg*NQ + ix;
556
+
557
+ float yl[NQ];
558
+
559
+ device const float * yb = y + ib0*QK8_0 + il*NQ;
560
+
561
+ // each thread in a SIMD group deals with NQ quants at a time
562
+ for (int ib = ib0; ib < nb; ib += NSG*NQ) {
563
+ for (short i = 0; i < NQ; ++i) {
564
+ yl[i] = yb[i];
565
+ }
566
+
567
+ for (short row = 0; row < NR0; row++) {
568
+ device const int8_t * qs = ax[row][ib].qs + il*NQ;
569
+
570
+ float sumq = 0.f;
571
+ FOR_UNROLL (short i = 0; i < NQ; ++i) {
572
+ sumq += qs[i] * yl[i];
573
+ }
574
+
575
+ sumf[row] += sumq*ax[row][ib].d;
576
+ }
577
+
578
+ yb += NSG*NQ*QK8_0;
579
+ }
580
+
581
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
582
+
583
+ helper_mv_reduce_and_write<NR0>(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem);
584
+ }
585
+
586
+ [[host_name("kernel_mul_mv_q8_0_f32")]]
587
+ kernel void kernel_mul_mv_q8_0_f32(
588
+ constant ggml_metal_kargs_mul_mv & args,
589
+ device const char * src0,
590
+ device const char * src1,
591
+ device char * dst,
592
+ threadgroup char * shmem [[threadgroup(0)]],
593
+ uint3 tgpig[[threadgroup_position_in_grid]],
594
+ ushort tiisg[[thread_index_in_simdgroup]],
595
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
596
+ kernel_mul_mv_q8_0_f32_impl<N_R0_Q8_0, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
597
+ }
598
+
599
+ // mat-vec kernel processing in chunks of float4
600
+ // chpb - chunks per quantization block
601
+ template<short r1ptg, typename q_t, short chpb, void (*deq_t4)(device const q_t *, short, thread float4 &) >
602
+ void kernel_mul_mv_ext_q4_f32_impl(
603
+ constant ggml_metal_kargs_mul_mv_ext & args,
604
+ device const char * src0,
605
+ device const char * src1,
606
+ device char * dst,
607
+ uint3 tgpig[[threadgroup_position_in_grid]],
608
+ ushort tiisg[[thread_index_in_simdgroup]],
609
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
610
+ const short NSG = FC_mul_mv_nsg;
611
+ const short nxpsg = FC_mul_mv_nxpsg;
612
+
613
+ const short chpt = 4; // chunks per thread
614
+
615
+ //const short nxpsg = (32);
616
+ const short nypsg = (32/nxpsg);
617
+
618
+ const short tx = tiisg%nxpsg;
619
+ const short ty = tiisg/nxpsg;
620
+
621
+ const int i01 = tgpig.x*(nypsg*NSG) + nypsg*sgitg + ty;
622
+ const int i11 = tgpig.y*r1ptg;
623
+ const int i1m = tgpig.z;
624
+
625
+ const int i12 = i1m%FC_mul_mv_ne12;
626
+ const int i13 = i1m/FC_mul_mv_ne12;
627
+
628
+ const uint64_t offset0 = i01*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
629
+ const uint64_t offset1 = i11*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
630
+
631
+ device const q_t * xq = (i01 < args.ne01) ? (device const q_t *) (src0 + offset0) + tx/chpb : (device const q_t *) src0;
632
+
633
+ device const float4 * y4[r1ptg];
634
+
635
+ for (int ir1 = 0; ir1 < r1ptg; ++ir1) {
636
+ y4[ir1] = (i11 + ir1 < args.ne11) ? (device const float4 *) (src1 + offset1 + ir1*args.nb11) + tx : (device const float4 *) src1;
637
+ }
638
+
639
+ float sumf[r1ptg] = { [ 0 ... r1ptg - 1 ] = 0.0f };
640
+
641
+ short cch = tx%chpb; // current chunk index
642
+
643
+ for (int ich = tx; 4*ich < args.ne00; ich += chpt*nxpsg) {
644
+ float4 lx[chpt];
645
+
646
+ #pragma unroll(chpt)
647
+ for (short ch = 0; ch < chpt; ++ch) {
648
+ deq_t4(xq, cch, lx[ch]);
649
+
650
+ cch += nxpsg;
651
+ if (cch >= chpb) {
652
+ xq += cch/chpb;
653
+ cch %= chpb;
654
+ }
655
+ }
656
+
657
+ #pragma unroll(chpt)
658
+ for (short ch = 0; ch < chpt; ++ch) {
659
+ #pragma unroll(r1ptg)
660
+ for (short ir1 = 0; ir1 < r1ptg; ++ir1) {
661
+ sumf[ir1] += dot(lx[ch], y4[ir1][ch*nxpsg]);
662
+ }
663
+ }
664
+
665
+ #pragma unroll(r1ptg)
666
+ for (short ir1 = 0; ir1 < r1ptg; ++ir1) {
667
+ y4[ir1] += chpt*nxpsg;
668
+ }
669
+ }
670
+
671
+ // reduce only the threads in each row
672
+ for (short ir1 = 0; ir1 < r1ptg; ++ir1) {
673
+ if (nxpsg >= 32) {
674
+ sumf[ir1] += simd_shuffle_down(sumf[ir1], 16);
675
+ }
676
+ if (nxpsg >= 16) {
677
+ sumf[ir1] += simd_shuffle_down(sumf[ir1], 8);
678
+ }
679
+ if (nxpsg >= 8) {
680
+ sumf[ir1] += simd_shuffle_down(sumf[ir1], 4);
681
+ }
682
+ if (nxpsg >= 4) {
683
+ sumf[ir1] += simd_shuffle_down(sumf[ir1], 2);
684
+ }
685
+ if (nxpsg >= 2) {
686
+ sumf[ir1] += simd_shuffle_down(sumf[ir1], 1);
687
+ }
688
+
689
+ //sumf[ir1] = simd_sum(sumf[ir1]);
690
+ }
691
+
692
+ if (tx == 0) {
693
+ for (short ir1 = 0; ir1 < r1ptg && i11 + ir1 < args.ne11; ++ir1) {
694
+ device float * dst_f32 = (device float *) dst + (uint64_t)i1m*args.ne0*args.ne1 + (uint64_t)(i11 + ir1)*args.ne0;
695
+
696
+ if (i01 < args.ne01) {
697
+ dst_f32[i01] = sumf[ir1];
698
+ }
699
+ }
700
+ }
701
+ }
702
+
703
+ // mat-vec kernel processing in chunks of float4x4
704
+ template<short r1ptg, typename q_t, short chpb, void (*deq_t4x4)(device const q_t *, short, thread float4x4 &) >
705
+ void kernel_mul_mv_ext_q4x4_f32_impl(
706
+ constant ggml_metal_kargs_mul_mv_ext & args,
707
+ device const char * src0,
708
+ device const char * src1,
709
+ device char * dst,
710
+ uint3 tgpig[[threadgroup_position_in_grid]],
711
+ ushort tiisg[[thread_index_in_simdgroup]],
712
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
713
+ const short NSG = FC_mul_mv_nsg;
714
+ const short nxpsg = FC_mul_mv_nxpsg;
715
+
716
+ const short chpt = 1;
717
+
718
+ //const short nxpsg = (32);
719
+ const short nypsg = (32/nxpsg);
720
+
721
+ const short tx = tiisg%nxpsg;
722
+ const short ty = tiisg/nxpsg;
723
+
724
+ const int i01 = tgpig.x*(nypsg*NSG) + nypsg*sgitg + ty;
725
+ const int i11 = tgpig.y*r1ptg;
726
+ const int i1m = tgpig.z;
727
+
728
+ const int i12 = i1m%FC_mul_mv_ne12;
729
+ const int i13 = i1m/FC_mul_mv_ne12;
730
+
731
+ const uint64_t offset0 = i01*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
732
+ const uint64_t offset1 = i11*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
733
+
734
+ device const q_t * xq = (i01 < args.ne01) ? (device const q_t *) (src0 + offset0) + tx/chpb : (device const q_t *) src0;
735
+
736
+ device const float4x4 * y4x4[r1ptg];
737
+
738
+ for (int ir1 = 0; ir1 < r1ptg; ++ir1) {
739
+ y4x4[ir1] = (i11 + ir1 < args.ne11) ? (device const float4x4 *) (src1 + offset1 + ir1*args.nb11) + tx : (device const float4x4 *) src1;
740
+ }
741
+
742
+ float sumf[r1ptg] = { [ 0 ... r1ptg - 1 ] = 0.0f };
743
+
744
+ short cch = tx%chpb;
745
+
746
+ for (int ich = tx; 16*ich < args.ne00; ich += chpt*nxpsg) {
747
+ float4x4 lx[chpt];
748
+
749
+ #pragma unroll(chpt)
750
+ for (short ch = 0; ch < chpt; ++ch) {
751
+ deq_t4x4(xq, cch, lx[ch]);
752
+
753
+ cch += nxpsg;
754
+ if (cch >= chpb) {
755
+ xq += cch/chpb;
756
+ cch %= chpb;
757
+ }
758
+ }
759
+
760
+ #pragma unroll(chpt)
761
+ for (short ch = 0; ch < chpt; ++ch) {
762
+ #pragma unroll(r1ptg)
763
+ for (short ir1 = 0; ir1 < r1ptg; ++ir1) {
764
+ sumf[ir1] +=
765
+ dot(lx[ch][0], y4x4[ir1][ch*nxpsg][0]) +
766
+ dot(lx[ch][1], y4x4[ir1][ch*nxpsg][1]) +
767
+ dot(lx[ch][2], y4x4[ir1][ch*nxpsg][2]) +
768
+ dot(lx[ch][3], y4x4[ir1][ch*nxpsg][3]);
769
+
770
+ }
771
+ }
772
+
773
+ #pragma unroll(r1ptg)
774
+ for (short ir1 = 0; ir1 < r1ptg; ++ir1) {
775
+ y4x4[ir1] += chpt*nxpsg;
776
+ }
777
+ }
778
+
779
+ for (short ir1 = 0; ir1 < r1ptg; ++ir1) {
780
+ if (nxpsg >= 32) {
781
+ sumf[ir1] += simd_shuffle_down(sumf[ir1], 16);
782
+ }
783
+ if (nxpsg >= 16) {
784
+ sumf[ir1] += simd_shuffle_down(sumf[ir1], 8);
785
+ }
786
+ if (nxpsg >= 8) {
787
+ sumf[ir1] += simd_shuffle_down(sumf[ir1], 4);
788
+ }
789
+ if (nxpsg >= 4) {
790
+ sumf[ir1] += simd_shuffle_down(sumf[ir1], 2);
791
+ }
792
+ if (nxpsg >= 2) {
793
+ sumf[ir1] += simd_shuffle_down(sumf[ir1], 1);
794
+ }
795
+
796
+ //sumf[ir1] = simd_sum(sumf[ir1]);
797
+ }
798
+
799
+ if (tx == 0) {
800
+ for (short ir1 = 0; ir1 < r1ptg && i11 + ir1 < args.ne11; ++ir1) {
801
+ device float * dst_f32 = (device float *) dst + (uint64_t)i1m*args.ne0*args.ne1 + (uint64_t)(i11 + ir1)*args.ne0;
802
+
803
+ if (i01 < args.ne01) {
804
+ dst_f32[i01] = sumf[ir1];
805
+ }
806
+ }
807
+ }
808
+ }
809
+
810
+ // dispatchers needed for compile-time nxpsg
811
+ // epb - elements per quantization block
812
+ template<short r1ptg, typename q_t, short epb, void (*deq_t4)(device const q_t *, short, thread float4 &)>
813
+ kernel void kernel_mul_mv_ext_q4_f32_disp(
814
+ constant ggml_metal_kargs_mul_mv_ext & args,
815
+ device const char * src0,
816
+ device const char * src1,
817
+ device char * dst,
818
+ uint3 tgpig[[threadgroup_position_in_grid]],
819
+ ushort tiisg[[thread_index_in_simdgroup]],
820
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
821
+ kernel_mul_mv_ext_q4_f32_impl<r1ptg, q_t, epb/4, deq_t4>(args, src0, src1, dst, tgpig, tiisg, sgitg);
822
+ }
823
+
824
+ template<short r1ptg, typename q_t, short epb, void (*deq_t4x4)(device const q_t *, short, thread float4x4 &)>
825
+ kernel void kernel_mul_mv_ext_q4x4_f32_disp(
826
+ constant ggml_metal_kargs_mul_mv_ext & args,
827
+ device const char * src0,
828
+ device const char * src1,
829
+ device char * dst,
830
+ uint3 tgpig[[threadgroup_position_in_grid]],
831
+ ushort tiisg[[thread_index_in_simdgroup]],
832
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
833
+ kernel_mul_mv_ext_q4x4_f32_impl<r1ptg, q_t, epb/16, deq_t4x4>(args, src0, src1, dst, tgpig, tiisg, sgitg);
834
+ }
835
+
836
+ typedef decltype(kernel_mul_mv_ext_q4_f32_disp <2, block_q8_0, 32, dequantize_q8_0_t4>) mul_mv_ext_q4_f32_t;
837
+ typedef decltype(kernel_mul_mv_ext_q4x4_f32_disp<2, block_q4_K, 256, dequantize_q4_K>) mul_mv_ext_q4x4_f32_t;
838
+
839
+ template [[host_name("kernel_mul_mv_ext_f32_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, float4, 4, dequantize_f32_t4>;
840
+ template [[host_name("kernel_mul_mv_ext_f32_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, float4, 4, dequantize_f32_t4>;
841
+ template [[host_name("kernel_mul_mv_ext_f32_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, float4, 4, dequantize_f32_t4>;
842
+ template [[host_name("kernel_mul_mv_ext_f32_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, float4, 4, dequantize_f32_t4>;
843
+
844
+ template [[host_name("kernel_mul_mv_ext_f16_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, half4, 4, dequantize_f16_t4>;
845
+ template [[host_name("kernel_mul_mv_ext_f16_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, half4, 4, dequantize_f16_t4>;
846
+ template [[host_name("kernel_mul_mv_ext_f16_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, half4, 4, dequantize_f16_t4>;
847
+ template [[host_name("kernel_mul_mv_ext_f16_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, half4, 4, dequantize_f16_t4>;
848
+
849
+ #if defined(GGML_METAL_HAS_BF16)
850
+ template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, bfloat4, 4, dequantize_bf16_t4>;
851
+ template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, bfloat4, 4, dequantize_bf16_t4>;
852
+ template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, bfloat4, 4, dequantize_bf16_t4>;
853
+ template [[host_name("kernel_mul_mv_ext_bf16_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, bfloat4, 4, dequantize_bf16_t4>;
854
+ #endif
855
+
856
+ template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q1_0, 128, dequantize_q1_0_t4>;
857
+ template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q1_0, 128, dequantize_q1_0_t4>;
858
+ template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q1_0, 128, dequantize_q1_0_t4>;
859
+ template [[host_name("kernel_mul_mv_ext_q1_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q1_0, 128, dequantize_q1_0_t4>;
860
+
861
+ template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q2_0, 64, dequantize_q2_0_t4>;
862
+ template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q2_0, 64, dequantize_q2_0_t4>;
863
+ template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q2_0, 64, dequantize_q2_0_t4>;
864
+ template [[host_name("kernel_mul_mv_ext_q2_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q2_0, 64, dequantize_q2_0_t4>;
865
+
866
+ template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q4_0, 32, dequantize_q4_0_t4>;
867
+ template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q4_0, 32, dequantize_q4_0_t4>;
868
+ template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_0, 32, dequantize_q4_0_t4>;
869
+ template [[host_name("kernel_mul_mv_ext_q4_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q4_0, 32, dequantize_q4_0_t4>;
870
+
871
+ template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q4_1, 32, dequantize_q4_1_t4>;
872
+ template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q4_1, 32, dequantize_q4_1_t4>;
873
+ template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q4_1, 32, dequantize_q4_1_t4>;
874
+ template [[host_name("kernel_mul_mv_ext_q4_1_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q4_1, 32, dequantize_q4_1_t4>;
875
+
876
+ template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q5_0, 32, dequantize_q5_0_t4>;
877
+ template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q5_0, 32, dequantize_q5_0_t4>;
878
+ template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q5_0, 32, dequantize_q5_0_t4>;
879
+ template [[host_name("kernel_mul_mv_ext_q5_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q5_0, 32, dequantize_q5_0_t4>;
880
+
881
+ template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q5_1, 32, dequantize_q5_1_t4>;
882
+ template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q5_1, 32, dequantize_q5_1_t4>;
883
+ template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q5_1, 32, dequantize_q5_1_t4>;
884
+ template [[host_name("kernel_mul_mv_ext_q5_1_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q5_1, 32, dequantize_q5_1_t4>;
885
+
886
+ template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_q8_0, 32, dequantize_q8_0_t4>;
887
+ template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_q8_0, 32, dequantize_q8_0_t4>;
888
+ template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_q8_0, 32, dequantize_q8_0_t4>;
889
+ template [[host_name("kernel_mul_mv_ext_q8_0_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_q8_0, 32, dequantize_q8_0_t4>;
890
+
891
+ template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_mxfp4, 32, dequantize_mxfp4_t4>;
892
+ template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_mxfp4, 32, dequantize_mxfp4_t4>;
893
+ template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_mxfp4, 32, dequantize_mxfp4_t4>;
894
+ template [[host_name("kernel_mul_mv_ext_mxfp4_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_mxfp4, 32, dequantize_mxfp4_t4>;
895
+
896
+ template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_2")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<2, block_iq4_nl, 32, dequantize_iq4_nl_t4>;
897
+ template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_3")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<3, block_iq4_nl, 32, dequantize_iq4_nl_t4>;
898
+ template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_4")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<4, block_iq4_nl, 32, dequantize_iq4_nl_t4>;
899
+ template [[host_name("kernel_mul_mv_ext_iq4_nl_f32_r1_5")]] kernel mul_mv_ext_q4_f32_t kernel_mul_mv_ext_q4_f32_disp<5, block_iq4_nl, 32, dequantize_iq4_nl_t4>;
900
+
901
+ template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q4_K, 256, dequantize_q4_K>;
902
+ template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q4_K, 256, dequantize_q4_K>;
903
+ template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q4_K, 256, dequantize_q4_K>;
904
+ template [[host_name("kernel_mul_mv_ext_q4_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q4_K, 256, dequantize_q4_K>;
905
+
906
+ template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q5_K, 256, dequantize_q5_K>;
907
+ template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q5_K, 256, dequantize_q5_K>;
908
+ template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q5_K, 256, dequantize_q5_K>;
909
+ template [[host_name("kernel_mul_mv_ext_q5_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q5_K, 256, dequantize_q5_K>;
910
+
911
+ template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q6_K, 256, dequantize_q6_K>;
912
+ template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q6_K, 256, dequantize_q6_K>;
913
+ template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q6_K, 256, dequantize_q6_K>;
914
+ template [[host_name("kernel_mul_mv_ext_q6_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q6_K, 256, dequantize_q6_K>;
915
+
916
+ template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q2_K, 256, dequantize_q2_K>;
917
+ template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q2_K, 256, dequantize_q2_K>;
918
+ template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q2_K, 256, dequantize_q2_K>;
919
+ template [[host_name("kernel_mul_mv_ext_q2_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q2_K, 256, dequantize_q2_K>;
920
+
921
+ template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_2")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<2, block_q3_K, 256, dequantize_q3_K>;
922
+ template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_3")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<3, block_q3_K, 256, dequantize_q3_K>;
923
+ template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_4")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<4, block_q3_K, 256, dequantize_q3_K>;
924
+ template [[host_name("kernel_mul_mv_ext_q3_K_f32_r1_5")]] kernel mul_mv_ext_q4x4_f32_t kernel_mul_mv_ext_q4x4_f32_disp<5, block_q3_K, 256, dequantize_q3_K>;
925
+
926
+ template<typename T0, typename T1, short NR0, typename args_t>
927
+ void kernel_mul_mv_t_t_impl(
928
+ args_t args,
929
+ device const char * src0,
930
+ device const char * src1,
931
+ device char * dst,
932
+ threadgroup char * shmem,
933
+ uint3 tgpig,
934
+ ushort tiisg,
935
+ ushort sgitg) {
936
+ const short NSG = FC_mul_mv_nsg;
937
+
938
+ constexpr short NW = N_SIMDWIDTH;
939
+ constexpr short NB = 32;
940
+ constexpr short NF = 8;
941
+
942
+ const int nb = args.ne00/NB;
943
+
944
+ const int r0 = tgpig.x*NR0;
945
+ const int r1 = tgpig.y;
946
+ const int im = tgpig.z;
947
+
948
+ const uint i12 = im%FC_mul_mv_ne12;
949
+ const uint i13 = im/FC_mul_mv_ne12;
950
+
951
+ //const uint64_t offset0 = r0*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
952
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
953
+
954
+ //device const T0 * x = (device const T0 *) (src0 + offset0);
955
+ device const T1 * y = (device const T1 *) (src1 + offset1);
956
+
957
+ // pointers to src0 rows
958
+ device const T0 * ax [NR0];
959
+ FOR_UNROLL (short row = 0; row < NR0; ++row) {
960
+ const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
961
+
962
+ ax[row] = (device const T0 *) ((device char *) src0 + offset0);
963
+ }
964
+
965
+ float sumf[NR0] = { 0.f };
966
+
967
+ const short ix = tiisg/(NW/NF);
968
+ const short il = tiisg%(NW/NF);
969
+
970
+ const int ib0 = sgitg*NF + ix;
971
+
972
+ T1 yl[NF];
973
+
974
+ device const T1 * yb = y + (ib0*NB + il*NF);
975
+
976
+ for (int ib = ib0; ib < nb; ib += NSG*NF) {
977
+ for (short i = 0; i < NF; ++i) {
978
+ yl[i] = yb[i];
979
+ }
980
+
981
+ for (short row = 0; row < NR0; row++) {
982
+ device const T0 * xb = ax[row] + (ib*NB + il*NF);
983
+
984
+ float sumq = 0.f;
985
+ FOR_UNROLL (short i = 0; i < NF; ++i) {
986
+ sumq += xb[i] * yl[i];
987
+ }
988
+
989
+ sumf[row] += sumq;
990
+ }
991
+
992
+ yb += NSG*NF*NW;
993
+ }
994
+
995
+ for (int i = nb*NB + sgitg*NW + tiisg; i < args.ne00; i += NW*NSG) {
996
+ for (short row = 0; row < NR0; row++) {
997
+ sumf[row] += ax[row][i] * y[i];
998
+ }
999
+ }
1000
+
1001
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
1002
+
1003
+ helper_mv_reduce_and_write<NR0>(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem);
1004
+ }
1005
+
1006
+ template<typename T0, typename T1, typename args_t>
1007
+ void kernel_mul_mv_t_t_disp(
1008
+ args_t args,
1009
+ device const char * src0,
1010
+ device const char * src1,
1011
+ device char * dst,
1012
+ threadgroup char * shmem,
1013
+ uint3 tgpig,
1014
+ ushort tiisg,
1015
+ ushort sgitg) {
1016
+ switch (args.nr0) {
1017
+ //case 1: kernel_mul_mv_t_t_impl<T0, T1, 1, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;
1018
+ case 2: kernel_mul_mv_t_t_impl<T0, T1, 2, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;
1019
+ //case 3: kernel_mul_mv_t_t_impl<T0, T1, 3, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;
1020
+ //case 4: kernel_mul_mv_t_t_impl<T0, T1, 4, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;
1021
+ }
1022
+ }
1023
+
1024
+ template<typename T0, typename T1>
1025
+ kernel void kernel_mul_mv_t_t(
1026
+ constant ggml_metal_kargs_mul_mv & args,
1027
+ device const char * src0,
1028
+ device const char * src1,
1029
+ device char * dst,
1030
+ threadgroup char * shmem [[threadgroup(0)]],
1031
+ uint3 tgpig[[threadgroup_position_in_grid]],
1032
+ ushort tiisg[[thread_index_in_simdgroup]],
1033
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
1034
+ kernel_mul_mv_t_t_disp<T0, T1, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
1035
+ }
1036
+
1037
+ typedef decltype(kernel_mul_mv_t_t<half, half>) mul_mv_t_t;
1038
+
1039
+ template [[host_name("kernel_mul_mv_f32_f32")]] kernel mul_mv_t_t kernel_mul_mv_t_t<float, float>;
1040
+ template [[host_name("kernel_mul_mv_f16_f32")]] kernel mul_mv_t_t kernel_mul_mv_t_t<half, float>;
1041
+ template [[host_name("kernel_mul_mv_f16_f16")]] kernel mul_mv_t_t kernel_mul_mv_t_t<half, half>;
1042
+ #if defined(GGML_METAL_HAS_BF16)
1043
+ template [[host_name("kernel_mul_mv_bf16_f32")]] kernel mul_mv_t_t kernel_mul_mv_t_t<bfloat, float>;
1044
+ template [[host_name("kernel_mul_mv_bf16_bf16")]] kernel mul_mv_t_t kernel_mul_mv_t_t<bfloat, bfloat>;
1045
+ #endif
1046
+
1047
+ template<typename T0, typename T04, typename T1, typename T14, short NR0, typename args_t>
1048
+ void kernel_mul_mv_t_t_4_impl(
1049
+ args_t args,
1050
+ device const char * src0,
1051
+ device const char * src1,
1052
+ device char * dst,
1053
+ threadgroup char * shmem,
1054
+ uint3 tgpig,
1055
+ ushort tiisg,
1056
+ ushort sgitg) {
1057
+ const short NSG = FC_mul_mv_nsg;
1058
+
1059
+ constexpr short NW = N_SIMDWIDTH;
1060
+ constexpr short NB = 32;
1061
+ constexpr short NF = 16;
1062
+ constexpr short NF4 = NF/4;
1063
+
1064
+ const int nb = args.ne00/NB;
1065
+
1066
+ const int r0 = tgpig.x*NR0;
1067
+ const int r1 = tgpig.y;
1068
+ const int im = tgpig.z;
1069
+
1070
+ const uint i12 = im%FC_mul_mv_ne12;
1071
+ const uint i13 = im/FC_mul_mv_ne12;
1072
+
1073
+ //const uint64_t offset0 = r0*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
1074
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
1075
+
1076
+ device const T1 * y = (device const T1 *) (src1 + offset1);
1077
+ device const T14 * y4 = (device const T14 *) (src1 + offset1);
1078
+
1079
+ // pointers to src0 rows
1080
+ device const T0 * ax [NR0];
1081
+ device const T04 * ax4[NR0];
1082
+ FOR_UNROLL (short row = 0; row < NR0; ++row) {
1083
+ const uint64_t offset0 = (r0 + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
1084
+
1085
+ ax [row] = (device const T0 *) ((device char *) src0 + offset0);
1086
+ ax4[row] = (device const T04 *) ((device char *) src0 + offset0);
1087
+ }
1088
+
1089
+ float sumf[NR0] = { 0.f };
1090
+
1091
+ const short ix = tiisg/(NW/NF);
1092
+ const short il = tiisg%(NW/NF);
1093
+
1094
+ const int ib0 = sgitg*NF + ix;
1095
+
1096
+ T14 yl4[NF4];
1097
+
1098
+ device const T14 * yb4 = y4 + (ib0*NB + il*NF)/4;
1099
+
1100
+ for (int ib = ib0; ib < nb; ib += NSG*NF) {
1101
+ for (short i = 0; i < NF4; ++i) {
1102
+ yl4[i] = yb4[i];
1103
+ }
1104
+
1105
+ for (short row = 0; row < NR0; row++) {
1106
+ device const T04 * xb4 = ax4[row] + (ib*NB + il*NF)/4;
1107
+
1108
+ float sumq = 0.f;
1109
+ FOR_UNROLL (short i = 0; i < NF4; ++i) {
1110
+ sumq += dot(float4(xb4[i]), float4(yl4[i]));
1111
+ }
1112
+
1113
+ sumf[row] += sumq;
1114
+ }
1115
+
1116
+ yb4 += NSG*NF*NW/4;
1117
+ }
1118
+
1119
+ for (int i = nb*NB + sgitg*NW + tiisg; i < args.ne00; i += NW*NSG) {
1120
+ for (short row = 0; row < NR0; row++) {
1121
+ sumf[row] += ax[row][i] * y[i];
1122
+ }
1123
+ }
1124
+
1125
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
1126
+
1127
+ helper_mv_reduce_and_write<NR0>(dst_f32, sumf, r0, args.ne01, tiisg, sgitg, shmem);
1128
+ }
1129
+
1130
+ template<typename T0, typename T04, typename T1, typename T14, typename args_t>
1131
+ void kernel_mul_mv_t_t_4_disp(
1132
+ args_t args,
1133
+ device const char * src0,
1134
+ device const char * src1,
1135
+ device char * dst,
1136
+ threadgroup char * shmem,
1137
+ uint3 tgpig,
1138
+ ushort tiisg,
1139
+ ushort sgitg) {
1140
+ switch (args.nr0) {
1141
+ //case 1: kernel_mul_mv_t_t_4_impl<T0, T04, T1, T14, 1, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;
1142
+ case 2: kernel_mul_mv_t_t_4_impl<T0, T04, T1, T14, 2, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;
1143
+ //case 3: kernel_mul_mv_t_t_4_impl<T0, T04, T1, T14, 3, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;
1144
+ //case 4: kernel_mul_mv_t_t_4_impl<T0, T04, T1, T14, 4, args_t>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg); break;
1145
+ };
1146
+ }
1147
+
1148
+ template<typename T0, typename T04, typename T1, typename T14>
1149
+ kernel void kernel_mul_mv_t_t_4(
1150
+ constant ggml_metal_kargs_mul_mv & args,
1151
+ device const char * src0,
1152
+ device const char * src1,
1153
+ device char * dst,
1154
+ threadgroup char * shmem [[threadgroup(0)]],
1155
+ uint3 tgpig[[threadgroup_position_in_grid]],
1156
+ ushort tiisg[[thread_index_in_simdgroup]],
1157
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
1158
+ kernel_mul_mv_t_t_4_disp<T0, T04, T1, T14, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
1159
+ }
1160
+
1161
+ typedef decltype(kernel_mul_mv_t_t_4<half, half4, half, half4>) mul_mv_t_t_4;
1162
+
1163
+ template [[host_name("kernel_mul_mv_f32_f32_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4<float, float4, float, float4>;
1164
+ template [[host_name("kernel_mul_mv_f16_f32_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4<half, half4, float, float4>;
1165
+ template [[host_name("kernel_mul_mv_f16_f16_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4<half, half4, half, half4>;
1166
+ #if defined(GGML_METAL_HAS_BF16)
1167
+ template [[host_name("kernel_mul_mv_bf16_f32_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4<bfloat, bfloat4, float, float4>;
1168
+ template [[host_name("kernel_mul_mv_bf16_bf16_4")]] kernel mul_mv_t_t_4 kernel_mul_mv_t_t_4<bfloat, bfloat4, bfloat, bfloat4>;
1169
+ #endif
1170
+
1171
+ template<typename T0, typename T1, typename args_t>
1172
+ void kernel_mul_mv_t_t_short_impl(
1173
+ args_t args,
1174
+ device const char * src0,
1175
+ device const char * src1,
1176
+ device char * dst,
1177
+ uint3 tgpig,
1178
+ ushort tiisg) {
1179
+ const int r0 = tgpig.x*32 + tiisg;
1180
+ const int r1 = tgpig.y;
1181
+ const int im = tgpig.z;
1182
+
1183
+ if (r0 >= args.ne01) {
1184
+ return;
1185
+ }
1186
+
1187
+ const uint i12 = im%FC_mul_mv_ne12;
1188
+ const uint i13 = im/FC_mul_mv_ne12;
1189
+
1190
+ const uint64_t offset0 = r0*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
1191
+
1192
+ device const T0 * x = (device const T0 *) (src0 + offset0);
1193
+
1194
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1;
1195
+
1196
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
1197
+
1198
+ device const T1 * y = (device const T1 *) (src1 + offset1);
1199
+
1200
+ float res = 0.0f;
1201
+
1202
+ for (int i = 0; i < args.ne00; ++i) {
1203
+ res += (float) x[i] * (float) y[i];
1204
+ }
1205
+
1206
+ dst_f32[(uint64_t)r1*args.ne0 + r0] = res;
1207
+ }
1208
+
1209
+ template<typename T0, typename T1>
1210
+ kernel void kernel_mul_mv_t_t_short(
1211
+ constant ggml_metal_kargs_mul_mv & args,
1212
+ device const char * src0,
1213
+ device const char * src1,
1214
+ device char * dst,
1215
+ uint3 tgpig[[threadgroup_position_in_grid]],
1216
+ ushort tiisg[[thread_index_in_simdgroup]]) {
1217
+ kernel_mul_mv_t_t_short_impl<T0, T1, constant ggml_metal_kargs_mul_mv &>(
1218
+ args,
1219
+ src0,
1220
+ src1,
1221
+ dst,
1222
+ tgpig,
1223
+ tiisg);
1224
+ }
1225
+
1226
+ typedef decltype(kernel_mul_mv_t_t_short<half, half>) mul_mv_t_t_short_t;
1227
+
1228
+ template [[host_name("kernel_mul_mv_f32_f32_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short<float, float>;
1229
+ template [[host_name("kernel_mul_mv_f16_f32_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short<half, float>;
1230
+ template [[host_name("kernel_mul_mv_f16_f16_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short<half, half>;
1231
+ #if defined(GGML_METAL_HAS_BF16)
1232
+ template [[host_name("kernel_mul_mv_bf16_f32_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short<bfloat, float>;
1233
+ template [[host_name("kernel_mul_mv_bf16_bf16_short")]] kernel mul_mv_t_t_short_t kernel_mul_mv_t_t_short<bfloat, bfloat>;
1234
+ #endif
1235
+
1236
+ template<int nr0, typename args_t>
1237
+ void kernel_mul_mv_q2_K_f32_impl(
1238
+ args_t args,
1239
+ device const char * src0,
1240
+ device const char * src1,
1241
+ device char * dst,
1242
+ threadgroup char * shmem,
1243
+ uint3 tgpig,
1244
+ ushort tiisg,
1245
+ ushort sgitg) {
1246
+ const short NSG = FC_mul_mv_nsg;
1247
+
1248
+ const int nb = args.ne00/QK_K;
1249
+
1250
+ const int r0 = tgpig.x;
1251
+ const int r1 = tgpig.y;
1252
+ const int im = tgpig.z;
1253
+
1254
+ const int first_row = (r0 * NSG + sgitg) * nr0;
1255
+
1256
+ const uint i12 = im%FC_mul_mv_ne12;
1257
+ const uint i13 = im/FC_mul_mv_ne12;
1258
+
1259
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
1260
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
1261
+
1262
+ device const block_q2_K * x = (device const block_q2_K *) (src0 + offset0);
1263
+ device const float * y = (device const float *) (src1 + offset1);
1264
+
1265
+ float yl[32];
1266
+ float sumf[nr0]={0.f};
1267
+
1268
+ const short ix = tiisg/8; // 0...3
1269
+ const short it = tiisg%8; // 0...7
1270
+ const short iq = it/4; // 0 or 1
1271
+ const short ir = it%4; // 0...3
1272
+ const short is = (8*ir)/16;// 0 or 1
1273
+
1274
+ device const float * y4 = y + ix * QK_K + 128 * iq + 8 * ir;
1275
+
1276
+ for (int ib = ix; ib < nb; ib += 4) {
1277
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
1278
+ for (short i = 0; i < 8; ++i) {
1279
+ yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0];
1280
+ yl[i+ 8] = y4[i+32]; sumy[1] += yl[i+ 8];
1281
+ yl[i+16] = y4[i+64]; sumy[2] += yl[i+16];
1282
+ yl[i+24] = y4[i+96]; sumy[3] += yl[i+24];
1283
+ }
1284
+
1285
+ device const uint8_t * sc = (device const uint8_t *)x[ib].scales + 8*iq + is;
1286
+ device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;
1287
+ device const half * dh = &x[ib].d;
1288
+
1289
+ for (short row = 0; row < nr0; row++) {
1290
+ float4 acc1 = {0.f, 0.f, 0.f, 0.f};
1291
+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};
1292
+ for (int i = 0; i < 8; i += 2) {
1293
+ acc1[0] += yl[i+ 0] * (qs[i/2] & 0x0003);
1294
+ acc2[0] += yl[i+ 1] * (qs[i/2] & 0x0300);
1295
+ acc1[1] += yl[i+ 8] * (qs[i/2] & 0x000c);
1296
+ acc2[1] += yl[i+ 9] * (qs[i/2] & 0x0c00);
1297
+ acc1[2] += yl[i+16] * (qs[i/2] & 0x0030);
1298
+ acc2[2] += yl[i+17] * (qs[i/2] & 0x3000);
1299
+ acc1[3] += yl[i+24] * (qs[i/2] & 0x00c0);
1300
+ acc2[3] += yl[i+25] * (qs[i/2] & 0xc000);
1301
+ }
1302
+ float dall = dh[0];
1303
+ float dmin = dh[1] * 1.f/16.f;
1304
+ sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc2[0]) * (sc[0] & 0xF) * 1.f/ 1.f +
1305
+ (acc1[1] + 1.f/256.f * acc2[1]) * (sc[2] & 0xF) * 1.f/ 4.f +
1306
+ (acc1[2] + 1.f/256.f * acc2[2]) * (sc[4] & 0xF) * 1.f/16.f +
1307
+ (acc1[3] + 1.f/256.f * acc2[3]) * (sc[6] & 0xF) * 1.f/64.f) -
1308
+ dmin * (sumy[0] * (sc[0] & 0xF0) + sumy[1] * (sc[2] & 0xF0) + sumy[2] * (sc[4] & 0xF0) + sumy[3] * (sc[6] & 0xF0));
1309
+
1310
+ qs += args.nb01/2;
1311
+ sc += args.nb01;
1312
+ dh += args.nb01/2;
1313
+ }
1314
+
1315
+ y4 += 4 * QK_K;
1316
+ }
1317
+
1318
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
1319
+
1320
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
1321
+ float sum_all = simd_sum(sumf[row]);
1322
+ if (tiisg == 0) {
1323
+ dst_f32[first_row + row] = sum_all;
1324
+ }
1325
+ }
1326
+ }
1327
+
1328
+ [[host_name("kernel_mul_mv_q2_K_f32")]]
1329
+ kernel void kernel_mul_mv_q2_K_f32(
1330
+ constant ggml_metal_kargs_mul_mv & args,
1331
+ device const char * src0,
1332
+ device const char * src1,
1333
+ device char * dst,
1334
+ uint3 tgpig[[threadgroup_position_in_grid]],
1335
+ ushort tiisg[[thread_index_in_simdgroup]],
1336
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
1337
+
1338
+ kernel_mul_mv_q2_K_f32_impl<N_R0_Q2_K, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
1339
+ }
1340
+
1341
+ template<int nr0, typename args_t>
1342
+ void kernel_mul_mv_q3_K_f32_impl(
1343
+ args_t args,
1344
+ device const char * src0,
1345
+ device const char * src1,
1346
+ device char * dst,
1347
+ threadgroup char * shmem,
1348
+ uint3 tgpig,
1349
+ ushort tiisg,
1350
+ ushort sgitg) {
1351
+ const short NSG = FC_mul_mv_nsg;
1352
+
1353
+ const int nb = args.ne00/QK_K;
1354
+
1355
+ const int r0 = tgpig.x;
1356
+ const int r1 = tgpig.y;
1357
+ const int im = tgpig.z;
1358
+
1359
+ const int first_row = (r0 * NSG + sgitg) * nr0;
1360
+
1361
+ const uint i12 = im%FC_mul_mv_ne12;
1362
+ const uint i13 = im/FC_mul_mv_ne12;
1363
+
1364
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
1365
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
1366
+
1367
+ device const block_q3_K * x = (device const block_q3_K *) (src0 + offset0);
1368
+ device const float * yy = (device const float *) (src1 + offset1);
1369
+
1370
+ float yl[32];
1371
+
1372
+ //const uint16_t kmask1 = 0x3030;
1373
+ //const uint16_t kmask2 = 0x0f0f;
1374
+
1375
+ const short tid = tiisg/4;
1376
+ const short ix = tiisg%4;
1377
+ const short ip = tid/4; // 0 or 1
1378
+ const short il = 2*((tid%4)/2); // 0 or 2
1379
+ const short ir = tid%2;
1380
+ const short l0 = 8*ir;
1381
+
1382
+ // One would think that the Metal compiler would figure out that ip and il can only have
1383
+ // 4 possible states, and optimize accordingly. Well, no. It needs help, and we do it
1384
+ // with these two tales.
1385
+ //
1386
+ // Possible masks for the high bit
1387
+ const ushort4 mm[4] = {{0x0001, 0x0100, 0x0002, 0x0200}, // ip = 0, il = 0
1388
+ {0x0004, 0x0400, 0x0008, 0x0800}, // ip = 0, il = 2
1389
+ {0x0010, 0x1000, 0x0020, 0x2000}, // ip = 1, il = 0
1390
+ {0x0040, 0x4000, 0x0080, 0x8000}}; // ip = 1, il = 2
1391
+
1392
+ // Possible masks for the low 2 bits
1393
+ const int4 qm[2] = {{0x0003, 0x0300, 0x000c, 0x0c00}, {0x0030, 0x3000, 0x00c0, 0xc000}};
1394
+
1395
+ const ushort4 hm = mm[2*ip + il/2];
1396
+
1397
+ const short shift = 2*il;
1398
+
1399
+ const float v1 = il == 0 ? 4.f : 64.f;
1400
+ const float v2 = 4.f * v1;
1401
+
1402
+ const uint16_t s_shift1 = 4*ip;
1403
+ const uint16_t s_shift2 = s_shift1 + il;
1404
+
1405
+ const short q_offset = 32*ip + l0;
1406
+ const short y_offset = 128*ip + 32*il + l0;
1407
+
1408
+ device const float * y1 = yy + ix*QK_K + y_offset;
1409
+
1410
+ uint32_t scales32, aux32;
1411
+ thread uint16_t * scales16 = (thread uint16_t *)&scales32;
1412
+ thread const int8_t * scales = (thread const int8_t *)&scales32;
1413
+
1414
+ float sumf1[nr0] = {0.f};
1415
+ float sumf2[nr0] = {0.f};
1416
+
1417
+ for (int i = ix; i < nb; i += 4) {
1418
+ for (short l = 0; l < 8; ++l) {
1419
+ yl[l+ 0] = y1[l+ 0];
1420
+ yl[l+ 8] = y1[l+16];
1421
+ yl[l+16] = y1[l+32];
1422
+ yl[l+24] = y1[l+48];
1423
+ }
1424
+
1425
+ device const uint16_t * q = (device const uint16_t *)(x[i].qs + q_offset);
1426
+ device const uint16_t * h = (device const uint16_t *)(x[i].hmask + l0);
1427
+ device const uint16_t * a = (device const uint16_t *)(x[i].scales);
1428
+ device const half * dh = &x[i].d;
1429
+
1430
+ for (short row = 0; row < nr0; ++row) {
1431
+ const float d_all = (float)dh[0];
1432
+
1433
+ scales16[0] = a[4];
1434
+ scales16[1] = a[5];
1435
+ aux32 = ((scales32 >> s_shift2) << 4) & 0x30303030;
1436
+ scales16[0] = a[il+0];
1437
+ scales16[1] = a[il+1];
1438
+ scales32 = ((scales32 >> s_shift1) & 0x0f0f0f0f) | aux32;
1439
+
1440
+ float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0;
1441
+ for (short l = 0; l < 8; l += 2) {
1442
+ const int32_t qs = q[l/2];
1443
+ s1 += yl[l+0] * (qs & qm[il/2][0]);
1444
+ s2 += yl[l+1] * (qs & qm[il/2][1]);
1445
+ s3 += ((h[l/2] & hm[0]) ? 0.f : yl[l+0]) + ((h[l/2] & hm[1]) ? 0.f : yl[l+1]);
1446
+ s4 += yl[l+16] * (qs & qm[il/2][2]);
1447
+ s5 += yl[l+17] * (qs & qm[il/2][3]);
1448
+ s6 += ((h[l/2] & hm[2]) ? 0.f : yl[l+16]) + ((h[l/2] & hm[3]) ? 0.f : yl[l+17]);
1449
+ }
1450
+ float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
1451
+ float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
1452
+ sumf1[row] += d1 * (scales[0] - 32);
1453
+ sumf2[row] += d2 * (scales[2] - 32);
1454
+
1455
+ s1 = s2 = s3 = s4 = s5 = s6 = 0;
1456
+ for (short l = 0; l < 8; l += 2) {
1457
+ const int32_t qs = q[l/2+8];
1458
+ s1 += yl[l+8] * (qs & qm[il/2][0]);
1459
+ s2 += yl[l+9] * (qs & qm[il/2][1]);
1460
+ s3 += ((h[l/2+8] & hm[0]) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm[1]) ? 0.f : yl[l+9]);
1461
+ s4 += yl[l+24] * (qs & qm[il/2][2]);
1462
+ s5 += yl[l+25] * (qs & qm[il/2][3]);
1463
+ s6 += ((h[l/2+8] & hm[2]) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm[3]) ? 0.f : yl[l+25]);
1464
+ }
1465
+ d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1);
1466
+ d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2);
1467
+ sumf1[row] += d1 * (scales[1] - 32);
1468
+ sumf2[row] += d2 * (scales[3] - 32);
1469
+
1470
+ q += args.nb01/2;
1471
+ h += args.nb01/2;
1472
+ a += args.nb01/2;
1473
+ dh += args.nb01/2;
1474
+ }
1475
+
1476
+ y1 += 4 * QK_K;
1477
+ }
1478
+
1479
+ for (int row = 0; row < nr0; ++row) {
1480
+ const float sumf = (sumf1[row] + 0.25f * sumf2[row]) / (1 << shift);
1481
+ sumf1[row] = simd_sum(sumf);
1482
+ }
1483
+
1484
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
1485
+
1486
+ if (tiisg == 0) {
1487
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
1488
+ dst_f32[first_row + row] = sumf1[row];
1489
+ }
1490
+ }
1491
+ }
1492
+
1493
+ [[host_name("kernel_mul_mv_q3_K_f32")]]
1494
+ kernel void kernel_mul_mv_q3_K_f32(
1495
+ constant ggml_metal_kargs_mul_mv & args,
1496
+ device const char * src0,
1497
+ device const char * src1,
1498
+ device char * dst,
1499
+ uint3 tgpig[[threadgroup_position_in_grid]],
1500
+ ushort tiisg[[thread_index_in_simdgroup]],
1501
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
1502
+
1503
+ kernel_mul_mv_q3_K_f32_impl<N_R0_Q3_K, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
1504
+ }
1505
+
1506
+ template<int nr0, typename args_t>
1507
+ void kernel_mul_mv_q4_K_f32_impl(
1508
+ args_t args,
1509
+ device const char * src0,
1510
+ device const char * src1,
1511
+ device char * dst,
1512
+ threadgroup char * shmem,
1513
+ uint3 tgpig,
1514
+ ushort tiisg,
1515
+ ushort sgitg) {
1516
+ const short NSG = FC_mul_mv_nsg;
1517
+
1518
+ constexpr uint16_t kmask1 = 0x3f3f;
1519
+ constexpr uint16_t kmask2 = 0x0f0f;
1520
+ constexpr uint16_t kmask3 = 0xc0c0;
1521
+
1522
+ const short ix = tiisg/8; // 0...3
1523
+ const short it = tiisg%8; // 0...7
1524
+ const short iq = it/4; // 0 or 1
1525
+ const short ir = it%4; // 0...3
1526
+
1527
+ const int nb = args.ne00/QK_K;
1528
+
1529
+ const int r0 = tgpig.x;
1530
+ const int r1 = tgpig.y;
1531
+ const int im = tgpig.z;
1532
+
1533
+ const int first_row = (r0 * NSG + sgitg) * nr0;
1534
+
1535
+ const uint i12 = im%FC_mul_mv_ne12;
1536
+ const uint i13 = im/FC_mul_mv_ne12;
1537
+
1538
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
1539
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
1540
+
1541
+ device const block_q4_K * x = (device const block_q4_K *) (src0 + offset0);
1542
+ device const float * y = (device const float *) (src1 + offset1);
1543
+
1544
+ float yl[16];
1545
+ float yh[16];
1546
+
1547
+ float sumf[nr0]={0.f};
1548
+
1549
+ device const float * y4 = y + ix * QK_K + 64 * iq + 8 * ir;
1550
+
1551
+ uint16_t sc16[4];
1552
+ thread const uint8_t * sc8 = (thread const uint8_t *)sc16;
1553
+
1554
+ for (int ib = ix; ib < nb; ib += 4) {
1555
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
1556
+
1557
+ for (short i = 0; i < 8; ++i) {
1558
+ yl[i+0] = y4[i+ 0]; sumy[0] += yl[i+0];
1559
+ yl[i+8] = y4[i+ 32]; sumy[1] += yl[i+8];
1560
+ yh[i+0] = y4[i+128]; sumy[2] += yh[i+0];
1561
+ yh[i+8] = y4[i+160]; sumy[3] += yh[i+8];
1562
+ }
1563
+
1564
+ device const uint16_t * sc = (device const uint16_t *)x[ib].scales + iq;
1565
+ device const uint16_t * q1 = (device const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;
1566
+ device const half * dh = &x[ib].d;
1567
+
1568
+ for (short row = 0; row < nr0; row++) {
1569
+ sc16[0] = sc[0] & kmask1;
1570
+ sc16[1] = sc[2] & kmask1;
1571
+ sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2);
1572
+ sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2);
1573
+
1574
+ device const uint16_t * q2 = q1 + 32;
1575
+
1576
+ float4 acc1 = {0.f, 0.f, 0.f, 0.f};
1577
+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};
1578
+
1579
+ FOR_UNROLL (short i = 0; i < 4; ++i) {
1580
+ acc1[0] += yl[2*i + 0] * (q1[i] & 0x000F);
1581
+ acc1[1] += yl[2*i + 1] * (q1[i] & 0x0F00);
1582
+ acc1[2] += yl[2*i + 8] * (q1[i] & 0x00F0);
1583
+ acc1[3] += yl[2*i + 9] * (q1[i] & 0xF000);
1584
+ acc2[0] += yh[2*i + 0] * (q2[i] & 0x000F);
1585
+ acc2[1] += yh[2*i + 1] * (q2[i] & 0x0F00);
1586
+ acc2[2] += yh[2*i + 8] * (q2[i] & 0x00F0);
1587
+ acc2[3] += yh[2*i + 9] * (q2[i] & 0xF000);
1588
+ }
1589
+
1590
+ sumf[row] += dh[0] * ((acc1[0] + 1.f/256.f * acc1[1]) * sc8[0] +
1591
+ (acc1[2] + 1.f/256.f * acc1[3]) * sc8[1] * 1.f/16.f +
1592
+ (acc2[0] + 1.f/256.f * acc2[1]) * sc8[4] +
1593
+ (acc2[2] + 1.f/256.f * acc2[3]) * sc8[5] * 1.f/16.f) -
1594
+ dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]);
1595
+
1596
+ q1 += args.nb01/2;
1597
+ sc += args.nb01/2;
1598
+ dh += args.nb01/2;
1599
+ }
1600
+
1601
+ y4 += 4 * QK_K;
1602
+ }
1603
+
1604
+ device float * dst_f32 = (device float *) dst + (int64_t)im*args.ne0*args.ne1 + (int64_t)r1*args.ne0;
1605
+
1606
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
1607
+ float sum_all = simd_sum(sumf[row]);
1608
+ if (tiisg == 0) {
1609
+ dst_f32[first_row + row] = sum_all;
1610
+ }
1611
+ }
1612
+ }
1613
+
1614
+ [[host_name("kernel_mul_mv_q4_K_f32")]]
1615
+ kernel void kernel_mul_mv_q4_K_f32(
1616
+ constant ggml_metal_kargs_mul_mv & args,
1617
+ device const char * src0,
1618
+ device const char * src1,
1619
+ device char * dst,
1620
+ uint3 tgpig[[threadgroup_position_in_grid]],
1621
+ ushort tiisg[[thread_index_in_simdgroup]],
1622
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
1623
+
1624
+ kernel_mul_mv_q4_K_f32_impl<N_R0_Q4_K, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
1625
+ }
1626
+
1627
+ template<int nr0, typename args_t>
1628
+ void kernel_mul_mv_q5_K_f32_impl(
1629
+ args_t args,
1630
+ device const char * src0,
1631
+ device const char * src1,
1632
+ device char * dst,
1633
+ threadgroup char * shmem,
1634
+ uint3 tgpig,
1635
+ ushort tiisg,
1636
+ ushort sgitg) {
1637
+ const short NSG = FC_mul_mv_nsg;
1638
+
1639
+ const int nb = args.ne00/QK_K;
1640
+
1641
+ const int r0 = tgpig.x;
1642
+ const int r1 = tgpig.y;
1643
+ const int im = tgpig.z;
1644
+
1645
+ const int first_row = (r0 * NSG + sgitg) * nr0;
1646
+
1647
+ const uint i12 = im%FC_mul_mv_ne12;
1648
+ const uint i13 = im/FC_mul_mv_ne12;
1649
+
1650
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
1651
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
1652
+
1653
+ device const block_q5_K * x = (device const block_q5_K *) (src0 + offset0);
1654
+ device const float * yy = (device const float *) (src1 + offset1);
1655
+
1656
+ float sumf[nr0]={0.f};
1657
+
1658
+ float yl[16], yh[16];
1659
+
1660
+ constexpr uint16_t kmask1 = 0x3f3f;
1661
+ constexpr uint16_t kmask2 = 0x0f0f;
1662
+ constexpr uint16_t kmask3 = 0xc0c0;
1663
+
1664
+ const short tid = tiisg/4;
1665
+ const short ix = tiisg%4;
1666
+ const short iq = tid/4;
1667
+ const short ir = tid%4;
1668
+
1669
+ const short l0 = 8*ir;
1670
+ const short q_offset = 32*iq + l0;
1671
+ const short y_offset = 64*iq + l0;
1672
+
1673
+ const uint8_t hm1 = 1u << (2*iq);
1674
+ const uint8_t hm2 = hm1 << 1;
1675
+ const uint8_t hm3 = hm1 << 4;
1676
+ const uint8_t hm4 = hm2 << 4;
1677
+
1678
+ uint16_t sc16[4];
1679
+ thread const uint8_t * sc8 = (thread const uint8_t *)sc16;
1680
+
1681
+ device const float * y1 = yy + ix*QK_K + y_offset;
1682
+
1683
+ for (int i = ix; i < nb; i += 4) {
1684
+ device const uint8_t * q1 = x[i].qs + q_offset;
1685
+ device const uint8_t * qh = x[i].qh + l0;
1686
+ device const half * dh = &x[i].d;
1687
+ device const uint16_t * a = (device const uint16_t *)x[i].scales + iq;
1688
+
1689
+ device const float * y2 = y1 + 128;
1690
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
1691
+ for (short l = 0; l < 8; ++l) {
1692
+ yl[l+0] = y1[l+ 0]; sumy[0] += yl[l+0];
1693
+ yl[l+8] = y1[l+32]; sumy[1] += yl[l+8];
1694
+ yh[l+0] = y2[l+ 0]; sumy[2] += yh[l+0];
1695
+ yh[l+8] = y2[l+32]; sumy[3] += yh[l+8];
1696
+ }
1697
+
1698
+ for (short row = 0; row < nr0; ++row) {
1699
+ device const uint8_t * q2 = q1 + 64;
1700
+
1701
+ sc16[0] = a[0] & kmask1;
1702
+ sc16[1] = a[2] & kmask1;
1703
+ sc16[2] = ((a[4] >> 0) & kmask2) | ((a[0] & kmask3) >> 2);
1704
+ sc16[3] = ((a[4] >> 4) & kmask2) | ((a[2] & kmask3) >> 2);
1705
+
1706
+ float4 acc1 = {0.f};
1707
+ float4 acc2 = {0.f};
1708
+ FOR_UNROLL (short l = 0; l < 8; ++l) {
1709
+ uint8_t h = qh[l];
1710
+ acc1[0] += yl[l+0] * (q1[l] & 0x0F);
1711
+ acc1[1] += yl[l+8] * (q1[l] & 0xF0);
1712
+ acc1[2] += yh[l+0] * (q2[l] & 0x0F);
1713
+ acc1[3] += yh[l+8] * (q2[l] & 0xF0);
1714
+ acc2[0] += h & hm1 ? yl[l+0] : 0.f;
1715
+ acc2[1] += h & hm2 ? yl[l+8] : 0.f;
1716
+ acc2[2] += h & hm3 ? yh[l+0] : 0.f;
1717
+ acc2[3] += h & hm4 ? yh[l+8] : 0.f;
1718
+ }
1719
+
1720
+ sumf[row] += dh[0] * (sc8[0] * (acc1[0] + 16.f*acc2[0]) +
1721
+ sc8[1] * (acc1[1]/16.f + 16.f*acc2[1]) +
1722
+ sc8[4] * (acc1[2] + 16.f*acc2[2]) +
1723
+ sc8[5] * (acc1[3]/16.f + 16.f*acc2[3])) -
1724
+ dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]);
1725
+
1726
+ q1 += args.nb01;
1727
+ qh += args.nb01;
1728
+ dh += args.nb01/2;
1729
+ a += args.nb01/2;
1730
+ }
1731
+
1732
+ y1 += 4 * QK_K;
1733
+ }
1734
+
1735
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
1736
+
1737
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
1738
+ const float tot = simd_sum(sumf[row]);
1739
+ if (tiisg == 0) {
1740
+ dst_f32[first_row + row] = tot;
1741
+ }
1742
+ }
1743
+ }
1744
+
1745
+ [[host_name("kernel_mul_mv_q5_K_f32")]]
1746
+ kernel void kernel_mul_mv_q5_K_f32(
1747
+ constant ggml_metal_kargs_mul_mv & args,
1748
+ device const char * src0,
1749
+ device const char * src1,
1750
+ device char * dst,
1751
+ uint3 tgpig[[threadgroup_position_in_grid]],
1752
+ ushort tiisg[[thread_index_in_simdgroup]],
1753
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
1754
+
1755
+ kernel_mul_mv_q5_K_f32_impl<N_R0_Q5_K, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
1756
+ }
1757
+
1758
+ template<int nr0, typename args_t>
1759
+ void kernel_mul_mv_q6_K_f32_impl(
1760
+ args_t args,
1761
+ device const char * src0,
1762
+ device const char * src1,
1763
+ device char * dst,
1764
+ threadgroup char * shmem,
1765
+ uint3 tgpig,
1766
+ ushort tiisg,
1767
+ ushort sgitg) {
1768
+ const short NSG = FC_mul_mv_nsg;
1769
+
1770
+ constexpr uint8_t kmask1 = 0x03;
1771
+ constexpr uint8_t kmask2 = 0x0C;
1772
+ constexpr uint8_t kmask3 = 0x30;
1773
+ constexpr uint8_t kmask4 = 0xC0;
1774
+
1775
+ const int nb = args.ne00/QK_K;
1776
+
1777
+ const int r0 = tgpig.x;
1778
+ const int r1 = tgpig.y;
1779
+ const int im = tgpig.z;
1780
+
1781
+ const int first_row = (r0 * NSG + sgitg) * nr0;
1782
+
1783
+ const uint i12 = im%FC_mul_mv_ne12;
1784
+ const uint i13 = im/FC_mul_mv_ne12;
1785
+
1786
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
1787
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
1788
+
1789
+ device const block_q6_K * x = (device const block_q6_K *) (src0 + offset0);
1790
+ device const float * yy = (device const float *) (src1 + offset1);
1791
+
1792
+ float sumf[nr0] = { 0.f };
1793
+
1794
+ float yl[16];
1795
+
1796
+ const short tid = tiisg/2;
1797
+ const short ix = tiisg%2;
1798
+ const short ip = tid/8; // 0 or 1
1799
+ const short il = tid%8;
1800
+ const short l0 = 4*il;
1801
+ const short is = 8*ip + l0/16;
1802
+
1803
+ const short y_offset = 128*ip + l0;
1804
+ const short q_offset_l = 64*ip + l0;
1805
+ const short q_offset_h = 32*ip + l0;
1806
+
1807
+ for (int i = ix; i < nb; i += 2) {
1808
+ device const uint8_t * q1 = x[i].ql + q_offset_l;
1809
+ device const uint8_t * q2 = q1 + 32;
1810
+ device const uint8_t * qh = x[i].qh + q_offset_h;
1811
+ device const int8_t * sc = x[i].scales + is;
1812
+ device const half * dh = &x[i].d;
1813
+
1814
+ device const float * y = yy + i * QK_K + y_offset;
1815
+
1816
+ for (short l = 0; l < 4; ++l) {
1817
+ yl[4*l + 0] = y[l + 0];
1818
+ yl[4*l + 1] = y[l + 32];
1819
+ yl[4*l + 2] = y[l + 64];
1820
+ yl[4*l + 3] = y[l + 96];
1821
+ }
1822
+
1823
+ for (short row = 0; row < nr0; ++row) {
1824
+ float4 sums = {0.f, 0.f, 0.f, 0.f};
1825
+
1826
+ FOR_UNROLL (short l = 0; l < 4; ++l) {
1827
+ sums[0] += yl[4*l + 0] * ((int8_t)((q1[l] & 0xF) | ((qh[l] & kmask1) << 4)) - 32);
1828
+ sums[1] += yl[4*l + 1] * ((int8_t)((q2[l] & 0xF) | ((qh[l] & kmask2) << 2)) - 32);
1829
+ sums[2] += yl[4*l + 2] * ((int8_t)((q1[l] >> 4) | ((qh[l] & kmask3) << 0)) - 32);
1830
+ sums[3] += yl[4*l + 3] * ((int8_t)((q2[l] >> 4) | ((qh[l] & kmask4) >> 2)) - 32);
1831
+ }
1832
+
1833
+ sumf[row] += dh[0] * (sums[0] * sc[0] + sums[1] * sc[2] + sums[2] * sc[4] + sums[3] * sc[6]);
1834
+
1835
+ q1 += args.nb01;
1836
+ q2 += args.nb01;
1837
+ qh += args.nb01;
1838
+ sc += args.nb01;
1839
+ dh += args.nb01/2;
1840
+ }
1841
+ }
1842
+
1843
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
1844
+
1845
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
1846
+ float sum_all = simd_sum(sumf[row]);
1847
+ if (tiisg == 0) {
1848
+ dst_f32[first_row + row] = sum_all;
1849
+ }
1850
+ }
1851
+ }
1852
+
1853
+ [[host_name("kernel_mul_mv_q6_K_f32")]]
1854
+ kernel void kernel_mul_mv_q6_K_f32(
1855
+ constant ggml_metal_kargs_mul_mv & args,
1856
+ device const char * src0,
1857
+ device const char * src1,
1858
+ device char * dst,
1859
+ uint3 tgpig[[threadgroup_position_in_grid]],
1860
+ ushort tiisg[[thread_index_in_simdgroup]],
1861
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
1862
+
1863
+ kernel_mul_mv_q6_K_f32_impl<N_R0_Q6_K, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
1864
+ }
1865
+
1866
+ // ======================= "True" 2-bit
1867
+
1868
+ template<int nr0, typename args_t>
1869
+ void kernel_mul_mv_iq2_xxs_f32_impl(
1870
+ args_t args,
1871
+ device const char * src0,
1872
+ device const char * src1,
1873
+ device char * dst,
1874
+ threadgroup char * shmem,
1875
+ uint3 tgpig,
1876
+ ushort tiisg,
1877
+ ushort sgitg) {
1878
+ const short NSG = FC_mul_mv_nsg;
1879
+
1880
+ const int nb = args.ne00/QK_K;
1881
+
1882
+ const int r0 = tgpig.x;
1883
+ const int r1 = tgpig.y;
1884
+ const int im = tgpig.z;
1885
+
1886
+ const int first_row = (r0 * NSG + sgitg) * nr0;
1887
+
1888
+ const uint i12 = im%FC_mul_mv_ne12;
1889
+ const uint i13 = im/FC_mul_mv_ne12;
1890
+
1891
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
1892
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
1893
+
1894
+ device const block_iq2_xxs * x = (device const block_iq2_xxs *) (src0 + offset0);
1895
+ device const float * y = (device const float *) (src1 + offset1);
1896
+
1897
+ float yl[32];
1898
+ float sumf[nr0]={0.f};
1899
+
1900
+ const int nb32 = nb * (QK_K / 32);
1901
+
1902
+ threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem);
1903
+ threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256);
1904
+ {
1905
+ int nval = 4;
1906
+ int pos = (32*sgitg + tiisg)*nval;
1907
+ for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xxs_grid[pos + i];
1908
+ nval = 2;
1909
+ pos = (32*sgitg + tiisg)*nval;
1910
+ for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i];
1911
+ threadgroup_barrier(mem_flags::mem_threadgroup);
1912
+ }
1913
+
1914
+ const int ix = tiisg;
1915
+
1916
+ device const float * y4 = y + 32 * ix;
1917
+
1918
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
1919
+ for (short i = 0; i < 32; ++i) {
1920
+ yl[i] = y4[i];
1921
+ }
1922
+
1923
+ const int ibl = ib32 / (QK_K / 32);
1924
+ const int ib = ib32 % (QK_K / 32);
1925
+
1926
+ device const block_iq2_xxs * xr = x + ibl;
1927
+ device const uint16_t * q2 = xr->qs + 4 * ib;
1928
+ device const half * dh = &xr->d;
1929
+
1930
+ for (short row = 0; row < nr0; row++) {
1931
+ const float db = dh[0];
1932
+ device const uint8_t * aux8 = (device const uint8_t *)q2;
1933
+ const uint32_t aux32 = q2[2] | (q2[3] << 16);
1934
+ const float d = db * (0.5f + (aux32 >> 28));
1935
+
1936
+ float sum = 0;
1937
+ for (short l = 0; l < 4; ++l) {
1938
+ const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + aux8[l]);
1939
+ const uint8_t signs = ssigns[(aux32 >> 7*l) & 127];
1940
+ for (short j = 0; j < 8; ++j) {
1941
+ sum += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
1942
+ }
1943
+ }
1944
+ sumf[row] += d * sum;
1945
+
1946
+ dh += args.nb01/2;
1947
+ q2 += args.nb01/2;
1948
+ }
1949
+
1950
+ y4 += 32 * 32;
1951
+ }
1952
+
1953
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
1954
+
1955
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
1956
+ float sum_all = simd_sum(sumf[row]);
1957
+ if (tiisg == 0) {
1958
+ dst_f32[first_row + row] = sum_all * 0.25f;
1959
+ }
1960
+ }
1961
+ }
1962
+
1963
+ [[host_name("kernel_mul_mv_iq2_xxs_f32")]]
1964
+ kernel void kernel_mul_mv_iq2_xxs_f32(
1965
+ constant ggml_metal_kargs_mul_mv & args,
1966
+ device const char * src0,
1967
+ device const char * src1,
1968
+ device char * dst,
1969
+ threadgroup char * shmem [[threadgroup(0)]],
1970
+ uint3 tgpig[[threadgroup_position_in_grid]],
1971
+ ushort tiisg[[thread_index_in_simdgroup]],
1972
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
1973
+ kernel_mul_mv_iq2_xxs_f32_impl<N_R0_IQ2_XXS, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
1974
+ }
1975
+
1976
+ template<int nr0, typename args_t>
1977
+ void kernel_mul_mv_iq2_xs_f32_impl(
1978
+ args_t args,
1979
+ device const char * src0,
1980
+ device const char * src1,
1981
+ device char * dst,
1982
+ threadgroup char * shmem,
1983
+ uint3 tgpig,
1984
+ ushort tiisg,
1985
+ ushort sgitg) {
1986
+ const short NSG = FC_mul_mv_nsg;
1987
+
1988
+ const int nb = args.ne00/QK_K;
1989
+
1990
+ const int r0 = tgpig.x;
1991
+ const int r1 = tgpig.y;
1992
+ const int im = tgpig.z;
1993
+
1994
+ const int first_row = (r0 * NSG + sgitg) * nr0;
1995
+
1996
+ const uint i12 = im%FC_mul_mv_ne12;
1997
+ const uint i13 = im/FC_mul_mv_ne12;
1998
+
1999
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
2000
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
2001
+
2002
+ device const block_iq2_xs * x = (device const block_iq2_xs *) (src0 + offset0);
2003
+ device const float * y = (device const float *) (src1 + offset1);
2004
+
2005
+ float yl[32];
2006
+ float sumf[nr0]={0.f};
2007
+
2008
+ const int nb32 = nb * (QK_K / 32);
2009
+
2010
+ threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem);
2011
+ threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 512);
2012
+ {
2013
+ int nval = 8;
2014
+ int pos = (32*sgitg + tiisg)*nval;
2015
+ for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2xs_grid[pos + i];
2016
+ nval = 2;
2017
+ pos = (32*sgitg + tiisg)*nval;
2018
+ for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i];
2019
+ threadgroup_barrier(mem_flags::mem_threadgroup);
2020
+ }
2021
+
2022
+ const int ix = tiisg;
2023
+
2024
+ device const float * y4 = y + 32 * ix;
2025
+
2026
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
2027
+ for (short i = 0; i < 32; ++i) {
2028
+ yl[i] = y4[i];
2029
+ }
2030
+
2031
+ const int ibl = ib32 / (QK_K / 32);
2032
+ const int ib = ib32 % (QK_K / 32);
2033
+
2034
+ device const block_iq2_xs * xr = x + ibl;
2035
+ device const uint16_t * q2 = xr->qs + 4 * ib;
2036
+ device const uint8_t * sc = xr->scales + ib;
2037
+ device const half * dh = &xr->d;
2038
+
2039
+ for (short row = 0; row < nr0; row++) {
2040
+ const float db = dh[0];
2041
+ const uint8_t ls1 = sc[0] & 0xf;
2042
+ const uint8_t ls2 = sc[0] >> 4;
2043
+ const float d1 = db * (0.5f + ls1);
2044
+ const float d2 = db * (0.5f + ls2);
2045
+
2046
+ float sum1 = 0, sum2 = 0;
2047
+ for (short l = 0; l < 2; ++l) {
2048
+ const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511));
2049
+ const uint8_t signs = ssigns[(q2[l] >> 9)];
2050
+ for (short j = 0; j < 8; ++j) {
2051
+ sum1 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
2052
+ }
2053
+ }
2054
+ for (short l = 2; l < 4; ++l) {
2055
+ const threadgroup uint8_t * grid = (const threadgroup uint8_t *)(svalues + (q2[l] & 511));
2056
+ const uint8_t signs = ssigns[(q2[l] >> 9)];
2057
+ for (short j = 0; j < 8; ++j) {
2058
+ sum2 += yl[8*l + j] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);
2059
+ }
2060
+ }
2061
+ sumf[row] += d1 * sum1 + d2 * sum2;
2062
+
2063
+ dh += args.nb01/2;
2064
+ q2 += args.nb01/2;
2065
+ sc += args.nb01;
2066
+ }
2067
+
2068
+ y4 += 32 * 32;
2069
+ }
2070
+
2071
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
2072
+
2073
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
2074
+ float sum_all = simd_sum(sumf[row]);
2075
+ if (tiisg == 0) {
2076
+ dst_f32[first_row + row] = sum_all * 0.25f;
2077
+ }
2078
+ }
2079
+ }
2080
+
2081
+ [[host_name("kernel_mul_mv_iq2_xs_f32")]]
2082
+ kernel void kernel_mul_mv_iq2_xs_f32(
2083
+ constant ggml_metal_kargs_mul_mv & args,
2084
+ device const char * src0,
2085
+ device const char * src1,
2086
+ device char * dst,
2087
+ threadgroup char * shmem [[threadgroup(0)]],
2088
+ uint3 tgpig[[threadgroup_position_in_grid]],
2089
+ ushort tiisg[[thread_index_in_simdgroup]],
2090
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
2091
+
2092
+ kernel_mul_mv_iq2_xs_f32_impl<N_R0_IQ2_XS, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
2093
+ }
2094
+
2095
+ template<int nr0, typename args_t>
2096
+ void kernel_mul_mv_iq3_xxs_f32_impl(
2097
+ args_t args,
2098
+ device const char * src0,
2099
+ device const char * src1,
2100
+ device char * dst,
2101
+ threadgroup char * shmem,
2102
+ uint3 tgpig,
2103
+ ushort tiisg,
2104
+ ushort sgitg) {
2105
+ const short NSG = FC_mul_mv_nsg;
2106
+
2107
+ const int nb = args.ne00/QK_K;
2108
+
2109
+ const int r0 = tgpig.x;
2110
+ const int r1 = tgpig.y;
2111
+ const int im = tgpig.z;
2112
+
2113
+ const int first_row = (r0 * NSG + sgitg) * nr0;
2114
+
2115
+ const uint i12 = im%FC_mul_mv_ne12;
2116
+ const uint i13 = im/FC_mul_mv_ne12;
2117
+
2118
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
2119
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
2120
+
2121
+ device const block_iq3_xxs * x = (device const block_iq3_xxs *) (src0 + offset0);
2122
+ device const float * y = (device const float *) (src1 + offset1);
2123
+
2124
+ float yl[32];
2125
+ float sumf[nr0]={0.f};
2126
+
2127
+ const int nb32 = nb * (QK_K / 32);
2128
+
2129
+ threadgroup uint32_t * svalues = (threadgroup uint32_t *)(shmem);
2130
+ threadgroup uint8_t * ssigns = (threadgroup uint8_t *)(svalues + 256);
2131
+ {
2132
+ int nval = 4;
2133
+ int pos = (32*sgitg + tiisg)*nval;
2134
+ for (int i = 0; i < nval; ++i) svalues[pos + i] = iq3xxs_grid[pos + i];
2135
+ nval = 2;
2136
+ pos = (32*sgitg + tiisg)*nval;
2137
+ for (int i = 0; i < nval; ++i) ssigns[pos+i] = ksigns_iq2xs[pos+i];
2138
+ threadgroup_barrier(mem_flags::mem_threadgroup);
2139
+ }
2140
+
2141
+ const int ix = tiisg;
2142
+
2143
+ device const float * y4 = y + 32 * ix;
2144
+
2145
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
2146
+ for (short i = 0; i < 32; ++i) {
2147
+ yl[i] = y4[i];
2148
+ }
2149
+
2150
+ const int ibl = ib32 / (QK_K / 32);
2151
+ const int ib = ib32 % (QK_K / 32);
2152
+
2153
+ device const block_iq3_xxs * xr = x + ibl;
2154
+ device const uint8_t * q3 = xr->qs + 8 * ib;
2155
+ device const uint16_t * gas = (device const uint16_t *)(xr->qs + QK_K/4) + 2 * ib;
2156
+ device const half * dh = &xr->d;
2157
+
2158
+ for (short row = 0; row < nr0; row++) {
2159
+ const float db = dh[0];
2160
+ const uint32_t aux32 = gas[0] | (gas[1] << 16);
2161
+ const float d = db * (0.5f + (aux32 >> 28));
2162
+
2163
+ float2 sum = {0};
2164
+ for (short l = 0; l < 4; ++l) {
2165
+ const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(svalues + q3[2*l+0]);
2166
+ const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(svalues + q3[2*l+1]);
2167
+ const uint8_t signs = ssigns[(aux32 >> 7*l) & 127];
2168
+ for (short j = 0; j < 4; ++j) {
2169
+ sum[0] += yl[8*l + j + 0] * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f);
2170
+ sum[1] += yl[8*l + j + 4] * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f);
2171
+ }
2172
+ }
2173
+ sumf[row] += d * (sum[0] + sum[1]);
2174
+
2175
+ dh += args.nb01/2;
2176
+ q3 += args.nb01;
2177
+ gas += args.nb01/2;
2178
+ }
2179
+
2180
+ y4 += 32 * 32;
2181
+ }
2182
+
2183
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
2184
+
2185
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
2186
+ float sum_all = simd_sum(sumf[row]);
2187
+ if (tiisg == 0) {
2188
+ dst_f32[first_row + row] = sum_all * 0.5f;
2189
+ }
2190
+ }
2191
+ }
2192
+
2193
+ [[host_name("kernel_mul_mv_iq3_xxs_f32")]]
2194
+ kernel void kernel_mul_mv_iq3_xxs_f32(
2195
+ constant ggml_metal_kargs_mul_mv & args,
2196
+ device const char * src0,
2197
+ device const char * src1,
2198
+ device char * dst,
2199
+ threadgroup char * shmem [[threadgroup(0)]],
2200
+ uint3 tgpig[[threadgroup_position_in_grid]],
2201
+ ushort tiisg[[thread_index_in_simdgroup]],
2202
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
2203
+
2204
+ kernel_mul_mv_iq3_xxs_f32_impl<N_R0_IQ3_XXS, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
2205
+ }
2206
+
2207
+ template<int nr0, typename args_t>
2208
+ void kernel_mul_mv_iq3_s_f32_impl(
2209
+ args_t args,
2210
+ device const char * src0,
2211
+ device const char * src1,
2212
+ device char * dst,
2213
+ threadgroup char * shmem,
2214
+ uint3 tgpig,
2215
+ ushort tiisg,
2216
+ ushort sgitg) {
2217
+ const short NSG = FC_mul_mv_nsg;
2218
+
2219
+ const int nb = args.ne00/QK_K;
2220
+
2221
+ const int r0 = tgpig.x;
2222
+ const int r1 = tgpig.y;
2223
+ const int im = tgpig.z;
2224
+
2225
+ const int first_row = (r0 * NSG + sgitg) * nr0;
2226
+
2227
+ const uint i12 = im%FC_mul_mv_ne12;
2228
+ const uint i13 = im/FC_mul_mv_ne12;
2229
+
2230
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
2231
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
2232
+
2233
+ device const block_iq3_s * x = (device const block_iq3_s *) (src0 + offset0);
2234
+ device const float * y = (device const float *) (src1 + offset1);
2235
+
2236
+ float yl[32];
2237
+ float sumf[nr0]={0.f};
2238
+
2239
+ const int nb32 = nb * (QK_K / 32);
2240
+
2241
+ threadgroup uint32_t * svalues = (threadgroup uint32_t *) shmem;
2242
+ {
2243
+ int nval = 8;
2244
+ int pos = (32*sgitg + tiisg)*nval;
2245
+ for (int i = 0; i < nval; ++i) svalues[pos + i] = iq3s_grid[pos + i];
2246
+ threadgroup_barrier(mem_flags::mem_threadgroup);
2247
+ }
2248
+
2249
+ const int ix = tiisg;
2250
+
2251
+ device const float * y4 = y + 32 * ix;
2252
+
2253
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
2254
+ for (short i = 0; i < 32; ++i) {
2255
+ yl[i] = y4[i];
2256
+ }
2257
+
2258
+ const int ibl = ib32 / (QK_K / 32);
2259
+ const int ib = ib32 % (QK_K / 32);
2260
+
2261
+ device const block_iq3_s * xr = x + ibl;
2262
+ device const uint8_t * qs = xr->qs + 8 * ib;
2263
+ device const uint8_t * qh = xr->qh + ib;
2264
+ device const uint8_t * sc = xr->scales + (ib/2);
2265
+ device const uint8_t * signs = xr->signs + 4 * ib;
2266
+ device const half * dh = &xr->d;
2267
+
2268
+ for (short row = 0; row < nr0; row++) {
2269
+ const float db = dh[0];
2270
+ const float d = db * (1 + 2*((sc[0] >> 4*(ib%2)) & 0xf));
2271
+
2272
+ float2 sum = {0};
2273
+ for (short l = 0; l < 4; ++l) {
2274
+ const threadgroup uint32_t * table1 = qh[0] & kmask_iq2xs[2*l+0] ? svalues + 256 : svalues;
2275
+ const threadgroup uint32_t * table2 = qh[0] & kmask_iq2xs[2*l+1] ? svalues + 256 : svalues;
2276
+ const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(table1 + qs[2*l+0]);
2277
+ const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(table2 + qs[2*l+1]);
2278
+ for (short j = 0; j < 4; ++j) {
2279
+ sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l] & kmask_iq2xs[j+0]);
2280
+ sum[1] += yl[8*l + j + 4] * grid2[j] * select(1, -1, signs[l] & kmask_iq2xs[j+4]);
2281
+ }
2282
+ }
2283
+ sumf[row] += d * (sum[0] + sum[1]);
2284
+
2285
+ dh += args.nb01/2;
2286
+ qs += args.nb01;
2287
+ qh += args.nb01;
2288
+ sc += args.nb01;
2289
+ signs += args.nb01;
2290
+ }
2291
+
2292
+ y4 += 32 * 32;
2293
+ }
2294
+
2295
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
2296
+
2297
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
2298
+ float sum_all = simd_sum(sumf[row]);
2299
+ if (tiisg == 0) {
2300
+ dst_f32[first_row + row] = sum_all;
2301
+ }
2302
+ }
2303
+ }
2304
+
2305
+ [[host_name("kernel_mul_mv_iq3_s_f32")]]
2306
+ kernel void kernel_mul_mv_iq3_s_f32(
2307
+ constant ggml_metal_kargs_mul_mv & args,
2308
+ device const char * src0,
2309
+ device const char * src1,
2310
+ device char * dst,
2311
+ threadgroup char * shmem [[threadgroup(0)]],
2312
+ uint3 tgpig[[threadgroup_position_in_grid]],
2313
+ ushort tiisg[[thread_index_in_simdgroup]],
2314
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
2315
+
2316
+ kernel_mul_mv_iq3_s_f32_impl<N_R0_IQ3_S, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
2317
+ }
2318
+
2319
+ template<int nr0, typename args_t>
2320
+ void kernel_mul_mv_iq2_s_f32_impl(
2321
+ args_t args,
2322
+ device const char * src0,
2323
+ device const char * src1,
2324
+ device char * dst,
2325
+ threadgroup char * shmem,
2326
+ uint3 tgpig,
2327
+ ushort tiisg,
2328
+ ushort sgitg) {
2329
+ const short NSG = FC_mul_mv_nsg;
2330
+
2331
+ const int nb = args.ne00/QK_K;
2332
+
2333
+ const int r0 = tgpig.x;
2334
+ const int r1 = tgpig.y;
2335
+ const int im = tgpig.z;
2336
+
2337
+ const int first_row = (r0 * NSG + sgitg) * nr0;
2338
+
2339
+ const uint i12 = im%FC_mul_mv_ne12;
2340
+ const uint i13 = im/FC_mul_mv_ne12;
2341
+
2342
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
2343
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
2344
+
2345
+ device const block_iq2_s * x = (device const block_iq2_s *) (src0 + offset0);
2346
+ device const float * y = (device const float *) (src1 + offset1);
2347
+
2348
+ float yl[32];
2349
+ float sumf[nr0]={0.f};
2350
+
2351
+ const int nb32 = nb * (QK_K / 32);
2352
+
2353
+ //threadgroup uint64_t * svalues = (threadgroup uint64_t *) shmem;
2354
+ //{
2355
+ // int nval = 32;
2356
+ // int pos = (32*sgitg + tiisg)*nval;
2357
+ // for (int i = 0; i < nval; ++i) svalues[pos + i] = iq2s_grid[pos + i];
2358
+ // threadgroup_barrier(mem_flags::mem_threadgroup);
2359
+ //}
2360
+
2361
+ const short ix = tiisg;
2362
+
2363
+ device const float * y4 = y + 32 * ix;
2364
+
2365
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
2366
+ for (short i = 0; i < 32; ++i) {
2367
+ yl[i] = y4[i];
2368
+ }
2369
+
2370
+ const int ibl = ib32 / (QK_K / 32);
2371
+ const int ib = ib32 % (QK_K / 32);
2372
+
2373
+ device const block_iq2_s * xr = x + ibl;
2374
+ device const uint8_t * qs = xr->qs + 4 * ib;
2375
+ device const uint8_t * qh = xr->qh + ib;
2376
+ device const uint8_t * sc = xr->scales + ib;
2377
+ device const uint8_t * signs = qs + QK_K/8;
2378
+ device const half * dh = &xr->d;
2379
+
2380
+ for (short row = 0; row < nr0; row++) {
2381
+ const float db = dh[0];
2382
+ const float d1 = db * (0.5f + (sc[0] & 0xf));
2383
+ const float d2 = db * (0.5f + (sc[0] >> 4));
2384
+
2385
+ float2 sum = {0};
2386
+ for (short l = 0; l < 2; ++l) {
2387
+ //const threadgroup uint8_t * grid1 = (const threadgroup uint8_t *)(svalues + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300)));
2388
+ //const threadgroup uint8_t * grid2 = (const threadgroup uint8_t *)(svalues + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300)));
2389
+ constant uint8_t * grid1 = (constant uint8_t *)(iq2s_grid + (qs[l+0] | ((qh[0] << (8-2*l)) & 0x300)));
2390
+ constant uint8_t * grid2 = (constant uint8_t *)(iq2s_grid + (qs[l+2] | ((qh[0] << (4-2*l)) & 0x300)));
2391
+ for (short j = 0; j < 8; ++j) {
2392
+ sum[0] += yl[8*l + j + 0] * grid1[j] * select(1, -1, signs[l+0] & kmask_iq2xs[j]);
2393
+ sum[1] += yl[8*l + j + 16] * grid2[j] * select(1, -1, signs[l+2] & kmask_iq2xs[j]);
2394
+ }
2395
+ }
2396
+ sumf[row] += d1 * sum[0] + d2 * sum[1];
2397
+
2398
+ dh += args.nb01/2;
2399
+ qs += args.nb01;
2400
+ qh += args.nb01;
2401
+ sc += args.nb01;
2402
+ signs += args.nb01;
2403
+ }
2404
+
2405
+ y4 += 32 * 32;
2406
+ }
2407
+
2408
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
2409
+
2410
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
2411
+ float sum_all = simd_sum(sumf[row]);
2412
+ if (tiisg == 0) {
2413
+ dst_f32[first_row + row] = sum_all * 0.25f;
2414
+ }
2415
+ }
2416
+ }
2417
+
2418
+ [[host_name("kernel_mul_mv_iq2_s_f32")]]
2419
+ kernel void kernel_mul_mv_iq2_s_f32(
2420
+ constant ggml_metal_kargs_mul_mv & args,
2421
+ device const char * src0,
2422
+ device const char * src1,
2423
+ device char * dst,
2424
+ threadgroup char * shmem [[threadgroup(0)]],
2425
+ uint3 tgpig[[threadgroup_position_in_grid]],
2426
+ ushort tiisg[[thread_index_in_simdgroup]],
2427
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
2428
+
2429
+ kernel_mul_mv_iq2_s_f32_impl<N_R0_IQ2_S, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
2430
+ }
2431
+
2432
+ template<int nr0, typename args_t>
2433
+ void kernel_mul_mv_iq1_s_f32_impl(
2434
+ args_t args,
2435
+ device const char * src0,
2436
+ device const char * src1,
2437
+ device char * dst,
2438
+ threadgroup char * shmem,
2439
+ uint3 tgpig,
2440
+ ushort tiisg,
2441
+ ushort sgitg) {
2442
+ const short NSG = FC_mul_mv_nsg;
2443
+
2444
+ const int nb = args.ne00/QK_K;
2445
+
2446
+ const int r0 = tgpig.x;
2447
+ const int r1 = tgpig.y;
2448
+ const int im = tgpig.z;
2449
+
2450
+ const int first_row = (r0 * NSG + sgitg) * nr0;
2451
+
2452
+ const uint i12 = im%FC_mul_mv_ne12;
2453
+ const uint i13 = im/FC_mul_mv_ne12;
2454
+
2455
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
2456
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
2457
+
2458
+ device const block_iq1_s * x = (device const block_iq1_s *) (src0 + offset0);
2459
+ device const float * y = (device const float *) (src1 + offset1);
2460
+
2461
+ float yl[32];
2462
+ float sumf[nr0]={0.f};
2463
+
2464
+ const int nb32 = nb * (QK_K / 32);
2465
+
2466
+ const short ix = tiisg;
2467
+
2468
+ device const float * y4 = y + 32 * ix;
2469
+
2470
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
2471
+ float sumy = 0;
2472
+ for (short i = 0; i < 32; ++i) {
2473
+ yl[i] = y4[i];
2474
+ sumy += yl[i];
2475
+ }
2476
+
2477
+ const int ibl = ib32 / (QK_K / 32);
2478
+ const int ib = ib32 % (QK_K / 32);
2479
+
2480
+ device const block_iq1_s * xr = x + ibl;
2481
+ device const uint8_t * qs = xr->qs + 4 * ib;
2482
+ device const uint16_t * qh = xr->qh + ib;
2483
+ device const half * dh = &xr->d;
2484
+
2485
+ for (short row = 0; row < nr0; row++) {
2486
+ constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700)));
2487
+ constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 5) & 0x700)));
2488
+ constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[0] << 2) & 0x700)));
2489
+ constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[0] >> 1) & 0x700)));
2490
+
2491
+ float sum = 0;
2492
+ for (short j = 0; j < 4; ++j) {
2493
+ sum += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4)
2494
+ + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4)
2495
+ + yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4)
2496
+ + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4);
2497
+ }
2498
+ sumf[row] += (float)dh[0] * (sum + sumy * (qh[0] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA)) * (2*((qh[0] >> 12) & 7) + 1);
2499
+
2500
+ dh += args.nb01/2;
2501
+ qs += args.nb01;
2502
+ qh += args.nb01/2;
2503
+ }
2504
+
2505
+ y4 += 32 * 32;
2506
+ }
2507
+
2508
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
2509
+
2510
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
2511
+ float sum_all = simd_sum(sumf[row]);
2512
+ if (tiisg == 0) {
2513
+ dst_f32[first_row + row] = sum_all;
2514
+ }
2515
+ }
2516
+ }
2517
+
2518
+ [[host_name("kernel_mul_mv_iq1_s_f32")]]
2519
+ kernel void kernel_mul_mv_iq1_s_f32(
2520
+ constant ggml_metal_kargs_mul_mv & args,
2521
+ device const char * src0,
2522
+ device const char * src1,
2523
+ device char * dst,
2524
+ uint3 tgpig[[threadgroup_position_in_grid]],
2525
+ ushort tiisg[[thread_index_in_simdgroup]],
2526
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
2527
+
2528
+ kernel_mul_mv_iq1_s_f32_impl<N_R0_IQ1_S, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
2529
+ }
2530
+
2531
+ template<int nr0, typename args_t>
2532
+ void kernel_mul_mv_iq1_m_f32_impl(
2533
+ args_t args,
2534
+ device const char * src0,
2535
+ device const char * src1,
2536
+ device char * dst,
2537
+ threadgroup char * shmem,
2538
+ uint3 tgpig,
2539
+ ushort tiisg,
2540
+ ushort sgitg) {
2541
+ const short NSG = FC_mul_mv_nsg;
2542
+
2543
+ const int nb = args.ne00/QK_K;
2544
+
2545
+ const int r0 = tgpig.x;
2546
+ const int r1 = tgpig.y;
2547
+ const int im = tgpig.z;
2548
+
2549
+ const int first_row = (r0 * NSG + sgitg) * nr0;
2550
+
2551
+ const uint i12 = im%FC_mul_mv_ne12;
2552
+ const uint i13 = im/FC_mul_mv_ne12;
2553
+
2554
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
2555
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
2556
+
2557
+ device const block_iq1_m * x = (device const block_iq1_m *) (src0 + offset0);
2558
+ device const float * y = (device const float *) (src1 + offset1);
2559
+
2560
+ float yl[32];
2561
+ float sumf[nr0]={0.f};
2562
+
2563
+ const int nb32 = nb * (QK_K / 32);
2564
+
2565
+ const short ix = tiisg;
2566
+
2567
+ device const float * y4 = y + 32 * ix;
2568
+
2569
+ iq1m_scale_t scale;
2570
+
2571
+ for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
2572
+ float4 sumy = {0.f};
2573
+ for (short i = 0; i < 8; ++i) {
2574
+ yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0];
2575
+ yl[i+ 8] = y4[i+ 8]; sumy[1] += yl[i+ 8];
2576
+ yl[i+16] = y4[i+16]; sumy[2] += yl[i+16];
2577
+ yl[i+24] = y4[i+24]; sumy[3] += yl[i+24];
2578
+ }
2579
+
2580
+ const int ibl = ib32 / (QK_K / 32);
2581
+ const int ib = ib32 % (QK_K / 32);
2582
+
2583
+ device const block_iq1_m * xr = x + ibl;
2584
+ device const uint8_t * qs = xr->qs + 4 * ib;
2585
+ device const uint8_t * qh = xr->qh + 2 * ib;
2586
+ device const uint16_t * sc = (device const uint16_t *)xr->scales;
2587
+
2588
+ for (short row = 0; row < nr0; row++) {
2589
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
2590
+
2591
+ constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700)));
2592
+ constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 4) & 0x700)));
2593
+ constant uint8_t * grid3 = (constant uint8_t *)(iq1s_grid_gpu + (qs[2] | ((qh[1] << 8) & 0x700)));
2594
+ constant uint8_t * grid4 = (constant uint8_t *)(iq1s_grid_gpu + (qs[3] | ((qh[1] << 4) & 0x700)));
2595
+
2596
+ float2 sum = {0.f};
2597
+ for (short j = 0; j < 4; ++j) {
2598
+ sum[0] += yl[j+ 0] * (grid1[j] & 0xf) + yl[j+ 4] * (grid1[j] >> 4)
2599
+ + yl[j+ 8] * (grid2[j] & 0xf) + yl[j+12] * (grid2[j] >> 4);
2600
+ sum[1] += yl[j+16] * (grid3[j] & 0xf) + yl[j+20] * (grid3[j] >> 4)
2601
+ + yl[j+24] * (grid4[j] & 0xf) + yl[j+28] * (grid4[j] >> 4);
2602
+ }
2603
+ const float delta1 = sumy[0] * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[1] * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA);
2604
+ const float delta2 = sumy[2] * (qh[1] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[3] * (qh[1] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA);
2605
+
2606
+ sumf[row] += (float)scale.f16 * ((sum[0] + delta1) * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 7) + 1) +
2607
+ (sum[1] + delta2) * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 7) + 1));
2608
+
2609
+ sc += args.nb01/2;
2610
+ qs += args.nb01;
2611
+ qh += args.nb01;
2612
+ }
2613
+
2614
+ y4 += 32 * 32;
2615
+ }
2616
+
2617
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
2618
+
2619
+ for (int row = 0; row < nr0 && first_row + row < args.ne0; ++row) {
2620
+ float sum_all = simd_sum(sumf[row]);
2621
+ if (tiisg == 0) {
2622
+ dst_f32[first_row + row] = sum_all;
2623
+ }
2624
+ }
2625
+ }
2626
+
2627
+ [[host_name("kernel_mul_mv_iq1_m_f32")]]
2628
+ kernel void kernel_mul_mv_iq1_m_f32(
2629
+ constant ggml_metal_kargs_mul_mv & args,
2630
+ device const char * src0,
2631
+ device const char * src1,
2632
+ device char * dst,
2633
+ uint3 tgpig[[threadgroup_position_in_grid]],
2634
+ ushort tiisg[[thread_index_in_simdgroup]],
2635
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
2636
+
2637
+ kernel_mul_mv_iq1_m_f32_impl<N_R0_IQ1_M, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
2638
+ }
2639
+
2640
+ template<int NR0, typename args_t>
2641
+ void kernel_mul_mv_iq4_nl_f32_impl(
2642
+ args_t args,
2643
+ device const char * src0,
2644
+ device const char * src1,
2645
+ device char * dst,
2646
+ threadgroup char * shmem,
2647
+ uint3 tgpig,
2648
+ ushort tiisg,
2649
+ ushort sgitg) {
2650
+ const short NSG = FC_mul_mv_nsg;
2651
+
2652
+ threadgroup float * shmem_f32 = (threadgroup float *) shmem;
2653
+
2654
+ const int r0 = tgpig.x;
2655
+ const int r1 = tgpig.y;
2656
+ const int im = tgpig.z;
2657
+
2658
+ const int first_row = (r0 * NSG + sgitg) * NR0;
2659
+
2660
+ const uint i12 = im%FC_mul_mv_ne12;
2661
+ const uint i13 = im/FC_mul_mv_ne12;
2662
+
2663
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
2664
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
2665
+
2666
+ device const block_iq4_nl * x = (device const block_iq4_nl *) (src0 + offset0);
2667
+ device const float * y = (device const float *) (src1 + offset1);
2668
+
2669
+ const int nb = args.ne00/QK4_NL;
2670
+ const int ns01 = args.nb01/args.nb00;
2671
+
2672
+ const short ix = tiisg/2; // 0...15
2673
+ const short it = tiisg%2; // 0 or 1
2674
+
2675
+ shmem_f32[tiisg] = kvalues_iq4nl_f[tiisg%16];
2676
+ threadgroup_barrier(mem_flags::mem_threadgroup);
2677
+
2678
+ float4 yl[4];
2679
+ float sumf[NR0]={0.f};
2680
+
2681
+ device const float * yb = y + ix*QK4_NL + it*8;
2682
+
2683
+ uint32_t aux32[2];
2684
+ thread const uint8_t * q8 = (thread const uint8_t *)aux32;
2685
+
2686
+ float4 qf1, qf2;
2687
+
2688
+ // [TAG_MUL_MV_WEIRD]
2689
+ for (int ib = ix; ib < nb && ib < ns01; ib += 16) {
2690
+ device const float4 * y4 = (device const float4 *)yb;
2691
+ yl[0] = y4[0];
2692
+ yl[1] = y4[4];
2693
+ yl[2] = y4[1];
2694
+ yl[3] = y4[5];
2695
+
2696
+ for (short row = 0; row < NR0; row++) {
2697
+ device const block_iq4_nl & xb = x[row*ns01 + ib];
2698
+ device const uint16_t * q4 = (device const uint16_t *)(xb.qs + 8*it);
2699
+
2700
+ float4 acc1 = {0.f}, acc2 = {0.f};
2701
+
2702
+ aux32[0] = q4[0] | (q4[1] << 16);
2703
+ aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f;
2704
+ aux32[0] &= 0x0f0f0f0f;
2705
+ qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]};
2706
+ qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]};
2707
+ acc1 += yl[0] * qf1;
2708
+ acc2 += yl[1] * qf2;
2709
+
2710
+ aux32[0] = q4[2] | (q4[3] << 16);
2711
+ aux32[1] = (aux32[0] >> 4) & 0x0f0f0f0f;
2712
+ aux32[0] &= 0x0f0f0f0f;
2713
+ qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]};
2714
+ qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]};
2715
+ acc1 += yl[2] * qf1;
2716
+ acc2 += yl[3] * qf2;
2717
+
2718
+ acc1 += acc2;
2719
+
2720
+ sumf[row] += (float)xb.d * (acc1[0] + acc1[1] + acc1[2] + acc1[3]);
2721
+ }
2722
+
2723
+ yb += 16 * QK4_NL;
2724
+ }
2725
+
2726
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
2727
+
2728
+ for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) {
2729
+ float sum_all = simd_sum(sumf[row]);
2730
+ if (tiisg == 0) {
2731
+ dst_f32[first_row + row] = sum_all;
2732
+ }
2733
+ }
2734
+ }
2735
+
2736
+ [[host_name("kernel_mul_mv_iq4_nl_f32")]]
2737
+ kernel void kernel_mul_mv_iq4_nl_f32(
2738
+ constant ggml_metal_kargs_mul_mv & args,
2739
+ device const char * src0,
2740
+ device const char * src1,
2741
+ device char * dst,
2742
+ threadgroup char * shmem [[threadgroup(0)]],
2743
+ uint3 tgpig[[threadgroup_position_in_grid]],
2744
+ ushort tiisg[[thread_index_in_simdgroup]],
2745
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
2746
+
2747
+ kernel_mul_mv_iq4_nl_f32_impl<N_R0_IQ4_NL, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
2748
+ }
2749
+
2750
+ template<int NR0, typename args_t>
2751
+ void kernel_mul_mv_iq4_xs_f32_impl(
2752
+ args_t args,
2753
+ device const char * src0,
2754
+ device const char * src1,
2755
+ device char * dst,
2756
+ threadgroup char * shmem,
2757
+ uint3 tgpig,
2758
+ ushort tiisg,
2759
+ ushort sgitg) {
2760
+ const short NSG = FC_mul_mv_nsg;
2761
+
2762
+ threadgroup float * shmem_f32 = (threadgroup float *) shmem;
2763
+
2764
+ const int r0 = tgpig.x;
2765
+ const int r1 = tgpig.y;
2766
+ const int im = tgpig.z;
2767
+ const int first_row = (r0 * NSG + sgitg) * NR0;
2768
+
2769
+ const uint i12 = im%FC_mul_mv_ne12;
2770
+ const uint i13 = im/FC_mul_mv_ne12;
2771
+
2772
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
2773
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
2774
+
2775
+ device const block_iq4_xs * x = (device const block_iq4_xs *) (src0 + offset0);
2776
+ device const float * y = (device const float *) (src1 + offset1);
2777
+
2778
+ const int nb = args.ne00/QK_K;
2779
+ const int ns01 = args.nb01/args.nb00;
2780
+
2781
+ const short ix = tiisg/16; // 0 or 1
2782
+ const short it = tiisg%16; // 0...15
2783
+ const short ib = it/2;
2784
+ const short il = it%2;
2785
+
2786
+ shmem_f32[tiisg] = kvalues_iq4nl_f[tiisg%16];
2787
+ threadgroup_barrier(mem_flags::mem_threadgroup);
2788
+
2789
+ float4 yl[4];
2790
+ float sumf[NR0]={0.f};
2791
+
2792
+ device const float * yb = y + ix * QK_K + ib * 32 + il * 8;
2793
+
2794
+ uint32_t aux32[2];
2795
+ thread const uint8_t * q8 = (thread const uint8_t *)aux32;
2796
+
2797
+ float4 qf1, qf2;
2798
+
2799
+ // [TAG_MUL_MV_WEIRD]
2800
+ for (int ibl = ix; ibl < nb && ibl < ns01; ibl += 2) {
2801
+ device const float4 * y4 = (device const float4 *)yb;
2802
+ yl[0] = y4[0];
2803
+ yl[1] = y4[4];
2804
+ yl[2] = y4[1];
2805
+ yl[3] = y4[5];
2806
+
2807
+ for (short row = 0; row < NR0; ++row) {
2808
+ device const block_iq4_xs & xb = x[row*ns01 + ibl];
2809
+ device const uint32_t * q4 = (device const uint32_t *)(xb.qs + 16*ib + 8*il);
2810
+
2811
+ float4 acc1 = {0.f}, acc2 = {0.f};
2812
+
2813
+ aux32[0] = (q4[0] ) & 0x0f0f0f0f;
2814
+ aux32[1] = (q4[0] >> 4) & 0x0f0f0f0f;
2815
+ qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]};
2816
+ qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]};
2817
+ acc1 += yl[0] * qf1;
2818
+ acc2 += yl[1] * qf2;
2819
+
2820
+ aux32[0] = (q4[1] ) & 0x0f0f0f0f;
2821
+ aux32[1] = (q4[1] >> 4) & 0x0f0f0f0f;
2822
+ qf1 = {shmem_f32[q8[0]], shmem_f32[q8[1]], shmem_f32[q8[2]], shmem_f32[q8[3]]};
2823
+ qf2 = {shmem_f32[q8[4]], shmem_f32[q8[5]], shmem_f32[q8[6]], shmem_f32[q8[7]]};
2824
+ acc1 += yl[2] * qf1;
2825
+ acc2 += yl[3] * qf2;
2826
+
2827
+ acc1 += acc2;
2828
+
2829
+ const int ls = (((xb.scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((xb.scales_h >> 2*ib) & 3) << 4)) - 32;
2830
+ sumf[row] += (float)xb.d * ls * (acc1[0] + acc1[1] + acc1[2] + acc1[3]);
2831
+ }
2832
+
2833
+ yb += 2 * QK_K;
2834
+ }
2835
+
2836
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
2837
+
2838
+ for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) {
2839
+ float sum_all = simd_sum(sumf[row]);
2840
+ if (tiisg == 0) {
2841
+ dst_f32[first_row + row] = sum_all;
2842
+ }
2843
+ }
2844
+ }
2845
+
2846
+ [[host_name("kernel_mul_mv_iq4_xs_f32")]]
2847
+ kernel void kernel_mul_mv_iq4_xs_f32(
2848
+ constant ggml_metal_kargs_mul_mv & args,
2849
+ device const char * src0,
2850
+ device const char * src1,
2851
+ device char * dst,
2852
+ threadgroup char * shmem [[threadgroup(0)]],
2853
+ uint3 tgpig[[threadgroup_position_in_grid]],
2854
+ ushort tiisg[[thread_index_in_simdgroup]],
2855
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
2856
+
2857
+ kernel_mul_mv_iq4_xs_f32_impl<N_R0_IQ4_XS, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
2858
+ }
2859
+
2860
+ template<int NR0, typename args_t>
2861
+ void kernel_mul_mv_mxfp4_f32_impl(
2862
+ args_t args,
2863
+ device const char * src0,
2864
+ device const char * src1,
2865
+ device char * dst,
2866
+ threadgroup char * shmem,
2867
+ uint3 tgpig,
2868
+ ushort tiisg,
2869
+ ushort sgitg) {
2870
+ const short NSG = FC_mul_mv_nsg;
2871
+
2872
+ threadgroup float * shmem_f32 = (threadgroup float *) shmem;
2873
+
2874
+ const int r0 = tgpig.x;
2875
+ const int r1 = tgpig.y;
2876
+ const int im = tgpig.z;
2877
+
2878
+ const int first_row = (r0 * NSG + sgitg) * NR0;
2879
+
2880
+ const uint i12 = im%FC_mul_mv_ne12;
2881
+ const uint i13 = im/FC_mul_mv_ne12;
2882
+
2883
+ const uint64_t offset0 = first_row*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
2884
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
2885
+
2886
+ device const block_mxfp4 * x = (device const block_mxfp4 *) (src0 + offset0);
2887
+ device const float * y = (device const float *) (src1 + offset1);
2888
+
2889
+ const int nb = args.ne00/QK_MXFP4;
2890
+ const int ns01 = args.nb01/args.nb00; // this can be larger than nb for permuted src0 tensors
2891
+
2892
+ const short ix = tiisg/2; // 0...15
2893
+ const short it = tiisg%2; // 0 or 1
2894
+
2895
+ shmem_f32[tiisg] = kvalues_mxfp4_f[tiisg%16];
2896
+ threadgroup_barrier(mem_flags::mem_threadgroup);
2897
+
2898
+ float4 yl[4];
2899
+ float sumf[NR0]={0.f};
2900
+
2901
+ device const float * yb = y + ix*QK_MXFP4 + it*8;
2902
+
2903
+ // note: just the check `ib < nb` is enough, but adding the redundant `&& ib < ns01` check makes the kernel a bit faster
2904
+ // no idea why that is - needs some deeper investigation [TAG_MUL_MV_WEIRD]
2905
+ for (int ib = ix; ib < nb && ib < ns01; ib += 16) {
2906
+ device const float4 * y4 = (device const float4 *) yb;
2907
+
2908
+ yl[0] = y4[0];
2909
+ yl[1] = y4[4];
2910
+ yl[2] = y4[1];
2911
+ yl[3] = y4[5];
2912
+
2913
+ FOR_UNROLL (short row = 0; row < NR0; row++) {
2914
+ device const block_mxfp4 & xb = x[row*ns01 + ib];
2915
+ device const uint8_t * q2 = (device const uint8_t *)(xb.qs + 8*it);
2916
+
2917
+ float4 acc1 = yl[0]*float4(shmem_f32[q2[0] & 0x0F], shmem_f32[q2[1] & 0x0F], shmem_f32[q2[2] & 0x0F], shmem_f32[q2[3] & 0x0F]);
2918
+ float4 acc2 = yl[1]*float4(shmem_f32[q2[0] >> 4 ], shmem_f32[q2[1] >> 4 ], shmem_f32[q2[2] >> 4 ], shmem_f32[q2[3] >> 4 ]);
2919
+ float4 acc3 = yl[2]*float4(shmem_f32[q2[4] & 0x0F], shmem_f32[q2[5] & 0x0F], shmem_f32[q2[6] & 0x0F], shmem_f32[q2[7] & 0x0F]);
2920
+ float4 acc4 = yl[3]*float4(shmem_f32[q2[4] >> 4 ], shmem_f32[q2[5] >> 4 ], shmem_f32[q2[6] >> 4 ], shmem_f32[q2[7] >> 4 ]);
2921
+
2922
+ acc1 = (acc1 + acc3) + (acc2 + acc4);
2923
+
2924
+ sumf[row] += e8m0_to_fp32(xb.e) * ((acc1[0] + acc1[1]) + (acc1[2] + acc1[3]));
2925
+ }
2926
+
2927
+ yb += 16 * QK_MXFP4;
2928
+ }
2929
+
2930
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
2931
+
2932
+ for (int row = 0; row < NR0 && first_row + row < args.ne0; ++row) {
2933
+ float sum_all = simd_sum(sumf[row]);
2934
+ if (tiisg == 0) {
2935
+ dst_f32[first_row + row] = sum_all;
2936
+ }
2937
+ }
2938
+ }
2939
+
2940
+ [[host_name("kernel_mul_mv_mxfp4_f32")]]
2941
+ kernel void kernel_mul_mv_mxfp4_f32(
2942
+ constant ggml_metal_kargs_mul_mv & args,
2943
+ device const char * src0,
2944
+ device const char * src1,
2945
+ device char * dst,
2946
+ threadgroup char * shmem [[threadgroup(0)]],
2947
+ uint3 tgpig[[threadgroup_position_in_grid]],
2948
+ ushort tiisg[[thread_index_in_simdgroup]],
2949
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
2950
+
2951
+ kernel_mul_mv_mxfp4_f32_impl<N_R0_MXFP4, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
2952
+ }
2953
+
2954
+ template<int nr0, typename args_t>
2955
+ void kernel_mul_mv_tq2_0_f32_impl(
2956
+ args_t args,
2957
+ device const char * src0,
2958
+ device const char * src1,
2959
+ device char * dst,
2960
+ threadgroup char * shmem,
2961
+ uint3 tgpig,
2962
+ ushort tiisg,
2963
+ ushort sgitg) {
2964
+ const short NSG = FC_mul_mv_nsg;
2965
+
2966
+ const int nb = args.ne00/QK_K;
2967
+
2968
+ const int r0 = tgpig.x;
2969
+ const int r1 = tgpig.y;
2970
+ const int im = tgpig.z;
2971
+
2972
+ const int first_row = (r0 * NSG + sgitg) * nr0;
2973
+
2974
+ const uint i12 = im%FC_mul_mv_ne12;
2975
+ const uint i13 = im/FC_mul_mv_ne12;
2976
+
2977
+ const uint64_t offset1 = r1*args.nb11 + (i12 )*args.nb12 + (i13 )*args.nb13;
2978
+
2979
+ device const float * y = (device const float *) (src1 + offset1);
2980
+
2981
+ device const block_tq2_0 * ax[nr0];
2982
+ for (int row = 0; row < nr0; ++row) {
2983
+ const uint64_t offset0 = (first_row + row)*args.nb01 + (i12/FC_mul_mv_r2)*args.nb02 + (i13/FC_mul_mv_r3)*args.nb03;
2984
+ ax[row] = (device const block_tq2_0 *) ((device char *) src0 + offset0);
2985
+ }
2986
+
2987
+ float sumf[nr0] = {0.f};
2988
+
2989
+ // 8 threads per block, NBLOCK blocks per pass, 2 halves per block per pass
2990
+ constexpr short NBLOCK = 4;
2991
+
2992
+ constexpr short NB = N_SIMDWIDTH/NBLOCK; // threads per block
2993
+
2994
+ const short blk = tiisg / NB; // 0..NBLOCK-1, block handled by this thread
2995
+ const short htg = tiisg % NB; // 0..NB-1, thread within block (0..7)
2996
+
2997
+ // byte and y base offsets within the block (32 elements per thread, 4 per byte)
2998
+ device const float4 * yb4 = (device const float4 *)(y + 4*htg + blk*QK_K);
2999
+
3000
+ // hoisted per-byte coefficients (from y) and total y-sum, shared across rows
3001
+ // ref: https://github.com/ggml-org/llama.cpp/pull/26980
3002
+ float4 coef[4];
3003
+
3004
+ for (int ib = blk; ib < nb; ib += NBLOCK) {
3005
+ FOR_UNROLL (short h0 = 0; h0 < 2; ++h0) {
3006
+ const float4 y0 = yb4[ 0 + 32*h0];
3007
+ const float4 y1 = yb4[ 8 + 32*h0];
3008
+ const float4 y2 = yb4[16 + 32*h0];
3009
+ const float4 y3 = yb4[24 + 32*h0];
3010
+
3011
+ float sumy = 0.f;
3012
+ FOR_UNROLL (short j = 0; j < 4; ++j) {
3013
+ coef[j] = float4(
3014
+ y0[j],
3015
+ y1[j] - 4.0f*y0[j],
3016
+ y2[j] - 4.0f*y1[j],
3017
+ y3[j] - 4.0f*y2[j]);
3018
+
3019
+ sumy += (y0[j] + y1[j]) + (y2[j] + y3[j]);
3020
+ }
3021
+
3022
+ FOR_UNROLL (short row = 0; row < nr0; ++row) {
3023
+ device const block_tq2_0 & xb = ax[row][ib];
3024
+ device const uchar * qs = xb.qs + 4*htg + 32*h0;
3025
+
3026
+ float sum = -sumy;
3027
+ FOR_UNROLL (short j = 0; j < 4; ++j) {
3028
+ // express the 2-bit field shifts (v>>2, v>>4, v>>6) as float floor ops
3029
+ const float v = (float)qs[j];
3030
+
3031
+ const float f0 = v;
3032
+ const float f1 = floor(v*0.25f); // v>>2
3033
+ const float f2 = floor(v*0.0625); // v>>4
3034
+ const float f3 = floor(v*0.015625); // v>>6
3035
+
3036
+ sum += coef[j][0]*f0 + coef[j][1]*f1 + coef[j][2]*f2 + coef[j][3]*f3;
3037
+ }
3038
+
3039
+ sumf[row] += xb.d * sum;
3040
+ }
3041
+ }
3042
+
3043
+ yb4 += QK_K * NBLOCK / 4;
3044
+ }
3045
+
3046
+ device float * dst_f32 = (device float *) dst + (uint64_t)im*args.ne0*args.ne1 + (uint64_t)r1*args.ne0;
3047
+
3048
+ for (int row = 0; row < nr0; ++row) {
3049
+ const float tot = simd_sum(sumf[row]);
3050
+ if (tiisg == 0 && first_row + row < args.ne01) {
3051
+ dst_f32[first_row + row] = tot;
3052
+ }
3053
+ }
3054
+ }
3055
+
3056
+ [[host_name("kernel_mul_mv_tq2_0_f32")]]
3057
+ kernel void kernel_mul_mv_tq2_0_f32(
3058
+ constant ggml_metal_kargs_mul_mv & args,
3059
+ device const char * src0,
3060
+ device const char * src1,
3061
+ device char * dst,
3062
+ uint3 tgpig[[threadgroup_position_in_grid]],
3063
+ ushort tiisg[[thread_index_in_simdgroup]],
3064
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
3065
+
3066
+ kernel_mul_mv_tq2_0_f32_impl<N_R0_TQ2_0, constant ggml_metal_kargs_mul_mv &>(args, src0, src1, dst, nullptr, tgpig, tiisg, sgitg);
3067
+ }
3068
+
3069
+ //
3070
+ // matrix-vector multiplication
3071
+ //
3072
+
3073
+ typedef void (kernel_mul_mv_disp_t)(
3074
+ ggml_metal_kargs_mul_mv args,
3075
+ device const char * src0,
3076
+ device const char * src1,
3077
+ device char * dst,
3078
+ uint3 tgpig,
3079
+ ushort tiisg);
3080
+
3081
+ typedef void (kernel_mul_mv2_disp_t)(
3082
+ ggml_metal_kargs_mul_mv args,
3083
+ device const char * src0,
3084
+ device const char * src1,
3085
+ device char * dst,
3086
+ threadgroup char * shmem,
3087
+ uint3 tgpig,
3088
+ ushort tiisg,
3089
+ ushort sgitg);
3090
+
3091
+ template<kernel_mul_mv_disp_t disp_fn>
3092
+ void mmv_fn(
3093
+ ggml_metal_kargs_mul_mv args,
3094
+ device const char * src0,
3095
+ device const char * src1,
3096
+ device char * dst,
3097
+ threadgroup char * shmem,
3098
+ uint3 tgpig,
3099
+ ushort tiitg,
3100
+ ushort tiisg,
3101
+ ushort sgitg) {
3102
+ disp_fn(args, src0, src1, dst, tgpig, tiisg);
3103
+ }
3104
+
3105
+ template<kernel_mul_mv2_disp_t disp_fn>
3106
+ void mmv_fn(
3107
+ ggml_metal_kargs_mul_mv args,
3108
+ device const char * src0,
3109
+ device const char * src1,
3110
+ device char * dst,
3111
+ threadgroup char * shmem,
3112
+ uint3 tgpig,
3113
+ ushort tiitg,
3114
+ ushort tiisg,
3115
+ ushort sgitg) {
3116
+ disp_fn(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
3117
+ }
3118
+
3119
+ typedef decltype(mmv_fn<kernel_mul_mv_t_t_disp<half, half, ggml_metal_kargs_mul_mv>>) mul_mv_disp_fn_t;
3120
+
3121
+ template<mul_mv_disp_fn_t disp_fn>
3122
+ kernel void kernel_mul_mv_id(
3123
+ constant ggml_metal_kargs_mul_mv_id & args,
3124
+ device const char * src0s,
3125
+ device const char * src1,
3126
+ device char * dst,
3127
+ device const char * ids,
3128
+ threadgroup char * shmem [[threadgroup(0)]],
3129
+ uint3 tgpig[[threadgroup_position_in_grid]],
3130
+ ushort tiitg[[thread_index_in_threadgroup]],
3131
+ ushort tiisg[[thread_index_in_simdgroup]],
3132
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
3133
+ const int iid1 = tgpig.z/args.nei0;
3134
+ const int idx = tgpig.z%args.nei0;
3135
+
3136
+ tgpig.z = 0;
3137
+
3138
+ const int32_t i02 = ((device const int32_t *) (ids + iid1*args.nbi1))[idx];
3139
+
3140
+ const int64_t i11 = idx % args.ne11;
3141
+ const int64_t i12 = iid1;
3142
+
3143
+ const int64_t i1 = idx;
3144
+ const int64_t i2 = i12;
3145
+
3146
+ device const char * src0_cur = src0s + i02*args.nb02;
3147
+ device const char * src1_cur = src1 + i11*args.nb11 + i12*args.nb12;
3148
+
3149
+ device char * dst_cur = dst + (i1*args.ne0 + i2*args.ne1*args.ne0)*sizeof(float);
3150
+
3151
+ ggml_metal_kargs_mul_mv args0 = {
3152
+ /*.ne00 =*/ args.ne00,
3153
+ /*.ne01 =*/ args.ne01,
3154
+ /*.ne02 =*/ 1, // args.ne02,
3155
+ /*.nb00 =*/ args.nb00,
3156
+ /*.nb01 =*/ args.nb01,
3157
+ /*.nb02 =*/ args.nb02,
3158
+ /*.nb03 =*/ args.nb02, // args.ne02 == 1
3159
+ /*.ne10 =*/ args.ne10,
3160
+ /*.ne11 =*/ 1, // args.ne11,
3161
+ /*.ne12 =*/ 1, // args.ne12,
3162
+ /*.nb10 =*/ args.nb10,
3163
+ /*.nb11 =*/ args.nb11,
3164
+ /*.nb12 =*/ args.nb12,
3165
+ /*.nb13 =*/ args.nb12, // ne12 == 1
3166
+ /*.ne0 =*/ args.ne0,
3167
+ /*.ne1 =*/ 1, // args.ne1,
3168
+ /*.nr0 =*/ args.nr0,
3169
+ /*.r2 =*/ 1,
3170
+ /*.r3 =*/ 1,
3171
+ };
3172
+
3173
+ disp_fn(
3174
+ args0,
3175
+ /* src0 */ src0_cur,
3176
+ /* src1 */ src1_cur,
3177
+ /* dst */ dst_cur,
3178
+ shmem,
3179
+ tgpig,
3180
+ tiitg,
3181
+ tiisg,
3182
+ sgitg);
3183
+ }
3184
+
3185
+ typedef decltype(kernel_mul_mv_id<mmv_fn<kernel_mul_mv_t_t_disp<float, float>>>) kernel_mul_mv_id_t;
3186
+
3187
+ typedef decltype(kernel_mul_mv_id<mmv_fn<kernel_mul_mv_t_t_4_disp<float, float4, float, float4>>>) kernel_mul_mv_id_4_t;
3188
+
3189
+ template [[host_name("kernel_mul_mv_id_f32_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_t_t_disp<float, float>>>;
3190
+ template [[host_name("kernel_mul_mv_id_f16_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_t_t_disp<half, float>>>;
3191
+ #if defined(GGML_METAL_HAS_BF16)
3192
+ template [[host_name("kernel_mul_mv_id_bf16_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_t_t_disp<bfloat, float>>>;
3193
+ #endif
3194
+ template [[host_name("kernel_mul_mv_id_f32_f32_4")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_t_t_4_disp<float, float4, float, float4>>>;
3195
+ template [[host_name("kernel_mul_mv_id_f16_f32_4")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_t_t_4_disp<half, half4, float, float4>>>;
3196
+ #if defined(GGML_METAL_HAS_BF16)
3197
+ template [[host_name("kernel_mul_mv_id_bf16_f32_4")]] kernel kernel_mul_mv_id_4_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_t_t_4_disp<bfloat, bfloat4, float, float4>>>;
3198
+ #endif
3199
+
3200
+ template [[host_name("kernel_mul_mv_id_q8_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q8_0_f32_impl<N_R0_Q8_0>>>;
3201
+
3202
+ template [[host_name("kernel_mul_mv_id_q1_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q1_0_f32_impl<N_R0_Q1_0>>>;
3203
+ template [[host_name("kernel_mul_mv_id_q2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q2_0_f32_impl<N_R0_Q2_0>>>;
3204
+ template [[host_name("kernel_mul_mv_id_q4_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<mul_vec_q_n_f32_impl<block_q4_0, N_R0_Q4_0>>>;
3205
+ template [[host_name("kernel_mul_mv_id_q4_1_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<mul_vec_q_n_f32_impl<block_q4_1, N_R0_Q4_1>>>;
3206
+ template [[host_name("kernel_mul_mv_id_q5_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<mul_vec_q_n_f32_impl<block_q5_0, N_R0_Q5_0>>>;
3207
+ template [[host_name("kernel_mul_mv_id_q5_1_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<mul_vec_q_n_f32_impl<block_q5_1, N_R0_Q5_1>>>;
3208
+
3209
+ template [[host_name("kernel_mul_mv_id_mxfp4_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_mxfp4_f32_impl<N_R0_MXFP4>>>;
3210
+
3211
+ template [[host_name("kernel_mul_mv_id_q2_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q2_K_f32_impl <N_R0_Q2_K>>>;
3212
+ template [[host_name("kernel_mul_mv_id_q3_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q3_K_f32_impl <N_R0_Q3_K>>>;
3213
+ template [[host_name("kernel_mul_mv_id_q4_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q4_K_f32_impl <N_R0_Q4_K>>>;
3214
+ template [[host_name("kernel_mul_mv_id_q5_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q5_K_f32_impl <N_R0_Q5_K>>>;
3215
+ template [[host_name("kernel_mul_mv_id_q6_K_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_q6_K_f32_impl <N_R0_Q6_K>>>;
3216
+ template [[host_name("kernel_mul_mv_id_iq1_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_s_f32_impl <N_R0_IQ1_S>>>;
3217
+ template [[host_name("kernel_mul_mv_id_iq1_m_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq1_m_f32_impl <N_R0_IQ1_M>>>;
3218
+ template [[host_name("kernel_mul_mv_id_iq2_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xxs_f32_impl<N_R0_IQ2_XXS>>>;
3219
+ template [[host_name("kernel_mul_mv_id_iq2_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_xs_f32_impl <N_R0_IQ2_XS>>>;
3220
+ template [[host_name("kernel_mul_mv_id_iq3_xxs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq3_xxs_f32_impl<N_R0_IQ3_XXS>>>;
3221
+ template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq3_s_f32_impl <N_R0_IQ3_S>>>;
3222
+ template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq2_s_f32_impl <N_R0_IQ2_S>>>;
3223
+ template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_nl_f32_impl <N_R0_IQ4_NL>>>;
3224
+ template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_iq4_xs_f32_impl <N_R0_IQ4_XS>>>;
3225
+ template [[host_name("kernel_mul_mv_id_tq2_0_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id<mmv_fn<kernel_mul_mv_tq2_0_f32_impl <N_R0_TQ2_0>>>;