elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,3033 @@
1
+ #include "mmvq.hpp"
2
+
3
+ #include "ggml.h"
4
+ #include "common.hpp"
5
+ #include "element_wise.hpp"
6
+ #include "quants.hpp"
7
+ #include "vecdotq.hpp"
8
+
9
+ // Minimum weight-row count at which the Q4_K multi-column MMVQ kernel handles two output rows per
10
+ // subgroup (rows_per_sg == 2) instead of one, when ncols_dst == 2.
11
+ //
12
+ // Pairing rows lets a subgroup load each activation block once and apply it to two rows, at the cost
13
+ // of halving the number of subgroups in the launch. With only two destination columns there is too
14
+ // little work per row to hide that loss of parallelism, so pairing only pays off once there are
15
+ // enough rows to keep the device occupied. This is a measured performance crossover, not a
16
+ // correctness or hardware limit - both variants compute the same result for any nrows.
17
+ //
18
+ // Derived on Intel Arc Pro B70 with `test-backend-ops perf -o MUL_MAT` (Q4_K, ncols_dst == 2),
19
+ // sweeping nrows over 5120..6912 at ncols 17408 and 19968: one row per subgroup was up to 9% faster
20
+ // below the crossover, two rows per subgroup 8-15% faster above it, and the crossover fell inside
21
+ // (6144, 6272] for both ncols with no measurable ncols dependence. A later 32-row granularity sweep
22
+ // narrowed it to (6144, 6176], so 6272 is a conservative gate rather than the exact crossover.
23
+ // ncols_dst >= 3 amortizes the activation loads over more columns and is faster with two rows at
24
+ // every row count, so it does not consult this threshold.
25
+ static constexpr int Q4_K_MMVQ_ROW_PAIR_MIN_NROWS = 6272;
26
+
27
+ template <typename reorder_vec_dot_q_sycl>
28
+ static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,
29
+ const int ncols, const int nrows, const sycl::nd_item<3> & nd_item) {
30
+ using block_type = ggml_sycl_reordered::block_q_t<reorder_vec_dot_q_sycl::gtype>;
31
+ using block_traits = typename block_type::traits;
32
+
33
+ const auto sg = nd_item.get_sub_group();
34
+ const int sg_range = sg.get_group_linear_range();
35
+ const int workgroup_id = nd_item.get_group_linear_id();
36
+ const int sg_id = sg.get_group_linear_id();
37
+ const int row = workgroup_id * sg_range + sg_id;
38
+
39
+ if (row >= nrows) {
40
+ return;
41
+ }
42
+
43
+ const int blocks_per_row = ncols / block_traits::qk;
44
+ constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);
45
+ constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;
46
+ const int nblocks = nrows * (ncols / block_traits::qk);
47
+
48
+ static_assert(blocks_per_subgroup > 0);
49
+ static_assert(block_elements_per_subgroup > 0);
50
+
51
+ float partial_sum = 0.0f;
52
+ for (int i = sg.get_local_linear_id() / block_elements_per_subgroup; i < blocks_per_row; i += blocks_per_subgroup) {
53
+ const int ibx = row * blocks_per_row + i; // x block index
54
+
55
+ const auto bx_offset = block_type::get_block_offset(ibx, nblocks);
56
+ const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);
57
+ // Y block index that aligns with ibx
58
+ const int iby = i * block_type::block_to_q8_1_ratio();
59
+ const int8_t* q8_1_quant_ptr = (const int8_t*)vy + iby * QK8_1;
60
+ const sycl::half2* q8_1_ds_ptr = (const sycl::half2*)((const char*)vy + ncols + iby * sizeof(sycl::half2));
61
+
62
+ #pragma unroll
63
+ for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {
64
+ // x block quant index when casting the quants to int
65
+ const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
66
+
67
+ partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
68
+ }
69
+ }
70
+
71
+ auto sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum, std::plus<>());
72
+
73
+ if (sg.leader()) {
74
+ dst[row] = sum;
75
+ }
76
+ }
77
+
78
+ // With has_fusion, `vgate` is a second weight matrix sharing vx's shape, stride and reorder
79
+ // layout: one pass computes both row dot products and the epilogue writes glu(gate, up).
80
+ template <typename reorder_vec_dot_q_sycl, int ncols_dst, bool has_fusion = false, int rows_per_sg = 1>
81
+ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void * __restrict__ vgate,
82
+ const void * __restrict__ vy, float * __restrict__ dst, const int ncols,
83
+ const int nrows, const int stride_col_y_bytes, const int stride_col_dst,
84
+ const ggml_glu_op glu_op, const sycl::nd_item<3> & nd_item) {
85
+ using block_type = ggml_sycl_reordered::block_q_t<reorder_vec_dot_q_sycl::gtype>;
86
+ using block_traits = typename block_type::traits;
87
+
88
+ const auto sg = nd_item.get_sub_group();
89
+ const int sg_range = sg.get_group_linear_range();
90
+ const int workgroup_id = nd_item.get_group_linear_id();
91
+ const int sg_id = sg.get_group_linear_id();
92
+ const int row0 = (workgroup_id * sg_range + sg_id) * rows_per_sg;
93
+
94
+ // row is sub-group uniform, so this retires whole sub-groups and the collectives below
95
+ // stay convergent
96
+ if (row0 >= nrows) {
97
+ return;
98
+ }
99
+
100
+ static_assert(rows_per_sg == 1 ||
101
+ reorder_vec_dot_shared_activations<reorder_vec_dot_q_sycl::gtype>::value);
102
+
103
+ const int blocks_per_row = ncols / block_traits::qk;
104
+ constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);
105
+ constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;
106
+ const int nblocks = nrows * (ncols / block_traits::qk);
107
+
108
+ static_assert(blocks_per_subgroup > 0);
109
+ static_assert(block_elements_per_subgroup > 0);
110
+
111
+ float partial_sum[ncols_dst][rows_per_sg] = {};
112
+ // sized 1 rather than 0 when unused: zero-length arrays are not standard C++, and the
113
+ // array is dead and eliminated in that case
114
+ [[maybe_unused]] float partial_gate[has_fusion ? ncols_dst : 1][has_fusion ? rows_per_sg : 1] = {};
115
+ for (int i = sg.get_local_linear_id() / block_elements_per_subgroup; i < blocks_per_row; i += blocks_per_subgroup) {
116
+ const int iby = i * block_type::block_to_q8_1_ratio();
117
+
118
+ #pragma unroll
119
+ for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {
120
+ const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
121
+
122
+ if constexpr (rows_per_sg > 1) {
123
+ typename reorder_vec_dot_q_sycl::weights wx[rows_per_sg];
124
+ [[maybe_unused]] typename reorder_vec_dot_q_sycl::weights wg[rows_per_sg];
125
+ #pragma unroll
126
+ for (int r = 0; r < rows_per_sg; ++r) {
127
+ const int row = sycl::min(row0 + r, nrows - 1);
128
+ const int ibx = row * blocks_per_row + i;
129
+ const auto bx_offset = block_type::get_block_offset(ibx, nblocks);
130
+ const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);
131
+ wx[r] = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs);
132
+ if constexpr (has_fusion) {
133
+ wg[r] = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs);
134
+ }
135
+ }
136
+ #pragma unroll
137
+ for (int j = 0; j < ncols_dst; ++j) {
138
+ const char * vy_j = (const char *) vy + j * stride_col_y_bytes;
139
+ const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;
140
+ const sycl::half2 * q8_1_ds_ptr =
141
+ (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));
142
+ const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs);
143
+ #pragma unroll
144
+ for (int r = 0; r < rows_per_sg; ++r) {
145
+ partial_sum[j][r] += reorder_vec_dot_q_sycl::apply(wx[r], a);
146
+ if constexpr (has_fusion) {
147
+ partial_gate[j][r] += reorder_vec_dot_q_sycl::apply(wg[r], a);
148
+ }
149
+ }
150
+ }
151
+ } else if constexpr (reorder_vec_dot_shared_weights<reorder_vec_dot_q_sycl::gtype>::value) {
152
+ const int ibx = row0 * blocks_per_row + i;
153
+ const auto bx_offset = block_type::get_block_offset(ibx, nblocks);
154
+ const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);
155
+ const auto wx = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs);
156
+ if constexpr (has_fusion) {
157
+ const auto wg = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs);
158
+
159
+ #pragma unroll
160
+ for (int j = 0; j < ncols_dst; ++j) {
161
+ const char * vy_j = (const char *) vy + j * stride_col_y_bytes;
162
+ const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;
163
+ const sycl::half2 * q8_1_ds_ptr =
164
+ (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));
165
+
166
+ // up and gate share the activation, so load it once and apply it twice
167
+ const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs);
168
+
169
+ partial_sum[j][0] += reorder_vec_dot_q_sycl::apply(wx, a);
170
+ partial_gate[j][0] += reorder_vec_dot_q_sycl::apply(wg, a);
171
+ }
172
+ } else {
173
+ #pragma unroll
174
+ for (int j = 0; j < ncols_dst; ++j) {
175
+ const char * vy_j = (const char *) vy + j * stride_col_y_bytes;
176
+ const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;
177
+ const sycl::half2 * q8_1_ds_ptr =
178
+ (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));
179
+
180
+ partial_sum[j][0] += reorder_vec_dot_q_sycl::dot(wx, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
181
+ }
182
+ }
183
+ } else {
184
+ const int ibx = row0 * blocks_per_row + i;
185
+ const auto bx_offset = block_type::get_block_offset(ibx, nblocks);
186
+ const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);
187
+ #pragma unroll
188
+ for (int j = 0; j < ncols_dst; ++j) {
189
+ const char * vy_j = (const char *) vy + j * stride_col_y_bytes;
190
+ const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;
191
+ const sycl::half2 * q8_1_ds_ptr =
192
+ (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));
193
+
194
+ partial_sum[j][0] +=
195
+ reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
196
+
197
+ if constexpr (has_fusion) {
198
+ partial_gate[j][0] +=
199
+ reorder_vec_dot_q_sycl()(vgate, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
200
+ }
201
+ }
202
+ }
203
+ }
204
+ }
205
+
206
+ #pragma unroll
207
+ for (int j = 0; j < ncols_dst; ++j) {
208
+ #pragma unroll
209
+ for (int r = 0; r < rows_per_sg; ++r) {
210
+ float sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum[j][r], std::plus<>());
211
+
212
+ if constexpr (has_fusion) {
213
+ const float gate = sycl::reduce_over_group(nd_item.get_sub_group(), partial_gate[j][r], std::plus<>());
214
+
215
+ // uniform across the launch; the launcher only instantiates SWIGLU and GEGLU
216
+ sum *= glu_op == GGML_GLU_OP_SWIGLU ? op_silu(gate) : op_gelu(gate);
217
+ }
218
+
219
+ if (sg.leader() && row0 + r < nrows) {
220
+ dst[j * stride_col_dst + row0 + r] = sum;
221
+ }
222
+ }
223
+ }
224
+ }
225
+
226
+ template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl>
227
+ static void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,
228
+ const int ncols, const int nrows, const sycl::nd_item<3> & item_ct1) {
229
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1);
230
+
231
+ if (row >= nrows) {
232
+ return;
233
+ }
234
+
235
+ const int blocks_per_row = ncols / qk;
236
+ constexpr int blocks_per_warp = (vdr * WARP_SIZE + qi - 1) / qi; // Ensuring blocks_per_warp > 0
237
+
238
+ assert(blocks_per_warp > 0);
239
+
240
+ // partial sum for each thread
241
+ float tmp = 0.0f;
242
+
243
+ const block_q_t * x = (const block_q_t *) vx;
244
+ const block_q8_1 * y = (const block_q8_1 *) vy;
245
+
246
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) {
247
+ const int ibx = row * blocks_per_row + i; // x block index
248
+
249
+ const int iby = i * (qk / QK8_1); // y block index that aligns with ibx
250
+
251
+ for (size_t elem = 0; elem < qi / vdr; elem += WARP_SIZE) {
252
+ const int iqs = elem + vdr * (item_ct1.get_local_id(2) %
253
+ (qi / vdr)); // x block quant index when casting the quants to int
254
+
255
+ tmp += vec_dot_q_sycl(&x[ibx], &y[iby], iqs);
256
+ }
257
+ }
258
+
259
+ // sum up partial sums and write back result
260
+ #pragma unroll
261
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
262
+ tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
263
+ }
264
+
265
+ if (item_ct1.get_local_id(2) == 0) {
266
+ dst[row] = tmp;
267
+ }
268
+ }
269
+
270
+ template <int qk, int qi, typename block_q_t, int vdr,
271
+ vec_dot_q_sycl_t vec_dot_q_sycl, int ncols_dst>
272
+ static void mul_mat_vec_q_ncols(
273
+ const void * __restrict__ vx,
274
+ const void * __restrict__ vy,
275
+ float * __restrict__ dst,
276
+ const int ncols,
277
+ const int nrows,
278
+ const int stride_col_y,
279
+ const int stride_col_dst,
280
+ const sycl::nd_item<3> & item_ct1) {
281
+
282
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1)
283
+ + item_ct1.get_local_id(1);
284
+
285
+ if (row >= nrows) {
286
+ return;
287
+ }
288
+
289
+ const int blocks_per_row = ncols / qk;
290
+ constexpr int blocks_per_warp = (vdr * WARP_SIZE + qi - 1) / qi;
291
+
292
+ // partial sums: one per output column
293
+ float tmp[ncols_dst] = {0.0f};
294
+
295
+ const block_q_t * x = (const block_q_t *) vx;
296
+ const block_q8_1 * y = (const block_q8_1 *) vy;
297
+
298
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr);
299
+ i < blocks_per_row;
300
+ i += blocks_per_warp) {
301
+
302
+ const int ibx = row * blocks_per_row + i;
303
+ const int iby = i * (qk / QK8_1);
304
+
305
+ // read weight block once, dot against all columns
306
+ for (size_t elem = 0; elem < qi / vdr; elem += WARP_SIZE) {
307
+ const int iqs = elem + vdr * (item_ct1.get_local_id(2) % (qi / vdr));
308
+
309
+ #pragma unroll
310
+ for (int j = 0; j < ncols_dst; ++j) {
311
+ tmp[j] += vec_dot_q_sycl(&x[ibx], &y[j * stride_col_y + iby], iqs);
312
+ }
313
+ }
314
+ }
315
+
316
+ // reduce within subgroup
317
+ #pragma unroll
318
+ for (int j = 0; j < ncols_dst; ++j) {
319
+ #pragma unroll
320
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
321
+ tmp[j] += dpct::permute_sub_group_by_xor(
322
+ item_ct1.get_sub_group(), tmp[j], mask);
323
+ }
324
+ }
325
+
326
+ if (item_ct1.get_local_id(2) == 0) {
327
+ #pragma unroll
328
+ for (int j = 0; j < ncols_dst; ++j) {
329
+ dst[j * stride_col_dst + row] = tmp[j];
330
+ }
331
+ }
332
+ }
333
+
334
+ template <int qk, int qi, typename block_q_t, int vdr>
335
+ static void mul_mat_vec_q_iq2_xxs_q8_1(const void *__restrict__ vx,
336
+ const void *__restrict__ vy,
337
+ float *__restrict__ dst, const int ncols,
338
+ const int nrows,
339
+ const sycl::nd_item<3> &item_ct1) {
340
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
341
+ item_ct1.get_local_id(1);
342
+
343
+ if (row >= nrows) {
344
+ return;
345
+ }
346
+
347
+ const int blocks_per_row = ncols / qk;
348
+ const int blocks_per_warp = vdr * WARP_SIZE / qi;
349
+ assert(blocks_per_warp>0);
350
+
351
+ // partial sum for each thread
352
+ float tmp = 0.0f;
353
+
354
+ const block_q_t * x = (const block_q_t *) vx;
355
+ const block_q8_1 * y = (const block_q8_1 *) vy;
356
+
357
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
358
+ i += blocks_per_warp) {
359
+ const int ibx = row*blocks_per_row + i; // x block index
360
+
361
+ const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
362
+
363
+ const int iqs =
364
+ vdr *
365
+ (item_ct1.get_local_id(2) %
366
+ (qi / vdr)); // x block quant index when casting the quants to int
367
+
368
+ tmp += vec_dot_iq2_xxs_q8_1(&x[ibx], &y[iby], iqs, iq2xxs_grid, ksigns_iq2xs, kmask_iq2xs);
369
+ }
370
+
371
+ // sum up partial sums and write back result
372
+ #pragma unroll
373
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
374
+ tmp +=
375
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
376
+ }
377
+
378
+ if (item_ct1.get_local_id(2) == 0) {
379
+ dst[row] = tmp;
380
+ }
381
+ }
382
+
383
+ template <int qk, int qi, typename block_q_t, int vdr>
384
+ static void mul_mat_vec_q_iq2_xs_q8_1(const void *__restrict__ vx,
385
+ const void *__restrict__ vy,
386
+ float *__restrict__ dst, const int ncols,
387
+ const int nrows,
388
+ const sycl::nd_item<3> &item_ct1) {
389
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
390
+ item_ct1.get_local_id(1);
391
+
392
+ if (row >= nrows) {
393
+ return;
394
+ }
395
+
396
+ const int blocks_per_row = ncols / qk;
397
+ const int blocks_per_warp = vdr * WARP_SIZE / qi;
398
+ assert(blocks_per_warp>0);
399
+ // partial sum for each thread
400
+ float tmp = 0.0f;
401
+
402
+ const block_q_t * x = (const block_q_t *) vx;
403
+ const block_q8_1 * y = (const block_q8_1 *) vy;
404
+
405
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
406
+ i += blocks_per_warp) {
407
+ const int ibx = row*blocks_per_row + i; // x block index
408
+
409
+ const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
410
+
411
+ const int iqs =
412
+ vdr *
413
+ (item_ct1.get_local_id(2) %
414
+ (qi / vdr)); // x block quant index when casting the quants to int
415
+
416
+ tmp += vec_dot_iq2_xs_q8_1(&x[ibx], &y[iby], iqs, iq2xs_grid, ksigns64);
417
+ }
418
+
419
+ // sum up partial sums and write back result
420
+ #pragma unroll
421
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
422
+ tmp +=
423
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
424
+ }
425
+
426
+ if (item_ct1.get_local_id(2) == 0) {
427
+ dst[row] = tmp;
428
+ }
429
+ }
430
+
431
+ template <int qk, int qi, typename block_q_t, int vdr>
432
+ static void mul_mat_vec_q_iq2_s_q8_1(const void *__restrict__ vx,
433
+ const void *__restrict__ vy,
434
+ float *__restrict__ dst, const int ncols,
435
+ const int nrows,
436
+ const sycl::nd_item<3> &item_ct1) {
437
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
438
+ item_ct1.get_local_id(1);
439
+
440
+ if (row >= nrows) {
441
+ return;
442
+ }
443
+
444
+ const int blocks_per_row = ncols / qk;
445
+ const int blocks_per_warp = vdr * WARP_SIZE / qi;
446
+ assert(blocks_per_warp>0);
447
+ // partial sum for each thread
448
+ float tmp = 0.0f;
449
+
450
+ const block_q_t * x = (const block_q_t *) vx;
451
+ const block_q8_1 * y = (const block_q8_1 *) vy;
452
+
453
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
454
+ i += blocks_per_warp) {
455
+ const int ibx = row*blocks_per_row + i; // x block index
456
+
457
+ const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
458
+
459
+ const int iqs =
460
+ vdr *
461
+ (item_ct1.get_local_id(2) %
462
+ (qi / vdr)); // x block quant index when casting the quants to int
463
+
464
+ tmp += vec_dot_iq2_s_q8_1(&x[ibx], &y[iby], iqs);
465
+ }
466
+
467
+ // sum up partial sums and write back result
468
+ #pragma unroll
469
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
470
+ tmp +=
471
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
472
+ }
473
+
474
+ if (item_ct1.get_local_id(2) == 0) {
475
+ dst[row] = tmp;
476
+ }
477
+ }
478
+
479
+ template <int qk, int qi, typename block_q_t, int vdr>
480
+ static void mul_mat_vec_q_iq3_xxs_q8_1(const void *__restrict__ vx,
481
+ const void *__restrict__ vy,
482
+ float *__restrict__ dst, const int ncols,
483
+ const int nrows,
484
+ const sycl::nd_item<3> &item_ct1) {
485
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
486
+ item_ct1.get_local_id(1);
487
+
488
+ if (row >= nrows) {
489
+ return;
490
+ }
491
+
492
+ const int blocks_per_row = ncols / qk;
493
+ const int blocks_per_warp = vdr * WARP_SIZE / qi;
494
+ assert(blocks_per_warp>0);
495
+ // partial sum for each thread
496
+ float tmp = 0.0f;
497
+
498
+ const block_q_t * x = (const block_q_t *) vx;
499
+ const block_q8_1 * y = (const block_q8_1 *) vy;
500
+
501
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
502
+ i += blocks_per_warp) {
503
+ const int ibx = row*blocks_per_row + i; // x block index
504
+
505
+ const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
506
+
507
+ const int iqs =
508
+ vdr *
509
+ (item_ct1.get_local_id(2) %
510
+ (qi / vdr)); // x block quant index when casting the quants to int
511
+
512
+ tmp += vec_dot_iq3_xxs_q8_1(&x[ibx], &y[iby], iqs, iq3xxs_grid, ksigns64);
513
+ }
514
+
515
+ // sum up partial sums and write back result
516
+ #pragma unroll
517
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
518
+ tmp +=
519
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
520
+ }
521
+
522
+ if (item_ct1.get_local_id(2) == 0) {
523
+ dst[row] = tmp;
524
+ }
525
+ }
526
+
527
+ template <int qk, int qi, typename block_q_t, int vdr>
528
+ static void mul_mat_vec_q_iq3_s_q8_1(const void *__restrict__ vx,
529
+ const void *__restrict__ vy,
530
+ float *__restrict__ dst, const int ncols,
531
+ const int nrows,
532
+ const sycl::nd_item<3> &item_ct1) {
533
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
534
+ item_ct1.get_local_id(1);
535
+
536
+ if (row >= nrows) {
537
+ return;
538
+ }
539
+
540
+ const int blocks_per_row = ncols / qk;
541
+ const int blocks_per_warp = vdr * WARP_SIZE / qi;
542
+ assert(blocks_per_warp>0);
543
+ // partial sum for each thread
544
+ float tmp = 0.0f;
545
+
546
+ const block_q_t * x = (const block_q_t *) vx;
547
+ const block_q8_1 * y = (const block_q8_1 *) vy;
548
+
549
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
550
+ i += blocks_per_warp) {
551
+ const int ibx = row*blocks_per_row + i; // x block index
552
+
553
+ const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
554
+
555
+ const int iqs =
556
+ vdr *
557
+ (item_ct1.get_local_id(2) %
558
+ (qi / vdr)); // x block quant index when casting the quants to int
559
+
560
+ tmp += vec_dot_iq3_s_q8_1(&x[ibx], &y[iby], iqs, iq3s_grid);
561
+ }
562
+
563
+ // sum up partial sums and write back result
564
+ #pragma unroll
565
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
566
+ tmp +=
567
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
568
+ }
569
+
570
+ if (item_ct1.get_local_id(2) == 0) {
571
+ dst[row] = tmp;
572
+ }
573
+ }
574
+
575
+ template <int qk, int qi, typename block_q_t, int vdr>
576
+ static void mul_mat_vec_q_iq1_s_q8_1(const void *__restrict__ vx,
577
+ const void *__restrict__ vy,
578
+ float *__restrict__ dst, const int ncols,
579
+ const int nrows,
580
+ const sycl::nd_item<3> &item_ct1) {
581
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
582
+ item_ct1.get_local_id(1);
583
+
584
+ if (row >= nrows) {
585
+ return;
586
+ }
587
+
588
+ const int blocks_per_row = ncols / qk;
589
+ const int blocks_per_warp = vdr * WARP_SIZE / qi;
590
+ assert(blocks_per_warp>0);
591
+ // partial sum for each thread
592
+ float tmp = 0.0f;
593
+
594
+ const block_q_t * x = (const block_q_t *) vx;
595
+ const block_q8_1 * y = (const block_q8_1 *) vy;
596
+
597
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
598
+ i += blocks_per_warp) {
599
+ const int ibx = row*blocks_per_row + i; // x block index
600
+
601
+ const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
602
+
603
+ const int iqs =
604
+ vdr *
605
+ (item_ct1.get_local_id(2) %
606
+ (qi / vdr)); // x block quant index when casting the quants to int
607
+
608
+ tmp += vec_dot_iq1_s_q8_1(&x[ibx], &y[iby], iqs, iq1s_grid_gpu);
609
+ }
610
+
611
+ // sum up partial sums and write back result
612
+ #pragma unroll
613
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
614
+ tmp +=
615
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
616
+ }
617
+
618
+ if (item_ct1.get_local_id(2) == 0) {
619
+ dst[row] = tmp;
620
+ }
621
+ }
622
+
623
+ template <int qk, int qi, typename block_q_t, int vdr>
624
+ static void mul_mat_vec_q_iq1_m_q8_1(const void *__restrict__ vx,
625
+ const void *__restrict__ vy,
626
+ float *__restrict__ dst, const int ncols,
627
+ const int nrows,
628
+ const sycl::nd_item<3> &item_ct1) {
629
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
630
+ item_ct1.get_local_id(1);
631
+
632
+ if (row >= nrows) {
633
+ return;
634
+ }
635
+
636
+ const int blocks_per_row = ncols / qk;
637
+ const int blocks_per_warp = vdr * WARP_SIZE / qi;
638
+ assert(blocks_per_warp>0);
639
+ // partial sum for each thread
640
+ float tmp = 0.0f;
641
+
642
+ const block_q_t * x = (const block_q_t *) vx;
643
+ const block_q8_1 * y = (const block_q8_1 *) vy;
644
+
645
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
646
+ i += blocks_per_warp) {
647
+ const int ibx = row*blocks_per_row + i; // x block index
648
+
649
+ const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
650
+
651
+ const int iqs =
652
+ vdr *
653
+ (item_ct1.get_local_id(2) %
654
+ (qi / vdr)); // x block quant index when casting the quants to int
655
+
656
+ tmp += vec_dot_iq1_m_q8_1(&x[ibx], &y[iby], iqs);
657
+ }
658
+
659
+ // sum up partial sums and write back result
660
+ #pragma unroll
661
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
662
+ tmp +=
663
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
664
+ }
665
+
666
+ if (item_ct1.get_local_id(2) == 0) {
667
+ dst[row] = tmp;
668
+ }
669
+ }
670
+
671
+ template <int qk, int qi, typename block_q_t, int vdr>
672
+ static void mul_mat_vec_q_iq4_nl_q8_1(const void *__restrict__ vx,
673
+ const void *__restrict__ vy,
674
+ float *__restrict__ dst, const int ncols,
675
+ const int nrows,
676
+ const sycl::nd_item<3> &item_ct1) {
677
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
678
+ item_ct1.get_local_id(1);
679
+
680
+ if (row >= nrows) {
681
+ return;
682
+ }
683
+
684
+ const int blocks_per_row = ncols / qk;
685
+ const int blocks_per_warp = vdr * WARP_SIZE / qi;
686
+ assert(blocks_per_warp>0);
687
+ // partial sum for each thread
688
+ float tmp = 0.0f;
689
+
690
+ const block_q_t * x = (const block_q_t *) vx;
691
+ const block_q8_1 * y = (const block_q8_1 *) vy;
692
+
693
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
694
+ i += blocks_per_warp) {
695
+ const int ibx = row*blocks_per_row + i; // x block index
696
+
697
+ const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
698
+
699
+ const int iqs =
700
+ vdr *
701
+ (item_ct1.get_local_id(2) %
702
+ (qi / vdr)); // x block quant index when casting the quants to int
703
+
704
+ tmp += vec_dot_iq4_nl_q8_1(&x[ibx], &y[iby], iqs);
705
+ }
706
+
707
+ // sum up partial sums and write back result
708
+ #pragma unroll
709
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
710
+ tmp +=
711
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
712
+ }
713
+
714
+ if (item_ct1.get_local_id(2) == 0) {
715
+ dst[row] = tmp;
716
+ }
717
+ }
718
+
719
+
720
+ template <int qk, int qi, typename block_q_t, int vdr>
721
+ static void mul_mat_vec_q_iq4_xs_q8_1(const void *__restrict__ vx,
722
+ const void *__restrict__ vy,
723
+ float *__restrict__ dst, const int ncols,
724
+ const int nrows,
725
+ const sycl::nd_item<3> &item_ct1) {
726
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
727
+ item_ct1.get_local_id(1);
728
+
729
+ if (row >= nrows) {
730
+ return;
731
+ }
732
+
733
+ const int blocks_per_row = ncols / qk;
734
+ const int blocks_per_warp = vdr * WARP_SIZE / qi;
735
+ assert(blocks_per_warp>0);
736
+ // partial sum for each thread
737
+ float tmp = 0.0f;
738
+
739
+ const block_q_t * x = (const block_q_t *) vx;
740
+ const block_q8_1 * y = (const block_q8_1 *) vy;
741
+
742
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
743
+ i += blocks_per_warp) {
744
+ const int ibx = row*blocks_per_row + i; // x block index
745
+
746
+ const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
747
+
748
+ const int iqs =
749
+ vdr *
750
+ (item_ct1.get_local_id(2) %
751
+ (qi / vdr)); // x block quant index when casting the quants to int
752
+
753
+ tmp += vec_dot_iq4_xs_q8_1(&x[ibx], &y[iby], iqs);
754
+ }
755
+
756
+ // sum up partial sums and write back result
757
+ #pragma unroll
758
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
759
+ tmp +=
760
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
761
+ }
762
+
763
+ if (item_ct1.get_local_id(2) == 0) {
764
+ dst[row] = tmp;
765
+ }
766
+ }
767
+
768
+ static void reorder_mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
769
+ const int nrows, dpct::queue_ptr stream) {
770
+ GGML_ASSERT(ncols % QK4_0 == 0);
771
+ // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
772
+ constexpr size_t num_subgroups = WARP_SIZE;
773
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
774
+ const sycl::range<3> block_nums(1, 1, block_num_y);
775
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
776
+
777
+ stream->submit([&](sycl::handler & cgh) {
778
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
779
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
780
+ mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_0>>(vx, vy, dst, ncols, nrows,
781
+ nd_item);
782
+ });
783
+ });
784
+ }
785
+
786
+ template <int ncols_dst>
787
+ static void reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols(
788
+ const void * vx, const void * vy, float * dst,
789
+ const int ncols, const int nrows,
790
+ const int stride_col_y_bytes, const int stride_col_dst,
791
+ dpct::queue_ptr stream) {
792
+ GGML_ASSERT(ncols % QK4_0 == 0);
793
+ constexpr size_t num_subgroups = WARP_SIZE;
794
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
795
+ const sycl::range<3> block_nums(1, 1, block_num_y);
796
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
797
+
798
+ stream->submit([&](sycl::handler & cgh) {
799
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
800
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
801
+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_0>, ncols_dst>(
802
+ vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
803
+ /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
804
+ });
805
+ });
806
+ }
807
+
808
+ static void reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(
809
+ const void * vx, const void * vy, float * dst,
810
+ const int ncols, const int nrows, const int ncols_dst,
811
+ const int stride_col_y_bytes, const int stride_col_dst,
812
+ dpct::queue_ptr stream) {
813
+ switch (ncols_dst) {
814
+ case 1: reorder_mul_mat_vec_q4_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
815
+ case 2: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
816
+ case 3: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
817
+ case 4: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
818
+ case 5: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
819
+ case 6: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
820
+ case 7: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
821
+ case 8: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
822
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q4_0 reorder multi-col MMVQ", ncols_dst);
823
+ }
824
+ }
825
+
826
+ static void mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows,
827
+ dpct::queue_ptr stream) {
828
+ GGML_ASSERT(ncols % QK4_0 == 0);
829
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
830
+ const sycl::range<3> block_nums(1, 1, block_num_y);
831
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
832
+
833
+ {
834
+ stream->submit([&](sycl::handler & cgh) {
835
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
836
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
837
+ mul_mat_vec_q<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>(
838
+ vx, vy, dst, ncols, nrows, item_ct1);
839
+ });
840
+ });
841
+ }
842
+ }
843
+
844
+ template <int ncols_dst>
845
+ static void mul_mat_vec_q4_0_q8_1_sycl_ncols(
846
+ const void * vx, const void * vy, float * dst,
847
+ const int ncols, const int nrows,
848
+ const int stride_col_y, const int stride_col_dst,
849
+ dpct::queue_ptr stream) {
850
+ GGML_ASSERT(ncols % QK4_0 == 0);
851
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
852
+ const sycl::range<3> block_nums(1, 1, block_num_y);
853
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
854
+ stream->submit([&](sycl::handler & cgh) {
855
+ cgh.parallel_for(
856
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
857
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
858
+ mul_mat_vec_q_ncols<QK4_0, QI4_0, block_q4_0,
859
+ VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1, ncols_dst>(
860
+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
861
+ });
862
+ });
863
+ }
864
+
865
+ static void mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(
866
+ const void * vx, const void * vy, float * dst,
867
+ const int ncols, const int nrows, const int ncols_dst,
868
+ const int stride_col_y, const int stride_col_dst,
869
+ dpct::queue_ptr stream) {
870
+ switch (ncols_dst) {
871
+ case 1: mul_mat_vec_q4_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
872
+ case 2: mul_mat_vec_q4_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
873
+ case 3: mul_mat_vec_q4_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
874
+ case 4: mul_mat_vec_q4_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
875
+ case 5: mul_mat_vec_q4_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
876
+ case 6: mul_mat_vec_q4_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
877
+ case 7: mul_mat_vec_q4_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
878
+ case 8: mul_mat_vec_q4_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
879
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q4_0 multi-col MMVQ", ncols_dst);
880
+ }
881
+ }
882
+
883
+ static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy,
884
+ float *dst, const int ncols,
885
+ const int nrows,
886
+ dpct::queue_ptr stream) {
887
+ GGML_ASSERT(ncols % QK4_1 == 0);
888
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
889
+ const sycl::range<3> block_nums(1, 1, block_num_y);
890
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
891
+ {
892
+
893
+ stream->submit([&](sycl::handler &cgh) {
894
+
895
+ cgh.parallel_for(
896
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
897
+ [=](sycl::nd_item<3> item_ct1)
898
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
899
+ mul_mat_vec_q<QK4_0, QI4_1, block_q4_1,
900
+ VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>(
901
+ vx, vy, dst, ncols, nrows, item_ct1);
902
+ });
903
+ });
904
+ }
905
+ }
906
+
907
+ template <int ncols_dst>
908
+ static void mul_mat_vec_q4_1_q8_1_sycl_ncols(
909
+ const void * vx, const void * vy, float * dst,
910
+ const int ncols, const int nrows,
911
+ const int stride_col_y, const int stride_col_dst,
912
+ dpct::queue_ptr stream) {
913
+ GGML_ASSERT(ncols % QK4_1 == 0);
914
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
915
+ const sycl::range<3> block_nums(1, 1, block_num_y);
916
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
917
+ stream->submit([&](sycl::handler & cgh) {
918
+ cgh.parallel_for(
919
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
920
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
921
+ mul_mat_vec_q_ncols<QK4_0, QI4_1, block_q4_1,
922
+ VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1, ncols_dst>(
923
+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
924
+ });
925
+ });
926
+ }
927
+
928
+ static void mul_mat_vec_q4_1_q8_1_sycl_switch_ncols(
929
+ const void * vx, const void * vy, float * dst,
930
+ const int ncols, const int nrows, const int ncols_dst,
931
+ const int stride_col_y, const int stride_col_dst,
932
+ dpct::queue_ptr stream) {
933
+ switch (ncols_dst) {
934
+ case 1: mul_mat_vec_q4_1_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
935
+ case 2: mul_mat_vec_q4_1_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
936
+ case 3: mul_mat_vec_q4_1_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
937
+ case 4: mul_mat_vec_q4_1_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
938
+ case 5: mul_mat_vec_q4_1_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
939
+ case 6: mul_mat_vec_q4_1_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
940
+ case 7: mul_mat_vec_q4_1_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
941
+ case 8: mul_mat_vec_q4_1_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
942
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q4_1 multi-col MMVQ", ncols_dst);
943
+ }
944
+ }
945
+
946
+ static void mul_mat_vec_mxfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows,
947
+ dpct::queue_ptr stream) {
948
+ GGML_ASSERT(ncols % QK_MXFP4 == 0);
949
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
950
+ const sycl::range<3> block_nums(1, 1, block_num_y);
951
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
952
+
953
+ {
954
+ stream->submit([&](sycl::handler & cgh) {
955
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
956
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
957
+ mul_mat_vec_q<QK_MXFP4, QI_MXFP4, block_mxfp4, VDR_MXFP4_Q8_1_MMVQ, vec_dot_mxfp4_q8_1>(
958
+ vx, vy, dst, ncols, nrows, item_ct1);
959
+ });
960
+ });
961
+ }
962
+ }
963
+
964
+ template <int ncols_dst>
965
+ static void mul_mat_vec_mxfp4_q8_1_sycl_ncols(
966
+ const void * vx, const void * vy, float * dst,
967
+ const int ncols, const int nrows,
968
+ const int stride_col_y, const int stride_col_dst,
969
+ dpct::queue_ptr stream) {
970
+ GGML_ASSERT(ncols % QK_MXFP4 == 0);
971
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
972
+ const sycl::range<3> block_nums(1, 1, block_num_y);
973
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
974
+ stream->submit([&](sycl::handler & cgh) {
975
+ cgh.parallel_for(
976
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
977
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
978
+ mul_mat_vec_q_ncols<QK_MXFP4, QI_MXFP4, block_mxfp4,
979
+ VDR_MXFP4_Q8_1_MMVQ, vec_dot_mxfp4_q8_1, ncols_dst>(
980
+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
981
+ });
982
+ });
983
+ }
984
+
985
+ static void mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols(
986
+ const void * vx, const void * vy, float * dst,
987
+ const int ncols, const int nrows, const int ncols_dst,
988
+ const int stride_col_y, const int stride_col_dst,
989
+ dpct::queue_ptr stream) {
990
+ switch (ncols_dst) {
991
+ case 1: mul_mat_vec_mxfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
992
+ case 2: mul_mat_vec_mxfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
993
+ case 3: mul_mat_vec_mxfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
994
+ case 4: mul_mat_vec_mxfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
995
+ case 5: mul_mat_vec_mxfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
996
+ case 6: mul_mat_vec_mxfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
997
+ case 7: mul_mat_vec_mxfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
998
+ case 8: mul_mat_vec_mxfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
999
+ default: GGML_ABORT("unsupported ncols_dst=%d for MXFP4 multi-col MMVQ", ncols_dst);
1000
+ }
1001
+ }
1002
+
1003
+ static void mul_mat_vec_nvfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows,
1004
+ dpct::queue_ptr stream) {
1005
+ GGML_ASSERT(ncols % QK_NVFP4 == 0);
1006
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1007
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1008
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1009
+
1010
+ {
1011
+ stream->submit([&](sycl::handler & cgh) {
1012
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1013
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1014
+ mul_mat_vec_q<QK_NVFP4, QI_NVFP4, block_nvfp4, VDR_NVFP4_Q8_1_MMVQ, vec_dot_nvfp4_q8_1>(
1015
+ vx, vy, dst, ncols, nrows, item_ct1);
1016
+ });
1017
+ });
1018
+ }
1019
+ }
1020
+
1021
+ template <int ncols_dst>
1022
+ static void mul_mat_vec_nvfp4_q8_1_sycl_ncols(
1023
+ const void * vx, const void * vy, float * dst,
1024
+ const int ncols, const int nrows,
1025
+ const int stride_col_y, const int stride_col_dst,
1026
+ dpct::queue_ptr stream) {
1027
+ GGML_ASSERT(ncols % QK_NVFP4 == 0);
1028
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1029
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1030
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1031
+ stream->submit([&](sycl::handler & cgh) {
1032
+ cgh.parallel_for(
1033
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1034
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1035
+ mul_mat_vec_q_ncols<QK_NVFP4, QI_NVFP4, block_nvfp4,
1036
+ VDR_NVFP4_Q8_1_MMVQ, vec_dot_nvfp4_q8_1, ncols_dst>(
1037
+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
1038
+ });
1039
+ });
1040
+ }
1041
+
1042
+ static void mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols(
1043
+ const void * vx, const void * vy, float * dst,
1044
+ const int ncols, const int nrows, const int ncols_dst,
1045
+ const int stride_col_y, const int stride_col_dst,
1046
+ dpct::queue_ptr stream) {
1047
+ switch (ncols_dst) {
1048
+ case 1: mul_mat_vec_nvfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1049
+ case 2: mul_mat_vec_nvfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1050
+ case 3: mul_mat_vec_nvfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1051
+ case 4: mul_mat_vec_nvfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1052
+ case 5: mul_mat_vec_nvfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1053
+ case 6: mul_mat_vec_nvfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1054
+ case 7: mul_mat_vec_nvfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1055
+ case 8: mul_mat_vec_nvfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1056
+ default: GGML_ABORT("unsupported ncols_dst=%d for NVFP4 multi-col MMVQ", ncols_dst);
1057
+ }
1058
+ }
1059
+
1060
+ static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy,
1061
+ float *dst, const int ncols,
1062
+ const int nrows,
1063
+ dpct::queue_ptr stream) {
1064
+ GGML_ASSERT(ncols % QK5_0 == 0);
1065
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1066
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1067
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1068
+ {
1069
+
1070
+ stream->submit([&](sycl::handler &cgh) {
1071
+
1072
+ cgh.parallel_for(
1073
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1074
+ [=](sycl::nd_item<3> item_ct1)
1075
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1076
+ mul_mat_vec_q<QK5_0, QI5_0, block_q5_0,
1077
+ VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>(
1078
+ vx, vy, dst, ncols, nrows, item_ct1);
1079
+ });
1080
+ });
1081
+ }
1082
+ }
1083
+
1084
+ template <int ncols_dst>
1085
+ static void mul_mat_vec_q5_0_q8_1_sycl_ncols(
1086
+ const void * vx, const void * vy, float * dst,
1087
+ const int ncols, const int nrows,
1088
+ const int stride_col_y, const int stride_col_dst,
1089
+ dpct::queue_ptr stream) {
1090
+ GGML_ASSERT(ncols % QK5_0 == 0);
1091
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1092
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1093
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1094
+ stream->submit([&](sycl::handler & cgh) {
1095
+ cgh.parallel_for(
1096
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1097
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1098
+ mul_mat_vec_q_ncols<QK5_0, QI5_0, block_q5_0,
1099
+ VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1, ncols_dst>(
1100
+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
1101
+ });
1102
+ });
1103
+ }
1104
+
1105
+ static void mul_mat_vec_q5_0_q8_1_sycl_switch_ncols(
1106
+ const void * vx, const void * vy, float * dst,
1107
+ const int ncols, const int nrows, const int ncols_dst,
1108
+ const int stride_col_y, const int stride_col_dst,
1109
+ dpct::queue_ptr stream) {
1110
+ switch (ncols_dst) {
1111
+ case 1: mul_mat_vec_q5_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1112
+ case 2: mul_mat_vec_q5_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1113
+ case 3: mul_mat_vec_q5_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1114
+ case 4: mul_mat_vec_q5_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1115
+ case 5: mul_mat_vec_q5_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1116
+ case 6: mul_mat_vec_q5_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1117
+ case 7: mul_mat_vec_q5_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1118
+ case 8: mul_mat_vec_q5_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1119
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q5_0 multi-col MMVQ", ncols_dst);
1120
+ }
1121
+ }
1122
+
1123
+ static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy,
1124
+ float *dst, const int ncols,
1125
+ const int nrows,
1126
+ dpct::queue_ptr stream) {
1127
+ GGML_ASSERT(ncols % QK5_1 == 0);
1128
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1129
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1130
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1131
+ {
1132
+
1133
+ stream->submit([&](sycl::handler &cgh) {
1134
+
1135
+ cgh.parallel_for(
1136
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1137
+ [=](sycl::nd_item<3> item_ct1)
1138
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1139
+ mul_mat_vec_q<QK5_1, QI5_1, block_q5_1,
1140
+ VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>(
1141
+ vx, vy, dst, ncols, nrows, item_ct1);
1142
+ });
1143
+ });
1144
+ }
1145
+ }
1146
+
1147
+ template <int ncols_dst>
1148
+ static void mul_mat_vec_q5_1_q8_1_sycl_ncols(
1149
+ const void * vx, const void * vy, float * dst,
1150
+ const int ncols, const int nrows,
1151
+ const int stride_col_y, const int stride_col_dst,
1152
+ dpct::queue_ptr stream) {
1153
+ GGML_ASSERT(ncols % QK5_1 == 0);
1154
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1155
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1156
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1157
+ stream->submit([&](sycl::handler & cgh) {
1158
+ cgh.parallel_for(
1159
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1160
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1161
+ mul_mat_vec_q_ncols<QK5_1, QI5_1, block_q5_1,
1162
+ VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1, ncols_dst>(
1163
+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
1164
+ });
1165
+ });
1166
+ }
1167
+
1168
+ static void mul_mat_vec_q5_1_q8_1_sycl_switch_ncols(
1169
+ const void * vx, const void * vy, float * dst,
1170
+ const int ncols, const int nrows, const int ncols_dst,
1171
+ const int stride_col_y, const int stride_col_dst,
1172
+ dpct::queue_ptr stream) {
1173
+ switch (ncols_dst) {
1174
+ case 1: mul_mat_vec_q5_1_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1175
+ case 2: mul_mat_vec_q5_1_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1176
+ case 3: mul_mat_vec_q5_1_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1177
+ case 4: mul_mat_vec_q5_1_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1178
+ case 5: mul_mat_vec_q5_1_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1179
+ case 6: mul_mat_vec_q5_1_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1180
+ case 7: mul_mat_vec_q5_1_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1181
+ case 8: mul_mat_vec_q5_1_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1182
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q5_1 multi-col MMVQ", ncols_dst);
1183
+ }
1184
+ }
1185
+
1186
+ static void reorder_mul_mat_vec_q8_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
1187
+ const int nrows, dpct::queue_ptr stream) {
1188
+ GGML_ASSERT(ncols % QK8_0 == 0);
1189
+ // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
1190
+ constexpr size_t num_subgroups = WARP_SIZE;
1191
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
1192
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1193
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1194
+
1195
+ stream->submit([&](sycl::handler & cgh) {
1196
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1197
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1198
+ mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0>>(vx, vy, dst, ncols, nrows,
1199
+ nd_item);
1200
+ });
1201
+ });
1202
+ }
1203
+
1204
+ template <int ncols_dst>
1205
+ static void reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols(
1206
+ const void * vx, const void * vy, float * dst,
1207
+ const int ncols, const int nrows,
1208
+ const int stride_col_y_bytes, const int stride_col_dst,
1209
+ dpct::queue_ptr stream) {
1210
+ GGML_ASSERT(ncols % QK8_0 == 0);
1211
+ constexpr size_t num_subgroups = WARP_SIZE;
1212
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
1213
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1214
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1215
+
1216
+ stream->submit([&](sycl::handler & cgh) {
1217
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1218
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1219
+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0>, ncols_dst>(
1220
+ vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
1221
+ /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
1222
+ });
1223
+ });
1224
+ }
1225
+
1226
+ static void reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(
1227
+ const void * vx, const void * vy, float * dst,
1228
+ const int ncols, const int nrows, const int ncols_dst,
1229
+ const int stride_col_y_bytes, const int stride_col_dst,
1230
+ dpct::queue_ptr stream) {
1231
+ switch (ncols_dst) {
1232
+ case 1: reorder_mul_mat_vec_q8_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1233
+ case 2: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1234
+ case 3: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1235
+ case 4: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1236
+ case 5: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1237
+ case 6: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1238
+ case 7: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1239
+ case 8: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1240
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q8_0 reorder multi-col MMVQ", ncols_dst);
1241
+ }
1242
+ }
1243
+
1244
+ static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy,
1245
+ float *dst, const int ncols,
1246
+ const int nrows,
1247
+ dpct::queue_ptr stream) {
1248
+ GGML_ASSERT(ncols % QK8_0 == 0);
1249
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1250
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1251
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1252
+ {
1253
+
1254
+ stream->submit([&](sycl::handler &cgh) {
1255
+
1256
+ cgh.parallel_for(
1257
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1258
+ [=](sycl::nd_item<3> item_ct1)
1259
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1260
+ mul_mat_vec_q<QK8_0, QI8_0, block_q8_0,
1261
+ VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>(
1262
+ vx, vy, dst, ncols, nrows, item_ct1);
1263
+ });
1264
+ });
1265
+ }
1266
+ }
1267
+
1268
+ template <int ncols_dst>
1269
+ static void mul_mat_vec_q8_0_q8_1_sycl_ncols(
1270
+ const void * vx, const void * vy, float * dst,
1271
+ const int ncols, const int nrows,
1272
+ const int stride_col_y, const int stride_col_dst,
1273
+ dpct::queue_ptr stream) {
1274
+ GGML_ASSERT(ncols % QK8_0 == 0);
1275
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1276
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1277
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1278
+ stream->submit([&](sycl::handler & cgh) {
1279
+ cgh.parallel_for(
1280
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1281
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1282
+ mul_mat_vec_q_ncols<QK8_0, QI8_0, block_q8_0,
1283
+ VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1, ncols_dst>(
1284
+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
1285
+ });
1286
+ });
1287
+ }
1288
+
1289
+ static void mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(
1290
+ const void * vx, const void * vy, float * dst,
1291
+ const int ncols, const int nrows, const int ncols_dst,
1292
+ const int stride_col_y, const int stride_col_dst,
1293
+ dpct::queue_ptr stream) {
1294
+ switch (ncols_dst) {
1295
+ case 1: mul_mat_vec_q8_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1296
+ case 2: mul_mat_vec_q8_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1297
+ case 3: mul_mat_vec_q8_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1298
+ case 4: mul_mat_vec_q8_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1299
+ case 5: mul_mat_vec_q8_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1300
+ case 6: mul_mat_vec_q8_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1301
+ case 7: mul_mat_vec_q8_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1302
+ case 8: mul_mat_vec_q8_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1303
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q8_0 multi-col MMVQ", ncols_dst);
1304
+ }
1305
+ }
1306
+
1307
+ static void mul_mat_vec_q1_0_q8_1_sycl(const void * vx, const void * vy,
1308
+ float * dst, const int ncols,
1309
+ const int nrows,
1310
+ dpct::queue_ptr stream) {
1311
+ GGML_ASSERT(ncols % QK1_0 == 0);
1312
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1313
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1314
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1315
+
1316
+ stream->submit([&](sycl::handler & cgh) {
1317
+ cgh.parallel_for(
1318
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1319
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1320
+ mul_mat_vec_q<QK1_0, QI1_0, block_q1_0,
1321
+ VDR_Q1_0_Q8_1_MMVQ, vec_dot_q1_0_q8_1>(
1322
+ vx, vy, dst, ncols, nrows, item_ct1);
1323
+ });
1324
+ });
1325
+ }
1326
+
1327
+ template <int ncols_dst>
1328
+ static void mul_mat_vec_q1_0_q8_1_sycl_ncols(
1329
+ const void * vx, const void * vy, float * dst,
1330
+ const int ncols, const int nrows,
1331
+ const int stride_col_y, const int stride_col_dst,
1332
+ dpct::queue_ptr stream) {
1333
+ GGML_ASSERT(ncols % QK1_0 == 0);
1334
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1335
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1336
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1337
+
1338
+ stream->submit([&](sycl::handler & cgh) {
1339
+ cgh.parallel_for(
1340
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1341
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1342
+ mul_mat_vec_q_ncols<QK1_0, QI1_0, block_q1_0,
1343
+ VDR_Q1_0_Q8_1_MMVQ, vec_dot_q1_0_q8_1, ncols_dst>(
1344
+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
1345
+ });
1346
+ });
1347
+ }
1348
+
1349
+ static void mul_mat_vec_q1_0_q8_1_sycl_switch_ncols(
1350
+ const void * vx, const void * vy, float * dst,
1351
+ const int ncols, const int nrows, const int ncols_dst,
1352
+ const int stride_col_y, const int stride_col_dst,
1353
+ dpct::queue_ptr stream) {
1354
+ switch (ncols_dst) {
1355
+ case 1: mul_mat_vec_q1_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1356
+ case 2: mul_mat_vec_q1_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1357
+ case 3: mul_mat_vec_q1_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1358
+ case 4: mul_mat_vec_q1_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1359
+ case 5: mul_mat_vec_q1_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1360
+ case 6: mul_mat_vec_q1_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1361
+ case 7: mul_mat_vec_q1_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1362
+ case 8: mul_mat_vec_q1_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1363
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q1_0 multi-col MMVQ", ncols_dst);
1364
+ }
1365
+ }
1366
+
1367
+ static void mul_mat_vec_q2_0_q8_1_sycl(const void * vx, const void * vy,
1368
+ float * dst, const int ncols,
1369
+ const int nrows,
1370
+ dpct::queue_ptr stream) {
1371
+ GGML_ASSERT(ncols % QK2_0 == 0);
1372
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1373
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1374
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1375
+
1376
+ stream->submit([&](sycl::handler & cgh) {
1377
+ cgh.parallel_for(
1378
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1379
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1380
+ mul_mat_vec_q<QK2_0, QI2_0, block_q2_0,
1381
+ VDR_Q2_0_Q8_1_MMVQ, vec_dot_q2_0_q8_1>(
1382
+ vx, vy, dst, ncols, nrows, item_ct1);
1383
+ });
1384
+ });
1385
+ }
1386
+
1387
+ template <int ncols_dst>
1388
+ static void mul_mat_vec_q2_0_q8_1_sycl_ncols(
1389
+ const void * vx, const void * vy, float * dst,
1390
+ const int ncols, const int nrows,
1391
+ const int stride_col_y, const int stride_col_dst,
1392
+ dpct::queue_ptr stream) {
1393
+ GGML_ASSERT(ncols % QK2_0 == 0);
1394
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1395
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1396
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1397
+
1398
+ stream->submit([&](sycl::handler & cgh) {
1399
+ cgh.parallel_for(
1400
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1401
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1402
+ mul_mat_vec_q_ncols<QK2_0, QI2_0, block_q2_0,
1403
+ VDR_Q2_0_Q8_1_MMVQ, vec_dot_q2_0_q8_1, ncols_dst>(
1404
+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
1405
+ });
1406
+ });
1407
+ }
1408
+
1409
+ static void mul_mat_vec_q2_0_q8_1_sycl_switch_ncols(
1410
+ const void * vx, const void * vy, float * dst,
1411
+ const int ncols, const int nrows, const int ncols_dst,
1412
+ const int stride_col_y, const int stride_col_dst,
1413
+ dpct::queue_ptr stream) {
1414
+ switch (ncols_dst) {
1415
+ case 1: mul_mat_vec_q2_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1416
+ case 2: mul_mat_vec_q2_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1417
+ case 3: mul_mat_vec_q2_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1418
+ case 4: mul_mat_vec_q2_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1419
+ case 5: mul_mat_vec_q2_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1420
+ case 6: mul_mat_vec_q2_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1421
+ case 7: mul_mat_vec_q2_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1422
+ case 8: mul_mat_vec_q2_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1423
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q2_0 multi-col MMVQ", ncols_dst);
1424
+ }
1425
+ }
1426
+
1427
+ static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy,
1428
+ float *dst, const int ncols,
1429
+ const int nrows,
1430
+ dpct::queue_ptr stream) {
1431
+ GGML_ASSERT(ncols % QK_K == 0);
1432
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1433
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1434
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1435
+ {
1436
+
1437
+ stream->submit([&](sycl::handler &cgh) {
1438
+
1439
+ cgh.parallel_for(
1440
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1441
+ [=](sycl::nd_item<3> item_ct1)
1442
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1443
+ mul_mat_vec_q<QK_K, QI2_K, block_q2_K,
1444
+ VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>(
1445
+ vx, vy, dst, ncols, nrows, item_ct1);
1446
+ });
1447
+ });
1448
+ }
1449
+ }
1450
+
1451
+ template <int ncols_dst>
1452
+ static void mul_mat_vec_q2_K_q8_1_sycl_ncols(
1453
+ const void * vx, const void * vy, float * dst,
1454
+ const int ncols, const int nrows,
1455
+ const int stride_col_y, const int stride_col_dst,
1456
+ dpct::queue_ptr stream) {
1457
+ GGML_ASSERT(ncols % QK_K == 0);
1458
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1459
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1460
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1461
+ stream->submit([&](sycl::handler & cgh) {
1462
+ cgh.parallel_for(
1463
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1464
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1465
+ mul_mat_vec_q_ncols<QK_K, QI2_K, block_q2_K,
1466
+ VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1, ncols_dst>(
1467
+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
1468
+ });
1469
+ });
1470
+ }
1471
+
1472
+ static void mul_mat_vec_q2_K_q8_1_sycl_switch_ncols(
1473
+ const void * vx, const void * vy, float * dst,
1474
+ const int ncols, const int nrows, const int ncols_dst,
1475
+ const int stride_col_y, const int stride_col_dst,
1476
+ dpct::queue_ptr stream) {
1477
+ switch (ncols_dst) {
1478
+ case 1: mul_mat_vec_q2_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1479
+ case 2: mul_mat_vec_q2_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1480
+ case 3: mul_mat_vec_q2_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1481
+ case 4: mul_mat_vec_q2_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1482
+ case 5: mul_mat_vec_q2_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1483
+ case 6: mul_mat_vec_q2_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1484
+ case 7: mul_mat_vec_q2_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1485
+ case 8: mul_mat_vec_q2_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1486
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q2_K multi-col MMVQ", ncols_dst);
1487
+ }
1488
+ }
1489
+
1490
+ static void reorder_mul_mat_vec_q2_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
1491
+ const int nrows, dpct::queue_ptr stream) {
1492
+ GGML_ASSERT(ncols % QK_K == 0);
1493
+
1494
+ // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
1495
+ constexpr size_t num_subgroups = WARP_SIZE;
1496
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
1497
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1498
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1499
+
1500
+ stream->submit([&](sycl::handler & cgh) {
1501
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1502
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1503
+ mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q2_K>>(vx, vy, dst, ncols, nrows,
1504
+ nd_item);
1505
+ });
1506
+ });
1507
+ }
1508
+
1509
+ template <int ncols_dst>
1510
+ static void reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols(
1511
+ const void * vx, const void * vy, float * dst,
1512
+ const int ncols, const int nrows,
1513
+ const int stride_col_y_bytes, const int stride_col_dst,
1514
+ dpct::queue_ptr stream) {
1515
+ GGML_ASSERT(ncols % QK_K == 0);
1516
+ constexpr size_t num_subgroups = WARP_SIZE;
1517
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
1518
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1519
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1520
+
1521
+ stream->submit([&](sycl::handler & cgh) {
1522
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1523
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1524
+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q2_K>, ncols_dst>(
1525
+ vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
1526
+ /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
1527
+ });
1528
+ });
1529
+ }
1530
+
1531
+ static void reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols(
1532
+ const void * vx, const void * vy, float * dst,
1533
+ const int ncols, const int nrows, const int ncols_dst,
1534
+ const int stride_col_y_bytes, const int stride_col_dst,
1535
+ dpct::queue_ptr stream) {
1536
+ switch (ncols_dst) {
1537
+ case 1: reorder_mul_mat_vec_q2_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1538
+ case 2: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1539
+ case 3: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1540
+ case 4: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1541
+ case 5: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1542
+ case 6: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1543
+ case 7: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1544
+ case 8: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1545
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q2_K reorder multi-col MMVQ", ncols_dst);
1546
+ }
1547
+ }
1548
+
1549
+ static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy,
1550
+ float *dst, const int ncols,
1551
+ const int nrows,
1552
+ dpct::queue_ptr stream) {
1553
+ GGML_ASSERT(ncols % QK_K == 0);
1554
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1555
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1556
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1557
+ {
1558
+
1559
+ stream->submit([&](sycl::handler &cgh) {
1560
+
1561
+ cgh.parallel_for(
1562
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1563
+ [=](sycl::nd_item<3> item_ct1)
1564
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1565
+ mul_mat_vec_q<QK_K, QI3_K, block_q3_K,
1566
+ VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>(
1567
+ vx, vy, dst, ncols, nrows, item_ct1);
1568
+ });
1569
+ });
1570
+ }
1571
+ }
1572
+
1573
+ static void reorder_mul_mat_vec_q3_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
1574
+ const int nrows, dpct::queue_ptr stream) {
1575
+ GGML_ASSERT(ncols % QK_K == 0);
1576
+
1577
+ // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
1578
+ constexpr size_t num_subgroups = WARP_SIZE;
1579
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
1580
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1581
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1582
+
1583
+ stream->submit([&](sycl::handler & cgh) {
1584
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1585
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1586
+ mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q3_K>>(vx, vy, dst, ncols, nrows,
1587
+ nd_item);
1588
+ });
1589
+ });
1590
+ }
1591
+
1592
+ template <int ncols_dst>
1593
+ static void reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols(
1594
+ const void * vx, const void * vy, float * dst,
1595
+ const int ncols, const int nrows,
1596
+ const int stride_col_y_bytes, const int stride_col_dst,
1597
+ dpct::queue_ptr stream) {
1598
+ GGML_ASSERT(ncols % QK_K == 0);
1599
+ constexpr size_t num_subgroups = WARP_SIZE;
1600
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
1601
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1602
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1603
+
1604
+ stream->submit([&](sycl::handler & cgh) {
1605
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1606
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1607
+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q3_K>, ncols_dst>(
1608
+ vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
1609
+ /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
1610
+ });
1611
+ });
1612
+ }
1613
+
1614
+ static void reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols(
1615
+ const void * vx, const void * vy, float * dst,
1616
+ const int ncols, const int nrows, const int ncols_dst,
1617
+ const int stride_col_y_bytes, const int stride_col_dst,
1618
+ dpct::queue_ptr stream) {
1619
+ switch (ncols_dst) {
1620
+ case 1: reorder_mul_mat_vec_q3_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1621
+ case 2: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1622
+ case 3: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1623
+ case 4: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1624
+ case 5: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1625
+ case 6: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1626
+ case 7: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1627
+ case 8: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1628
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q3_K reorder multi-col MMVQ", ncols_dst);
1629
+ }
1630
+ }
1631
+
1632
+ template <int ncols_dst>
1633
+ static void mul_mat_vec_q3_K_q8_1_sycl_ncols(
1634
+ const void * vx, const void * vy, float * dst,
1635
+ const int ncols, const int nrows,
1636
+ const int stride_col_y, const int stride_col_dst,
1637
+ dpct::queue_ptr stream) {
1638
+ GGML_ASSERT(ncols % QK_K == 0);
1639
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1640
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1641
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1642
+ stream->submit([&](sycl::handler & cgh) {
1643
+ cgh.parallel_for(
1644
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1645
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1646
+ mul_mat_vec_q_ncols<QK_K, QI3_K, block_q3_K,
1647
+ VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1, ncols_dst>(
1648
+ vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
1649
+ });
1650
+ });
1651
+ }
1652
+
1653
+ static void mul_mat_vec_q3_K_q8_1_sycl_switch_ncols(
1654
+ const void * vx, const void * vy, float * dst,
1655
+ const int ncols, const int nrows, const int ncols_dst,
1656
+ const int stride_col_y, const int stride_col_dst,
1657
+ dpct::queue_ptr stream) {
1658
+ switch (ncols_dst) {
1659
+ case 1: mul_mat_vec_q3_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1660
+ case 2: mul_mat_vec_q3_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1661
+ case 3: mul_mat_vec_q3_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1662
+ case 4: mul_mat_vec_q3_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1663
+ case 5: mul_mat_vec_q3_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1664
+ case 6: mul_mat_vec_q3_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1665
+ case 7: mul_mat_vec_q3_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1666
+ case 8: mul_mat_vec_q3_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1667
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q3_K multi-col MMVQ", ncols_dst);
1668
+ }
1669
+ }
1670
+
1671
+
1672
+ static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy,
1673
+ float *dst, const int ncols,
1674
+ const int nrows,
1675
+ dpct::queue_ptr stream) {
1676
+ GGML_ASSERT(ncols % QK_K == 0);
1677
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1678
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1679
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1680
+ {
1681
+
1682
+ stream->submit([&](sycl::handler &cgh) {
1683
+
1684
+ cgh.parallel_for(
1685
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1686
+ [=](sycl::nd_item<3> item_ct1)
1687
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1688
+ mul_mat_vec_q<QK_K, QI4_K, block_q4_K,
1689
+ VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>(
1690
+ vx, vy, dst, ncols, nrows, item_ct1);
1691
+ });
1692
+ });
1693
+ }
1694
+ }
1695
+
1696
+ template <int ncols_dst>
1697
+ static void mul_mat_vec_q4_K_q8_1_sycl_ncols(
1698
+ const void * vx, const void * vy, float * dst,
1699
+ const int ncols, const int nrows,
1700
+ const int stride_col_y, const int stride_col_dst,
1701
+ dpct::queue_ptr stream) {
1702
+ GGML_ASSERT(ncols % QK_K == 0);
1703
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1704
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1705
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1706
+
1707
+ stream->submit([&](sycl::handler & cgh) {
1708
+ cgh.parallel_for(
1709
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1710
+ [=](sycl::nd_item<3> item_ct1)
1711
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1712
+ mul_mat_vec_q_ncols<QK_K, QI4_K, block_q4_K,
1713
+ VDR_Q4_K_Q8_1_MMVQ,
1714
+ vec_dot_q4_K_q8_1,
1715
+ ncols_dst>(
1716
+ vx, vy, dst, ncols, nrows,
1717
+ stride_col_y, stride_col_dst, item_ct1);
1718
+ });
1719
+ });
1720
+ }
1721
+
1722
+ static void mul_mat_vec_q4_K_q8_1_sycl_switch_ncols(
1723
+ const void * vx, const void * vy, float * dst,
1724
+ const int ncols, const int nrows,
1725
+ const int ncols_dst,
1726
+ const int stride_col_y, const int stride_col_dst,
1727
+ dpct::queue_ptr stream) {
1728
+ switch (ncols_dst) {
1729
+ case 1: mul_mat_vec_q4_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1730
+ case 2: mul_mat_vec_q4_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1731
+ case 3: mul_mat_vec_q4_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1732
+ case 4: mul_mat_vec_q4_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1733
+ case 5: mul_mat_vec_q4_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1734
+ case 6: mul_mat_vec_q4_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1735
+ case 7: mul_mat_vec_q4_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1736
+ case 8: mul_mat_vec_q4_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1737
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q4_K multi-col MMVQ", ncols_dst);
1738
+ }
1739
+ }
1740
+
1741
+ static void reorder_mul_mat_vec_q4_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
1742
+ const int nrows, dpct::queue_ptr stream) {
1743
+ GGML_ASSERT(ncols % QK_K == 0);
1744
+
1745
+ // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
1746
+ constexpr size_t num_subgroups = WARP_SIZE;
1747
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
1748
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1749
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1750
+
1751
+ stream->submit([&](sycl::handler & cgh) {
1752
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1753
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1754
+ mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>>(vx, vy, dst, ncols,
1755
+ nrows, nd_item);
1756
+ });
1757
+ });
1758
+ }
1759
+
1760
+ template <int ncols_dst, int rows_per_sg>
1761
+ static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl(
1762
+ const void * vx, const void * vy, float * dst,
1763
+ const int ncols, const int nrows,
1764
+ const int stride_col_y_bytes, const int stride_col_dst,
1765
+ dpct::queue_ptr stream) {
1766
+ GGML_ASSERT(ncols % QK_K == 0);
1767
+
1768
+ constexpr size_t num_subgroups = WARP_SIZE;
1769
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg);
1770
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1771
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1772
+
1773
+ stream->submit([&](sycl::handler & cgh) {
1774
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1775
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1776
+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>, ncols_dst,
1777
+ /*has_fusion=*/ false, rows_per_sg>(
1778
+ vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
1779
+ /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
1780
+ });
1781
+ });
1782
+ }
1783
+
1784
+ template <int ncols_dst>
1785
+ static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols(
1786
+ const void * vx, const void * vy, float * dst,
1787
+ const int ncols, const int nrows,
1788
+ const int stride_col_y_bytes, const int stride_col_dst,
1789
+ dpct::queue_ptr stream) {
1790
+ constexpr int rows_per_sg = ncols_dst >= 3 && ncols_dst <= 4 ? 2 : 1;
1791
+ reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<ncols_dst, rows_per_sg>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream);
1792
+ }
1793
+
1794
+ static void reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols(
1795
+ const void * vx, const void * vy, float * dst,
1796
+ const int ncols, const int nrows, const int ncols_dst,
1797
+ const int stride_col_y_bytes, const int stride_col_dst,
1798
+ dpct::queue_ptr stream) {
1799
+ switch (ncols_dst) {
1800
+ case 1: reorder_mul_mat_vec_q4_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1801
+ case 2:
1802
+ if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) {
1803
+ reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream);
1804
+ } else {
1805
+ reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 1>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream);
1806
+ }
1807
+ break;
1808
+ case 3: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1809
+ case 4: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1810
+ case 5: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1811
+ case 6: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1812
+ case 7: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1813
+ case 8: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1814
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q4_K reorder multi-col MMVQ", ncols_dst);
1815
+ }
1816
+ }
1817
+
1818
+ static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy,
1819
+ float *dst, const int ncols,
1820
+ const int nrows,
1821
+ dpct::queue_ptr stream) {
1822
+ GGML_ASSERT(ncols % QK_K == 0);
1823
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1824
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1825
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1826
+ {
1827
+
1828
+ stream->submit([&](sycl::handler &cgh) {
1829
+
1830
+ cgh.parallel_for(
1831
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1832
+ [=](sycl::nd_item<3> item_ct1)
1833
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1834
+ mul_mat_vec_q<QK_K, QI5_K, block_q5_K,
1835
+ VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>(
1836
+ vx, vy, dst, ncols, nrows, item_ct1);
1837
+ });
1838
+ });
1839
+ }
1840
+ }
1841
+
1842
+ template <int ncols_dst>
1843
+ static void mul_mat_vec_q5_K_q8_1_sycl_ncols(
1844
+ const void * vx, const void * vy, float * dst,
1845
+ const int ncols, const int nrows,
1846
+ const int stride_col_y, const int stride_col_dst,
1847
+ dpct::queue_ptr stream) {
1848
+ GGML_ASSERT(ncols % QK_K == 0);
1849
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1850
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1851
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1852
+
1853
+ stream->submit([&](sycl::handler & cgh) {
1854
+ cgh.parallel_for(
1855
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1856
+ [=](sycl::nd_item<3> item_ct1)
1857
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1858
+ mul_mat_vec_q_ncols<QK_K, QI5_K, block_q5_K,
1859
+ VDR_Q5_K_Q8_1_MMVQ,
1860
+ vec_dot_q5_K_q8_1,
1861
+ ncols_dst>(
1862
+ vx, vy, dst, ncols, nrows,
1863
+ stride_col_y, stride_col_dst, item_ct1);
1864
+ });
1865
+ });
1866
+ }
1867
+
1868
+ static void mul_mat_vec_q5_K_q8_1_sycl_switch_ncols(
1869
+ const void * vx, const void * vy, float * dst,
1870
+ const int ncols, const int nrows,
1871
+ const int ncols_dst,
1872
+ const int stride_col_y, const int stride_col_dst,
1873
+ dpct::queue_ptr stream) {
1874
+ switch (ncols_dst) {
1875
+ case 1: mul_mat_vec_q5_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1876
+ case 2: mul_mat_vec_q5_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1877
+ case 3: mul_mat_vec_q5_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1878
+ case 4: mul_mat_vec_q5_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1879
+ case 5: mul_mat_vec_q5_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1880
+ case 6: mul_mat_vec_q5_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1881
+ case 7: mul_mat_vec_q5_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1882
+ case 8: mul_mat_vec_q5_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
1883
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q5_K multi-col MMVQ", ncols_dst);
1884
+ }
1885
+ }
1886
+
1887
+ static void reorder_mul_mat_vec_q5_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
1888
+ const int nrows, dpct::queue_ptr stream) {
1889
+ GGML_ASSERT(ncols % QK_K == 0);
1890
+
1891
+ constexpr size_t num_subgroups = WARP_SIZE;
1892
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
1893
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1894
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1895
+
1896
+ stream->submit([&](sycl::handler & cgh) {
1897
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1898
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1899
+ mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K>>(vx, vy, dst, ncols,
1900
+ nrows, nd_item);
1901
+ });
1902
+ });
1903
+ }
1904
+
1905
+ template <int ncols_dst>
1906
+ static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols(
1907
+ const void * vx, const void * vy, float * dst,
1908
+ const int ncols, const int nrows,
1909
+ const int stride_col_y_bytes, const int stride_col_dst,
1910
+ dpct::queue_ptr stream) {
1911
+ GGML_ASSERT(ncols % QK_K == 0);
1912
+
1913
+ constexpr size_t num_subgroups = WARP_SIZE;
1914
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
1915
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1916
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1917
+
1918
+ stream->submit([&](sycl::handler & cgh) {
1919
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1920
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1921
+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K>, ncols_dst>(
1922
+ vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
1923
+ /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
1924
+ });
1925
+ });
1926
+ }
1927
+
1928
+ static void reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols(
1929
+ const void * vx, const void * vy, float * dst,
1930
+ const int ncols, const int nrows, const int ncols_dst,
1931
+ const int stride_col_y_bytes, const int stride_col_dst,
1932
+ dpct::queue_ptr stream) {
1933
+ switch (ncols_dst) {
1934
+ case 1: reorder_mul_mat_vec_q5_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1935
+ case 2: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1936
+ case 3: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1937
+ case 4: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1938
+ case 5: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1939
+ case 6: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1940
+ case 7: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1941
+ case 8: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1942
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q5_K reorder multi-col MMVQ", ncols_dst);
1943
+ }
1944
+ }
1945
+
1946
+ static void reorder_mul_mat_vec_q6_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
1947
+ const int nrows, dpct::queue_ptr stream) {
1948
+ GGML_ASSERT(ncols % QK_K == 0);
1949
+ // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
1950
+ constexpr size_t num_subgroups = WARP_SIZE;
1951
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
1952
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1953
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1954
+
1955
+
1956
+ stream->submit([&](sycl::handler & cgh) {
1957
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1958
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1959
+ mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q6_K>>(vx, vy, dst, ncols, nrows,
1960
+ nd_item);
1961
+ });
1962
+ });
1963
+ }
1964
+
1965
+ template <int ncols_dst>
1966
+ static void reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols(
1967
+ const void * vx, const void * vy, float * dst,
1968
+ const int ncols, const int nrows,
1969
+ const int stride_col_y_bytes, const int stride_col_dst,
1970
+ dpct::queue_ptr stream) {
1971
+ GGML_ASSERT(ncols % QK_K == 0);
1972
+ constexpr size_t num_subgroups = WARP_SIZE;
1973
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
1974
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1975
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
1976
+
1977
+ stream->submit([&](sycl::handler & cgh) {
1978
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
1979
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1980
+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q6_K>, ncols_dst>(
1981
+ vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
1982
+ /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
1983
+ });
1984
+ });
1985
+ }
1986
+
1987
+ static void reorder_mul_mat_vec_q6_k_q8_1_sycl_switch_ncols(
1988
+ const void * vx, const void * vy, float * dst,
1989
+ const int ncols, const int nrows, const int ncols_dst,
1990
+ const int stride_col_y_bytes, const int stride_col_dst,
1991
+ dpct::queue_ptr stream) {
1992
+ switch (ncols_dst) {
1993
+ case 1: reorder_mul_mat_vec_q6_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
1994
+ case 2: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1995
+ case 3: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1996
+ case 4: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1997
+ case 5: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1998
+ case 6: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
1999
+ case 7: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
2000
+ case 8: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
2001
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q6_K reorder multi-col MMVQ", ncols_dst);
2002
+ }
2003
+ }
2004
+
2005
+ static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy,
2006
+ float *dst, const int ncols,
2007
+ const int nrows,
2008
+ dpct::queue_ptr stream) {
2009
+ GGML_ASSERT(ncols % QK_K == 0);
2010
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2011
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2012
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2013
+ {
2014
+
2015
+ stream->submit([&](sycl::handler &cgh) {
2016
+
2017
+ cgh.parallel_for(
2018
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2019
+ [=](sycl::nd_item<3> item_ct1)
2020
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2021
+ mul_mat_vec_q<QK_K, QI6_K, block_q6_K,
2022
+ VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>(
2023
+ vx, vy, dst, ncols, nrows, item_ct1);
2024
+ });
2025
+ });
2026
+ }
2027
+ }
2028
+
2029
+ template <int ncols_dst>
2030
+ static void mul_mat_vec_q6_K_q8_1_sycl_ncols(
2031
+ const void * vx, const void * vy, float * dst,
2032
+ const int ncols, const int nrows,
2033
+ const int stride_col_y, const int stride_col_dst,
2034
+ dpct::queue_ptr stream) {
2035
+ GGML_ASSERT(ncols % QK_K == 0);
2036
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2037
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2038
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2039
+
2040
+ stream->submit([&](sycl::handler & cgh) {
2041
+ cgh.parallel_for(
2042
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2043
+ [=](sycl::nd_item<3> item_ct1)
2044
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2045
+ mul_mat_vec_q_ncols<QK_K, QI6_K, block_q6_K,
2046
+ VDR_Q6_K_Q8_1_MMVQ,
2047
+ vec_dot_q6_K_q8_1,
2048
+ ncols_dst>(
2049
+ vx, vy, dst, ncols, nrows,
2050
+ stride_col_y, stride_col_dst, item_ct1);
2051
+ });
2052
+ });
2053
+ }
2054
+
2055
+ static void mul_mat_vec_q6_K_q8_1_sycl_switch_ncols(
2056
+ const void * vx, const void * vy, float * dst,
2057
+ const int ncols, const int nrows,
2058
+ const int ncols_dst,
2059
+ const int stride_col_y, const int stride_col_dst,
2060
+ dpct::queue_ptr stream) {
2061
+ switch (ncols_dst) {
2062
+ case 1: mul_mat_vec_q6_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
2063
+ case 2: mul_mat_vec_q6_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2064
+ case 3: mul_mat_vec_q6_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2065
+ case 4: mul_mat_vec_q6_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2066
+ case 5: mul_mat_vec_q6_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2067
+ case 6: mul_mat_vec_q6_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2068
+ case 7: mul_mat_vec_q6_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2069
+ case 8: mul_mat_vec_q6_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2070
+ default: GGML_ABORT("unsupported ncols_dst=%d for Q6_K multi-col MMVQ", ncols_dst);
2071
+ }
2072
+ }
2073
+
2074
+
2075
+ static void mul_mat_vec_iq2_xxs_q8_1_sycl(const void *vx, const void *vy,
2076
+ float *dst, const int ncols,
2077
+ const int nrows,
2078
+ dpct::queue_ptr stream) {
2079
+ GGML_ASSERT(ncols % QK_K == 0);
2080
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2081
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2082
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2083
+ {
2084
+ stream->submit([&](sycl::handler &cgh) {
2085
+ cgh.parallel_for(
2086
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2087
+ [=](sycl::nd_item<3> item_ct1)
2088
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2089
+ mul_mat_vec_q_iq2_xxs_q8_1<QK_K, QI2_XXS/2, block_iq2_xxs, 1>(
2090
+ vx, vy, dst, ncols, nrows, item_ct1);
2091
+ });
2092
+ });
2093
+ }
2094
+ }
2095
+
2096
+ static void mul_mat_vec_iq2_xs_q8_1_sycl(const void *vx, const void *vy,
2097
+ float *dst, const int ncols,
2098
+ const int nrows,
2099
+ dpct::queue_ptr stream) {
2100
+ GGML_ASSERT(ncols % QK_K == 0);
2101
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2102
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2103
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2104
+ {
2105
+ stream->submit([&](sycl::handler & cgh) {
2106
+ cgh.parallel_for(
2107
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2108
+ [=](sycl::nd_item<3> item_ct1)
2109
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2110
+ mul_mat_vec_q_iq2_xs_q8_1<QK_K, QI2_XS/2, block_iq2_xs, 1>(
2111
+ vx, vy, dst, ncols, nrows, item_ct1);
2112
+ });
2113
+ });
2114
+ }
2115
+ }
2116
+
2117
+ static void mul_mat_vec_iq2_s_q8_1_sycl(const void *vx, const void *vy,
2118
+ float *dst, const int ncols,
2119
+ const int nrows,
2120
+ dpct::queue_ptr stream) {
2121
+ GGML_ASSERT(ncols % QK_K == 0);
2122
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2123
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2124
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2125
+ {
2126
+
2127
+ stream->submit([&](sycl::handler &cgh) {
2128
+ cgh.parallel_for(
2129
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2130
+ [=](sycl::nd_item<3> item_ct1)
2131
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2132
+ mul_mat_vec_q_iq2_s_q8_1<QK_K, QI2_S/2, block_iq2_s, 1>(
2133
+ vx, vy, dst, ncols, nrows, item_ct1);
2134
+ });
2135
+ });
2136
+ }
2137
+ }
2138
+
2139
+ static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy,
2140
+ float *dst, const int ncols,
2141
+ const int nrows,
2142
+ dpct::queue_ptr stream) {
2143
+ GGML_ASSERT(ncols % QK_K == 0);
2144
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2145
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2146
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2147
+ {
2148
+
2149
+ stream->submit([&](sycl::handler &cgh) {
2150
+ cgh.parallel_for(
2151
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2152
+ [=](sycl::nd_item<3> item_ct1)
2153
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2154
+ mul_mat_vec_q_iq3_xxs_q8_1<QK_K, QI3_XXS/2, block_iq3_xxs, 1>(
2155
+ vx, vy, dst, ncols, nrows, item_ct1);
2156
+ });
2157
+ });
2158
+ }
2159
+ }
2160
+
2161
+ static void mul_mat_vec_iq3_s_q8_1_sycl(const void *vx, const void *vy,
2162
+ float *dst, const int ncols,
2163
+ const int nrows,
2164
+ dpct::queue_ptr stream) {
2165
+ GGML_ASSERT(ncols % QK_K == 0);
2166
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2167
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2168
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2169
+ {
2170
+
2171
+ stream->submit([&](sycl::handler &cgh) {
2172
+ cgh.parallel_for(
2173
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2174
+ [=](sycl::nd_item<3> item_ct1)
2175
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2176
+ mul_mat_vec_q_iq3_s_q8_1<QK_K, QI3_S/2, block_iq3_s, 1>(
2177
+ vx, vy, dst, ncols, nrows, item_ct1);
2178
+ });
2179
+ });
2180
+ }
2181
+ }
2182
+
2183
+ static void mul_mat_vec_iq1_s_q8_1_sycl(const void *vx, const void *vy,
2184
+ float *dst, const int ncols,
2185
+ const int nrows,
2186
+ dpct::queue_ptr stream) {
2187
+ GGML_ASSERT(ncols % QK_K == 0);
2188
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2189
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2190
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2191
+ {
2192
+
2193
+ stream->submit([&](sycl::handler &cgh) {
2194
+ cgh.parallel_for(
2195
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2196
+ [=](sycl::nd_item<3> item_ct1)
2197
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2198
+ mul_mat_vec_q_iq1_s_q8_1<QK_K, QI1_S, block_iq1_s, 1>(
2199
+ vx, vy, dst, ncols, nrows, item_ct1);
2200
+ });
2201
+ });
2202
+ }
2203
+ }
2204
+
2205
+ static void mul_mat_vec_iq1_m_q8_1_sycl(const void *vx, const void *vy,
2206
+ float *dst, const int ncols,
2207
+ const int nrows,
2208
+ dpct::queue_ptr stream) {
2209
+ GGML_ASSERT(ncols % QK_K == 0);
2210
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2211
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2212
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2213
+ {
2214
+ stream->submit([&](sycl::handler &cgh) {
2215
+ cgh.parallel_for(
2216
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2217
+ [=](sycl::nd_item<3> item_ct1)
2218
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2219
+ mul_mat_vec_q_iq1_m_q8_1<QK_K, QI1_S, block_iq1_m, 1>(
2220
+ vx, vy, dst, ncols, nrows, item_ct1);
2221
+ });
2222
+ });
2223
+ }
2224
+ }
2225
+
2226
+ static void mul_mat_vec_iq4_nl_q8_1_sycl(const void *vx, const void *vy,
2227
+ float *dst, const int ncols,
2228
+ const int nrows,
2229
+ dpct::queue_ptr stream) {
2230
+ GGML_ASSERT(ncols % QK4_NL == 0);
2231
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2232
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2233
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2234
+ {
2235
+
2236
+ stream->submit([&](sycl::handler &cgh) {
2237
+ cgh.parallel_for(
2238
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2239
+ [=](sycl::nd_item<3> item_ct1)
2240
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2241
+ mul_mat_vec_q_iq4_nl_q8_1<QK4_NL, QI4_NL, block_iq4_nl, 2>(
2242
+ vx, vy, dst, ncols, nrows, item_ct1);
2243
+ });
2244
+ });
2245
+ }
2246
+ }
2247
+
2248
+ static void mul_mat_vec_iq4_xs_q8_1_sycl(const void *vx, const void *vy,
2249
+ float *dst, const int ncols,
2250
+ const int nrows,
2251
+ dpct::queue_ptr stream) {
2252
+ GGML_ASSERT(ncols % QK_K == 0);
2253
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2254
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2255
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2256
+ {
2257
+
2258
+ stream->submit([&](sycl::handler &cgh) {
2259
+ cgh.parallel_for(
2260
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2261
+ [=](sycl::nd_item<3> item_ct1)
2262
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2263
+ mul_mat_vec_q_iq4_xs_q8_1<QK_K, QI4_XS/4, block_iq4_xs, 1>(
2264
+ vx, vy, dst, ncols, nrows, item_ct1);
2265
+ });
2266
+ });
2267
+ }
2268
+ }
2269
+
2270
+ template <int ncols_dst>
2271
+ static void mul_mat_vec_iq4_xs_q8_1_sycl_ncols(
2272
+ const void * vx, const void * vy, float * dst,
2273
+ const int ncols, const int nrows,
2274
+ const int stride_col_y, const int stride_col_dst,
2275
+ dpct::queue_ptr stream) {
2276
+ GGML_ASSERT(ncols % QK_K == 0);
2277
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2278
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2279
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2280
+
2281
+ stream->submit([&](sycl::handler & cgh) {
2282
+ cgh.parallel_for(
2283
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2284
+ [=](sycl::nd_item<3> item_ct1)
2285
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2286
+ mul_mat_vec_q_ncols<QK_K, QI4_XS/4, block_iq4_xs,
2287
+ 1,
2288
+ vec_dot_iq4_xs_q8_1,
2289
+ ncols_dst>(
2290
+ vx, vy, dst, ncols, nrows,
2291
+ stride_col_y, stride_col_dst, item_ct1);
2292
+ });
2293
+ });
2294
+ }
2295
+
2296
+ static void mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols(
2297
+ const void * vx, const void * vy, float * dst,
2298
+ const int ncols, const int nrows,
2299
+ const int ncols_dst,
2300
+ const int stride_col_y, const int stride_col_dst,
2301
+ dpct::queue_ptr stream) {
2302
+ switch (ncols_dst) {
2303
+ case 1: mul_mat_vec_iq4_xs_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
2304
+ case 2: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2305
+ case 3: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2306
+ case 4: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2307
+ case 5: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2308
+ case 6: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2309
+ case 7: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2310
+ case 8: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
2311
+ default: GGML_ABORT("unsupported ncols_dst=%d for IQ4_XS multi-col MMVQ", ncols_dst);
2312
+ }
2313
+ }
2314
+
2315
+ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1,
2316
+ ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
2317
+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low,
2318
+ const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_col_size,
2319
+ const dpct::queue_ptr & stream) {
2320
+ const int64_t ne10 = src1->ne[0];
2321
+ GGML_ASSERT(ne10 % QK8_1 == 0);
2322
+
2323
+ const int64_t ne00 = src0->ne[0];
2324
+ const int64_t row_diff = row_high - row_low;
2325
+
2326
+ int id;
2327
+ SYCL_CHECK(CHECK_TRY_ERROR(id = get_current_device_id()));
2328
+ const size_t q8_1_ts = sizeof(block_q8_1);
2329
+ const size_t q8_1_bs = QK8_1;
2330
+ // the main device has a larger memory buffer to hold the results from all GPUs
2331
+ // nrows_dst == nrows of the matrix that the kernel writes into
2332
+
2333
+ for (int i = 0; i < src1_ncols; i++) {
2334
+ const size_t src1_ddq_i_offset = i * src1_padded_col_size * q8_1_ts / q8_1_bs;
2335
+ const char * src1_ddq_i_bs = src1_ddq_i + src1_ddq_i_offset;
2336
+ float * dst_dd_i_bs = dst_dd_i + i * dst->ne[0];
2337
+ switch (src0->type) {
2338
+ case GGML_TYPE_Q4_0:
2339
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2340
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2341
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2342
+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
2343
+ const int stride_col_dst = dst->ne[0];
2344
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2345
+ reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(
2346
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2347
+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
2348
+ return;
2349
+ } else {
2350
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_0_q8_1_sycl\n");
2351
+ reorder_mul_mat_vec_q4_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2352
+ }
2353
+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2354
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2355
+ const int stride_col_dst = dst->ne[0];
2356
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2357
+ mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(
2358
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2359
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2360
+ return;
2361
+ } else if (i == 0 || src1_ncols == 1) {
2362
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_0_q8_1_sycl\n");
2363
+ mul_mat_vec_q4_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2364
+ }
2365
+ break;
2366
+ case GGML_TYPE_Q4_1:
2367
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2368
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2369
+ const int stride_col_dst = dst->ne[0];
2370
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_1_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2371
+ mul_mat_vec_q4_1_q8_1_sycl_switch_ncols(
2372
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2373
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2374
+ return;
2375
+ } else if (i == 0 || src1_ncols == 1) {
2376
+ mul_mat_vec_q4_1_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2377
+ }
2378
+ break;
2379
+ case GGML_TYPE_Q5_0:
2380
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2381
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2382
+ const int stride_col_dst = dst->ne[0];
2383
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2384
+ mul_mat_vec_q5_0_q8_1_sycl_switch_ncols(
2385
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2386
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2387
+ return;
2388
+ } else if (i == 0 || src1_ncols == 1) {
2389
+ mul_mat_vec_q5_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2390
+ }
2391
+ break;
2392
+ case GGML_TYPE_Q5_1:
2393
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2394
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2395
+ const int stride_col_dst = dst->ne[0];
2396
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_1_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2397
+ mul_mat_vec_q5_1_q8_1_sycl_switch_ncols(
2398
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2399
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2400
+ return;
2401
+ } else if (i == 0 || src1_ncols == 1) {
2402
+ mul_mat_vec_q5_1_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2403
+ }
2404
+ break;
2405
+ case GGML_TYPE_Q8_0:
2406
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2407
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2408
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2409
+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
2410
+ const int stride_col_dst = dst->ne[0];
2411
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2412
+ reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(
2413
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2414
+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
2415
+ return;
2416
+ } else {
2417
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q8_0_q8_1_sycl\n");
2418
+ reorder_mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2419
+ }
2420
+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2421
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2422
+ const int stride_col_dst = dst->ne[0];
2423
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q8_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2424
+ mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(
2425
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2426
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2427
+ return;
2428
+ } else if (i == 0 || src1_ncols == 1) {
2429
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q8_0_q8_1_sycl\n");
2430
+ mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2431
+ }
2432
+ break;
2433
+ case GGML_TYPE_Q1_0:
2434
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2435
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2436
+ const int stride_col_dst = dst->ne[0];
2437
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q1_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2438
+ mul_mat_vec_q1_0_q8_1_sycl_switch_ncols(
2439
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2440
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2441
+ return;
2442
+ } else if (i == 0 || src1_ncols == 1) {
2443
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q1_0_q8_1_sycl\n");
2444
+ mul_mat_vec_q1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2445
+ }
2446
+ break;
2447
+ case GGML_TYPE_Q2_0:
2448
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2449
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2450
+ const int stride_col_dst = dst->ne[0];
2451
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2452
+ mul_mat_vec_q2_0_q8_1_sycl_switch_ncols(
2453
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2454
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2455
+ return;
2456
+ } else if (i == 0 || src1_ncols == 1) {
2457
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_0_q8_1_sycl\n");
2458
+ mul_mat_vec_q2_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2459
+ }
2460
+ break;
2461
+ case GGML_TYPE_Q2_K:
2462
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2463
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2464
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2465
+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
2466
+ const int stride_col_dst = dst->ne[0];
2467
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2468
+ reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols(
2469
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2470
+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
2471
+ return;
2472
+ } else {
2473
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q2_k_q8_1_sycl\n");
2474
+ reorder_mul_mat_vec_q2_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2475
+ }
2476
+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2477
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2478
+ const int stride_col_dst = dst->ne[0];
2479
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2480
+ mul_mat_vec_q2_K_q8_1_sycl_switch_ncols(
2481
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2482
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2483
+ return;
2484
+ } else if (i == 0 || src1_ncols == 1) {
2485
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_K_q8_1_sycl\n");
2486
+ mul_mat_vec_q2_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2487
+ }
2488
+ break;
2489
+ case GGML_TYPE_Q3_K:
2490
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2491
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2492
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2493
+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
2494
+ const int stride_col_dst = dst->ne[0];
2495
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2496
+ reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols(
2497
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2498
+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
2499
+ return;
2500
+ } else {
2501
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q3_k_q8_1_sycl\n");
2502
+ reorder_mul_mat_vec_q3_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2503
+ }
2504
+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2505
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2506
+ const int stride_col_dst = dst->ne[0];
2507
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q3_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2508
+ mul_mat_vec_q3_K_q8_1_sycl_switch_ncols(
2509
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2510
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2511
+ return;
2512
+ } else if (i == 0 || src1_ncols == 1) {
2513
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q3_K_q8_1_sycl\n");
2514
+ mul_mat_vec_q3_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2515
+ }
2516
+ break;
2517
+ case GGML_TYPE_Q4_K:
2518
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2519
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2520
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2521
+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
2522
+ const int stride_col_dst = dst->ne[0];
2523
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2524
+ reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols(
2525
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2526
+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
2527
+ return;
2528
+ } else {
2529
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_k_q8_1_sycl\n");
2530
+ reorder_mul_mat_vec_q4_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2531
+ }
2532
+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2533
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2534
+ const int stride_col_dst = dst->ne[0];
2535
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2536
+ mul_mat_vec_q4_K_q8_1_sycl_switch_ncols(
2537
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2538
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2539
+ return;
2540
+ } else if (i == 0 || src1_ncols == 1) {
2541
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_K_q8_1_sycl\n");
2542
+ mul_mat_vec_q4_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2543
+ }
2544
+ break;
2545
+ case GGML_TYPE_Q5_K:
2546
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2547
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2548
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2549
+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
2550
+ const int stride_col_dst = dst->ne[0];
2551
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2552
+ reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols(
2553
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2554
+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
2555
+ return;
2556
+ } else {
2557
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q5_k_q8_1_sycl\n");
2558
+ reorder_mul_mat_vec_q5_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2559
+ }
2560
+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2561
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2562
+ const int stride_col_dst = dst->ne[0];
2563
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2564
+ mul_mat_vec_q5_K_q8_1_sycl_switch_ncols(
2565
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2566
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2567
+ return;
2568
+ } else if (i == 0 || src1_ncols == 1) {
2569
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_K_q8_1_sycl\n");
2570
+ mul_mat_vec_q5_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2571
+ }
2572
+ break;
2573
+ case GGML_TYPE_Q6_K:
2574
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2575
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2576
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2577
+ const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
2578
+ const int stride_col_dst = dst->ne[0];
2579
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q6_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2580
+ reorder_mul_mat_vec_q6_k_q8_1_sycl_switch_ncols(
2581
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2582
+ src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
2583
+ return;
2584
+ } else {
2585
+ GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q6_k_q8_1_sycl\n");
2586
+ reorder_mul_mat_vec_q6_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2587
+ }
2588
+ } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2589
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2590
+ const int stride_col_dst = dst->ne[0];
2591
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q6_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2592
+ mul_mat_vec_q6_K_q8_1_sycl_switch_ncols(
2593
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2594
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2595
+ return;
2596
+ } else if (i == 0 || src1_ncols == 1) {
2597
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_q6_k_q8_1_sycl\n");
2598
+ mul_mat_vec_q6_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2599
+ }
2600
+ break;
2601
+ case GGML_TYPE_IQ1_S:
2602
+ mul_mat_vec_iq1_s_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2603
+ break;
2604
+ case GGML_TYPE_IQ1_M:
2605
+ mul_mat_vec_iq1_m_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2606
+ break;
2607
+ case GGML_TYPE_IQ2_XXS:
2608
+ mul_mat_vec_iq2_xxs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2609
+ break;
2610
+ case GGML_TYPE_IQ2_XS:
2611
+ mul_mat_vec_iq2_xs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2612
+ break;
2613
+ case GGML_TYPE_IQ2_S:
2614
+ mul_mat_vec_iq2_s_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2615
+ break;
2616
+ case GGML_TYPE_IQ3_XXS:
2617
+ mul_mat_vec_iq3_xxs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2618
+ break;
2619
+ case GGML_TYPE_IQ3_S:
2620
+ mul_mat_vec_iq3_s_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2621
+ break;
2622
+ case GGML_TYPE_IQ4_NL:
2623
+ mul_mat_vec_iq4_nl_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2624
+ break;
2625
+ case GGML_TYPE_IQ4_XS:
2626
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2627
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2628
+ const int stride_col_dst = dst->ne[0];
2629
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2630
+ mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols(
2631
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2632
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2633
+ return;
2634
+ } else if (i == 0 || src1_ncols == 1) {
2635
+ mul_mat_vec_iq4_xs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2636
+ }
2637
+ break;
2638
+ case GGML_TYPE_MXFP4:
2639
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2640
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2641
+ const int stride_col_dst = dst->ne[0];
2642
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2643
+ mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols(
2644
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2645
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2646
+ return;
2647
+ } else if (i == 0 || src1_ncols == 1) {
2648
+ mul_mat_vec_mxfp4_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2649
+ }
2650
+ break;
2651
+ case GGML_TYPE_NVFP4:
2652
+ if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
2653
+ const int stride_col_y = src1_padded_col_size / QK8_1;
2654
+ const int stride_col_dst = dst->ne[0];
2655
+ GGML_SYCL_DEBUG("Calling mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
2656
+ mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols(
2657
+ src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
2658
+ src1_ncols, stride_col_y, stride_col_dst, stream);
2659
+ return;
2660
+ } else if (i == 0 || src1_ncols == 1) {
2661
+ mul_mat_vec_nvfp4_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
2662
+ }
2663
+ break;
2664
+ default:
2665
+ GGML_ABORT("fatal error: unsupport src0 data type %s\n", ggml_type_name(src0->type));
2666
+ }
2667
+ }
2668
+ GGML_UNUSED(src1);
2669
+ GGML_UNUSED(dst);
2670
+ GGML_UNUSED(src1_ddf_i);
2671
+ GGML_UNUSED(ctx);
2672
+ }
2673
+
2674
+ // src1_row_stride: 0 for shared src1 (gate/up proj), else per-expert stride (down proj).
2675
+ template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl>
2676
+ static void mul_mat_vec_q_moe(
2677
+ const void * __restrict__ vx_base, const void * __restrict__ vy_base,
2678
+ float * __restrict__ dst_base, const int32_t * __restrict__ ids_dev,
2679
+ const int ncols, const int nrows,
2680
+ const size_t expert_weight_stride, const size_t dst_row_stride,
2681
+ const size_t src1_row_stride,
2682
+ const sycl::nd_item<3> & item_ct1) {
2683
+
2684
+ const int expert_idx = item_ct1.get_group(1);
2685
+ const int i02 = ids_dev[expert_idx];
2686
+
2687
+ const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride;
2688
+ const char * vy = (const char *) vy_base + (size_t) expert_idx * src1_row_stride;
2689
+ float * dst = (float *) ((char *) dst_base + (size_t) expert_idx * dst_row_stride);
2690
+
2691
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1);
2692
+
2693
+ if (row >= nrows) {
2694
+ return;
2695
+ }
2696
+
2697
+ const int blocks_per_row = ncols / qk;
2698
+ constexpr int blocks_per_warp = (vdr * WARP_SIZE + qi - 1) / qi;
2699
+
2700
+ float tmp = 0.0f;
2701
+
2702
+ const block_q_t * x = (const block_q_t *) vx;
2703
+ const block_q8_1 * y = (const block_q8_1 *) vy;
2704
+
2705
+ for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) {
2706
+ const int ibx = row * blocks_per_row + i;
2707
+ const int iby = i * (qk / QK8_1);
2708
+
2709
+ for (size_t elem = 0; elem < qi / vdr; elem += WARP_SIZE) {
2710
+ const int iqs = elem + vdr * (item_ct1.get_local_id(2) % (qi / vdr));
2711
+ tmp += vec_dot_q_sycl(&x[ibx], &y[iby], iqs);
2712
+ }
2713
+ }
2714
+
2715
+ #pragma unroll
2716
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
2717
+ tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
2718
+ }
2719
+
2720
+ if (item_ct1.get_local_id(2) == 0) {
2721
+ dst[row] = tmp;
2722
+ }
2723
+ }
2724
+
2725
+ template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl>
2726
+ static void launch_mul_mat_vec_q_moe(
2727
+ const void * vx_base, const void * vy, const int32_t * ids_dev,
2728
+ float * dst_base, const int ncols, const int nrows, const int n_experts_used,
2729
+ const size_t expert_weight_stride, const size_t dst_row_stride,
2730
+ const size_t src1_row_stride,
2731
+ dpct::queue_ptr stream) {
2732
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2733
+ const sycl::range<3> block_nums(1, (unsigned) n_experts_used, (unsigned) block_num_y);
2734
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2735
+ stream->submit([&](sycl::handler & cgh) {
2736
+ cgh.parallel_for(
2737
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2738
+ [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2739
+ mul_mat_vec_q_moe<qk, qi, block_q_t, vdr, vec_dot_q_sycl>(
2740
+ vx_base, vy, dst_base, ids_dev, ncols, nrows,
2741
+ expert_weight_stride, dst_row_stride, src1_row_stride, item);
2742
+ });
2743
+ });
2744
+ }
2745
+
2746
+ bool ggml_sycl_mul_mat_vec_q_id(
2747
+ enum ggml_type src0_type,
2748
+ const void * vx_base,
2749
+ const void * vy,
2750
+ const int32_t * ids_dev,
2751
+ float * dst_base,
2752
+ int ncols,
2753
+ int nrows,
2754
+ int n_experts_used,
2755
+ size_t expert_weight_stride,
2756
+ size_t dst_row_stride,
2757
+ size_t src1_row_stride,
2758
+ dpct::queue_ptr stream) {
2759
+ switch (src0_type) {
2760
+ case GGML_TYPE_Q4_0:
2761
+ launch_mul_mat_vec_q_moe<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>(
2762
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2763
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2764
+ return true;
2765
+ case GGML_TYPE_Q4_1:
2766
+ launch_mul_mat_vec_q_moe<QK4_1, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>(
2767
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2768
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2769
+ return true;
2770
+ case GGML_TYPE_Q5_0:
2771
+ launch_mul_mat_vec_q_moe<QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>(
2772
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2773
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2774
+ return true;
2775
+ case GGML_TYPE_Q5_1:
2776
+ launch_mul_mat_vec_q_moe<QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>(
2777
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2778
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2779
+ return true;
2780
+ case GGML_TYPE_Q8_0:
2781
+ launch_mul_mat_vec_q_moe<QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>(
2782
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2783
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2784
+ return true;
2785
+ case GGML_TYPE_Q2_0:
2786
+ launch_mul_mat_vec_q_moe<QK2_0, QI2_0, block_q2_0, VDR_Q2_0_Q8_1_MMVQ, vec_dot_q2_0_q8_1>(
2787
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2788
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2789
+ return true;
2790
+ case GGML_TYPE_Q2_K:
2791
+ launch_mul_mat_vec_q_moe<QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>(
2792
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2793
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2794
+ return true;
2795
+ case GGML_TYPE_Q3_K:
2796
+ launch_mul_mat_vec_q_moe<QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>(
2797
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2798
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2799
+ return true;
2800
+ case GGML_TYPE_Q4_K:
2801
+ launch_mul_mat_vec_q_moe<QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>(
2802
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2803
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2804
+ return true;
2805
+ case GGML_TYPE_Q5_K:
2806
+ launch_mul_mat_vec_q_moe<QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>(
2807
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2808
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2809
+ return true;
2810
+ case GGML_TYPE_Q6_K:
2811
+ launch_mul_mat_vec_q_moe<QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>(
2812
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2813
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2814
+ return true;
2815
+ case GGML_TYPE_MXFP4:
2816
+ launch_mul_mat_vec_q_moe<QK_MXFP4, QI_MXFP4, block_mxfp4, VDR_MXFP4_Q8_1_MMVQ, vec_dot_mxfp4_q8_1>(
2817
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2818
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2819
+ return true;
2820
+ case GGML_TYPE_NVFP4:
2821
+ launch_mul_mat_vec_q_moe<QK_NVFP4, QI_NVFP4, block_nvfp4, VDR_NVFP4_Q8_1_MMVQ, vec_dot_nvfp4_q8_1>(
2822
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2823
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2824
+ return true;
2825
+ default:
2826
+ return false;
2827
+ }
2828
+ }
2829
+
2830
+ // Reorder (SoA) MoE expert GEMV: MoE expert/row/lane indexing (from mul_mat_vec_q_moe) with the
2831
+ // dense-reorder per-block reads (from mul_mat_vec_q_reorder). Each expert slice in vx_base is a
2832
+ // self-contained SoA, so nblocks = nrows*(ncols/qk) per expert and the constant expert stride holds.
2833
+ template <typename reorder_vec_dot_q_sycl>
2834
+ static void mul_mat_vec_q_moe_reorder(
2835
+ const void * __restrict__ vx_base, const void * __restrict__ vy_base,
2836
+ float * __restrict__ dst_base, const int32_t * __restrict__ ids_dev,
2837
+ const int ncols, const int nrows,
2838
+ const size_t expert_weight_stride, const size_t dst_row_stride,
2839
+ const size_t src1_row_stride,
2840
+ const sycl::nd_item<3> & item_ct1) {
2841
+ using block_type = ggml_sycl_reordered::block_q_t<reorder_vec_dot_q_sycl::gtype>;
2842
+ using block_traits = typename block_type::traits;
2843
+
2844
+ const int expert_idx = item_ct1.get_group(1);
2845
+ const int i02 = ids_dev[expert_idx];
2846
+
2847
+ const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride;
2848
+ const char * vy = (const char *) vy_base + (size_t) expert_idx * src1_row_stride;
2849
+ float * dst = (float *) ((char *) dst_base + (size_t) expert_idx * dst_row_stride);
2850
+
2851
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1);
2852
+ if (row >= nrows) {
2853
+ return;
2854
+ }
2855
+
2856
+ const auto sg = item_ct1.get_sub_group();
2857
+
2858
+ const int blocks_per_row = ncols / block_traits::qk;
2859
+ constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);
2860
+ constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;
2861
+ const int nblocks = nrows * (ncols / block_traits::qk);
2862
+
2863
+ static_assert(blocks_per_subgroup > 0);
2864
+ static_assert(block_elements_per_subgroup > 0);
2865
+
2866
+ float partial_sum = 0.0f;
2867
+ for (int i = sg.get_local_linear_id() / block_elements_per_subgroup; i < blocks_per_row; i += blocks_per_subgroup) {
2868
+ const int ibx = row * blocks_per_row + i;
2869
+
2870
+ const auto bx_offset = block_type::get_block_offset(ibx, nblocks);
2871
+ const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);
2872
+
2873
+ const int iby = i * block_type::block_to_q8_1_ratio();
2874
+ const int8_t * q8_1_quant_ptr = (const int8_t *) vy + iby * QK8_1;
2875
+ const sycl::half2 * q8_1_ds_ptr = (const sycl::half2 *) ((const char *) vy + ncols + iby * sizeof(sycl::half2));
2876
+
2877
+ #pragma unroll
2878
+ for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {
2879
+ const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
2880
+ partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
2881
+ }
2882
+ }
2883
+
2884
+ auto sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());
2885
+ if (sg.leader()) {
2886
+ dst[row] = sum;
2887
+ }
2888
+ }
2889
+
2890
+ template <typename reorder_vec_dot_q_sycl>
2891
+ static void launch_mul_mat_vec_q_moe_reorder(
2892
+ const void * vx_base, const void * vy, const int32_t * ids_dev,
2893
+ float * dst_base, const int ncols, const int nrows, const int n_experts_used,
2894
+ const size_t expert_weight_stride, const size_t dst_row_stride,
2895
+ const size_t src1_row_stride,
2896
+ dpct::queue_ptr stream) {
2897
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
2898
+ const sycl::range<3> block_nums(1, (unsigned) n_experts_used, (unsigned) block_num_y);
2899
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
2900
+ stream->submit([&](sycl::handler & cgh) {
2901
+ cgh.parallel_for(
2902
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2903
+ [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2904
+ mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl>(
2905
+ vx_base, vy, dst_base, ids_dev, ncols, nrows,
2906
+ expert_weight_stride, dst_row_stride, src1_row_stride, item);
2907
+ });
2908
+ });
2909
+ }
2910
+
2911
+ bool ggml_sycl_mul_mat_vec_q_id_reorder(
2912
+ enum ggml_type src0_type,
2913
+ const void * vx_base,
2914
+ const void * vy,
2915
+ const int32_t * ids_dev,
2916
+ float * dst_base,
2917
+ int ncols,
2918
+ int nrows,
2919
+ int n_experts_used,
2920
+ size_t expert_weight_stride,
2921
+ size_t dst_row_stride,
2922
+ size_t src1_row_stride,
2923
+ dpct::queue_ptr stream) {
2924
+ switch (src0_type) {
2925
+ case GGML_TYPE_Q4_K:
2926
+ launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>>(
2927
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2928
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2929
+ return true;
2930
+ case GGML_TYPE_Q5_K:
2931
+ launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K>>(
2932
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2933
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2934
+ return true;
2935
+ case GGML_TYPE_Q6_K:
2936
+ launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q6_K>>(
2937
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
2938
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
2939
+ return true;
2940
+ default:
2941
+ return false;
2942
+ }
2943
+ }
2944
+
2945
+ template <typename reorder_vec_dot_q_sycl, int ncols_dst, int rows_per_sg>
2946
+ static void launch_mul_mat_vec_q_reorder_glu_impl(const void * vx, const void * vgate, const void * vy, float * dst,
2947
+ const int ncols, const int nrows, const int stride_col_y_bytes,
2948
+ const int stride_col_dst, const ggml_glu_op glu_op,
2949
+ dpct::queue_ptr stream) {
2950
+ GGML_ASSERT(ncols % QK_K == 0);
2951
+
2952
+ constexpr size_t num_subgroups = WARP_SIZE;
2953
+
2954
+ const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg);
2955
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2956
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
2957
+
2958
+ stream->submit([&](sycl::handler & cgh) {
2959
+ cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
2960
+ [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2961
+ mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl, ncols_dst, /*has_fusion=*/ true,
2962
+ rows_per_sg>(
2963
+ vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op,
2964
+ nd_item);
2965
+ });
2966
+ });
2967
+ }
2968
+
2969
+ template <typename reorder_vec_dot_q_sycl, int ncols_dst>
2970
+ static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate, const void * vy, float * dst,
2971
+ const int ncols, const int nrows, const int stride_col_y_bytes,
2972
+ const int stride_col_dst, const ggml_glu_op glu_op,
2973
+ dpct::queue_ptr stream) {
2974
+ constexpr int rows_per_sg =
2975
+ reorder_vec_dot_shared_activations<reorder_vec_dot_q_sycl::gtype>::value && ncols_dst >= 3 && ncols_dst <= 4
2976
+ ? 2
2977
+ : 1;
2978
+ launch_mul_mat_vec_q_reorder_glu_impl<reorder_vec_dot_q_sycl, ncols_dst, rows_per_sg>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream);
2979
+ }
2980
+
2981
+ bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx,
2982
+ const void * vgate, const void * vy, float * dst, int ncols, int nrows,
2983
+ int ncols_dst, int stride_col_y_bytes, int stride_col_dst,
2984
+ dpct::queue_ptr stream) {
2985
+ if (src0_type != GGML_TYPE_Q4_K) {
2986
+ return false;
2987
+ }
2988
+ if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) {
2989
+ return false;
2990
+ }
2991
+
2992
+ using vec_dot = reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>;
2993
+
2994
+ switch (ncols_dst) {
2995
+ case 1:
2996
+ launch_mul_mat_vec_q_reorder_glu<vec_dot, 1>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
2997
+ stride_col_dst, glu_op, stream);
2998
+ return true;
2999
+ case 2:
3000
+ if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) {
3001
+ launch_mul_mat_vec_q_reorder_glu_impl<vec_dot, 2, 2>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream);
3002
+ } else {
3003
+ launch_mul_mat_vec_q_reorder_glu_impl<vec_dot, 2, 1>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream);
3004
+ }
3005
+ return true;
3006
+ case 3:
3007
+ launch_mul_mat_vec_q_reorder_glu<vec_dot, 3>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
3008
+ stride_col_dst, glu_op, stream);
3009
+ return true;
3010
+ case 4:
3011
+ launch_mul_mat_vec_q_reorder_glu<vec_dot, 4>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
3012
+ stride_col_dst, glu_op, stream);
3013
+ return true;
3014
+ case 5:
3015
+ launch_mul_mat_vec_q_reorder_glu<vec_dot, 5>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
3016
+ stride_col_dst, glu_op, stream);
3017
+ return true;
3018
+ case 6:
3019
+ launch_mul_mat_vec_q_reorder_glu<vec_dot, 6>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
3020
+ stride_col_dst, glu_op, stream);
3021
+ return true;
3022
+ case 7:
3023
+ launch_mul_mat_vec_q_reorder_glu<vec_dot, 7>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
3024
+ stride_col_dst, glu_op, stream);
3025
+ return true;
3026
+ case 8:
3027
+ launch_mul_mat_vec_q_reorder_glu<vec_dot, 8>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
3028
+ stride_col_dst, glu_op, stream);
3029
+ return true;
3030
+ default:
3031
+ return false;
3032
+ }
3033
+ }