elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,2227 @@
1
+ #include "convert.hpp"
2
+ #include "dmmv.hpp"
3
+ #include "dequantize.hpp"
4
+ #include "presets.hpp"
5
+
6
+ #if defined(__INTEL_LLVM_COMPILER)
7
+ #if __has_include(<sycl/ext/oneapi/bfloat16.hpp>)
8
+ #include <sycl/ext/oneapi/bfloat16.hpp>
9
+ #define GGML_SYCL_DMMV_HAS_BF16
10
+ #endif
11
+ #include <sycl/ext/intel/esimd.hpp>
12
+ #include "esimd.hpp"
13
+ #define GGML_SYCL_DMMV_HAS_ESIMD
14
+ #endif
15
+
16
+ static void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
17
+ const sycl::half *x = (const sycl::half *)vx;
18
+
19
+ // automatic half -> float type cast if dfloat == float
20
+ v.x() = x[ib + iqs + 0];
21
+ v.y() = x[ib + iqs + 1];
22
+ }
23
+
24
+ #ifdef GGML_SYCL_DMMV_HAS_BF16
25
+ static void convert_bf16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
26
+ const sycl::ext::oneapi::bfloat16 *x = (const sycl::ext::oneapi::bfloat16 *)vx;
27
+
28
+ // automatic bfloat16 -> float type cast if dfloat == float
29
+ v.x() = x[ib + iqs + 0];
30
+ v.y() = x[ib + iqs + 1];
31
+ }
32
+ #endif
33
+
34
+ static void convert_f32(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
35
+ const float * x = (const float *) vx;
36
+
37
+ // automatic half -> float type cast if dfloat == float
38
+ v.x() = x[ib + iqs + 0];
39
+ v.y() = x[ib + iqs + 1];
40
+ }
41
+
42
+ template <int qk, int qr, dequantize_kernel_t dequantize_kernel>
43
+ static void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows,
44
+ const sycl::nd_item<3> &item_ct1) {
45
+ // qk = quantized weights per x block
46
+ // qr = number of quantized weights per data value in x block
47
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
48
+ item_ct1.get_local_id(1);
49
+
50
+ if (row >= nrows) {
51
+ return;
52
+ }
53
+
54
+ const int tid = item_ct1.get_local_id(2);
55
+
56
+ const int iter_stride = 2*GGML_SYCL_DMMV_X;
57
+ const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter
58
+ const int y_offset = qr == 1 ? 1 : qk/2;
59
+
60
+ // partial sum for each thread
61
+ #ifdef GGML_SYCL_F16
62
+ sycl::half2 tmp = {0.0f, 0.0f}; // two sums for f16 to take advantage of half2 intrinsics
63
+ #else
64
+ float tmp = 0.0f;
65
+ #endif // GGML_SYCL_F16
66
+
67
+ for (int i = 0; i < ncols; i += iter_stride) {
68
+ const int col = i + vals_per_iter*tid;
69
+ const int ib = (row*ncols + col)/qk; // x block index
70
+ const int iqs = (col%qk)/qr; // x quant index
71
+ const int iybs = col - col%qk; // y block start index
72
+
73
+ // processing >2 values per i iter is faster for fast GPUs
74
+ #pragma unroll
75
+ for (int j = 0; j < vals_per_iter; j += 2) {
76
+ // process 2 vals per j iter
77
+
78
+ // dequantize
79
+ // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val
80
+ dfloat2 v;
81
+ dequantize_kernel(vx, ib, iqs + j/qr, v);
82
+
83
+ // matrix multiplication
84
+ // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2
85
+ #ifdef GGML_SYCL_F16
86
+ dfloat2 t1{y[iybs + iqs + j / qr + 0],
87
+ y[iybs + iqs + j / qr + y_offset]};
88
+
89
+ tmp += v * t1;
90
+ #else
91
+ tmp += v.x() * y[iybs + iqs + j / qr + 0];
92
+ tmp += v.y() * y[iybs + iqs + j / qr + y_offset];
93
+ #endif // GGML_SYCL_F16
94
+ }
95
+ }
96
+
97
+ // sum up partial sums and write back result
98
+ const int mask_start = ncols > GGML_SYCL_DMMV_X ? WARP_SIZE >> 1 : WARP_SIZE >> 2;
99
+ for (int mask = mask_start; mask > 0; mask >>= 1) {
100
+ tmp +=
101
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
102
+ }
103
+
104
+ if (tid == 0) {
105
+ #ifdef GGML_SYCL_F16
106
+ dst[row] = tmp.x() + tmp.y();
107
+ #else
108
+ dst[row] = tmp;
109
+ #endif // GGML_SYCL_F16
110
+ }
111
+ }
112
+
113
+ template <int qk, int qr, dequantize_kernel_t_reorder dequantize_kernel_reorder>
114
+ static void dequantize_mul_mat_vec_reorder(const void * __restrict__ vx, const dfloat * __restrict__ y, float * __restrict__ dst, const int ncols, const int nrows,
115
+ const sycl::nd_item<3> &item_ct1) {
116
+ // qk = quantized weights per x block
117
+ // qr = number of quantized weights per data value in x block
118
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
119
+ item_ct1.get_local_id(1);
120
+
121
+ if (row >= nrows) {
122
+ return;
123
+ }
124
+
125
+ const int tid = item_ct1.get_local_id(2);
126
+
127
+
128
+ const int ncols_left = ncols % (QK4_0*WARP_SIZE);
129
+ const int ncols_align = ncols - ncols_left;
130
+ const int iter_stride = 8*2*GGML_SYCL_DMMV_X;
131
+ const int vals_per_iter = iter_stride / WARP_SIZE; // num quantized vals per thread and i iter //64/16=4, 512/16/2= 16
132
+ const int y_offset = qr == 1 ? 1 : qk/2;
133
+
134
+ // partial sum for each thread
135
+ #ifdef GGML_SYCL_F16
136
+ sycl::half2 tmp = {0.0f, 0.0f}; // two sums for f16 to take advantage of half2 intrinsics
137
+ #else
138
+ float tmp = 0.0f;
139
+ #endif // GGML_SYCL_F16
140
+ const char *d_ptr = (const char*)vx+ncols*nrows/2;
141
+ int i=0;
142
+ for (i = 0; i < ncols_align; i += iter_stride) {
143
+ const int col = i + vals_per_iter*tid;
144
+ const int ib = (row*ncols + col)/qk; // x block index
145
+ const int iqs = (col%qk)/qr; // x quant index
146
+ const int iybs = col - col%qk; // y block start index
147
+
148
+ // processing >2 values per i iter is faster for fast GPUs
149
+ #pragma unroll
150
+ for (int j = 0; j < vals_per_iter; j += 2) {
151
+ // process 2 vals per j iter
152
+
153
+ // dequantize
154
+ // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val
155
+ dfloat2 v;
156
+ dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v);
157
+
158
+ // matrix multiplication
159
+ // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2
160
+ #ifdef GGML_SYCL_F16
161
+ dfloat2 t1{y[iybs + iqs + j / qr + 0],
162
+ y[iybs + iqs + j / qr + y_offset]};
163
+
164
+ tmp += v * t1;
165
+ #else
166
+ tmp += v.x() * y[iybs + iqs + j / qr + 0];
167
+ tmp += v.y() * y[iybs + iqs + j / qr + y_offset];
168
+ #endif // GGML_SYCL_F16
169
+ }
170
+ }
171
+
172
+ for (; i < ncols; i += iter_stride) {
173
+ if (tid>=ncols_left/QK4_0) continue;
174
+ const int col = i + vals_per_iter*tid;
175
+ const int ib = (row*ncols + col)/qk; // x block index
176
+ const int iqs = (col%qk)/qr; // x quant index
177
+ const int iybs = col - col%qk; // y block start index
178
+
179
+ // processing >2 values per i iter is faster for fast GPUs
180
+ #pragma unroll
181
+ for (int j = 0; j < vals_per_iter; j += 2) {
182
+ // process 2 vals per j iter
183
+
184
+ // dequantize
185
+ // for qr = 2 the iqs needs to increase by 1 per j iter because 2 weights per data val
186
+ dfloat2 v;
187
+ dequantize_kernel_reorder((const void *)d_ptr, ib, (const void *)vx, ib * QK4_0 / 2 +iqs+j/qr, v);
188
+
189
+ // matrix multiplication
190
+ // for qr = 2 the y index needs to increase by 1 per j iter because of y_offset = qk/2
191
+ #ifdef GGML_SYCL_F16
192
+ dfloat2 t1{y[iybs + iqs + j / qr + 0],
193
+ y[iybs + iqs + j / qr + y_offset]};
194
+
195
+ tmp += v * t1;
196
+ #else
197
+ tmp += v.x() * y[iybs + iqs + j / qr + 0];
198
+ tmp += v.y() * y[iybs + iqs + j / qr + y_offset];
199
+ #endif // GGML_SYCL_F16
200
+ }
201
+ }
202
+
203
+ // sum up partial sums and write back result
204
+ const int mask_start = ncols > GGML_SYCL_DMMV_X ? WARP_SIZE >> 1 : WARP_SIZE >> 2;
205
+ for (int mask = mask_start; mask > 0; mask >>= 1) {
206
+ tmp +=
207
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
208
+ }
209
+
210
+ if (tid == 0) {
211
+ #ifdef GGML_SYCL_F16
212
+ dst[row] = tmp.x() + tmp.y();
213
+ #else
214
+ dst[row] = tmp;
215
+ #endif // GGML_SYCL_F16
216
+ }
217
+ }
218
+
219
+ static void convert_mul_mat_vec_f16_sycl(const void *vx, const dfloat *y,
220
+ float *dst, const int ncols,
221
+ const int nrows,
222
+ dpct::queue_ptr stream) {
223
+ GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
224
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
225
+ const sycl::range<3> block_nums(1, 1, block_num_y);
226
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
227
+ {
228
+ dpct::has_capability_or_fail(stream->get_device(),
229
+ {sycl::aspect::fp16});
230
+
231
+ stream->parallel_for(
232
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
233
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
234
+ dequantize_mul_mat_vec<1, 1, convert_f16>(vx, y, dst, ncols,
235
+ nrows, item_ct1);
236
+ });
237
+ }
238
+ }
239
+
240
+ #ifdef GGML_SYCL_DMMV_HAS_BF16
241
+ static void convert_mul_mat_vec_bf16_sycl(const void *vx, const dfloat *y,
242
+ float *dst, const int ncols,
243
+ const int nrows,
244
+ dpct::queue_ptr stream) {
245
+ // The qk=1 kernel iterates with stride 2*GGML_SYCL_DMMV_X, so ncols must be a
246
+ // multiple of that — not just GGML_SYCL_DMMV_X — to avoid out-of-bounds reads.
247
+ GGML_ASSERT(ncols % (2*GGML_SYCL_DMMV_X) == 0);
248
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
249
+ const sycl::range<3> block_nums(1, 1, block_num_y);
250
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
251
+ {
252
+ stream->parallel_for(
253
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
254
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
255
+ dequantize_mul_mat_vec<1, 1, convert_bf16>(vx, y, dst, ncols,
256
+ nrows, item_ct1);
257
+ });
258
+ }
259
+ }
260
+ #endif
261
+
262
+ static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx,
263
+ const float *__restrict__ yy,
264
+ float *__restrict__ dst,
265
+ const int ncols, int nrows,
266
+ const sycl::nd_item<3> &item_ct1) {
267
+
268
+ static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");
269
+
270
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
271
+ item_ct1.get_local_id(1);
272
+ if (row >= nrows) return;
273
+
274
+ const int num_blocks_per_row = ncols / QK_K;
275
+ const int ib0 = row*num_blocks_per_row;
276
+
277
+ const block_q2_K * x = (const block_q2_K *)vx + ib0;
278
+
279
+ float tmp = 0; // partial sum for thread in warp
280
+
281
+ #if QK_K == 256
282
+ const int tid =
283
+ item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15
284
+ const int ix =
285
+ item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1
286
+
287
+ const int step = 16/K_QUANTS_PER_ITERATION;
288
+
289
+ const int in = tid % step; // 0...15 or 0...7
290
+
291
+ const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 or 0...14 in steps of 2
292
+
293
+ uint32_t aux[4];
294
+ const uint8_t * d = (const uint8_t *)aux;
295
+ const uint8_t * m = (const uint8_t *)(aux + 2);
296
+
297
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
298
+
299
+ const float dall = x[i].dm[0];
300
+ const float dmin = x[i].dm[1];
301
+
302
+ for (int im = 0; im < 2; ++im) {
303
+ const int q_offset = 32*im + l0;
304
+ const int s_offset = 8*im;
305
+ const int y_offset = 128*im + l0;
306
+
307
+ const float * y = yy + i * QK_K + y_offset;
308
+ const uint8_t * q = x[i].qs + q_offset;
309
+
310
+ const uint32_t * a = (const uint32_t *)(x[i].scales + s_offset);
311
+ aux[0] = a[0] & 0x0f0f0f0f;
312
+ aux[1] = a[1] & 0x0f0f0f0f;
313
+ aux[2] = (a[0] >> 4) & 0x0f0f0f0f;
314
+ aux[3] = (a[1] >> 4) & 0x0f0f0f0f;
315
+
316
+ float sum1 = 0, sum2 = 0;
317
+ for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {
318
+ sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3)
319
+ + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3)
320
+ + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3)
321
+ + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3)
322
+ + y[l+16] * d[1] * ((q[l+16] >> 0) & 3)
323
+ + y[l+48] * d[3] * ((q[l+16] >> 2) & 3)
324
+ + y[l+80] * d[5] * ((q[l+16] >> 4) & 3)
325
+ +y[l+112] * d[7] * ((q[l+16] >> 6) & 3);
326
+ sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6]
327
+ + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7];
328
+
329
+ }
330
+ tmp += dall * sum1 - dmin * sum2;
331
+ }
332
+
333
+ }
334
+ #else
335
+ const int tid = item_ct1.get_local_id(2) /
336
+ (2 * K_QUANTS_PER_ITERATION); // 0...15 or 0...7
337
+ const int ix = item_ct1.get_local_id(2) %
338
+ (2 * K_QUANTS_PER_ITERATION); // 0....1 or 0...3
339
+ const int offset = tid * K_QUANTS_PER_ITERATION;
340
+
341
+ uint32_t uaux[2];
342
+ const uint8_t * d = (const uint8_t *)uaux;
343
+
344
+
345
+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {
346
+
347
+ const float * y = yy + i * QK_K + offset;
348
+ const uint8_t * q = x[i].qs + offset;
349
+ const uint32_t * s = (const uint32_t *)x[i].scales;
350
+
351
+ uaux[0] = s[0] & 0x0f0f0f0f;
352
+ uaux[1] = (s[0] >> 4) & 0x0f0f0f0f;
353
+
354
+ const sycl::float2 dall =
355
+ x[i].dm.convert<float, sycl::rounding_mode::automatic>();
356
+
357
+ float sum1 = 0, sum2 = 0;
358
+ for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {
359
+ const uint8_t ql = q[l];
360
+ sum1 += y[l+ 0] * d[0] * ((ql >> 0) & 3)
361
+ + y[l+16] * d[1] * ((ql >> 2) & 3)
362
+ + y[l+32] * d[2] * ((ql >> 4) & 3)
363
+ + y[l+48] * d[3] * ((ql >> 6) & 3);
364
+ sum2 += y[l+0] * d[4] + y[l+16] * d[5] + y[l+32] * d[6] + y[l+48] * d[7];
365
+ }
366
+ tmp += dall.x() * sum1 - dall.y() * sum2;
367
+ }
368
+
369
+ #endif
370
+
371
+ // sum up partial sums and write back result
372
+ #pragma unroll
373
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
374
+ tmp +=
375
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
376
+ }
377
+
378
+ if (item_ct1.get_local_id(2) == 0) {
379
+ dst[row] = tmp;
380
+ }
381
+ }
382
+
383
+ static void dequantize_mul_mat_vec_q2_k_reorder(const void *__restrict__ vx,
384
+ const float *__restrict__ yy,
385
+ float *__restrict__ dst,
386
+ const int ncols, int nrows,
387
+ const sycl::nd_item<3> &item_ct1) {
388
+
389
+ static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");
390
+
391
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
392
+ item_ct1.get_local_id(1);
393
+ if (row >= nrows) return;
394
+
395
+ const int num_blocks_per_row = ncols / QK_K;
396
+ const int ib0 = row*num_blocks_per_row;
397
+
398
+ // SOA base pointers for the reordered layout:
399
+ // [qs: nb * (QK_K/4)] [scales: nb * (QK_K/16)] [dm: nb * sizeof(half2)]
400
+ const int nb = nrows * num_blocks_per_row;
401
+ const uint8_t * qs_base = (const uint8_t *)vx;
402
+ const uint8_t * scales_base = qs_base + (size_t)nb * (QK_K / 4);
403
+ const sycl::half2 * dm_base = (const sycl::half2 *)(scales_base + (size_t)nb * (QK_K / 16));
404
+
405
+ float tmp = 0; // partial sum for thread in warp
406
+
407
+ #if QK_K == 256
408
+ const int tid =
409
+ item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15
410
+ const int ix =
411
+ item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1
412
+
413
+ const int step = 16/K_QUANTS_PER_ITERATION;
414
+
415
+ const int in = tid % step; // 0...15 or 0...7
416
+
417
+ const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15 or 0...14 in steps of 2
418
+
419
+ uint32_t aux[4];
420
+ const uint8_t * d = (const uint8_t *)aux;
421
+ const uint8_t * m = (const uint8_t *)(aux + 2);
422
+
423
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
424
+ const int bi = ib0 + i;
425
+
426
+ const sycl::half2 dm_val = dm_base[bi];
427
+ const float dall = dm_val[0];
428
+ const float dmin = dm_val[1];
429
+
430
+ for (int im = 0; im < 2; ++im) {
431
+ const int q_offset = 32*im + l0;
432
+ const int s_offset = 8*im;
433
+ const int y_offset = 128*im + l0;
434
+
435
+ const float * y = yy + i * QK_K + y_offset;
436
+ const uint8_t * q = qs_base + bi * (QK_K / 4) + q_offset;
437
+
438
+ const uint32_t * a = (const uint32_t *)(scales_base + bi * (QK_K / 16) + s_offset);
439
+ aux[0] = a[0] & 0x0f0f0f0f;
440
+ aux[1] = a[1] & 0x0f0f0f0f;
441
+ aux[2] = (a[0] >> 4) & 0x0f0f0f0f;
442
+ aux[3] = (a[1] >> 4) & 0x0f0f0f0f;
443
+
444
+ float sum1 = 0, sum2 = 0;
445
+ for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {
446
+ sum1 += y[l+ 0] * d[0] * ((q[l+ 0] >> 0) & 3)
447
+ + y[l+32] * d[2] * ((q[l+ 0] >> 2) & 3)
448
+ + y[l+64] * d[4] * ((q[l+ 0] >> 4) & 3)
449
+ + y[l+96] * d[6] * ((q[l+ 0] >> 6) & 3)
450
+ + y[l+16] * d[1] * ((q[l+16] >> 0) & 3)
451
+ + y[l+48] * d[3] * ((q[l+16] >> 2) & 3)
452
+ + y[l+80] * d[5] * ((q[l+16] >> 4) & 3)
453
+ +y[l+112] * d[7] * ((q[l+16] >> 6) & 3);
454
+ sum2 += y[l+ 0] * m[0] + y[l+32] * m[2] + y[l+64] * m[4] + y[ l+96] * m[6]
455
+ + y[l+16] * m[1] + y[l+48] * m[3] + y[l+80] * m[5] + y[l+112] * m[7];
456
+
457
+ }
458
+ tmp += dall * sum1 - dmin * sum2;
459
+ }
460
+ }
461
+ #else
462
+ GGML_UNUSED(vx);
463
+ GGML_UNUSED(yy);
464
+ GGML_UNUSED(ncols);
465
+ GGML_UNUSED(item_ct1);
466
+ GGML_ABORT("Q2_K reorder DMMV not supported for QK_K != 256");
467
+ #endif
468
+
469
+ // sum up partial sums and write back result
470
+ #pragma unroll
471
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
472
+ tmp +=
473
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
474
+ }
475
+
476
+ if (item_ct1.get_local_id(2) == 0) {
477
+ dst[row] = tmp;
478
+ }
479
+ }
480
+
481
+ static void dequantize_mul_mat_vec_q3_k(const void *__restrict__ vx,
482
+ const float *__restrict__ yy,
483
+ float *__restrict__ dst,
484
+ const int ncols, int nrows,
485
+ const sycl::nd_item<3> &item_ct1) {
486
+
487
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
488
+ item_ct1.get_local_id(1);
489
+ if (row >= nrows) return;
490
+
491
+ const int num_blocks_per_row = ncols / QK_K;
492
+ const int ib0 = row*num_blocks_per_row;
493
+
494
+ const block_q3_K * x = (const block_q3_K *)vx + ib0;
495
+
496
+ float tmp = 0; // partial sum for thread in warp
497
+
498
+ #if QK_K == 256
499
+
500
+ const uint16_t kmask1 = 0x0303;
501
+ const uint16_t kmask2 = 0x0f0f;
502
+
503
+ const int tid =
504
+ item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15
505
+ const int ix =
506
+ item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1
507
+
508
+ const int n = K_QUANTS_PER_ITERATION; // iterations in the inner loop
509
+ const int step = 16/K_QUANTS_PER_ITERATION;
510
+ const int in = tid % step; // 0...15 or 0...7
511
+
512
+ const int l0 = n*in; // 0...15 or 0...14 in steps of 2
513
+
514
+ uint16_t utmp[4];
515
+ const int8_t * s = (const int8_t *)utmp;
516
+
517
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
518
+
519
+ const uint8_t * h = x[i].hmask + l0;
520
+ const float d = x[i].d;
521
+
522
+ for (int im = 0; im < 2; ++im) {
523
+ const int q_offset = 32*im + l0;
524
+ const int y_offset = 128*im + l0;
525
+ const uint16_t s_shift = 4*im;
526
+ const uint8_t m = 1 << (4*im);
527
+
528
+ const float * y = yy + i * QK_K + y_offset;
529
+ const uint8_t * q = x[i].qs + q_offset;
530
+
531
+ const uint16_t * a = (const uint16_t *)x[i].scales;
532
+ utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4);
533
+ utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4);
534
+ utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4);
535
+ utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4);
536
+
537
+ float sum = 0;
538
+ for (int l = 0; l < n; ++l) {
539
+ sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4))
540
+ + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4))
541
+ + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4))
542
+ + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4));
543
+ sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4))
544
+ + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4))
545
+ + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4))
546
+ + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4));
547
+ }
548
+ tmp += d * sum;
549
+ }
550
+
551
+ }
552
+ #else
553
+
554
+ const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7
555
+ const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3
556
+ const int offset = tid * K_QUANTS_PER_ITERATION; // 0...15 or 0...14
557
+ const int in = offset/8; // 0 or 1
558
+ const int im = offset%8; // 0...7
559
+
560
+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {
561
+
562
+ const float * y = yy + i * QK_K + offset;
563
+ const uint8_t * q = x[i].qs + offset;
564
+ const uint8_t * s = x[i].scales;
565
+
566
+ const float dall = (float)x[i].d;
567
+
568
+ float sum = 0;
569
+ for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) {
570
+ const uint8_t hl = x[i].hmask[im+l] >> in;
571
+ const uint8_t ql = q[l];
572
+ sum += y[l+ 0] * dall * ((s[0] & 0xF) - 8) * ((int8_t)((ql >> 0) & 3) - ((hl >> 0) & 1 ? 0 : 4))
573
+ + y[l+16] * dall * ((s[0] >> 4) - 8) * ((int8_t)((ql >> 2) & 3) - ((hl >> 2) & 1 ? 0 : 4))
574
+ + y[l+32] * dall * ((s[1] & 0xF) - 8) * ((int8_t)((ql >> 4) & 3) - ((hl >> 4) & 1 ? 0 : 4))
575
+ + y[l+48] * dall * ((s[1] >> 4) - 8) * ((int8_t)((ql >> 6) & 3) - ((hl >> 6) & 1 ? 0 : 4));
576
+ }
577
+ tmp += sum;
578
+ }
579
+ #endif
580
+
581
+ // sum up partial sums and write back result
582
+ #pragma unroll
583
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
584
+ tmp +=
585
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
586
+ }
587
+
588
+ if (item_ct1.get_local_id(2) == 0) {
589
+ dst[row] = tmp;
590
+ }
591
+ }
592
+
593
+ static void dequantize_mul_mat_vec_q3_k_reorder(const void *__restrict__ vx,
594
+ const float *__restrict__ yy,
595
+ float *__restrict__ dst,
596
+ const int ncols, int nrows,
597
+ const sycl::nd_item<3> &item_ct1) {
598
+
599
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
600
+ item_ct1.get_local_id(1);
601
+ if (row >= nrows) return;
602
+
603
+ const int num_blocks_per_row = ncols / QK_K;
604
+ const int ib0 = row*num_blocks_per_row;
605
+
606
+ // SOA base pointers for the reordered layout:
607
+ // [qs: nb * (QK_K/4)] [hmask: nb * (QK_K/8)] [scales: nb * 12] [d: nb * sizeof(half)]
608
+ const int nb = nrows * num_blocks_per_row;
609
+ const uint8_t * qs_base = (const uint8_t *)vx;
610
+ const uint8_t * hmask_base = qs_base + (size_t)nb * (QK_K / 4);
611
+ const uint8_t * scales_base = hmask_base + (size_t)nb * (QK_K / 8);
612
+ const sycl::half * d_base = (const sycl::half *)(scales_base + (size_t)nb * 12);
613
+
614
+ float tmp = 0; // partial sum for thread in warp
615
+
616
+ #if QK_K == 256
617
+
618
+ const uint16_t kmask1 = 0x0303;
619
+ const uint16_t kmask2 = 0x0f0f;
620
+
621
+ const int tid =
622
+ item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15
623
+ const int ix =
624
+ item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1
625
+
626
+ const int n = K_QUANTS_PER_ITERATION; // iterations in the inner loop
627
+ const int step = 16/K_QUANTS_PER_ITERATION;
628
+ const int in = tid % step; // 0...15 or 0...7
629
+
630
+ const int l0 = n*in; // 0...15 or 0...14 in steps of 2
631
+
632
+ uint16_t utmp[4];
633
+ const int8_t * s = (const int8_t *)utmp;
634
+
635
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
636
+ const int bi = ib0 + i;
637
+
638
+ const uint8_t * h = hmask_base + bi * (QK_K / 8) + l0;
639
+
640
+ const float d = d_base[bi];
641
+
642
+ for (int im = 0; im < 2; ++im) {
643
+ const int q_offset = 32*im + l0;
644
+ const int y_offset = 128*im + l0;
645
+ const uint16_t s_shift = 4*im;
646
+ const uint8_t m = 1 << (4*im);
647
+
648
+ const float * y = yy + i * QK_K + y_offset;
649
+ const uint8_t * q = qs_base + bi * (QK_K / 4) + q_offset;
650
+
651
+ const uint16_t * a = (const uint16_t *)(scales_base + bi * 12);
652
+ utmp[0] = ((a[0] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 0)) & kmask1) << 4);
653
+ utmp[1] = ((a[1] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 0)) & kmask1) << 4);
654
+ utmp[2] = ((a[2] >> s_shift) & kmask2) | (((a[4] >> (s_shift + 2)) & kmask1) << 4);
655
+ utmp[3] = ((a[3] >> s_shift) & kmask2) | (((a[5] >> (s_shift + 2)) & kmask1) << 4);
656
+
657
+ float sum = 0;
658
+ for (int l = 0; l < n; ++l) {
659
+ sum += y[l+ 0] * (s[0] - 32) * (((q[l] >> 0) & 3) - (h[l] & (m << 0) ? 0 : 4))
660
+ + y[l+32] * (s[2] - 32) * (((q[l] >> 2) & 3) - (h[l] & (m << 1) ? 0 : 4))
661
+ + y[l+64] * (s[4] - 32) * (((q[l] >> 4) & 3) - (h[l] & (m << 2) ? 0 : 4))
662
+ + y[l+96] * (s[6] - 32) * (((q[l] >> 6) & 3) - (h[l] & (m << 3) ? 0 : 4));
663
+ sum += y[l+16] * (s[1] - 32) * (((q[l+16] >> 0) & 3) - (h[l+16] & (m << 0) ? 0 : 4))
664
+ + y[l+48] * (s[3] - 32) * (((q[l+16] >> 2) & 3) - (h[l+16] & (m << 1) ? 0 : 4))
665
+ + y[l+80] * (s[5] - 32) * (((q[l+16] >> 4) & 3) - (h[l+16] & (m << 2) ? 0 : 4))
666
+ + y[l+112] * (s[7] - 32) * (((q[l+16] >> 6) & 3) - (h[l+16] & (m << 3) ? 0 : 4));
667
+ }
668
+ tmp += d * sum;
669
+ }
670
+ }
671
+ #else
672
+ GGML_UNUSED(vx);
673
+ GGML_UNUSED(yy);
674
+ GGML_UNUSED(ncols);
675
+ GGML_UNUSED(item_ct1);
676
+ GGML_ABORT("Q3_K reorder DMMV not supported for QK_K != 256");
677
+ #endif
678
+
679
+ // sum up partial sums and write back result
680
+ #pragma unroll
681
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
682
+ tmp +=
683
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
684
+ }
685
+
686
+ if (item_ct1.get_local_id(2) == 0) {
687
+ dst[row] = tmp;
688
+ }
689
+ }
690
+
691
+ static void dequantize_mul_mat_vec_q4_k(const void *__restrict__ vx,
692
+ const float *__restrict__ yy,
693
+ float *__restrict__ dst,
694
+ const int ncols, int nrows,
695
+ const sycl::nd_item<3> &item_ct1) {
696
+
697
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
698
+ item_ct1.get_local_id(1);
699
+ if (row >= nrows) return;
700
+ const int num_blocks_per_row = ncols / QK_K;
701
+ const int ib0 = row*num_blocks_per_row;
702
+
703
+ const block_q4_K * x = (const block_q4_K *)vx + ib0;
704
+
705
+ #if QK_K == 256
706
+ const uint16_t kmask1 = 0x3f3f;
707
+ const uint16_t kmask2 = 0x0f0f;
708
+ const uint16_t kmask3 = 0xc0c0;
709
+
710
+ const int tid =
711
+ item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15
712
+ const int ix =
713
+ item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1
714
+
715
+ const int step = 8/K_QUANTS_PER_ITERATION; // 8 or 4
716
+
717
+ const int il_base = tid/step; // 0 or 1 (was 0...3)
718
+ const int ir = tid - step*il_base; // 0...7 or 0...3
719
+ const int n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4
720
+
721
+ const int in = il_base%2;
722
+
723
+ const int l0 = n*(2*ir + in);
724
+
725
+ uint16_t aux[4];
726
+ const uint8_t * sc = (const uint8_t *)aux;
727
+
728
+ #if K_QUANTS_PER_ITERATION == 2
729
+ uint32_t q32[4];
730
+ const uint8_t * q4 = (const uint8_t *)q32;
731
+ #else
732
+ uint16_t q16[4];
733
+ const uint8_t * q4 = (const uint8_t *)q16;
734
+ #endif
735
+
736
+ float tmp = 0; // partial sum for thread in warp
737
+
738
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
739
+
740
+ const float dall = x[i].dm[0];
741
+ const float dmin = x[i].dm[1];
742
+
743
+ for (int im = 0; im < 2; ++im) {
744
+ const int q_offset = 32*im + l0;
745
+ const int y_offset = 64*im + l0;
746
+
747
+ const float * y1 = yy + i*QK_K + y_offset;
748
+ const float * y2 = y1 + 128;
749
+
750
+ const uint16_t * a = (const uint16_t *)x[i].scales;
751
+ aux[0] = a[im+0] & kmask1;
752
+ aux[1] = a[im+2] & kmask1;
753
+ aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);
754
+ aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);
755
+
756
+ #if K_QUANTS_PER_ITERATION == 2
757
+ const uint32_t * q1 = (const uint32_t *)(x[i].qs + q_offset);
758
+ const uint32_t * q2 = q1 + 16;
759
+
760
+ q32[0] = q1[0] & 0x0f0f0f0f;
761
+ q32[1] = q1[0] & 0xf0f0f0f0;
762
+ q32[2] = q2[0] & 0x0f0f0f0f;
763
+ q32[3] = q2[0] & 0xf0f0f0f0;
764
+
765
+ sycl::float4 s = {0.f, 0.f, 0.f, 0.f};
766
+ float smin = 0;
767
+ for (int l = 0; l < 4; ++l) {
768
+ s.x() += y1[l] * q4[l + 0]; s.y() += y1[l + 32] * q4[l + 4];
769
+ s.z() += y2[l] * q4[l + 8]; s.w() += y2[l + 32] * q4[l + 12];
770
+ smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];
771
+ }
772
+ tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f / 16.f +
773
+ s.z() * sc[4] + s.w() * sc[5] * 1.f / 16.f) -
774
+ dmin * smin;
775
+ #else
776
+ const uint16_t * q1 = (const uint16_t *)(x[i].qs + q_offset);
777
+ const uint16_t * q2 = q1 + 32;
778
+
779
+ q16[0] = q1[0] & 0x0f0f;
780
+ q16[1] = q1[0] & 0xf0f0;
781
+ q16[2] = q2[0] & 0x0f0f;
782
+ q16[3] = q2[0] & 0xf0f0;
783
+
784
+ sycl::float4 s = {0.f, 0.f, 0.f, 0.f};
785
+ float smin = 0;
786
+ for (int l = 0; l < 2; ++l) {
787
+ s.x() += y1[l] * q4[l+0]; s.y() += y1[l+32] * q4[l+2];
788
+ s.z() += y2[l] * q4[l+4]; s.w() += y2[l+32] * q4[l+6];
789
+ smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];
790
+ }
791
+ tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f/16.f + s.z() * sc[4] + s.w() * sc[5] * 1.f/16.f) - dmin * smin;
792
+ #endif
793
+ }
794
+
795
+ }
796
+ #else
797
+ const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15
798
+ const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION);
799
+
800
+ const int step = tid * K_QUANTS_PER_ITERATION;
801
+
802
+ uint16_t aux16[2];
803
+ const uint8_t * s = (const uint8_t *)aux16;
804
+
805
+ float tmp = 0;
806
+
807
+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {
808
+ const uint8_t * q = x[i].qs + step;
809
+ const float * y = yy + i*QK_K + step;
810
+ const uint16_t * a = (const uint16_t *)x[i].scales;
811
+ aux16[0] = a[0] & 0x0f0f;
812
+ aux16[1] = (a[0] >> 4) & 0x0f0f;
813
+ const float d = (float)x[i].dm[0];
814
+ const float m = (float)x[i].dm[1];
815
+ float sum = 0.f;
816
+ for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {
817
+ sum += y[j+ 0] * (d * s[0] * (q[j+ 0] & 0xF) - m * s[2])
818
+ + y[j+16] * (d * s[0] * (q[j+16] & 0xF) - m * s[2])
819
+ + y[j+32] * (d * s[1] * (q[j+ 0] >> 4) - m * s[3])
820
+ + y[j+48] * (d * s[1] * (q[j+16] >> 4) - m * s[3]);
821
+ }
822
+ tmp += sum;
823
+ }
824
+
825
+ #endif
826
+
827
+ // sum up partial sums and write back result
828
+ #pragma unroll
829
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
830
+ tmp +=
831
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
832
+ }
833
+
834
+ if (tid == 0) {
835
+ dst[row] = tmp;
836
+ }
837
+ }
838
+
839
+ static void dequantize_mul_mat_vec_q4_k_reorder(const void *__restrict__ vx,
840
+ const float *__restrict__ yy,
841
+ float *__restrict__ dst,
842
+ const int ncols, int nrows,
843
+ const sycl::nd_item<3> &item_ct1) {
844
+
845
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
846
+ item_ct1.get_local_id(1);
847
+ if (row >= nrows) return;
848
+ const int num_blocks_per_row = ncols / QK_K;
849
+ const int ib0 = row*num_blocks_per_row;
850
+
851
+ // SOA base pointers for the reordered layout:
852
+ // [qs: nb * QK_K/2] [scales: nb * K_SCALE_SIZE] [dm: nb * sizeof(half2)]
853
+ const int nb = nrows * num_blocks_per_row;
854
+ const uint8_t * qs_base = (const uint8_t *)vx;
855
+ const uint8_t * scales_base = qs_base + (size_t)nb * (QK_K / 2);
856
+ const sycl::half2 * dm_base = (const sycl::half2 *)(scales_base + (size_t)nb * K_SCALE_SIZE);
857
+
858
+ #if QK_K == 256
859
+ const uint16_t kmask1 = 0x3f3f;
860
+ const uint16_t kmask2 = 0x0f0f;
861
+ const uint16_t kmask3 = 0xc0c0;
862
+
863
+ const int tid =
864
+ item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15
865
+ const int ix =
866
+ item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0,1
867
+
868
+ const int step = 8/K_QUANTS_PER_ITERATION; // 8 or 4
869
+
870
+ const int il_base = tid/step; // 0 or 1 (was 0...3)
871
+ const int ir = tid - step*il_base; // 0...7 or 0...3
872
+ const int n = 2 * K_QUANTS_PER_ITERATION; // 2 or 4
873
+
874
+ const int in = il_base%2;
875
+
876
+ const int l0 = n*(2*ir + in);
877
+
878
+ uint16_t aux[4];
879
+ const uint8_t * sc = (const uint8_t *)aux;
880
+
881
+ #if K_QUANTS_PER_ITERATION == 2
882
+ uint32_t q32[4];
883
+ const uint8_t * q4 = (const uint8_t *)q32;
884
+ #else
885
+ uint16_t q16[4];
886
+ const uint8_t * q4 = (const uint8_t *)q16;
887
+ #endif
888
+
889
+ float tmp = 0; // partial sum for thread in warp
890
+
891
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
892
+ const int bi = ib0 + i;
893
+
894
+ const sycl::half2 dm_val = dm_base[bi];
895
+ const float dall = dm_val[0];
896
+ const float dmin = dm_val[1];
897
+
898
+ for (int im = 0; im < 2; ++im) {
899
+ const int q_offset = 32*im + l0;
900
+ const int y_offset = 64*im + l0;
901
+
902
+ const float * y1 = yy + i*QK_K + y_offset;
903
+ const float * y2 = y1 + 128;
904
+
905
+ const uint16_t * a = (const uint16_t *)(scales_base + bi * K_SCALE_SIZE);
906
+ aux[0] = a[im+0] & kmask1;
907
+ aux[1] = a[im+2] & kmask1;
908
+ aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);
909
+ aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);
910
+
911
+ #if K_QUANTS_PER_ITERATION == 2
912
+ const uint32_t * q1 = (const uint32_t *)(qs_base + bi * (QK_K / 2) + q_offset);
913
+ const uint32_t * q2 = q1 + 16;
914
+
915
+ q32[0] = q1[0] & 0x0f0f0f0f;
916
+ q32[1] = q1[0] & 0xf0f0f0f0;
917
+ q32[2] = q2[0] & 0x0f0f0f0f;
918
+ q32[3] = q2[0] & 0xf0f0f0f0;
919
+
920
+ sycl::float4 s = {0.f, 0.f, 0.f, 0.f};
921
+ float smin = 0;
922
+ for (int l = 0; l < 4; ++l) {
923
+ s.x() += y1[l] * q4[l + 0]; s.y() += y1[l + 32] * q4[l + 4];
924
+ s.z() += y2[l] * q4[l + 8]; s.w() += y2[l + 32] * q4[l + 12];
925
+ smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];
926
+ }
927
+ tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f / 16.f +
928
+ s.z() * sc[4] + s.w() * sc[5] * 1.f / 16.f) -
929
+ dmin * smin;
930
+ #else
931
+ const uint16_t * q1 = (const uint16_t *)(qs_base + bi * (QK_K / 2) + q_offset);
932
+ const uint16_t * q2 = q1 + 32;
933
+
934
+ q16[0] = q1[0] & 0x0f0f;
935
+ q16[1] = q1[0] & 0xf0f0;
936
+ q16[2] = q2[0] & 0x0f0f;
937
+ q16[3] = q2[0] & 0xf0f0;
938
+
939
+ sycl::float4 s = {0.f, 0.f, 0.f, 0.f};
940
+ float smin = 0;
941
+ for (int l = 0; l < 2; ++l) {
942
+ s.x() += y1[l] * q4[l+0]; s.y() += y1[l+32] * q4[l+2];
943
+ s.z() += y2[l] * q4[l+4]; s.w() += y2[l+32] * q4[l+6];
944
+ smin += y1[l] * sc[2] + y1[l+32] * sc[3] + y2[l] * sc[6] + y2[l+32] * sc[7];
945
+ }
946
+ tmp += dall * (s.x() * sc[0] + s.y() * sc[1] * 1.f/16.f + s.z() * sc[4] + s.w() * sc[5] * 1.f/16.f) - dmin * smin;
947
+ #endif
948
+ }
949
+
950
+ }
951
+ #else
952
+ const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15
953
+ const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION);
954
+
955
+ const int step = tid * K_QUANTS_PER_ITERATION;
956
+
957
+ uint16_t aux16[2];
958
+ const uint8_t * s = (const uint8_t *)aux16;
959
+
960
+ float tmp = 0;
961
+
962
+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {
963
+ const int bi = ib0 + i;
964
+
965
+ const uint8_t * q = qs_base + bi * (QK_K / 2) + step;
966
+ const float * y = yy + i*QK_K + step;
967
+ const uint16_t * a = (const uint16_t *)(scales_base + bi * K_SCALE_SIZE);
968
+ aux16[0] = a[0] & 0x0f0f;
969
+ aux16[1] = (a[0] >> 4) & 0x0f0f;
970
+ const sycl::half2 dm_val = dm_base[bi];
971
+ const float d = (float)dm_val[0];
972
+ const float m = (float)dm_val[1];
973
+ float sum = 0.f;
974
+ for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {
975
+ sum += y[j+ 0] * (d * s[0] * (q[j+ 0] & 0xF) - m * s[2])
976
+ + y[j+16] * (d * s[0] * (q[j+16] & 0xF) - m * s[2])
977
+ + y[j+32] * (d * s[1] * (q[j+ 0] >> 4) - m * s[3])
978
+ + y[j+48] * (d * s[1] * (q[j+16] >> 4) - m * s[3]);
979
+ }
980
+ tmp += sum;
981
+ }
982
+
983
+ #endif
984
+
985
+ // sum up partial sums and write back result
986
+ #pragma unroll
987
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
988
+ tmp +=
989
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
990
+ }
991
+
992
+ if (tid == 0) {
993
+ dst[row] = tmp;
994
+ }
995
+ }
996
+
997
+ static void dequantize_mul_mat_vec_q5_k(const void *__restrict__ vx,
998
+ const float *__restrict__ yy,
999
+ float *__restrict__ dst,
1000
+ const int ncols, int nrows,
1001
+ const sycl::nd_item<3> &item_ct1) {
1002
+
1003
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
1004
+ item_ct1.get_local_id(1);
1005
+ if (row >= nrows) return;
1006
+ const int num_blocks_per_row = ncols / QK_K;
1007
+ const int ib0 = row*num_blocks_per_row;
1008
+
1009
+ const block_q5_K * x = (const block_q5_K *)vx + ib0;
1010
+
1011
+ float tmp = 0; // partial sum for thread in warp
1012
+
1013
+ #if QK_K == 256
1014
+ const uint16_t kmask1 = 0x3f3f;
1015
+ const uint16_t kmask2 = 0x0f0f;
1016
+ const uint16_t kmask3 = 0xc0c0;
1017
+
1018
+ const int tid = item_ct1.get_local_id(2) / 2; // 0...7
1019
+ const int ix = item_ct1.get_local_id(2) % 2;
1020
+
1021
+ const int il_base = tid/4; // 0 or 1 (was 0...3)
1022
+ const int ir = tid - 4*il_base;// 0...3
1023
+ const int n = 2;
1024
+
1025
+ const int in = il_base%2;
1026
+
1027
+ const int l0 = n*(2*ir + in);
1028
+
1029
+ uint16_t aux[4];
1030
+ const uint8_t * sc = (const uint8_t *)aux;
1031
+
1032
+ uint16_t q16[8];
1033
+ const uint8_t * q4 = (const uint8_t *)q16;
1034
+
1035
+ for (int i = ix; i < num_blocks_per_row; i += 2) {
1036
+
1037
+ const uint8_t * qh = x[i].qh + l0;
1038
+ const float dall = x[i].dm[0];
1039
+ const float dmin = x[i].dm[1];
1040
+
1041
+ for (int im = 0; im < 2; ++im) {
1042
+ const int q_offset = 32*im + l0;
1043
+ const int y_offset = 64*im + l0;
1044
+
1045
+ const uint8_t hm1 = 1 << (2*im);
1046
+ const uint8_t hm2 = hm1 << 4;
1047
+
1048
+ const uint8_t * ql1 = x[i].qs + q_offset;
1049
+ const float * y1 = yy + i*QK_K + y_offset;
1050
+ const float * y2 = y1 + 128;
1051
+
1052
+ const uint16_t * a = (const uint16_t *)x[i].scales;
1053
+ aux[0] = a[im+0] & kmask1;
1054
+ aux[1] = a[im+2] & kmask1;
1055
+ aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);
1056
+ aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);
1057
+
1058
+ sycl::float4 sum = {0.f, 0.f, 0.f, 0.f};
1059
+ float smin = 0;
1060
+ const uint16_t * q1 = (const uint16_t *)ql1;
1061
+ const uint16_t * q2 = q1 + 32;
1062
+ q16[0] = q1[0] & 0x0f0f;
1063
+ q16[1] = q1[8] & 0x0f0f;
1064
+ q16[2] = (q1[0] >> 4) & 0x0f0f;
1065
+ q16[3] = (q1[8] >> 4) & 0x0f0f;
1066
+ q16[4] = q2[0] & 0x0f0f;
1067
+ q16[5] = q2[8] & 0x0f0f;
1068
+ q16[6] = (q2[0] >> 4) & 0x0f0f;
1069
+ q16[7] = (q2[8] >> 4) & 0x0f0f;
1070
+ for (int l = 0; l < n; ++l) {
1071
+ sum.x() +=
1072
+ y1[l + 0] * (q4[l + 0] + (qh[l + 0] & (hm1 << 0) ? 16 : 0)) +
1073
+ y1[l + 16] * (q4[l + 2] + (qh[l + 16] & (hm1 << 0) ? 16 : 0));
1074
+ sum.y() +=
1075
+ y1[l + 32] * (q4[l + 4] + (qh[l + 0] & (hm1 << 1) ? 16 : 0)) +
1076
+ y1[l + 48] * (q4[l + 6] + (qh[l + 16] & (hm1 << 1) ? 16 : 0));
1077
+ sum.z() +=
1078
+ y2[l + 0] * (q4[l + 8] + (qh[l + 0] & (hm2 << 0) ? 16 : 0)) +
1079
+ y2[l + 16] * (q4[l + 10] + (qh[l + 16] & (hm2 << 0) ? 16 : 0));
1080
+ sum.w() +=
1081
+ y2[l + 32] * (q4[l + 12] + (qh[l + 0] & (hm2 << 1) ? 16 : 0)) +
1082
+ y2[l + 48] * (q4[l + 14] + (qh[l + 16] & (hm2 << 1) ? 16 : 0));
1083
+ smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3]
1084
+ + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7];
1085
+ }
1086
+ tmp += dall * (sum.x() * sc[0] + sum.y() * sc[1] + sum.z() * sc[4] +
1087
+ sum.w() * sc[5]) -
1088
+ dmin * smin;
1089
+ }
1090
+ }
1091
+
1092
+ #else
1093
+ const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15
1094
+ const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION);
1095
+ const int step = tid * K_QUANTS_PER_ITERATION;
1096
+ const int im = step/8;
1097
+ const int in = step%8;
1098
+
1099
+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {
1100
+ const uint8_t * q = x[i].qs + step;
1101
+ const int8_t * s = x[i].scales;
1102
+ const float * y = yy + i*QK_K + step;
1103
+ const float d = x[i].d;
1104
+ float sum = 0.f;
1105
+ for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {
1106
+ const uint8_t h = x[i].qh[in+j] >> im;
1107
+ sum += y[j+ 0] * d * s[0] * ((q[j+ 0] & 0xF) - ((h >> 0) & 1 ? 0 : 16))
1108
+ + y[j+16] * d * s[1] * ((q[j+16] & 0xF) - ((h >> 2) & 1 ? 0 : 16))
1109
+ + y[j+32] * d * s[2] * ((q[j+ 0] >> 4) - ((h >> 4) & 1 ? 0 : 16))
1110
+ + y[j+48] * d * s[3] * ((q[j+16] >> 4) - ((h >> 6) & 1 ? 0 : 16));
1111
+ }
1112
+ tmp += sum;
1113
+ }
1114
+ #endif
1115
+
1116
+ // sum up partial sums and write back result
1117
+ #pragma unroll
1118
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
1119
+ tmp +=
1120
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
1121
+ }
1122
+
1123
+ if (item_ct1.get_local_id(2) == 0) {
1124
+ dst[row] = tmp;
1125
+ }
1126
+ }
1127
+
1128
+ static void dequantize_mul_mat_vec_q5_k_reorder(const void *__restrict__ vx,
1129
+ const float *__restrict__ yy,
1130
+ float *__restrict__ dst,
1131
+ const int ncols, int nrows,
1132
+ const sycl::nd_item<3> &item_ct1) {
1133
+
1134
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
1135
+ item_ct1.get_local_id(1);
1136
+ if (row >= nrows) return;
1137
+ const int num_blocks_per_row = ncols / QK_K;
1138
+ const int ib0 = row*num_blocks_per_row;
1139
+
1140
+ // SOA base pointers for the reordered layout:
1141
+ // [qs: nb * QK_K/2] [qh: nb * QK_K/8] [scales: nb * K_SCALE_SIZE] [dm: nb * sizeof(half2)]
1142
+ const int nb = nrows * num_blocks_per_row;
1143
+ const uint8_t * qs_base = (const uint8_t *)vx;
1144
+ const uint8_t * qh_base = qs_base + (size_t)nb * (QK_K / 2);
1145
+ const uint8_t * scales_base = qh_base + (size_t)nb * (QK_K / 8);
1146
+ const sycl::half2 * dm_base = (const sycl::half2 *)(scales_base + (size_t)nb * K_SCALE_SIZE);
1147
+
1148
+ float tmp = 0; // partial sum for thread in warp
1149
+
1150
+ #if QK_K == 256
1151
+ const uint16_t kmask1 = 0x3f3f;
1152
+ const uint16_t kmask2 = 0x0f0f;
1153
+ const uint16_t kmask3 = 0xc0c0;
1154
+
1155
+ const int tid = item_ct1.get_local_id(2) / 2; // 0...15
1156
+ const int ix = item_ct1.get_local_id(2) % 2;
1157
+
1158
+ const int il_base = tid/4; // 0...3
1159
+ const int ir = tid - 4*il_base;// 0...3
1160
+ const int n = 2;
1161
+
1162
+ const int in = il_base%2;
1163
+
1164
+ const int l0 = n*(2*ir + in);
1165
+
1166
+ uint16_t aux[4];
1167
+ const uint8_t * sc = (const uint8_t *)aux;
1168
+
1169
+ uint16_t q16[8];
1170
+ const uint8_t * q4 = (const uint8_t *)q16;
1171
+
1172
+ for (int i = ix; i < num_blocks_per_row; i += 2) {
1173
+ const int bi = ib0 + i;
1174
+
1175
+ const uint8_t * qh = qh_base + bi * (QK_K / 8) + l0;
1176
+ const sycl::half2 dm_val = dm_base[bi];
1177
+ const float dall = dm_val[0];
1178
+ const float dmin = dm_val[1];
1179
+
1180
+ for (int im = 0; im < 2; ++im) {
1181
+ const int q_offset = 32*im + l0;
1182
+ const int y_offset = 64*im + l0;
1183
+
1184
+ const uint8_t hm1 = 1 << (2*im);
1185
+ const uint8_t hm2 = hm1 << 4;
1186
+
1187
+ const uint8_t * ql1 = qs_base + bi * (QK_K / 2) + q_offset;
1188
+ const float * y1 = yy + i*QK_K + y_offset;
1189
+ const float * y2 = y1 + 128;
1190
+
1191
+ const uint16_t * a = (const uint16_t *)(scales_base + bi * K_SCALE_SIZE);
1192
+ aux[0] = a[im+0] & kmask1;
1193
+ aux[1] = a[im+2] & kmask1;
1194
+ aux[2] = ((a[im+4] >> 0) & kmask2) | ((a[im+0] & kmask3) >> 2);
1195
+ aux[3] = ((a[im+4] >> 4) & kmask2) | ((a[im+2] & kmask3) >> 2);
1196
+
1197
+ sycl::float4 sum = {0.f, 0.f, 0.f, 0.f};
1198
+ float smin = 0;
1199
+ const uint16_t * q1 = (const uint16_t *)ql1;
1200
+ const uint16_t * q2 = q1 + 32;
1201
+ q16[0] = q1[0] & 0x0f0f;
1202
+ q16[1] = q1[8] & 0x0f0f;
1203
+ q16[2] = (q1[0] >> 4) & 0x0f0f;
1204
+ q16[3] = (q1[8] >> 4) & 0x0f0f;
1205
+ q16[4] = q2[0] & 0x0f0f;
1206
+ q16[5] = q2[8] & 0x0f0f;
1207
+ q16[6] = (q2[0] >> 4) & 0x0f0f;
1208
+ q16[7] = (q2[8] >> 4) & 0x0f0f;
1209
+ for (int l = 0; l < n; ++l) {
1210
+ sum.x() +=
1211
+ y1[l + 0] * (q4[l + 0] + (qh[l + 0] & (hm1 << 0) ? 16 : 0)) +
1212
+ y1[l + 16] * (q4[l + 2] + (qh[l + 16] & (hm1 << 0) ? 16 : 0));
1213
+ sum.y() +=
1214
+ y1[l + 32] * (q4[l + 4] + (qh[l + 0] & (hm1 << 1) ? 16 : 0)) +
1215
+ y1[l + 48] * (q4[l + 6] + (qh[l + 16] & (hm1 << 1) ? 16 : 0));
1216
+ sum.z() +=
1217
+ y2[l + 0] * (q4[l + 8] + (qh[l + 0] & (hm2 << 0) ? 16 : 0)) +
1218
+ y2[l + 16] * (q4[l + 10] + (qh[l + 16] & (hm2 << 0) ? 16 : 0));
1219
+ sum.w() +=
1220
+ y2[l + 32] * (q4[l + 12] + (qh[l + 0] & (hm2 << 1) ? 16 : 0)) +
1221
+ y2[l + 48] * (q4[l + 14] + (qh[l + 16] & (hm2 << 1) ? 16 : 0));
1222
+ smin += (y1[l] + y1[l+16]) * sc[2] + (y1[l+32] + y1[l+48]) * sc[3]
1223
+ + (y2[l] + y2[l+16]) * sc[6] + (y2[l+32] + y2[l+48]) * sc[7];
1224
+ }
1225
+ tmp += dall * (sum.x() * sc[0] + sum.y() * sc[1] + sum.z() * sc[4] +
1226
+ sum.w() * sc[5]) -
1227
+ dmin * smin;
1228
+ }
1229
+ }
1230
+ #else
1231
+ // The reordered Q5_K layout is only produced for QK_K == 256.
1232
+ #endif
1233
+
1234
+ // sum up partial sums and write back result
1235
+ #pragma unroll
1236
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
1237
+ tmp +=
1238
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
1239
+ }
1240
+
1241
+ if (item_ct1.get_local_id(2) == 0) {
1242
+ dst[row] = tmp;
1243
+ }
1244
+ }
1245
+
1246
+ static void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows,
1247
+ const sycl::nd_item<3> &item_ct1) {
1248
+
1249
+ static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");
1250
+
1251
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
1252
+ item_ct1.get_local_id(1);
1253
+ if (row >= nrows) return;
1254
+
1255
+ const int num_blocks_per_row = ncols / QK_K;
1256
+ const int ib0 = row*num_blocks_per_row;
1257
+
1258
+ const block_q6_K * x = (const block_q6_K *)vx + ib0;
1259
+
1260
+ #if QK_K == 256
1261
+
1262
+ const int tid =
1263
+ item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15
1264
+ const int ix =
1265
+ item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0, 1
1266
+
1267
+ const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8
1268
+
1269
+ const int in = tid % step; // 0...15 or 0...7
1270
+
1271
+ #if K_QUANTS_PER_ITERATION == 1
1272
+ const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15
1273
+ const int is = 0;
1274
+ #else
1275
+ const int l0 = 4 * in; // 0, 4, 8, ..., 28
1276
+ const int is = in / 4;
1277
+ #endif
1278
+
1279
+ float tmp = 0; // partial sum for thread in warp
1280
+
1281
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
1282
+
1283
+ const float d = x[i].d;
1284
+
1285
+ for (int im = 0; im < 2; ++im) {
1286
+ const int ql_offset = 64*im + l0;
1287
+ const int qh_offset = 32*im + l0;
1288
+ const int s_offset = 8*im + is;
1289
+ const int y_offset = 128*im + l0;
1290
+
1291
+ const float * y = yy + i * QK_K + y_offset;
1292
+ const uint8_t * ql = x[i].ql + ql_offset;
1293
+ const uint8_t * qh = x[i].qh + qh_offset;
1294
+ const int8_t * s = x[i].scales + s_offset;
1295
+
1296
+ #if K_QUANTS_PER_ITERATION == 1
1297
+ float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32)
1298
+ + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32)
1299
+ + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32)
1300
+ + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32)
1301
+ + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32)
1302
+ + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32)
1303
+ + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32)
1304
+ +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32);
1305
+ tmp += sum;
1306
+ #else
1307
+ float sum = 0;
1308
+ for (int l = 0; l < 4; ++l) {
1309
+ sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32)
1310
+ + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32)
1311
+ + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32)
1312
+ + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32);
1313
+ }
1314
+ tmp += sum;
1315
+ #endif
1316
+ }
1317
+
1318
+ }
1319
+
1320
+ #else
1321
+
1322
+ const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...7
1323
+ const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); // 0...3
1324
+
1325
+ const int step = tid * K_QUANTS_PER_ITERATION;
1326
+
1327
+ float tmp = 0; // partial sum for thread in warp
1328
+
1329
+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {
1330
+
1331
+ const float * y = yy + i * QK_K + step;
1332
+ const uint8_t * ql = x[i].ql + step;
1333
+ const uint8_t * qh = x[i].qh + step;
1334
+ const int8_t * s = x[i].scales;
1335
+
1336
+ const float d = x[i+0].d;
1337
+
1338
+ float sum = 0;
1339
+ for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {
1340
+ sum += y[j+ 0] * s[0] * d * ((int8_t)((ql[j+ 0] & 0xF) | ((qh[j] & 0x03) << 4)) - 32)
1341
+ + y[j+16] * s[1] * d * ((int8_t)((ql[j+16] & 0xF) | ((qh[j] & 0x0c) << 2)) - 32)
1342
+ + y[j+32] * s[2] * d * ((int8_t)((ql[j+ 0] >> 4) | ((qh[j] & 0x30) >> 0)) - 32)
1343
+ + y[j+48] * s[3] * d * ((int8_t)((ql[j+16] >> 4) | ((qh[j] & 0xc0) >> 2)) - 32);
1344
+ }
1345
+ tmp += sum;
1346
+
1347
+ }
1348
+
1349
+ #endif
1350
+
1351
+ // sum up partial sums and write back result
1352
+ #pragma unroll
1353
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
1354
+ tmp +=
1355
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
1356
+ }
1357
+
1358
+ if (tid == 0) {
1359
+ dst[row] = tmp;
1360
+ }
1361
+ }
1362
+
1363
+ static void dequantize_mul_mat_vec_q6_k_reorder(const void * __restrict__ vx, const float * __restrict__ yy, float * __restrict__ dst, const int ncols, int nrows,
1364
+ const sycl::nd_item<3> &item_ct1) {
1365
+
1366
+ static_assert(16%K_QUANTS_PER_ITERATION == 0, "16 must be divisible by K_QUANTS_PER_ITERATION");
1367
+
1368
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
1369
+ item_ct1.get_local_id(1);
1370
+ if (row >= nrows) return;
1371
+
1372
+ const int num_blocks_per_row = ncols / QK_K;
1373
+ const int ib0 = row*num_blocks_per_row;
1374
+
1375
+ // SOA base pointers for the reordered layout:
1376
+ // [ql: nb * QK_K/2] [qh: nb * QK_K/4] [scales: nb * QK_K/16] [d: nb * sizeof(half)]
1377
+ const int nb = nrows * num_blocks_per_row;
1378
+ const uint8_t * ql_base = (const uint8_t *)vx;
1379
+ const uint8_t * qh_base = ql_base + (size_t)nb * (QK_K / 2);
1380
+ const int8_t * scales_base = (const int8_t *)(qh_base + (size_t)nb * (QK_K / 4));
1381
+ const sycl::half * d_base = (const sycl::half *)((const uint8_t *)scales_base + (size_t)nb * (QK_K / 16));
1382
+
1383
+ #if QK_K == 256
1384
+
1385
+ const int tid =
1386
+ item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...7 or 0...15
1387
+ const int ix =
1388
+ item_ct1.get_local_id(2) % K_QUANTS_PER_ITERATION; // 0 or 0, 1
1389
+
1390
+ const int step = 16/K_QUANTS_PER_ITERATION; // 16 or 8
1391
+
1392
+ const int in = tid % step; // 0...15 or 0...7
1393
+
1394
+ #if K_QUANTS_PER_ITERATION == 1
1395
+ const int l0 = K_QUANTS_PER_ITERATION*in; // 0...15
1396
+ const int is = 0;
1397
+ #else
1398
+ const int l0 = 4 * in; // 0, 4, 8, ..., 28
1399
+ const int is = in / 4;
1400
+ #endif
1401
+
1402
+ float tmp = 0; // partial sum for thread in warp
1403
+
1404
+ for (int i = ix; i < num_blocks_per_row; i += K_QUANTS_PER_ITERATION) {
1405
+ const int bi = ib0 + i;
1406
+
1407
+ const float d = d_base[bi];
1408
+
1409
+ for (int im = 0; im < 2; ++im) {
1410
+ const int ql_offset = 64*im + l0;
1411
+ const int qh_offset = 32*im + l0;
1412
+ const int s_offset = 8*im + is;
1413
+ const int y_offset = 128*im + l0;
1414
+
1415
+ const float * y = yy + i * QK_K + y_offset;
1416
+ const uint8_t * ql = ql_base + bi * (QK_K / 2) + ql_offset;
1417
+ const uint8_t * qh = qh_base + bi * (QK_K / 4) + qh_offset;
1418
+ const int8_t * s = scales_base + bi * (QK_K / 16) + s_offset;
1419
+
1420
+ #if K_QUANTS_PER_ITERATION == 1
1421
+ float sum = y[ 0] * s[0] * d * ((int8_t)((ql[ 0] & 0xF) | ((qh[ 0] & 0x03) << 4)) - 32)
1422
+ + y[16] * s[1] * d * ((int8_t)((ql[16] & 0xF) | ((qh[16] & 0x03) << 4)) - 32)
1423
+ + y[32] * s[2] * d * ((int8_t)((ql[32] & 0xF) | ((qh[ 0] & 0x0c) << 2)) - 32)
1424
+ + y[48] * s[3] * d * ((int8_t)((ql[48] & 0xF) | ((qh[16] & 0x0c) << 2)) - 32)
1425
+ + y[64] * s[4] * d * ((int8_t)((ql[ 0] >> 4) | ((qh[ 0] & 0x30) >> 0)) - 32)
1426
+ + y[80] * s[5] * d * ((int8_t)((ql[16] >> 4) | ((qh[16] & 0x30) >> 0)) - 32)
1427
+ + y[96] * s[6] * d * ((int8_t)((ql[32] >> 4) | ((qh[ 0] & 0xc0) >> 2)) - 32)
1428
+ +y[112] * s[7] * d * ((int8_t)((ql[48] >> 4) | ((qh[16] & 0xc0) >> 2)) - 32);
1429
+ tmp += sum;
1430
+ #else
1431
+ float sum = 0;
1432
+ for (int l = 0; l < 4; ++l) {
1433
+ sum += y[l+ 0] * s[0] * d * ((int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32)
1434
+ + y[l+32] * s[2] * d * ((int8_t)((ql[l+32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32)
1435
+ + y[l+64] * s[4] * d * ((int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32)
1436
+ + y[l+96] * s[6] * d * ((int8_t)((ql[l+32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32);
1437
+ }
1438
+ tmp += sum;
1439
+ #endif
1440
+ }
1441
+
1442
+ }
1443
+
1444
+ #else
1445
+
1446
+ const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...7
1447
+ const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); // 0...3
1448
+
1449
+ const int step = tid * K_QUANTS_PER_ITERATION;
1450
+
1451
+ float tmp = 0; // partial sum for thread in warp
1452
+
1453
+ for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) {
1454
+ const int bi = ib0 + i;
1455
+
1456
+ const float * y = yy + i * QK_K + step;
1457
+ const uint8_t * ql = ql_base + bi * (QK_K / 2) + step;
1458
+ const uint8_t * qh = qh_base + bi * (QK_K / 4) + step;
1459
+ const int8_t * s = scales_base + bi * (QK_K / 16);
1460
+
1461
+ const float d = d_base[bi];
1462
+
1463
+ float sum = 0;
1464
+ for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) {
1465
+ sum += y[j+ 0] * s[0] * d * ((int8_t)((ql[j+ 0] & 0xF) | ((qh[j] & 0x03) << 4)) - 32)
1466
+ + y[j+16] * s[1] * d * ((int8_t)((ql[j+16] & 0xF) | ((qh[j] & 0x0c) << 2)) - 32)
1467
+ + y[j+32] * s[2] * d * ((int8_t)((ql[j+ 0] >> 4) | ((qh[j] & 0x30) >> 0)) - 32)
1468
+ + y[j+48] * s[3] * d * ((int8_t)((ql[j+16] >> 4) | ((qh[j] & 0xc0) >> 2)) - 32);
1469
+ }
1470
+ tmp += sum;
1471
+
1472
+ }
1473
+
1474
+ #endif
1475
+
1476
+ // sum up partial sums and write back result
1477
+ #pragma unroll
1478
+ for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
1479
+ tmp +=
1480
+ dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
1481
+ }
1482
+
1483
+ if (tid == 0) {
1484
+ dst[row] = tmp;
1485
+ }
1486
+ }
1487
+
1488
+ static void dequantize_mul_mat_vec_q4_0_sycl_reorder(const void *vx, const dfloat *y,
1489
+ float *dst, const int ncols,
1490
+ const int nrows,
1491
+ dpct::queue_ptr stream) {
1492
+ GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
1493
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1494
+ // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
1495
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1496
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1497
+ {
1498
+ dpct::has_capability_or_fail(stream->get_device(),
1499
+ {sycl::aspect::fp16});
1500
+
1501
+ stream->parallel_for(
1502
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1503
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1504
+ dequantize_mul_mat_vec_reorder<QK4_0, QR4_0, dequantize_q4_0_reorder>(
1505
+ vx, y, dst, ncols, nrows, item_ct1);
1506
+ });
1507
+ }
1508
+ }
1509
+
1510
+
1511
+ static void dequantize_mul_mat_vec_q4_0_sycl(const void *vx, const dfloat *y,
1512
+ float *dst, const int ncols,
1513
+ const int nrows,
1514
+ dpct::queue_ptr stream) {
1515
+ GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
1516
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1517
+ // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
1518
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1519
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1520
+ {
1521
+ dpct::has_capability_or_fail(stream->get_device(),
1522
+ {sycl::aspect::fp16});
1523
+
1524
+ stream->parallel_for(
1525
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1526
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1527
+ dequantize_mul_mat_vec<QK4_0, QR4_0, dequantize_q4_0>(
1528
+ vx, y, dst, ncols, nrows, item_ct1);
1529
+ });
1530
+ }
1531
+ }
1532
+
1533
+ static void dequantize_mul_mat_vec_q1_0_sycl_reorder(const void *vx, const dfloat *y,
1534
+ float *dst, const int ncols,
1535
+ const int nrows,
1536
+ dpct::queue_ptr stream) {
1537
+ GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
1538
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1539
+ // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
1540
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1541
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1542
+ {
1543
+ dpct::has_capability_or_fail(stream->get_device(),
1544
+ {sycl::aspect::fp16});
1545
+
1546
+ stream->parallel_for(
1547
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1548
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1549
+ dequantize_mul_mat_vec_reorder<QK1_0, QR1_0, dequantize_q1_0_reorder>(
1550
+ vx, y, dst, ncols, nrows, item_ct1);
1551
+ });
1552
+ }
1553
+ }
1554
+
1555
+ static void dequantize_mul_mat_vec_q1_0_sycl(const void *vx, const dfloat *y,
1556
+ float *dst, const int ncols,
1557
+ const int nrows,
1558
+ dpct::queue_ptr stream) {
1559
+ GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
1560
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1561
+ // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead
1562
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1563
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1564
+ {
1565
+ dpct::has_capability_or_fail(stream->get_device(),
1566
+ {sycl::aspect::fp16});
1567
+
1568
+ stream->parallel_for(
1569
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1570
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1571
+ dequantize_mul_mat_vec<QK1_0, QR1_0, dequantize_q1_0>(
1572
+ vx, y, dst, ncols, nrows, item_ct1);
1573
+ });
1574
+ }
1575
+ }
1576
+
1577
+ static void dequantize_mul_mat_vec_q4_1_sycl(const void *vx, const dfloat *y,
1578
+ float *dst, const int ncols,
1579
+ const int nrows,
1580
+ dpct::queue_ptr stream) {
1581
+ GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
1582
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1583
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1584
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1585
+ {
1586
+ dpct::has_capability_or_fail(stream->get_device(),
1587
+ {sycl::aspect::fp16});
1588
+
1589
+ stream->parallel_for(
1590
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1591
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1592
+ dequantize_mul_mat_vec<QK4_1, QR4_1, dequantize_q4_1>(
1593
+ vx, y, dst, ncols, nrows, item_ct1);
1594
+ });
1595
+ }
1596
+ }
1597
+
1598
+ static void dequantize_mul_mat_vec_q5_0_sycl(const void *vx, const dfloat *y,
1599
+ float *dst, const int ncols,
1600
+ const int nrows,
1601
+ dpct::queue_ptr stream) {
1602
+ GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
1603
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1604
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1605
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1606
+ {
1607
+ dpct::has_capability_or_fail(stream->get_device(),
1608
+ {sycl::aspect::fp16});
1609
+
1610
+ stream->parallel_for(
1611
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1612
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1613
+ dequantize_mul_mat_vec<QK5_0, QR5_0, dequantize_q5_0>(
1614
+ vx, y, dst, ncols, nrows, item_ct1);
1615
+ });
1616
+ }
1617
+ }
1618
+
1619
+ static void dequantize_mul_mat_vec_q5_1_sycl(const void *vx, const dfloat *y,
1620
+ float *dst, const int ncols,
1621
+ const int nrows,
1622
+ dpct::queue_ptr stream) {
1623
+ GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
1624
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1625
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1626
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1627
+ {
1628
+ dpct::has_capability_or_fail(stream->get_device(),
1629
+ {sycl::aspect::fp16});
1630
+
1631
+ stream->parallel_for(
1632
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1633
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1634
+ dequantize_mul_mat_vec<QK5_1, QR5_1, dequantize_q5_1>(
1635
+ vx, y, dst, ncols, nrows, item_ct1);
1636
+ });
1637
+ }
1638
+ }
1639
+
1640
+ static void dequantize_mul_mat_vec_q8_0_sycl_reorder(const void *vx, const dfloat *y,
1641
+ float *dst, const int ncols,
1642
+ const int nrows,
1643
+ dpct::queue_ptr stream) {
1644
+ GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
1645
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1646
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1647
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1648
+ {
1649
+ dpct::has_capability_or_fail(stream->get_device(),
1650
+ {sycl::aspect::fp16});
1651
+
1652
+ stream->parallel_for(
1653
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1654
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1655
+ // Q8_0 reorder layout: [all qs (ncols*nrows bytes)][all d values]
1656
+ // Cannot reuse dequantize_mul_mat_vec_reorder template because it has
1657
+ // Q4_0-specific constants hardcoded (d_ptr offset and qs stride).
1658
+ const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
1659
+ item_ct1.get_local_id(1);
1660
+ if (row >= nrows) return;
1661
+
1662
+ const int tid = item_ct1.get_local_id(2);
1663
+ const int iter_stride = 8*2*GGML_SYCL_DMMV_X;
1664
+ const int vals_per_iter = iter_stride / WARP_SIZE;
1665
+ const int ncols_left = ncols % (QK8_0*WARP_SIZE);
1666
+ const int ncols_align = ncols - ncols_left;
1667
+
1668
+ #ifdef GGML_SYCL_F16
1669
+ sycl::half2 tmp = {0.0f, 0.0f};
1670
+ #else
1671
+ float tmp = 0.0f;
1672
+ #endif
1673
+ const char *d_ptr = (const char*)vx + ncols*nrows; // d after all qs
1674
+
1675
+ int i = 0;
1676
+ for (i = 0; i < ncols_align; i += iter_stride) {
1677
+ const int col = i + vals_per_iter*tid;
1678
+ const int ib = (row*ncols + col)/QK8_0;
1679
+ const int iqs = col % QK8_0;
1680
+
1681
+ #pragma unroll
1682
+ for (int j = 0; j < vals_per_iter; j += 2) {
1683
+ dfloat2 v;
1684
+ dequantize_q8_0_reorder((const void *)d_ptr, ib, (const void *)vx,
1685
+ ib * QK8_0 + iqs + j, v);
1686
+
1687
+ #ifdef GGML_SYCL_F16
1688
+ dfloat2 t1{y[col + j + 0], y[col + j + 1]};
1689
+ tmp += v * t1;
1690
+ #else
1691
+ tmp += v.x() * y[col + j + 0];
1692
+ tmp += v.y() * y[col + j + 1];
1693
+ #endif
1694
+ }
1695
+ }
1696
+
1697
+ // handle remaining columns
1698
+ for (; i < ncols; i += iter_stride) {
1699
+ if (tid >= ncols_left/QK8_0) continue;
1700
+ const int col = i + vals_per_iter*tid;
1701
+ const int ib = (row*ncols + col)/QK8_0;
1702
+ const int iqs = col % QK8_0;
1703
+
1704
+ #pragma unroll
1705
+ for (int j = 0; j < vals_per_iter; j += 2) {
1706
+ dfloat2 v;
1707
+ dequantize_q8_0_reorder((const void *)d_ptr, ib, (const void *)vx,
1708
+ ib * QK8_0 + iqs + j, v);
1709
+
1710
+ #ifdef GGML_SYCL_F16
1711
+ dfloat2 t1{y[col + j + 0], y[col + j + 1]};
1712
+ tmp += v * t1;
1713
+ #else
1714
+ tmp += v.x() * y[col + j + 0];
1715
+ tmp += v.y() * y[col + j + 1];
1716
+ #endif
1717
+ }
1718
+ }
1719
+
1720
+ // reduce
1721
+ const int mask_start = ncols > GGML_SYCL_DMMV_X ? WARP_SIZE >> 1 : WARP_SIZE >> 2;
1722
+ for (int mask = mask_start; mask > 0; mask >>= 1) {
1723
+ tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
1724
+ }
1725
+
1726
+ if (tid == 0) {
1727
+ #ifdef GGML_SYCL_F16
1728
+ dst[row] = tmp.x() + tmp.y();
1729
+ #else
1730
+ dst[row] = tmp;
1731
+ #endif
1732
+ }
1733
+ });
1734
+ }
1735
+ }
1736
+
1737
+ static void dequantize_mul_mat_vec_q8_0_sycl(const void *vx, const dfloat *y,
1738
+ float *dst, const int ncols,
1739
+ const int nrows,
1740
+ dpct::queue_ptr stream) {
1741
+ GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0);
1742
+ const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
1743
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1744
+ const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
1745
+ {
1746
+ dpct::has_capability_or_fail(stream->get_device(),
1747
+ {sycl::aspect::fp16});
1748
+
1749
+ stream->parallel_for(
1750
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1751
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1752
+ dequantize_mul_mat_vec<QK8_0, QR8_0, dequantize_q8_0>(
1753
+ vx, y, dst, ncols, nrows, item_ct1);
1754
+ });
1755
+ }
1756
+ }
1757
+
1758
+ static void dequantize_mul_mat_vec_q2_K_sycl(const void *vx, const float *y,
1759
+ float *dst, const int ncols,
1760
+ const int nrows,
1761
+ dpct::queue_ptr stream) {
1762
+ GGML_ASSERT(ncols % QK_K == 0);
1763
+ const int ny = 2; // very slightly faster than 1 even when K_QUANTS_PER_ITERATION = 2
1764
+ const int block_num_y = (nrows + ny - 1) / ny;
1765
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1766
+ const sycl::range<3> block_dims(1, ny, WARP_SIZE);
1767
+ stream->parallel_for(
1768
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1769
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1770
+ dequantize_mul_mat_vec_q2_k(vx, y, dst, ncols, nrows, item_ct1);
1771
+ });
1772
+ }
1773
+
1774
+ static void dequantize_mul_mat_vec_q2_K_sycl_reorder(const void *vx, const float *y,
1775
+ float *dst, const int ncols,
1776
+ const int nrows,
1777
+ dpct::queue_ptr stream) {
1778
+ GGML_ASSERT(ncols % QK_K == 0);
1779
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
1780
+ const int block_num_y = (nrows + ny - 1) / ny;
1781
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1782
+ const sycl::range<3> block_dims(1, ny, WARP_SIZE);
1783
+ stream->parallel_for(
1784
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1785
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1786
+ dequantize_mul_mat_vec_q2_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
1787
+ });
1788
+ }
1789
+
1790
+ static void dequantize_mul_mat_vec_q3_K_sycl(const void *vx, const float *y,
1791
+ float *dst, const int ncols,
1792
+ const int nrows,
1793
+ dpct::queue_ptr stream) {
1794
+ GGML_ASSERT(ncols % QK_K == 0);
1795
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
1796
+ const int block_num_y = (nrows + ny - 1) / ny;
1797
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1798
+ const sycl::range<3> block_dims(1, ny, WARP_SIZE);
1799
+ stream->parallel_for(
1800
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1801
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1802
+ dequantize_mul_mat_vec_q3_k(vx, y, dst, ncols, nrows, item_ct1);
1803
+ });
1804
+ }
1805
+
1806
+ static void dequantize_mul_mat_vec_q3_K_sycl_reorder(const void *vx, const float *y,
1807
+ float *dst, const int ncols,
1808
+ const int nrows,
1809
+ dpct::queue_ptr stream) {
1810
+ GGML_ASSERT(ncols % QK_K == 0);
1811
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
1812
+ const int block_num_y = (nrows + ny - 1) / ny;
1813
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1814
+ const sycl::range<3> block_dims(1, ny, WARP_SIZE);
1815
+ stream->parallel_for(
1816
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1817
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1818
+ dequantize_mul_mat_vec_q3_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
1819
+ });
1820
+ }
1821
+
1822
+ static void dequantize_mul_mat_vec_q4_K_sycl(const void *vx, const float *y,
1823
+ float *dst, const int ncols,
1824
+ const int nrows,
1825
+ dpct::queue_ptr stream) {
1826
+ GGML_ASSERT(ncols % QK_K == 0);
1827
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
1828
+ const int block_num_y = (nrows + ny - 1) / ny;
1829
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1830
+ const sycl::range<3> block_dims(1, ny, WARP_SIZE);
1831
+ stream->parallel_for(
1832
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1833
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1834
+ dequantize_mul_mat_vec_q4_k(vx, y, dst, ncols, nrows, item_ct1);
1835
+ });
1836
+ }
1837
+
1838
+ static void dequantize_mul_mat_vec_q5_K_sycl(const void *vx, const float *y,
1839
+ float *dst, const int ncols,
1840
+ const int nrows,
1841
+ dpct::queue_ptr stream) {
1842
+ GGML_ASSERT(ncols % QK_K == 0);
1843
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
1844
+ const int block_num_y = (nrows + ny - 1) / ny;
1845
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1846
+ const sycl::range<3> block_dims(1, ny, WARP_SIZE);
1847
+ stream->parallel_for(
1848
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1849
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1850
+ dequantize_mul_mat_vec_q5_k(vx, y, dst, ncols, nrows, item_ct1);
1851
+ });
1852
+ }
1853
+
1854
+ static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y,
1855
+ float *dst, const int ncols,
1856
+ const int nrows,
1857
+ dpct::queue_ptr stream) {
1858
+ GGML_ASSERT(ncols % QK_K == 0);
1859
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
1860
+ const int block_num_y = (nrows + ny - 1) / ny;
1861
+ const sycl::range<3> block_nums(1, 1, block_num_y);
1862
+ const sycl::range<3> block_dims(1, ny, WARP_SIZE);
1863
+ stream->parallel_for(
1864
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
1865
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
1866
+ dequantize_mul_mat_vec_q6_k(vx, y, dst, ncols, nrows, item_ct1);
1867
+ });
1868
+ }
1869
+
1870
+ #ifdef GGML_SYCL_DMMV_HAS_ESIMD
1871
+ using ggml_sycl_esimd::GGML_SYCL_DMMV_ESIMD_WG_SIZE;
1872
+
1873
+ // generic reordered dequantize-matvec: each work-group owns a pair of
1874
+ // consecutive output rows and updates one 32-wide accumulator per row
1875
+ template <ggml_type T>
1876
+ ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd(
1877
+ const void * vx, const float * y, float * dst,
1878
+ const int ncols, const int nrows,
1879
+ sycl::local_accessor<float, 1> lmem,
1880
+ const sycl::nd_item<1> & it) {
1881
+ using namespace sycl::ext::intel::esimd;
1882
+ using traits = ggml_sycl_esimd::esimd_reorder_q_traits<T>;
1883
+
1884
+ const int num_blocks_per_row = ncols / QK_K;
1885
+ const size_t nb = (size_t) nrows * num_blocks_per_row;
1886
+ const auto ps = traits::make_ptrs(vx, nb);
1887
+
1888
+ const int tid = it.get_local_id(0);
1889
+ const int row_pair = it.get_group(0);
1890
+ const int row0 = row_pair * 2; // two consecutive output rows
1891
+ const bool has_row1 = row0 + 1 < nrows;
1892
+
1893
+ // one 32-wide accumulator per output row (small footprint, no spill)
1894
+ simd<float, 32> acc0 = 0.0f;
1895
+ simd<float, 32> acc1 = 0.0f;
1896
+
1897
+ for (int ib = tid; ib < num_blocks_per_row; ib += GGML_SYCL_DMMV_ESIMD_WG_SIZE) {
1898
+ simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) ib * QK_K);
1899
+
1900
+ const size_t bi0 = (size_t) (row0 + 0) * num_blocks_per_row + ib;
1901
+ const size_t bi1 = (size_t) (row0 + 1) * num_blocks_per_row + ib;
1902
+
1903
+ traits::mac_pair(ps, bi0, ps, bi1, has_row1, y_vec, acc0, acc1);
1904
+ }
1905
+
1906
+ lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{});
1907
+ lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{});
1908
+ it.barrier(sycl::access::fence_space::local_space);
1909
+
1910
+ if (tid == 0) {
1911
+ float sum0 = 0.0f;
1912
+ float sum1 = 0.0f;
1913
+ for (int p = 0; p < GGML_SYCL_DMMV_ESIMD_WG_SIZE; ++p) {
1914
+ sum0 += lmem[p * 2 + 0];
1915
+ sum1 += lmem[p * 2 + 1];
1916
+ }
1917
+ dst[row0 + 0] = sum0;
1918
+ if (has_row1) {
1919
+ dst[row0 + 1] = sum1;
1920
+ }
1921
+ }
1922
+ }
1923
+
1924
+ static void dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(const void *vx, const float *y,
1925
+ float *dst, const int ncols,
1926
+ const int nrows,
1927
+ dpct::queue_ptr stream) {
1928
+ GGML_ASSERT(ncols % QK_K == 0);
1929
+ const int workgroups = (nrows + 1) / 2;
1930
+ stream->submit([&](sycl::handler &h) {
1931
+ sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
1932
+ h.parallel_for(
1933
+ sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
1934
+ [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
1935
+ dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q2_K>(
1936
+ vx, y, dst, ncols, nrows, lmem, it);
1937
+ });
1938
+ });
1939
+ }
1940
+
1941
+ static void dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(const void *vx, const float *y,
1942
+ float *dst, const int ncols,
1943
+ const int nrows,
1944
+ dpct::queue_ptr stream) {
1945
+ GGML_ASSERT(ncols % QK_K == 0);
1946
+ const int workgroups = (nrows + 1) / 2;
1947
+ stream->submit([&](sycl::handler &h) {
1948
+ sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
1949
+ h.parallel_for(
1950
+ sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
1951
+ [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
1952
+ dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q3_K>(
1953
+ vx, y, dst, ncols, nrows, lmem, it);
1954
+ });
1955
+ });
1956
+ }
1957
+
1958
+ static void dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(const void *vx, const float *y,
1959
+ float *dst, const int ncols,
1960
+ const int nrows,
1961
+ dpct::queue_ptr stream) {
1962
+ GGML_ASSERT(ncols % QK_K == 0);
1963
+ const int workgroups = (nrows + 1) / 2;
1964
+ stream->submit([&](sycl::handler &h) {
1965
+ sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
1966
+ h.parallel_for(
1967
+ sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
1968
+ [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
1969
+ dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q4_K>(
1970
+ vx, y, dst, ncols, nrows, lmem, it);
1971
+ });
1972
+ });
1973
+ }
1974
+
1975
+ static void dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(const void *vx, const float *y,
1976
+ float *dst, const int ncols,
1977
+ const int nrows,
1978
+ dpct::queue_ptr stream) {
1979
+ GGML_ASSERT(ncols % QK_K == 0);
1980
+ const int workgroups = (nrows + 1) / 2;
1981
+ stream->submit([&](sycl::handler &h) {
1982
+ sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
1983
+ h.parallel_for(
1984
+ sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
1985
+ [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
1986
+ dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q5_K>(
1987
+ vx, y, dst, ncols, nrows, lmem, it);
1988
+ });
1989
+ });
1990
+ }
1991
+
1992
+ static void dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(const void *vx, const float *y,
1993
+ float *dst, const int ncols,
1994
+ const int nrows,
1995
+ dpct::queue_ptr stream) {
1996
+ GGML_ASSERT(ncols % QK_K == 0);
1997
+ const int workgroups = (nrows + 1) / 2;
1998
+ stream->submit([&](sycl::handler &h) {
1999
+ sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
2000
+ h.parallel_for(
2001
+ sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
2002
+ [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
2003
+ dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q6_K>(
2004
+ vx, y, dst, ncols, nrows, lmem, it);
2005
+ });
2006
+ });
2007
+ }
2008
+
2009
+ #endif // GGML_SYCL_DMMV_HAS_ESIMD
2010
+
2011
+ static void dequantize_mul_mat_vec_q4_K_sycl_reorder(const void *vx, const float *y,
2012
+ float *dst, const int ncols,
2013
+ const int nrows,
2014
+ dpct::queue_ptr stream) {
2015
+ GGML_ASSERT(ncols % QK_K == 0);
2016
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
2017
+ const int block_num_y = (nrows + ny - 1) / ny;
2018
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2019
+ const sycl::range<3> block_dims(1, ny, WARP_SIZE);
2020
+ stream->parallel_for(
2021
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2022
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2023
+ dequantize_mul_mat_vec_q4_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
2024
+ });
2025
+ }
2026
+
2027
+ static void dequantize_mul_mat_vec_q5_K_sycl_reorder(const void *vx, const float *y,
2028
+ float *dst, const int ncols,
2029
+ const int nrows,
2030
+ dpct::queue_ptr stream) {
2031
+ GGML_ASSERT(ncols % QK_K == 0);
2032
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
2033
+ const int block_num_y = (nrows + ny - 1) / ny;
2034
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2035
+ const sycl::range<3> block_dims(1, ny, WARP_SIZE);
2036
+ stream->parallel_for(
2037
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2038
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2039
+ dequantize_mul_mat_vec_q5_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
2040
+ });
2041
+ }
2042
+
2043
+ static void dequantize_mul_mat_vec_q6_K_sycl_reorder(const void *vx, const float *y,
2044
+ float *dst, const int ncols,
2045
+ const int nrows,
2046
+ dpct::queue_ptr stream) {
2047
+ GGML_ASSERT(ncols % QK_K == 0);
2048
+ const int ny = 2 / K_QUANTS_PER_ITERATION;
2049
+ const int block_num_y = (nrows + ny - 1) / ny;
2050
+ const sycl::range<3> block_nums(1, 1, block_num_y);
2051
+ const sycl::range<3> block_dims(1, ny, WARP_SIZE);
2052
+ stream->parallel_for(
2053
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
2054
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
2055
+ dequantize_mul_mat_vec_q6_k_reorder(vx, y, dst, ncols, nrows, item_ct1);
2056
+ });
2057
+ }
2058
+
2059
+ void ggml_sycl_op_dequantize_mul_mat_vec(
2060
+ ggml_backend_sycl_context & ctx,
2061
+ const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst,
2062
+ const char *src0_dd_i, const float *src1_ddf_i, const char *src1_ddq_i,
2063
+ float *dst_dd_i, const int64_t row_low, const int64_t row_high,
2064
+ const int64_t src1_ncols, const int64_t src1_padded_row_size,
2065
+ const dpct::queue_ptr &stream) {
2066
+
2067
+ const int64_t ne00 = src0->ne[0];
2068
+ const int64_t row_diff = row_high - row_low;
2069
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
2070
+ // on some GPUs it is faster to convert src1 to half and to use half precision intrinsics
2071
+ #ifdef GGML_SYCL_F16
2072
+ ggml_sycl_pool_alloc<sycl::half> src1_dfloat_a(ctx.pool());
2073
+ sycl::half *src1_dfloat = nullptr; // dfloat == half
2074
+
2075
+ bool src1_convert_f16 =
2076
+ src0->type == GGML_TYPE_Q1_0 ||
2077
+ src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 ||
2078
+ src0->type == GGML_TYPE_Q5_0 || src0->type == GGML_TYPE_Q5_1 ||
2079
+ src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16 ||
2080
+ src0->type == GGML_TYPE_BF16;
2081
+
2082
+ if (src1_convert_f16) {
2083
+ scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2,
2084
+ " : converting src1 to fp16");
2085
+ src1_dfloat = src1_dfloat_a.alloc(ne00);
2086
+ const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src1->type, dst);
2087
+ GGML_ASSERT(to_fp16_sycl != nullptr);
2088
+ to_fp16_sycl(src1_ddf_i, src1_dfloat, ne00, stream);
2089
+ }
2090
+ #else
2091
+ const dfloat * src1_dfloat = (const dfloat *) src1_ddf_i; // dfloat == float, no conversion
2092
+ #endif // GGML_SYCL_F16
2093
+
2094
+ switch (src0->type) {
2095
+ case GGML_TYPE_Q1_0:
2096
+ if ((ggml_tensor_extra_gpu*)dst->src[0]->extra &&
2097
+ ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {
2098
+ dequantize_mul_mat_vec_q1_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
2099
+ } else {
2100
+ dequantize_mul_mat_vec_q1_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
2101
+ }
2102
+ break;
2103
+ case GGML_TYPE_Q4_0:
2104
+ if ((ggml_tensor_extra_gpu*)dst->src[0]->extra &&
2105
+ ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) {
2106
+ dequantize_mul_mat_vec_q4_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
2107
+ } else {
2108
+ dequantize_mul_mat_vec_q4_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
2109
+ }
2110
+ break;
2111
+ case GGML_TYPE_Q4_1:
2112
+ dequantize_mul_mat_vec_q4_1_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
2113
+ break;
2114
+ case GGML_TYPE_Q5_0:
2115
+ dequantize_mul_mat_vec_q5_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
2116
+ break;
2117
+ case GGML_TYPE_Q5_1:
2118
+ dequantize_mul_mat_vec_q5_1_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
2119
+ break;
2120
+ case GGML_TYPE_Q8_0:
2121
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2122
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2123
+ dequantize_mul_mat_vec_q8_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
2124
+ } else {
2125
+ dequantize_mul_mat_vec_q8_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
2126
+ }
2127
+ break;
2128
+ case GGML_TYPE_Q2_K:
2129
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2130
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2131
+ #ifdef GGML_SYCL_DMMV_HAS_ESIMD
2132
+ if (g_ggml_sycl_enable_esimd) {
2133
+ dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2134
+ }
2135
+ else
2136
+ #endif
2137
+ {
2138
+ dequantize_mul_mat_vec_q2_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2139
+ }
2140
+ } else {
2141
+ dequantize_mul_mat_vec_q2_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2142
+ }
2143
+ break;
2144
+ case GGML_TYPE_Q3_K:
2145
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2146
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2147
+ #ifdef GGML_SYCL_DMMV_HAS_ESIMD
2148
+ if (g_ggml_sycl_enable_esimd) {
2149
+ dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2150
+ }
2151
+ else
2152
+ #endif
2153
+ {
2154
+ dequantize_mul_mat_vec_q3_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2155
+ }
2156
+ } else {
2157
+ dequantize_mul_mat_vec_q3_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2158
+ }
2159
+ break;
2160
+ case GGML_TYPE_Q4_K:
2161
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2162
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2163
+ #ifdef GGML_SYCL_DMMV_HAS_ESIMD
2164
+ if (g_ggml_sycl_enable_esimd) {
2165
+ dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2166
+ }
2167
+ else
2168
+ #endif
2169
+ {
2170
+ dequantize_mul_mat_vec_q4_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2171
+ }
2172
+ } else {
2173
+ dequantize_mul_mat_vec_q4_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2174
+ }
2175
+ break;
2176
+ case GGML_TYPE_Q5_K:
2177
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2178
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2179
+ #ifdef GGML_SYCL_DMMV_HAS_ESIMD
2180
+ if (g_ggml_sycl_enable_esimd) {
2181
+ dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2182
+ }
2183
+ else
2184
+ #endif
2185
+ {
2186
+ dequantize_mul_mat_vec_q5_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2187
+ }
2188
+ } else {
2189
+ dequantize_mul_mat_vec_q5_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2190
+ }
2191
+ break;
2192
+ case GGML_TYPE_Q6_K:
2193
+ if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
2194
+ ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
2195
+ #ifdef GGML_SYCL_DMMV_HAS_ESIMD
2196
+ if (g_ggml_sycl_enable_esimd) {
2197
+ dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2198
+ }
2199
+ else
2200
+ #endif
2201
+ {
2202
+ dequantize_mul_mat_vec_q6_K_sycl_reorder(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2203
+ }
2204
+ } else {
2205
+ dequantize_mul_mat_vec_q6_K_sycl(src0_dd_i, src1_ddf_i, dst_dd_i, ne00, row_diff, stream);
2206
+ }
2207
+ break;
2208
+ case GGML_TYPE_F16:
2209
+ convert_mul_mat_vec_f16_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
2210
+ break;
2211
+ #ifdef GGML_SYCL_DMMV_HAS_BF16
2212
+ case GGML_TYPE_BF16:
2213
+ convert_mul_mat_vec_bf16_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream);
2214
+ break;
2215
+ #endif
2216
+ default:
2217
+ printf("ggml_sycl_op_dequantize_mul_mat_vec unsupported GGML_TYPE %d\n", src0->type);
2218
+ GGML_ABORT("fatal error");
2219
+ }
2220
+
2221
+ GGML_UNUSED(src1);
2222
+ GGML_UNUSED(dst);
2223
+ GGML_UNUSED(src1_ddq_i);
2224
+ GGML_UNUSED(src1_ncols);
2225
+ GGML_UNUSED(src1_padded_row_size);
2226
+ GGML_UNUSED(ctx);
2227
+ }