elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,1726 @@
1
+ //
2
+ // MIT license
3
+ // Copyright (C) 2025 Intel Corporation
4
+ // SPDX-License-Identifier: MIT
5
+ //
6
+
7
+ //
8
+ // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
9
+ // See https://llvm.org/LICENSE.txt for license information.
10
+ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
11
+ //
12
+
13
+ #ifndef GGML_SYCL_VECDOTQ_HPP
14
+ #define GGML_SYCL_VECDOTQ_HPP
15
+
16
+ #include "dpct/helper.hpp"
17
+ #include "ggml.h"
18
+ #include "type.hpp"
19
+ #include "quants.hpp"
20
+
21
+ typedef float (*vec_dot_q_sycl_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1,
22
+ const int & iqs);
23
+
24
+ static __dpct_inline__ int get_int_b1(const void * x, const int & i32) {
25
+ const uint8_t * x8 = (const uint8_t *) x;
26
+
27
+ int x32 = x8[4*i32 + 0] << 0;
28
+ x32 |= x8[4*i32 + 1] << 8;
29
+ x32 |= x8[4*i32 + 2] << 16;
30
+ x32 |= x8[4*i32 + 3] << 24;
31
+
32
+ return x32;
33
+ }
34
+
35
+ static __dpct_inline__ int get_int_b2(const void * x, const int & i32) {
36
+ const uint16_t * x16 = (const uint16_t *) x; // assume at least 2 byte alignment
37
+
38
+ int x32 = x16[2*i32 + 0] << 0;
39
+ x32 |= x16[2*i32 + 1] << 16;
40
+
41
+ return x32;
42
+ }
43
+
44
+ static __dpct_inline__ int get_int_b4(const void * x, const int & i32) {
45
+ return ((const int *) x)[i32]; // assume at least 4 byte alignment
46
+ }
47
+
48
+ static __dpct_inline__ int get_int_from_int8(const int8_t* x8, const int& i32) {
49
+ const uint16_t* x16 =
50
+ (const uint16_t*)(x8 + sizeof(int) * i32); // assume at least 2 byte
51
+ // alignment
52
+
53
+ int x32 = 0;
54
+ x32 |= x16[0] << 0;
55
+ x32 |= x16[1] << 16;
56
+
57
+ return x32;
58
+ }
59
+
60
+ static __dpct_inline__ int get_int_from_uint8(
61
+ const uint8_t* x8,
62
+ const int& i32) {
63
+ const uint16_t* x16 =
64
+ (const uint16_t*)(x8 + sizeof(int) * i32); // assume at least 2 byte
65
+ // alignment
66
+
67
+ int x32 = 0;
68
+ x32 |= x16[0] << 0;
69
+ x32 |= x16[1] << 16;
70
+
71
+ return x32;
72
+ }
73
+
74
+ static __dpct_inline__ int get_int_from_int8_aligned(
75
+ const int8_t* x8,
76
+ const int& i32) {
77
+ return *(
78
+ (const int*)(x8 + sizeof(int) * i32)); // assume at least 4 byte alignment
79
+ }
80
+
81
+ static __dpct_inline__ int get_int_from_uint8_aligned(
82
+ const uint8_t* x8,
83
+ const int& i32) {
84
+ return *(
85
+ (const int*)(x8 + sizeof(int) * i32)); // assume at least 4 byte alignment
86
+ }
87
+
88
+ static __dpct_inline__ int byte_sub_4(const int a, const int b) {
89
+ const uint32_t ua = static_cast<uint32_t>(a);
90
+ const uint32_t ub = static_cast<uint32_t>(b);
91
+ return static_cast<int>(((ua | 0x80808080u) - ub) ^ 0x80808080u);
92
+ }
93
+
94
+ static __dpct_inline__ float vec_dot_q6_K_q8_1_impl_mmvq_scalar(
95
+ const int vl, const int vh, const int u0, const int u1, const int8_t sc0,
96
+ const int8_t sc1, const float d, const float d80, const float d81) {
97
+ static_assert(QR6_K == 2, "q6_K MMVQ scalar fast path assumes QR6_K == 2");
98
+
99
+ const int vil0 = (vl >> 0) & 0x0F0F0F0F;
100
+ const int vih0 = ((vh >> 0) << 4) & 0x30303030;
101
+ const int vi0 = byte_sub_4(vil0 | vih0, 0x20202020);
102
+
103
+ const int vil1 = (vl >> 4) & 0x0F0F0F0F;
104
+ const int vih1 = ((vh >> 4) << 4) & 0x30303030;
105
+ const int vi1 = byte_sub_4(vil1 | vih1, 0x20202020);
106
+
107
+ const float sumf =
108
+ d80 * (dpct::dp4a(vi0, u0, 0) * sc0) +
109
+ d81 * (dpct::dp4a(vi1, u1, 0) * sc1);
110
+
111
+ return d * sumf;
112
+ }
113
+
114
+ static __dpct_inline__ void get_int_from_table_16(const uint32_t &q4,
115
+ const uint8_t *values,
116
+ int &val1, int &val2) {
117
+
118
+ uint32_t aux32; const uint8_t * q8 = (const uint8_t *)&aux32;
119
+ aux32 = q4 & 0x0f0f0f0f;
120
+ uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8);
121
+ uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8);
122
+ val1 = v1 | (v2 << 16);
123
+ aux32 = (q4 >> 4) & 0x0f0f0f0f;
124
+ v1 = values[q8[0]] | (values[q8[1]] << 8);
125
+ v2 = values[q8[2]] | (values[q8[3]] << 8);
126
+ val2 = v1 | (v2 << 16);
127
+ }
128
+
129
+ static __dpct_inline__ sycl::int2 get_int_from_table_16(
130
+ const int& q4, const int8_t* table) {
131
+ const uint32_t* table32 = (const uint32_t*)table;
132
+ uint32_t tmp[2];
133
+ const uint32_t low_high_selection_indices =
134
+ (0x32103210 | ((q4 & 0x88888888) >> 1));
135
+ #pragma unroll
136
+ for (uint32_t i = 0; i < 2; ++i) {
137
+ const uint32_t shift = 16 * i;
138
+
139
+ const uint32_t low =
140
+ dpct::byte_level_permute(table32[0], table32[1], q4 >> shift);
141
+ const uint32_t high =
142
+ dpct::byte_level_permute(table32[2], table32[3], q4 >> shift);
143
+ tmp[i] = dpct::byte_level_permute(
144
+ low, high, low_high_selection_indices >> shift);
145
+ }
146
+ return sycl::int2(
147
+ dpct::byte_level_permute(tmp[0], tmp[1], 0x6420),
148
+ dpct::byte_level_permute(tmp[0], tmp[1], 0x7531));
149
+ }
150
+
151
+ #define VDR_Q2_K_Q8_1_MMVQ 1
152
+
153
+ // contiguous v/x values
154
+ static __dpct_inline__ float vec_dot_q2_K_q8_1_impl_mmvq(
155
+ const int &v, const int *__restrict__ u, const uint8_t *__restrict__ scales,
156
+ const sycl::half2 &dm2, const float *__restrict__ d8) {
157
+
158
+ float sumf_d = 0.0f;
159
+ float sumf_m = 0.0f;
160
+
161
+ #pragma unroll
162
+ for (int i = 0; i < QR2_K; ++i) {
163
+ const int sc = scales[2*i];
164
+
165
+ const int vi = (v >> (2*i)) & 0x03030303;
166
+
167
+ sumf_d +=
168
+ d8[i] * (dpct::dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product
169
+
170
+ // fill int with 4x m
171
+ int m = sc >> 4;
172
+ m |= m << 8;
173
+ m |= m << 16;
174
+ sumf_m += d8[i] *
175
+ dpct::dp4a(
176
+ m, u[i],
177
+ 0); // multiply constant q2_K part with sum of q8_1 values
178
+ }
179
+
180
+ const sycl::float2 dm2f =
181
+ dm2.convert<float, sycl::rounding_mode::automatic>();
182
+
183
+ return dm2f.x() * sumf_d - dm2f.y() * sumf_m;
184
+ }
185
+
186
+
187
+ #define VDR_Q3_K_Q8_1_MMVQ 1
188
+
189
+ // contiguous v/x values
190
+ static __dpct_inline__ float vec_dot_q3_K_q8_1_impl_mmvq(
191
+ const int &vl, const int &vh, const int *__restrict__ u,
192
+ const uint8_t *__restrict__ scales, const int &scale_offset,
193
+ const float &d3, const float *__restrict__ d8) {
194
+
195
+ float sumf = 0.0f;
196
+
197
+ #pragma unroll
198
+ for (int i = 0; i < QR3_K; ++i) {
199
+ const int isc = scale_offset + 2*i;
200
+
201
+ const int isc_low = isc % (QK_K/32);
202
+ const int sc_shift_low = 4 * (isc / (QK_K/32));
203
+ const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;
204
+
205
+ const int isc_high = isc % (QK_K/64);
206
+ const int sc_shift_high = 2 * (isc / (QK_K/64));
207
+ const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;
208
+
209
+ const int sc = (sc_low | sc_high) - 32;
210
+
211
+ const int vil = (vl >> (2*i)) & 0x03030303;
212
+
213
+ const int vih = ((vh >> i) << 2) & 0x04040404;
214
+
215
+ const int vi =
216
+ dpct::vectorized_binary<sycl::char4>(vil, vih, dpct::sub_sat());
217
+
218
+ sumf += d8[i] * (dpct::dp4a(vi, u[i], 0) * sc); // SIMD dot product
219
+ }
220
+
221
+ return d3 * sumf;
222
+ }
223
+
224
+ #define VDR_Q4_K_Q8_1_MMVQ 2
225
+
226
+ // contiguous v/x values
227
+ static __dpct_inline__ float vec_dot_q4_K_q8_1_impl_vmmq(
228
+ const int *__restrict__ v, const int *__restrict__ u,
229
+ const uint8_t *__restrict__ sc, const uint8_t *__restrict__ m,
230
+ const sycl::half2 &dm4, const float *__restrict__ d8) {
231
+
232
+ float sumf_d = 0.0f;
233
+ float sumf_m = 0.0f;
234
+
235
+ #pragma unroll
236
+ for (int i = 0; i < QR4_K; ++i) {
237
+ const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;
238
+ const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;
239
+
240
+ const int dot1 =
241
+ dpct::dp4a(v1i, u[2 * i + 1],
242
+ dpct::dp4a(v0i, u[2 * i + 0], 0)); // SIMD dot product
243
+ const int dot2 =
244
+ dpct::dp4a(0x01010101, u[2 * i + 1],
245
+ dpct::dp4a(0x01010101, u[2 * i + 0], 0)); // sum of u
246
+
247
+ sumf_d += d8[i] * (dot1 * sc[i]);
248
+ sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values
249
+ }
250
+
251
+ const sycl::float2 dm4f =
252
+ dm4.convert<float, sycl::rounding_mode::automatic>();
253
+
254
+ return dm4f.x() * sumf_d - dm4f.y() * sumf_m;
255
+ }
256
+
257
+
258
+ #define VDR_Q5_K_Q8_1_MMVQ 2
259
+
260
+ // contiguous v/x values
261
+ static __dpct_inline__ float vec_dot_q5_K_q8_1_impl_vmmq(
262
+ const int *__restrict__ vl, const int *__restrict__ vh,
263
+ const int *__restrict__ u, const uint8_t *__restrict__ sc,
264
+ const uint8_t *__restrict__ m, const sycl::half2 &dm5,
265
+ const float *__restrict__ d8) {
266
+
267
+ float sumf_d = 0.0f;
268
+ float sumf_m = 0.0f;
269
+
270
+ #pragma unroll
271
+ for (int i = 0; i < QR5_K; ++i) {
272
+ const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;
273
+ const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;
274
+
275
+ const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;
276
+ const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;
277
+
278
+ const int v0i = vl0i | vh0i;
279
+ const int v1i = vl1i | vh1i;
280
+
281
+ const int dot1 =
282
+ dpct::dp4a(v0i, u[2 * i + 0],
283
+ dpct::dp4a(v1i, u[2 * i + 1], 0)); // SIMD dot product
284
+ const int dot2 =
285
+ dpct::dp4a(0x01010101, u[2 * i + 0],
286
+ dpct::dp4a(0x01010101, u[2 * i + 1], 0)); // sum of u
287
+
288
+ sumf_d += d8[i] * (dot1 * sc[i]);
289
+ sumf_m += d8[i] * (dot2 * m[i]);
290
+
291
+ }
292
+
293
+ const sycl::float2 dm5f =
294
+ dm5.convert<float, sycl::rounding_mode::automatic>();
295
+
296
+ return dm5f.x() * sumf_d - dm5f.y() * sumf_m;
297
+ }
298
+
299
+
300
+ #define VDR_Q6_K_Q8_1_MMVQ 1
301
+
302
+ // contiguous v/x values
303
+ static __dpct_inline__ float
304
+ vec_dot_q6_K_q8_1_impl_mmvq(const int &vl, const int &vh,
305
+ const int *__restrict__ u,
306
+ const int8_t *__restrict__ scales, const float &d,
307
+ const float *__restrict__ d8) {
308
+ return vec_dot_q6_K_q8_1_impl_mmvq_scalar(
309
+ vl, vh, u[0], u[1], scales[0], scales[4], d, d8[0], d8[1]);
310
+ }
311
+
312
+ #define VDR_Q1_0_Q8_1_MMVQ 1
313
+ #define VDR_Q1_0_Q8_1_MMQ 4
314
+
315
+ static __dpct_inline__ float
316
+ vec_dot_q1_0_q8_1(const void *__restrict__ vbq,
317
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
318
+
319
+ const block_q1_0 * bq1_0 = (const block_q1_0 *) vbq;
320
+
321
+ const block_q8_1 * bq8_1_chunk = bq8_1 + iqs;
322
+ const float d1 = bq1_0->d;
323
+ const int v = get_int_from_uint8_aligned(bq1_0->qs, iqs);
324
+
325
+ int vi_bytes[8];
326
+ #pragma unroll
327
+ for (int j = 0; j < 8; ++j) {
328
+ const int shift = j * 4;
329
+ const int bits4 = (v >> shift) & 0x0F;
330
+ const int b0 = (bits4 & 0x01) ? 1 : -1;
331
+ const int b1 = (bits4 & 0x02) ? 1 : -1;
332
+ const int b2 = (bits4 & 0x04) ? 1 : -1;
333
+ const int b3 = (bits4 & 0x08) ? 1 : -1;
334
+ vi_bytes[j] = (b0 & 0xFF) | ((b1 & 0xFF) << 8) | ((b2 & 0xFF) << 16) | ((b3 & 0xFF) << 24);
335
+ }
336
+
337
+ int sumi = 0;
338
+ #pragma unroll
339
+ for (int j = 0; j < 8; ++j) {
340
+ const int u = get_int_from_int8_aligned(bq8_1_chunk->qs, j);
341
+ sumi = ggml_sycl_dp4a(vi_bytes[j], u, sumi);
342
+ }
343
+
344
+ return d1 * bq8_1_chunk->ds[0] * sumi;
345
+ }
346
+
347
+ // VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called
348
+ // MMVQ = mul_mat_vec_q, MMQ = mul_mat_q
349
+
350
+ template <ggml_type T> struct reorder_vec_dot_q_sycl {
351
+ static_assert(T != T, "ggml_type for reorder vecdot not implemented");
352
+ };
353
+
354
+ // For some types the weight side of the dot product does not depend on the destination column, so a
355
+ // multi-column mul_mat_vec can unpack it once per block instead of once per column. Such a type adds
356
+ // load() and dot() next to operator() and opts in here. See reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>.
357
+ template <ggml_type T> struct reorder_vec_dot_shared_weights {
358
+ static constexpr bool value = false;
359
+ };
360
+
361
+ template <> struct reorder_vec_dot_shared_weights<GGML_TYPE_Q4_K> {
362
+ static constexpr bool value = true;
363
+ };
364
+
365
+ template <ggml_type T> struct reorder_vec_dot_shared_activations {
366
+ static constexpr bool value = false;
367
+ };
368
+
369
+ template <> struct reorder_vec_dot_shared_activations<GGML_TYPE_Q4_K> {
370
+ static constexpr bool value = true;
371
+ };
372
+
373
+ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q4_0> {
374
+ static constexpr ggml_type gtype = GGML_TYPE_Q4_0;
375
+
376
+ using q4_0_block = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q4_0>;
377
+ using q4_0_traits = typename q4_0_block::traits;
378
+
379
+ __dpct_inline__ float vec_dot_q4_0_q8_1_impl(const int * v, const int * u, const float & d4, const sycl::half2 & ds8) {
380
+ int sumi = 0;
381
+
382
+ #pragma unroll
383
+ for (size_t i = 0; i < q4_0_traits::vdr_mmvq; ++i) {
384
+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
385
+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
386
+
387
+ // SIMD dot product of quantized values
388
+ sumi = dpct::dp4a(vi0, u[2 * i + 0], sumi);
389
+ sumi = dpct::dp4a(vi1, u[2 * i + 1], sumi);
390
+ }
391
+
392
+ const sycl::float2 ds8f = ds8.convert<float, sycl::rounding_mode::automatic>();
393
+
394
+ // second part effectively subtracts 8 from each quant value
395
+ return d4 * (sumi * ds8f.x() - (8 * q4_0_traits::vdr_mmvq / q4_0_traits::qi) * ds8f.y());
396
+ }
397
+
398
+ __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
399
+ const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
400
+ const sycl::half2 * q8_1_ds, const int & iqs) {
401
+ const uint8_t * bq4_0 = static_cast<const uint8_t *>(vbq) + ibx_offset.first;
402
+ const ggml_half d = *(reinterpret_cast<const ggml_half *>(static_cast<const uint8_t *>(vbq) + d_offset.first));
403
+ int v[q4_0_traits::vdr_mmvq];
404
+ int u[2 * q4_0_traits::vdr_mmvq];
405
+
406
+
407
+ #pragma unroll
408
+ for (size_t i = 0; i < q4_0_traits::vdr_mmvq; ++i) {
409
+ v[i] = get_int_from_uint8(bq4_0, iqs + i);
410
+ u[2 * i + 0] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i);
411
+ u[2 * i + 1] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i + q4_0_traits::qi);
412
+ }
413
+
414
+ return vec_dot_q4_0_q8_1_impl(v, u, d, *q8_1_ds);
415
+ };
416
+ };
417
+
418
+ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {
419
+ static constexpr ggml_type gtype = GGML_TYPE_Q8_0;
420
+
421
+ using q8_0_block = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q8_0>;
422
+ using q8_0_traits = typename q8_0_block::traits;
423
+
424
+ __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
425
+ const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
426
+ const sycl::half2 * q8_1_ds, const int & iqs) {
427
+ const uint8_t * base = static_cast<const uint8_t *>(vbq);
428
+ const int8_t * qs = reinterpret_cast<const int8_t *>(base + ibx_offset.first);
429
+ const ggml_half d = *reinterpret_cast<const ggml_half *>(base + d_offset.first);
430
+
431
+ int v[q8_0_traits::vdr_mmvq];
432
+ int u[q8_0_traits::vdr_mmvq];
433
+
434
+ #pragma unroll
435
+ for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
436
+ v[i] = get_int_from_int8(qs, iqs + i);
437
+ u[i] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i);
438
+ }
439
+
440
+ int sumi = 0;
441
+ #pragma unroll
442
+ for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
443
+ sumi = dpct::dp4a(v[i], u[i], sumi);
444
+ }
445
+
446
+ const sycl::half2 ds_values = *q8_1_ds;
447
+ return static_cast<float>(d) * static_cast<float>(ds_values[0]) * sumi;
448
+ }
449
+ };
450
+
451
+ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q2_K> {
452
+ static constexpr ggml_type gtype = GGML_TYPE_Q2_K;
453
+
454
+ using q2_k_block = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q2_K>;
455
+ using q2_k_traits = typename q2_k_block::traits;
456
+
457
+ __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
458
+ const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
459
+ const sycl::half2 * q8_1_ds, const int & iqs) {
460
+ const uint8_t * base = static_cast<const uint8_t *>(vbq);
461
+ const uint8_t * qs = base + ibx_offset.first;
462
+ const uint8_t * scales = base + d_offset.first;
463
+ const ggml_half2 * dm = reinterpret_cast<const ggml_half2 *>(base + d_offset.second);
464
+
465
+ const int bq8_offset = QR2_K * (iqs / QI8_1);
466
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1 / 2);
467
+
468
+ const int v = get_int_from_uint8_aligned(qs, iqs);
469
+
470
+ int u[QR2_K];
471
+ float d8[QR2_K];
472
+
473
+ #pragma unroll
474
+ for (int i = 0; i < QR2_K; ++i) {
475
+ const int8_t * quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1;
476
+ u[i] = get_int_from_int8_aligned(quant_base_ptr, iqs % QI8_1);
477
+ d8[i] = (*(q8_1_ds + bq8_offset + i))[0];
478
+ }
479
+
480
+ return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales + scale_offset, *dm, d8);
481
+ }
482
+ };
483
+
484
+ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q3_K> {
485
+ static constexpr ggml_type gtype = GGML_TYPE_Q3_K;
486
+
487
+ using q3_k_block = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q3_K>;
488
+ using q3_k_traits = typename q3_k_block::traits;
489
+
490
+ __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
491
+ const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
492
+ const sycl::half2 * q8_1_ds, const int & iqs) {
493
+ const uint8_t * base = static_cast<const uint8_t *>(vbq);
494
+ const uint8_t * qs = base + ibx_offset.first;
495
+ const uint8_t * hmask = base + ibx_offset.second;
496
+ const uint8_t * scales = base + d_offset.first;
497
+ const ggml_half d = *reinterpret_cast<const ggml_half *>(base + d_offset.second);
498
+
499
+ const int bq8_offset = QR3_K * (iqs / (QI3_K / 2));
500
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1 / 2);
501
+
502
+ const int vl = get_int_from_uint8(qs, iqs);
503
+ const int vh = ~get_int_from_uint8(hmask, iqs % (QI3_K / 2)) >> bq8_offset;
504
+
505
+ int u[QR3_K];
506
+ float d8[QR3_K];
507
+
508
+ #pragma unroll
509
+ for (int i = 0; i < QR3_K; ++i) {
510
+ const int8_t * quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1;
511
+ u[i] = get_int_from_int8_aligned(quant_base_ptr, iqs % QI8_1);
512
+ d8[i] = (*(q8_1_ds + bq8_offset + i))[0];
513
+ }
514
+
515
+ return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, scales, scale_offset, static_cast<float>(d), d8);
516
+ }
517
+ };
518
+
519
+ static inline float vec_dot_q4_K_q8_1_common(const int * __restrict__ q4, const uint16_t * __restrict__ scales,
520
+ const ggml_half2 & dm, const block_q8_1 * __restrict__ bq8_1,
521
+ const int & iqs) {
522
+ int v[2];
523
+ int u[2 * QR4_K];
524
+ float d8[QR4_K];
525
+
526
+ v[0] = q4[0];
527
+ v[1] = q4[4];
528
+
529
+ uint16_t aux[2];
530
+ const int j = (QR4_K * ((iqs / 2) / (QI8_1 / 2))) / 2;
531
+ if (j < 2) {
532
+ aux[0] = scales[j + 0] & 0x3f3f;
533
+ aux[1] = scales[j + 2] & 0x3f3f;
534
+ } else {
535
+ aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2);
536
+ aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2);
537
+ }
538
+
539
+ const uint8_t * sc = (const uint8_t *) aux;
540
+ const uint8_t * m = sc + 2;
541
+
542
+ const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));
543
+
544
+ for (int i = 0; i < QR4_K; ++i) {
545
+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
546
+ d8[i] = bq8i->ds[0];
547
+
548
+ const int * q8 = (const int *) bq8i->qs + ((iqs / 2) % 4);
549
+ u[2 * i + 0] = q8[0];
550
+ u[2 * i + 1] = q8[4];
551
+ }
552
+
553
+ return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, dm, d8);
554
+ }
555
+
556
+ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K> {
557
+ static constexpr ggml_type gtype = GGML_TYPE_Q4_K;
558
+
559
+ using q4_k_block = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q4_K>;
560
+ using q4_k_traits = typename q4_k_block::traits;
561
+
562
+ struct weights {
563
+ int v[2];
564
+ uint16_t aux[2];
565
+ ggml_half2 dm;
566
+ int bq8_offset;
567
+ };
568
+
569
+ struct activations {
570
+ int u[2 * QR4_K];
571
+ float d8[QR4_K];
572
+ };
573
+
574
+ __dpct_inline__ static weights load(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
575
+ const std::pair<int, int> d_offset, const int & iqs) {
576
+ const uint8_t * base = static_cast<const uint8_t *>(vbq);
577
+ const uint8_t * qs = base + ibx_offset.first;
578
+ const uint8_t * scs = base + d_offset.first;
579
+ const ggml_half2 * dms = reinterpret_cast<const ggml_half2 *>(base + d_offset.second);
580
+
581
+ weights w;
582
+ w.bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));
583
+
584
+ const int * q4 = (const int *) (qs + 16 * w.bq8_offset + 4 * ((iqs / 2) % 4));
585
+ const uint16_t * scales = (const uint16_t *) scs;
586
+
587
+ w.v[0] = q4[0];
588
+ w.v[1] = q4[4];
589
+
590
+ const int j = (QR4_K * ((iqs / 2) / (QI8_1 / 2))) / 2;
591
+ if (j < 2) {
592
+ w.aux[0] = scales[j + 0] & 0x3f3f;
593
+ w.aux[1] = scales[j + 2] & 0x3f3f;
594
+ } else {
595
+ w.aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2);
596
+ w.aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2);
597
+ }
598
+
599
+ w.dm = *dms;
600
+
601
+ return w;
602
+ }
603
+
604
+ __dpct_inline__ static activations load_activations(const int8_t * q8_1_quant_ptr,
605
+ const sycl::half2 * q8_1_ds, const int & iqs) {
606
+ activations a;
607
+ const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));
608
+ for (int i = 0; i < QR4_K; ++i) {
609
+ const int8_t * quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1;
610
+ sycl::half2 ds_values = *(q8_1_ds + bq8_offset + i);
611
+
612
+ a.d8[i] = ds_values[0];
613
+
614
+ const int * q8 = (const int *) quant_base_ptr + ((iqs / 2) % 4);
615
+ a.u[2 * i + 0] = q8[0];
616
+ a.u[2 * i + 1] = q8[4];
617
+ }
618
+
619
+ return a;
620
+ }
621
+
622
+ __dpct_inline__ static float apply(const weights & w, const activations & a) {
623
+ const uint8_t * sc = (const uint8_t *) w.aux;
624
+ const uint8_t * m = sc + 2;
625
+
626
+ return vec_dot_q4_K_q8_1_impl_vmmq(w.v, a.u, sc, m, w.dm, a.d8);
627
+ }
628
+
629
+ __dpct_inline__ static float dot(const weights & w, const int8_t * q8_1_quant_ptr,
630
+ const sycl::half2 * q8_1_ds, const int & iqs) {
631
+ const auto a = load_activations(q8_1_quant_ptr, q8_1_ds, iqs);
632
+
633
+ return apply(w, a);
634
+ }
635
+
636
+ __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
637
+ const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
638
+ const sycl::half2 * q8_1_ds, const int & iqs) {
639
+ return dot(load(vbq, ibx_offset, d_offset, iqs), q8_1_quant_ptr, q8_1_ds, iqs);
640
+ }
641
+ };
642
+
643
+ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K> {
644
+ static constexpr ggml_type gtype = GGML_TYPE_Q5_K;
645
+
646
+ using q5_k_block = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q5_K>;
647
+ using q5_k_traits = typename q5_k_block::traits;
648
+
649
+ __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
650
+ const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
651
+ const sycl::half2 * q8_1_ds, const int & iqs) {
652
+ const uint8_t * base = static_cast<const uint8_t *>(vbq);
653
+ const uint8_t * qs = base + ibx_offset.first; // low 4 bits
654
+ const uint8_t * qh_base = base + ibx_offset.second; // high bit
655
+ const uint8_t * scs = base + d_offset.first;
656
+ const ggml_half2 * dms = reinterpret_cast<const ggml_half2 *>(base + d_offset.second);
657
+
658
+ const int bq8_offset = QR5_K * ((iqs / 2) / (QI8_1 / 2));
659
+ const int * ql_ptr = (const int *) (qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));
660
+ const int * qh_ptr = (const int *) (qh_base + 4 * ((iqs / 2) % 4));
661
+ const uint16_t * scales = (const uint16_t *) scs;
662
+
663
+ int vl[2];
664
+ int vh[2];
665
+ int u[2 * QR5_K];
666
+ float d8[QR5_K];
667
+
668
+ vl[0] = ql_ptr[0];
669
+ vl[1] = ql_ptr[4];
670
+
671
+ vh[0] = qh_ptr[0] >> bq8_offset;
672
+ vh[1] = qh_ptr[4] >> bq8_offset;
673
+
674
+ uint16_t aux[2];
675
+ const int j = (QR5_K * ((iqs / 2) / (QI8_1 / 2))) / 2;
676
+ if (j < 2) {
677
+ aux[0] = scales[j + 0] & 0x3f3f;
678
+ aux[1] = scales[j + 2] & 0x3f3f;
679
+ } else {
680
+ aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2);
681
+ aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2);
682
+ }
683
+
684
+ const uint8_t * sc = (const uint8_t *) aux;
685
+ const uint8_t * m = sc + 2;
686
+
687
+ for (int i = 0; i < QR5_K; ++i) {
688
+ const int8_t* quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1;
689
+ sycl::half2 ds_values = *(q8_1_ds + bq8_offset + i);
690
+
691
+ d8[i] = ds_values[0];
692
+
693
+ const int * q8 = (const int *) quant_base_ptr + ((iqs / 2) % 4);
694
+ u[2 * i + 0] = q8[0];
695
+ u[2 * i + 1] = q8[4];
696
+ }
697
+
698
+ return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, *dms, d8);
699
+ }
700
+ };
701
+
702
+ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q6_K> {
703
+ static constexpr ggml_type gtype = GGML_TYPE_Q6_K;
704
+
705
+ using q6_k_block = ggml_sycl_reordered::block_q_t<GGML_TYPE_Q6_K>;
706
+ using q6_k_traits = typename q6_k_block::traits;
707
+
708
+ __dpct_inline__ float vec_dot_q6_K_q8_1_impl_mmvq(const int vl, const int vh, const int * __restrict__ u,
709
+ const int8_t * __restrict__ scales, const float d,
710
+ const float * __restrict__ d8) {
711
+ return vec_dot_q6_K_q8_1_impl_mmvq_scalar(
712
+ vl, vh, u[0], u[1], scales[0], scales[4], d, d8[0], d8[1]);
713
+ }
714
+
715
+ __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
716
+ const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr, const sycl::half2 * q8_1_ds,
717
+ const int iqs) {
718
+ const uint8_t * base = static_cast<const uint8_t *>(vbq);
719
+ const uint8_t * ql = base + ibx_offset.first;
720
+ const uint8_t * qh = base + ibx_offset.second;
721
+ const int8_t * scales = reinterpret_cast<const int8_t *>(base + d_offset.first);
722
+ const ggml_half * d = (const ggml_half *) (base + d_offset.second);
723
+
724
+ const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 4);
725
+ const int scale_offset = (QI6_K / 4) * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 8);
726
+ const int vh_shift = 2 * ((iqs % (QI6_K / 2)) / (QI6_K / 4));
727
+
728
+ const int vl = get_int_from_uint8(ql, iqs);
729
+ const int vh = get_int_from_uint8(qh, (QI6_K / 4) * (iqs / (QI6_K / 2)) + iqs % (QI6_K / 4)) >> vh_shift;
730
+
731
+ const int8_t * scs = scales + scale_offset;
732
+
733
+ const int u0 = get_int_from_int8_aligned(
734
+ q8_1_quant_ptr + bq8_offset * QK8_1, iqs % QI8_1);
735
+ const int u1 = get_int_from_int8_aligned(
736
+ q8_1_quant_ptr + (bq8_offset + 2) * QK8_1, iqs % QI8_1);
737
+ const float d80 = (*(q8_1_ds + bq8_offset + 0))[0];
738
+ const float d81 = (*(q8_1_ds + bq8_offset + 2))[0];
739
+
740
+ return vec_dot_q6_K_q8_1_impl_mmvq_scalar(
741
+ vl, vh, u0, u1, scs[0], scs[4], *d, d80, d81);
742
+ }
743
+ };
744
+ #define VDR_Q4_0_Q8_1_MMVQ 2
745
+ #define VDR_Q4_0_Q8_1_MMQ 4
746
+
747
+ #define VDR_Q2_0_Q8_1_MMVQ 1
748
+
749
+ template <int vdr>
750
+ static __dpct_inline__ float vec_dot_q2_0_q8_1_impl(
751
+ const int * v,
752
+ const int * u,
753
+ const float & d2,
754
+ const sycl::half2 & ds8) {
755
+ int sumi = 0;
756
+
757
+ #pragma unroll
758
+ for (int i = 0; i < vdr; ++i) {
759
+ #pragma unroll
760
+ for (int j = 0; j < 4; ++j) {
761
+ const uint8_t q = (uint8_t) ((uint32_t) v[i] >> (8 * j));
762
+
763
+ // unpack 2-bit values to byte lanes (0..3), then apply zero-point
764
+ // correction with ds8f.y() below, mirroring the q4_0 style.
765
+ int vi = 0;
766
+ vi |= (((q >> 0) & 0x3) & 0xFF) << 0;
767
+ vi |= (((q >> 2) & 0x3) & 0xFF) << 8;
768
+ vi |= (((q >> 4) & 0x3) & 0xFF) << 16;
769
+ vi |= (((q >> 6) & 0x3) & 0xFF) << 24;
770
+
771
+ sumi = dpct::dp4a(vi, u[4 * i + j], sumi);
772
+ }
773
+ }
774
+
775
+ const sycl::float2 ds8f = ds8.convert<float, sycl::rounding_mode::automatic>();
776
+ // q2_0 has zero-point 1. Scale ds8f.y() by processed-lane ratio,
777
+ // consistent with q4_0's explicit zero-point subtraction style.
778
+ return d2 * (sumi * ds8f.x() - ((float) vdr / (float) QI2_0) * ds8f.y());
779
+ }
780
+
781
+ template <int vdr>
782
+ static __dpct_inline__ float vec_dot_q4_0_q8_1_impl(const int * v, const int * u, const float & d4,
783
+ const sycl::half2 & ds8) {
784
+ int sumi = 0;
785
+ #pragma unroll
786
+ for (int i = 0; i < vdr; ++i) {
787
+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
788
+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
789
+
790
+ // SIMD dot product of quantized values
791
+ sumi = dpct::dp4a(vi0, u[2 * i + 0], sumi);
792
+ sumi = dpct::dp4a(vi1, u[2 * i + 1], sumi);
793
+ }
794
+
795
+ const sycl::float2 ds8f = ds8.convert<float, sycl::rounding_mode::automatic>();
796
+
797
+ // second part effectively subtracts 8 from each quant value
798
+ return d4 * (sumi * ds8f.x() - (8 * vdr / QI4_0) * ds8f.y());
799
+ }
800
+
801
+ #define VDR_Q4_1_Q8_1_MMVQ 2
802
+ #define VDR_Q4_1_Q8_1_MMQ 4
803
+
804
+ template <int vdr>
805
+ static __dpct_inline__ float vec_dot_q4_1_q8_1_impl(const int *v, const int *u,
806
+ const sycl::half2 &dm4,
807
+ const sycl::half2 &ds8) {
808
+
809
+ int sumi = 0;
810
+
811
+ #pragma unroll
812
+ for (int i = 0; i < vdr; ++i) {
813
+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
814
+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
815
+
816
+ // SIMD dot product of quantized values
817
+ sumi = dpct::dp4a(vi0, u[2 * i + 0], sumi);
818
+ sumi = dpct::dp4a(vi1, u[2 * i + 1], sumi);
819
+ }
820
+
821
+ #ifdef GGML_SYCL_F16
822
+ const sycl::float2 tmp =
823
+ (dm4 * ds8).convert<float, sycl::rounding_mode::automatic>();
824
+ const float d4d8 = tmp.x();
825
+ const float m4s8 = tmp.y();
826
+ #else
827
+ const sycl::float2 dm4f =
828
+ dm4.convert<float, sycl::rounding_mode::automatic>();
829
+ const sycl::float2 ds8f =
830
+ ds8.convert<float, sycl::rounding_mode::automatic>();
831
+ const float d4d8 = dm4f.x() * ds8f.x();
832
+ const float m4s8 = dm4f.y() * ds8f.y();
833
+ #endif // GGML_SYCL_F16
834
+
835
+ // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it
836
+ return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));
837
+ }
838
+
839
+ #define VDR_Q5_0_Q8_1_MMVQ 2
840
+ #define VDR_Q5_0_Q8_1_MMQ 4
841
+
842
+ template <int vdr>
843
+ static __dpct_inline__ float
844
+ vec_dot_q5_0_q8_1_impl(const int *vl, const int *vh, const int *u,
845
+ const float &d5, const sycl::half2 &ds8) {
846
+ int sumi = 0;
847
+
848
+ #pragma unroll
849
+ for (int i = 0; i < vdr; ++i) {
850
+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
851
+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
852
+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
853
+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
854
+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
855
+ sumi = dpct::dp4a(vi0, u[2 * i + 0],
856
+ sumi); // SIMD dot product of quantized values
857
+
858
+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
859
+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
860
+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
861
+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
862
+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
863
+ sumi = dpct::dp4a(vi1, u[2 * i + 1],
864
+ sumi); // SIMD dot product of quantized values
865
+ }
866
+
867
+ const sycl::float2 ds8f =
868
+ ds8.convert<float, sycl::rounding_mode::automatic>();
869
+
870
+ // second part effectively subtracts 16 from each quant value
871
+ return d5 * (sumi * ds8f.x() - (16 * vdr / QI5_0) * ds8f.y());
872
+ }
873
+
874
+ #define VDR_Q5_1_Q8_1_MMVQ 2
875
+ #define VDR_Q5_1_Q8_1_MMQ 4
876
+
877
+ template <int vdr>
878
+ static __dpct_inline__ float
879
+ vec_dot_q5_1_q8_1_impl(const int *vl, const int *vh, const int *u,
880
+ const sycl::half2 &dm5, const sycl::half2 &ds8) {
881
+
882
+ int sumi = 0;
883
+
884
+ #pragma unroll
885
+ for (int i = 0; i < vdr; ++i) {
886
+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
887
+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
888
+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
889
+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
890
+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
891
+ sumi = dpct::dp4a(vi0, u[2 * i + 0],
892
+ sumi); // SIMD dot product of quantized values
893
+
894
+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
895
+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
896
+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
897
+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
898
+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
899
+ sumi = dpct::dp4a(vi1, u[2 * i + 1],
900
+ sumi); // SIMD dot product of quantized values
901
+ }
902
+
903
+ #ifdef GGML_SYCL_F16
904
+ const sycl::float2 tmp =
905
+ (dm5 * ds8).convert<float, sycl::rounding_mode::automatic>();
906
+ const float d5d8 = tmp.x();
907
+ const float m5s8 = tmp.y();
908
+
909
+
910
+ #else
911
+ const sycl::float2 dm5f =
912
+ dm5.convert<float, sycl::rounding_mode::automatic>();
913
+ const sycl::float2 ds8f =
914
+ ds8.convert<float, sycl::rounding_mode::automatic>();
915
+ const float d5d8 = dm5f.x() * ds8f.x();
916
+ const float m5s8 = dm5f.y() * ds8f.y();
917
+ #endif // GGML_SYCL_F16
918
+
919
+ // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it
920
+ return sumi*d5d8 + m5s8 / (QI5_1 / vdr);
921
+ }
922
+
923
+ #define VDR_Q8_0_Q8_1_MMVQ 2
924
+ #define VDR_Q8_0_Q8_1_MMQ 8
925
+
926
+ template <int vdr>
927
+ static __dpct_inline__ float vec_dot_q8_0_q8_1_impl(const int *v, const int *u,
928
+ const float &d8_0,
929
+ const float &d8_1) {
930
+
931
+ int sumi = 0;
932
+
933
+ #pragma unroll
934
+ for (int i = 0; i < vdr; ++i) {
935
+ // SIMD dot product of quantized values
936
+ sumi = dpct::dp4a(v[i], u[i], sumi);
937
+ }
938
+
939
+ return d8_0*d8_1 * sumi;
940
+ }
941
+
942
+ template <typename T, int vdr>
943
+ static __dpct_inline__ T vec_dot_q8_0_q8_1_impl(const int * v, const int * u, const T & d8_0, const T & d8_1) {
944
+ int sumi = 0;
945
+
946
+ #pragma unroll
947
+ for (int i = 0; i < vdr; ++i) {
948
+ // SIMD dot product of quantized values
949
+ sumi = ggml_sycl_dp4a(v[i], u[i], sumi);
950
+ }
951
+
952
+ return d8_0*d8_1 * ((T) sumi);
953
+ }
954
+
955
+ template <int vdr>
956
+ static __dpct_inline__ float vec_dot_q8_1_q8_1_impl(const int *v, const int *u,
957
+ const sycl::half2 &dm8,
958
+ const sycl::half2 &ds8) {
959
+
960
+ int sumi = 0;
961
+
962
+ #pragma unroll
963
+ for (int i = 0; i < vdr; ++i) {
964
+ // SIMD dot product of quantized values
965
+ sumi = dpct::dp4a(v[i], u[i], sumi);
966
+ }
967
+
968
+ #ifdef GGML_SYCL_F16
969
+ const sycl::float2 tmp =
970
+ (dm8 * ds8).convert<float, sycl::rounding_mode::automatic>();
971
+ const float d8d8 = tmp.x();
972
+ const float m8s8 = tmp.y();
973
+ #else
974
+ const sycl::float2 dm8f =
975
+ dm8.convert<float, sycl::rounding_mode::automatic>();
976
+ const sycl::float2 ds8f =
977
+ ds8.convert<float, sycl::rounding_mode::automatic>();
978
+ const float d8d8 = dm8f.x() * ds8f.x();
979
+ const float m8s8 = dm8f.y() * ds8f.y();
980
+ #endif // GGML_SYCL_F16
981
+
982
+ // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it
983
+ return sumi*d8d8 + m8s8 / (QI8_1 / vdr);
984
+ }
985
+
986
+ static __dpct_inline__ float
987
+ vec_dot_q4_0_q8_1(const void *__restrict__ vbq,
988
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
989
+
990
+ const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq;
991
+
992
+ int v[VDR_Q4_0_Q8_1_MMVQ];
993
+ int u[2 * VDR_Q4_0_Q8_1_MMVQ];
994
+
995
+ #pragma unroll
996
+ for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {
997
+ v[i] = get_int_from_uint8(bq4_0->qs, iqs + i);
998
+ u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
999
+ u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_0);
1000
+ }
1001
+
1002
+ return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, bq4_0->d, bq8_1->ds);
1003
+ }
1004
+
1005
+ static __dpct_inline__ float
1006
+ vec_dot_q2_0_q8_1(const void *__restrict__ vbq,
1007
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1008
+
1009
+ const block_q2_0 * bq2_0 = (const block_q2_0 *) vbq;
1010
+
1011
+ int v[2 * VDR_Q2_0_Q8_1_MMVQ];
1012
+ int u[8 * VDR_Q2_0_Q8_1_MMVQ];
1013
+
1014
+ #pragma unroll
1015
+ for (int i = 0; i < VDR_Q2_0_Q8_1_MMVQ; ++i) {
1016
+ const int base = 4 * (iqs + i);
1017
+
1018
+ // Q2_0 has QK2_0 = 64 and uses 2 x QK8_1 blocks on the RHS.
1019
+ v[2 * i + 0] = get_int_from_uint8(bq2_0->qs, iqs + i);
1020
+ v[2 * i + 1] = get_int_from_uint8(bq2_0->qs, iqs + i + QI2_0);
1021
+
1022
+ u[8 * i + 0] = get_int_from_int8_aligned(bq8_1[0].qs, base + 0);
1023
+ u[8 * i + 1] = get_int_from_int8_aligned(bq8_1[0].qs, base + 1);
1024
+ u[8 * i + 2] = get_int_from_int8_aligned(bq8_1[0].qs, base + 2);
1025
+ u[8 * i + 3] = get_int_from_int8_aligned(bq8_1[0].qs, base + 3);
1026
+
1027
+ u[8 * i + 4] = get_int_from_int8_aligned(bq8_1[1].qs, base + 0);
1028
+ u[8 * i + 5] = get_int_from_int8_aligned(bq8_1[1].qs, base + 1);
1029
+ u[8 * i + 6] = get_int_from_int8_aligned(bq8_1[1].qs, base + 2);
1030
+ u[8 * i + 7] = get_int_from_int8_aligned(bq8_1[1].qs, base + 3);
1031
+ }
1032
+
1033
+ const float sum0 = vec_dot_q2_0_q8_1_impl<VDR_Q2_0_Q8_1_MMVQ>(
1034
+ v + 0, u + 0, bq2_0->d, bq8_1[0].ds);
1035
+ const float sum1 = vec_dot_q2_0_q8_1_impl<VDR_Q2_0_Q8_1_MMVQ>(
1036
+ v + VDR_Q2_0_Q8_1_MMVQ, u + 4 * VDR_Q2_0_Q8_1_MMVQ, bq2_0->d, bq8_1[1].ds);
1037
+ return sum0 + sum1;
1038
+ }
1039
+
1040
+ static __dpct_inline__ float
1041
+ vec_dot_q4_1_q8_1(const void *__restrict__ vbq,
1042
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1043
+
1044
+ const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq;
1045
+
1046
+ int v[VDR_Q4_1_Q8_1_MMVQ];
1047
+ int u[2*VDR_Q4_1_Q8_1_MMVQ];
1048
+
1049
+ #pragma unroll
1050
+ for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {
1051
+ v[i] = get_int_from_uint8_aligned(bq4_1->qs, iqs + i);
1052
+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
1053
+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_1);
1054
+ }
1055
+
1056
+ return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);
1057
+ }
1058
+
1059
+ #define VDR_MXFP4_Q8_1_MMVQ 2
1060
+ #define VDR_MXFP4_Q8_1_MMQ 4
1061
+
1062
+ static __dpct_inline__ float vec_dot_mxfp4_q8_1(const void * __restrict__ vbq,
1063
+ const block_q8_1 * __restrict__ bq8_1,
1064
+ const int & iqs) {
1065
+ const block_mxfp4 * bq4 = (const block_mxfp4 *) vbq;
1066
+
1067
+ const int * q8 = (const int *) bq8_1->qs + iqs;
1068
+
1069
+ int sumi = 0;
1070
+ #pragma unroll
1071
+ for (int l = 0; l < VDR_MXFP4_Q8_1_MMVQ; ++l) {
1072
+ const int aux_q4 = get_int_b1(bq4->qs, iqs + l);
1073
+ const sycl::int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4);
1074
+ sumi = ggml_sycl_dp4a(v.x(), q8[l + 0], sumi);
1075
+ sumi = ggml_sycl_dp4a(v.y(), q8[l + 4], sumi);
1076
+ }
1077
+
1078
+ const float d = ggml_sycl_e8m0_to_fp32(bq4->e) * 0.5f * (bq8_1->ds)[0];
1079
+ return d * sumi;
1080
+ }
1081
+
1082
+ #define VDR_NVFP4_Q8_1_MMVQ 4
1083
+ #define VDR_NVFP4_Q8_1_MMQ 8
1084
+
1085
+ static __dpct_inline__ float vec_dot_nvfp4_q8_1(const void * __restrict__ vbq,
1086
+ const block_q8_1 * __restrict__ bq8_1,
1087
+ const int32_t & iqs) {
1088
+ const block_nvfp4 * bq4 = (const block_nvfp4 *) vbq;
1089
+ float sum = 0.0f;
1090
+ #pragma unroll
1091
+ for (int i = 0; i < VDR_NVFP4_Q8_1_MMVQ/2; i++) {
1092
+ const int32_t iqs0 = iqs + 2*i;
1093
+ const int32_t iqs1 = iqs0 + 1;
1094
+ const int32_t is = iqs0 >> 1;
1095
+ const sycl::int2 v0 = get_int_from_table_16(get_int_b4(bq4->qs, iqs0), kvalues_mxfp4);
1096
+ const sycl::int2 v1 = get_int_from_table_16(get_int_b4(bq4->qs, iqs1), kvalues_mxfp4);
1097
+ const block_q8_1 * bq8 = bq8_1 + (is >> 1);
1098
+ const int32_t i8 = ((is & 1) << 2);
1099
+
1100
+ int sumi = ggml_sycl_dp4a(v0.x(), get_int_b4(bq8->qs, i8 + 0), 0);
1101
+ sumi = ggml_sycl_dp4a(v0.y(), get_int_b4(bq8->qs, i8 + 2), sumi);
1102
+ sumi = ggml_sycl_dp4a(v1.x(), get_int_b4(bq8->qs, i8 + 1), sumi);
1103
+ sumi = ggml_sycl_dp4a(v1.y(), get_int_b4(bq8->qs, i8 + 3), sumi);
1104
+
1105
+ const float d = ggml_sycl_ue4m3_to_fp32(bq4->d[is]) * (bq8->ds)[0];
1106
+ sum += d * float(sumi);
1107
+ }
1108
+
1109
+ return sum;
1110
+ }
1111
+
1112
+ static __dpct_inline__ float
1113
+ vec_dot_q5_0_q8_1(const void *__restrict__ vbq,
1114
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1115
+
1116
+ const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq;
1117
+
1118
+ int vl[VDR_Q5_0_Q8_1_MMVQ];
1119
+ int vh[VDR_Q5_0_Q8_1_MMVQ];
1120
+ int u[2*VDR_Q5_0_Q8_1_MMVQ];
1121
+
1122
+ #pragma unroll
1123
+ for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {
1124
+ vl[i] = get_int_from_uint8(bq5_0->qs, iqs + i);
1125
+ vh[i] = get_int_from_uint8(bq5_0->qh, 0) >> (4 * (iqs + i));
1126
+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
1127
+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_0);
1128
+ }
1129
+
1130
+ return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, bq5_0->d, bq8_1->ds);
1131
+ }
1132
+
1133
+ static __dpct_inline__ float
1134
+ vec_dot_q5_1_q8_1(const void *__restrict__ vbq,
1135
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1136
+
1137
+ const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq;
1138
+
1139
+ int vl[VDR_Q5_1_Q8_1_MMVQ];
1140
+ int vh[VDR_Q5_1_Q8_1_MMVQ];
1141
+ int u[2*VDR_Q5_1_Q8_1_MMVQ];
1142
+
1143
+ #pragma unroll
1144
+ for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {
1145
+ vl[i] = get_int_from_uint8_aligned(bq5_1->qs, iqs + i);
1146
+ vh[i] = get_int_from_uint8_aligned(bq5_1->qh, 0) >> (4 * (iqs + i));
1147
+ u[2*i+0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
1148
+ u[2*i+1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_1);
1149
+ }
1150
+
1151
+ return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);
1152
+ }
1153
+
1154
+ static __dpct_inline__ float
1155
+ vec_dot_q8_0_q8_1(const void *__restrict__ vbq,
1156
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1157
+
1158
+ const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq;
1159
+
1160
+ int v[VDR_Q8_0_Q8_1_MMVQ];
1161
+ int u[VDR_Q8_0_Q8_1_MMVQ];
1162
+
1163
+ #pragma unroll
1164
+ for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {
1165
+ v[i] = get_int_from_int8(bq8_0->qs, iqs + i);
1166
+ u[i] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);
1167
+ }
1168
+
1169
+ return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMVQ>(v, u, bq8_0->d,
1170
+ bq8_1->ds[0]);
1171
+ }
1172
+
1173
+ static __dpct_inline__ float
1174
+ vec_dot_q2_K_q8_1(const void *__restrict__ vbq,
1175
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1176
+
1177
+ const block_q2_K * bq2_K = (const block_q2_K *) vbq;
1178
+
1179
+ const int bq8_offset = QR2_K * (iqs / QI8_1);
1180
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
1181
+
1182
+ const uint8_t * scales = bq2_K->scales + scale_offset;
1183
+
1184
+ const int v = get_int_from_uint8_aligned(bq2_K->qs, iqs);
1185
+ int u[QR2_K];
1186
+ float d8[QR2_K];
1187
+
1188
+ #pragma unroll
1189
+ for (int i = 0; i < QR2_K; ++ i) {
1190
+ u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
1191
+ d8[i] = bq8_1[bq8_offset + i].ds[0];
1192
+ }
1193
+
1194
+ return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);
1195
+ }
1196
+
1197
+ static __dpct_inline__ float
1198
+ vec_dot_q3_K_q8_1(const void *__restrict__ vbq,
1199
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1200
+
1201
+ const block_q3_K * bq3_K = (const block_q3_K *) vbq;
1202
+
1203
+ const int bq8_offset = QR3_K * (iqs / (QI3_K/2));
1204
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
1205
+
1206
+ const float d = bq3_K->d;
1207
+
1208
+ const int vl = get_int_from_uint8(bq3_K->qs, iqs);
1209
+
1210
+ // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
1211
+ const int vh = ~get_int_from_uint8(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;
1212
+
1213
+ int u[QR3_K];
1214
+ float d8[QR3_K];
1215
+
1216
+ #pragma unroll
1217
+ for (int i = 0; i < QR3_K; ++i) {
1218
+ u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
1219
+ d8[i] = bq8_1[bq8_offset + i].ds[0];
1220
+ }
1221
+
1222
+ return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);
1223
+ }
1224
+
1225
+ static __dpct_inline__ float vec_dot_q4_K_q8_1(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1,
1226
+ const int & iqs) {
1227
+ #ifndef GGML_QKK_64
1228
+
1229
+ const block_q4_K * bq4_K = (const block_q4_K *) vbq;
1230
+
1231
+ const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));
1232
+ const int * q4 = (const int *) (bq4_K->qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));
1233
+ const uint16_t * scales = (const uint16_t *) bq4_K->scales;
1234
+
1235
+ return vec_dot_q4_K_q8_1_common(q4, scales, bq4_K->dm, bq8_1, iqs);
1236
+
1237
+ #else
1238
+
1239
+ #if __SYCL_ARCH__ >= VER_4VEC // lowest compute capability for integer intrinsics
1240
+ const block_q4_K * bq4_K = (const block_q4_K *) vbq;
1241
+
1242
+ float sumf_d = 0.0f;
1243
+ float sumf_m = 0.0f;
1244
+
1245
+ uint16_t aux16[2];
1246
+ const uint8_t * s = (const uint8_t *)aux16;
1247
+
1248
+ const uint16_t * a = (const uint16_t *)bq4_K->scales;
1249
+ aux16[0] = a[0] & 0x0f0f;
1250
+ aux16[1] = (a[0] >> 4) & 0x0f0f;
1251
+
1252
+ const float dall = bq4_K->dm[0];
1253
+ const float dmin = bq4_K->dm[1];
1254
+
1255
+ const float d8_1 = bq8_1[0].ds[0];
1256
+ const float d8_2 = bq8_1[1].ds[1];
1257
+
1258
+ const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2));
1259
+ const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4);
1260
+ const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2));
1261
+ const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4);
1262
+
1263
+ const int * q4 = (const int *)bq4_K->qs + (iqs/2);
1264
+ const int v1 = q4[0];
1265
+ const int v2 = q4[4];
1266
+
1267
+ const int dot1 = dpct::dp4a(ui2, v2 & 0x0f0f0f0f, dpct::dp4a(ui1, v1 & 0x0f0f0f0f, 0));
1268
+ const int dot2 = dpct::dp4a(ui4, (v2 >> 4) & 0x0f0f0f0f, dpct::dp4a(ui3, (v1 >> 4) & 0x0f0f0f0f, 0));
1269
+ const int dot3 = dpct::dp4a(0x01010101, ui2, dpct::dp4a(0x01010101, ui1, 0));
1270
+ const int dot4 = dpct::dp4a(0x01010101, ui4, dpct::dp4a(0x01010101, ui3, 0));
1271
+
1272
+ sumf_d += d8_1 * (dot1 * s[0]) + d8_2 * (dot2 * s[1]);
1273
+ sumf_m += d8_1 * (dot3 * s[2]) + d8_2 * (dot4 * s[3]);
1274
+
1275
+ return dall * sumf_d - dmin * sumf_m;
1276
+
1277
+ #else
1278
+ bad_arch();
1279
+ #endif // __SYCL_ARCH__ >= VER_4VEC
1280
+
1281
+ #endif
1282
+ }
1283
+
1284
+ static __dpct_inline__ float
1285
+ vec_dot_q5_K_q8_1(const void *__restrict__ vbq,
1286
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1287
+
1288
+ #ifndef GGML_QKK_64
1289
+ const block_q5_K * bq5_K = (const block_q5_K *) vbq;
1290
+
1291
+ int vl[2];
1292
+ int vh[2];
1293
+ int u[2*QR5_K];
1294
+ float d8[QR5_K];
1295
+
1296
+ const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));
1297
+ const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
1298
+ const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));
1299
+
1300
+ vl[0] = ql[0];
1301
+ vl[1] = ql[4];
1302
+
1303
+ vh[0] = qh[0] >> bq8_offset;
1304
+ vh[1] = qh[4] >> bq8_offset;
1305
+
1306
+ const uint16_t * scales = (const uint16_t *)bq5_K->scales;
1307
+ uint16_t aux[2];
1308
+ const int j = bq8_offset/2;
1309
+ if (j < 2) {
1310
+ aux[0] = scales[j+0] & 0x3f3f;
1311
+ aux[1] = scales[j+2] & 0x3f3f;
1312
+ } else {
1313
+ aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
1314
+ aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
1315
+ }
1316
+ const uint8_t * sc = (const uint8_t *)aux;
1317
+ const uint8_t * m = sc + 2;
1318
+
1319
+ #pragma unroll
1320
+ for (int i = 0; i < QR5_K; ++i) {
1321
+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
1322
+ d8[i] = bq8i->ds[0];
1323
+
1324
+ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
1325
+ u[2*i+0] = q8[0];
1326
+ u[2*i+1] = q8[4];
1327
+ }
1328
+
1329
+ return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);
1330
+
1331
+ #else
1332
+
1333
+ #if __SYCL_ARCH__ >= VER_4VEC // lowest compute capability for integer intrinsics
1334
+ const block_q5_K * bq5_K = (const block_q5_K *) vbq;
1335
+
1336
+ const int8_t * s = bq5_K->scales;
1337
+
1338
+ const float d = bq5_K->d;
1339
+
1340
+ const float d8_1 = bq8_1[0].ds[0];
1341
+ const float d8_2 = bq8_1[1].ds[1];
1342
+
1343
+ const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2));
1344
+ const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4);
1345
+ const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2));
1346
+ const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4);
1347
+
1348
+ const int * ql = (const int *)bq5_K->qs + (iqs/2);
1349
+ const int vl1 = ql[0];
1350
+ const int vl2 = ql[4];
1351
+
1352
+ const int step = 4 * (iqs/2); // 0, 4, 8, 12
1353
+ const int im = step/8; // = 0 for iqs = 0, 2, = 1 for iqs = 4, 6
1354
+ const int in = step%8; // 0, 4, 0, 4
1355
+ const int vh = (*((const int *)(bq5_K->qh + in))) >> im;
1356
+
1357
+ const int v1 = (((vh << 4) & 0x10101010) ^ 0x10101010) | ((vl1 >> 0) & 0x0f0f0f0f);
1358
+ const int v2 = (((vh << 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 0) & 0x0f0f0f0f);
1359
+ const int v3 = (((vh >> 0) & 0x10101010) ^ 0x10101010) | ((vl1 >> 4) & 0x0f0f0f0f);
1360
+ const int v4 = (((vh >> 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 4) & 0x0f0f0f0f);
1361
+
1362
+ const float sumf_d = d8_1 * (dpct::dp4a(ui1, v1, 0) * s[0] + dpct::dp4a(ui2, v2, 0) * s[1])
1363
+ + d8_2 * (dpct::dp4a(ui3, v3, 0) * s[2] + dpct::dp4a(ui4, v4, 0) * s[3]);
1364
+
1365
+ return d * sumf_d;
1366
+
1367
+ #else
1368
+ bad_arch();
1369
+ #endif // __SYCL_ARCH__ >= VER_4VEC
1370
+
1371
+ #endif
1372
+ }
1373
+
1374
+ static __dpct_inline__ float
1375
+ vec_dot_q6_K_q8_1(const void *__restrict__ vbq,
1376
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1377
+
1378
+ const block_q6_K * bq6_K = (const block_q6_K *) vbq;
1379
+
1380
+ const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);
1381
+ const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);
1382
+ const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));
1383
+
1384
+ const int vl = get_int_from_uint8(bq6_K->ql, iqs);
1385
+ const int vh = get_int_from_uint8(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;
1386
+
1387
+ const int8_t * scales = bq6_K->scales + scale_offset;
1388
+
1389
+ const int u0 = get_int_from_int8_aligned(
1390
+ bq8_1[bq8_offset + 0].qs, iqs % QI8_1);
1391
+ const int u1 = get_int_from_int8_aligned(
1392
+ bq8_1[bq8_offset + 2].qs, iqs % QI8_1);
1393
+ const float d80 = bq8_1[bq8_offset + 0].ds[0];
1394
+ const float d81 = bq8_1[bq8_offset + 2].ds[0];
1395
+
1396
+ return vec_dot_q6_K_q8_1_impl_mmvq_scalar(
1397
+ vl, vh, u0, u1, scales[0], scales[4], bq6_K->d, d80, d81);
1398
+ }
1399
+
1400
+
1401
+ static __dpct_inline__ float
1402
+ vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq,
1403
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs,
1404
+ const uint64_t *iq2xxs_grid, const uint8_t *ksigns_iq2xs,
1405
+ const uint8_t *kmask_iq2xs) {
1406
+ #if QK_K == 256
1407
+ const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq;
1408
+
1409
+ const int ib32 = iqs;
1410
+ const uint16_t * q2 = bq2->qs + 4*ib32;
1411
+ const uint8_t * aux8 = (const uint8_t *)q2;
1412
+ const int8_t * q8 = bq8_1[ib32].qs;
1413
+ uint32_t aux32 = q2[2] | (q2[3] << 16);
1414
+ int sumi = 0;
1415
+ for (int l = 0; l < 4; ++l) {
1416
+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);
1417
+ const uint8_t signs = ksigns_iq2xs[aux32 & 127];
1418
+ for (int j = 0; j < 8; ++j) {
1419
+ sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1420
+ }
1421
+ q8 += 8;
1422
+ aux32 >>= 7;
1423
+ }
1424
+ const float d = (float)bq2->d * (0.5f + aux32) * bq8_1[ib32].ds[0] * 0.25f;
1425
+ return d * sumi;
1426
+ #else
1427
+ assert(false);
1428
+ return 0.f;
1429
+ #endif
1430
+ }
1431
+
1432
+ static __dpct_inline__ float
1433
+ vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq,
1434
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs,
1435
+ const uint64_t *iq2xs_grid, const uint64_t *ksigns64) {
1436
+ #if DPCT_COMPATIBILITY_TEMP >= \
1437
+ MIN_CC_DP4A // lowest compute capability for integer intrinsics
1438
+ #if QK_K == 256
1439
+ const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq;
1440
+
1441
+ const int ib32 = iqs;
1442
+ const uint16_t * q2 = bq2->qs + 4*ib32;
1443
+ const int8_t * q8 = bq8_1[ib32].qs;
1444
+ const uint8_t ls1 = bq2->scales[ib32] & 0xf;
1445
+ const uint8_t ls2 = bq2->scales[ib32] >> 4;
1446
+ int sumi1 = 0;
1447
+ for (int l = 0; l < 2; ++l) {
1448
+ const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));
1449
+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));
1450
+ const int grid_l = dpct::vectorized_binary<sycl::uchar4>(
1451
+ grid[0] ^ signs[0], signs[0], std::minus<>());
1452
+ const int grid_h = dpct::vectorized_binary<sycl::uchar4>(
1453
+ grid[1] ^ signs[1], signs[1], std::minus<>());
1454
+ sumi1 = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi1);
1455
+ sumi1 = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi1);
1456
+ q8 += 8;
1457
+ }
1458
+ int sumi2 = 0;
1459
+ for (int l = 2; l < 4; ++l) {
1460
+ const uint32_t * grid = (const uint32_t *)(iq2xs_grid + (q2[l] & 511));
1461
+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l] >> 9));
1462
+ const int grid_l = dpct::vectorized_binary<sycl::uchar4>(
1463
+ grid[0] ^ signs[0], signs[0], std::minus<>());
1464
+ const int grid_h = dpct::vectorized_binary<sycl::uchar4>(
1465
+ grid[1] ^ signs[1], signs[1], std::minus<>());
1466
+ sumi2 = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi2);
1467
+ sumi2 = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi2);
1468
+ q8 += 8;
1469
+ }
1470
+ const float d = (float)bq2->d * bq8_1[ib32].ds[0] * 0.25f;
1471
+ return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);
1472
+ #else
1473
+ assert(false);
1474
+ return 0.f;
1475
+ #endif
1476
+ #else
1477
+ assert(false);
1478
+ return 0.f;
1479
+ #endif
1480
+ }
1481
+
1482
+ static __dpct_inline__ float
1483
+ vec_dot_iq2_s_q8_1(const void *__restrict__ vbq,
1484
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1485
+ #if QK_K == 256
1486
+ const block_iq2_s * bq2 = (const block_iq2_s *) vbq;
1487
+
1488
+ const int ib32 = iqs;
1489
+ const int8_t * q8 = bq8_1[ib32].qs;
1490
+ const uint8_t * signs = bq2->qs + QK_K/8 + 4*ib32;
1491
+ const uint8_t ls1 = bq2->scales[ib32] & 0xf;
1492
+ const uint8_t ls2 = bq2->scales[ib32] >> 4;
1493
+ int sumi1 = 0;
1494
+ for (int l = 0; l < 2; ++l) {
1495
+ const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));
1496
+ const uint32_t signs0 = dpct::vectorized_binary<sycl::uchar4>(
1497
+ ((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201,
1498
+ std::equal_to<>());
1499
+ const uint32_t signs1 = dpct::vectorized_binary<sycl::uchar4>(
1500
+ ((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201,
1501
+ std::equal_to<>());
1502
+ const int grid_l = dpct::vectorized_binary<sycl::uchar4>(
1503
+ grid[0] ^ signs0, signs0, std::minus<>());
1504
+ const int grid_h = dpct::vectorized_binary<sycl::uchar4>(
1505
+ grid[1] ^ signs1, signs1, std::minus<>());
1506
+ sumi1 = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi1);
1507
+ sumi1 = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi1);
1508
+ q8 += 8;
1509
+ }
1510
+ int sumi2 = 0;
1511
+ for (int l = 2; l < 4; ++l) {
1512
+ const uint32_t * grid = (const uint32_t *)(iq2s_grid + (bq2->qs[4*ib32+l] | ((bq2->qh[ib32] << (8-2*l)) & 0x300)));
1513
+ const uint32_t signs0 = dpct::vectorized_binary<sycl::uchar4>(
1514
+ ((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201,
1515
+ std::equal_to<>());
1516
+ const uint32_t signs1 = dpct::vectorized_binary<sycl::uchar4>(
1517
+ ((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201,
1518
+ std::equal_to<>());
1519
+ const int grid_l = dpct::vectorized_binary<sycl::uchar4>(
1520
+ grid[0] ^ signs0, signs0, std::minus<>());
1521
+ const int grid_h = dpct::vectorized_binary<sycl::uchar4>(
1522
+ grid[1] ^ signs1, signs1, std::minus<>());
1523
+ sumi2 = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi2);
1524
+ sumi2 = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi2);
1525
+ q8 += 8;
1526
+ }
1527
+ const float d = (float)bq2->d * bq8_1[ib32].ds[0] * 0.25f;
1528
+ return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);
1529
+ #else
1530
+ assert(false);
1531
+ #endif
1532
+ }
1533
+
1534
+ static __dpct_inline__ float
1535
+ vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq,
1536
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs,
1537
+ const uint32_t *iq3xxs_grid, const uint64_t *ksigns64) {
1538
+ #if DPCT_COMPATIBILITY_TEMP >= \
1539
+ MIN_CC_DP4A // lowest compute capability for integer intrinsics
1540
+ #if QK_K == 256
1541
+ const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq;
1542
+
1543
+ const int ib32 = iqs;
1544
+ const uint8_t * q3 = bq2->qs + 8*ib32;
1545
+ const uint16_t * gas = (const uint16_t *)(bq2->qs + QK_K/4) + 2*ib32;
1546
+ const int8_t * q8 = bq8_1[ib32].qs;
1547
+ uint32_t aux32 = gas[0] | (gas[1] << 16);
1548
+ int sumi = 0;
1549
+ for (int l = 0; l < 4; ++l) {
1550
+ const uint32_t * grid1 = iq3xxs_grid + q3[2*l+0];
1551
+ const uint32_t * grid2 = iq3xxs_grid + q3[2*l+1];
1552
+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (aux32 & 127));
1553
+ const int grid_l = dpct::vectorized_binary<sycl::uchar4>(
1554
+ grid1[0] ^ signs[0], signs[0], std::minus<>());
1555
+ const int grid_h = dpct::vectorized_binary<sycl::uchar4>(
1556
+ grid2[0] ^ signs[1], signs[1], std::minus<>());
1557
+ sumi = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi);
1558
+ sumi = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi);
1559
+ q8 += 8;
1560
+ aux32 >>= 7;
1561
+ }
1562
+ const float d = (float)bq2->d * (0.5f + aux32) * bq8_1[ib32].ds[0] * 0.5f;
1563
+ return d * sumi;
1564
+ #else
1565
+ assert(false);
1566
+ return 0.f;
1567
+ #endif
1568
+ #else
1569
+ assert(false);
1570
+ return 0.f;
1571
+ #endif
1572
+ }
1573
+
1574
+ static __dpct_inline__ float
1575
+ vec_dot_iq3_s_q8_1(const void *__restrict__ vbq,
1576
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs,
1577
+ const uint32_t *iq3s_grid) {
1578
+ #if QK_K == 256
1579
+ const block_iq3_s * bq2 = (const block_iq3_s *) vbq;
1580
+
1581
+ const int ib32 = iqs;
1582
+ const uint8_t * qs = bq2->qs + 8*ib32;
1583
+ const int8_t * q8 = bq8_1[ib32].qs;
1584
+ int sumi = 0;
1585
+ for (int l = 0; l < 4; ++l) {
1586
+ const uint32_t * grid1 = iq3s_grid + (qs[2*l+0] | ((bq2->qh[ib32] << (8 - 2*l)) & 256));
1587
+ const uint32_t * grid2 = iq3s_grid + (qs[2*l+1] | ((bq2->qh[ib32] << (7 - 2*l)) & 256));
1588
+ uint32_t signs0 = dpct::vectorized_binary<sycl::uchar4>(
1589
+ ((bq2->signs[4 * ib32 + l] & 0xf) * 0x01010101) & 0x08040201,
1590
+ 0x08040201, std::equal_to<>());
1591
+ uint32_t signs1 = dpct::vectorized_binary<sycl::uchar4>(
1592
+ ((bq2->signs[4 * ib32 + l] >> 4) * 0x01010101) & 0x08040201,
1593
+ 0x08040201, std::equal_to<>());
1594
+ const int grid_l = dpct::vectorized_binary<sycl::uchar4>(
1595
+ grid1[0] ^ signs0, signs0, std::minus<>());
1596
+ const int grid_h = dpct::vectorized_binary<sycl::uchar4>(
1597
+ grid2[0] ^ signs1, signs1, std::minus<>());
1598
+ sumi = dpct::dp4a(grid_l, *((const int *)q8 + 0), sumi);
1599
+ sumi = dpct::dp4a(grid_h, *((const int *)q8 + 1), sumi);
1600
+ q8 += 8;
1601
+ }
1602
+ const float d =
1603
+ (float)bq2->d *
1604
+ (1 + 2 * ((bq2->scales[ib32 / 2] >> 4 * (ib32 % 2)) & 0xf)) *
1605
+ bq8_1[ib32].ds[0];
1606
+ return d * sumi;
1607
+ #else
1608
+ assert(false);
1609
+ #endif
1610
+ }
1611
+
1612
+ static __dpct_inline__ float
1613
+ vec_dot_iq1_s_q8_1(const void *__restrict__ vbq,
1614
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs,
1615
+ const uint32_t *iq1s_grid_gpu) {
1616
+ #if QK_K == 256
1617
+ const block_iq1_s * bq1 = (const block_iq1_s *) vbq;
1618
+
1619
+ const int ib32 = iqs;
1620
+ int sumi = 0;
1621
+ const int * q8 = (const int *)bq8_1[ib32].qs;
1622
+ for (int l = 0; l < 4; ++l) {
1623
+ const int * grid = (const int *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[ib32] >> 3*l) & 7) << 8)));
1624
+ int grid0 = grid[0] & 0x0f0f0f0f;
1625
+ int grid1 = (grid[0] >> 4) & 0x0f0f0f0f;
1626
+ sumi = dpct::dp4a(q8[2 * l + 1], grid1,
1627
+ dpct::dp4a(q8[2 * l + 0], grid0, sumi));
1628
+ }
1629
+
1630
+ const float delta = bq1->qh[ib32] & 0x8000 ? -1-IQ1S_DELTA : -1+IQ1S_DELTA;
1631
+ const float d1q = (float)bq1->d * (2*((bq1->qh[ib32] >> 12) & 7) + 1);
1632
+ const float d = d1q * bq8_1[ib32].ds[0];
1633
+ const float m = d1q * bq8_1[ib32].ds[1];
1634
+ return d * sumi + m * delta;
1635
+ #else
1636
+ assert(false);
1637
+ #endif
1638
+ }
1639
+
1640
+ static __dpct_inline__ float
1641
+ vec_dot_iq1_m_q8_1(const void *__restrict__ vbq,
1642
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1643
+ #if QK_K == 256
1644
+ const block_iq1_m * bq1 = (const block_iq1_m *) vbq;
1645
+
1646
+ const int ib32 = iqs;
1647
+ int sumi[2] = {0, 0};
1648
+ float sumf[2] = {0.f, 0.f};
1649
+
1650
+ const int * q8 = (const int *)bq8_1[ib32].qs;
1651
+ for (int l = 0; l < 4; ++l) {
1652
+ const int * grid = (const int *)(iq1s_grid_gpu + (bq1->qs[4*ib32+l] | (((bq1->qh[2*ib32+l/2] >> 4*(l%2)) & 7) << 8)));
1653
+ int grid0 = grid[0] & 0x0f0f0f0f;
1654
+ int grid1 = (grid[0] >> 4) & 0x0f0f0f0f;
1655
+ sumi[l / 2] = dpct::dp4a(q8[2 * l + 1], grid1,
1656
+ dpct::dp4a(q8[2 * l + 0], grid0, sumi[l / 2]));
1657
+ const float delta = (bq1->qh[2*ib32+l/2] >> 4*(l%2)) & 0x08 ? -1-IQ1M_DELTA : -1+IQ1M_DELTA;
1658
+ const int sumy = dpct::dp4a(q8[2 * l + 1], 0x01010101,
1659
+ dpct::dp4a(q8[2 * l + 0], 0x01010101, 0));
1660
+ sumf[l/2] += delta*sumy;
1661
+ }
1662
+
1663
+ iq1m_scale_t scale;
1664
+ const uint16_t * sc = (const uint16_t *)bq1->scales;
1665
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);
1666
+ const float d = (float)scale.f16 * bq8_1[ib32].ds[0];
1667
+ return d * ((sumi[0] + sumf[0]) * (2*((sc[ib32/2] >> 6*(ib32%2)) & 0x7) + 1) + (sumi[1] + sumf[1]) * (2*((sc[ib32/2] >> (6*(ib32%2)+3)) & 0x7) + 1));
1668
+ #else
1669
+ assert(false);
1670
+ #endif
1671
+ }
1672
+
1673
+
1674
+ static __dpct_inline__ float
1675
+ vec_dot_iq4_nl_q8_1(const void *__restrict__ vbq,
1676
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1677
+
1678
+ const block_iq4_nl * bq = (const block_iq4_nl *) vbq;
1679
+
1680
+ const uint16_t * q4 = (const uint16_t *)bq->qs + 2*iqs;
1681
+ const int32_t * q8 = (const int32_t *)bq8_1->qs + iqs;
1682
+
1683
+ const uint8_t * values = (const uint8_t *)kvalues_iq4nl;
1684
+
1685
+ int v1, v2;
1686
+ int sumi1 = 0, sumi2 = 0;
1687
+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {
1688
+ const uint32_t aux = q4[2*l] | (q4[2*l+1] << 16);
1689
+ get_int_from_table_16(aux, values, v1, v2);
1690
+ sumi1 = dpct::dp4a(v1, q8[l + 0], sumi1);
1691
+ sumi2 = dpct::dp4a(v2, q8[l + 4], sumi2);
1692
+ }
1693
+
1694
+ const float d = (float)bq->d * bq8_1->ds[0];
1695
+ return d * (sumi1 + sumi2);
1696
+ }
1697
+
1698
+
1699
+ static __dpct_inline__ float
1700
+ vec_dot_iq4_xs_q8_1(const void *__restrict__ vbq,
1701
+ const block_q8_1 *__restrict__ bq8_1, const int &iqs) {
1702
+
1703
+ #if QK_K == 256
1704
+ const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq;
1705
+ const uint8_t * values = (const uint8_t *)kvalues_iq4nl;
1706
+
1707
+ // iqs is 0...7
1708
+ const int ib32 = iqs;
1709
+ const int32_t * q8 = (const int *)bq8_1[ib32].qs;
1710
+ const uint32_t * q4 = (const uint32_t *)bq4->qs + 4*ib32;
1711
+ const int8_t ls = ((bq4->scales_l[ib32/2] >> 4*(ib32%2)) & 0xf) | (((bq4->scales_h >> 2*ib32) & 3) << 4);
1712
+ const float d = (float)bq4->d * (ls - 32) * bq8_1[ib32].ds[0];
1713
+ int v1, v2;
1714
+ int sumi1 = 0, sumi2 = 0;
1715
+ for (int j = 0; j < 4; ++j) {
1716
+ get_int_from_table_16(q4[j], values, v1, v2);
1717
+ sumi1 = dpct::dp4a(v1, q8[j + 0], sumi1);
1718
+ sumi2 = dpct::dp4a(v2, q8[j + 4], sumi2);
1719
+ }
1720
+ return d * (sumi1 + sumi2);
1721
+ #else
1722
+ assert(false);
1723
+ #endif
1724
+ }
1725
+
1726
+ #endif // GGML_SYCL_VECDOTQ_HPP