elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,2011 @@
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+ #ifdef cl_khr_integer_dot_product
3
+ #pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
4
+ #define FA_HAVE_INT_DOT 1
5
+ #endif
6
+
7
+ #ifdef cl_khr_subgroup_shuffle
8
+ #pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable
9
+ #define HAS_SUBGROUP_SHUFFLE 1
10
+ #elif defined(cl_qcom_subgroup_shuffle)
11
+ #pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable
12
+ #define HAS_SUBGROUP_SHUFFLE 1
13
+ // Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR
14
+ // name, so calling it is an implicit declaration and the program fails to build.
15
+ // Route it to the qcom builtin.
16
+ #define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f)
17
+ #endif
18
+
19
+ // Flash attention: Q=f32, K=q4_0, V=q4_0.
20
+ // Block = half d + uchar qs[16]; qs[j] low/high nibble -> elem j / j+16.
21
+ // Dequant: val[i] = d * (nibble_i - 8). dp4a path runs on raw 0..15 nibbles
22
+ // and applies the -8*sum(q) correction once per block (needs Q q_sum).
23
+
24
+ #define ACC_TYPE float
25
+ #define ACC_TYPE4 float4
26
+ #define Q_DATA_TYPE4 float4
27
+ #define O_DATA_TYPE4 float4
28
+ #define MASK_DATA_TYPE half
29
+ #define CONVERT_Q_ACC4(x) (x)
30
+ #define CONVERT_O_DATA4(x) (x)
31
+
32
+ #define DK_VEC (DK/4)
33
+ #define DV_VEC (DV/4)
34
+
35
+ #ifndef FA_SG
36
+ #define FA_SG 64
37
+ #endif
38
+ #define Q1_WG_SIZE FA_SG
39
+
40
+ // The kernels are built with -cl-finite-math-only. On some older Adreno GPUs,
41
+ // infinite operand can cause undefined behavior and miscompilation for exp.
42
+ // Therefore, a large negative value is used instead.
43
+ #define FA_M_INIT (-3.0e38f)
44
+
45
+ #define QK4_0 32
46
+ #define Q4_0_BLOCK_SIZE 18
47
+
48
+ #define DK_Q4_BLOCKS (DK / QK4_0)
49
+ #define DV_Q4_BLOCKS (DV / QK4_0)
50
+
51
+ inline float dot_q4_0_f32(const global char * block_ptr, ACC_TYPE4 * q_slice) {
52
+ float d = vload_half(0, (const global half *)block_ptr);
53
+ const global uchar * qs = (const global uchar *)(block_ptr + 2);
54
+
55
+ float sum = 0.0f;
56
+ // Low nibbles -> elems 0..15.
57
+ #pragma unroll
58
+ for (int g = 0; g < 4; ++g) {
59
+ float4 nv = (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f,
60
+ (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f,
61
+ (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f,
62
+ (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f);
63
+ sum += dot(q_slice[g], nv);
64
+ }
65
+ // High nibbles -> elems 16..31.
66
+ #pragma unroll
67
+ for (int g = 0; g < 4; ++g) {
68
+ float4 nv = (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f,
69
+ (float)(int)(qs[g*4 + 1] >> 4) - 8.0f,
70
+ (float)(int)(qs[g*4 + 2] >> 4) - 8.0f,
71
+ (float)(int)(qs[g*4 + 3] >> 4) - 8.0f);
72
+ sum += dot(q_slice[4 + g], nv);
73
+ }
74
+ return sum * d;
75
+ }
76
+
77
+ #ifdef FA_HAVE_INT_DOT
78
+ inline uint pack_i8x4(char a, char b, char c, char d) {
79
+ return ((uint)(uchar)a) |
80
+ ((uint)(uchar)b) << 8 |
81
+ ((uint)(uchar)c) << 16 |
82
+ ((uint)(uchar)d) << 24;
83
+ }
84
+
85
+ // Returns (qd, q_sum); q_sum feeds the -8*sum(q) bias correction.
86
+ typedef struct {
87
+ float qd;
88
+ int q_sum;
89
+ } q4_q_block_info;
90
+
91
+ inline q4_q_block_info quant_q_block_int8_packed_q4(const ACC_TYPE4 * q_block,
92
+ uint * out_packed) {
93
+ float amax = 0.0f;
94
+ #pragma unroll
95
+ for (int i = 0; i < 8; ++i) {
96
+ float4 av = fabs(q_block[i]);
97
+ amax = fmax(amax, fmax(fmax(av.s0, av.s1), fmax(av.s2, av.s3)));
98
+ }
99
+ float qd = amax / 127.0f;
100
+ float qid = (amax > 0.0f) ? 127.0f / amax : 0.0f;
101
+
102
+ int q_sum = 0;
103
+ #pragma unroll
104
+ for (int i = 0; i < 8; ++i) {
105
+ float4 v = q_block[i] * qid;
106
+ char a = (char)((int)round(v.s0));
107
+ char b = (char)((int)round(v.s1));
108
+ char c = (char)((int)round(v.s2));
109
+ char d = (char)((int)round(v.s3));
110
+ out_packed[i] = pack_i8x4(a, b, c, d);
111
+ q_sum += (int)a + (int)b + (int)c + (int)d;
112
+ }
113
+ q4_q_block_info info = { qd, q_sum };
114
+ return info;
115
+ }
116
+
117
+ // k_packed[0..3] = low nibbles (Q elems 0..15), k_packed[4..7] = high (16..31).
118
+ inline void pack_q4_0_nibbles(const global uchar * qs, uint * k_packed) {
119
+ #pragma unroll
120
+ for (int g = 0; g < 4; ++g) {
121
+ uchar b0 = qs[g*4 + 0];
122
+ uchar b1 = qs[g*4 + 1];
123
+ uchar b2 = qs[g*4 + 2];
124
+ uchar b3 = qs[g*4 + 3];
125
+ k_packed[g] =
126
+ ((uint)(b0 & 0x0F)) |
127
+ ((uint)(b1 & 0x0F)) << 8 |
128
+ ((uint)(b2 & 0x0F)) << 16 |
129
+ ((uint)(b3 & 0x0F)) << 24;
130
+ k_packed[4 + g] =
131
+ ((uint)(b0 >> 4)) |
132
+ ((uint)(b1 >> 4)) << 8 |
133
+ ((uint)(b2 >> 4)) << 16 |
134
+ ((uint)(b3 >> 4)) << 24;
135
+ }
136
+ }
137
+
138
+ inline float dot_q4_0_int(const global char * k_block_ptr,
139
+ const uint * q_packed,
140
+ float q_d,
141
+ int q_sum) {
142
+ float kd = vload_half(0, (const global half *)k_block_ptr);
143
+ const global uchar * k_qs = (const global uchar *)(k_block_ptr + 2);
144
+
145
+ uint k_packed[8];
146
+ pack_q4_0_nibbles(k_qs, k_packed);
147
+
148
+ int sum = 0;
149
+ #pragma unroll
150
+ for (int i = 0; i < 8; ++i) {
151
+ sum = dot_acc_sat_4x8packed_ss_int(q_packed[i], k_packed[i], sum);
152
+ }
153
+ // Correct raw-nibble sum: (nibble - 8) bias -> subtract 8 * q_sum.
154
+ return (float)(sum - 8 * q_sum) * q_d * kd;
155
+ }
156
+ #endif // FA_HAVE_INT_DOT
157
+
158
+ inline void dequant_q4_0_f32(const global char * block_ptr, ACC_TYPE4 * out) {
159
+ float d = vload_half(0, (const global half *)block_ptr);
160
+ const global uchar * qs = (const global uchar *)(block_ptr + 2);
161
+
162
+ #pragma unroll
163
+ for (int g = 0; g < 4; ++g) {
164
+ out[g] = d * (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f,
165
+ (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f,
166
+ (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f,
167
+ (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f);
168
+ }
169
+ #pragma unroll
170
+ for (int g = 0; g < 4; ++g) {
171
+ out[4 + g] = d * (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f,
172
+ (float)(int)(qs[g*4 + 1] >> 4) - 8.0f,
173
+ (float)(int)(qs[g*4 + 2] >> 4) - 8.0f,
174
+ (float)(int)(qs[g*4 + 3] >> 4) - 8.0f);
175
+ }
176
+ }
177
+
178
+ // max_bias<=0 returns 1.0 so score += 1.0 * mask[k] stays a no-op multiplier.
179
+ inline float get_alibi_slope(float max_bias, int head_idx, int n_head_log2, float m0, float m1) {
180
+ if (max_bias <= 0.0f) return 1.0f;
181
+ float base = (head_idx < n_head_log2) ? m0 : m1;
182
+ int exph = (head_idx < n_head_log2) ? (head_idx + 1) : (2*(head_idx - n_head_log2) + 1);
183
+ return pow(base, (float)exph);
184
+ }
185
+
186
+ // q1 decode: one query row per WG, threads sweep KV positions.
187
+ __kernel void flash_attn_f32_q4_0_q1(
188
+ const global void * q_void, ulong q_offset,
189
+ const global void * k_void, ulong k_offset,
190
+ const global void * v_void, ulong v_offset,
191
+ global void * o_void, ulong o_offset,
192
+ const float scale,
193
+ const int n_q,
194
+ const int n_kv,
195
+ const int is_causal,
196
+ const int n_head,
197
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
198
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
199
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
200
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
201
+ const float max_bias,
202
+ const float m0,
203
+ const float m1,
204
+ const int n_head_log2,
205
+ const float logit_softcap,
206
+ const int n_head_kv,
207
+ const global void* mask_void,
208
+ const ulong mask_offset,
209
+ const ulong mask_nb1,
210
+ const ulong mask_nb2,
211
+ const ulong mask_nb3,
212
+ const int mask_ne2,
213
+ const int mask_ne3,
214
+ const global void* sinks_void,
215
+ const ulong sinks_offset
216
+ ) {
217
+ const int tid = get_local_id(0);
218
+ const int head_batch_idx = get_global_id(1);
219
+
220
+ const int batch_idx = head_batch_idx / n_head;
221
+ const int head_idx = head_batch_idx % n_head;
222
+
223
+ const int gqa_ratio = n_head / n_head_kv;
224
+ const int head_kv_idx = head_idx / gqa_ratio;
225
+
226
+ const global char* q_base = (const global char*)q_void + q_offset;
227
+ const global char* k_base = (const global char*)k_void + k_offset;
228
+ const global char* v_base = (const global char*)v_void + v_offset;
229
+ global char* o_base = (global char*)o_void + o_offset;
230
+
231
+ const global char* mask_base = NULL;
232
+ if (mask_void != NULL) {
233
+ const int mask_head_idx = head_idx % mask_ne2;
234
+ const int mask_batch_idx = batch_idx % mask_ne3;
235
+ mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;
236
+ }
237
+
238
+ ACC_TYPE4 q_priv[DK_VEC];
239
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2;
240
+ const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset);
241
+ #pragma unroll
242
+ for (int i = 0; i < DK_VEC; ++i) {
243
+ q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]);
244
+ }
245
+
246
+ #ifdef FA_HAVE_INT_DOT
247
+ // Quantise Q once per thread: 8 uints + qd + q_sum per block.
248
+ uint q_packed[DK_Q4_BLOCKS * 8];
249
+ float q_d_scale[DK_Q4_BLOCKS];
250
+ int q_sum_arr[DK_Q4_BLOCKS];
251
+ #pragma unroll
252
+ for (int b = 0; b < DK_Q4_BLOCKS; ++b) {
253
+ q4_q_block_info info = quant_q_block_int8_packed_q4(&q_priv[b * 8], &q_packed[b * 8]);
254
+ q_d_scale[b] = info.qd;
255
+ q_sum_arr[b] = info.q_sum;
256
+ }
257
+ #endif
258
+
259
+ float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
260
+
261
+ const global ACC_TYPE* sinks_ptr = NULL;
262
+ if (sinks_void != NULL) {
263
+ sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);
264
+ }
265
+
266
+ // One-pass online softmax (FA-2): single sweep over kv positions,
267
+ // updating per-thread (m_i, l_i, o_acc) per K. Eliminates the second
268
+ // K read of the original two-pass implementation.
269
+ ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT;
270
+ ACC_TYPE l_i = 0.0f;
271
+ ACC_TYPE4 o_acc[DV_VEC];
272
+ #pragma unroll
273
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);
274
+
275
+ for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) {
276
+ const global char* k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
277
+ const global char* v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
278
+
279
+ ACC_TYPE score = 0.0f;
280
+ #pragma unroll
281
+ for (int b = 0; b < DK_Q4_BLOCKS; b++) {
282
+ #ifdef FA_HAVE_INT_DOT
283
+ score += dot_q4_0_int(k_row + b * Q4_0_BLOCK_SIZE,
284
+ &q_packed[b * 8], q_d_scale[b], q_sum_arr[b]);
285
+ #else
286
+ score += dot_q4_0_f32(k_row + b * Q4_0_BLOCK_SIZE, &q_priv[b * 8]);
287
+ #endif
288
+ }
289
+ score *= scale;
290
+
291
+ if (mask_base != NULL) {
292
+ const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);
293
+ score += slope * (ACC_TYPE)mask_ptr[k_idx];
294
+ }
295
+ if (logit_softcap > 0.0f) {
296
+ score = logit_softcap * tanh(score / logit_softcap);
297
+ }
298
+
299
+ // Online softmax step.
300
+ const ACC_TYPE m_new = max(m_i, score);
301
+ const ACC_TYPE alpha = exp(m_i - m_new);
302
+ const ACC_TYPE p = exp(score - m_new);
303
+
304
+ l_i = alpha * l_i + p;
305
+ #pragma unroll
306
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha;
307
+
308
+ #pragma unroll
309
+ for (int b = 0; b < DV_Q4_BLOCKS; b++) {
310
+ ACC_TYPE4 v_dequant[8];
311
+ dequant_q4_0_f32(v_row + b * Q4_0_BLOCK_SIZE, v_dequant);
312
+ #pragma unroll
313
+ for (int i = 0; i < 8; i++) {
314
+ o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]);
315
+ }
316
+ }
317
+
318
+ m_i = m_new;
319
+ }
320
+
321
+ // Cross-thread reduce: max(m_i) -> m_final, rescale per-thread l_i and
322
+ // o_acc by alpha = exp(m_i_thread - m_final) before sum-reduce.
323
+ __local ACC_TYPE local_m[Q1_WG_SIZE];
324
+ local_m[tid] = m_i;
325
+ barrier(CLK_LOCAL_MEM_FENCE);
326
+ #pragma unroll
327
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
328
+ if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]);
329
+ barrier(CLK_LOCAL_MEM_FENCE);
330
+ }
331
+ const ACC_TYPE m_final = local_m[0];
332
+
333
+ const ACC_TYPE alpha_final = exp(m_i - m_final);
334
+ l_i *= alpha_final;
335
+ #pragma unroll
336
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final;
337
+
338
+ __local ACC_TYPE local_l[Q1_WG_SIZE];
339
+ __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE];
340
+ local_l[tid] = l_i;
341
+ barrier(CLK_LOCAL_MEM_FENCE);
342
+ #pragma unroll
343
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
344
+ if (tid < s) local_l[tid] += local_l[tid + s];
345
+ barrier(CLK_LOCAL_MEM_FENCE);
346
+ }
347
+
348
+ const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1;
349
+ global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset);
350
+ ACC_TYPE l_final = local_l[0];
351
+
352
+ if (sinks_ptr != NULL) {
353
+ l_final += exp(sinks_ptr[head_idx] - m_final);
354
+ }
355
+
356
+ if (l_final > 0.0f) {
357
+ const ACC_TYPE l_inv = 1.0f / l_final;
358
+ for (int i = 0; i < DV_VEC; i++) {
359
+ local_o_comp[tid] = o_acc[i];
360
+ barrier(CLK_LOCAL_MEM_FENCE);
361
+ #pragma unroll
362
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
363
+ if (tid < s) local_o_comp[tid] += local_o_comp[tid + s];
364
+ barrier(CLK_LOCAL_MEM_FENCE);
365
+ }
366
+ if (tid == 0) {
367
+ o_row[i] = CONVERT_O_DATA4(local_o_comp[0] * l_inv);
368
+ }
369
+ }
370
+ } else if (tid == 0) {
371
+ #pragma unroll
372
+ for (int i = 0; i < DV_VEC; ++i) o_row[i] = (O_DATA_TYPE4)(0.0f);
373
+ }
374
+ }
375
+
376
+ #ifdef cl_intel_subgroups
377
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
378
+ #else
379
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
380
+ #endif
381
+
382
+ #ifdef cl_qcom_reqd_sub_group_size
383
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
384
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
385
+ #else
386
+ #define REQD_SUBGROUP_SIZE_64
387
+ #endif
388
+
389
+ #define VEC_NSG 4
390
+ #define VEC_WG_SIZE (Q1_WG_SIZE * VEC_NSG)
391
+ #define Q1V_DV_PER_THREAD ((DV_VEC + Q1_WG_SIZE - 1) / Q1_WG_SIZE)
392
+
393
+ // Dequant one float4 lane (0..7) from a q4_0 block.
394
+ // Lanes 0..3 → low nibbles of qs[0..15], lanes 4..7 → high nibbles.
395
+ inline float4 dequant_q4_0_lane(const global char * block_ptr, int lane) {
396
+ const float d = vload_half(0, (const global half *)block_ptr);
397
+ const global uchar * qs = (const global uchar *)(block_ptr + 2);
398
+ const int g = lane & 3;
399
+ const int shift = (lane < 4) ? 0 : 4;
400
+ return d * (float4)((float)((qs[g*4+0] >> shift) & 0x0F) - 8.0f,
401
+ (float)((qs[g*4+1] >> shift) & 0x0F) - 8.0f,
402
+ (float)((qs[g*4+2] >> shift) & 0x0F) - 8.0f,
403
+ (float)((qs[g*4+3] >> shift) & 0x0F) - 8.0f);
404
+ }
405
+
406
+ REQD_SUBGROUP_SIZE_64
407
+ __kernel void flash_attn_f32_q4_0_q1_vec(
408
+ const global void * q_void, ulong q_offset,
409
+ const global void * k_void, ulong k_offset,
410
+ const global void * v_void, ulong v_offset,
411
+ global void * o_void, ulong o_offset,
412
+ const float scale,
413
+ const int n_q,
414
+ const int n_kv,
415
+ const int is_causal,
416
+ const int n_head,
417
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
418
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
419
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
420
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
421
+ const float max_bias,
422
+ const float m0,
423
+ const float m1,
424
+ const int n_head_log2,
425
+ const float logit_softcap,
426
+ const int n_head_kv,
427
+ const global void* mask_void,
428
+ const ulong mask_offset,
429
+ const ulong mask_nb1,
430
+ const ulong mask_nb2,
431
+ const ulong mask_nb3,
432
+ const int mask_ne2,
433
+ const int mask_ne3,
434
+ const global void* sinks_void,
435
+ const ulong sinks_offset
436
+ ) {
437
+ const int tid = get_local_id(0);
438
+ const int sgid = tid / Q1_WG_SIZE;
439
+ const int tid_sg = tid % Q1_WG_SIZE;
440
+ const int head_batch_idx = get_global_id(1);
441
+
442
+ const int batch_idx = head_batch_idx / n_head;
443
+ const int head_idx = head_batch_idx % n_head;
444
+
445
+ const int gqa_ratio = n_head / n_head_kv;
446
+ const int head_kv_idx = head_idx / gqa_ratio;
447
+
448
+ const global char * q_base = (const global char *) q_void + q_offset;
449
+ const global char * k_base = (const global char *) k_void + k_offset;
450
+ const global char * v_base = (const global char *) v_void + v_offset;
451
+ global char * o_base = (global char *) o_void + o_offset;
452
+
453
+ const global char * mask_base = NULL;
454
+ if (mask_void != NULL) {
455
+ const int mask_head_idx = head_idx % mask_ne2;
456
+ const int mask_batch_idx = batch_idx % mask_ne3;
457
+ mask_base = (const global char *) mask_void + mask_offset +
458
+ mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;
459
+ }
460
+
461
+ __local ACC_TYPE4 q_shared[DK_VEC];
462
+ {
463
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2;
464
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
465
+ for (int i = tid; i < DK_VEC; i += VEC_WG_SIZE) {
466
+ q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]);
467
+ }
468
+ }
469
+ barrier(CLK_LOCAL_MEM_FENCE);
470
+
471
+ #ifdef FA_HAVE_INT_DOT
472
+ // quantize Q to int8-packed uints + per-block (qd, q_sum) once per WG for dp4a
473
+ // one thread per Q block, remaining threads idle this step
474
+ __local uint q_packed_shared[DK_Q4_BLOCKS * 8];
475
+ __local float q_d_shared[DK_Q4_BLOCKS];
476
+ __local int q_sum_shared[DK_Q4_BLOCKS];
477
+ if (tid < DK_Q4_BLOCKS) {
478
+ ACC_TYPE4 q_block[8];
479
+ #pragma unroll
480
+ for (int i = 0; i < 8; ++i) q_block[i] = q_shared[tid * 8 + i];
481
+ uint packed[8];
482
+ q4_q_block_info info = quant_q_block_int8_packed_q4(q_block, packed);
483
+ #pragma unroll
484
+ for (int i = 0; i < 8; ++i) q_packed_shared[tid * 8 + i] = packed[i];
485
+ q_d_shared[tid] = info.qd;
486
+ q_sum_shared[tid] = info.q_sum;
487
+ }
488
+ barrier(CLK_LOCAL_MEM_FENCE);
489
+ #endif
490
+
491
+ const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
492
+
493
+ const global ACC_TYPE * sinks_ptr = NULL;
494
+ if (sinks_void != NULL) {
495
+ sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset);
496
+ }
497
+
498
+ ACC_TYPE4 o_acc[Q1V_DV_PER_THREAD];
499
+ #pragma unroll
500
+ for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);
501
+
502
+ ACC_TYPE m_i = FA_M_INIT;
503
+ ACC_TYPE l_i = 0.0f;
504
+
505
+ const int kv_per_sg = (n_kv + VEC_NSG - 1) / VEC_NSG;
506
+ const int kv_start = sgid * kv_per_sg;
507
+ const int kv_end = min(n_kv, kv_start + kv_per_sg);
508
+
509
+ for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) {
510
+ const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
511
+ const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
512
+
513
+ #ifdef FA_HAVE_INT_DOT
514
+ // per-lane dp4a: each lane packs 4 raw q4_0 nibbles into a uint,
515
+ // then dot_acc_sat_4x8packed_ss_int against the matching uint.
516
+ ACC_TYPE lane_contrib = 0.0f;
517
+ for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) {
518
+ const int block_idx = qk / 8;
519
+ const int lane_in_block = qk % 8;
520
+ const int g = lane_in_block & 3;
521
+ const int shift = (lane_in_block < 4) ? 0 : 4;
522
+ const global char * k_block = k_row + block_idx * Q4_0_BLOCK_SIZE;
523
+ const float kd = vload_half(0, (const global half *)k_block);
524
+ const global uchar * k_qs = (const global uchar *)(k_block + 2);
525
+ const uchar b0 = k_qs[g*4 + 0];
526
+ const uchar b1 = k_qs[g*4 + 1];
527
+ const uchar b2 = k_qs[g*4 + 2];
528
+ const uchar b3 = k_qs[g*4 + 3];
529
+ const uint k_packed = ((uint)((b0 >> shift) & 0x0F)) |
530
+ ((uint)((b1 >> shift) & 0x0F)) << 8 |
531
+ ((uint)((b2 >> shift) & 0x0F)) << 16 |
532
+ ((uint)((b3 >> shift) & 0x0F)) << 24;
533
+ const uint q_packed_lane = q_packed_shared[block_idx * 8 + lane_in_block];
534
+ const int raw_dot = dot_acc_sat_4x8packed_ss_int(q_packed_lane, k_packed, 0);
535
+ const float qd = q_d_shared[block_idx];
536
+ const float block_scale = qd * kd;
537
+ float contrib = (float)raw_dot * block_scale;
538
+ if (lane_in_block == 0) {
539
+ // block bias correction is per-block
540
+ const int q_sum_b = q_sum_shared[block_idx];
541
+ contrib -= 8.0f * block_scale * (float)q_sum_b;
542
+ }
543
+ lane_contrib += contrib;
544
+ }
545
+ ACC_TYPE score = sub_group_reduce_add(lane_contrib) * scale;
546
+ #else
547
+ ACC_TYPE4 dot4 = (ACC_TYPE4)(0.0f);
548
+ for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) {
549
+ const int block_idx = qk / 8;
550
+ const int lane = qk % 8;
551
+ const float4 k_v = dequant_q4_0_lane(k_row + block_idx * Q4_0_BLOCK_SIZE, lane);
552
+ dot4 = mad(q_shared[qk], k_v, dot4);
553
+ }
554
+ ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3;
555
+ ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale;
556
+ #endif
557
+
558
+ if (mask_base != NULL) {
559
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base;
560
+ score += slope * (ACC_TYPE) mask_ptr[k_idx];
561
+ }
562
+ if (logit_softcap > 0.0f) {
563
+ score = logit_softcap * tanh(score / logit_softcap);
564
+ }
565
+
566
+ const ACC_TYPE m_new = max(m_i, score);
567
+ const ACC_TYPE scale_prev = native_exp(m_i - m_new);
568
+ const ACC_TYPE p = native_exp(score - m_new);
569
+
570
+ int idx = 0;
571
+ for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) {
572
+ const int block_idx = dv / 8;
573
+ const int lane = dv % 8;
574
+ const float4 v_v = dequant_q4_0_lane(v_row + block_idx * Q4_0_BLOCK_SIZE, lane);
575
+ o_acc[idx] = mad(p, v_v, o_acc[idx] * scale_prev);
576
+ }
577
+ l_i = l_i * scale_prev + p;
578
+ m_i = m_new;
579
+ }
580
+
581
+ __local ACC_TYPE sg_m[VEC_NSG];
582
+ __local ACC_TYPE sg_l[VEC_NSG];
583
+ __local ACC_TYPE4 sg_o[VEC_NSG][DV_VEC];
584
+
585
+ if (tid_sg == 0) {
586
+ sg_m[sgid] = m_i;
587
+ sg_l[sgid] = l_i;
588
+ }
589
+ {
590
+ int idx = 0;
591
+ for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) {
592
+ sg_o[sgid][dv] = o_acc[idx];
593
+ }
594
+ }
595
+ barrier(CLK_LOCAL_MEM_FENCE);
596
+
597
+ if (sgid == 0) {
598
+ ACC_TYPE m_final = sg_m[0];
599
+ #pragma unroll
600
+ for (int s = 1; s < VEC_NSG; ++s) {
601
+ m_final = max(m_final, sg_m[s]);
602
+ }
603
+ if (sinks_ptr != NULL) {
604
+ m_final = max(m_final, sinks_ptr[head_idx]);
605
+ }
606
+
607
+ ACC_TYPE l_final = 0.0f;
608
+ #pragma unroll
609
+ for (int s = 0; s < VEC_NSG; ++s) {
610
+ l_final += sg_l[s] * native_exp(sg_m[s] - m_final);
611
+ }
612
+ if (sinks_ptr != NULL) {
613
+ l_final += native_exp(sinks_ptr[head_idx] - m_final);
614
+ }
615
+ const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f;
616
+
617
+ const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1;
618
+ global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset);
619
+
620
+ int idx = 0;
621
+ for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) {
622
+ ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f);
623
+ #pragma unroll
624
+ for (int s = 0; s < VEC_NSG; ++s) {
625
+ const ACC_TYPE alpha = native_exp(sg_m[s] - m_final);
626
+ o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv], o_merged);
627
+ }
628
+ o_row[dv] = CONVERT_O_DATA4(o_merged * l_inv);
629
+ }
630
+ }
631
+ }
632
+
633
+ // Flash-decoding split pass for q4_0 KV. Merge kernel is type-agnostic and
634
+ // shared with the f16/q8_0 FA kernels.
635
+ #define FA_PARTIAL_FLOATS (2 + DV)
636
+
637
+ __kernel void flash_attn_f32_q4_0_q1_split(
638
+ const global void * q_void, ulong q_offset,
639
+ const global void * k_void, ulong k_offset,
640
+ const global void * v_void, ulong v_offset,
641
+ const float scale,
642
+ const int n_q,
643
+ const int n_kv,
644
+ const int n_head,
645
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
646
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
647
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
648
+ const float max_bias,
649
+ const float m0,
650
+ const float m1,
651
+ const int n_head_log2,
652
+ const float logit_softcap,
653
+ const int n_head_kv,
654
+ const global void * mask_void,
655
+ const ulong mask_offset,
656
+ const ulong mask_nb1,
657
+ const ulong mask_nb2,
658
+ const ulong mask_nb3,
659
+ const int mask_ne2,
660
+ const int mask_ne3,
661
+ global float * partial_void,
662
+ const int n_splits,
663
+ const int kv_per_split
664
+ ) {
665
+ const int tid = get_local_id(0);
666
+ const int head_batch_idx = get_global_id(1);
667
+ const int split_q_idx = get_global_id(2);
668
+ const int split_idx = split_q_idx % n_splits;
669
+ const int q_idx = split_q_idx / n_splits;
670
+ const int batch_idx = head_batch_idx / n_head;
671
+ const int head_idx = head_batch_idx % n_head;
672
+ const int gqa_ratio = n_head / n_head_kv;
673
+ const int head_kv_idx = head_idx / gqa_ratio;
674
+
675
+ const int kv_start = split_idx * kv_per_split;
676
+ const int kv_end = min(kv_start + kv_per_split, n_kv);
677
+
678
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
679
+ const ulong record_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
680
+ * n_splits + split_idx);
681
+ global float * rec = partial_void + record_idx * record_stride;
682
+ global float4 * rec_o = (global float4 *) (rec + 2);
683
+
684
+ if (kv_start >= kv_end) {
685
+ if (tid == 0) {
686
+ rec[0] = FA_M_INIT;
687
+ rec[1] = 0.0f;
688
+ }
689
+ return;
690
+ }
691
+
692
+ const global char * q_base = (const global char *) q_void + q_offset;
693
+ const global char * k_base = (const global char *) k_void + k_offset;
694
+ const global char * v_base = (const global char *) v_void + v_offset;
695
+
696
+ const global char * mask_base = NULL;
697
+ if (mask_void != NULL) {
698
+ const int mask_head_idx = head_idx % mask_ne2;
699
+ const int mask_batch_idx = batch_idx % mask_ne3;
700
+ mask_base = (const global char *) mask_void + mask_offset +
701
+ mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 +
702
+ (ulong) q_idx * mask_nb1;
703
+ }
704
+
705
+ ACC_TYPE4 q_priv[DK_VEC];
706
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
707
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
708
+ #pragma unroll
709
+ for (int i = 0; i < DK_VEC; ++i) {
710
+ q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]);
711
+ }
712
+
713
+ #ifdef FA_HAVE_INT_DOT
714
+ uint q_packed[DK_Q4_BLOCKS * 8];
715
+ float q_d_scale[DK_Q4_BLOCKS];
716
+ int q_sum_arr[DK_Q4_BLOCKS];
717
+ #pragma unroll
718
+ for (int b = 0; b < DK_Q4_BLOCKS; ++b) {
719
+ q4_q_block_info info = quant_q_block_int8_packed_q4(&q_priv[b * 8], &q_packed[b * 8]);
720
+ q_d_scale[b] = info.qd;
721
+ q_sum_arr[b] = info.q_sum;
722
+ }
723
+ #endif
724
+
725
+ const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
726
+
727
+ // One-pass online softmax (FA-2): single sweep over the split's K range.
728
+ ACC_TYPE m_i = FA_M_INIT;
729
+ ACC_TYPE l_i = 0.0f;
730
+ ACC_TYPE4 o_acc[DV_VEC];
731
+ #pragma unroll
732
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);
733
+
734
+ for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) {
735
+ const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
736
+ const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
737
+ ACC_TYPE score = 0.0f;
738
+ #pragma unroll
739
+ for (int b = 0; b < DK_Q4_BLOCKS; ++b) {
740
+ #ifdef FA_HAVE_INT_DOT
741
+ score += dot_q4_0_int(k_row + b * Q4_0_BLOCK_SIZE,
742
+ &q_packed[b * 8], q_d_scale[b], q_sum_arr[b]);
743
+ #else
744
+ score += dot_q4_0_f32(k_row + b * Q4_0_BLOCK_SIZE, &q_priv[b * 8]);
745
+ #endif
746
+ }
747
+ score *= scale;
748
+ if (mask_base != NULL) {
749
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base);
750
+ score += slope * (ACC_TYPE) mask_ptr[k_idx];
751
+ }
752
+ if (logit_softcap > 0.0f) {
753
+ score = logit_softcap * tanh(score / logit_softcap);
754
+ }
755
+
756
+ // Online softmax step.
757
+ const ACC_TYPE m_new = max(m_i, score);
758
+ const ACC_TYPE alpha = exp(m_i - m_new);
759
+ const ACC_TYPE p = exp(score - m_new);
760
+
761
+ l_i = alpha * l_i + p;
762
+ #pragma unroll
763
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha;
764
+
765
+ #pragma unroll
766
+ for (int b = 0; b < DV_Q4_BLOCKS; ++b) {
767
+ ACC_TYPE4 v_dequant[8];
768
+ dequant_q4_0_f32(v_row + b * Q4_0_BLOCK_SIZE, v_dequant);
769
+ #pragma unroll
770
+ for (int i = 0; i < 8; ++i) {
771
+ o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]);
772
+ }
773
+ }
774
+
775
+ m_i = m_new;
776
+ }
777
+
778
+ // Cross-thread reduce: max(m_i) -> m_c, rescale per-thread l_i and o_acc
779
+ // by alpha = exp(m_i_thread - m_c) before sum-reduce.
780
+ __local ACC_TYPE local_m[Q1_WG_SIZE];
781
+ local_m[tid] = m_i;
782
+ barrier(CLK_LOCAL_MEM_FENCE);
783
+ #pragma unroll
784
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
785
+ if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]);
786
+ barrier(CLK_LOCAL_MEM_FENCE);
787
+ }
788
+ const ACC_TYPE m_c = local_m[0];
789
+
790
+ const ACC_TYPE alpha_final = exp(m_i - m_c);
791
+ l_i *= alpha_final;
792
+ #pragma unroll
793
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final;
794
+
795
+ __local ACC_TYPE local_l[Q1_WG_SIZE];
796
+ __local ACC_TYPE4 local_o[Q1_WG_SIZE];
797
+ local_l[tid] = l_i;
798
+ barrier(CLK_LOCAL_MEM_FENCE);
799
+ #pragma unroll
800
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
801
+ if (tid < s) local_l[tid] += local_l[tid + s];
802
+ barrier(CLK_LOCAL_MEM_FENCE);
803
+ }
804
+ const ACC_TYPE l_c = local_l[0];
805
+
806
+ if (tid == 0) {
807
+ rec[0] = (float) m_c;
808
+ rec[1] = (float) l_c;
809
+ }
810
+ for (int i = 0; i < DV_VEC; ++i) {
811
+ local_o[tid] = o_acc[i];
812
+ barrier(CLK_LOCAL_MEM_FENCE);
813
+ #pragma unroll
814
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
815
+ if (tid < s) local_o[tid] += local_o[tid + s];
816
+ barrier(CLK_LOCAL_MEM_FENCE);
817
+ }
818
+ if (tid == 0) {
819
+ rec_o[i] = local_o[0];
820
+ }
821
+ }
822
+ }
823
+
824
+ // Prefill: q4_0 K/V, n_q > 1. BLOCK_M × BLOCK_N tiling.
825
+ // K in local as packed nibbles + per-block scale; V dequant -> half in local.
826
+ // Requires DK % QK4_0 == 0 and DV % QK4_0 == 0.
827
+ #define KV_DATA_TYPE4 half4
828
+ #define CONVERT_KV_ACC4(x) convert_float4(x)
829
+
830
+ #define DK_Q4_BLOCKS_PREFILL (DK / QK4_0)
831
+ #define DV_Q4_BLOCKS_PREFILL (DV / QK4_0)
832
+
833
+ // N_SPLIT>1 splits DK/DV across N_SPLIT threads per query row; needs
834
+ // sub_group_shuffle_xor and DK_Q4_BLOCKS_PREFILL % N_SPLIT == 0.
835
+ #ifndef N_SPLIT
836
+ #define N_SPLIT 1
837
+ #endif
838
+
839
+ #if N_SPLIT > 1
840
+ #define SPLIT_DK_VEC (DK_VEC / N_SPLIT)
841
+ #define SPLIT_DV_VEC (DV_VEC / N_SPLIT)
842
+ #define SPLIT_DK_Q4_BLOCKS (DK_Q4_BLOCKS_PREFILL / N_SPLIT)
843
+ #define WG_SIZE (BLOCK_M * N_SPLIT)
844
+ #else
845
+ #define SPLIT_DK_VEC DK_VEC
846
+ #define SPLIT_DV_VEC DV_VEC
847
+ #define SPLIT_DK_Q4_BLOCKS DK_Q4_BLOCKS_PREFILL
848
+ #define WG_SIZE BLOCK_M
849
+ #endif
850
+
851
+ #ifndef MQ_GQA
852
+ #define MQ_GQA 4
853
+ #endif
854
+ #ifndef MQ_NSG_SPLIT
855
+ #define MQ_NSG_SPLIT 4
856
+ #endif
857
+ #define MQ_SPLIT_WG_SIZE_Q4 (Q1_WG_SIZE * MQ_NSG_SPLIT)
858
+
859
+ REQD_SUBGROUP_SIZE_64
860
+ __kernel void flash_attn_f32_q4_0_q1_vec_mq_split(
861
+ const global void * q_void, ulong q_offset,
862
+ const global void * k_void, ulong k_offset,
863
+ const global void * v_void, ulong v_offset,
864
+ const float scale,
865
+ const int n_q,
866
+ const int n_kv,
867
+ const int n_head,
868
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
869
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
870
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
871
+ const float max_bias,
872
+ const float m0,
873
+ const float m1,
874
+ const int n_head_log2,
875
+ const float logit_softcap,
876
+ const int n_head_kv,
877
+ const global void * mask_void,
878
+ const ulong mask_offset,
879
+ const ulong mask_nb1,
880
+ const ulong mask_nb2,
881
+ const ulong mask_nb3,
882
+ const int mask_ne2,
883
+ const int mask_ne3,
884
+ global float * partial_void,
885
+ const int n_splits,
886
+ const int kv_per_split
887
+ ) {
888
+ const int tid = get_local_id(0);
889
+ const int sgid = tid / Q1_WG_SIZE;
890
+ const int tid_sg = tid % Q1_WG_SIZE;
891
+ const int kvhead_batch_idx = get_global_id(1);
892
+ const int split_q_idx = get_global_id(2);
893
+ const int split_idx = split_q_idx % n_splits;
894
+ const int q_idx = split_q_idx / n_splits;
895
+
896
+ const int batch_idx = kvhead_batch_idx / n_head_kv;
897
+ const int head_kv_idx = kvhead_batch_idx % n_head_kv;
898
+
899
+ const int kv_start = split_idx * kv_per_split;
900
+ const int kv_end = min(kv_start + kv_per_split, n_kv);
901
+
902
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
903
+
904
+ if (kv_start >= kv_end) {
905
+ if (tid == 0) {
906
+ #pragma unroll
907
+ for (int h = 0; h < MQ_GQA; ++h) {
908
+ const int head_idx = head_kv_idx * MQ_GQA + h;
909
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
910
+ * n_splits + split_idx);
911
+ global float * rec = partial_void + rec_idx * record_stride;
912
+ rec[0] = FA_M_INIT;
913
+ rec[1] = 0.0f;
914
+ }
915
+ }
916
+ return;
917
+ }
918
+
919
+ const global char * q_base = (const global char *) q_void + q_offset;
920
+ const global char * k_base = (const global char *) k_void + k_offset;
921
+ const global char * v_base = (const global char *) v_void + v_offset;
922
+
923
+ __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC];
924
+ for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE_Q4) {
925
+ const int h = i / DK_VEC;
926
+ const int k = i % DK_VEC;
927
+ const int head_idx = head_kv_idx * MQ_GQA + h;
928
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
929
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
930
+ q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);
931
+ }
932
+ barrier(CLK_LOCAL_MEM_FENCE);
933
+
934
+ #ifdef FA_HAVE_INT_DOT
935
+ __local uint q_packed_shared[MQ_GQA * DK_Q4_BLOCKS * 8];
936
+ __local float q_d_shared[MQ_GQA * DK_Q4_BLOCKS];
937
+ __local int q_sum_shared[MQ_GQA * DK_Q4_BLOCKS];
938
+ {
939
+ const int active = MQ_GQA * DK_Q4_BLOCKS;
940
+ if (tid < active) {
941
+ const int h = tid / DK_Q4_BLOCKS;
942
+ const int block_id = tid % DK_Q4_BLOCKS;
943
+ ACC_TYPE4 q_block[8];
944
+ #pragma unroll
945
+ for (int i = 0; i < 8; ++i) q_block[i] = q_shared[h * DK_VEC + block_id * 8 + i];
946
+ uint packed[8];
947
+ q4_q_block_info info = quant_q_block_int8_packed_q4(q_block, packed);
948
+ #pragma unroll
949
+ for (int i = 0; i < 8; ++i) q_packed_shared[(h * DK_Q4_BLOCKS + block_id) * 8 + i] = packed[i];
950
+ q_d_shared[h * DK_Q4_BLOCKS + block_id] = info.qd;
951
+ q_sum_shared[h * DK_Q4_BLOCKS + block_id] = info.q_sum;
952
+ }
953
+ }
954
+ barrier(CLK_LOCAL_MEM_FENCE);
955
+ #endif
956
+
957
+ float slope[MQ_GQA];
958
+ #pragma unroll
959
+ for (int h = 0; h < MQ_GQA; ++h) {
960
+ slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);
961
+ }
962
+
963
+ const global char * mask_base[MQ_GQA];
964
+ if (mask_void != NULL) {
965
+ const int mask_batch_idx = batch_idx % mask_ne3;
966
+ const global char * mask_base_b = (const global char *) mask_void + mask_offset +
967
+ mask_batch_idx * mask_nb3 +
968
+ (ulong) q_idx * mask_nb1;
969
+ #pragma unroll
970
+ for (int h = 0; h < MQ_GQA; ++h) {
971
+ const int head_idx = head_kv_idx * MQ_GQA + h;
972
+ const int mask_head_idx = head_idx % mask_ne2;
973
+ mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;
974
+ }
975
+ } else {
976
+ #pragma unroll
977
+ for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL;
978
+ }
979
+
980
+ ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD];
981
+ ACC_TYPE m_i[MQ_GQA];
982
+ ACC_TYPE l_i[MQ_GQA];
983
+ #pragma unroll
984
+ for (int h = 0; h < MQ_GQA; ++h) {
985
+ m_i[h] = FA_M_INIT;
986
+ l_i[h] = 0.0f;
987
+ #pragma unroll
988
+ for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f);
989
+ }
990
+
991
+ const int kv_len = kv_end - kv_start;
992
+ const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT;
993
+ const int kv_lo = kv_start + sgid * kv_per_sg;
994
+ const int kv_hi = min(kv_end, kv_lo + kv_per_sg);
995
+
996
+ for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) {
997
+ const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
998
+ const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
999
+
1000
+ #ifdef FA_HAVE_INT_DOT
1001
+ ACC_TYPE lane_contrib[MQ_GQA];
1002
+ #pragma unroll
1003
+ for (int h = 0; h < MQ_GQA; ++h) lane_contrib[h] = 0.0f;
1004
+
1005
+ for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) {
1006
+ const int block_idx = qk / 8;
1007
+ const int lane_in_block = qk % 8;
1008
+ const int g = lane_in_block & 3;
1009
+ const int shift = (lane_in_block < 4) ? 0 : 4;
1010
+ const global char * k_block = k_row + block_idx * Q4_0_BLOCK_SIZE;
1011
+ const float kd = vload_half(0, (const global half *)k_block);
1012
+ const global uchar * k_qs = (const global uchar *)(k_block + 2);
1013
+ const uchar b0 = k_qs[g*4 + 0];
1014
+ const uchar b1 = k_qs[g*4 + 1];
1015
+ const uchar b2 = k_qs[g*4 + 2];
1016
+ const uchar b3 = k_qs[g*4 + 3];
1017
+ const uint k_packed = ((uint)((b0 >> shift) & 0x0F)) |
1018
+ ((uint)((b1 >> shift) & 0x0F)) << 8 |
1019
+ ((uint)((b2 >> shift) & 0x0F)) << 16 |
1020
+ ((uint)((b3 >> shift) & 0x0F)) << 24;
1021
+ #pragma unroll
1022
+ for (int h = 0; h < MQ_GQA; ++h) {
1023
+ const uint q_packed_lane = q_packed_shared[(h * DK_Q4_BLOCKS + block_idx) * 8 + lane_in_block];
1024
+ const int raw_dot = dot_acc_sat_4x8packed_ss_int(q_packed_lane, k_packed, 0);
1025
+ const float qd = q_d_shared[h * DK_Q4_BLOCKS + block_idx];
1026
+ const float block_scale = qd * kd;
1027
+ float contrib = (float) raw_dot * block_scale;
1028
+ if (lane_in_block == 0) {
1029
+ const int q_sum_b = q_sum_shared[h * DK_Q4_BLOCKS + block_idx];
1030
+ contrib -= 8.0f * block_scale * (float) q_sum_b;
1031
+ }
1032
+ lane_contrib[h] += contrib;
1033
+ }
1034
+ }
1035
+
1036
+ ACC_TYPE score[MQ_GQA];
1037
+ #pragma unroll
1038
+ for (int h = 0; h < MQ_GQA; ++h) {
1039
+ ACC_TYPE s = sub_group_reduce_add(lane_contrib[h]) * scale;
1040
+ if (mask_base[h] != NULL) {
1041
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
1042
+ s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
1043
+ }
1044
+ if (logit_softcap > 0.0f) {
1045
+ s = logit_softcap * tanh(s / logit_softcap);
1046
+ }
1047
+ score[h] = s;
1048
+ }
1049
+ #else
1050
+ // fallback float-dequant K dot
1051
+ ACC_TYPE4 dot4[MQ_GQA];
1052
+ #pragma unroll
1053
+ for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f);
1054
+
1055
+ for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) {
1056
+ const int block_idx = qk / 8;
1057
+ const int lane = qk % 8;
1058
+ const float4 k_v = dequant_q4_0_lane(k_row + block_idx * Q4_0_BLOCK_SIZE, lane);
1059
+ #pragma unroll
1060
+ for (int h = 0; h < MQ_GQA; ++h) {
1061
+ dot4[h] = mad(q_shared[h * DK_VEC + qk], k_v, dot4[h]);
1062
+ }
1063
+ }
1064
+
1065
+ ACC_TYPE score[MQ_GQA];
1066
+ #pragma unroll
1067
+ for (int h = 0; h < MQ_GQA; ++h) {
1068
+ const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
1069
+ ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale;
1070
+ if (mask_base[h] != NULL) {
1071
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
1072
+ s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
1073
+ }
1074
+ if (logit_softcap > 0.0f) {
1075
+ s = logit_softcap * tanh(s / logit_softcap);
1076
+ }
1077
+ score[h] = s;
1078
+ }
1079
+ #endif
1080
+
1081
+ ACC_TYPE p_h[MQ_GQA];
1082
+ ACC_TYPE sp_h[MQ_GQA];
1083
+ #pragma unroll
1084
+ for (int h = 0; h < MQ_GQA; ++h) {
1085
+ const ACC_TYPE m_new = max(m_i[h], score[h]);
1086
+ sp_h[h] = native_exp(m_i[h] - m_new);
1087
+ p_h[h] = native_exp(score[h] - m_new);
1088
+ l_i[h] = l_i[h] * sp_h[h] + p_h[h];
1089
+ m_i[h] = m_new;
1090
+ }
1091
+
1092
+ int idx = 0;
1093
+ for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) {
1094
+ const int block_idx = dv / 8;
1095
+ const int lane = dv % 8;
1096
+ const float4 v_v = dequant_q4_0_lane(v_row + block_idx * Q4_0_BLOCK_SIZE, lane);
1097
+ #pragma unroll
1098
+ for (int h = 0; h < MQ_GQA; ++h) {
1099
+ o_acc[h][idx] = mad(p_h[h], v_v, o_acc[h][idx] * sp_h[h]);
1100
+ }
1101
+ }
1102
+ }
1103
+
1104
+ // per-h cross-subgroup merge
1105
+ __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT];
1106
+ __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT];
1107
+ __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC];
1108
+
1109
+ if (tid_sg == 0) {
1110
+ #pragma unroll
1111
+ for (int h = 0; h < MQ_GQA; ++h) {
1112
+ sg_m[h][sgid] = m_i[h];
1113
+ sg_l[h][sgid] = l_i[h];
1114
+ }
1115
+ }
1116
+
1117
+ #pragma unroll
1118
+ for (int h = 0; h < MQ_GQA; ++h) {
1119
+ {
1120
+ int idx = 0;
1121
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) {
1122
+ sg_o[sgid][dv_idx] = o_acc[h][idx];
1123
+ }
1124
+ }
1125
+ barrier(CLK_LOCAL_MEM_FENCE);
1126
+
1127
+ if (sgid == 0) {
1128
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1129
+
1130
+ ACC_TYPE m_c = sg_m[h][0];
1131
+ #pragma unroll
1132
+ for (int s = 1; s < MQ_NSG_SPLIT; ++s) {
1133
+ m_c = max(m_c, sg_m[h][s]);
1134
+ }
1135
+ ACC_TYPE l_c = 0.0f;
1136
+ #pragma unroll
1137
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
1138
+ l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c);
1139
+ }
1140
+
1141
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
1142
+ * n_splits + split_idx);
1143
+ global float * rec = partial_void + rec_idx * record_stride;
1144
+ global float4 * rec_o = (global float4 *) (rec + 2);
1145
+
1146
+ if (tid_sg == 0) {
1147
+ rec[0] = (float) m_c;
1148
+ rec[1] = (float) l_c;
1149
+ }
1150
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) {
1151
+ ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f);
1152
+ #pragma unroll
1153
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
1154
+ const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c);
1155
+ o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged);
1156
+ }
1157
+ rec_o[dv_idx] = o_merged;
1158
+ }
1159
+ }
1160
+ barrier(CLK_LOCAL_MEM_FENCE);
1161
+ }
1162
+ }
1163
+
1164
+ // flash_attn_f32_q4_0_q1_vec_mq_split_c8 — cluster-parallel variant of the MQ
1165
+ // split, port of flash_attn_f32_f16_q1_vec_mq_split_c8
1166
+ // Requires dp4a + subgroup shuffles
1167
+
1168
+ #if defined(FA_HAVE_INT_DOT) && defined(HAS_SUBGROUP_SHUFFLE)
1169
+
1170
+ #ifndef FA_CL_C
1171
+ #define FA_CL_C 8
1172
+ #endif
1173
+
1174
+ // Lane striping requires DK/DV to divide across the cluster (see f16 c8).
1175
+ #if (DK_VEC % FA_CL_C) == 0 && (DV_VEC % FA_CL_C) == 0
1176
+ #define FA_CL_NCL (Q1_WG_SIZE / FA_CL_C) // clusters (position streams) per subgroup
1177
+ #define FA_CL_DKQ (DK_VEC / FA_CL_C) // K quartets per lane per row
1178
+ #define FA_CL_DVQ (DV_VEC / FA_CL_C) // V quartets (o_acc float4s) per lane per head
1179
+
1180
+ #ifdef FA_C8_NO_SG_PIN
1181
+ #define FA_C8_SG_ATTR_Q4
1182
+ #else
1183
+ #define FA_C8_SG_ATTR_Q4 REQD_SUBGROUP_SIZE_64
1184
+ #endif
1185
+
1186
+ FA_C8_SG_ATTR_Q4
1187
+ __kernel void flash_attn_f32_q4_0_q1_vec_mq_split_c8(
1188
+ const global void * q_void, ulong q_offset,
1189
+ const global void * k_void, ulong k_offset,
1190
+ const global void * v_void, ulong v_offset,
1191
+ const float scale,
1192
+ const int n_q,
1193
+ const int n_kv,
1194
+ const int n_head,
1195
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
1196
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
1197
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
1198
+ const float max_bias,
1199
+ const float m0,
1200
+ const float m1,
1201
+ const int n_head_log2,
1202
+ const float logit_softcap,
1203
+ const int n_head_kv,
1204
+ const global void * mask_void,
1205
+ const ulong mask_offset,
1206
+ const ulong mask_nb1,
1207
+ const ulong mask_nb2,
1208
+ const ulong mask_nb3,
1209
+ const int mask_ne2,
1210
+ const int mask_ne3,
1211
+ global float * partial_void,
1212
+ const int n_splits,
1213
+ const int kv_per_split
1214
+ ) {
1215
+ const int tid = get_local_id(0);
1216
+ const int sgid = tid / Q1_WG_SIZE;
1217
+ const int tid_sg = tid % Q1_WG_SIZE;
1218
+ const int cl = tid_sg / FA_CL_C; // cluster id
1219
+ const int lic = tid_sg % FA_CL_C; // lane in cluster
1220
+ const int kvhead_batch_idx = get_global_id(1);
1221
+ const int split_q_idx = get_global_id(2);
1222
+ const int split_idx = split_q_idx % n_splits;
1223
+ const int q_idx = split_q_idx / n_splits;
1224
+
1225
+ const int batch_idx = kvhead_batch_idx / n_head_kv;
1226
+ const int head_kv_idx = kvhead_batch_idx % n_head_kv;
1227
+
1228
+ const int kv_start = split_idx * kv_per_split;
1229
+ const int kv_end = min(kv_start + kv_per_split, n_kv);
1230
+
1231
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
1232
+
1233
+ if (kv_start >= kv_end) {
1234
+ if (tid == 0) {
1235
+ #pragma unroll
1236
+ for (int h = 0; h < MQ_GQA; ++h) {
1237
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1238
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
1239
+ * n_splits + split_idx);
1240
+ global float * rec = partial_void + rec_idx * record_stride;
1241
+ rec[0] = FA_M_INIT;
1242
+ rec[1] = 0.0f;
1243
+ }
1244
+ }
1245
+ return;
1246
+ }
1247
+
1248
+ const global char * q_base = (const global char *) q_void + q_offset;
1249
+ const global char * k_base = (const global char *) k_void + k_offset;
1250
+ const global char * v_base = (const global char *) v_void + v_offset;
1251
+
1252
+ // Stage MQ_GQA Q rows in __local as float4 (source for the quantize pass).
1253
+ __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC];
1254
+ for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE_Q4) {
1255
+ const int h = i / DK_VEC;
1256
+ const int k = i % DK_VEC;
1257
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1258
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
1259
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
1260
+ q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);
1261
+ }
1262
+ barrier(CLK_LOCAL_MEM_FENCE);
1263
+
1264
+ // Per-(h, block) int8-packed Q + (qd, q_sum), quantized once per WG.
1265
+ __local uint q_packed_shared[MQ_GQA * DK_Q4_BLOCKS * 8];
1266
+ __local float q_d_shared[MQ_GQA * DK_Q4_BLOCKS];
1267
+ __local int q_sum_shared[MQ_GQA * DK_Q4_BLOCKS];
1268
+ {
1269
+ const int active = MQ_GQA * DK_Q4_BLOCKS;
1270
+ if (tid < active) {
1271
+ const int h = tid / DK_Q4_BLOCKS;
1272
+ const int block_id = tid % DK_Q4_BLOCKS;
1273
+ ACC_TYPE4 q_block[8];
1274
+ #pragma unroll
1275
+ for (int i = 0; i < 8; ++i) q_block[i] = q_shared[h * DK_VEC + block_id * 8 + i];
1276
+ uint packed[8];
1277
+ q4_q_block_info info = quant_q_block_int8_packed_q4(q_block, packed);
1278
+ #pragma unroll
1279
+ for (int i = 0; i < 8; ++i) q_packed_shared[(h * DK_Q4_BLOCKS + block_id) * 8 + i] = packed[i];
1280
+ q_d_shared[h * DK_Q4_BLOCKS + block_id] = info.qd;
1281
+ q_sum_shared[h * DK_Q4_BLOCKS + block_id] = info.q_sum;
1282
+ }
1283
+ }
1284
+ barrier(CLK_LOCAL_MEM_FENCE);
1285
+
1286
+ float slope[MQ_GQA];
1287
+ #pragma unroll
1288
+ for (int h = 0; h < MQ_GQA; ++h) {
1289
+ slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);
1290
+ }
1291
+
1292
+ const global char * mask_base[MQ_GQA];
1293
+ if (mask_void != NULL) {
1294
+ const int mask_batch_idx = batch_idx % mask_ne3;
1295
+ const global char * mask_base_b = (const global char *) mask_void + mask_offset +
1296
+ mask_batch_idx * mask_nb3 +
1297
+ (ulong) q_idx * mask_nb1;
1298
+ #pragma unroll
1299
+ for (int h = 0; h < MQ_GQA; ++h) {
1300
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1301
+ const int mask_head_idx = head_idx % mask_ne2;
1302
+ mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;
1303
+ }
1304
+ } else {
1305
+ #pragma unroll
1306
+ for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL;
1307
+ }
1308
+
1309
+ // Per-CLUSTER online state; o_acc holds this lane's V quartets {lic + FA_CL_C*i}.
1310
+ ACC_TYPE4 o_acc[MQ_GQA][FA_CL_DVQ];
1311
+ ACC_TYPE m_i[MQ_GQA];
1312
+ ACC_TYPE l_i[MQ_GQA];
1313
+ #pragma unroll
1314
+ for (int h = 0; h < MQ_GQA; ++h) {
1315
+ m_i[h] = FA_M_INIT;
1316
+ l_i[h] = 0.0f;
1317
+ #pragma unroll
1318
+ for (int i = 0; i < FA_CL_DVQ; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f);
1319
+ }
1320
+
1321
+ const int kv_len = kv_end - kv_start;
1322
+ const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT;
1323
+ const int kv_lo = kv_start + sgid * kv_per_sg;
1324
+ const int kv_hi = min(kv_end, kv_lo + kv_per_sg);
1325
+
1326
+ // Uniform trip count; tail clamps the row address and drops the score to
1327
+ // FA_M_INIT (p underflows to 0) so shuffles stay convergent.
1328
+ const int n_iter = (kv_hi - kv_lo + FA_CL_NCL - 1) / FA_CL_NCL;
1329
+ const ulong k_row_base = batch_idx * k_nb3 + head_kv_idx * k_nb2;
1330
+ const ulong v_row_base = batch_idx * v_nb3 + head_kv_idx * v_nb2;
1331
+
1332
+ for (int it = 0; it < n_iter; ++it) {
1333
+ const int k_idx = kv_lo + cl + it * FA_CL_NCL;
1334
+ const int valid = k_idx < kv_hi;
1335
+ const int k_safe = valid ? k_idx : (kv_hi - 1);
1336
+
1337
+ const global char * k_row = k_base + k_row_base + (ulong) k_safe * k_nb1;
1338
+ const global char * v_row = v_base + v_row_base + (ulong) k_safe * v_nb1;
1339
+
1340
+ // dp4a K dot over this lane's quartets of the cluster's row.
1341
+ ACC_TYPE lane_contrib[MQ_GQA];
1342
+ #pragma unroll
1343
+ for (int h = 0; h < MQ_GQA; ++h) lane_contrib[h] = 0.0f;
1344
+
1345
+ #pragma unroll
1346
+ for (int i = 0; i < FA_CL_DKQ; ++i) {
1347
+ const int qk = lic + FA_CL_C * i;
1348
+ const int block_idx = qk / 8;
1349
+ const int lane_in_block = qk % 8;
1350
+ const int g = lane_in_block & 3;
1351
+ const int shift = (lane_in_block < 4) ? 0 : 4;
1352
+ const global char * k_block = k_row + block_idx * Q4_0_BLOCK_SIZE;
1353
+ const float kd = vload_half(0, (const global half *)k_block);
1354
+ const global uchar * k_qs = (const global uchar *)(k_block + 2);
1355
+ const uchar b0 = k_qs[g*4 + 0];
1356
+ const uchar b1 = k_qs[g*4 + 1];
1357
+ const uchar b2 = k_qs[g*4 + 2];
1358
+ const uchar b3 = k_qs[g*4 + 3];
1359
+ const uint k_packed = ((uint)((b0 >> shift) & 0x0F)) |
1360
+ ((uint)((b1 >> shift) & 0x0F)) << 8 |
1361
+ ((uint)((b2 >> shift) & 0x0F)) << 16 |
1362
+ ((uint)((b3 >> shift) & 0x0F)) << 24;
1363
+ #pragma unroll
1364
+ for (int h = 0; h < MQ_GQA; ++h) {
1365
+ const uint q_packed_lane = q_packed_shared[(h * DK_Q4_BLOCKS + block_idx) * 8 + lane_in_block];
1366
+ const int raw_dot = dot_acc_sat_4x8packed_ss_int(q_packed_lane, k_packed, 0);
1367
+ const float qd = q_d_shared[h * DK_Q4_BLOCKS + block_idx];
1368
+ const float block_scale = qd * kd;
1369
+ float contrib = (float) raw_dot * block_scale;
1370
+ if (lane_in_block == 0) {
1371
+ const int q_sum_b = q_sum_shared[h * DK_Q4_BLOCKS + block_idx];
1372
+ contrib -= 8.0f * block_scale * (float) q_sum_b;
1373
+ }
1374
+ lane_contrib[h] += contrib;
1375
+ }
1376
+ }
1377
+
1378
+ // Cluster-reduce + score.
1379
+ ACC_TYPE score[MQ_GQA];
1380
+ #pragma unroll
1381
+ for (int h = 0; h < MQ_GQA; ++h) {
1382
+ ACC_TYPE s = lane_contrib[h];
1383
+ #pragma unroll
1384
+ for (int step = 1; step < FA_CL_C; step <<= 1) {
1385
+ s += sub_group_shuffle_xor(s, step);
1386
+ }
1387
+ s *= scale;
1388
+ if (mask_base[h] != NULL) {
1389
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
1390
+ s += slope[h] * (ACC_TYPE) mask_ptr[k_safe];
1391
+ }
1392
+ if (logit_softcap > 0.0f) {
1393
+ s = logit_softcap * tanh(s / logit_softcap);
1394
+ }
1395
+ score[h] = valid ? s : FA_M_INIT;
1396
+ }
1397
+
1398
+ // Per-cluster online update (serial chain depth n_iter, not kv_per_sg).
1399
+ ACC_TYPE p_h[MQ_GQA];
1400
+ ACC_TYPE sp_h[MQ_GQA];
1401
+ #pragma unroll
1402
+ for (int h = 0; h < MQ_GQA; ++h) {
1403
+ const ACC_TYPE m_new = max(m_i[h], score[h]);
1404
+ sp_h[h] = native_exp(m_i[h] - m_new);
1405
+ p_h[h] = native_exp(score[h] - m_new);
1406
+ l_i[h] = l_i[h] * sp_h[h] + p_h[h];
1407
+ m_i[h] = m_new;
1408
+ }
1409
+
1410
+ // V accumulate on this lane's quartets (p = 0 on tail -> inert).
1411
+ #pragma unroll
1412
+ for (int i = 0; i < FA_CL_DVQ; ++i) {
1413
+ const int dv = lic + FA_CL_C * i;
1414
+ const float4 v_v = dequant_q4_0_lane(v_row + (dv / 8) * Q4_0_BLOCK_SIZE, dv % 8);
1415
+ #pragma unroll
1416
+ for (int h = 0; h < MQ_GQA; ++h) {
1417
+ o_acc[h][i] = mad(p_h[h], v_v, o_acc[h][i] * sp_h[h]);
1418
+ }
1419
+ }
1420
+ }
1421
+
1422
+ // Merge stage 1: fold cluster partials inside the subgroup via shuffles.
1423
+ // Lanes with equal lic across clusters hold the SAME dv slice.
1424
+ #pragma unroll
1425
+ for (int h = 0; h < MQ_GQA; ++h) {
1426
+ ACC_TYPE m_c = m_i[h];
1427
+ #pragma unroll
1428
+ for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) {
1429
+ m_c = max(m_c, sub_group_shuffle_xor(m_c, step));
1430
+ }
1431
+ const ACC_TYPE alpha = native_exp(m_i[h] - m_c);
1432
+ ACC_TYPE l_c = l_i[h] * alpha;
1433
+ #pragma unroll
1434
+ for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) {
1435
+ l_c += sub_group_shuffle_xor(l_c, step);
1436
+ }
1437
+ #pragma unroll
1438
+ for (int i = 0; i < FA_CL_DVQ; ++i) {
1439
+ ACC_TYPE4 o = o_acc[h][i] * alpha;
1440
+ #pragma unroll
1441
+ for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) {
1442
+ o.s0 += sub_group_shuffle_xor(o.s0, step);
1443
+ o.s1 += sub_group_shuffle_xor(o.s1, step);
1444
+ o.s2 += sub_group_shuffle_xor(o.s2, step);
1445
+ o.s3 += sub_group_shuffle_xor(o.s3, step);
1446
+ }
1447
+ o_acc[h][i] = o;
1448
+ }
1449
+ m_i[h] = m_c;
1450
+ l_i[h] = l_c;
1451
+ }
1452
+
1453
+ // Merge stage 2: baseline cross-subgroup LDS merge (o published by
1454
+ // cluster 0's lanes; layout identical to the baseline sg_o).
1455
+ __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT];
1456
+ __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT];
1457
+ __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC];
1458
+
1459
+ if (tid_sg == 0) {
1460
+ #pragma unroll
1461
+ for (int h = 0; h < MQ_GQA; ++h) {
1462
+ sg_m[h][sgid] = m_i[h];
1463
+ sg_l[h][sgid] = l_i[h];
1464
+ }
1465
+ }
1466
+
1467
+ #pragma unroll
1468
+ for (int h = 0; h < MQ_GQA; ++h) {
1469
+ if (cl == 0) {
1470
+ #pragma unroll
1471
+ for (int i = 0; i < FA_CL_DVQ; ++i) {
1472
+ sg_o[sgid][lic + FA_CL_C * i] = o_acc[h][i];
1473
+ }
1474
+ }
1475
+ barrier(CLK_LOCAL_MEM_FENCE);
1476
+
1477
+ if (sgid == 0) {
1478
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1479
+
1480
+ ACC_TYPE m_c = sg_m[h][0];
1481
+ #pragma unroll
1482
+ for (int s = 1; s < MQ_NSG_SPLIT; ++s) {
1483
+ m_c = max(m_c, sg_m[h][s]);
1484
+ }
1485
+ ACC_TYPE l_c = 0.0f;
1486
+ #pragma unroll
1487
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
1488
+ l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c);
1489
+ }
1490
+
1491
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
1492
+ * n_splits + split_idx);
1493
+ global float * rec = partial_void + rec_idx * record_stride;
1494
+ global float4 * rec_o = (global float4 *) (rec + 2);
1495
+
1496
+ if (tid_sg == 0) {
1497
+ rec[0] = (float) m_c;
1498
+ rec[1] = (float) l_c;
1499
+ }
1500
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) {
1501
+ ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f);
1502
+ #pragma unroll
1503
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
1504
+ const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c);
1505
+ o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged);
1506
+ }
1507
+ rec_o[dv_idx] = o_merged;
1508
+ }
1509
+ }
1510
+ barrier(CLK_LOCAL_MEM_FENCE);
1511
+ }
1512
+ }
1513
+
1514
+ #endif // DK_VEC/DV_VEC divisible by FA_CL_C
1515
+ #endif // FA_HAVE_INT_DOT && HAS_SUBGROUP_SHUFFLE (q1_vec_mq_split_c8)
1516
+
1517
+ __kernel void flash_attn_f32_q4_0(
1518
+ const global void * q_void, ulong q_offset,
1519
+ const global void * k_void, ulong k_offset,
1520
+ const global void * v_void, ulong v_offset,
1521
+ global void * o_void, ulong o_offset,
1522
+ const float scale,
1523
+ const int n_q,
1524
+ const int n_kv,
1525
+ const int is_causal,
1526
+ const int n_head,
1527
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
1528
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
1529
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
1530
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
1531
+ const float max_bias,
1532
+ const float m0,
1533
+ const float m1,
1534
+ const int n_head_log2,
1535
+ const float logit_softcap,
1536
+ const int n_head_kv,
1537
+ const global void* mask_void,
1538
+ const ulong mask_offset,
1539
+ const ulong mask_nb1,
1540
+ const ulong mask_nb2,
1541
+ const ulong mask_nb3,
1542
+ const int mask_ne2,
1543
+ const int mask_ne3,
1544
+ const global void* sinks_void,
1545
+ const ulong sinks_offset,
1546
+ // blk: per-(qblock,kvblock) class from flash_attn_blk_f16
1547
+ // (0=masked, 1=mixed, 2=unmasked). NULL disables the prepass opt.
1548
+ const global void * blk_void
1549
+ ) {
1550
+ const int tid = get_local_id(0);
1551
+ const int block_q_idx = get_group_id(0);
1552
+ const int head_batch_idx = get_global_id(1);
1553
+
1554
+ #if N_SPLIT > 1
1555
+ const int q_lane = tid / N_SPLIT;
1556
+ const int split_idx = tid % N_SPLIT;
1557
+ #else
1558
+ const int q_lane = tid;
1559
+ const int split_idx = 0;
1560
+ #endif
1561
+ const int my_query_row = block_q_idx * BLOCK_M + q_lane;
1562
+ const int query_valid = my_query_row < n_q;
1563
+
1564
+ const int batch_idx = head_batch_idx / n_head;
1565
+ const int head_idx = head_batch_idx % n_head;
1566
+
1567
+ const int gqa_ratio = n_head / n_head_kv;
1568
+ const int head_kv_idx = head_idx / gqa_ratio;
1569
+ const int mask_head_idx = mask_void != NULL ? head_idx % mask_ne2 : 0;
1570
+ const int mask_batch_idx = mask_void != NULL ? batch_idx % mask_ne3 : 0;
1571
+
1572
+ const global char * q_base = (const global char *) q_void + q_offset;
1573
+ const global char * k_base = (const global char *) k_void + k_offset;
1574
+ const global char * v_base = (const global char *) v_void + v_offset;
1575
+ global char * o_base = (global char *) o_void + o_offset;
1576
+
1577
+ const global char * mask_base = NULL;
1578
+ if (mask_void != NULL) {
1579
+ mask_base = (const global char *) mask_void + mask_offset +
1580
+ mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;
1581
+ }
1582
+
1583
+ // BLK_PREPASS_BM may differ from this kernel's BLOCK_M; scale q-block idx.
1584
+ #ifndef BLK_PREPASS_BM
1585
+ #define BLK_PREPASS_BM BLOCK_M
1586
+ #endif
1587
+ const global char * blk_base = NULL;
1588
+ int n_kv_blocks = 0;
1589
+ if (blk_void != NULL) {
1590
+ n_kv_blocks = (n_kv + BLOCK_N - 1) / BLOCK_N;
1591
+ const int n_q_blocks_prepass = (n_q + BLK_PREPASS_BM - 1) / BLK_PREPASS_BM;
1592
+ const int prepass_q_block = (block_q_idx * BLOCK_M) / BLK_PREPASS_BM;
1593
+ blk_base = (const global char *) blk_void +
1594
+ (((mask_batch_idx * mask_ne2) + mask_head_idx) * n_q_blocks_prepass + prepass_q_block) * n_kv_blocks;
1595
+ }
1596
+
1597
+ const int dk_off_vec = split_idx * SPLIT_DK_VEC;
1598
+ ACC_TYPE4 q_priv[SPLIT_DK_VEC];
1599
+ if (query_valid) {
1600
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1;
1601
+ const global float4 * q_ptr = (const global float4 *) (q_base + q_row_offset);
1602
+ #pragma unroll
1603
+ for (int i = 0; i < SPLIT_DK_VEC; ++i) {
1604
+ q_priv[i] = q_ptr[dk_off_vec + i];
1605
+ }
1606
+ } else {
1607
+ #pragma unroll
1608
+ for (int i = 0; i < SPLIT_DK_VEC; ++i) q_priv[i] = (ACC_TYPE4)(0.0f);
1609
+ }
1610
+
1611
+ #ifdef FA_HAVE_INT_DOT
1612
+ uint q_packed_pf[SPLIT_DK_Q4_BLOCKS * 8];
1613
+ float q_d_pf[SPLIT_DK_Q4_BLOCKS];
1614
+ int q_sum_pf[SPLIT_DK_Q4_BLOCKS];
1615
+ #pragma unroll
1616
+ for (int b = 0; b < SPLIT_DK_Q4_BLOCKS; ++b) {
1617
+ q4_q_block_info info = quant_q_block_int8_packed_q4(&q_priv[b * 8], &q_packed_pf[b * 8]);
1618
+ q_d_pf[b] = info.qd;
1619
+ q_sum_pf[b] = info.q_sum;
1620
+ }
1621
+ #endif
1622
+
1623
+ const int dv_off_vec = split_idx * SPLIT_DV_VEC;
1624
+ ACC_TYPE4 o_acc[SPLIT_DV_VEC];
1625
+ #pragma unroll
1626
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);
1627
+
1628
+ ACC_TYPE m_i = FA_M_INIT;
1629
+ ACC_TYPE l_i = 0.0f;
1630
+
1631
+ float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
1632
+
1633
+ #ifdef FA_HAVE_INT_DOT
1634
+ // Accessors so the staging code is layout-agnostic.
1635
+ #ifdef FA_K_LDS_T
1636
+ #define FA_K_PACKED(ROW, IDX) l_k_packed[IDX][ROW]
1637
+ #define FA_K_SCALE(ROW, BLK) l_k_scale[BLK][ROW]
1638
+ #else
1639
+ #define FA_K_PACKED(ROW, IDX) l_k_packed[ROW][IDX]
1640
+ #define FA_K_SCALE(ROW, BLK) l_k_scale[ROW][BLK]
1641
+ #endif
1642
+
1643
+ #ifdef FA_K_LDS_T
1644
+ // K tile transposed: the 4 KV rows the QK loop walks together become adjacent, so each
1645
+ // (block, group) step is ONE 128-bit local read instead of four 32-bit ones. The QK
1646
+ // loop is LDS-read-issue-bound.
1647
+ __local uint l_k_packed[DK_Q4_BLOCKS_PREFILL * 8][BLOCK_N];
1648
+ __local float l_k_scale [DK_Q4_BLOCKS_PREFILL][BLOCK_N];
1649
+ #else
1650
+ __local uint l_k_packed[BLOCK_N][DK_Q4_BLOCKS_PREFILL * 8];
1651
+ __local float l_k_scale [BLOCK_N][DK_Q4_BLOCKS_PREFILL];
1652
+ #endif
1653
+ #else
1654
+ __local half4 l_k[BLOCK_N][DK_VEC];
1655
+ #endif
1656
+
1657
+ __local half4 l_v[BLOCK_N][DV_VEC];
1658
+
1659
+ for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) {
1660
+ // Skip fully-masked KV tiles (uniform branch across WG).
1661
+ char blk_cur = 1;
1662
+ if (blk_base != NULL) {
1663
+ blk_cur = blk_base[k_start / BLOCK_N];
1664
+ if (blk_cur == 0) continue;
1665
+ }
1666
+
1667
+ {
1668
+ #ifdef FA_HAVE_INT_DOT
1669
+ const int k_blocks_per_row = DK_Q4_BLOCKS_PREFILL;
1670
+ const int n_blocks_total = BLOCK_N * k_blocks_per_row;
1671
+ for (int i = tid; i < n_blocks_total; i += WG_SIZE) {
1672
+ const int row = i / k_blocks_per_row;
1673
+ const int blk = i % k_blocks_per_row;
1674
+ const int k_row_idx = k_start + row;
1675
+ if (k_row_idx < n_kv) {
1676
+ const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1;
1677
+ const global char * blk_ptr = k_base + k_row_off + blk * Q4_0_BLOCK_SIZE;
1678
+ const float df = (float) vload_half(0, (const global half *) blk_ptr);
1679
+ const global uchar * qs = (const global uchar *)(blk_ptr + 2);
1680
+ FA_K_SCALE(row, blk) = df;
1681
+ uint k_packed[8];
1682
+ pack_q4_0_nibbles(qs, k_packed);
1683
+ #pragma unroll
1684
+ for (int j = 0; j < 8; ++j) {
1685
+ FA_K_PACKED(row, blk * 8 + j) = k_packed[j];
1686
+ }
1687
+ } else {
1688
+ FA_K_SCALE(row, blk) = 0.0f;
1689
+ #pragma unroll
1690
+ for (int j = 0; j < 8; ++j) FA_K_PACKED(row, blk * 8 + j) = 0u;
1691
+ }
1692
+ }
1693
+ #else
1694
+ // Fallback: dequant q4_0 -> half in local memory.
1695
+ const int k_blocks_per_row = DK_Q4_BLOCKS_PREFILL;
1696
+ const int n_blocks_total = BLOCK_N * k_blocks_per_row;
1697
+ for (int i = tid; i < n_blocks_total; i += WG_SIZE) {
1698
+ const int row = i / k_blocks_per_row;
1699
+ const int blk = i % k_blocks_per_row;
1700
+ const int k_row_idx = k_start + row;
1701
+ if (k_row_idx < n_kv) {
1702
+ const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1;
1703
+ const global char * blk_ptr = k_base + k_row_off + blk * Q4_0_BLOCK_SIZE;
1704
+ const float df = (float) vload_half(0, (const global half *) blk_ptr);
1705
+ const global uchar * qs = (const global uchar *)(blk_ptr + 2);
1706
+ #pragma unroll
1707
+ for (int g = 0; g < 4; ++g) {
1708
+ float4 vlo = df * (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f,
1709
+ (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f,
1710
+ (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f,
1711
+ (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f);
1712
+ float4 vhi = df * (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f,
1713
+ (float)(int)(qs[g*4 + 1] >> 4) - 8.0f,
1714
+ (float)(int)(qs[g*4 + 2] >> 4) - 8.0f,
1715
+ (float)(int)(qs[g*4 + 3] >> 4) - 8.0f);
1716
+ l_k[row][blk * 8 + g ] = (half4)((half)vlo.s0, (half)vlo.s1, (half)vlo.s2, (half)vlo.s3);
1717
+ l_k[row][blk * 8 + 4 + g] = (half4)((half)vhi.s0, (half)vhi.s1, (half)vhi.s2, (half)vhi.s3);
1718
+ }
1719
+ } else {
1720
+ #pragma unroll
1721
+ for (int j = 0; j < 8; ++j) l_k[row][blk * 8 + j] = (half4)(0.0h);
1722
+ }
1723
+ }
1724
+ #endif
1725
+ }
1726
+ // V tile load — dequant V -> half in local memory.
1727
+ {
1728
+ const int v_blocks_per_row = DV_Q4_BLOCKS_PREFILL;
1729
+ const int n_blocks_total = BLOCK_N * v_blocks_per_row;
1730
+ for (int i = tid; i < n_blocks_total; i += WG_SIZE) {
1731
+ const int row = i / v_blocks_per_row;
1732
+ const int blk = i % v_blocks_per_row;
1733
+ const int v_row_idx = k_start + row;
1734
+ if (v_row_idx < n_kv) {
1735
+ const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1;
1736
+ const global char * blk_ptr = v_base + v_row_off + blk * Q4_0_BLOCK_SIZE;
1737
+ const float df = (float) vload_half(0, (const global half *) blk_ptr);
1738
+ const global uchar * qs = (const global uchar *)(blk_ptr + 2);
1739
+ #pragma unroll
1740
+ for (int g = 0; g < 4; ++g) {
1741
+ float4 vlo = df * (float4)((float)(int)(qs[g*4 + 0] & 0x0F) - 8.0f,
1742
+ (float)(int)(qs[g*4 + 1] & 0x0F) - 8.0f,
1743
+ (float)(int)(qs[g*4 + 2] & 0x0F) - 8.0f,
1744
+ (float)(int)(qs[g*4 + 3] & 0x0F) - 8.0f);
1745
+ float4 vhi = df * (float4)((float)(int)(qs[g*4 + 0] >> 4) - 8.0f,
1746
+ (float)(int)(qs[g*4 + 1] >> 4) - 8.0f,
1747
+ (float)(int)(qs[g*4 + 2] >> 4) - 8.0f,
1748
+ (float)(int)(qs[g*4 + 3] >> 4) - 8.0f);
1749
+ l_v[row][blk * 8 + g ] = (half4)((half)vlo.s0, (half)vlo.s1, (half)vlo.s2, (half)vlo.s3);
1750
+ l_v[row][blk * 8 + 4 + g] = (half4)((half)vhi.s0, (half)vhi.s1, (half)vhi.s2, (half)vhi.s3);
1751
+ }
1752
+ } else {
1753
+ #pragma unroll
1754
+ for (int j = 0; j < 8; ++j) l_v[row][blk * 8 + j] = (half4)(0.0h);
1755
+ }
1756
+ }
1757
+ }
1758
+ barrier(CLK_LOCAL_MEM_FENCE);
1759
+
1760
+ // QK dot + online softmax. N_SPLIT>1 reduces per-thread partials via shuffle_xor.
1761
+ #if N_SPLIT > 1
1762
+ {
1763
+ #else
1764
+ if (query_valid) {
1765
+ #endif
1766
+ const int k_blk_base = split_idx * SPLIT_DK_Q4_BLOCKS;
1767
+ for (int j = 0; j < BLOCK_N; j += 4) {
1768
+ const int k_row0 = k_start + j;
1769
+ const int k_row1 = k_start + j + 1;
1770
+ const int k_row2 = k_start + j + 2;
1771
+ const int k_row3 = k_start + j + 3;
1772
+
1773
+ ACC_TYPE s0, s1, s2, s3;
1774
+ #ifdef FA_HAVE_INT_DOT
1775
+ s0 = 0.0f; s1 = 0.0f; s2 = 0.0f; s3 = 0.0f;
1776
+ #pragma unroll
1777
+ for (int b_local = 0; b_local < SPLIT_DK_Q4_BLOCKS; ++b_local) {
1778
+ const int b = k_blk_base + b_local;
1779
+ int sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
1780
+ #ifdef FA_K_LDS_T
1781
+ // 4 KV rows are adjacent in the transposed tile: one 128-bit local
1782
+ // read per (block, group) instead of four 32-bit ones.
1783
+ #pragma unroll
1784
+ for (int g = 0; g < 8; ++g) {
1785
+ const uint qp = q_packed_pf[b_local * 8 + g];
1786
+ const uint4 kq4 = vload4(0, &l_k_packed[b * 8 + g][j]);
1787
+ sum0 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s0, sum0);
1788
+ sum1 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s1, sum1);
1789
+ sum2 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s2, sum2);
1790
+ sum3 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s3, sum3);
1791
+ }
1792
+ #else
1793
+ #pragma unroll
1794
+ for (int g = 0; g < 8; ++g) {
1795
+ const uint qp = q_packed_pf[b_local * 8 + g];
1796
+ sum0 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j ][b * 8 + g], sum0);
1797
+ sum1 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+1][b * 8 + g], sum1);
1798
+ sum2 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+2][b * 8 + g], sum2);
1799
+ sum3 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+3][b * 8 + g], sum3);
1800
+ }
1801
+ #endif
1802
+ const float qd = q_d_pf[b_local];
1803
+ const int q_sum = q_sum_pf[b_local];
1804
+ #ifdef FA_K_LDS_T
1805
+ const float4 ks4 = vload4(0, &l_k_scale[b][j]);
1806
+ s0 += (float)(sum0 - 8 * q_sum) * qd * ks4.s0;
1807
+ s1 += (float)(sum1 - 8 * q_sum) * qd * ks4.s1;
1808
+ s2 += (float)(sum2 - 8 * q_sum) * qd * ks4.s2;
1809
+ s3 += (float)(sum3 - 8 * q_sum) * qd * ks4.s3;
1810
+ #else
1811
+ s0 += (float)(sum0 - 8 * q_sum) * qd * l_k_scale[j ][b];
1812
+ s1 += (float)(sum1 - 8 * q_sum) * qd * l_k_scale[j+1][b];
1813
+ s2 += (float)(sum2 - 8 * q_sum) * qd * l_k_scale[j+2][b];
1814
+ s3 += (float)(sum3 - 8 * q_sum) * qd * l_k_scale[j+3][b];
1815
+ #endif
1816
+ }
1817
+ #else
1818
+ ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f);
1819
+ ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f);
1820
+ ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f);
1821
+ ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f);
1822
+ #pragma unroll
1823
+ for (int k = 0; k < SPLIT_DK_VEC; ++k) {
1824
+ const ACC_TYPE4 qk = q_priv[k];
1825
+ const int k_abs = dk_off_vec + k;
1826
+ dot_acc0 = mad(qk, CONVERT_KV_ACC4(l_k[j ][k_abs]), dot_acc0);
1827
+ dot_acc1 = mad(qk, CONVERT_KV_ACC4(l_k[j+1][k_abs]), dot_acc1);
1828
+ dot_acc2 = mad(qk, CONVERT_KV_ACC4(l_k[j+2][k_abs]), dot_acc2);
1829
+ dot_acc3 = mad(qk, CONVERT_KV_ACC4(l_k[j+3][k_abs]), dot_acc3);
1830
+ }
1831
+ s0 = dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3;
1832
+ s1 = dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3;
1833
+ s2 = dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3;
1834
+ s3 = dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3;
1835
+ #endif
1836
+
1837
+ #if N_SPLIT > 1
1838
+ // Power-of-2 N_SPLIT: shuffle_xor butterfly. N_SPLIT=3 (DK=96):
1839
+ // explicit 3-lane shuffle.
1840
+ #if (N_SPLIT & (N_SPLIT - 1)) == 0
1841
+ #pragma unroll
1842
+ for (int step = 1; step < N_SPLIT; step <<= 1) {
1843
+ s0 += sub_group_shuffle_xor(s0, step);
1844
+ s1 += sub_group_shuffle_xor(s1, step);
1845
+ s2 += sub_group_shuffle_xor(s2, step);
1846
+ s3 += sub_group_shuffle_xor(s3, step);
1847
+ }
1848
+ #else
1849
+ const uint tri_base = (get_sub_group_local_id() / N_SPLIT) * N_SPLIT;
1850
+ s0 = sub_group_shuffle(s0, tri_base + 0) + sub_group_shuffle(s0, tri_base + 1) + sub_group_shuffle(s0, tri_base + 2);
1851
+ s1 = sub_group_shuffle(s1, tri_base + 0) + sub_group_shuffle(s1, tri_base + 1) + sub_group_shuffle(s1, tri_base + 2);
1852
+ s2 = sub_group_shuffle(s2, tri_base + 0) + sub_group_shuffle(s2, tri_base + 1) + sub_group_shuffle(s2, tri_base + 2);
1853
+ s3 = sub_group_shuffle(s3, tri_base + 0) + sub_group_shuffle(s3, tri_base + 1) + sub_group_shuffle(s3, tri_base + 2);
1854
+ #endif
1855
+ if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; }
1856
+ #endif
1857
+ s0 *= scale; s1 *= scale; s2 *= scale; s3 *= scale;
1858
+
1859
+ if (is_causal) {
1860
+ const int causal_limit = n_kv - n_q + my_query_row;
1861
+ if (k_row0 > causal_limit) s0 = FA_M_INIT;
1862
+ if (k_row1 > causal_limit) s1 = FA_M_INIT;
1863
+ if (k_row2 > causal_limit) s2 = FA_M_INIT;
1864
+ if (k_row3 > causal_limit) s3 = FA_M_INIT;
1865
+ }
1866
+ if (k_row0 >= n_kv) s0 = FA_M_INIT;
1867
+ if (k_row1 >= n_kv) s1 = FA_M_INIT;
1868
+ if (k_row2 >= n_kv) s2 = FA_M_INIT;
1869
+ if (k_row3 >= n_kv) s3 = FA_M_INIT;
1870
+
1871
+ if (query_valid && mask_base != NULL && blk_cur != 2) {
1872
+ const global MASK_DATA_TYPE * mask_ptr =
1873
+ (const global MASK_DATA_TYPE *) (mask_base + my_query_row * mask_nb1);
1874
+ if (k_row0 < n_kv) s0 += slope * (ACC_TYPE) mask_ptr[k_row0];
1875
+ if (k_row1 < n_kv) s1 += slope * (ACC_TYPE) mask_ptr[k_row1];
1876
+ if (k_row2 < n_kv) s2 += slope * (ACC_TYPE) mask_ptr[k_row2];
1877
+ if (k_row3 < n_kv) s3 += slope * (ACC_TYPE) mask_ptr[k_row3];
1878
+ }
1879
+ if (logit_softcap > 0.0f) {
1880
+ s0 = logit_softcap * tanh(s0 / logit_softcap);
1881
+ s1 = logit_softcap * tanh(s1 / logit_softcap);
1882
+ s2 = logit_softcap * tanh(s2 / logit_softcap);
1883
+ s3 = logit_softcap * tanh(s3 / logit_softcap);
1884
+ }
1885
+
1886
+ const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3)));
1887
+ // Whole tile masked (m_new == FA_M_INIT): force the exp() args
1888
+ // far negative so the tile contributes 0, not exp(0)=1.
1889
+ const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new;
1890
+ const ACC_TYPE scale_prev = native_exp(m_i - m_exp);
1891
+ const ACC_TYPE p0 = native_exp(s0 - m_exp);
1892
+ const ACC_TYPE p1 = native_exp(s1 - m_exp);
1893
+ const ACC_TYPE p2 = native_exp(s2 - m_exp);
1894
+ const ACC_TYPE p3 = native_exp(s3 - m_exp);
1895
+
1896
+ #pragma unroll
1897
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) {
1898
+ const int i_abs = dv_off_vec + i;
1899
+ o_acc[i] = mad(p3, CONVERT_KV_ACC4(l_v[j+3][i_abs]),
1900
+ mad(p2, CONVERT_KV_ACC4(l_v[j+2][i_abs]),
1901
+ mad(p1, CONVERT_KV_ACC4(l_v[j+1][i_abs]),
1902
+ mad(p0, CONVERT_KV_ACC4(l_v[j ][i_abs]),
1903
+ o_acc[i] * scale_prev))));
1904
+ }
1905
+ l_i = l_i * scale_prev + p0 + p1 + p2 + p3;
1906
+ m_i = m_new;
1907
+ }
1908
+ }
1909
+ barrier(CLK_LOCAL_MEM_FENCE);
1910
+ }
1911
+
1912
+ // Write output.
1913
+ if (query_valid) {
1914
+ if (sinks_void != NULL) {
1915
+ const global ACC_TYPE * sinks_ptr =
1916
+ (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset);
1917
+ const ACC_TYPE m_sink = sinks_ptr[head_idx];
1918
+ const ACC_TYPE m_final = max(m_i, m_sink);
1919
+ const ACC_TYPE scale_o = exp(m_i - m_final);
1920
+ #pragma unroll
1921
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] *= scale_o;
1922
+ l_i = l_i * scale_o + exp(m_sink - m_final);
1923
+ m_i = m_final;
1924
+ }
1925
+ const ACC_TYPE l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f;
1926
+ const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1;
1927
+ global float4 * o_row = (global float4 *) (o_base + o_row_offset);
1928
+ if (l_inv > 0.0f) {
1929
+ #pragma unroll
1930
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = o_acc[i] * l_inv;
1931
+ } else {
1932
+ #pragma unroll
1933
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = (float4)(0.0f);
1934
+ }
1935
+ }
1936
+ }
1937
+
1938
+ // FD Pass 2: merge split partials. Identical across q4_0/q8_0/f16; each FA
1939
+ // source owns a copy since kernels compile per-source-program.
1940
+ __kernel void flash_attn_f32_merge(
1941
+ const global float * partial_void,
1942
+ global void * o_void,
1943
+ const ulong o_offset,
1944
+ const int n_head,
1945
+ const int n_splits,
1946
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
1947
+ const global void * sinks_void,
1948
+ const ulong sinks_offset,
1949
+ const int n_q
1950
+ ) {
1951
+ const int lane = get_local_id(0);
1952
+ const int head_batch_idx = get_global_id(1);
1953
+ const int q_idx = get_global_id(2);
1954
+ const int batch_idx = head_batch_idx / n_head;
1955
+ const int head_idx = head_batch_idx % n_head;
1956
+
1957
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
1958
+ const ulong record_idx_0 = (((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) * n_splits;
1959
+ const global float * rec0 = partial_void + record_idx_0 * record_stride;
1960
+
1961
+ __local ACC_TYPE m_final_shared;
1962
+ __local ACC_TYPE l_final_shared;
1963
+ if (lane == 0) {
1964
+ ACC_TYPE m = FA_M_INIT;
1965
+ for (int c = 0; c < n_splits; ++c) {
1966
+ const ACC_TYPE m_c = rec0[c * record_stride + 0];
1967
+ m = max(m, m_c);
1968
+ }
1969
+ ACC_TYPE m_sink = 0.0f;
1970
+ bool has_sink = false;
1971
+ if (sinks_void != NULL) {
1972
+ const global ACC_TYPE * sinks_ptr =
1973
+ (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset);
1974
+ m_sink = sinks_ptr[head_idx];
1975
+ has_sink = true;
1976
+ m = max(m, m_sink);
1977
+ }
1978
+ ACC_TYPE l = 0.0f;
1979
+ for (int c = 0; c < n_splits; ++c) {
1980
+ const ACC_TYPE m_c = rec0[c * record_stride + 0];
1981
+ const ACC_TYPE l_c = rec0[c * record_stride + 1];
1982
+ if (m_c > FA_M_INIT) {
1983
+ l += l_c * exp(m_c - m);
1984
+ }
1985
+ }
1986
+ if (has_sink) {
1987
+ l += exp(m_sink - m);
1988
+ }
1989
+ m_final_shared = m;
1990
+ l_final_shared = l;
1991
+ }
1992
+ barrier(CLK_LOCAL_MEM_FENCE);
1993
+ const ACC_TYPE m_final = m_final_shared;
1994
+ const ACC_TYPE l_final = l_final_shared;
1995
+ const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f;
1996
+
1997
+ ACC_TYPE4 o = (ACC_TYPE4)(0.0f);
1998
+ for (int c = 0; c < n_splits; ++c) {
1999
+ const global float * rec_c = rec0 + c * record_stride;
2000
+ const ACC_TYPE m_c = rec_c[0];
2001
+ if (m_c <= FA_M_INIT) continue;
2002
+ const global float4 * rec_oc = (const global float4 *) (rec_c + 2);
2003
+ const ACC_TYPE scale_c = exp(m_c - m_final);
2004
+ o = mad((ACC_TYPE4)(scale_c), rec_oc[lane], o);
2005
+ }
2006
+ o = o * l_inv;
2007
+
2008
+ const ulong o_row_offset = (ulong) batch_idx * o_nb3 + (ulong) q_idx * o_nb2 + (ulong) head_idx * o_nb1;
2009
+ global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) ((global char *) o_void + o_offset + o_row_offset);
2010
+ o_row[lane] = CONVERT_O_DATA4(o);
2011
+ }