elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,2656 @@
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_qcom_reqd_sub_group_size
10
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
11
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
12
+ #else
13
+ #define REQD_SUBGROUP_SIZE_64
14
+ #endif
15
+
16
+ // subgroup size for q1 kernels
17
+ #ifndef FA_SG
18
+ #define FA_SG 64
19
+ #endif
20
+ #ifdef cl_intel_required_subgroup_size
21
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
22
+ #define REQD_FA_SG __attribute__((intel_reqd_sub_group_size(FA_SG)))
23
+ #else
24
+ #define REQD_FA_SG
25
+ #endif
26
+
27
+ #ifdef cl_khr_subgroup_shuffle
28
+ #pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable
29
+ #define HAS_SUBGROUP_SHUFFLE 1
30
+ #elif defined(cl_qcom_subgroup_shuffle)
31
+ #pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable
32
+ #define HAS_SUBGROUP_SHUFFLE 1
33
+ // Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR
34
+ // name, so calling it is an implicit declaration and the program fails to build.
35
+ // Route it to the qcom builtin.
36
+ #define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f)
37
+ #endif
38
+
39
+ #define ACC_TYPE float
40
+ #define ACC_TYPE4 float4
41
+ #define Q_DATA_TYPE4 float4
42
+ #define KV_DATA_TYPE4 half4
43
+ #define O_DATA_TYPE4 float4
44
+ #define MASK_DATA_TYPE half
45
+ #define CONVERT_Q_ACC4(x) (x)
46
+ #define CONVERT_KV_ACC4(x) convert_float4(x)
47
+ #define CONVERT_O_DATA4(x) (x)
48
+
49
+ #define DK_VEC (DK/4)
50
+ #define DV_VEC (DV/4)
51
+
52
+ #ifndef FA_PARTIAL_FLOATS
53
+ #define FA_PARTIAL_FLOATS (2 + DV)
54
+ #endif
55
+ #define Q1_WG_SIZE FA_SG
56
+
57
+ // The kernels are built with -cl-finite-math-only. On some older Adreno GPUs,
58
+ // infinite operand can cause undefined behavior and miscompilation for exp.
59
+ // Therefore, a large negative value is used instead.
60
+ #define FA_M_INIT (-3.0e38f)
61
+
62
+ // Drop full unroll at DK>=192 — Adreno compiler host-memory budget.
63
+ #if DK >= 192
64
+ #define FA_UNROLL
65
+ #else
66
+ #define FA_UNROLL _Pragma("unroll")
67
+ #endif
68
+
69
+ // N_SPLIT>1 splits DK/DV across threads to cut per-thread register use.
70
+ #ifndef N_SPLIT
71
+ #define N_SPLIT 1
72
+ #endif
73
+
74
+ #define SPLIT_DK_VEC (DK_VEC / N_SPLIT)
75
+ #define SPLIT_DV_VEC (DV_VEC / N_SPLIT)
76
+
77
+ #if N_SPLIT > 1
78
+ #define WG_SIZE (BLOCK_M * N_SPLIT)
79
+ #else
80
+ #define WG_SIZE (BLOCK_M)
81
+ #endif
82
+
83
+ inline float get_alibi_slope(
84
+ const float max_bias, const uint h, const uint n_head_log2, const float m0, const float m1
85
+ ) {
86
+ if (max_bias <= 0.0f) {
87
+ return 1.0f;
88
+ }
89
+ const float base = h < n_head_log2 ? m0 : m1;
90
+ const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;
91
+
92
+ return pow(base, exph);
93
+ }
94
+
95
+ // Adreno compiler crashes when attempting to compile the entire program for DK=512,
96
+ // FA_DECODE_ONLY allows bypass the encoding kernel.
97
+ #if !defined(FA_DECODE_ONLY) && !defined(FA_MQ_ONLY)
98
+ #ifndef FA_TILE_NAME
99
+ #define FA_TILE_NAME flash_attn_f32_f16
100
+ #endif
101
+ __kernel void FA_TILE_NAME(
102
+ const global void * q_void, ulong q_offset,
103
+ #ifdef FA_K_IMG
104
+ __read_only image1d_buffer_t k_img, ulong k_offset_unused,
105
+ #else
106
+ const global void * k_void, ulong k_offset,
107
+ #endif
108
+ const global void * v_void, ulong v_offset,
109
+ global void * o_void, ulong o_offset,
110
+ const float scale,
111
+ const int n_q,
112
+ const int n_kv,
113
+ const int is_causal,
114
+ const int n_head,
115
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
116
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
117
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
118
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
119
+ const float max_bias,
120
+ const float m0,
121
+ const float m1,
122
+ const int n_head_log2,
123
+ const float logit_softcap,
124
+ const int n_head_kv,
125
+ const global void* mask_void,
126
+ const ulong mask_offset,
127
+ const ulong mask_nb1,
128
+ const ulong mask_nb2,
129
+ const ulong mask_nb3,
130
+ const int mask_ne2,
131
+ const int mask_ne3,
132
+ const global void* sinks_void,
133
+ const ulong sinks_offset,
134
+ const global void * k_pad_void,
135
+ const global void * v_pad_void,
136
+ const global void * mask_pad_void,
137
+ const global char * blk,
138
+ const int n_kv_blocks,
139
+ const ulong mask_pad_nb1,
140
+ const ulong mask_pad_nb2,
141
+ const ulong mask_pad_nb3
142
+ ) {
143
+ const int tid = get_local_id(0);
144
+ const int block_q_idx = get_group_id(0);
145
+ const int head_batch_idx = get_global_id(1);
146
+
147
+ #if N_SPLIT > 1
148
+ const int q_lane = tid / N_SPLIT;
149
+ const int split_idx = tid % N_SPLIT;
150
+ #else
151
+ const int q_lane = tid;
152
+ const int split_idx = 0;
153
+ #endif
154
+
155
+ const int my_query_row = block_q_idx * BLOCK_M + q_lane;
156
+ const int query_valid = my_query_row < n_q;
157
+
158
+ const int batch_idx = head_batch_idx / n_head;
159
+ const int head_idx = head_batch_idx % n_head;
160
+
161
+ const int gqa_ratio = n_head / n_head_kv;
162
+ const int head_kv_idx = head_idx / gqa_ratio;
163
+ const int mask_head_idx = mask_void != NULL ? head_idx % mask_ne2 : 0;
164
+ const int mask_batch_idx = mask_void != NULL ? batch_idx % mask_ne3 : 0;
165
+
166
+ const global char* q_base = (const global char*)q_void + q_offset;
167
+ #ifndef FA_K_IMG
168
+ const global char* k_base = (const global char*)k_void + k_offset;
169
+ #endif
170
+ const global char* v_base = (const global char*)v_void + v_offset;
171
+ global char* o_base = (global char*)o_void + o_offset;
172
+
173
+ const global char* mask_base = NULL;
174
+ if (mask_void != NULL) {
175
+ mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;
176
+ }
177
+ const global char* mask_pad_base = NULL;
178
+ if (mask_pad_void != NULL) {
179
+ mask_pad_base = (const global char*)mask_pad_void + mask_batch_idx * mask_pad_nb3 + mask_head_idx * mask_pad_nb2;
180
+ }
181
+ const global char* blk_base = NULL;
182
+ if (blk != NULL) {
183
+ const int n_q_blocks = (n_q + BLOCK_M - 1) / BLOCK_M;
184
+ blk_base = blk + (((mask_batch_idx * mask_ne2) + mask_head_idx) * n_q_blocks + block_q_idx) * n_kv_blocks;
185
+ }
186
+
187
+ ACC_TYPE4 q_priv[SPLIT_DK_VEC];
188
+ const int dk_off = split_idx * SPLIT_DK_VEC;
189
+ if (query_valid) {
190
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1;
191
+ const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset);
192
+ FA_UNROLL
193
+ for (int i = 0; i < SPLIT_DK_VEC; ++i) {
194
+ q_priv[i] = CONVERT_Q_ACC4(q_ptr[dk_off + i]);
195
+ }
196
+ } else {
197
+ FA_UNROLL
198
+ for (int i = 0; i < SPLIT_DK_VEC; ++i) {
199
+ q_priv[i] = (ACC_TYPE4)(0.0f);
200
+ }
201
+ }
202
+
203
+ ACC_TYPE4 o_acc[SPLIT_DV_VEC];
204
+ FA_UNROLL
205
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) {
206
+ o_acc[i] = (ACC_TYPE4)(0.0f);
207
+ }
208
+
209
+ ACC_TYPE m_i = FA_M_INIT;
210
+ ACC_TYPE l_i = 0.0f;
211
+
212
+ float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
213
+
214
+ #ifdef FA_K_LDS_T
215
+ // K tile transposed: [dk vec][kv row] instead of [kv row][dk vec].
216
+ //
217
+ // The QK loop walks 2 or 4 KV rows at a time against the same dk element. Row-major
218
+ // those are DK_VEC half4s apart, so each is its own 64-bit local read. Transposed they
219
+ // are adjacent, so a pair is one 128-bit read -- half the LDS issues for the same bytes,
220
+ // no extra registers, arithmetic untouched.
221
+ //
222
+ // This kernel looked like it should be FMA-bound (a half4 mad does ~4 ALU ops per LDS
223
+ // read, unlike the 1:1 of the dp4a loop), but it is NOT: a wrong-math probe that kept
224
+ // every FMA and removed the LDS reads ran it 38.6% faster (18.92 -> 11.62 ms/op).
225
+ // Explicitly 16-byte aligned: FA_LK_PAIR below reads two adjacent half4 as one float4,
226
+ // and the element type only obliges the compiler to align this array to 8. The indices
227
+ // are even so the offset is a multiple of 16, but the base has to be too, and relying
228
+ // on the compiler to over-align it is relying on luck.
229
+ __local KV_DATA_TYPE4 l_k[DK_VEC][BLOCK_N] __attribute__((aligned(16)));
230
+ #define FA_LK(ROW, C) l_k[C][ROW]
231
+ // Two adjacent KV rows as one 128-bit local read (half4 pair == 16 B). j is even and
232
+ // BLOCK_N is even, so &l_k[c][j] is 16 B past a 16 B-aligned base.
233
+ #define FA_LK_PAIR(C, J) as_half8(*(__local const float4 *)(&l_k[C][J]))
234
+ #else
235
+ __local KV_DATA_TYPE4 l_k[BLOCK_N][DK_VEC];
236
+ #define FA_LK(ROW, C) l_k[ROW][C]
237
+ #endif
238
+ __local KV_DATA_TYPE4 l_v[BLOCK_N][DV_VEC];
239
+
240
+ #if N_SPLIT > 1 && !defined(HAS_SUBGROUP_SHUFFLE)
241
+ __local ACC_TYPE local_partial[BLOCK_N][WG_SIZE];
242
+ __local ACC_TYPE local_p[BLOCK_M][BLOCK_N];
243
+ __local ACC_TYPE local_softmax_scale[BLOCK_M];
244
+ __local ACC_TYPE local_l_inv[BLOCK_M];
245
+ #endif
246
+
247
+ for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) {
248
+ char blk_cur = 1;
249
+ if (blk_base != NULL) {
250
+ blk_cur = blk_base[k_start / BLOCK_N];
251
+ if (blk_cur == 0) continue;
252
+ }
253
+
254
+ const int use_kv_pad = k_pad_void != NULL && k_start + BLOCK_N > n_kv;
255
+ const int k_tile_start = use_kv_pad ? 0 : k_start;
256
+ const ulong k_tile_nb2 = use_kv_pad ? (ulong) BLOCK_N * k_nb1 : k_nb2;
257
+ const ulong k_tile_nb3 = use_kv_pad ? (ulong) n_head_kv * k_tile_nb2 : k_nb3;
258
+ const ulong v_tile_nb2 = use_kv_pad ? (ulong) BLOCK_N * v_nb1 : v_nb2;
259
+ const ulong v_tile_nb3 = use_kv_pad ? (ulong) n_head_kv * v_tile_nb2 : v_nb3;
260
+ #ifdef FA_K_IMG
261
+ // K via texture cache for the bulk (aligned) tiles; the ragged last
262
+ // tile (use_kv_pad) still reads the f32-strided pad buffer from global.
263
+ const global char* k_tile_base = use_kv_pad ? (const global char*) k_pad_void : (const global char*) 0;
264
+ const int k_pitch_px_row = (int)(k_nb1 >> 3);
265
+ const int k_pitch_px_head = (int)(k_nb2 >> 3);
266
+ const int k_pitch_px_batch = (int)(k_nb3 >> 3);
267
+ #else
268
+ const global char* k_tile_base = use_kv_pad ? (const global char*) k_pad_void : k_base;
269
+ #endif
270
+ const global char* v_tile_base = use_kv_pad ? (const global char*) v_pad_void : v_base;
271
+
272
+ for (int i = tid; i < BLOCK_N * DK_VEC; i += WG_SIZE) {
273
+ const int row = i / DK_VEC;
274
+ const int col = i % DK_VEC;
275
+ const int k_row_idx = k_tile_start + row;
276
+ if (use_kv_pad || k_row_idx < n_kv) {
277
+ #ifdef FA_K_IMG
278
+ if (use_kv_pad) {
279
+ const ulong k_row_offset = batch_idx * k_tile_nb3 + head_kv_idx * k_tile_nb2 + k_row_idx * k_nb1;
280
+ FA_LK(row, col) = ((__global KV_DATA_TYPE4*)(k_tile_base + k_row_offset))[col];
281
+ } else {
282
+ const int k_row_px = batch_idx * k_pitch_px_batch + head_kv_idx * k_pitch_px_head + k_row_idx * k_pitch_px_row;
283
+ FA_LK(row, col) = read_imageh(k_img, k_row_px + col);
284
+ }
285
+ #else
286
+ const ulong k_row_offset = batch_idx * k_tile_nb3 + head_kv_idx * k_tile_nb2 + k_row_idx * k_nb1;
287
+ FA_LK(row, col) = ((__global KV_DATA_TYPE4*)(k_tile_base + k_row_offset))[col];
288
+ #endif
289
+ } else {
290
+ FA_LK(row, col) = (KV_DATA_TYPE4)(0.0h);
291
+ }
292
+ }
293
+ for (int i = tid; i < BLOCK_N * DV_VEC; i += WG_SIZE) {
294
+ const int row = i / DV_VEC;
295
+ const int col = i % DV_VEC;
296
+ const int v_row_idx = k_tile_start + row;
297
+ if (use_kv_pad || v_row_idx < n_kv) {
298
+ const ulong v_row_offset = batch_idx * v_tile_nb3 + head_kv_idx * v_tile_nb2 + v_row_idx * v_nb1;
299
+ l_v[row][col] = ((__global KV_DATA_TYPE4*)(v_tile_base + v_row_offset))[col];
300
+ } else {
301
+ l_v[row][col] = (KV_DATA_TYPE4)(0.0h);
302
+ }
303
+ }
304
+ barrier(CLK_LOCAL_MEM_FENCE);
305
+
306
+ #if N_SPLIT > 1 && defined(HAS_SUBGROUP_SHUFFLE)
307
+ {
308
+ const int dv_off = split_idx * SPLIT_DV_VEC;
309
+ for (int j = 0; j < BLOCK_N; j += 2) {
310
+ const int k_row0 = k_start + j;
311
+ const int k_row1 = k_start + j + 1;
312
+
313
+ ACC_TYPE partial0 = 0.0f;
314
+ ACC_TYPE partial1 = 0.0f;
315
+ FA_UNROLL
316
+ for (int k = 0; k < SPLIT_DK_VEC; k++) {
317
+ const ACC_TYPE4 qk = q_priv[k];
318
+ #if defined(FA_K_LDS_T)
319
+ // 2 KV rows adjacent in the transposed tile: one 128-bit local read.
320
+ const half8 kk = FA_LK_PAIR(dk_off + k, j);
321
+ ACC_TYPE4 dot0 = qk * CONVERT_KV_ACC4(kk.lo);
322
+ ACC_TYPE4 dot1 = qk * CONVERT_KV_ACC4(kk.hi);
323
+ #else
324
+ ACC_TYPE4 dot0 = qk * CONVERT_KV_ACC4(l_k[j ][dk_off + k]);
325
+ ACC_TYPE4 dot1 = qk * CONVERT_KV_ACC4(l_k[j+1][dk_off + k]);
326
+ #endif
327
+ partial0 += dot0.s0 + dot0.s1 + dot0.s2 + dot0.s3;
328
+ partial1 += dot1.s0 + dot1.s1 + dot1.s2 + dot1.s3;
329
+ }
330
+
331
+ FA_UNROLL
332
+ for (int step = 1; step < N_SPLIT; step <<= 1) {
333
+ partial0 += sub_group_shuffle_xor(partial0, step);
334
+ partial1 += sub_group_shuffle_xor(partial1, step);
335
+ }
336
+
337
+ ACC_TYPE score0 = partial0 * scale;
338
+ ACC_TYPE score1 = partial1 * scale;
339
+
340
+ if (!query_valid) { score0 = FA_M_INIT; score1 = FA_M_INIT; }
341
+ if (is_causal) {
342
+ if (k_row0 > (n_kv - n_q + my_query_row)) score0 = FA_M_INIT;
343
+ if (k_row1 > (n_kv - n_q + my_query_row)) score1 = FA_M_INIT;
344
+ }
345
+ if (k_row0 >= n_kv) score0 = FA_M_INIT;
346
+ if (k_row1 >= n_kv) score1 = FA_M_INIT;
347
+
348
+ if (query_valid && mask_base != NULL && blk_cur != 2) {
349
+ if (use_kv_pad && mask_pad_base != NULL) {
350
+ const global MASK_DATA_TYPE* mask_ptr =
351
+ (const global MASK_DATA_TYPE*)(mask_pad_base + my_query_row * mask_pad_nb1);
352
+ score0 += slope * (ACC_TYPE)mask_ptr[j];
353
+ score1 += slope * (ACC_TYPE)mask_ptr[j + 1];
354
+ } else {
355
+ const global MASK_DATA_TYPE* mask_ptr =
356
+ (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1);
357
+ if (k_row0 < n_kv) score0 += slope * (ACC_TYPE)mask_ptr[k_row0];
358
+ if (k_row1 < n_kv) score1 += slope * (ACC_TYPE)mask_ptr[k_row1];
359
+ }
360
+ }
361
+
362
+ if (logit_softcap > 0.0f) {
363
+ score0 = logit_softcap * tanh(score0 / logit_softcap);
364
+ score1 = logit_softcap * tanh(score1 / logit_softcap);
365
+ }
366
+
367
+ const ACC_TYPE m_new = max(m_i, max(score0, score1));
368
+ // Whole tile masked (m_new == FA_M_INIT): force the exp() args
369
+ // far negative so the tile contributes 0, not exp(0)=1.
370
+ const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new;
371
+ const ACC_TYPE sp = native_exp(m_i - m_exp);
372
+ const ACC_TYPE p0 = native_exp(score0 - m_exp);
373
+ const ACC_TYPE p1 = native_exp(score1 - m_exp);
374
+
375
+ FA_UNROLL
376
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) {
377
+ o_acc[i] = o_acc[i] * sp
378
+ + p0 * CONVERT_KV_ACC4(l_v[j ][dv_off + i])
379
+ + p1 * CONVERT_KV_ACC4(l_v[j+1][dv_off + i]);
380
+ }
381
+ l_i = l_i * sp + p0 + p1;
382
+ m_i = m_new;
383
+ }
384
+ }
385
+ #elif N_SPLIT > 1
386
+ // N_SPLIT>1 fallback (no shuffle): 3-phase local-memory reduction.
387
+ // Phase 1 — partial dots for all BLOCK_N tokens.
388
+ for (int j = 0; j < BLOCK_N; ++j) {
389
+ ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f);
390
+ FA_UNROLL
391
+ for (int k = 0; k < SPLIT_DK_VEC; k++) {
392
+ dot_acc = mad(q_priv[k], CONVERT_KV_ACC4(FA_LK(j, dk_off + k)), dot_acc);
393
+ }
394
+ local_partial[j][tid] =
395
+ dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3;
396
+ }
397
+ barrier(CLK_LOCAL_MEM_FENCE); // 1 barrier: partial dots visible
398
+
399
+ // Phase 2 — split_idx==0 reduces partial sums and computes block softmax.
400
+ if (split_idx == 0) {
401
+ if (query_valid) {
402
+ ACC_TYPE m_new = m_i;
403
+ for (int j = 0; j < BLOCK_N; ++j) {
404
+ const int k_row = k_start + j;
405
+ ACC_TYPE score = 0.0f;
406
+ FA_UNROLL
407
+ for (int s = 0; s < N_SPLIT; s++) {
408
+ score += local_partial[j][q_lane * N_SPLIT + s];
409
+ }
410
+ score *= scale;
411
+
412
+ if (is_causal && k_row > (n_kv - n_q + my_query_row)) score = FA_M_INIT;
413
+ if (k_row >= n_kv) score = FA_M_INIT;
414
+
415
+ if (mask_base != NULL && blk_cur != 2) {
416
+ if (use_kv_pad && mask_pad_base != NULL) {
417
+ const global MASK_DATA_TYPE* mask_ptr =
418
+ (const global MASK_DATA_TYPE*)(mask_pad_base + my_query_row * mask_pad_nb1);
419
+ score += slope * (ACC_TYPE)mask_ptr[j];
420
+ } else {
421
+ const global MASK_DATA_TYPE* mask_ptr =
422
+ (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1);
423
+ if (k_row < n_kv) score += slope * (ACC_TYPE)mask_ptr[k_row];
424
+ }
425
+ }
426
+
427
+ if (logit_softcap > 0.0f) {
428
+ score = logit_softcap * tanh(score / logit_softcap);
429
+ }
430
+
431
+ m_new = max(m_new, score);
432
+ local_p[q_lane][j] = score;
433
+ }
434
+
435
+ const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new;
436
+ const ACC_TYPE sp = native_exp(m_i - m_exp);
437
+ ACC_TYPE l_new = l_i * sp;
438
+ for (int j = 0; j < BLOCK_N; ++j) {
439
+ const ACC_TYPE p = native_exp(local_p[q_lane][j] - m_exp);
440
+ local_p[q_lane][j] = p;
441
+ l_new += p;
442
+ }
443
+ local_softmax_scale[q_lane] = sp;
444
+ l_i = l_new;
445
+ m_i = m_new;
446
+ } else {
447
+ local_softmax_scale[q_lane] = 1.0f;
448
+ for (int j = 0; j < BLOCK_N; ++j) local_p[q_lane][j] = 0.0f;
449
+ }
450
+ }
451
+ barrier(CLK_LOCAL_MEM_FENCE);
452
+
453
+ // Phase 3 — V accumulate using broadcast probabilities.
454
+ {
455
+ const ACC_TYPE sp_block = local_softmax_scale[q_lane];
456
+ const int dv_off = split_idx * SPLIT_DV_VEC;
457
+ FA_UNROLL
458
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) {
459
+ o_acc[i] *= sp_block;
460
+ }
461
+ for (int j = 0; j < BLOCK_N; ++j) {
462
+ const ACC_TYPE p = local_p[q_lane][j];
463
+ FA_UNROLL
464
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) {
465
+ o_acc[i] = mad(p, CONVERT_KV_ACC4(l_v[j][dv_off + i]), o_acc[i]);
466
+ }
467
+ }
468
+ }
469
+ #else
470
+ // N_SPLIT==1: j+=4 unroll. Requires BLOCK_N % 4 == 0.
471
+ if (query_valid) {
472
+ for (int j = 0; j < BLOCK_N; j += 4) {
473
+ const int k_row0 = k_start + j;
474
+ const int k_row1 = k_start + j + 1;
475
+ const int k_row2 = k_start + j + 2;
476
+ const int k_row3 = k_start + j + 3;
477
+
478
+ ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f);
479
+ ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f);
480
+ ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f);
481
+ ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f);
482
+ FA_UNROLL
483
+ for (int k = 0; k < DK_VEC; k++) {
484
+ const ACC_TYPE4 qk = q_priv[k];
485
+ #if defined(FA_K_LDS_T)
486
+ // 4 KV rows adjacent in the transposed tile: two 128-bit local reads
487
+ // instead of four 64-bit ones.
488
+ const half8 kk01 = FA_LK_PAIR(k, j);
489
+ const half8 kk23 = FA_LK_PAIR(k, j + 2);
490
+ dot_acc0 = mad(qk, CONVERT_KV_ACC4(kk01.lo), dot_acc0);
491
+ dot_acc1 = mad(qk, CONVERT_KV_ACC4(kk01.hi), dot_acc1);
492
+ dot_acc2 = mad(qk, CONVERT_KV_ACC4(kk23.lo), dot_acc2);
493
+ dot_acc3 = mad(qk, CONVERT_KV_ACC4(kk23.hi), dot_acc3);
494
+ #else
495
+ dot_acc0 = mad(qk, CONVERT_KV_ACC4(l_k[j][k]), dot_acc0);
496
+ dot_acc1 = mad(qk, CONVERT_KV_ACC4(l_k[j+1][k]), dot_acc1);
497
+ dot_acc2 = mad(qk, CONVERT_KV_ACC4(l_k[j+2][k]), dot_acc2);
498
+ dot_acc3 = mad(qk, CONVERT_KV_ACC4(l_k[j+3][k]), dot_acc3);
499
+ #endif
500
+ }
501
+ ACC_TYPE s0 = (dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3) * scale;
502
+ ACC_TYPE s1 = (dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3) * scale;
503
+ ACC_TYPE s2 = (dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3) * scale;
504
+ ACC_TYPE s3 = (dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3) * scale;
505
+
506
+ if (is_causal) {
507
+ const int causal_limit = n_kv - n_q + my_query_row;
508
+ if (k_row0 > causal_limit) s0 = FA_M_INIT;
509
+ if (k_row1 > causal_limit) s1 = FA_M_INIT;
510
+ if (k_row2 > causal_limit) s2 = FA_M_INIT;
511
+ if (k_row3 > causal_limit) s3 = FA_M_INIT;
512
+ }
513
+ if (k_row0 >= n_kv) s0 = FA_M_INIT;
514
+ if (k_row1 >= n_kv) s1 = FA_M_INIT;
515
+ if (k_row2 >= n_kv) s2 = FA_M_INIT;
516
+ if (k_row3 >= n_kv) s3 = FA_M_INIT;
517
+
518
+ if (mask_base != NULL && blk_cur != 2) {
519
+ if (use_kv_pad && mask_pad_base != NULL) {
520
+ const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_pad_base + my_query_row * mask_pad_nb1);
521
+ s0 += slope * (ACC_TYPE)mask_ptr[j];
522
+ s1 += slope * (ACC_TYPE)mask_ptr[j + 1];
523
+ s2 += slope * (ACC_TYPE)mask_ptr[j + 2];
524
+ s3 += slope * (ACC_TYPE)mask_ptr[j + 3];
525
+ } else {
526
+ const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base + my_query_row * mask_nb1);
527
+ if (k_row0 < n_kv) s0 += slope * (ACC_TYPE)mask_ptr[k_row0];
528
+ if (k_row1 < n_kv) s1 += slope * (ACC_TYPE)mask_ptr[k_row1];
529
+ if (k_row2 < n_kv) s2 += slope * (ACC_TYPE)mask_ptr[k_row2];
530
+ if (k_row3 < n_kv) s3 += slope * (ACC_TYPE)mask_ptr[k_row3];
531
+ }
532
+ }
533
+
534
+ if (logit_softcap > 0.0f) {
535
+ s0 = logit_softcap * tanh(s0 / logit_softcap);
536
+ s1 = logit_softcap * tanh(s1 / logit_softcap);
537
+ s2 = logit_softcap * tanh(s2 / logit_softcap);
538
+ s3 = logit_softcap * tanh(s3 / logit_softcap);
539
+ }
540
+
541
+ const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3)));
542
+ // Whole tile masked (m_new == FA_M_INIT): force the exp() args
543
+ // far negative so the tile contributes 0, not exp(0)=1.
544
+ const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new;
545
+ const ACC_TYPE scale_prev = native_exp(m_i - m_exp);
546
+ const ACC_TYPE p0 = native_exp(s0 - m_exp);
547
+ const ACC_TYPE p1 = native_exp(s1 - m_exp);
548
+ const ACC_TYPE p2 = native_exp(s2 - m_exp);
549
+ const ACC_TYPE p3 = native_exp(s3 - m_exp);
550
+
551
+ FA_UNROLL
552
+ for (int i = 0; i < DV_VEC; ++i) {
553
+ o_acc[i] = mad(p3, CONVERT_KV_ACC4(l_v[j+3][i]),
554
+ mad(p2, CONVERT_KV_ACC4(l_v[j+2][i]),
555
+ mad(p1, CONVERT_KV_ACC4(l_v[j+1][i]),
556
+ mad(p0, CONVERT_KV_ACC4(l_v[j][i]),
557
+ o_acc[i] * scale_prev))));
558
+ }
559
+ l_i = l_i * scale_prev + p0 + p1 + p2 + p3;
560
+ m_i = m_new;
561
+ }
562
+ }
563
+ #endif
564
+ // End of tile: every thread must finish reading l_k/l_v before the
565
+ // next iteration's load overwrites them (WAR hazard on local memory).
566
+ barrier(CLK_LOCAL_MEM_FENCE);
567
+ }
568
+
569
+ // Write output.
570
+ #if N_SPLIT > 1 && defined(HAS_SUBGROUP_SHUFFLE)
571
+ if (query_valid) {
572
+ ACC_TYPE sinks_sp = 1.0f;
573
+ if (sinks_void != NULL) {
574
+ const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);
575
+ const ACC_TYPE m_sink = sinks_ptr[head_idx];
576
+ const ACC_TYPE m_final = max(m_i, m_sink);
577
+ sinks_sp = exp(m_i - m_final);
578
+ l_i = l_i * sinks_sp + exp(m_sink - m_final);
579
+ m_i = m_final;
580
+ }
581
+ const ACC_TYPE l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f;
582
+ const int dv_off = split_idx * SPLIT_DV_VEC;
583
+ const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1;
584
+ global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset);
585
+ if (l_inv > 0.0f) {
586
+ FA_UNROLL
587
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) {
588
+ o_row[dv_off + i] = CONVERT_O_DATA4(o_acc[i] * sinks_sp * l_inv);
589
+ }
590
+ } else {
591
+ FA_UNROLL
592
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) {
593
+ o_row[dv_off + i] = (O_DATA_TYPE4)(0.0f);
594
+ }
595
+ }
596
+ }
597
+ #elif N_SPLIT > 1
598
+ if (split_idx == 0) {
599
+ ACC_TYPE sinks_sp = 1.0f;
600
+ if (query_valid && sinks_void != NULL) {
601
+ const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);
602
+ const ACC_TYPE m_sink = sinks_ptr[head_idx];
603
+ const ACC_TYPE m_final = max(m_i, m_sink);
604
+ sinks_sp = exp(m_i - m_final);
605
+ l_i = l_i * sinks_sp + exp(m_sink - m_final);
606
+ m_i = m_final;
607
+ }
608
+ local_softmax_scale[q_lane] = sinks_sp;
609
+ local_l_inv[q_lane] = (query_valid && l_i > 0.0f) ? (1.0f / l_i) : 0.0f;
610
+ }
611
+ barrier(CLK_LOCAL_MEM_FENCE);
612
+
613
+ if (query_valid) {
614
+ const ACC_TYPE sinks_sp = local_softmax_scale[q_lane];
615
+ const ACC_TYPE l_inv = local_l_inv[q_lane];
616
+ const int dv_off = split_idx * SPLIT_DV_VEC;
617
+ const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1;
618
+ global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset);
619
+ if (l_inv > 0.0f) {
620
+ FA_UNROLL
621
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) {
622
+ o_row[dv_off + i] = CONVERT_O_DATA4(o_acc[i] * sinks_sp * l_inv);
623
+ }
624
+ } else {
625
+ FA_UNROLL
626
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) {
627
+ o_row[dv_off + i] = (O_DATA_TYPE4)(0.0f);
628
+ }
629
+ }
630
+ }
631
+ #else
632
+ if (query_valid) {
633
+ if (sinks_void != NULL) {
634
+ const global ACC_TYPE* sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);
635
+ const ACC_TYPE m_sink = sinks_ptr[head_idx];
636
+ const ACC_TYPE m_final = max(m_i, m_sink);
637
+
638
+ const ACC_TYPE scale_o = exp(m_i - m_final);
639
+ FA_UNROLL
640
+ for (int i = 0; i < DV_VEC; ++i) {
641
+ o_acc[i] *= scale_o;
642
+ }
643
+
644
+ l_i = l_i * exp(m_i - m_final) + exp(m_sink - m_final);
645
+ }
646
+
647
+ const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1;
648
+ global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset);
649
+ if (l_i > 0.0f) {
650
+ const ACC_TYPE l_inv = 1.0f / l_i;
651
+ FA_UNROLL
652
+ for (int i = 0; i < DV_VEC; ++i) {
653
+ o_row[i] = CONVERT_O_DATA4(o_acc[i] * l_inv);
654
+ }
655
+ } else {
656
+ FA_UNROLL
657
+ for (int i = 0; i < DV_VEC; ++i) {
658
+ o_row[i] = (O_DATA_TYPE4)(0.0f);
659
+ }
660
+ }
661
+ }
662
+ #endif
663
+ }
664
+ #endif // !FA_DECODE_ONLY
665
+
666
+ // allow bypassing decode kernels to avoid compiler crash for DK=512 on Adreno GPUs
667
+ #ifndef FA_PREFILL_ONLY
668
+ #ifndef FA_MQ_ONLY // q1 excluded from the MQ-only (g8) program
669
+ REQD_FA_SG
670
+ __kernel void flash_attn_f32_f16_q1(
671
+ const global void * q_void, ulong q_offset,
672
+ const global void * k_void, ulong k_offset,
673
+ const global void * v_void, ulong v_offset,
674
+ global void * o_void, ulong o_offset,
675
+ const float scale,
676
+ const int n_q,
677
+ const int n_kv,
678
+ const int is_causal,
679
+ const int n_head,
680
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
681
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
682
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
683
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
684
+ const float max_bias,
685
+ const float m0,
686
+ const float m1,
687
+ const int n_head_log2,
688
+ const float logit_softcap,
689
+ const int n_head_kv,
690
+ const global void* mask_void,
691
+ const ulong mask_offset,
692
+ const ulong mask_nb1,
693
+ const ulong mask_nb2,
694
+ const ulong mask_nb3,
695
+ const int mask_ne2,
696
+ const int mask_ne3,
697
+ const global void* sinks_void,
698
+ const ulong sinks_offset
699
+ ) {
700
+ const int tid = get_local_id(0);
701
+ const int head_batch_idx = get_global_id(1);
702
+
703
+ const int batch_idx = head_batch_idx / n_head;
704
+ const int head_idx = head_batch_idx % n_head;
705
+
706
+ const int gqa_ratio = n_head / n_head_kv;
707
+ const int head_kv_idx = head_idx / gqa_ratio;
708
+
709
+ const global char* q_base = (const global char*)q_void + q_offset;
710
+ #ifndef FA_K_IMG
711
+ const global char* k_base = (const global char*)k_void + k_offset;
712
+ #endif
713
+ const global char* v_base = (const global char*)v_void + v_offset;
714
+ global char* o_base = (global char*)o_void + o_offset;
715
+
716
+ const global char* mask_base = NULL;
717
+ if (mask_void != NULL) {
718
+ const int mask_head_idx = head_idx % mask_ne2;
719
+ const int mask_batch_idx = batch_idx % mask_ne3;
720
+ mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;
721
+ }
722
+
723
+ // Q is uniform across WG threads (n_q=1). Share via local memory to
724
+ // avoid per-thread q_priv[DK_VEC] dynamic-indexed private array that
725
+ // spills to DDR on Adreno.
726
+ __local ACC_TYPE4 q_shared[DK_VEC];
727
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2;
728
+ const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset);
729
+ for (int i = tid; i < DK_VEC; i += Q1_WG_SIZE) {
730
+ q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]);
731
+ }
732
+ sub_group_barrier(CLK_LOCAL_MEM_FENCE);
733
+
734
+ float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
735
+
736
+ const global ACC_TYPE* sinks_ptr = NULL;
737
+ if (sinks_void != NULL) {
738
+ sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);
739
+ }
740
+
741
+ ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT;
742
+ for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) {
743
+ const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
744
+ const global KV_DATA_TYPE4* k_ptr = (const global KV_DATA_TYPE4*)(k_base + k_row_offset);
745
+ ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f);
746
+ FA_UNROLL
747
+ for (int k = 0; k < DK_VEC; k++) {
748
+ dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc);
749
+ }
750
+ ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
751
+ if (mask_base != NULL) {
752
+ const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);
753
+ score += slope * (ACC_TYPE)mask_ptr[k_idx];
754
+ }
755
+ if (logit_softcap > 0.0f) {
756
+ score = logit_softcap * tanh(score / logit_softcap);
757
+ }
758
+ m_i = max(m_i, score);
759
+ }
760
+
761
+ const ACC_TYPE m_final = sub_group_reduce_max(m_i);
762
+
763
+ ACC_TYPE4 o_acc[DV_VEC];
764
+ FA_UNROLL
765
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);
766
+ ACC_TYPE l_i = 0.0f;
767
+
768
+ for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) {
769
+ const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
770
+ const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
771
+ const global KV_DATA_TYPE4* k_ptr = (const global KV_DATA_TYPE4*)(k_base + k_row_offset);
772
+ const global KV_DATA_TYPE4* v_ptr = (const global KV_DATA_TYPE4*)(v_base + v_row_offset);
773
+ ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f);
774
+ FA_UNROLL
775
+ for (int k = 0; k < DK_VEC; k++) {
776
+ dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc);
777
+ }
778
+ ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
779
+ if (mask_base != NULL) {
780
+ const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);
781
+ score += slope * (ACC_TYPE)mask_ptr[k_idx];
782
+ }
783
+ if (logit_softcap > 0.0f) {
784
+ score = logit_softcap * tanh(score / logit_softcap);
785
+ }
786
+ const ACC_TYPE p = exp(score - m_final);
787
+ l_i += p;
788
+ FA_UNROLL
789
+ for (int i = 0; i < DV_VEC; i++) {
790
+ o_acc[i] = mad(p, CONVERT_KV_ACC4(v_ptr[i]), o_acc[i]);
791
+ }
792
+ }
793
+
794
+ __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE];
795
+ const ACC_TYPE l_red = sub_group_reduce_add(l_i);
796
+
797
+ const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1;
798
+ global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset);
799
+ ACC_TYPE l_final = l_red;
800
+
801
+ if (sinks_ptr != NULL) {
802
+ l_final += exp(sinks_ptr[head_idx] - m_final);
803
+ }
804
+
805
+ if (l_final > 0.0f) {
806
+ const ACC_TYPE l_inv = 1.0f / l_final;
807
+ for (int i = 0; i < DV_VEC; i++) {
808
+ local_o_comp[tid] = o_acc[i];
809
+ sub_group_barrier(CLK_LOCAL_MEM_FENCE);
810
+ FA_UNROLL
811
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
812
+ if (tid < s) local_o_comp[tid] += local_o_comp[tid + s];
813
+ sub_group_barrier(CLK_LOCAL_MEM_FENCE);
814
+ }
815
+ if (tid == 0) {
816
+ o_row[i] = CONVERT_O_DATA4(local_o_comp[0] * l_inv);
817
+ }
818
+ }
819
+ } else if (tid == 0) {
820
+ FA_UNROLL
821
+ for (int i = 0; i < DV_VEC; ++i) o_row[i] = (O_DATA_TYPE4)(0.0f);
822
+ }
823
+ }
824
+
825
+ #endif // !FA_MQ_ONLY (q1)
826
+ // decode variant for large DV (e.g. Gemma-4 DK=DV=512 global layers).
827
+ #define VEC_NSG 4
828
+ #define VEC_WG_SIZE (Q1_WG_SIZE * VEC_NSG)
829
+ #define Q1V_DV_PER_THREAD ((DV_VEC + Q1_WG_SIZE - 1) / Q1_WG_SIZE)
830
+
831
+ // allow bypassing the kernel to avoid compiler crash for DK=512 on Adreno GPUs
832
+ #if !defined(FA_DECODE_MINIMAL) && !defined(FA_MQ_ONLY)
833
+ REQD_SUBGROUP_SIZE_64
834
+ __kernel void flash_attn_f32_f16_q1_vec(
835
+ const global void * q_void, ulong q_offset,
836
+ const global void * k_void, ulong k_offset,
837
+ const global void * v_void, ulong v_offset,
838
+ global void * o_void, ulong o_offset,
839
+ const float scale,
840
+ const int n_q,
841
+ const int n_kv,
842
+ const int is_causal,
843
+ const int n_head,
844
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
845
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
846
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
847
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
848
+ const float max_bias,
849
+ const float m0,
850
+ const float m1,
851
+ const int n_head_log2,
852
+ const float logit_softcap,
853
+ const int n_head_kv,
854
+ const global void* mask_void,
855
+ const ulong mask_offset,
856
+ const ulong mask_nb1,
857
+ const ulong mask_nb2,
858
+ const ulong mask_nb3,
859
+ const int mask_ne2,
860
+ const int mask_ne3,
861
+ const global void* sinks_void,
862
+ const ulong sinks_offset
863
+ ) {
864
+ const int tid = get_local_id(0);
865
+ const int sgid = tid / Q1_WG_SIZE; // subgroup index (0..VEC_NSG-1)
866
+ const int tid_sg = tid % Q1_WG_SIZE; // lane within subgroup
867
+ const int head_batch_idx = get_global_id(1);
868
+
869
+ const int batch_idx = head_batch_idx / n_head;
870
+ const int head_idx = head_batch_idx % n_head;
871
+
872
+ const int gqa_ratio = n_head / n_head_kv;
873
+ const int head_kv_idx = head_idx / gqa_ratio;
874
+
875
+ const global char * q_base = (const global char *) q_void + q_offset;
876
+ const global char * k_base = (const global char *) k_void + k_offset;
877
+ const global char * v_base = (const global char *) v_void + v_offset;
878
+ global char * o_base = (global char *) o_void + o_offset;
879
+
880
+ const global char * mask_base = NULL;
881
+ if (mask_void != NULL) {
882
+ const int mask_head_idx = head_idx % mask_ne2;
883
+ const int mask_batch_idx = batch_idx % mask_ne3;
884
+ mask_base = (const global char *) mask_void + mask_offset +
885
+ mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;
886
+ }
887
+
888
+ // Q is uniform across the WG — stage in __local once. All WG threads load.
889
+ __local ACC_TYPE4 q_shared[DK_VEC];
890
+ {
891
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2;
892
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
893
+ for (int i = tid; i < DK_VEC; i += VEC_WG_SIZE) {
894
+ q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]);
895
+ }
896
+ }
897
+ barrier(CLK_LOCAL_MEM_FENCE);
898
+
899
+ const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
900
+
901
+ const global ACC_TYPE * sinks_ptr = NULL;
902
+ if (sinks_void != NULL) {
903
+ sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset);
904
+ }
905
+
906
+ // per-thread DV slice within its subgroup
907
+ // DV=512 -> 2x float4 = 32 bytes; DV=256 -> 1x float4 - no spill
908
+ ACC_TYPE4 o_acc[Q1V_DV_PER_THREAD];
909
+ #pragma unroll
910
+ for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);
911
+
912
+ // each subgroup independently runs the FA-2 online softmax over its slice of n_kv.
913
+ // sinks are not folded into per-subgroup m_i — they're added once in
914
+ // the cross-subgroup merge to avoid double-counting.
915
+ ACC_TYPE m_i = FA_M_INIT;
916
+ ACC_TYPE l_i = 0.0f;
917
+
918
+ const int kv_per_sg = (n_kv + VEC_NSG - 1) / VEC_NSG;
919
+ const int kv_start = sgid * kv_per_sg;
920
+ const int kv_end = min(n_kv, kv_start + kv_per_sg);
921
+
922
+ for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) {
923
+ const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
924
+ const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
925
+ const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_off);
926
+ const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off);
927
+
928
+ // Q*K^T: each thread accumulates its DK slice; subgroup-reduce the partial.
929
+ ACC_TYPE4 dot4 = (ACC_TYPE4)(0.0f);
930
+ for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) {
931
+ dot4 = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot4);
932
+ }
933
+ ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3;
934
+ ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale;
935
+
936
+ if (mask_base != NULL) {
937
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base;
938
+ score += slope * (ACC_TYPE) mask_ptr[k_idx];
939
+ }
940
+ if (logit_softcap > 0.0f) {
941
+ score = logit_softcap * tanh(score / logit_softcap);
942
+ }
943
+
944
+ // FA-2 online update. All threads in the subgroup see the same score,
945
+ // so m_i and l_i evolve identically across lanes within the subgroup.
946
+ const ACC_TYPE m_new = max(m_i, score);
947
+ const ACC_TYPE scale_prev = native_exp(m_i - m_new);
948
+ const ACC_TYPE p = native_exp(score - m_new);
949
+
950
+ int idx = 0;
951
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) {
952
+ o_acc[idx] = mad(p, CONVERT_KV_ACC4(v_ptr[dv_idx]), o_acc[idx] * scale_prev);
953
+ }
954
+ l_i = l_i * scale_prev + p;
955
+ m_i = m_new;
956
+ }
957
+
958
+ // Cross-subgroup merge via __local. Each subgroup publishes (m_i, l_i)
959
+ // and its o_acc slice; subgroup 0 then folds them into the final norm
960
+ // and writes the row.
961
+ __local ACC_TYPE sg_m[VEC_NSG];
962
+ __local ACC_TYPE sg_l[VEC_NSG];
963
+ __local ACC_TYPE4 sg_o[VEC_NSG][DV_VEC];
964
+
965
+ if (tid_sg == 0) {
966
+ sg_m[sgid] = m_i;
967
+ sg_l[sgid] = l_i;
968
+ }
969
+ {
970
+ int idx = 0;
971
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) {
972
+ sg_o[sgid][dv_idx] = o_acc[idx];
973
+ }
974
+ }
975
+ barrier(CLK_LOCAL_MEM_FENCE);
976
+
977
+ if (sgid == 0) {
978
+ // m_final = max over all subgroups' m_i, plus the sink (if any).
979
+ ACC_TYPE m_final = sg_m[0];
980
+ #pragma unroll
981
+ for (int s = 1; s < VEC_NSG; ++s) {
982
+ m_final = max(m_final, sg_m[s]);
983
+ }
984
+ if (sinks_ptr != NULL) {
985
+ m_final = max(m_final, sinks_ptr[head_idx]);
986
+ }
987
+
988
+ ACC_TYPE l_final = 0.0f;
989
+ #pragma unroll
990
+ for (int s = 0; s < VEC_NSG; ++s) {
991
+ l_final += sg_l[s] * native_exp(sg_m[s] - m_final);
992
+ }
993
+ if (sinks_ptr != NULL) {
994
+ l_final += native_exp(sinks_ptr[head_idx] - m_final);
995
+ }
996
+ const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f;
997
+
998
+ const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1;
999
+ global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset);
1000
+
1001
+ // Each thread in subgroup 0 writes its DV slice, folding all subgroups'
1002
+ // contributions with the rescale factor.
1003
+ int idx = 0;
1004
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) {
1005
+ ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f);
1006
+ #pragma unroll
1007
+ for (int s = 0; s < VEC_NSG; ++s) {
1008
+ const ACC_TYPE alpha = native_exp(sg_m[s] - m_final);
1009
+ o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged);
1010
+ }
1011
+ o_row[dv_idx] = CONVERT_O_DATA4(o_merged * l_inv);
1012
+ }
1013
+ }
1014
+ }
1015
+
1016
+ #endif // !FA_DECODE_MINIMAL
1017
+
1018
+ #ifndef FA_DECODE_ONLY
1019
+
1020
+ // flash_attn_f32_f16_q1_local_tile
1021
+ // one WG per (q_idx, q_head)
1022
+
1023
+ #define LT_KC 32
1024
+ #define LT_WG 128
1025
+
1026
+ #ifndef FA_MQ_ONLY // q1_local_tile excluded from the MQ-only (g8) program
1027
+ REQD_SUBGROUP_SIZE_64
1028
+ __kernel void flash_attn_f32_f16_q1_local_tile(
1029
+ const global void * q_void, ulong q_offset,
1030
+ const global void * k_void, ulong k_offset,
1031
+ const global void * v_void, ulong v_offset,
1032
+ global void * o_void, ulong o_offset,
1033
+ const float scale,
1034
+ const int n_q,
1035
+ const int n_kv,
1036
+ const int is_causal,
1037
+ const int n_head,
1038
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
1039
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
1040
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
1041
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
1042
+ const float max_bias,
1043
+ const float m0,
1044
+ const float m1,
1045
+ const int n_head_log2,
1046
+ const float logit_softcap,
1047
+ const int n_head_kv,
1048
+ const global void * mask_void,
1049
+ const ulong mask_offset,
1050
+ const ulong mask_nb1,
1051
+ const ulong mask_nb2,
1052
+ const ulong mask_nb3,
1053
+ const int mask_ne2,
1054
+ const int mask_ne3,
1055
+ const global void * sinks_void,
1056
+ const ulong sinks_offset
1057
+ ) {
1058
+ const int q_idx = get_global_id(0) / LT_WG;
1059
+ const int head_idx = get_global_id(1);
1060
+ const int batch_idx = get_global_id(2);
1061
+ const int tid = get_local_id(0);
1062
+
1063
+ const int gqa_ratio = n_head_kv > 0 ? (n_head / n_head_kv) : 1;
1064
+ const int head_kv_idx = head_idx / gqa_ratio;
1065
+
1066
+ const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
1067
+
1068
+ __local half k_tile[LT_KC * DK]; // 32*128*2 = 8 KB at DK=128
1069
+ __local half v_tile[LT_KC * DV]; // 8 KB
1070
+ __local float red[LT_WG]; // 512 B reduction scratch
1071
+ __local float score_shared; // broadcast score (each K-step)
1072
+
1073
+ // Each thread owns one float of Q at index `tid` (assumes LT_WG == DK).
1074
+ const global char * q_row_base = (const global char *) q_void + q_offset +
1075
+ batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
1076
+ float q_val = ((const global float *) q_row_base)[tid];
1077
+
1078
+ const global char * mask_base = NULL;
1079
+ if (mask_void != NULL) {
1080
+ const int mask_head_idx = head_idx % mask_ne2;
1081
+ const int mask_batch_idx = batch_idx % mask_ne3;
1082
+ mask_base = (const global char *) mask_void + mask_offset +
1083
+ mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 +
1084
+ (ulong) q_idx * mask_nb1;
1085
+ }
1086
+
1087
+ float o_val = 0.0f;
1088
+ float m_i = FA_M_INIT;
1089
+ float l_i = 0.0f;
1090
+
1091
+ for (int kb = 0; kb < n_kv; kb += LT_KC) {
1092
+ const int tile_len = min(LT_KC, n_kv - kb);
1093
+
1094
+ // Stage K and V tiles into __local.
1095
+ for (int i = tid; i < tile_len * DK; i += LT_WG) {
1096
+ const int j = i / DK;
1097
+ const int d = i % DK;
1098
+ const int kv_idx = kb + j;
1099
+ const global char * k_row = (const global char *) k_void + k_offset +
1100
+ batch_idx * k_nb3 + head_kv_idx * k_nb2 +
1101
+ (ulong) kv_idx * k_nb1;
1102
+ const global char * v_row = (const global char *) v_void + v_offset +
1103
+ batch_idx * v_nb3 + head_kv_idx * v_nb2 +
1104
+ (ulong) kv_idx * v_nb1;
1105
+ k_tile[j * DK + d] = ((const global half *) k_row)[d];
1106
+ v_tile[j * DV + d] = ((const global half *) v_row)[d];
1107
+ }
1108
+ barrier(CLK_LOCAL_MEM_FENCE);
1109
+
1110
+ for (int j = 0; j < tile_len; ++j) {
1111
+ const int kv_idx = kb + j;
1112
+
1113
+ // Q·K dot via __local tree-reduce.
1114
+ red[tid] = q_val * convert_float(k_tile[j * DK + tid]);
1115
+ barrier(CLK_LOCAL_MEM_FENCE);
1116
+ for (int stride = LT_WG >> 1; stride > 0; stride >>= 1) {
1117
+ if (tid < stride) {
1118
+ red[tid] += red[tid + stride];
1119
+ }
1120
+ barrier(CLK_LOCAL_MEM_FENCE);
1121
+ }
1122
+
1123
+ if (tid == 0) {
1124
+ float s = red[0] * scale;
1125
+ if (mask_base != NULL) {
1126
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base;
1127
+ s += slope * (float) mask_ptr[kv_idx];
1128
+ }
1129
+ if (logit_softcap > 0.0f) {
1130
+ s = logit_softcap * tanh(s / logit_softcap);
1131
+ }
1132
+ score_shared = s;
1133
+ }
1134
+ barrier(CLK_LOCAL_MEM_FENCE);
1135
+
1136
+ const float s = score_shared;
1137
+ const float m_new = fmax(m_i, s);
1138
+ const float alpha = native_exp(m_i - m_new);
1139
+ const float beta = native_exp(s - m_new);
1140
+
1141
+ o_val = o_val * alpha + beta * convert_float(v_tile[j * DV + tid]);
1142
+ l_i = l_i * alpha + beta;
1143
+ m_i = m_new;
1144
+ }
1145
+ barrier(CLK_LOCAL_MEM_FENCE);
1146
+ }
1147
+
1148
+ // Fold attention sinks into the running (m, l, o), if present.
1149
+ if (sinks_void != NULL) {
1150
+ const global float * sinks_ptr =
1151
+ (const global float *) ((const global char *) sinks_void + sinks_offset);
1152
+ const float m_sink = sinks_ptr[head_idx];
1153
+ const float m_new = fmax(m_i, m_sink);
1154
+ const float alpha = native_exp(m_i - m_new);
1155
+ const float beta = native_exp(m_sink - m_new);
1156
+ o_val = o_val * alpha;
1157
+ l_i = l_i * alpha + beta;
1158
+ m_i = m_new;
1159
+ }
1160
+
1161
+ const float l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f;
1162
+ global float * o_row = (global float *) ((global char *) o_void + o_offset +
1163
+ batch_idx * o_nb3 + head_idx * o_nb1 +
1164
+ (ulong) q_idx * o_nb2);
1165
+ o_row[tid] = o_val * l_inv;
1166
+ }
1167
+
1168
+ // flash_attn_f32_f16_q1_local_mq_split
1169
+
1170
+ #define LMQ_WG 64
1171
+ #define LMQ_KC 32
1172
+ #define LMQ_DPL 2 // DK / LMQ_WG at DK=128
1173
+
1174
+ #endif // !FA_MQ_ONLY (q1_local_tile)
1175
+ #ifndef MQ_GQA
1176
+ #define MQ_GQA 4
1177
+ #endif
1178
+
1179
+ #ifndef FA_PARTIAL_FLOATS
1180
+ #define FA_PARTIAL_FLOATS (2 + DV)
1181
+ #endif
1182
+
1183
+ #ifndef FA_MQ_ONLY // q1_local_mq_split excluded from the MQ-only (g8) program
1184
+ REQD_SUBGROUP_SIZE_64
1185
+ __kernel void flash_attn_f32_f16_q1_local_mq_split(
1186
+ const global void * q_void, ulong q_offset,
1187
+ const global void * k_void, ulong k_offset,
1188
+ const global void * v_void, ulong v_offset,
1189
+ const float scale,
1190
+ const int n_q,
1191
+ const int n_kv,
1192
+ const int n_head,
1193
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
1194
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
1195
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
1196
+ const float max_bias,
1197
+ const float m0,
1198
+ const float m1,
1199
+ const int n_head_log2,
1200
+ const float logit_softcap,
1201
+ const int n_head_kv,
1202
+ const global void * mask_void,
1203
+ const ulong mask_offset,
1204
+ const ulong mask_nb1,
1205
+ const ulong mask_nb2,
1206
+ const ulong mask_nb3,
1207
+ const int mask_ne2,
1208
+ const int mask_ne3,
1209
+ global float * partial_void,
1210
+ const int n_splits,
1211
+ const int kv_per_split
1212
+ ) {
1213
+ const int tid = get_local_id(0); // 0..LMQ_WG-1
1214
+ const int kvhead_batch_idx = get_global_id(1);
1215
+ const int split_q_idx = get_global_id(2);
1216
+ const int split_idx = split_q_idx % n_splits;
1217
+ const int q_idx = split_q_idx / n_splits;
1218
+
1219
+ const int batch_idx = kvhead_batch_idx / n_head_kv;
1220
+ const int head_kv_idx = kvhead_batch_idx % n_head_kv;
1221
+
1222
+ const int kv_start = split_idx * kv_per_split;
1223
+ const int kv_end = min(kv_start + kv_per_split, n_kv);
1224
+
1225
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
1226
+
1227
+ if (kv_start >= kv_end) {
1228
+ // Empty split — write sentinel for each Q-head so merge treats it as 0.
1229
+ if (tid == 0) {
1230
+ #pragma unroll
1231
+ for (int h = 0; h < MQ_GQA; ++h) {
1232
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1233
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
1234
+ * n_splits + split_idx);
1235
+ global float * rec = partial_void + rec_idx * record_stride;
1236
+ rec[0] = FA_M_INIT;
1237
+ rec[1] = 0.0f;
1238
+ }
1239
+ }
1240
+ return;
1241
+ }
1242
+
1243
+ const global char * q_base = (const global char *) q_void + q_offset;
1244
+ const global char * k_base = (const global char *) k_void + k_offset;
1245
+ const global char * v_base = (const global char *) v_void + v_offset;
1246
+
1247
+ // Stage MQ_GQA Q rows in __local (MQ_GQA × DK floats).
1248
+ __local float q_shared[MQ_GQA * DK];
1249
+ for (int i = tid; i < MQ_GQA * DK; i += LMQ_WG) {
1250
+ const int h = i / DK;
1251
+ const int d = i % DK;
1252
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1253
+ const ulong q_row_off = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
1254
+ const global float * q_ptr = (const global float *) (q_base + q_row_off);
1255
+ q_shared[h * DK + d] = q_ptr[d];
1256
+ }
1257
+
1258
+ // K/V tile staging buffers (16 KB combined at DK=DV=128 KC=32).
1259
+ __local half k_tile[LMQ_KC * DK];
1260
+ __local half v_tile[LMQ_KC * DV];
1261
+
1262
+ // Per-h state held in private registers.
1263
+ float o_acc[MQ_GQA][LMQ_DPL];
1264
+ float m_i[MQ_GQA];
1265
+ float l_i[MQ_GQA];
1266
+ float slope[MQ_GQA];
1267
+ #pragma unroll
1268
+ for (int h = 0; h < MQ_GQA; ++h) {
1269
+ m_i[h] = FA_M_INIT;
1270
+ l_i[h] = 0.0f;
1271
+ slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);
1272
+ #pragma unroll
1273
+ for (int p = 0; p < LMQ_DPL; ++p) o_acc[h][p] = 0.0f;
1274
+ }
1275
+
1276
+ // Per-h mask pointers.
1277
+ const global char * mask_base[MQ_GQA];
1278
+ if (mask_void != NULL) {
1279
+ const int mask_batch_idx = batch_idx % mask_ne3;
1280
+ const global char * mask_base_b = (const global char *) mask_void + mask_offset +
1281
+ mask_batch_idx * mask_nb3 +
1282
+ (ulong) q_idx * mask_nb1;
1283
+ #pragma unroll
1284
+ for (int h = 0; h < MQ_GQA; ++h) {
1285
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1286
+ const int mask_head_idx = head_idx % mask_ne2;
1287
+ mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;
1288
+ }
1289
+ } else {
1290
+ #pragma unroll
1291
+ for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL;
1292
+ }
1293
+
1294
+ barrier(CLK_LOCAL_MEM_FENCE); // Ensure Q staged before first dot.
1295
+
1296
+ for (int kb = kv_start; kb < kv_end; kb += LMQ_KC) {
1297
+ const int tile_len = min((int) LMQ_KC, kv_end - kb);
1298
+
1299
+ // Cooperative load K + V tile.
1300
+ for (int i = tid; i < tile_len * DK; i += LMQ_WG) {
1301
+ const int j = i / DK;
1302
+ const int d = i % DK;
1303
+ const int kv_idx = kb + j;
1304
+ const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + (ulong) kv_idx * k_nb1;
1305
+ const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + (ulong) kv_idx * v_nb1;
1306
+ k_tile[j * DK + d] = ((const global half *) k_row)[d];
1307
+ v_tile[j * DV + d] = ((const global half *) v_row)[d];
1308
+ }
1309
+ barrier(CLK_LOCAL_MEM_FENCE);
1310
+
1311
+ // Process each cache row in the tile.
1312
+ for (int j = 0; j < tile_len; ++j) {
1313
+ const int kv_idx = kb + j;
1314
+
1315
+ // Dot product per h: lane owns LMQ_DPL D-elements at (tid*LMQ_DPL..).
1316
+ float score[MQ_GQA];
1317
+ #pragma unroll
1318
+ for (int h = 0; h < MQ_GQA; ++h) {
1319
+ float contrib = 0.0f;
1320
+ #pragma unroll
1321
+ for (int p = 0; p < LMQ_DPL; ++p) {
1322
+ const int d = tid * LMQ_DPL + p;
1323
+ contrib += q_shared[h * DK + d] * (float) k_tile[j * DK + d];
1324
+ }
1325
+ float s = sub_group_reduce_add(contrib) * scale;
1326
+ if (mask_base[h] != NULL) {
1327
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
1328
+ s += slope[h] * (float) mask_ptr[kv_idx];
1329
+ }
1330
+ if (logit_softcap > 0.0f) {
1331
+ s = logit_softcap * tanh(s / logit_softcap);
1332
+ }
1333
+ score[h] = s;
1334
+ }
1335
+
1336
+ // Online softmax update + V accumulation per h.
1337
+ float p_h[MQ_GQA];
1338
+ float sp_h[MQ_GQA];
1339
+ #pragma unroll
1340
+ for (int h = 0; h < MQ_GQA; ++h) {
1341
+ const float m_new = fmax(m_i[h], score[h]);
1342
+ sp_h[h] = native_exp(m_i[h] - m_new);
1343
+ p_h[h] = native_exp(score[h] - m_new);
1344
+ l_i[h] = l_i[h] * sp_h[h] + p_h[h];
1345
+ m_i[h] = m_new;
1346
+ }
1347
+
1348
+ #pragma unroll
1349
+ for (int p = 0; p < LMQ_DPL; ++p) {
1350
+ const int d = tid * LMQ_DPL + p;
1351
+ const float v_val = (float) v_tile[j * DV + d];
1352
+ #pragma unroll
1353
+ for (int h = 0; h < MQ_GQA; ++h) {
1354
+ o_acc[h][p] = o_acc[h][p] * sp_h[h] + p_h[h] * v_val;
1355
+ }
1356
+ }
1357
+ }
1358
+ barrier(CLK_LOCAL_MEM_FENCE); // Before next tile load overwrites k/v_tile.
1359
+ }
1360
+
1361
+ // write partial records: one per (h, split)
1362
+ #pragma unroll
1363
+ for (int h = 0; h < MQ_GQA; ++h) {
1364
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1365
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
1366
+ * n_splits + split_idx);
1367
+ global float * rec = partial_void + rec_idx * record_stride;
1368
+ global float * rec_o = rec + 2;
1369
+
1370
+ if (tid == 0) {
1371
+ rec[0] = m_i[h];
1372
+ rec[1] = l_i[h];
1373
+ }
1374
+ #pragma unroll
1375
+ for (int p = 0; p < LMQ_DPL; ++p) {
1376
+ const int d = tid * LMQ_DPL + p;
1377
+ rec_o[d] = o_acc[h][p];
1378
+ }
1379
+ }
1380
+ }
1381
+
1382
+ #endif // !FA_MQ_ONLY (q1_local_mq_split)
1383
+ #ifndef MQ_NSG
1384
+ #define MQ_NSG 4
1385
+ #endif
1386
+ #define MQ_WG_SIZE (Q1_WG_SIZE * MQ_NSG)
1387
+
1388
+ REQD_SUBGROUP_SIZE_64
1389
+ __kernel void flash_attn_f32_f16_q1_vec_mq(
1390
+ const global void * q_void, ulong q_offset,
1391
+ const global void * k_void, ulong k_offset,
1392
+ const global void * v_void, ulong v_offset,
1393
+ global void * o_void, ulong o_offset,
1394
+ const float scale,
1395
+ const int n_q,
1396
+ const int n_kv,
1397
+ const int is_causal,
1398
+ const int n_head,
1399
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
1400
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
1401
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
1402
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
1403
+ const float max_bias,
1404
+ const float m0,
1405
+ const float m1,
1406
+ const int n_head_log2,
1407
+ const float logit_softcap,
1408
+ const int n_head_kv,
1409
+ const global void* mask_void,
1410
+ const ulong mask_offset,
1411
+ const ulong mask_nb1,
1412
+ const ulong mask_nb2,
1413
+ const ulong mask_nb3,
1414
+ const int mask_ne2,
1415
+ const int mask_ne3,
1416
+ const global void* sinks_void,
1417
+ const ulong sinks_offset
1418
+ ) {
1419
+ const int tid = get_local_id(0);
1420
+ const int sgid = tid / Q1_WG_SIZE; // subgroup 0..MQ_NSG-1
1421
+ const int tid_sg = tid % Q1_WG_SIZE; // lane 0..63
1422
+ const int kvhead_batch_idx = get_global_id(1);
1423
+
1424
+ const int batch_idx = kvhead_batch_idx / n_head_kv;
1425
+ const int head_kv_idx = kvhead_batch_idx % n_head_kv;
1426
+
1427
+ const global char * q_base = (const global char *) q_void + q_offset;
1428
+ const global char * k_base = (const global char *) k_void + k_offset;
1429
+ const global char * v_base = (const global char *) v_void + v_offset;
1430
+ global char * o_base = (global char *) o_void + o_offset;
1431
+
1432
+ __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC];
1433
+ for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_WG_SIZE) {
1434
+ const int h = i / DK_VEC;
1435
+ const int k = i % DK_VEC;
1436
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1437
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2;
1438
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
1439
+ q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);
1440
+ }
1441
+ barrier(CLK_LOCAL_MEM_FENCE);
1442
+
1443
+ // per-h ALiBi slope
1444
+ float slope[MQ_GQA];
1445
+ #pragma unroll
1446
+ for (int h = 0; h < MQ_GQA; ++h) {
1447
+ slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);
1448
+ }
1449
+
1450
+ // per-h mask row pointer
1451
+ const global char * mask_base[MQ_GQA];
1452
+ if (mask_void != NULL) {
1453
+ const int mask_batch_idx = batch_idx % mask_ne3;
1454
+ const global char * mask_base_b = (const global char *) mask_void + mask_offset +
1455
+ mask_batch_idx * mask_nb3;
1456
+ #pragma unroll
1457
+ for (int h = 0; h < MQ_GQA; ++h) {
1458
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1459
+ const int mask_head_idx = head_idx % mask_ne2;
1460
+ mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;
1461
+ }
1462
+ } else {
1463
+ #pragma unroll
1464
+ for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL;
1465
+ }
1466
+
1467
+ const global ACC_TYPE * sinks_ptr = NULL;
1468
+ if (sinks_void != NULL) {
1469
+ sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset);
1470
+ }
1471
+
1472
+ // per-thread per-h DV slice.
1473
+ ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD];
1474
+ ACC_TYPE m_i[MQ_GQA];
1475
+ ACC_TYPE l_i[MQ_GQA];
1476
+ #pragma unroll
1477
+ for (int h = 0; h < MQ_GQA; ++h) {
1478
+ m_i[h] = FA_M_INIT;
1479
+ l_i[h] = 0.0f;
1480
+ #pragma unroll
1481
+ for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f);
1482
+ }
1483
+
1484
+ // each subgroup independently sweeps its slice of n_kv.
1485
+ const int kv_per_sg = (n_kv + MQ_NSG - 1) / MQ_NSG;
1486
+ const int kv_start = sgid * kv_per_sg;
1487
+ const int kv_end = min(n_kv, kv_start + kv_per_sg);
1488
+
1489
+ for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) {
1490
+ const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
1491
+ const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
1492
+ const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_off);
1493
+ const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off);
1494
+
1495
+ // Q*K^T: load each K stride once, dot against all MQ_GQA Q rows.
1496
+ ACC_TYPE4 dot4[MQ_GQA];
1497
+ #pragma unroll
1498
+ for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f);
1499
+ for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) {
1500
+ const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[k]);
1501
+ #pragma unroll
1502
+ for (int h = 0; h < MQ_GQA; ++h) {
1503
+ dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]);
1504
+ }
1505
+ }
1506
+
1507
+ ACC_TYPE score[MQ_GQA];
1508
+ #pragma unroll
1509
+ for (int h = 0; h < MQ_GQA; ++h) {
1510
+ const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
1511
+ ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale;
1512
+ if (mask_base[h] != NULL) {
1513
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
1514
+ s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
1515
+ }
1516
+ if (logit_softcap > 0.0f) {
1517
+ s = logit_softcap * tanh(s / logit_softcap);
1518
+ }
1519
+ score[h] = s;
1520
+ }
1521
+
1522
+ // FA-2 online softmax update — V load amortized across MQ_GQA heads.
1523
+ // p, scale_prev are computed per h; the V vector is loaded once
1524
+ // per dv stride and reused MQ_GQA times.
1525
+ ACC_TYPE p_h[MQ_GQA];
1526
+ ACC_TYPE sp_h[MQ_GQA];
1527
+ #pragma unroll
1528
+ for (int h = 0; h < MQ_GQA; ++h) {
1529
+ const ACC_TYPE m_new = max(m_i[h], score[h]);
1530
+ sp_h[h] = native_exp(m_i[h] - m_new);
1531
+ p_h[h] = native_exp(score[h] - m_new);
1532
+ l_i[h] = l_i[h] * sp_h[h] + p_h[h];
1533
+ m_i[h] = m_new;
1534
+ }
1535
+
1536
+ int idx = 0;
1537
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) {
1538
+ const ACC_TYPE4 v_vec = CONVERT_KV_ACC4(v_ptr[dv_idx]);
1539
+ #pragma unroll
1540
+ for (int h = 0; h < MQ_GQA; ++h) {
1541
+ o_acc[h][idx] = mad(p_h[h], v_vec, o_acc[h][idx] * sp_h[h]);
1542
+ }
1543
+ }
1544
+ }
1545
+
1546
+ // cross subgroup merge
1547
+ __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG];
1548
+ __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG];
1549
+ __local ACC_TYPE4 sg_o[MQ_NSG][DV_VEC];
1550
+
1551
+ if (tid_sg == 0) {
1552
+ #pragma unroll
1553
+ for (int h = 0; h < MQ_GQA; ++h) {
1554
+ sg_m[h][sgid] = m_i[h];
1555
+ sg_l[h][sgid] = l_i[h];
1556
+ }
1557
+ }
1558
+
1559
+ #pragma unroll
1560
+ for (int h = 0; h < MQ_GQA; ++h) {
1561
+ // each subgroup publishes its o_acc slice for head h.
1562
+ {
1563
+ int idx = 0;
1564
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) {
1565
+ sg_o[sgid][dv_idx] = o_acc[h][idx];
1566
+ }
1567
+ }
1568
+ barrier(CLK_LOCAL_MEM_FENCE);
1569
+
1570
+ if (sgid == 0) {
1571
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1572
+
1573
+ ACC_TYPE m_final = sg_m[h][0];
1574
+ #pragma unroll
1575
+ for (int s = 1; s < MQ_NSG; ++s) {
1576
+ m_final = max(m_final, sg_m[h][s]);
1577
+ }
1578
+ if (sinks_ptr != NULL) {
1579
+ m_final = max(m_final, sinks_ptr[head_idx]);
1580
+ }
1581
+
1582
+ ACC_TYPE l_final = 0.0f;
1583
+ #pragma unroll
1584
+ for (int s = 0; s < MQ_NSG; ++s) {
1585
+ l_final += sg_l[h][s] * native_exp(sg_m[h][s] - m_final);
1586
+ }
1587
+ if (sinks_ptr != NULL) {
1588
+ l_final += native_exp(sinks_ptr[head_idx] - m_final);
1589
+ }
1590
+ const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f;
1591
+
1592
+ const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1;
1593
+ global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset);
1594
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) {
1595
+ ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f);
1596
+ #pragma unroll
1597
+ for (int s = 0; s < MQ_NSG; ++s) {
1598
+ const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_final);
1599
+ o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged);
1600
+ }
1601
+ o_row[dv_idx] = CONVERT_O_DATA4(o_merged * l_inv);
1602
+ }
1603
+ }
1604
+ // Barrier guards next h's overwrite of sg_o.
1605
+ barrier(CLK_LOCAL_MEM_FENCE);
1606
+ }
1607
+ }
1608
+
1609
+ #ifndef MQ_NSG_SPLIT
1610
+ #define MQ_NSG_SPLIT 4
1611
+ #endif
1612
+ #define MQ_SPLIT_WG_SIZE (Q1_WG_SIZE * MQ_NSG_SPLIT)
1613
+
1614
+ #ifndef FA_PARTIAL_FLOATS
1615
+ #define FA_PARTIAL_FLOATS (2 + DV)
1616
+ #endif
1617
+
1618
+ REQD_SUBGROUP_SIZE_64
1619
+ __kernel void flash_attn_f32_f16_q1_vec_mq_split(
1620
+ const global void * q_void, ulong q_offset,
1621
+ const global void * k_void, ulong k_offset,
1622
+ const global void * v_void, ulong v_offset,
1623
+ const float scale,
1624
+ const int n_q,
1625
+ const int n_kv,
1626
+ const int n_head,
1627
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
1628
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
1629
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
1630
+ const float max_bias,
1631
+ const float m0,
1632
+ const float m1,
1633
+ const int n_head_log2,
1634
+ const float logit_softcap,
1635
+ const int n_head_kv,
1636
+ const global void * mask_void,
1637
+ const ulong mask_offset,
1638
+ const ulong mask_nb1,
1639
+ const ulong mask_nb2,
1640
+ const ulong mask_nb3,
1641
+ const int mask_ne2,
1642
+ const int mask_ne3,
1643
+ global float * partial_void,
1644
+ const int n_splits,
1645
+ const int kv_per_split
1646
+ ) {
1647
+ const int tid = get_local_id(0);
1648
+ const int sgid = tid / Q1_WG_SIZE;
1649
+ const int tid_sg = tid % Q1_WG_SIZE;
1650
+ const int kvhead_batch_idx = get_global_id(1);
1651
+ const int split_q_idx = get_global_id(2);
1652
+ const int split_idx = split_q_idx % n_splits;
1653
+ const int q_idx = split_q_idx / n_splits;
1654
+
1655
+ const int batch_idx = kvhead_batch_idx / n_head_kv;
1656
+ const int head_kv_idx = kvhead_batch_idx % n_head_kv;
1657
+
1658
+ const int kv_start = split_idx * kv_per_split;
1659
+ const int kv_end = min(kv_start + kv_per_split, n_kv);
1660
+
1661
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
1662
+
1663
+ if (kv_start >= kv_end) {
1664
+ // write sentinel for each of the MQ_GQA Q-heads so the
1665
+ // merge pass treats this slot as dropped
1666
+ if (tid == 0) {
1667
+ #pragma unroll
1668
+ for (int h = 0; h < MQ_GQA; ++h) {
1669
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1670
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
1671
+ * n_splits + split_idx);
1672
+ global float * rec = partial_void + rec_idx * record_stride;
1673
+ rec[0] = FA_M_INIT;
1674
+ rec[1] = 0.0f;
1675
+ }
1676
+ }
1677
+ return;
1678
+ }
1679
+
1680
+ const global char * q_base = (const global char *) q_void + q_offset;
1681
+ const global char * k_base = (const global char *) k_void + k_offset;
1682
+ const global char * v_base = (const global char *) v_void + v_offset;
1683
+
1684
+ // stage MQ_GQA Q rows in __local once (uniform across WG)
1685
+ __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC];
1686
+ for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) {
1687
+ const int h = i / DK_VEC;
1688
+ const int k = i % DK_VEC;
1689
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1690
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
1691
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
1692
+ q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);
1693
+ }
1694
+ barrier(CLK_LOCAL_MEM_FENCE);
1695
+
1696
+ float slope[MQ_GQA];
1697
+ #pragma unroll
1698
+ for (int h = 0; h < MQ_GQA; ++h) {
1699
+ slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);
1700
+ }
1701
+
1702
+ const global char * mask_base[MQ_GQA];
1703
+ if (mask_void != NULL) {
1704
+ const int mask_batch_idx = batch_idx % mask_ne3;
1705
+ const global char * mask_base_b = (const global char *) mask_void + mask_offset +
1706
+ mask_batch_idx * mask_nb3 +
1707
+ (ulong) q_idx * mask_nb1;
1708
+ #pragma unroll
1709
+ for (int h = 0; h < MQ_GQA; ++h) {
1710
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1711
+ const int mask_head_idx = head_idx % mask_ne2;
1712
+ mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;
1713
+ }
1714
+ } else {
1715
+ #pragma unroll
1716
+ for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL;
1717
+ }
1718
+
1719
+ ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD];
1720
+ ACC_TYPE m_i[MQ_GQA];
1721
+ ACC_TYPE l_i[MQ_GQA];
1722
+ #pragma unroll
1723
+ for (int h = 0; h < MQ_GQA; ++h) {
1724
+ m_i[h] = FA_M_INIT;
1725
+ l_i[h] = 0.0f;
1726
+ #pragma unroll
1727
+ for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f);
1728
+ }
1729
+
1730
+ // each subgroup independently sweeps its slice of the split's kv range.
1731
+ const int kv_len = kv_end - kv_start;
1732
+ const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT;
1733
+ const int kv_lo = kv_start + sgid * kv_per_sg;
1734
+ const int kv_hi = min(kv_end, kv_lo + kv_per_sg);
1735
+
1736
+ for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) {
1737
+ const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
1738
+ const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
1739
+ const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_off);
1740
+ const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off);
1741
+
1742
+ ACC_TYPE4 dot4[MQ_GQA];
1743
+ #pragma unroll
1744
+ for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f);
1745
+ for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) {
1746
+ const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[k]);
1747
+ #pragma unroll
1748
+ for (int h = 0; h < MQ_GQA; ++h) {
1749
+ dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]);
1750
+ }
1751
+ }
1752
+
1753
+ ACC_TYPE score[MQ_GQA];
1754
+ #pragma unroll
1755
+ for (int h = 0; h < MQ_GQA; ++h) {
1756
+ const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
1757
+ ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale;
1758
+ if (mask_base[h] != NULL) {
1759
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
1760
+ s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
1761
+ }
1762
+ if (logit_softcap > 0.0f) {
1763
+ s = logit_softcap * tanh(s / logit_softcap);
1764
+ }
1765
+ score[h] = s;
1766
+ }
1767
+
1768
+ ACC_TYPE p_h[MQ_GQA];
1769
+ ACC_TYPE sp_h[MQ_GQA];
1770
+ #pragma unroll
1771
+ for (int h = 0; h < MQ_GQA; ++h) {
1772
+ const ACC_TYPE m_new = max(m_i[h], score[h]);
1773
+ sp_h[h] = native_exp(m_i[h] - m_new);
1774
+ p_h[h] = native_exp(score[h] - m_new);
1775
+ l_i[h] = l_i[h] * sp_h[h] + p_h[h];
1776
+ m_i[h] = m_new;
1777
+ }
1778
+
1779
+ int idx = 0;
1780
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) {
1781
+ const ACC_TYPE4 v_vec = CONVERT_KV_ACC4(v_ptr[dv_idx]);
1782
+ #pragma unroll
1783
+ for (int h = 0; h < MQ_GQA; ++h) {
1784
+ o_acc[h][idx] = mad(p_h[h], v_vec, o_acc[h][idx] * sp_h[h]);
1785
+ }
1786
+ }
1787
+ }
1788
+
1789
+ // per-h cross-subgroup merge
1790
+ __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT];
1791
+ __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT];
1792
+ __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC];
1793
+
1794
+ if (tid_sg == 0) {
1795
+ #pragma unroll
1796
+ for (int h = 0; h < MQ_GQA; ++h) {
1797
+ sg_m[h][sgid] = m_i[h];
1798
+ sg_l[h][sgid] = l_i[h];
1799
+ }
1800
+ }
1801
+
1802
+ #pragma unroll
1803
+ for (int h = 0; h < MQ_GQA; ++h) {
1804
+ {
1805
+ int idx = 0;
1806
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) {
1807
+ sg_o[sgid][dv_idx] = o_acc[h][idx];
1808
+ }
1809
+ }
1810
+ barrier(CLK_LOCAL_MEM_FENCE);
1811
+
1812
+ if (sgid == 0) {
1813
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1814
+
1815
+ // fold per-subgroup (m, l) into split-level (m_c, l_c)
1816
+ ACC_TYPE m_c = sg_m[h][0];
1817
+ #pragma unroll
1818
+ for (int s = 1; s < MQ_NSG_SPLIT; ++s) {
1819
+ m_c = max(m_c, sg_m[h][s]);
1820
+ }
1821
+ ACC_TYPE l_c = 0.0f;
1822
+ #pragma unroll
1823
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
1824
+ l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c);
1825
+ }
1826
+
1827
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
1828
+ * n_splits + split_idx);
1829
+ global float * rec = partial_void + rec_idx * record_stride;
1830
+ global float4 * rec_o = (global float4 *) (rec + 2);
1831
+
1832
+ if (tid_sg == 0) {
1833
+ rec[0] = (float) m_c;
1834
+ rec[1] = (float) l_c;
1835
+ }
1836
+ // each thread writes its DV slice of the merged O.
1837
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) {
1838
+ ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f);
1839
+ #pragma unroll
1840
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
1841
+ const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c);
1842
+ o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged);
1843
+ }
1844
+ rec_o[dv_idx] = o_merged;
1845
+ }
1846
+ }
1847
+ barrier(CLK_LOCAL_MEM_FENCE);
1848
+ }
1849
+ }
1850
+
1851
+ // Cluster-parallel variant of _q1_vec_mq_split
1852
+ //
1853
+ // Tthe baseline keeps one 256B K row in flight per subgroup (32 lanes cooperate
1854
+ // on one position, serialized by the reduce+exp chain). This kernel
1855
+ // takes q1_split's memory-level parallelism at MQ's read-once traffic:
1856
+ // - the 64-lane subgroup is split into FA_CL_NCL clusters of FA_CL_C lanes;
1857
+ // - each cluster owns its own KV position stream (positions strided by
1858
+ // FA_CL_NCL) with private per-cluster online-softmax state, hence FA_CL_NCL
1859
+ // independent K rows in flight per subgroup, no cross-cluster serial chain;
1860
+ // - within a cluster, lanes split DK for the dot (cluster-reduce via
1861
+ // sub_group_shuffle_xor, steps < FA_CL_C stay inside the cluster) and
1862
+ // split DV for o_acc (each lane owns dv indices {lic + FA_CL_C*i} — the
1863
+ // same slice for every position, so accumulation is lane-local);
1864
+ // - merge stage 1 folds the FA_CL_NCL cluster partials with cross-cluster
1865
+ // shuffles (distances >= FA_CL_C); stage 2 is the baseline cross-subgroup
1866
+ // LDS merge (o published by cluster 0's lanes, layout-identical to the
1867
+ // baseline's sg_o).
1868
+ // The KV sweep runs a UNIFORM trip count (max over clusters) with a clamped
1869
+ // row address + FA_M_INIT score on the tail — keeps every shuffle convergent
1870
+ // (p = exp(FA_M_INIT - m) underflows to 0, so clamped-row reads are inert).
1871
+ // Register cost vs baseline: o_acc grows from DV_VEC/64 to DV_VEC/FA_CL_C
1872
+ // float4 per lane per head — FA_CL_C=8 / MQ_GQA=4 => 16 float4 (256B).
1873
+
1874
+ #ifdef HAS_SUBGROUP_SHUFFLE // cluster reduce/merge needs shuffles; absent -> kernel dropped, dispatch falls back
1875
+
1876
+ #ifndef FA_CL_C
1877
+ #define FA_CL_C 8
1878
+ #endif
1879
+
1880
+ // The lane striping requires DK/DV to divide evenly across the cluster;
1881
+ // otherwise (e.g. DK=40 with FA_CL_C=16 -> zero-size arrays) compile the
1882
+ // kernel out — host soft-create falls back silently.
1883
+ #if (DK_VEC % FA_CL_C) == 0 && (DV_VEC % FA_CL_C) == 0
1884
+ #define FA_CL_NCL (Q1_WG_SIZE / FA_CL_C) // clusters (position streams) per subgroup
1885
+ #define FA_CL_DK (DK_VEC / FA_CL_C) // half4s of K per lane per row
1886
+ #define FA_CL_DV (DV_VEC / FA_CL_C) // float4s of o_acc per lane per head
1887
+
1888
+ // explicit "half" sub-group attribute routes this fp16-heavy kernel to a slow
1889
+ // codegen path on the X1 compiler. X2 keeps the pin: its driver miscompile
1890
+ // without it.
1891
+ #ifdef FA_C8_NO_SG_PIN
1892
+ #define FA_C8_SG_ATTR
1893
+ #else
1894
+ // REQD_FA_SG pins the HW subgroup on Intel (intel_reqd_sub_group_size(FA_SG),
1895
+ // host passes -D FA_SG=32); empty on Adreno. REQD_SUBGROUP_SIZE_64 pins 64 on
1896
+ // Adreno; empty on Intel.
1897
+ #define FA_C8_SG_ATTR REQD_FA_SG REQD_SUBGROUP_SIZE_64
1898
+ #endif
1899
+
1900
+ FA_C8_SG_ATTR
1901
+ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
1902
+ const global void * q_void, ulong q_offset,
1903
+ const global void * k_void, ulong k_offset,
1904
+ const global void * v_void, ulong v_offset,
1905
+ const float scale,
1906
+ const int n_q,
1907
+ const int n_kv,
1908
+ const int n_head,
1909
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
1910
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
1911
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
1912
+ const float max_bias,
1913
+ const float m0,
1914
+ const float m1,
1915
+ const int n_head_log2,
1916
+ const float logit_softcap,
1917
+ const int n_head_kv,
1918
+ const global void * mask_void,
1919
+ const ulong mask_offset,
1920
+ const ulong mask_nb1,
1921
+ const ulong mask_nb2,
1922
+ const ulong mask_nb3,
1923
+ const int mask_ne2,
1924
+ const int mask_ne3,
1925
+ global float * partial_void,
1926
+ const int n_splits,
1927
+ const int kv_per_split
1928
+ ) {
1929
+ const int tid = get_local_id(0);
1930
+ const int sgid = tid / Q1_WG_SIZE;
1931
+ const int tid_sg = tid % Q1_WG_SIZE;
1932
+ const int cl = tid_sg / FA_CL_C; // cluster id
1933
+ const int lic = tid_sg % FA_CL_C; // lane in cluster
1934
+ const int kvhead_batch_idx = get_global_id(1);
1935
+ const int split_q_idx = get_global_id(2);
1936
+ const int split_idx = split_q_idx % n_splits;
1937
+ const int q_idx = split_q_idx / n_splits;
1938
+
1939
+ const int batch_idx = kvhead_batch_idx / n_head_kv;
1940
+ const int head_kv_idx = kvhead_batch_idx % n_head_kv;
1941
+
1942
+ const int kv_start = split_idx * kv_per_split;
1943
+ const int kv_end = min(kv_start + kv_per_split, n_kv);
1944
+
1945
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
1946
+
1947
+ if (kv_start >= kv_end) {
1948
+ if (tid == 0) {
1949
+ #pragma unroll
1950
+ for (int h = 0; h < MQ_GQA; ++h) {
1951
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1952
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
1953
+ * n_splits + split_idx);
1954
+ global float * rec = partial_void + rec_idx * record_stride;
1955
+ rec[0] = FA_M_INIT;
1956
+ rec[1] = 0.0f;
1957
+ }
1958
+ }
1959
+ return;
1960
+ }
1961
+
1962
+ const global char * q_base = (const global char *) q_void + q_offset;
1963
+ const global char * k_base = (const global char *) k_void + k_offset;
1964
+ const global char * v_base = (const global char *) v_void + v_offset;
1965
+
1966
+ // Stage MQ_GQA Q rows in __local once (uniform across WG).
1967
+ __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC];
1968
+ for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) {
1969
+ const int h = i / DK_VEC;
1970
+ const int k = i % DK_VEC;
1971
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1972
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
1973
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
1974
+ q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);
1975
+ }
1976
+ barrier(CLK_LOCAL_MEM_FENCE);
1977
+
1978
+ float slope[MQ_GQA];
1979
+ #pragma unroll
1980
+ for (int h = 0; h < MQ_GQA; ++h) {
1981
+ slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);
1982
+ }
1983
+
1984
+ const global char * mask_base[MQ_GQA];
1985
+ if (mask_void != NULL) {
1986
+ const int mask_batch_idx = batch_idx % mask_ne3;
1987
+ const global char * mask_base_b = (const global char *) mask_void + mask_offset +
1988
+ mask_batch_idx * mask_nb3 +
1989
+ (ulong) q_idx * mask_nb1;
1990
+ #pragma unroll
1991
+ for (int h = 0; h < MQ_GQA; ++h) {
1992
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1993
+ const int mask_head_idx = head_idx % mask_ne2;
1994
+ mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;
1995
+ }
1996
+ } else {
1997
+ #pragma unroll
1998
+ for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL;
1999
+ }
2000
+
2001
+ // Per-CLUSTER online-softmax state (uniform across the cluster's lanes);
2002
+ // o_acc holds this lane's DV slice {lic + FA_CL_C*i}.
2003
+ ACC_TYPE4 o_acc[MQ_GQA][FA_CL_DV];
2004
+ ACC_TYPE m_i[MQ_GQA];
2005
+ ACC_TYPE l_i[MQ_GQA];
2006
+ #pragma unroll
2007
+ for (int h = 0; h < MQ_GQA; ++h) {
2008
+ m_i[h] = FA_M_INIT;
2009
+ l_i[h] = 0.0f;
2010
+ #pragma unroll
2011
+ for (int i = 0; i < FA_CL_DV; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f);
2012
+ }
2013
+
2014
+ const int kv_len = kv_end - kv_start;
2015
+ const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT;
2016
+ const int kv_lo = kv_start + sgid * kv_per_sg;
2017
+ const int kv_hi = min(kv_end, kv_lo + kv_per_sg);
2018
+
2019
+ // Uniform trip count across the subgroup: every cluster runs n_iter
2020
+ // iterations; tail positions clamp the row address and drop the score to
2021
+ // FA_M_INIT so shuffles stay convergent and the contribution is exactly 0.
2022
+ const int n_iter = (kv_hi - kv_lo + FA_CL_NCL - 1) / FA_CL_NCL;
2023
+ const ulong kv_row_base = batch_idx * k_nb3 + head_kv_idx * k_nb2;
2024
+ const ulong v_row_base = batch_idx * v_nb3 + head_kv_idx * v_nb2;
2025
+
2026
+ for (int it = 0; it < n_iter; ++it) {
2027
+ const int k_idx = kv_lo + cl + it * FA_CL_NCL;
2028
+ const int valid = k_idx < kv_hi;
2029
+ const int k_safe = valid ? k_idx : (kv_hi - 1);
2030
+
2031
+ const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + kv_row_base + (ulong) k_safe * k_nb1);
2032
+ const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_base + (ulong) k_safe * v_nb1);
2033
+
2034
+ // Dot: this lane covers DK elements {lic + FA_CL_C*i} of the cluster's row.
2035
+ ACC_TYPE4 dot4[MQ_GQA];
2036
+ #pragma unroll
2037
+ for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f);
2038
+ #pragma unroll
2039
+ for (int i = 0; i < FA_CL_DK; ++i) {
2040
+ const int kk = lic + FA_CL_C * i;
2041
+ const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[kk]);
2042
+ #pragma unroll
2043
+ for (int h = 0; h < MQ_GQA; ++h) {
2044
+ dot4[h] = mad(q_shared[h * DK_VEC + kk], k_vec, dot4[h]);
2045
+ }
2046
+ }
2047
+
2048
+ // Cluster-reduce (xor steps < FA_CL_C stay inside the cluster) + score.
2049
+ ACC_TYPE score[MQ_GQA];
2050
+ #pragma unroll
2051
+ for (int h = 0; h < MQ_GQA; ++h) {
2052
+ ACC_TYPE s = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
2053
+ #pragma unroll
2054
+ for (int step = 1; step < FA_CL_C; step <<= 1) {
2055
+ s += sub_group_shuffle_xor(s, step);
2056
+ }
2057
+ s *= scale;
2058
+ if (mask_base[h] != NULL) {
2059
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
2060
+ s += slope[h] * (ACC_TYPE) mask_ptr[k_safe];
2061
+ }
2062
+ if (logit_softcap > 0.0f) {
2063
+ s = logit_softcap * tanh(s / logit_softcap);
2064
+ }
2065
+ score[h] = valid ? s : FA_M_INIT;
2066
+ }
2067
+
2068
+ // Per-cluster online update — identical math to the baseline, but the
2069
+ // serial chain is per cluster (depth n_iter, not kv_per_sg).
2070
+ ACC_TYPE p_h[MQ_GQA];
2071
+ ACC_TYPE sp_h[MQ_GQA];
2072
+ #pragma unroll
2073
+ for (int h = 0; h < MQ_GQA; ++h) {
2074
+ const ACC_TYPE m_new = max(m_i[h], score[h]);
2075
+ sp_h[h] = native_exp(m_i[h] - m_new);
2076
+ p_h[h] = native_exp(score[h] - m_new);
2077
+ l_i[h] = l_i[h] * sp_h[h] + p_h[h];
2078
+ m_i[h] = m_new;
2079
+ }
2080
+
2081
+ // V accumulate on this lane's DV slice (p = 0 on tail -> inert).
2082
+ #pragma unroll
2083
+ for (int i = 0; i < FA_CL_DV; ++i) {
2084
+ const ACC_TYPE4 v_vec = CONVERT_KV_ACC4(v_ptr[lic + FA_CL_C * i]);
2085
+ #pragma unroll
2086
+ for (int h = 0; h < MQ_GQA; ++h) {
2087
+ o_acc[h][i] = mad(p_h[h], v_vec, o_acc[h][i] * sp_h[h]);
2088
+ }
2089
+ }
2090
+ }
2091
+
2092
+ // Merge stage 1: fold the FA_CL_NCL cluster partials inside the subgroup.
2093
+ // Lanes with equal lic across clusters hold the SAME dv slice, so a
2094
+ // cross-cluster xor-reduce (distances FA_CL_C..Q1_WG_SIZE/2) sums o
2095
+ // slice-wise; m/l fold the same way. All shuffles are subgroup-convergent.
2096
+ #pragma unroll
2097
+ for (int h = 0; h < MQ_GQA; ++h) {
2098
+ ACC_TYPE m_c = m_i[h];
2099
+ #pragma unroll
2100
+ for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) {
2101
+ m_c = max(m_c, sub_group_shuffle_xor(m_c, step));
2102
+ }
2103
+ const ACC_TYPE alpha = native_exp(m_i[h] - m_c);
2104
+ ACC_TYPE l_c = l_i[h] * alpha;
2105
+ #pragma unroll
2106
+ for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) {
2107
+ l_c += sub_group_shuffle_xor(l_c, step);
2108
+ }
2109
+ #pragma unroll
2110
+ for (int i = 0; i < FA_CL_DV; ++i) {
2111
+ ACC_TYPE4 o = o_acc[h][i] * alpha;
2112
+ #pragma unroll
2113
+ for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) {
2114
+ o.s0 += sub_group_shuffle_xor(o.s0, step);
2115
+ o.s1 += sub_group_shuffle_xor(o.s1, step);
2116
+ o.s2 += sub_group_shuffle_xor(o.s2, step);
2117
+ o.s3 += sub_group_shuffle_xor(o.s3, step);
2118
+ }
2119
+ o_acc[h][i] = o;
2120
+ }
2121
+ m_i[h] = m_c;
2122
+ l_i[h] = l_c;
2123
+ }
2124
+
2125
+ // Merge stage 2: baseline cross-subgroup LDS merge. Cluster 0's lanes hold
2126
+ // the subgroup's merged o (dv indices {lic + FA_CL_C*i}) — same sg_o layout
2127
+ // and fold loop as q1_vec_mq_split.
2128
+ __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT];
2129
+ __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT];
2130
+ __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC];
2131
+
2132
+ if (tid_sg == 0) {
2133
+ #pragma unroll
2134
+ for (int h = 0; h < MQ_GQA; ++h) {
2135
+ sg_m[h][sgid] = m_i[h];
2136
+ sg_l[h][sgid] = l_i[h];
2137
+ }
2138
+ }
2139
+
2140
+ #pragma unroll
2141
+ for (int h = 0; h < MQ_GQA; ++h) {
2142
+ if (cl == 0) {
2143
+ #pragma unroll
2144
+ for (int i = 0; i < FA_CL_DV; ++i) {
2145
+ sg_o[sgid][lic + FA_CL_C * i] = o_acc[h][i];
2146
+ }
2147
+ }
2148
+ barrier(CLK_LOCAL_MEM_FENCE);
2149
+
2150
+ if (sgid == 0) {
2151
+ const int head_idx = head_kv_idx * MQ_GQA + h;
2152
+
2153
+ ACC_TYPE m_c = sg_m[h][0];
2154
+ #pragma unroll
2155
+ for (int s = 1; s < MQ_NSG_SPLIT; ++s) {
2156
+ m_c = max(m_c, sg_m[h][s]);
2157
+ }
2158
+ ACC_TYPE l_c = 0.0f;
2159
+ #pragma unroll
2160
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
2161
+ l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c);
2162
+ }
2163
+
2164
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
2165
+ * n_splits + split_idx);
2166
+ global float * rec = partial_void + rec_idx * record_stride;
2167
+ global float4 * rec_o = (global float4 *) (rec + 2);
2168
+
2169
+ if (tid_sg == 0) {
2170
+ rec[0] = (float) m_c;
2171
+ rec[1] = (float) l_c;
2172
+ }
2173
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) {
2174
+ ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f);
2175
+ #pragma unroll
2176
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
2177
+ const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c);
2178
+ o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged);
2179
+ }
2180
+ rec_o[dv_idx] = o_merged;
2181
+ }
2182
+ }
2183
+ barrier(CLK_LOCAL_MEM_FENCE);
2184
+ }
2185
+ }
2186
+
2187
+ #endif // DK_VEC/DV_VEC divisible by FA_CL_C
2188
+ #endif // HAS_SUBGROUP_SHUFFLE (q1_vec_mq_split_c8)
2189
+
2190
+ REQD_SUBGROUP_SIZE_64
2191
+ __kernel void flash_attn_f32_f16_q1_vec_mq_split_k_img(
2192
+ const global void * q_void, ulong q_offset,
2193
+ __read_only image1d_buffer_t k_img,
2194
+ const global void * v_void, ulong v_offset,
2195
+ const float scale,
2196
+ const int n_q,
2197
+ const int n_kv,
2198
+ const int n_head,
2199
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
2200
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
2201
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
2202
+ const float max_bias,
2203
+ const float m0,
2204
+ const float m1,
2205
+ const int n_head_log2,
2206
+ const float logit_softcap,
2207
+ const int n_head_kv,
2208
+ const global void * mask_void,
2209
+ const ulong mask_offset,
2210
+ const ulong mask_nb1,
2211
+ const ulong mask_nb2,
2212
+ const ulong mask_nb3,
2213
+ const int mask_ne2,
2214
+ const int mask_ne3,
2215
+ global float * partial_void,
2216
+ const int n_splits,
2217
+ const int kv_per_split
2218
+ ) {
2219
+ const int tid = get_local_id(0);
2220
+ const int sgid = tid / Q1_WG_SIZE;
2221
+ const int tid_sg = tid % Q1_WG_SIZE;
2222
+ const int kvhead_batch_idx = get_global_id(1);
2223
+ const int split_q_idx = get_global_id(2);
2224
+ const int split_idx = split_q_idx % n_splits;
2225
+ const int q_idx = split_q_idx / n_splits;
2226
+
2227
+ const int batch_idx = kvhead_batch_idx / n_head_kv;
2228
+ const int head_kv_idx = kvhead_batch_idx % n_head_kv;
2229
+
2230
+ const int kv_start = split_idx * kv_per_split;
2231
+ const int kv_end = min(kv_start + kv_per_split, n_kv);
2232
+
2233
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
2234
+
2235
+ if (kv_start >= kv_end) {
2236
+ if (tid == 0) {
2237
+ #pragma unroll
2238
+ for (int h = 0; h < MQ_GQA; ++h) {
2239
+ const int head_idx = head_kv_idx * MQ_GQA + h;
2240
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
2241
+ * n_splits + split_idx);
2242
+ global float * rec = partial_void + rec_idx * record_stride;
2243
+ rec[0] = FA_M_INIT;
2244
+ rec[1] = 0.0f;
2245
+ }
2246
+ }
2247
+ return;
2248
+ }
2249
+
2250
+ const global char * q_base = (const global char *) q_void + q_offset;
2251
+ const global char * v_base = (const global char *) v_void + v_offset;
2252
+
2253
+ __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC];
2254
+ for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) {
2255
+ const int h = i / DK_VEC;
2256
+ const int k = i % DK_VEC;
2257
+ const int head_idx = head_kv_idx * MQ_GQA + h;
2258
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
2259
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
2260
+ q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);
2261
+ }
2262
+ barrier(CLK_LOCAL_MEM_FENCE);
2263
+
2264
+ float slope[MQ_GQA];
2265
+ #pragma unroll
2266
+ for (int h = 0; h < MQ_GQA; ++h) {
2267
+ slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);
2268
+ }
2269
+
2270
+ const global char * mask_base[MQ_GQA];
2271
+ if (mask_void != NULL) {
2272
+ const int mask_batch_idx = batch_idx % mask_ne3;
2273
+ const global char * mask_base_b = (const global char *) mask_void + mask_offset +
2274
+ mask_batch_idx * mask_nb3 +
2275
+ (ulong) q_idx * mask_nb1;
2276
+ #pragma unroll
2277
+ for (int h = 0; h < MQ_GQA; ++h) {
2278
+ const int head_idx = head_kv_idx * MQ_GQA + h;
2279
+ const int mask_head_idx = head_idx % mask_ne2;
2280
+ mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;
2281
+ }
2282
+ } else {
2283
+ #pragma unroll
2284
+ for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL;
2285
+ }
2286
+
2287
+ ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD];
2288
+ ACC_TYPE m_i[MQ_GQA];
2289
+ ACC_TYPE l_i[MQ_GQA];
2290
+ #pragma unroll
2291
+ for (int h = 0; h < MQ_GQA; ++h) {
2292
+ m_i[h] = FA_M_INIT;
2293
+ l_i[h] = 0.0f;
2294
+ #pragma unroll
2295
+ for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f);
2296
+ }
2297
+
2298
+ // K pitches in pixel units, pixel = 1 half4 = 8 B -> byte_stride >> 3.
2299
+ const int pitch_px_row = (int)(k_nb1 >> 3);
2300
+ const int pitch_px_head = (int)(k_nb2 >> 3);
2301
+ const int pitch_px_batch = (int)(k_nb3 >> 3);
2302
+
2303
+ const int kv_len = kv_end - kv_start;
2304
+ const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT;
2305
+ const int kv_lo = kv_start + sgid * kv_per_sg;
2306
+ const int kv_hi = min(kv_end, kv_lo + kv_per_sg);
2307
+
2308
+ for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) {
2309
+ const int k_row_px = batch_idx * pitch_px_batch +
2310
+ head_kv_idx * pitch_px_head +
2311
+ k_idx * pitch_px_row;
2312
+
2313
+ const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
2314
+ const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_off);
2315
+
2316
+ ACC_TYPE4 dot4[MQ_GQA];
2317
+ #pragma unroll
2318
+ for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f);
2319
+ for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) {
2320
+ const half4 k_h4 = read_imageh(k_img, k_row_px + k);
2321
+ const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_h4);
2322
+ #pragma unroll
2323
+ for (int h = 0; h < MQ_GQA; ++h) {
2324
+ dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]);
2325
+ }
2326
+ }
2327
+
2328
+ ACC_TYPE score[MQ_GQA];
2329
+ #pragma unroll
2330
+ for (int h = 0; h < MQ_GQA; ++h) {
2331
+ const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
2332
+ ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale;
2333
+ if (mask_base[h] != NULL) {
2334
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
2335
+ s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
2336
+ }
2337
+ if (logit_softcap > 0.0f) {
2338
+ s = logit_softcap * tanh(s / logit_softcap);
2339
+ }
2340
+ score[h] = s;
2341
+ }
2342
+
2343
+ ACC_TYPE p_h[MQ_GQA];
2344
+ ACC_TYPE sp_h[MQ_GQA];
2345
+ #pragma unroll
2346
+ for (int h = 0; h < MQ_GQA; ++h) {
2347
+ const ACC_TYPE m_new = max(m_i[h], score[h]);
2348
+ sp_h[h] = native_exp(m_i[h] - m_new);
2349
+ p_h[h] = native_exp(score[h] - m_new);
2350
+ l_i[h] = l_i[h] * sp_h[h] + p_h[h];
2351
+ m_i[h] = m_new;
2352
+ }
2353
+
2354
+ int idx = 0;
2355
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) {
2356
+ const ACC_TYPE4 v_vec = CONVERT_KV_ACC4(v_ptr[dv_idx]);
2357
+ #pragma unroll
2358
+ for (int h = 0; h < MQ_GQA; ++h) {
2359
+ o_acc[h][idx] = mad(p_h[h], v_vec, o_acc[h][idx] * sp_h[h]);
2360
+ }
2361
+ }
2362
+ }
2363
+
2364
+ __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT];
2365
+ __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT];
2366
+ __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC];
2367
+
2368
+ if (tid_sg == 0) {
2369
+ #pragma unroll
2370
+ for (int h = 0; h < MQ_GQA; ++h) {
2371
+ sg_m[h][sgid] = m_i[h];
2372
+ sg_l[h][sgid] = l_i[h];
2373
+ }
2374
+ }
2375
+
2376
+ #pragma unroll
2377
+ for (int h = 0; h < MQ_GQA; ++h) {
2378
+ {
2379
+ int idx = 0;
2380
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) {
2381
+ sg_o[sgid][dv_idx] = o_acc[h][idx];
2382
+ }
2383
+ }
2384
+ barrier(CLK_LOCAL_MEM_FENCE);
2385
+
2386
+ if (sgid == 0) {
2387
+ const int head_idx = head_kv_idx * MQ_GQA + h;
2388
+
2389
+ ACC_TYPE m_c = sg_m[h][0];
2390
+ #pragma unroll
2391
+ for (int s = 1; s < MQ_NSG_SPLIT; ++s) {
2392
+ m_c = max(m_c, sg_m[h][s]);
2393
+ }
2394
+ ACC_TYPE l_c = 0.0f;
2395
+ #pragma unroll
2396
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
2397
+ l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c);
2398
+ }
2399
+
2400
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
2401
+ * n_splits + split_idx);
2402
+ global float * rec = partial_void + rec_idx * record_stride;
2403
+ global float4 * rec_o = (global float4 *) (rec + 2);
2404
+
2405
+ if (tid_sg == 0) {
2406
+ rec[0] = (float) m_c;
2407
+ rec[1] = (float) l_c;
2408
+ }
2409
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) {
2410
+ ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f);
2411
+ #pragma unroll
2412
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
2413
+ const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c);
2414
+ o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged);
2415
+ }
2416
+ rec_o[dv_idx] = o_merged;
2417
+ }
2418
+ }
2419
+ barrier(CLK_LOCAL_MEM_FENCE);
2420
+ }
2421
+ }
2422
+ #endif // !FA_DECODE_ONLY
2423
+
2424
+ #ifndef FA_MQ_ONLY // q1_split + merge excluded from the MQ-only (g8) program
2425
+ __kernel void flash_attn_f32_f16_q1_split(
2426
+ const global void * q_void, ulong q_offset,
2427
+ const global void * k_void, ulong k_offset,
2428
+ const global void * v_void, ulong v_offset,
2429
+ const float scale,
2430
+ const int n_q,
2431
+ const int n_kv,
2432
+ const int n_head,
2433
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
2434
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
2435
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
2436
+ const float max_bias,
2437
+ const float m0,
2438
+ const float m1,
2439
+ const int n_head_log2,
2440
+ const float logit_softcap,
2441
+ const int n_head_kv,
2442
+ const global void * mask_void,
2443
+ const ulong mask_offset,
2444
+ const ulong mask_nb1,
2445
+ const ulong mask_nb2,
2446
+ const ulong mask_nb3,
2447
+ const int mask_ne2,
2448
+ const int mask_ne3,
2449
+ global float * partial_void,
2450
+ const int n_splits,
2451
+ const int kv_per_split
2452
+ ) {
2453
+ const int tid = get_local_id(0);
2454
+ const int head_batch_idx = get_global_id(1);
2455
+ const int split_q_idx = get_global_id(2);
2456
+ const int split_idx = split_q_idx % n_splits;
2457
+ const int q_idx = split_q_idx / n_splits;
2458
+ const int batch_idx = head_batch_idx / n_head;
2459
+ const int head_idx = head_batch_idx % n_head;
2460
+ const int gqa_ratio = n_head / n_head_kv;
2461
+ const int head_kv_idx = head_idx / gqa_ratio;
2462
+
2463
+ const int kv_start = split_idx * kv_per_split;
2464
+ const int kv_end = min(kv_start + kv_per_split, n_kv);
2465
+
2466
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
2467
+ const ulong record_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
2468
+ * n_splits + split_idx);
2469
+ global float * rec = partial_void + record_idx * record_stride;
2470
+ global float4 * rec_o = (global float4 *) (rec + 2);
2471
+
2472
+ if (kv_start >= kv_end) {
2473
+ // Empty split: leave sentinel partial for merge.
2474
+ if (tid == 0) {
2475
+ rec[0] = FA_M_INIT;
2476
+ rec[1] = 0.0f;
2477
+ }
2478
+ return;
2479
+ }
2480
+
2481
+ const global char * q_base = (const global char *) q_void + q_offset;
2482
+ const global char * k_base = (const global char *) k_void + k_offset;
2483
+ const global char * v_base = (const global char *) v_void + v_offset;
2484
+
2485
+ const global char * mask_base = NULL;
2486
+ if (mask_void != NULL) {
2487
+ const int mask_head_idx = head_idx % mask_ne2;
2488
+ const int mask_batch_idx = batch_idx % mask_ne3;
2489
+ mask_base = (const global char *) mask_void + mask_offset +
2490
+ mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 +
2491
+ (ulong) q_idx * mask_nb1;
2492
+ }
2493
+
2494
+ // share Q via local memory (n_q=1 per split -> uniform across WG).
2495
+ __local ACC_TYPE4 q_shared[DK_VEC];
2496
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
2497
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
2498
+ for (int i = tid; i < DK_VEC; i += Q1_WG_SIZE) {
2499
+ q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]);
2500
+ }
2501
+ sub_group_barrier(CLK_LOCAL_MEM_FENCE);
2502
+
2503
+ const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
2504
+
2505
+ // pass 1a — split-local max.
2506
+ ACC_TYPE m_i = FA_M_INIT;
2507
+ for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) {
2508
+ const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
2509
+ const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_offset);
2510
+ ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f);
2511
+ #pragma unroll
2512
+ for (int k = 0; k < DK_VEC; ++k) {
2513
+ dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc);
2514
+ }
2515
+ ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
2516
+ if (mask_base != NULL) {
2517
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base);
2518
+ score += slope * (ACC_TYPE) mask_ptr[k_idx];
2519
+ }
2520
+ if (logit_softcap > 0.0f) {
2521
+ score = logit_softcap * tanh(score / logit_softcap);
2522
+ }
2523
+ m_i = max(m_i, score);
2524
+ }
2525
+
2526
+ const ACC_TYPE m_c = sub_group_reduce_max(m_i);
2527
+
2528
+ // pass 1b — softmax-weighted V accumulate.
2529
+ ACC_TYPE4 o_acc[DV_VEC];
2530
+ #pragma unroll
2531
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);
2532
+ ACC_TYPE l_i = 0.0f;
2533
+
2534
+ for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) {
2535
+ const ulong k_row_offset = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
2536
+ const ulong v_row_offset = batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
2537
+ const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + k_row_offset);
2538
+ const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_offset);
2539
+ ACC_TYPE4 dot_acc = (ACC_TYPE4)(0.0f);
2540
+ #pragma unroll
2541
+ for (int k = 0; k < DK_VEC; ++k) {
2542
+ dot_acc = mad(q_shared[k], CONVERT_KV_ACC4(k_ptr[k]), dot_acc);
2543
+ }
2544
+ ACC_TYPE score = (dot_acc.s0 + dot_acc.s1 + dot_acc.s2 + dot_acc.s3) * scale;
2545
+ if (mask_base != NULL) {
2546
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base);
2547
+ score += slope * (ACC_TYPE) mask_ptr[k_idx];
2548
+ }
2549
+ if (logit_softcap > 0.0f) {
2550
+ score = logit_softcap * tanh(score / logit_softcap);
2551
+ }
2552
+ const ACC_TYPE p = exp(score - m_c);
2553
+ l_i += p;
2554
+ #pragma unroll
2555
+ for (int i = 0; i < DV_VEC; ++i) {
2556
+ o_acc[i] = mad(p, CONVERT_KV_ACC4(v_ptr[i]), o_acc[i]);
2557
+ }
2558
+ }
2559
+
2560
+ __local ACC_TYPE4 local_o[Q1_WG_SIZE];
2561
+ const ACC_TYPE l_c = sub_group_reduce_add(l_i);
2562
+
2563
+ if (tid == 0) {
2564
+ rec[0] = (float) m_c;
2565
+ rec[1] = (float) l_c;
2566
+ }
2567
+ for (int i = 0; i < DV_VEC; ++i) {
2568
+ local_o[tid] = o_acc[i];
2569
+ sub_group_barrier(CLK_LOCAL_MEM_FENCE);
2570
+ #pragma unroll
2571
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
2572
+ if (tid < s) local_o[tid] += local_o[tid + s];
2573
+ sub_group_barrier(CLK_LOCAL_MEM_FENCE);
2574
+ }
2575
+ if (tid == 0) {
2576
+ rec_o[i] = local_o[0];
2577
+ }
2578
+ }
2579
+ }
2580
+
2581
+ // FD Pass 2: merge per-split partials into final O
2582
+ // empty splits drop via exp(-INF)=0.
2583
+ __kernel void flash_attn_f32_merge(
2584
+ const global float * partial_void,
2585
+ global void * o_void,
2586
+ const ulong o_offset,
2587
+ const int n_head,
2588
+ const int n_splits,
2589
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
2590
+ const global void * sinks_void,
2591
+ const ulong sinks_offset,
2592
+ const int n_q
2593
+ ) {
2594
+ const int lane = get_local_id(0); // 0..DV_VEC-1
2595
+ const int head_batch_idx = get_global_id(1);
2596
+ const int q_idx = get_global_id(2);
2597
+ const int batch_idx = head_batch_idx / n_head;
2598
+ const int head_idx = head_batch_idx % n_head;
2599
+
2600
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
2601
+ const ulong record_idx_0 = (((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) * n_splits;
2602
+ const global float * rec0 = partial_void + record_idx_0 * record_stride;
2603
+
2604
+ __local ACC_TYPE m_final_shared;
2605
+ __local ACC_TYPE l_final_shared;
2606
+ if (lane == 0) {
2607
+ ACC_TYPE m = FA_M_INIT;
2608
+ for (int c = 0; c < n_splits; ++c) {
2609
+ const ACC_TYPE m_c = rec0[c * record_stride + 0];
2610
+ m = max(m, m_c);
2611
+ }
2612
+ ACC_TYPE m_sink = 0.0f;
2613
+ bool has_sink = false;
2614
+ if (sinks_void != NULL) {
2615
+ const global ACC_TYPE * sinks_ptr =
2616
+ (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset);
2617
+ m_sink = sinks_ptr[head_idx];
2618
+ has_sink = true;
2619
+ m = max(m, m_sink);
2620
+ }
2621
+ ACC_TYPE l = 0.0f;
2622
+ for (int c = 0; c < n_splits; ++c) {
2623
+ const ACC_TYPE m_c = rec0[c * record_stride + 0];
2624
+ const ACC_TYPE l_c = rec0[c * record_stride + 1];
2625
+ if (m_c > FA_M_INIT) {
2626
+ l += l_c * exp(m_c - m);
2627
+ }
2628
+ }
2629
+ if (has_sink) {
2630
+ l += exp(m_sink - m);
2631
+ }
2632
+ m_final_shared = m;
2633
+ l_final_shared = l;
2634
+ }
2635
+ barrier(CLK_LOCAL_MEM_FENCE);
2636
+ const ACC_TYPE m_final = m_final_shared;
2637
+ const ACC_TYPE l_final = l_final_shared;
2638
+ const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f;
2639
+
2640
+ ACC_TYPE4 o = (ACC_TYPE4)(0.0f);
2641
+ for (int c = 0; c < n_splits; ++c) {
2642
+ const global float * rec_c = rec0 + c * record_stride;
2643
+ const ACC_TYPE m_c = rec_c[0];
2644
+ if (m_c <= FA_M_INIT) continue;
2645
+ const global float4 * rec_oc = (const global float4 *) (rec_c + 2);
2646
+ const ACC_TYPE scale_c = exp(m_c - m_final);
2647
+ o = mad((ACC_TYPE4)(scale_c), rec_oc[lane], o);
2648
+ }
2649
+ o = o * l_inv;
2650
+
2651
+ const ulong o_row_offset = (ulong) batch_idx * o_nb3 + (ulong) q_idx * o_nb2 + (ulong) head_idx * o_nb1;
2652
+ global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) ((global char *) o_void + o_offset + o_row_offset);
2653
+ o_row[lane] = CONVERT_O_DATA4(o);
2654
+ }
2655
+ #endif // !FA_MQ_ONLY (q1_split + merge)
2656
+ #endif // !FA_PREFILL_ONLY (decode kernels)