elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,1840 @@
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+ #ifdef cl_khr_integer_dot_product
3
+ #pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
4
+ #define FA_HAVE_INT_DOT 1
5
+ #endif
6
+
7
+ #ifdef cl_khr_subgroup_shuffle
8
+ #pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable
9
+ #define HAS_SUBGROUP_SHUFFLE 1
10
+ #elif defined(cl_qcom_subgroup_shuffle)
11
+ #pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable
12
+ #define HAS_SUBGROUP_SHUFFLE 1
13
+ // Adreno compilers that expose only cl_qcom_subgroup_shuffle do not declare the KHR
14
+ // name, so calling it is an implicit declaration and the program fails to build.
15
+ // Route it to the qcom builtin.
16
+ #define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f)
17
+ #endif
18
+
19
+ // Flash attention: Q=f32, K=q8_0, V=q8_0.
20
+
21
+ #define ACC_TYPE float
22
+ #define ACC_TYPE4 float4
23
+ #define Q_DATA_TYPE4 float4
24
+ #define O_DATA_TYPE4 float4
25
+ #define MASK_DATA_TYPE half
26
+ #define CONVERT_Q_ACC4(x) (x)
27
+ #define CONVERT_O_DATA4(x) (x)
28
+
29
+ #define DK_VEC (DK/4)
30
+ #define DV_VEC (DV/4)
31
+
32
+ #ifndef FA_SG
33
+ #define FA_SG 64
34
+ #endif
35
+ #define Q1_WG_SIZE FA_SG
36
+
37
+ // The kernels are built with -cl-finite-math-only. On some older Adreno GPUs,
38
+ // infinite operand can cause undefined behavior and miscompilation for exp.
39
+ // Therefore, a large negative value is used instead.
40
+ #define FA_M_INIT (-3.0e38f)
41
+
42
+ // q8_0 block: 2B scale (half) + 32B int8 quants.
43
+ #define QK8_0 32
44
+ #define Q8_0_BLOCK_SIZE 34
45
+
46
+ #define DK_Q8_BLOCKS (DK / QK8_0)
47
+ #define DV_Q8_BLOCKS (DV / QK8_0)
48
+
49
+ inline float dot_q8_0_f32(const global char * block_ptr, ACC_TYPE4 * q_slice) {
50
+ float d = vload_half(0, (const global half *)block_ptr);
51
+ const global char * qs = block_ptr + 2;
52
+
53
+ float sum = 0.0f;
54
+ #pragma unroll
55
+ for (int i = 0; i < 8; i++) {
56
+ float4 qv = (float4)((float)qs[i*4], (float)qs[i*4+1], (float)qs[i*4+2], (float)qs[i*4+3]);
57
+ sum += dot(q_slice[i], qv);
58
+ }
59
+ return sum * d;
60
+ }
61
+
62
+ #ifdef FA_HAVE_INT_DOT
63
+ inline uint pack_i8x4(char a, char b, char c, char d) {
64
+ return ((uint)(uchar)a) |
65
+ ((uint)(uchar)b) << 8 |
66
+ ((uint)(uchar)c) << 16 |
67
+ ((uint)(uchar)d) << 24;
68
+ }
69
+
70
+ inline float quant_q_block_int8_packed(const ACC_TYPE4 * q_block,
71
+ uint * out_packed) {
72
+ float amax = 0.0f;
73
+ #pragma unroll
74
+ for (int i = 0; i < 8; ++i) {
75
+ float4 av = fabs(q_block[i]);
76
+ amax = fmax(amax, fmax(fmax(av.s0, av.s1), fmax(av.s2, av.s3)));
77
+ }
78
+ float qd = amax / 127.0f;
79
+ float qid = (amax > 0.0f) ? 127.0f / amax : 0.0f;
80
+
81
+ #pragma unroll
82
+ for (int i = 0; i < 8; ++i) {
83
+ float4 v = q_block[i] * qid;
84
+ char a = (char)((int)round(v.s0));
85
+ char b = (char)((int)round(v.s1));
86
+ char c = (char)((int)round(v.s2));
87
+ char d = (char)((int)round(v.s3));
88
+ out_packed[i] = pack_i8x4(a, b, c, d);
89
+ }
90
+ return qd;
91
+ }
92
+
93
+ inline float dot_q8_0_int(const global char * k_block_ptr,
94
+ const uint * q_packed,
95
+ float q_d) {
96
+ float kd = vload_half(0, (const global half *)k_block_ptr);
97
+ const global uchar * k_qs = (const global uchar *)(k_block_ptr + 2);
98
+
99
+ // k_qs is 2-byte aligned; pack chars per iteration rather than cast to uint*.
100
+ int sum = 0;
101
+ #pragma unroll
102
+ for (int i = 0; i < 8; ++i) {
103
+ uint k_packed =
104
+ (uint)k_qs[i*4 + 0] |
105
+ ((uint)k_qs[i*4 + 1]) << 8 |
106
+ ((uint)k_qs[i*4 + 2]) << 16 |
107
+ ((uint)k_qs[i*4 + 3]) << 24;
108
+ sum = dot_acc_sat_4x8packed_ss_int(q_packed[i], k_packed, sum);
109
+ }
110
+ return (float)sum * q_d * kd;
111
+ }
112
+ #endif // FA_HAVE_INT_DOT
113
+
114
+ inline void dequant_q8_0_f32(const global char * block_ptr, ACC_TYPE4 * out) {
115
+ float d = vload_half(0, (const global half *)block_ptr);
116
+ const global char * qs = block_ptr + 2;
117
+
118
+ #pragma unroll
119
+ for (int i = 0; i < 8; i++) {
120
+ out[i] = d * (float4)((float)qs[i*4], (float)qs[i*4+1], (float)qs[i*4+2], (float)qs[i*4+3]);
121
+ }
122
+ }
123
+
124
+ // max_bias<=0 returns 1.0 so score += 1.0 * mask[k] stays a no-op multiplier.
125
+ inline float get_alibi_slope(float max_bias, int head_idx, int n_head_log2, float m0, float m1) {
126
+ if (max_bias <= 0.0f) return 1.0f;
127
+ float base = (head_idx < n_head_log2) ? m0 : m1;
128
+ int exph = (head_idx < n_head_log2) ? (head_idx + 1) : (2*(head_idx - n_head_log2) + 1);
129
+ return pow(base, (float)exph);
130
+ }
131
+
132
+ // q1 decode: one query row per WG, threads sweep KV positions.
133
+ __kernel void flash_attn_f32_q8_0_q1(
134
+ const global void * q_void, ulong q_offset,
135
+ const global void * k_void, ulong k_offset,
136
+ const global void * v_void, ulong v_offset,
137
+ global void * o_void, ulong o_offset,
138
+ const float scale,
139
+ const int n_q,
140
+ const int n_kv,
141
+ const int is_causal,
142
+ const int n_head,
143
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
144
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
145
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
146
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
147
+ const float max_bias,
148
+ const float m0,
149
+ const float m1,
150
+ const int n_head_log2,
151
+ const float logit_softcap,
152
+ const int n_head_kv,
153
+ const global void* mask_void,
154
+ const ulong mask_offset,
155
+ const ulong mask_nb1,
156
+ const ulong mask_nb2,
157
+ const ulong mask_nb3,
158
+ const int mask_ne2,
159
+ const int mask_ne3,
160
+ const global void* sinks_void,
161
+ const ulong sinks_offset
162
+ ) {
163
+ const int tid = get_local_id(0);
164
+ const int head_batch_idx = get_global_id(1);
165
+
166
+ const int batch_idx = head_batch_idx / n_head;
167
+ const int head_idx = head_batch_idx % n_head;
168
+
169
+ const int gqa_ratio = n_head / n_head_kv;
170
+ const int head_kv_idx = head_idx / gqa_ratio;
171
+
172
+ const global char* q_base = (const global char*)q_void + q_offset;
173
+ const global char* k_base = (const global char*)k_void + k_offset;
174
+ const global char* v_base = (const global char*)v_void + v_offset;
175
+ global char* o_base = (global char*)o_void + o_offset;
176
+
177
+ const global char* mask_base = NULL;
178
+ if (mask_void != NULL) {
179
+ const int mask_head_idx = head_idx % mask_ne2;
180
+ const int mask_batch_idx = batch_idx % mask_ne3;
181
+ mask_base = (const global char*)mask_void + mask_offset + mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;
182
+ }
183
+
184
+ ACC_TYPE4 q_priv[DK_VEC];
185
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2;
186
+ const global Q_DATA_TYPE4* q_ptr = (const global Q_DATA_TYPE4*)(q_base + q_row_offset);
187
+ #pragma unroll
188
+ for (int i = 0; i < DK_VEC; ++i) {
189
+ q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]);
190
+ }
191
+
192
+ #ifdef FA_HAVE_INT_DOT
193
+ // Quantise Q once per thread; q_priv stays as fp for the V accumulate.
194
+ uint q_packed[DK_Q8_BLOCKS * 8];
195
+ float q_d_scale[DK_Q8_BLOCKS];
196
+ #pragma unroll
197
+ for (int b = 0; b < DK_Q8_BLOCKS; ++b) {
198
+ q_d_scale[b] = quant_q_block_int8_packed(&q_priv[b * 8], &q_packed[b * 8]);
199
+ }
200
+ #endif
201
+
202
+ float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
203
+
204
+ const global ACC_TYPE* sinks_ptr = NULL;
205
+ if (sinks_void != NULL) {
206
+ sinks_ptr = (const global ACC_TYPE*)((const global char*)sinks_void + sinks_offset);
207
+ }
208
+
209
+ // One-pass online softmax: per-thread maintains running (m_i, l_i, o_acc),
210
+ // updating each as new K positions are processed. Eliminates the second
211
+ // K read of the original two-pass implementation. After the loop, threads
212
+ // are merged via the standard FA-2 cross-thread reduction (rescale each
213
+ // thread's l_i and o_acc by alpha=exp(m_i_thread - m_final), then sum).
214
+ ACC_TYPE m_i = (sinks_ptr != NULL) ? sinks_ptr[head_idx] : FA_M_INIT;
215
+ ACC_TYPE l_i = 0.0f;
216
+ ACC_TYPE4 o_acc[DV_VEC];
217
+ #pragma unroll
218
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);
219
+
220
+ for (int k_idx = tid; k_idx < n_kv; k_idx += Q1_WG_SIZE) {
221
+ const global char* k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
222
+ const global char* v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
223
+
224
+ ACC_TYPE score = 0.0f;
225
+ #pragma unroll
226
+ for (int b = 0; b < DK_Q8_BLOCKS; b++) {
227
+ #ifdef FA_HAVE_INT_DOT
228
+ score += dot_q8_0_int(k_row + b * Q8_0_BLOCK_SIZE,
229
+ &q_packed[b * 8], q_d_scale[b]);
230
+ #else
231
+ score += dot_q8_0_f32(k_row + b * Q8_0_BLOCK_SIZE, &q_priv[b * 8]);
232
+ #endif
233
+ }
234
+ score *= scale;
235
+
236
+ if (mask_base != NULL) {
237
+ const global MASK_DATA_TYPE* mask_ptr = (const global MASK_DATA_TYPE*)(mask_base);
238
+ score += slope * (ACC_TYPE)mask_ptr[k_idx];
239
+ }
240
+ if (logit_softcap > 0.0f) {
241
+ score = logit_softcap * tanh(score / logit_softcap);
242
+ }
243
+
244
+ // Online softmax step.
245
+ const ACC_TYPE m_new = max(m_i, score);
246
+ const ACC_TYPE alpha = exp(m_i - m_new);
247
+ const ACC_TYPE p = exp(score - m_new);
248
+
249
+ l_i = alpha * l_i + p;
250
+ #pragma unroll
251
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha;
252
+
253
+ #pragma unroll
254
+ for (int b = 0; b < DV_Q8_BLOCKS; b++) {
255
+ ACC_TYPE4 v_dequant[8];
256
+ dequant_q8_0_f32(v_row + b * Q8_0_BLOCK_SIZE, v_dequant);
257
+ #pragma unroll
258
+ for (int i = 0; i < 8; i++) {
259
+ o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]);
260
+ }
261
+ }
262
+
263
+ m_i = m_new;
264
+ }
265
+
266
+ // Cross-thread reduce: max(m_i) -> m_final, then rescale per-thread l_i
267
+ // and o_acc by alpha = exp(m_i_thread - m_final) before sum-reduce.
268
+ __local ACC_TYPE local_m[Q1_WG_SIZE];
269
+ local_m[tid] = m_i;
270
+ barrier(CLK_LOCAL_MEM_FENCE);
271
+ #pragma unroll
272
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
273
+ if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]);
274
+ barrier(CLK_LOCAL_MEM_FENCE);
275
+ }
276
+ const ACC_TYPE m_final = local_m[0];
277
+
278
+ const ACC_TYPE alpha_final = exp(m_i - m_final);
279
+ l_i *= alpha_final;
280
+ #pragma unroll
281
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final;
282
+
283
+ __local ACC_TYPE local_l[Q1_WG_SIZE];
284
+ __local ACC_TYPE4 local_o_comp[Q1_WG_SIZE];
285
+ local_l[tid] = l_i;
286
+ barrier(CLK_LOCAL_MEM_FENCE);
287
+ #pragma unroll
288
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
289
+ if (tid < s) local_l[tid] += local_l[tid + s];
290
+ barrier(CLK_LOCAL_MEM_FENCE);
291
+ }
292
+
293
+ const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1;
294
+ global O_DATA_TYPE4 *o_row = (global O_DATA_TYPE4 *)(o_base + o_row_offset);
295
+ ACC_TYPE l_final = local_l[0];
296
+
297
+ if (sinks_ptr != NULL) {
298
+ l_final += exp(sinks_ptr[head_idx] - m_final);
299
+ }
300
+
301
+ if (l_final > 0.0f) {
302
+ const ACC_TYPE l_inv = 1.0f / l_final;
303
+ for (int i = 0; i < DV_VEC; i++) {
304
+ local_o_comp[tid] = o_acc[i];
305
+ barrier(CLK_LOCAL_MEM_FENCE);
306
+ #pragma unroll
307
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
308
+ if (tid < s) local_o_comp[tid] += local_o_comp[tid + s];
309
+ barrier(CLK_LOCAL_MEM_FENCE);
310
+ }
311
+ if (tid == 0) {
312
+ o_row[i] = CONVERT_O_DATA4(local_o_comp[0] * l_inv);
313
+ }
314
+ }
315
+ } else if (tid == 0) {
316
+ #pragma unroll
317
+ for (int i = 0; i < DV_VEC; ++i) o_row[i] = (O_DATA_TYPE4)(0.0f);
318
+ }
319
+ }
320
+
321
+ #ifdef cl_intel_subgroups
322
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
323
+ #else
324
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
325
+ #endif
326
+
327
+ #ifdef cl_qcom_reqd_sub_group_size
328
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
329
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
330
+ #else
331
+ #define REQD_SUBGROUP_SIZE_64
332
+ #endif
333
+
334
+ #define VEC_NSG 4
335
+ #define VEC_WG_SIZE (Q1_WG_SIZE * VEC_NSG)
336
+ #define Q1V_DV_PER_THREAD ((DV_VEC + Q1_WG_SIZE - 1) / Q1_WG_SIZE)
337
+
338
+ inline float4 dequant_q8_0_lane(const global char * block_ptr, int lane) {
339
+ const float d = vload_half(0, (const global half *)block_ptr);
340
+ const global char * qs = block_ptr + 2 + lane * 4;
341
+ return d * (float4)((float)qs[0], (float)qs[1], (float)qs[2], (float)qs[3]);
342
+ }
343
+
344
+ REQD_SUBGROUP_SIZE_64
345
+ __kernel void flash_attn_f32_q8_0_q1_vec(
346
+ const global void * q_void, ulong q_offset,
347
+ const global void * k_void, ulong k_offset,
348
+ const global void * v_void, ulong v_offset,
349
+ global void * o_void, ulong o_offset,
350
+ const float scale,
351
+ const int n_q,
352
+ const int n_kv,
353
+ const int is_causal,
354
+ const int n_head,
355
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
356
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
357
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
358
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
359
+ const float max_bias,
360
+ const float m0,
361
+ const float m1,
362
+ const int n_head_log2,
363
+ const float logit_softcap,
364
+ const int n_head_kv,
365
+ const global void* mask_void,
366
+ const ulong mask_offset,
367
+ const ulong mask_nb1,
368
+ const ulong mask_nb2,
369
+ const ulong mask_nb3,
370
+ const int mask_ne2,
371
+ const int mask_ne3,
372
+ const global void* sinks_void,
373
+ const ulong sinks_offset
374
+ ) {
375
+ const int tid = get_local_id(0);
376
+ const int sgid = tid / Q1_WG_SIZE;
377
+ const int tid_sg = tid % Q1_WG_SIZE;
378
+ const int head_batch_idx = get_global_id(1);
379
+
380
+ const int batch_idx = head_batch_idx / n_head;
381
+ const int head_idx = head_batch_idx % n_head;
382
+
383
+ const int gqa_ratio = n_head / n_head_kv;
384
+ const int head_kv_idx = head_idx / gqa_ratio;
385
+
386
+ const global char * q_base = (const global char *) q_void + q_offset;
387
+ const global char * k_base = (const global char *) k_void + k_offset;
388
+ const global char * v_base = (const global char *) v_void + v_offset;
389
+ global char * o_base = (global char *) o_void + o_offset;
390
+
391
+ const global char * mask_base = NULL;
392
+ if (mask_void != NULL) {
393
+ const int mask_head_idx = head_idx % mask_ne2;
394
+ const int mask_batch_idx = batch_idx % mask_ne3;
395
+ mask_base = (const global char *) mask_void + mask_offset +
396
+ mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;
397
+ }
398
+
399
+ __local ACC_TYPE4 q_shared[DK_VEC];
400
+ {
401
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2;
402
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
403
+ for (int i = tid; i < DK_VEC; i += VEC_WG_SIZE) {
404
+ q_shared[i] = CONVERT_Q_ACC4(q_ptr[i]);
405
+ }
406
+ }
407
+ barrier(CLK_LOCAL_MEM_FENCE);
408
+
409
+ const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
410
+
411
+ const global ACC_TYPE * sinks_ptr = NULL;
412
+ if (sinks_void != NULL) {
413
+ sinks_ptr = (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset);
414
+ }
415
+
416
+ ACC_TYPE4 o_acc[Q1V_DV_PER_THREAD];
417
+ #pragma unroll
418
+ for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);
419
+
420
+ ACC_TYPE m_i = FA_M_INIT;
421
+ ACC_TYPE l_i = 0.0f;
422
+
423
+ const int kv_per_sg = (n_kv + VEC_NSG - 1) / VEC_NSG;
424
+ const int kv_start = sgid * kv_per_sg;
425
+ const int kv_end = min(n_kv, kv_start + kv_per_sg);
426
+
427
+ for (int k_idx = kv_start; k_idx < kv_end; ++k_idx) {
428
+ const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
429
+ const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
430
+
431
+ ACC_TYPE4 dot4 = (ACC_TYPE4)(0.0f);
432
+ for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) {
433
+ const int block_idx = qk / 8;
434
+ const int lane = qk % 8;
435
+ const float4 k_v = dequant_q8_0_lane(k_row + block_idx * Q8_0_BLOCK_SIZE, lane);
436
+ dot4 = mad(q_shared[qk], k_v, dot4);
437
+ }
438
+ ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3;
439
+ ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale;
440
+
441
+ if (mask_base != NULL) {
442
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base;
443
+ score += slope * (ACC_TYPE) mask_ptr[k_idx];
444
+ }
445
+ if (logit_softcap > 0.0f) {
446
+ score = logit_softcap * tanh(score / logit_softcap);
447
+ }
448
+
449
+ const ACC_TYPE m_new = max(m_i, score);
450
+ const ACC_TYPE scale_prev = native_exp(m_i - m_new);
451
+ const ACC_TYPE p = native_exp(score - m_new);
452
+
453
+ int idx = 0;
454
+ for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) {
455
+ const int block_idx = dv / 8;
456
+ const int lane = dv % 8;
457
+ const float4 v_v = dequant_q8_0_lane(v_row + block_idx * Q8_0_BLOCK_SIZE, lane);
458
+ o_acc[idx] = mad(p, v_v, o_acc[idx] * scale_prev);
459
+ }
460
+ l_i = l_i * scale_prev + p;
461
+ m_i = m_new;
462
+ }
463
+
464
+ __local ACC_TYPE sg_m[VEC_NSG];
465
+ __local ACC_TYPE sg_l[VEC_NSG];
466
+ __local ACC_TYPE4 sg_o[VEC_NSG][DV_VEC];
467
+
468
+ if (tid_sg == 0) {
469
+ sg_m[sgid] = m_i;
470
+ sg_l[sgid] = l_i;
471
+ }
472
+ {
473
+ int idx = 0;
474
+ for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) {
475
+ sg_o[sgid][dv] = o_acc[idx];
476
+ }
477
+ }
478
+ barrier(CLK_LOCAL_MEM_FENCE);
479
+
480
+ if (sgid == 0) {
481
+ ACC_TYPE m_final = sg_m[0];
482
+ #pragma unroll
483
+ for (int s = 1; s < VEC_NSG; ++s) {
484
+ m_final = max(m_final, sg_m[s]);
485
+ }
486
+ if (sinks_ptr != NULL) {
487
+ m_final = max(m_final, sinks_ptr[head_idx]);
488
+ }
489
+
490
+ ACC_TYPE l_final = 0.0f;
491
+ #pragma unroll
492
+ for (int s = 0; s < VEC_NSG; ++s) {
493
+ l_final += sg_l[s] * native_exp(sg_m[s] - m_final);
494
+ }
495
+ if (sinks_ptr != NULL) {
496
+ l_final += native_exp(sinks_ptr[head_idx] - m_final);
497
+ }
498
+ const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f;
499
+
500
+ const ulong o_row_offset = batch_idx * o_nb3 + head_idx * o_nb1;
501
+ global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) (o_base + o_row_offset);
502
+
503
+ int idx = 0;
504
+ for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) {
505
+ ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f);
506
+ #pragma unroll
507
+ for (int s = 0; s < VEC_NSG; ++s) {
508
+ const ACC_TYPE alpha = native_exp(sg_m[s] - m_final);
509
+ o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv], o_merged);
510
+ }
511
+ o_row[dv] = CONVERT_O_DATA4(o_merged * l_inv);
512
+ }
513
+ }
514
+ }
515
+
516
+ // Flash-decoding split pass for q8_0 KV. Partial record: [m, l, O[DV]].
517
+ // Merge kernel from flash_attn_f32_f16.cl is type-agnostic and reused.
518
+ #define FA_PARTIAL_FLOATS (2 + DV)
519
+
520
+ __kernel void flash_attn_f32_q8_0_q1_split(
521
+ const global void * q_void, ulong q_offset,
522
+ const global void * k_void, ulong k_offset,
523
+ const global void * v_void, ulong v_offset,
524
+ const float scale,
525
+ const int n_q,
526
+ const int n_kv,
527
+ const int n_head,
528
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
529
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
530
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
531
+ const float max_bias,
532
+ const float m0,
533
+ const float m1,
534
+ const int n_head_log2,
535
+ const float logit_softcap,
536
+ const int n_head_kv,
537
+ const global void * mask_void,
538
+ const ulong mask_offset,
539
+ const ulong mask_nb1,
540
+ const ulong mask_nb2,
541
+ const ulong mask_nb3,
542
+ const int mask_ne2,
543
+ const int mask_ne3,
544
+ global float * partial_void,
545
+ const int n_splits,
546
+ const int kv_per_split
547
+ ) {
548
+ const int tid = get_local_id(0);
549
+ const int head_batch_idx = get_global_id(1);
550
+ const int split_q_idx = get_global_id(2);
551
+ const int split_idx = split_q_idx % n_splits;
552
+ const int q_idx = split_q_idx / n_splits;
553
+ const int batch_idx = head_batch_idx / n_head;
554
+ const int head_idx = head_batch_idx % n_head;
555
+ const int gqa_ratio = n_head / n_head_kv;
556
+ const int head_kv_idx = head_idx / gqa_ratio;
557
+
558
+ const int kv_start = split_idx * kv_per_split;
559
+ const int kv_end = min(kv_start + kv_per_split, n_kv);
560
+
561
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
562
+ const ulong record_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
563
+ * n_splits + split_idx);
564
+ global float * rec = partial_void + record_idx * record_stride;
565
+ global float4 * rec_o = (global float4 *) (rec + 2);
566
+
567
+ if (kv_start >= kv_end) {
568
+ // Empty split: leave sentinel partial for merge.
569
+ if (tid == 0) {
570
+ rec[0] = FA_M_INIT;
571
+ rec[1] = 0.0f;
572
+ }
573
+ return;
574
+ }
575
+
576
+ const global char * q_base = (const global char *) q_void + q_offset;
577
+ const global char * k_base = (const global char *) k_void + k_offset;
578
+ const global char * v_base = (const global char *) v_void + v_offset;
579
+
580
+ const global char * mask_base = NULL;
581
+ if (mask_void != NULL) {
582
+ const int mask_head_idx = head_idx % mask_ne2;
583
+ const int mask_batch_idx = batch_idx % mask_ne3;
584
+ mask_base = (const global char *) mask_void + mask_offset +
585
+ mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2 +
586
+ (ulong) q_idx * mask_nb1;
587
+ }
588
+
589
+ ACC_TYPE4 q_priv[DK_VEC];
590
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
591
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
592
+ #pragma unroll
593
+ for (int i = 0; i < DK_VEC; ++i) {
594
+ q_priv[i] = CONVERT_Q_ACC4(q_ptr[i]);
595
+ }
596
+
597
+ #ifdef FA_HAVE_INT_DOT
598
+ uint q_packed[DK_Q8_BLOCKS * 8];
599
+ float q_d_scale[DK_Q8_BLOCKS];
600
+ #pragma unroll
601
+ for (int b = 0; b < DK_Q8_BLOCKS; ++b) {
602
+ q_d_scale[b] = quant_q_block_int8_packed(&q_priv[b * 8], &q_packed[b * 8]);
603
+ }
604
+ #endif
605
+
606
+ const float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
607
+
608
+ // One-pass online softmax (FA-2): single sweep over the split's K range,
609
+ // updating per-thread (m_i, l_i, o_acc) per position. Eliminates the
610
+ // second K read of the original two-pass implementation.
611
+ ACC_TYPE m_i = FA_M_INIT;
612
+ ACC_TYPE l_i = 0.0f;
613
+ ACC_TYPE4 o_acc[DV_VEC];
614
+ #pragma unroll
615
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);
616
+
617
+ for (int k_idx = kv_start + tid; k_idx < kv_end; k_idx += Q1_WG_SIZE) {
618
+ const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
619
+ const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
620
+ ACC_TYPE score = 0.0f;
621
+ #pragma unroll
622
+ for (int b = 0; b < DK_Q8_BLOCKS; ++b) {
623
+ #ifdef FA_HAVE_INT_DOT
624
+ score += dot_q8_0_int(k_row + b * Q8_0_BLOCK_SIZE, &q_packed[b * 8], q_d_scale[b]);
625
+ #else
626
+ score += dot_q8_0_f32(k_row + b * Q8_0_BLOCK_SIZE, &q_priv[b * 8]);
627
+ #endif
628
+ }
629
+ score *= scale;
630
+ if (mask_base != NULL) {
631
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base);
632
+ score += slope * (ACC_TYPE) mask_ptr[k_idx];
633
+ }
634
+ if (logit_softcap > 0.0f) {
635
+ score = logit_softcap * tanh(score / logit_softcap);
636
+ }
637
+
638
+ // Online softmax step.
639
+ const ACC_TYPE m_new = max(m_i, score);
640
+ const ACC_TYPE alpha = exp(m_i - m_new);
641
+ const ACC_TYPE p = exp(score - m_new);
642
+
643
+ l_i = alpha * l_i + p;
644
+ #pragma unroll
645
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha;
646
+
647
+ #pragma unroll
648
+ for (int b = 0; b < DV_Q8_BLOCKS; ++b) {
649
+ ACC_TYPE4 v_dequant[8];
650
+ dequant_q8_0_f32(v_row + b * Q8_0_BLOCK_SIZE, v_dequant);
651
+ #pragma unroll
652
+ for (int i = 0; i < 8; ++i) {
653
+ o_acc[b * 8 + i] = mad(p, v_dequant[i], o_acc[b * 8 + i]);
654
+ }
655
+ }
656
+
657
+ m_i = m_new;
658
+ }
659
+
660
+ // Cross-thread reduce: max(m_i) -> m_c, then rescale per-thread l_i and
661
+ // o_acc by alpha = exp(m_i_thread - m_c) before sum-reduce.
662
+ __local ACC_TYPE local_m[Q1_WG_SIZE];
663
+ local_m[tid] = m_i;
664
+ barrier(CLK_LOCAL_MEM_FENCE);
665
+ #pragma unroll
666
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
667
+ if (tid < s) local_m[tid] = max(local_m[tid], local_m[tid + s]);
668
+ barrier(CLK_LOCAL_MEM_FENCE);
669
+ }
670
+ const ACC_TYPE m_c = local_m[0];
671
+
672
+ const ACC_TYPE alpha_final = exp(m_i - m_c);
673
+ l_i *= alpha_final;
674
+ #pragma unroll
675
+ for (int i = 0; i < DV_VEC; ++i) o_acc[i] *= alpha_final;
676
+
677
+ __local ACC_TYPE local_l[Q1_WG_SIZE];
678
+ __local ACC_TYPE4 local_o[Q1_WG_SIZE];
679
+ local_l[tid] = l_i;
680
+ barrier(CLK_LOCAL_MEM_FENCE);
681
+ #pragma unroll
682
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
683
+ if (tid < s) local_l[tid] += local_l[tid + s];
684
+ barrier(CLK_LOCAL_MEM_FENCE);
685
+ }
686
+ const ACC_TYPE l_c = local_l[0];
687
+
688
+ if (tid == 0) {
689
+ rec[0] = (float) m_c;
690
+ rec[1] = (float) l_c;
691
+ }
692
+ for (int i = 0; i < DV_VEC; ++i) {
693
+ local_o[tid] = o_acc[i];
694
+ barrier(CLK_LOCAL_MEM_FENCE);
695
+ #pragma unroll
696
+ for (int s = Q1_WG_SIZE / 2; s > 0; s >>= 1) {
697
+ if (tid < s) local_o[tid] += local_o[tid + s];
698
+ barrier(CLK_LOCAL_MEM_FENCE);
699
+ }
700
+ if (tid == 0) {
701
+ rec_o[i] = local_o[0];
702
+ }
703
+ }
704
+ }
705
+
706
+ // Prefill: q8_0 K/V, n_q > 1. BLOCK_M × BLOCK_N tiling.
707
+ // K path keeps packed int8 in local for dp4a QK dot; V path dequant -> half in local.
708
+ // Requires DK % QK8_0 == 0 and DV % QK8_0 == 0 (gated in supports_op).
709
+ #define KV_DATA_TYPE4 half4
710
+ #define CONVERT_KV_ACC4(x) convert_float4(x)
711
+
712
+ #define DK_Q8_BLOCKS_PREFILL (DK / QK8_0)
713
+ #define DV_Q8_BLOCKS_PREFILL (DV / QK8_0)
714
+
715
+ // N_SPLIT>1 splits DK/DV across N_SPLIT threads per query row; needs
716
+ // sub_group_shuffle_xor and DK_Q8_BLOCKS_PREFILL % N_SPLIT == 0.
717
+ #ifndef N_SPLIT
718
+ #define N_SPLIT 1
719
+ #endif
720
+
721
+ #if N_SPLIT > 1
722
+ #define SPLIT_DK_VEC (DK_VEC / N_SPLIT)
723
+ #define SPLIT_DV_VEC (DV_VEC / N_SPLIT)
724
+ #define SPLIT_DK_Q8_BLOCKS (DK_Q8_BLOCKS_PREFILL / N_SPLIT)
725
+ #define WG_SIZE (BLOCK_M * N_SPLIT)
726
+ #else
727
+ #define SPLIT_DK_VEC DK_VEC
728
+ #define SPLIT_DV_VEC DV_VEC
729
+ #define SPLIT_DK_Q8_BLOCKS DK_Q8_BLOCKS_PREFILL
730
+ #define WG_SIZE BLOCK_M
731
+ #endif
732
+
733
+ // FA_V_STRATEGY: 0 = dequant V to half in local (default); 2 = keep packed
734
+ // int8 in local, dequant in the accumulate loop (smaller local, slightly slower).
735
+ #ifndef FA_V_STRATEGY
736
+ #define FA_V_STRATEGY 0
737
+ #endif
738
+
739
+ #ifndef MQ_GQA
740
+ #define MQ_GQA 4
741
+ #endif
742
+ #ifndef MQ_NSG_SPLIT
743
+ #define MQ_NSG_SPLIT 4
744
+ #endif
745
+ #define MQ_SPLIT_WG_SIZE_Q8 (Q1_WG_SIZE * MQ_NSG_SPLIT)
746
+
747
+ REQD_SUBGROUP_SIZE_64
748
+ __kernel void flash_attn_f32_q8_0_q1_vec_mq_split(
749
+ const global void * q_void, ulong q_offset,
750
+ const global void * k_void, ulong k_offset,
751
+ const global void * v_void, ulong v_offset,
752
+ const float scale,
753
+ const int n_q,
754
+ const int n_kv,
755
+ const int n_head,
756
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
757
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
758
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
759
+ const float max_bias,
760
+ const float m0,
761
+ const float m1,
762
+ const int n_head_log2,
763
+ const float logit_softcap,
764
+ const int n_head_kv,
765
+ const global void * mask_void,
766
+ const ulong mask_offset,
767
+ const ulong mask_nb1,
768
+ const ulong mask_nb2,
769
+ const ulong mask_nb3,
770
+ const int mask_ne2,
771
+ const int mask_ne3,
772
+ global float * partial_void,
773
+ const int n_splits,
774
+ const int kv_per_split
775
+ ) {
776
+ const int tid = get_local_id(0);
777
+ const int sgid = tid / Q1_WG_SIZE;
778
+ const int tid_sg = tid % Q1_WG_SIZE;
779
+ const int kvhead_batch_idx = get_global_id(1);
780
+ const int split_q_idx = get_global_id(2);
781
+ const int split_idx = split_q_idx % n_splits;
782
+ const int q_idx = split_q_idx / n_splits;
783
+
784
+ const int batch_idx = kvhead_batch_idx / n_head_kv;
785
+ const int head_kv_idx = kvhead_batch_idx % n_head_kv;
786
+
787
+ const int kv_start = split_idx * kv_per_split;
788
+ const int kv_end = min(kv_start + kv_per_split, n_kv);
789
+
790
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
791
+
792
+ if (kv_start >= kv_end) {
793
+ // Empty split — write sentinel for each of the MQ_GQA Q-heads.
794
+ if (tid == 0) {
795
+ #pragma unroll
796
+ for (int h = 0; h < MQ_GQA; ++h) {
797
+ const int head_idx = head_kv_idx * MQ_GQA + h;
798
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
799
+ * n_splits + split_idx);
800
+ global float * rec = partial_void + rec_idx * record_stride;
801
+ rec[0] = FA_M_INIT;
802
+ rec[1] = 0.0f;
803
+ }
804
+ }
805
+ return;
806
+ }
807
+
808
+ const global char * q_base = (const global char *) q_void + q_offset;
809
+ const global char * k_base = (const global char *) k_void + k_offset;
810
+ const global char * v_base = (const global char *) v_void + v_offset;
811
+
812
+ __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC];
813
+ for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE_Q8) {
814
+ const int h = i / DK_VEC;
815
+ const int k = i % DK_VEC;
816
+ const int head_idx = head_kv_idx * MQ_GQA + h;
817
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
818
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
819
+ q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);
820
+ }
821
+ barrier(CLK_LOCAL_MEM_FENCE);
822
+
823
+ float slope[MQ_GQA];
824
+ #pragma unroll
825
+ for (int h = 0; h < MQ_GQA; ++h) {
826
+ slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);
827
+ }
828
+
829
+ const global char * mask_base[MQ_GQA];
830
+ if (mask_void != NULL) {
831
+ const int mask_batch_idx = batch_idx % mask_ne3;
832
+ const global char * mask_base_b = (const global char *) mask_void + mask_offset +
833
+ mask_batch_idx * mask_nb3 +
834
+ (ulong) q_idx * mask_nb1;
835
+ #pragma unroll
836
+ for (int h = 0; h < MQ_GQA; ++h) {
837
+ const int head_idx = head_kv_idx * MQ_GQA + h;
838
+ const int mask_head_idx = head_idx % mask_ne2;
839
+ mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;
840
+ }
841
+ } else {
842
+ #pragma unroll
843
+ for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL;
844
+ }
845
+
846
+ ACC_TYPE4 o_acc[MQ_GQA][Q1V_DV_PER_THREAD];
847
+ ACC_TYPE m_i[MQ_GQA];
848
+ ACC_TYPE l_i[MQ_GQA];
849
+ #pragma unroll
850
+ for (int h = 0; h < MQ_GQA; ++h) {
851
+ m_i[h] = FA_M_INIT;
852
+ l_i[h] = 0.0f;
853
+ #pragma unroll
854
+ for (int i = 0; i < Q1V_DV_PER_THREAD; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f);
855
+ }
856
+
857
+ const int kv_len = kv_end - kv_start;
858
+ const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT;
859
+ const int kv_lo = kv_start + sgid * kv_per_sg;
860
+ const int kv_hi = min(kv_end, kv_lo + kv_per_sg);
861
+
862
+ for (int k_idx = kv_lo; k_idx < kv_hi; ++k_idx) {
863
+ const global char * k_row = k_base + batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_idx * k_nb1;
864
+ const global char * v_row = v_base + batch_idx * v_nb3 + head_kv_idx * v_nb2 + k_idx * v_nb1;
865
+
866
+ ACC_TYPE4 dot4[MQ_GQA];
867
+ #pragma unroll
868
+ for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f);
869
+
870
+ for (int qk = tid_sg; qk < DK_VEC; qk += Q1_WG_SIZE) {
871
+ const int block_idx = qk / 8;
872
+ const int lane = qk % 8;
873
+ const float4 k_v = dequant_q8_0_lane(k_row + block_idx * Q8_0_BLOCK_SIZE, lane);
874
+ #pragma unroll
875
+ for (int h = 0; h < MQ_GQA; ++h) {
876
+ dot4[h] = mad(q_shared[h * DK_VEC + qk], k_v, dot4[h]);
877
+ }
878
+ }
879
+
880
+ ACC_TYPE score[MQ_GQA];
881
+ #pragma unroll
882
+ for (int h = 0; h < MQ_GQA; ++h) {
883
+ const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
884
+ ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale;
885
+ if (mask_base[h] != NULL) {
886
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
887
+ s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
888
+ }
889
+ if (logit_softcap > 0.0f) {
890
+ s = logit_softcap * tanh(s / logit_softcap);
891
+ }
892
+ score[h] = s;
893
+ }
894
+
895
+ ACC_TYPE p_h[MQ_GQA];
896
+ ACC_TYPE sp_h[MQ_GQA];
897
+ #pragma unroll
898
+ for (int h = 0; h < MQ_GQA; ++h) {
899
+ const ACC_TYPE m_new = max(m_i[h], score[h]);
900
+ sp_h[h] = native_exp(m_i[h] - m_new);
901
+ p_h[h] = native_exp(score[h] - m_new);
902
+ l_i[h] = l_i[h] * sp_h[h] + p_h[h];
903
+ m_i[h] = m_new;
904
+ }
905
+
906
+ int idx = 0;
907
+ for (int dv = tid_sg; dv < DV_VEC; dv += Q1_WG_SIZE, ++idx) {
908
+ const int block_idx = dv / 8;
909
+ const int lane = dv % 8;
910
+ const float4 v_v = dequant_q8_0_lane(v_row + block_idx * Q8_0_BLOCK_SIZE, lane);
911
+ #pragma unroll
912
+ for (int h = 0; h < MQ_GQA; ++h) {
913
+ o_acc[h][idx] = mad(p_h[h], v_v, o_acc[h][idx] * sp_h[h]);
914
+ }
915
+ }
916
+ }
917
+
918
+ __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT];
919
+ __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT];
920
+ __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC];
921
+
922
+ if (tid_sg == 0) {
923
+ #pragma unroll
924
+ for (int h = 0; h < MQ_GQA; ++h) {
925
+ sg_m[h][sgid] = m_i[h];
926
+ sg_l[h][sgid] = l_i[h];
927
+ }
928
+ }
929
+
930
+ #pragma unroll
931
+ for (int h = 0; h < MQ_GQA; ++h) {
932
+ {
933
+ int idx = 0;
934
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE, ++idx) {
935
+ sg_o[sgid][dv_idx] = o_acc[h][idx];
936
+ }
937
+ }
938
+ barrier(CLK_LOCAL_MEM_FENCE);
939
+
940
+ if (sgid == 0) {
941
+ const int head_idx = head_kv_idx * MQ_GQA + h;
942
+
943
+ ACC_TYPE m_c = sg_m[h][0];
944
+ #pragma unroll
945
+ for (int s = 1; s < MQ_NSG_SPLIT; ++s) {
946
+ m_c = max(m_c, sg_m[h][s]);
947
+ }
948
+ ACC_TYPE l_c = 0.0f;
949
+ #pragma unroll
950
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
951
+ l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c);
952
+ }
953
+
954
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
955
+ * n_splits + split_idx);
956
+ global float * rec = partial_void + rec_idx * record_stride;
957
+ global float4 * rec_o = (global float4 *) (rec + 2);
958
+
959
+ if (tid_sg == 0) {
960
+ rec[0] = (float) m_c;
961
+ rec[1] = (float) l_c;
962
+ }
963
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) {
964
+ ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f);
965
+ #pragma unroll
966
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
967
+ const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c);
968
+ o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged);
969
+ }
970
+ rec_o[dv_idx] = o_merged;
971
+ }
972
+ }
973
+ barrier(CLK_LOCAL_MEM_FENCE);
974
+ }
975
+ }
976
+
977
+ // flash_attn_f32_q8_0_q1_vec_mq_split_c8 — cluster-parallel variant of the MQ
978
+ // split above, port of the f16/q4_0 c8 kernels
979
+
980
+ #ifdef HAS_SUBGROUP_SHUFFLE
981
+
982
+ #ifndef FA_CL_C
983
+ #define FA_CL_C 8
984
+ #endif
985
+
986
+ // Lane striping requires DK/DV to divide across the cluster (see f16 c8).
987
+ #if (DK_VEC % FA_CL_C) == 0 && (DV_VEC % FA_CL_C) == 0
988
+ #define FA_CL_NCL (Q1_WG_SIZE / FA_CL_C) // clusters (position streams) per subgroup
989
+ #define FA_CL_DKQ (DK_VEC / FA_CL_C) // K quartets per lane per row
990
+ #define FA_CL_DVQ (DV_VEC / FA_CL_C) // V quartets (o_acc float4s) per lane per head
991
+
992
+ #ifdef FA_C8_NO_SG_PIN
993
+ #define FA_C8_SG_ATTR_Q8
994
+ #else
995
+ #define FA_C8_SG_ATTR_Q8 REQD_SUBGROUP_SIZE_64
996
+ #endif
997
+
998
+ FA_C8_SG_ATTR_Q8
999
+ __kernel void flash_attn_f32_q8_0_q1_vec_mq_split_c8(
1000
+ const global void * q_void, ulong q_offset,
1001
+ const global void * k_void, ulong k_offset,
1002
+ const global void * v_void, ulong v_offset,
1003
+ const float scale,
1004
+ const int n_q,
1005
+ const int n_kv,
1006
+ const int n_head,
1007
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
1008
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
1009
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
1010
+ const float max_bias,
1011
+ const float m0,
1012
+ const float m1,
1013
+ const int n_head_log2,
1014
+ const float logit_softcap,
1015
+ const int n_head_kv,
1016
+ const global void * mask_void,
1017
+ const ulong mask_offset,
1018
+ const ulong mask_nb1,
1019
+ const ulong mask_nb2,
1020
+ const ulong mask_nb3,
1021
+ const int mask_ne2,
1022
+ const int mask_ne3,
1023
+ global float * partial_void,
1024
+ const int n_splits,
1025
+ const int kv_per_split
1026
+ ) {
1027
+ const int tid = get_local_id(0);
1028
+ const int sgid = tid / Q1_WG_SIZE;
1029
+ const int tid_sg = tid % Q1_WG_SIZE;
1030
+ const int cl = tid_sg / FA_CL_C; // cluster id
1031
+ const int lic = tid_sg % FA_CL_C; // lane in cluster
1032
+ const int kvhead_batch_idx = get_global_id(1);
1033
+ const int split_q_idx = get_global_id(2);
1034
+ const int split_idx = split_q_idx % n_splits;
1035
+ const int q_idx = split_q_idx / n_splits;
1036
+
1037
+ const int batch_idx = kvhead_batch_idx / n_head_kv;
1038
+ const int head_kv_idx = kvhead_batch_idx % n_head_kv;
1039
+
1040
+ const int kv_start = split_idx * kv_per_split;
1041
+ const int kv_end = min(kv_start + kv_per_split, n_kv);
1042
+
1043
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
1044
+
1045
+ if (kv_start >= kv_end) {
1046
+ if (tid == 0) {
1047
+ #pragma unroll
1048
+ for (int h = 0; h < MQ_GQA; ++h) {
1049
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1050
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
1051
+ * n_splits + split_idx);
1052
+ global float * rec = partial_void + rec_idx * record_stride;
1053
+ rec[0] = FA_M_INIT;
1054
+ rec[1] = 0.0f;
1055
+ }
1056
+ }
1057
+ return;
1058
+ }
1059
+
1060
+ const global char * q_base = (const global char *) q_void + q_offset;
1061
+ const global char * k_base = (const global char *) k_void + k_offset;
1062
+ const global char * v_base = (const global char *) v_void + v_offset;
1063
+
1064
+ // Stage MQ_GQA Q rows in __local once (uniform across WG).
1065
+ __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC];
1066
+ for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE_Q8) {
1067
+ const int h = i / DK_VEC;
1068
+ const int k = i % DK_VEC;
1069
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1070
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
1071
+ const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
1072
+ q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);
1073
+ }
1074
+ barrier(CLK_LOCAL_MEM_FENCE);
1075
+
1076
+ float slope[MQ_GQA];
1077
+ #pragma unroll
1078
+ for (int h = 0; h < MQ_GQA; ++h) {
1079
+ slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);
1080
+ }
1081
+
1082
+ const global char * mask_base[MQ_GQA];
1083
+ if (mask_void != NULL) {
1084
+ const int mask_batch_idx = batch_idx % mask_ne3;
1085
+ const global char * mask_base_b = (const global char *) mask_void + mask_offset +
1086
+ mask_batch_idx * mask_nb3 +
1087
+ (ulong) q_idx * mask_nb1;
1088
+ #pragma unroll
1089
+ for (int h = 0; h < MQ_GQA; ++h) {
1090
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1091
+ const int mask_head_idx = head_idx % mask_ne2;
1092
+ mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;
1093
+ }
1094
+ } else {
1095
+ #pragma unroll
1096
+ for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL;
1097
+ }
1098
+
1099
+ // Per-CLUSTER online state; o_acc holds this lane's V quartets {lic + FA_CL_C*i}.
1100
+ ACC_TYPE4 o_acc[MQ_GQA][FA_CL_DVQ];
1101
+ ACC_TYPE m_i[MQ_GQA];
1102
+ ACC_TYPE l_i[MQ_GQA];
1103
+ #pragma unroll
1104
+ for (int h = 0; h < MQ_GQA; ++h) {
1105
+ m_i[h] = FA_M_INIT;
1106
+ l_i[h] = 0.0f;
1107
+ #pragma unroll
1108
+ for (int i = 0; i < FA_CL_DVQ; ++i) o_acc[h][i] = (ACC_TYPE4)(0.0f);
1109
+ }
1110
+
1111
+ const int kv_len = kv_end - kv_start;
1112
+ const int kv_per_sg = (kv_len + MQ_NSG_SPLIT - 1) / MQ_NSG_SPLIT;
1113
+ const int kv_lo = kv_start + sgid * kv_per_sg;
1114
+ const int kv_hi = min(kv_end, kv_lo + kv_per_sg);
1115
+
1116
+ // Uniform trip count; tail clamps the row address and drops the score to
1117
+ // FA_M_INIT (p underflows to 0) so shuffles stay convergent.
1118
+ const int n_iter = (kv_hi - kv_lo + FA_CL_NCL - 1) / FA_CL_NCL;
1119
+ const ulong k_row_base = batch_idx * k_nb3 + head_kv_idx * k_nb2;
1120
+ const ulong v_row_base = batch_idx * v_nb3 + head_kv_idx * v_nb2;
1121
+
1122
+ for (int it = 0; it < n_iter; ++it) {
1123
+ const int k_idx = kv_lo + cl + it * FA_CL_NCL;
1124
+ const int valid = k_idx < kv_hi;
1125
+ const int k_safe = valid ? k_idx : (kv_hi - 1);
1126
+
1127
+ const global char * k_row = k_base + k_row_base + (ulong) k_safe * k_nb1;
1128
+ const global char * v_row = v_base + v_row_base + (ulong) k_safe * v_nb1;
1129
+
1130
+ // Float-dequant K dot over this lane's quartets of the cluster's row.
1131
+ ACC_TYPE4 dot4[MQ_GQA];
1132
+ #pragma unroll
1133
+ for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f);
1134
+ #pragma unroll
1135
+ for (int i = 0; i < FA_CL_DKQ; ++i) {
1136
+ const int qk = lic + FA_CL_C * i;
1137
+ const float4 k_v = dequant_q8_0_lane(k_row + (qk / 8) * Q8_0_BLOCK_SIZE, qk % 8);
1138
+ #pragma unroll
1139
+ for (int h = 0; h < MQ_GQA; ++h) {
1140
+ dot4[h] = mad(q_shared[h * DK_VEC + qk], k_v, dot4[h]);
1141
+ }
1142
+ }
1143
+
1144
+ // Cluster-reduce (xor steps < FA_CL_C stay inside the cluster) + score.
1145
+ ACC_TYPE score[MQ_GQA];
1146
+ #pragma unroll
1147
+ for (int h = 0; h < MQ_GQA; ++h) {
1148
+ ACC_TYPE s = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
1149
+ #pragma unroll
1150
+ for (int step = 1; step < FA_CL_C; step <<= 1) {
1151
+ s += sub_group_shuffle_xor(s, step);
1152
+ }
1153
+ s *= scale;
1154
+ if (mask_base[h] != NULL) {
1155
+ const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
1156
+ s += slope[h] * (ACC_TYPE) mask_ptr[k_safe];
1157
+ }
1158
+ if (logit_softcap > 0.0f) {
1159
+ s = logit_softcap * tanh(s / logit_softcap);
1160
+ }
1161
+ score[h] = valid ? s : FA_M_INIT;
1162
+ }
1163
+
1164
+ // Per-cluster online update (serial chain depth n_iter, not kv_per_sg).
1165
+ ACC_TYPE p_h[MQ_GQA];
1166
+ ACC_TYPE sp_h[MQ_GQA];
1167
+ #pragma unroll
1168
+ for (int h = 0; h < MQ_GQA; ++h) {
1169
+ const ACC_TYPE m_new = max(m_i[h], score[h]);
1170
+ sp_h[h] = native_exp(m_i[h] - m_new);
1171
+ p_h[h] = native_exp(score[h] - m_new);
1172
+ l_i[h] = l_i[h] * sp_h[h] + p_h[h];
1173
+ m_i[h] = m_new;
1174
+ }
1175
+
1176
+ // V accumulate on this lane's quartets (p = 0 on tail -> inert).
1177
+ #pragma unroll
1178
+ for (int i = 0; i < FA_CL_DVQ; ++i) {
1179
+ const int dv = lic + FA_CL_C * i;
1180
+ const float4 v_v = dequant_q8_0_lane(v_row + (dv / 8) * Q8_0_BLOCK_SIZE, dv % 8);
1181
+ #pragma unroll
1182
+ for (int h = 0; h < MQ_GQA; ++h) {
1183
+ o_acc[h][i] = mad(p_h[h], v_v, o_acc[h][i] * sp_h[h]);
1184
+ }
1185
+ }
1186
+ }
1187
+
1188
+ // Merge stage 1: fold cluster partials inside the subgroup via shuffles.
1189
+ #pragma unroll
1190
+ for (int h = 0; h < MQ_GQA; ++h) {
1191
+ ACC_TYPE m_c = m_i[h];
1192
+ #pragma unroll
1193
+ for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) {
1194
+ m_c = max(m_c, sub_group_shuffle_xor(m_c, step));
1195
+ }
1196
+ const ACC_TYPE alpha = native_exp(m_i[h] - m_c);
1197
+ ACC_TYPE l_c = l_i[h] * alpha;
1198
+ #pragma unroll
1199
+ for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) {
1200
+ l_c += sub_group_shuffle_xor(l_c, step);
1201
+ }
1202
+ #pragma unroll
1203
+ for (int i = 0; i < FA_CL_DVQ; ++i) {
1204
+ ACC_TYPE4 o = o_acc[h][i] * alpha;
1205
+ #pragma unroll
1206
+ for (int step = FA_CL_C; step < Q1_WG_SIZE; step <<= 1) {
1207
+ o.s0 += sub_group_shuffle_xor(o.s0, step);
1208
+ o.s1 += sub_group_shuffle_xor(o.s1, step);
1209
+ o.s2 += sub_group_shuffle_xor(o.s2, step);
1210
+ o.s3 += sub_group_shuffle_xor(o.s3, step);
1211
+ }
1212
+ o_acc[h][i] = o;
1213
+ }
1214
+ m_i[h] = m_c;
1215
+ l_i[h] = l_c;
1216
+ }
1217
+
1218
+ // Merge stage 2: baseline cross-subgroup LDS merge (o published by
1219
+ // cluster 0's lanes; layout identical to the baseline sg_o).
1220
+ __local ACC_TYPE sg_m[MQ_GQA][MQ_NSG_SPLIT];
1221
+ __local ACC_TYPE sg_l[MQ_GQA][MQ_NSG_SPLIT];
1222
+ __local ACC_TYPE4 sg_o[MQ_NSG_SPLIT][DV_VEC];
1223
+
1224
+ if (tid_sg == 0) {
1225
+ #pragma unroll
1226
+ for (int h = 0; h < MQ_GQA; ++h) {
1227
+ sg_m[h][sgid] = m_i[h];
1228
+ sg_l[h][sgid] = l_i[h];
1229
+ }
1230
+ }
1231
+
1232
+ #pragma unroll
1233
+ for (int h = 0; h < MQ_GQA; ++h) {
1234
+ if (cl == 0) {
1235
+ #pragma unroll
1236
+ for (int i = 0; i < FA_CL_DVQ; ++i) {
1237
+ sg_o[sgid][lic + FA_CL_C * i] = o_acc[h][i];
1238
+ }
1239
+ }
1240
+ barrier(CLK_LOCAL_MEM_FENCE);
1241
+
1242
+ if (sgid == 0) {
1243
+ const int head_idx = head_kv_idx * MQ_GQA + h;
1244
+
1245
+ ACC_TYPE m_c = sg_m[h][0];
1246
+ #pragma unroll
1247
+ for (int s = 1; s < MQ_NSG_SPLIT; ++s) {
1248
+ m_c = max(m_c, sg_m[h][s]);
1249
+ }
1250
+ ACC_TYPE l_c = 0.0f;
1251
+ #pragma unroll
1252
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
1253
+ l_c += sg_l[h][s] * native_exp(sg_m[h][s] - m_c);
1254
+ }
1255
+
1256
+ const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
1257
+ * n_splits + split_idx);
1258
+ global float * rec = partial_void + rec_idx * record_stride;
1259
+ global float4 * rec_o = (global float4 *) (rec + 2);
1260
+
1261
+ if (tid_sg == 0) {
1262
+ rec[0] = (float) m_c;
1263
+ rec[1] = (float) l_c;
1264
+ }
1265
+ for (int dv_idx = tid_sg; dv_idx < DV_VEC; dv_idx += Q1_WG_SIZE) {
1266
+ ACC_TYPE4 o_merged = (ACC_TYPE4)(0.0f);
1267
+ #pragma unroll
1268
+ for (int s = 0; s < MQ_NSG_SPLIT; ++s) {
1269
+ const ACC_TYPE alpha = native_exp(sg_m[h][s] - m_c);
1270
+ o_merged = mad((ACC_TYPE4)(alpha), sg_o[s][dv_idx], o_merged);
1271
+ }
1272
+ rec_o[dv_idx] = o_merged;
1273
+ }
1274
+ }
1275
+ barrier(CLK_LOCAL_MEM_FENCE);
1276
+ }
1277
+ }
1278
+
1279
+ #endif // DK_VEC/DV_VEC divisible by FA_CL_C
1280
+ #endif // HAS_SUBGROUP_SHUFFLE (q1_vec_mq_split_c8)
1281
+
1282
+ __kernel void flash_attn_f32_q8_0(
1283
+ const global void * q_void, ulong q_offset,
1284
+ const global void * k_void, ulong k_offset,
1285
+ const global void * v_void, ulong v_offset,
1286
+ global void * o_void, ulong o_offset,
1287
+ const float scale,
1288
+ const int n_q,
1289
+ const int n_kv,
1290
+ const int is_causal,
1291
+ const int n_head,
1292
+ const ulong q_nb1, const ulong q_nb2, const ulong q_nb3,
1293
+ const ulong k_nb1, const ulong k_nb2, const ulong k_nb3,
1294
+ const ulong v_nb1, const ulong v_nb2, const ulong v_nb3,
1295
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
1296
+ const float max_bias,
1297
+ const float m0,
1298
+ const float m1,
1299
+ const int n_head_log2,
1300
+ const float logit_softcap,
1301
+ const int n_head_kv,
1302
+ const global void* mask_void,
1303
+ const ulong mask_offset,
1304
+ const ulong mask_nb1,
1305
+ const ulong mask_nb2,
1306
+ const ulong mask_nb3,
1307
+ const int mask_ne2,
1308
+ const int mask_ne3,
1309
+ const global void* sinks_void,
1310
+ const ulong sinks_offset,
1311
+ // blk: per-(qblock,kvblock) class from flash_attn_blk_f16
1312
+ // (0=masked, 1=mixed, 2=unmasked). NULL disables the prepass opt.
1313
+ const global void * blk_void
1314
+ ) {
1315
+ const int tid = get_local_id(0);
1316
+ const int block_q_idx = get_group_id(0);
1317
+ const int head_batch_idx = get_global_id(1);
1318
+
1319
+ #if N_SPLIT > 1
1320
+ const int q_lane = tid / N_SPLIT;
1321
+ const int split_idx = tid % N_SPLIT;
1322
+ #else
1323
+ const int q_lane = tid;
1324
+ const int split_idx = 0;
1325
+ #endif
1326
+ const int my_query_row = block_q_idx * BLOCK_M + q_lane;
1327
+ const int query_valid = my_query_row < n_q;
1328
+
1329
+ const int batch_idx = head_batch_idx / n_head;
1330
+ const int head_idx = head_batch_idx % n_head;
1331
+
1332
+ const int gqa_ratio = n_head / n_head_kv;
1333
+ const int head_kv_idx = head_idx / gqa_ratio;
1334
+ const int mask_head_idx = mask_void != NULL ? head_idx % mask_ne2 : 0;
1335
+ const int mask_batch_idx = mask_void != NULL ? batch_idx % mask_ne3 : 0;
1336
+
1337
+ const global char * q_base = (const global char *) q_void + q_offset;
1338
+ const global char * k_base = (const global char *) k_void + k_offset;
1339
+ const global char * v_base = (const global char *) v_void + v_offset;
1340
+ global char * o_base = (global char *) o_void + o_offset;
1341
+
1342
+ const global char * mask_base = NULL;
1343
+ if (mask_void != NULL) {
1344
+ mask_base = (const global char *) mask_void + mask_offset +
1345
+ mask_batch_idx * mask_nb3 + mask_head_idx * mask_nb2;
1346
+ }
1347
+
1348
+ // BLK_PREPASS_BM may differ from this kernel's BLOCK_M; scale q-block idx.
1349
+ #ifndef BLK_PREPASS_BM
1350
+ #define BLK_PREPASS_BM BLOCK_M
1351
+ #endif
1352
+ const global char * blk_base = NULL;
1353
+ int n_kv_blocks = 0;
1354
+ if (blk_void != NULL) {
1355
+ n_kv_blocks = (n_kv + BLOCK_N - 1) / BLOCK_N;
1356
+ const int n_q_blocks_prepass = (n_q + BLK_PREPASS_BM - 1) / BLK_PREPASS_BM;
1357
+ const int prepass_q_block = (block_q_idx * BLOCK_M) / BLK_PREPASS_BM;
1358
+ blk_base = (const global char *) blk_void +
1359
+ (((mask_batch_idx * mask_ne2) + mask_head_idx) * n_q_blocks_prepass + prepass_q_block) * n_kv_blocks;
1360
+ }
1361
+
1362
+ const int dk_off_vec = split_idx * SPLIT_DK_VEC;
1363
+ ACC_TYPE4 q_priv[SPLIT_DK_VEC];
1364
+ if (query_valid) {
1365
+ const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + my_query_row * q_nb1;
1366
+ const global float4 * q_ptr = (const global float4 *) (q_base + q_row_offset);
1367
+ #pragma unroll
1368
+ for (int i = 0; i < SPLIT_DK_VEC; ++i) {
1369
+ q_priv[i] = q_ptr[dk_off_vec + i];
1370
+ }
1371
+ } else {
1372
+ #pragma unroll
1373
+ for (int i = 0; i < SPLIT_DK_VEC; ++i) q_priv[i] = (ACC_TYPE4)(0.0f);
1374
+ }
1375
+
1376
+ #ifdef FA_HAVE_INT_DOT
1377
+ uint q_packed_pf[SPLIT_DK_Q8_BLOCKS * 8];
1378
+ float q_d_pf[SPLIT_DK_Q8_BLOCKS];
1379
+ #pragma unroll
1380
+ for (int b = 0; b < SPLIT_DK_Q8_BLOCKS; ++b) {
1381
+ q_d_pf[b] = quant_q_block_int8_packed(&q_priv[b * 8], &q_packed_pf[b * 8]);
1382
+ }
1383
+ #endif
1384
+
1385
+ const int dv_off_vec = split_idx * SPLIT_DV_VEC;
1386
+ ACC_TYPE4 o_acc[SPLIT_DV_VEC];
1387
+ #pragma unroll
1388
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] = (ACC_TYPE4)(0.0f);
1389
+
1390
+ ACC_TYPE m_i = FA_M_INIT;
1391
+ ACC_TYPE l_i = 0.0f;
1392
+
1393
+ float slope = get_alibi_slope(max_bias, head_idx, n_head_log2, m0, m1);
1394
+
1395
+ #ifdef FA_HAVE_INT_DOT
1396
+ // Accessors so the staging code is layout-agnostic.
1397
+ #ifdef FA_K_LDS_T
1398
+ #define FA_K_PACKED(ROW, IDX) l_k_packed[IDX][ROW]
1399
+ #define FA_K_SCALE(ROW, BLK) l_k_scale[BLK][ROW]
1400
+ #else
1401
+ #define FA_K_PACKED(ROW, IDX) l_k_packed[ROW][IDX]
1402
+ #define FA_K_SCALE(ROW, BLK) l_k_scale[ROW][BLK]
1403
+ #endif
1404
+
1405
+ #ifdef FA_K_LDS_T
1406
+ // K tile transposed: [block*8 + g][kv row] instead of [kv row][block*8 + g].
1407
+ //
1408
+ // The QK loop walks 4 KV rows at a time against the same (b, g), so in the original
1409
+ // layout those 4 values are BLOCK_N*8 uints apart and cost 4 separate 32-bit local
1410
+ // reads. Transposed they are adjacent, so they are one 128-bit read -- 4x fewer LDS
1411
+ // issues for the same bytes and no extra registers. That matters because the QK loop
1412
+ // is LDS-read-issue-bound: a wrong-math probe that kept every dp4a but cut the LDS
1413
+ // reads ran the whole kernel 41% faster (18.51 -> 10.91 ms/op), and deleting QK
1414
+ // outright only reached 10.88 -- i.e. essentially ALL of QK's cost is these reads.
1415
+ __local uint l_k_packed[DK_Q8_BLOCKS_PREFILL * 8][BLOCK_N];
1416
+ __local float l_k_scale [DK_Q8_BLOCKS_PREFILL][BLOCK_N];
1417
+ #else
1418
+ __local uint l_k_packed[BLOCK_N][DK_Q8_BLOCKS_PREFILL * 8];
1419
+ __local float l_k_scale [BLOCK_N][DK_Q8_BLOCKS_PREFILL];
1420
+ #endif
1421
+ #else
1422
+ __local half4 l_k[BLOCK_N][DK_VEC];
1423
+ #endif
1424
+
1425
+ #if FA_V_STRATEGY == 2
1426
+ __local uint l_v_packed[BLOCK_N][DV_Q8_BLOCKS_PREFILL * 8];
1427
+ __local float l_v_scale [BLOCK_N][DV_Q8_BLOCKS_PREFILL];
1428
+ #else
1429
+ __local half4 l_v[BLOCK_N][DV_VEC];
1430
+ #endif
1431
+
1432
+ for (int k_start = 0; k_start < n_kv; k_start += BLOCK_N) {
1433
+ // Skip fully-masked KV tiles (uniform branch across WG).
1434
+ char blk_cur = 1;
1435
+ if (blk_base != NULL) {
1436
+ blk_cur = blk_base[k_start / BLOCK_N];
1437
+ if (blk_cur == 0) continue;
1438
+ }
1439
+
1440
+ {
1441
+ #ifdef FA_HAVE_INT_DOT
1442
+ const int k_blocks_per_row = DK_Q8_BLOCKS_PREFILL;
1443
+ const int n_blocks_total = BLOCK_N * k_blocks_per_row;
1444
+ for (int i = tid; i < n_blocks_total; i += WG_SIZE) {
1445
+ const int row = i / k_blocks_per_row;
1446
+ const int blk = i % k_blocks_per_row;
1447
+ const int k_row_idx = k_start + row;
1448
+ if (k_row_idx < n_kv) {
1449
+ const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1;
1450
+ const global char * blk_ptr = k_base + k_row_off + blk * Q8_0_BLOCK_SIZE;
1451
+ const float df = (float) vload_half(0, (const global half *) blk_ptr);
1452
+ const global uchar * qs = (const global uchar *)(blk_ptr + 2);
1453
+ FA_K_SCALE(row, blk) = df;
1454
+ #pragma unroll
1455
+ for (int j = 0; j < 8; ++j) {
1456
+ uint k_packed =
1457
+ (uint) qs[j*4 + 0] |
1458
+ ((uint) qs[j*4 + 1]) << 8 |
1459
+ ((uint) qs[j*4 + 2]) << 16 |
1460
+ ((uint) qs[j*4 + 3]) << 24;
1461
+ FA_K_PACKED(row, blk * 8 + j) = k_packed;
1462
+ }
1463
+ } else {
1464
+ FA_K_SCALE(row, blk) = 0.0f;
1465
+ #pragma unroll
1466
+ for (int j = 0; j < 8; ++j) FA_K_PACKED(row, blk * 8 + j) = 0u;
1467
+ }
1468
+ }
1469
+ #else
1470
+ // Fallback: dequant q8_0 -> half in local memory.
1471
+ const int k_blocks_per_row = DK / QK8_0;
1472
+ const int n_blocks_total = BLOCK_N * k_blocks_per_row;
1473
+ for (int i = tid; i < n_blocks_total; i += WG_SIZE) {
1474
+ const int row = i / k_blocks_per_row;
1475
+ const int blk = i % k_blocks_per_row;
1476
+ const int k_row_idx = k_start + row;
1477
+ if (k_row_idx < n_kv) {
1478
+ const ulong k_row_off = batch_idx * k_nb3 + head_kv_idx * k_nb2 + k_row_idx * k_nb1;
1479
+ const global char * blk_ptr = k_base + k_row_off + blk * Q8_0_BLOCK_SIZE;
1480
+ const float df = (float) vload_half(0, (const global half *) blk_ptr);
1481
+ const global char * qs = blk_ptr + 2;
1482
+ #pragma unroll
1483
+ for (int j = 0; j < 8; ++j) {
1484
+ const float4 v = df * (float4)((float) qs[j*4 + 0],
1485
+ (float) qs[j*4 + 1],
1486
+ (float) qs[j*4 + 2],
1487
+ (float) qs[j*4 + 3]);
1488
+ l_k[row][blk * 8 + j] = (half4)((half) v.s0, (half) v.s1, (half) v.s2, (half) v.s3);
1489
+ }
1490
+ } else {
1491
+ #pragma unroll
1492
+ for (int j = 0; j < 8; ++j) l_k[row][blk * 8 + j] = (half4)(0.0h);
1493
+ }
1494
+ }
1495
+ #endif
1496
+ }
1497
+ // V tile load — strategy-dependent.
1498
+ #if FA_V_STRATEGY == 2
1499
+ {
1500
+ // Int8 packed V in local memory + per-block scale. Accumulate
1501
+ // step unpacks inline.
1502
+ const int v_blocks_per_row = DV_Q8_BLOCKS_PREFILL;
1503
+ const int n_blocks_total = BLOCK_N * v_blocks_per_row;
1504
+ for (int i = tid; i < n_blocks_total; i += WG_SIZE) {
1505
+ const int row = i / v_blocks_per_row;
1506
+ const int blk = i % v_blocks_per_row;
1507
+ const int v_row_idx = k_start + row;
1508
+ if (v_row_idx < n_kv) {
1509
+ const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1;
1510
+ const global char * blk_ptr = v_base + v_row_off + blk * Q8_0_BLOCK_SIZE;
1511
+ const float df = (float) vload_half(0, (const global half *) blk_ptr);
1512
+ const global uchar * qs = (const global uchar *)(blk_ptr + 2);
1513
+ l_v_scale[row][blk] = df;
1514
+ #pragma unroll
1515
+ for (int j = 0; j < 8; ++j) {
1516
+ uint v_packed =
1517
+ (uint) qs[j*4 + 0] |
1518
+ ((uint) qs[j*4 + 1]) << 8 |
1519
+ ((uint) qs[j*4 + 2]) << 16 |
1520
+ ((uint) qs[j*4 + 3]) << 24;
1521
+ l_v_packed[row][blk * 8 + j] = v_packed;
1522
+ }
1523
+ } else {
1524
+ l_v_scale[row][blk] = 0.0f;
1525
+ #pragma unroll
1526
+ for (int j = 0; j < 8; ++j) l_v_packed[row][blk * 8 + j] = 0u;
1527
+ }
1528
+ }
1529
+ }
1530
+ #else
1531
+ {
1532
+ // Default: dequant V -> half in local memory.
1533
+ const int v_blocks_per_row = DV / QK8_0;
1534
+ const int n_blocks_total = BLOCK_N * v_blocks_per_row;
1535
+ for (int i = tid; i < n_blocks_total; i += WG_SIZE) {
1536
+ const int row = i / v_blocks_per_row;
1537
+ const int blk = i % v_blocks_per_row;
1538
+ const int v_row_idx = k_start + row;
1539
+ if (v_row_idx < n_kv) {
1540
+ const ulong v_row_off = batch_idx * v_nb3 + head_kv_idx * v_nb2 + v_row_idx * v_nb1;
1541
+ const global char * blk_ptr = v_base + v_row_off + blk * Q8_0_BLOCK_SIZE;
1542
+ const float df = (float) vload_half(0, (const global half *) blk_ptr);
1543
+ const global char * qs = blk_ptr + 2;
1544
+ #pragma unroll
1545
+ for (int j = 0; j < 8; ++j) {
1546
+ const float4 v = df * (float4)((float) qs[j*4 + 0],
1547
+ (float) qs[j*4 + 1],
1548
+ (float) qs[j*4 + 2],
1549
+ (float) qs[j*4 + 3]);
1550
+ l_v[row][blk * 8 + j] = (half4)((half) v.s0, (half) v.s1, (half) v.s2, (half) v.s3);
1551
+ }
1552
+ } else {
1553
+ #pragma unroll
1554
+ for (int j = 0; j < 8; ++j) l_v[row][blk * 8 + j] = (half4)(0.0h);
1555
+ }
1556
+ }
1557
+ }
1558
+ #endif
1559
+ barrier(CLK_LOCAL_MEM_FENCE);
1560
+
1561
+ // QK dot + online softmax. N_SPLIT>1 reduces per-thread partials via shuffle_xor.
1562
+ #if N_SPLIT > 1
1563
+ {
1564
+ #else
1565
+ if (query_valid) {
1566
+ #endif
1567
+ const int k_blk_base = split_idx * SPLIT_DK_Q8_BLOCKS;
1568
+ for (int j = 0; j < BLOCK_N; j += 4) {
1569
+ const int k_row0 = k_start + j;
1570
+ const int k_row1 = k_start + j + 1;
1571
+ const int k_row2 = k_start + j + 2;
1572
+ const int k_row3 = k_start + j + 3;
1573
+
1574
+ ACC_TYPE s0, s1, s2, s3;
1575
+ #ifdef FA_HAVE_INT_DOT
1576
+ // dp4a-accelerated QK dot over owned blocks.
1577
+ s0 = 0.0f; s1 = 0.0f; s2 = 0.0f; s3 = 0.0f;
1578
+ #pragma unroll
1579
+ for (int b_local = 0; b_local < SPLIT_DK_Q8_BLOCKS; ++b_local) {
1580
+ const int b = k_blk_base + b_local;
1581
+ int sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
1582
+ #if defined(FA_K_LDS_T)
1583
+ // The 4 KV rows are adjacent in the transposed tile, so each (b, g)
1584
+ // step is ONE 128-bit local read instead of four 32-bit ones.
1585
+ #pragma unroll
1586
+ for (int g = 0; g < 8; ++g) {
1587
+ const uint qp = q_packed_pf[b_local * 8 + g];
1588
+ const uint4 kq4 = vload4(0, &l_k_packed[b * 8 + g][j]);
1589
+ sum0 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s0, sum0);
1590
+ sum1 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s1, sum1);
1591
+ sum2 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s2, sum2);
1592
+ sum3 = dot_acc_sat_4x8packed_ss_int(qp, kq4.s3, sum3);
1593
+ }
1594
+ #else
1595
+ #pragma unroll
1596
+ for (int g = 0; g < 8; ++g) {
1597
+ const uint qp = q_packed_pf[b_local * 8 + g];
1598
+ sum0 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j ][b * 8 + g], sum0);
1599
+ sum1 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+1][b * 8 + g], sum1);
1600
+ sum2 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+2][b * 8 + g], sum2);
1601
+ sum3 = dot_acc_sat_4x8packed_ss_int(qp, l_k_packed[j+3][b * 8 + g], sum3);
1602
+ }
1603
+ #endif
1604
+ const float qd = q_d_pf[b_local];
1605
+ #ifdef FA_K_LDS_T
1606
+ const float4 ks4 = vload4(0, &l_k_scale[b][j]);
1607
+ s0 += (float)sum0 * qd * ks4.s0;
1608
+ s1 += (float)sum1 * qd * ks4.s1;
1609
+ s2 += (float)sum2 * qd * ks4.s2;
1610
+ s3 += (float)sum3 * qd * ks4.s3;
1611
+ #else
1612
+ s0 += (float)sum0 * qd * l_k_scale[j ][b];
1613
+ s1 += (float)sum1 * qd * l_k_scale[j+1][b];
1614
+ s2 += (float)sum2 * qd * l_k_scale[j+2][b];
1615
+ s3 += (float)sum3 * qd * l_k_scale[j+3][b];
1616
+ #endif
1617
+ }
1618
+ #else
1619
+ ACC_TYPE4 dot_acc0 = (ACC_TYPE4)(0.0f);
1620
+ ACC_TYPE4 dot_acc1 = (ACC_TYPE4)(0.0f);
1621
+ ACC_TYPE4 dot_acc2 = (ACC_TYPE4)(0.0f);
1622
+ ACC_TYPE4 dot_acc3 = (ACC_TYPE4)(0.0f);
1623
+ #pragma unroll
1624
+ for (int k = 0; k < SPLIT_DK_VEC; ++k) {
1625
+ const ACC_TYPE4 qk = q_priv[k];
1626
+ const int k_abs = dk_off_vec + k;
1627
+ dot_acc0 = mad(qk, CONVERT_KV_ACC4(l_k[j ][k_abs]), dot_acc0);
1628
+ dot_acc1 = mad(qk, CONVERT_KV_ACC4(l_k[j+1][k_abs]), dot_acc1);
1629
+ dot_acc2 = mad(qk, CONVERT_KV_ACC4(l_k[j+2][k_abs]), dot_acc2);
1630
+ dot_acc3 = mad(qk, CONVERT_KV_ACC4(l_k[j+3][k_abs]), dot_acc3);
1631
+ }
1632
+ s0 = dot_acc0.s0 + dot_acc0.s1 + dot_acc0.s2 + dot_acc0.s3;
1633
+ s1 = dot_acc1.s0 + dot_acc1.s1 + dot_acc1.s2 + dot_acc1.s3;
1634
+ s2 = dot_acc2.s0 + dot_acc2.s1 + dot_acc2.s2 + dot_acc2.s3;
1635
+ s3 = dot_acc3.s0 + dot_acc3.s1 + dot_acc3.s2 + dot_acc3.s3;
1636
+ #endif
1637
+
1638
+ #if N_SPLIT > 1
1639
+ // Power-of-2 N_SPLIT: shuffle_xor butterfly. N_SPLIT=3 (DK=96): 3-way shuffle.
1640
+ #if (N_SPLIT & (N_SPLIT - 1)) == 0
1641
+ #pragma unroll
1642
+ for (int step = 1; step < N_SPLIT; step <<= 1) {
1643
+ s0 += sub_group_shuffle_xor(s0, step);
1644
+ s1 += sub_group_shuffle_xor(s1, step);
1645
+ s2 += sub_group_shuffle_xor(s2, step);
1646
+ s3 += sub_group_shuffle_xor(s3, step);
1647
+ }
1648
+ #else
1649
+ const uint tri_base = (get_sub_group_local_id() / N_SPLIT) * N_SPLIT;
1650
+ s0 = sub_group_shuffle(s0, tri_base + 0) + sub_group_shuffle(s0, tri_base + 1) + sub_group_shuffle(s0, tri_base + 2);
1651
+ s1 = sub_group_shuffle(s1, tri_base + 0) + sub_group_shuffle(s1, tri_base + 1) + sub_group_shuffle(s1, tri_base + 2);
1652
+ s2 = sub_group_shuffle(s2, tri_base + 0) + sub_group_shuffle(s2, tri_base + 1) + sub_group_shuffle(s2, tri_base + 2);
1653
+ s3 = sub_group_shuffle(s3, tri_base + 0) + sub_group_shuffle(s3, tri_base + 1) + sub_group_shuffle(s3, tri_base + 2);
1654
+ #endif
1655
+ if (!query_valid) { s0 = FA_M_INIT; s1 = FA_M_INIT; s2 = FA_M_INIT; s3 = FA_M_INIT; }
1656
+ #endif
1657
+ s0 *= scale; s1 *= scale; s2 *= scale; s3 *= scale;
1658
+
1659
+ if (is_causal) {
1660
+ const int causal_limit = n_kv - n_q + my_query_row;
1661
+ if (k_row0 > causal_limit) s0 = FA_M_INIT;
1662
+ if (k_row1 > causal_limit) s1 = FA_M_INIT;
1663
+ if (k_row2 > causal_limit) s2 = FA_M_INIT;
1664
+ if (k_row3 > causal_limit) s3 = FA_M_INIT;
1665
+ }
1666
+ if (k_row0 >= n_kv) s0 = FA_M_INIT;
1667
+ if (k_row1 >= n_kv) s1 = FA_M_INIT;
1668
+ if (k_row2 >= n_kv) s2 = FA_M_INIT;
1669
+ if (k_row3 >= n_kv) s3 = FA_M_INIT;
1670
+
1671
+ if (query_valid && mask_base != NULL && blk_cur != 2) {
1672
+ const global MASK_DATA_TYPE * mask_ptr =
1673
+ (const global MASK_DATA_TYPE *) (mask_base + my_query_row * mask_nb1);
1674
+ if (k_row0 < n_kv) s0 += slope * (ACC_TYPE) mask_ptr[k_row0];
1675
+ if (k_row1 < n_kv) s1 += slope * (ACC_TYPE) mask_ptr[k_row1];
1676
+ if (k_row2 < n_kv) s2 += slope * (ACC_TYPE) mask_ptr[k_row2];
1677
+ if (k_row3 < n_kv) s3 += slope * (ACC_TYPE) mask_ptr[k_row3];
1678
+ }
1679
+ if (logit_softcap > 0.0f) {
1680
+ s0 = logit_softcap * tanh(s0 / logit_softcap);
1681
+ s1 = logit_softcap * tanh(s1 / logit_softcap);
1682
+ s2 = logit_softcap * tanh(s2 / logit_softcap);
1683
+ s3 = logit_softcap * tanh(s3 / logit_softcap);
1684
+ }
1685
+
1686
+ const ACC_TYPE m_new = max(m_i, max(max(s0, s1), max(s2, s3)));
1687
+ // Whole tile masked (m_new == FA_M_INIT): force the exp() args
1688
+ // far negative so the tile contributes 0, not exp(0)=1.
1689
+ const ACC_TYPE m_exp = (m_new == FA_M_INIT) ? 0.0f : m_new;
1690
+ const ACC_TYPE scale_prev = native_exp(m_i - m_exp);
1691
+ const ACC_TYPE p0 = native_exp(s0 - m_exp);
1692
+ const ACC_TYPE p1 = native_exp(s1 - m_exp);
1693
+ const ACC_TYPE p2 = native_exp(s2 - m_exp);
1694
+ const ACC_TYPE p3 = native_exp(s3 - m_exp);
1695
+
1696
+ #if FA_V_STRATEGY == 2
1697
+ #pragma unroll
1698
+ for (int b_local = 0; b_local < DV_Q8_BLOCKS_PREFILL / N_SPLIT; ++b_local) {
1699
+ const int b_abs = split_idx * (DV_Q8_BLOCKS_PREFILL / N_SPLIT) + b_local;
1700
+ const float d0 = l_v_scale[j ][b_abs];
1701
+ const float d1 = l_v_scale[j+1][b_abs];
1702
+ const float d2 = l_v_scale[j+2][b_abs];
1703
+ const float d3 = l_v_scale[j+3][b_abs];
1704
+ #pragma unroll
1705
+ for (int g = 0; g < 8; ++g) {
1706
+ const int lane_abs = b_abs * 8 + g;
1707
+ const int lane_local = b_local * 8 + g;
1708
+ uint pk0 = l_v_packed[j ][lane_abs];
1709
+ uint pk1 = l_v_packed[j+1][lane_abs];
1710
+ uint pk2 = l_v_packed[j+2][lane_abs];
1711
+ uint pk3 = l_v_packed[j+3][lane_abs];
1712
+ float4 v0 = d0 * (float4)((float)(char)(pk0 & 0xff), (float)(char)((pk0>>8)&0xff), (float)(char)((pk0>>16)&0xff), (float)(char)((pk0>>24)&0xff));
1713
+ float4 v1 = d1 * (float4)((float)(char)(pk1 & 0xff), (float)(char)((pk1>>8)&0xff), (float)(char)((pk1>>16)&0xff), (float)(char)((pk1>>24)&0xff));
1714
+ float4 v2 = d2 * (float4)((float)(char)(pk2 & 0xff), (float)(char)((pk2>>8)&0xff), (float)(char)((pk2>>16)&0xff), (float)(char)((pk2>>24)&0xff));
1715
+ float4 v3 = d3 * (float4)((float)(char)(pk3 & 0xff), (float)(char)((pk3>>8)&0xff), (float)(char)((pk3>>16)&0xff), (float)(char)((pk3>>24)&0xff));
1716
+ o_acc[lane_local] = mad(p3, v3,
1717
+ mad(p2, v2,
1718
+ mad(p1, v1,
1719
+ mad(p0, v0,
1720
+ o_acc[lane_local] * scale_prev))));
1721
+ }
1722
+ }
1723
+ #else // FA_V_STRATEGY == 0
1724
+ #pragma unroll
1725
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) {
1726
+ const int i_abs = dv_off_vec + i;
1727
+ o_acc[i] = mad(p3, CONVERT_KV_ACC4(l_v[j+3][i_abs]),
1728
+ mad(p2, CONVERT_KV_ACC4(l_v[j+2][i_abs]),
1729
+ mad(p1, CONVERT_KV_ACC4(l_v[j+1][i_abs]),
1730
+ mad(p0, CONVERT_KV_ACC4(l_v[j ][i_abs]),
1731
+ o_acc[i] * scale_prev))));
1732
+ }
1733
+ #endif
1734
+ l_i = l_i * scale_prev + p0 + p1 + p2 + p3;
1735
+ m_i = m_new;
1736
+ }
1737
+ }
1738
+ barrier(CLK_LOCAL_MEM_FENCE);
1739
+ }
1740
+
1741
+ // Write output. With N_SPLIT>1 each thread writes its SPLIT_DV_VEC slice.
1742
+ if (query_valid) {
1743
+ if (sinks_void != NULL) {
1744
+ const global ACC_TYPE * sinks_ptr =
1745
+ (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset);
1746
+ const ACC_TYPE m_sink = sinks_ptr[head_idx];
1747
+ const ACC_TYPE m_final = max(m_i, m_sink);
1748
+ const ACC_TYPE scale_o = exp(m_i - m_final);
1749
+ #pragma unroll
1750
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) o_acc[i] *= scale_o;
1751
+ l_i = l_i * scale_o + exp(m_sink - m_final);
1752
+ m_i = m_final;
1753
+ }
1754
+ const ACC_TYPE l_inv = (l_i > 0.0f) ? (1.0f / l_i) : 0.0f;
1755
+ const ulong o_row_offset = batch_idx * o_nb3 + my_query_row * o_nb2 + head_idx * o_nb1;
1756
+ global float4 * o_row = (global float4 *) (o_base + o_row_offset);
1757
+ if (l_inv > 0.0f) {
1758
+ #pragma unroll
1759
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = o_acc[i] * l_inv;
1760
+ } else {
1761
+ #pragma unroll
1762
+ for (int i = 0; i < SPLIT_DV_VEC; ++i) o_row[dv_off_vec + i] = (float4)(0.0f);
1763
+ }
1764
+ }
1765
+ }
1766
+
1767
+ // FD Pass 2: merge split partials. Identical across q4_0/q8_0/f16; each FA
1768
+ // source owns a copy since kernels compile per-source-program.
1769
+ __kernel void flash_attn_f32_merge(
1770
+ const global float * partial_void,
1771
+ global void * o_void,
1772
+ const ulong o_offset,
1773
+ const int n_head,
1774
+ const int n_splits,
1775
+ const ulong o_nb1, const ulong o_nb2, const ulong o_nb3,
1776
+ const global void * sinks_void,
1777
+ const ulong sinks_offset,
1778
+ const int n_q
1779
+ ) {
1780
+ const int lane = get_local_id(0);
1781
+ const int head_batch_idx = get_global_id(1);
1782
+ const int q_idx = get_global_id(2);
1783
+ const int batch_idx = head_batch_idx / n_head;
1784
+ const int head_idx = head_batch_idx % n_head;
1785
+
1786
+ const ulong record_stride = (ulong) FA_PARTIAL_FLOATS;
1787
+ const ulong record_idx_0 = (((ulong) batch_idx * n_head + head_idx) * n_q + q_idx) * n_splits;
1788
+ const global float * rec0 = partial_void + record_idx_0 * record_stride;
1789
+
1790
+ __local ACC_TYPE m_final_shared;
1791
+ __local ACC_TYPE l_final_shared;
1792
+ if (lane == 0) {
1793
+ ACC_TYPE m = FA_M_INIT;
1794
+ for (int c = 0; c < n_splits; ++c) {
1795
+ const ACC_TYPE m_c = rec0[c * record_stride + 0];
1796
+ m = max(m, m_c);
1797
+ }
1798
+ ACC_TYPE m_sink = 0.0f;
1799
+ bool has_sink = false;
1800
+ if (sinks_void != NULL) {
1801
+ const global ACC_TYPE * sinks_ptr =
1802
+ (const global ACC_TYPE *) ((const global char *) sinks_void + sinks_offset);
1803
+ m_sink = sinks_ptr[head_idx];
1804
+ has_sink = true;
1805
+ m = max(m, m_sink);
1806
+ }
1807
+ ACC_TYPE l = 0.0f;
1808
+ for (int c = 0; c < n_splits; ++c) {
1809
+ const ACC_TYPE m_c = rec0[c * record_stride + 0];
1810
+ const ACC_TYPE l_c = rec0[c * record_stride + 1];
1811
+ if (m_c > FA_M_INIT) {
1812
+ l += l_c * exp(m_c - m);
1813
+ }
1814
+ }
1815
+ if (has_sink) {
1816
+ l += exp(m_sink - m);
1817
+ }
1818
+ m_final_shared = m;
1819
+ l_final_shared = l;
1820
+ }
1821
+ barrier(CLK_LOCAL_MEM_FENCE);
1822
+ const ACC_TYPE m_final = m_final_shared;
1823
+ const ACC_TYPE l_final = l_final_shared;
1824
+ const ACC_TYPE l_inv = (l_final > 0.0f) ? (1.0f / l_final) : 0.0f;
1825
+
1826
+ ACC_TYPE4 o = (ACC_TYPE4)(0.0f);
1827
+ for (int c = 0; c < n_splits; ++c) {
1828
+ const global float * rec_c = rec0 + c * record_stride;
1829
+ const ACC_TYPE m_c = rec_c[0];
1830
+ if (m_c <= FA_M_INIT) continue;
1831
+ const global float4 * rec_oc = (const global float4 *) (rec_c + 2);
1832
+ const ACC_TYPE scale_c = exp(m_c - m_final);
1833
+ o = mad((ACC_TYPE4)(scale_c), rec_oc[lane], o);
1834
+ }
1835
+ o = o * l_inv;
1836
+
1837
+ const ulong o_row_offset = (ulong) batch_idx * o_nb3 + (ulong) q_idx * o_nb2 + (ulong) head_idx * o_nb1;
1838
+ global O_DATA_TYPE4 * o_row = (global O_DATA_TYPE4 *) ((global char *) o_void + o_offset + o_row_offset);
1839
+ o_row[lane] = CONVERT_O_DATA4(o);
1840
+ }