elelem-ggml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1367) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/exe/elelem-ggml +8 -0
  4. data/ext/elelem/ggml/elelem.c +466 -0
  5. data/ext/elelem/ggml/extconf.rb +52 -0
  6. data/lib/elelem/ggml/chat_ml.rb +23 -0
  7. data/lib/elelem/ggml/gguf.rb +82 -0
  8. data/lib/elelem/ggml/provider.rb +63 -0
  9. data/lib/elelem/ggml/tokenizer.rb +95 -0
  10. data/lib/elelem/ggml/version.rb +7 -0
  11. data/lib/elelem/ggml.rb +37 -0
  12. data/vendor/ggml/AUTHORS +335 -0
  13. data/vendor/ggml/CMakeLists.txt +499 -0
  14. data/vendor/ggml/CONTRIBUTING.md +3 -0
  15. data/vendor/ggml/LICENSE +21 -0
  16. data/vendor/ggml/README.md +49 -0
  17. data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
  18. data/vendor/ggml/cmake/GitVars.cmake +22 -0
  19. data/vendor/ggml/cmake/common.cmake +50 -0
  20. data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
  21. data/vendor/ggml/ggml.pc.in +10 -0
  22. data/vendor/ggml/include/ggml-alloc.h +86 -0
  23. data/vendor/ggml/include/ggml-backend.h +437 -0
  24. data/vendor/ggml/include/ggml-blas.h +25 -0
  25. data/vendor/ggml/include/ggml-cann.h +123 -0
  26. data/vendor/ggml/include/ggml-cpp.h +39 -0
  27. data/vendor/ggml/include/ggml-cpu.h +152 -0
  28. data/vendor/ggml/include/ggml-cuda.h +47 -0
  29. data/vendor/ggml/include/ggml-et.h +28 -0
  30. data/vendor/ggml/include/ggml-hexagon.h +19 -0
  31. data/vendor/ggml/include/ggml-metal.h +61 -0
  32. data/vendor/ggml/include/ggml-opencl.h +26 -0
  33. data/vendor/ggml/include/ggml-openvino.h +37 -0
  34. data/vendor/ggml/include/ggml-opt.h +256 -0
  35. data/vendor/ggml/include/ggml-rpc.h +35 -0
  36. data/vendor/ggml/include/ggml-sycl.h +57 -0
  37. data/vendor/ggml/include/ggml-virtgpu.h +14 -0
  38. data/vendor/ggml/include/ggml-vulkan.h +29 -0
  39. data/vendor/ggml/include/ggml-webgpu.h +19 -0
  40. data/vendor/ggml/include/ggml-zdnn.h +17 -0
  41. data/vendor/ggml/include/ggml-zendnn.h +22 -0
  42. data/vendor/ggml/include/ggml.h +2963 -0
  43. data/vendor/ggml/include/gguf.h +211 -0
  44. data/vendor/ggml/requirements.txt +12 -0
  45. data/vendor/ggml/scripts/gen-authors.sh +9 -0
  46. data/vendor/ggml/scripts/make-release-checks.sh +96 -0
  47. data/vendor/ggml/scripts/make-release-desc.sh +69 -0
  48. data/vendor/ggml/scripts/make-release-summary.txt +33 -0
  49. data/vendor/ggml/scripts/release.sh +195 -0
  50. data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
  51. data/vendor/ggml/scripts/sync-llama.last +1 -0
  52. data/vendor/ggml/scripts/sync-llama.sh +21 -0
  53. data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
  54. data/vendor/ggml/scripts/sync-whisper.last +1 -0
  55. data/vendor/ggml/scripts/sync-whisper.sh +17 -0
  56. data/vendor/ggml/src/CMakeLists.txt +634 -0
  57. data/vendor/ggml/src/ggml-alloc.c +1249 -0
  58. data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
  59. data/vendor/ggml/src/ggml-backend-dl.h +45 -0
  60. data/vendor/ggml/src/ggml-backend-impl.h +291 -0
  61. data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
  62. data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
  63. data/vendor/ggml/src/ggml-backend.cpp +2517 -0
  64. data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
  65. data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
  66. data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
  67. data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
  68. data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
  69. data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
  70. data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
  71. data/vendor/ggml/src/ggml-cann/common.h +651 -0
  72. data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
  73. data/vendor/ggml/src/ggml-common.h +1911 -0
  74. data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
  75. data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  76. data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
  77. data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
  78. data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
  79. data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  80. data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
  81. data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
  82. data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  83. data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
  84. data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  85. data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  86. data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  87. data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
  88. data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  89. data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  90. data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  91. data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
  92. data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  93. data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
  94. data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  95. data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
  96. data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  97. data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
  98. data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  99. data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
  100. data/vendor/ggml/src/ggml-cpu/common.h +95 -0
  101. data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  102. data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
  103. data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
  104. data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
  105. data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
  106. data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
  107. data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
  108. data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
  109. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
  110. data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
  111. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
  112. data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  113. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
  114. data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  115. data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
  116. data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
  117. data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
  118. data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
  119. data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
  120. data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
  121. data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  122. data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
  123. data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
  124. data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
  125. data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
  126. data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
  127. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
  128. data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
  129. data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
  130. data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
  131. data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
  132. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
  133. data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
  134. data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
  135. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
  136. data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
  137. data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
  138. data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
  139. data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
  140. data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
  141. data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
  142. data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
  143. data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
  144. data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
  145. data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
  146. data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
  147. data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
  148. data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
  149. data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
  150. data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
  151. data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
  152. data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
  153. data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
  154. data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
  155. data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
  156. data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
  157. data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
  158. data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  159. data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
  160. data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
  161. data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  162. data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  163. data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
  164. data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
  165. data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
  166. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
  167. data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  168. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  169. data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  170. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  171. data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  172. data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
  173. data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  174. data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
  175. data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
  176. data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
  177. data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  178. data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  179. data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  180. data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
  181. data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
  182. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  183. data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  184. data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
  185. data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  186. data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
  187. data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
  188. data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
  189. data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
  190. data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  191. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
  192. data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
  193. data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
  194. data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
  195. data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
  196. data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
  197. data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
  198. data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
  199. data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
  200. data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
  201. data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
  202. data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
  203. data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
  204. data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
  205. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
  206. data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
  207. data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
  208. data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
  209. data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
  210. data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
  211. data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
  212. data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
  213. data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
  214. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
  215. data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
  216. data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
  217. data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
  218. data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
  219. data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
  220. data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
  221. data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
  222. data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
  223. data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
  224. data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
  225. data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
  226. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
  227. data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
  228. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
  229. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
  230. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
  231. data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
  232. data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
  233. data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
  234. data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
  235. data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
  236. data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
  237. data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  238. data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
  239. data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
  240. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
  241. data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
  242. data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
  243. data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
  244. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  245. data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  246. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  247. data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  248. data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
  249. data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  250. data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
  251. data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
  252. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  253. data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  254. data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
  255. data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
  256. data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
  257. data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  258. data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
  259. data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
  260. data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
  261. data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
  262. data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
  263. data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
  264. data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
  265. data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
  266. data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
  267. data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
  268. data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  269. data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
  270. data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
  271. data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
  272. data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
  273. data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
  274. data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
  275. data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
  276. data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
  277. data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
  278. data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  279. data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
  280. data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  281. data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
  282. data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  283. data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
  284. data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
  285. data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
  286. data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  287. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
  288. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  289. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
  290. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  291. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
  292. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  293. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
  294. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  295. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  296. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
  297. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  298. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
  299. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
  300. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
  301. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  302. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
  303. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  304. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  305. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
  306. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  307. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
  308. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  309. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  310. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
  311. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  312. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  313. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  314. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  315. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  316. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  317. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  318. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  319. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  320. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  321. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  322. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  323. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  324. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  325. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  326. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  327. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  328. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  329. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  330. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  331. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  332. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  333. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  334. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  335. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  336. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  337. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  338. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  339. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  340. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  341. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  342. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  343. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  344. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  345. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  346. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  347. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  348. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  349. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  350. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  351. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  352. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  353. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  354. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  355. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  356. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  357. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  358. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  359. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  360. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  361. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  362. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  363. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  364. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  365. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  366. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  367. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  368. data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  369. data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
  370. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  371. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  372. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  373. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  374. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  375. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  376. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  377. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  378. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  379. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  380. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  381. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  382. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  383. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  384. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  385. data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  386. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  387. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  388. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  389. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  390. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  391. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  392. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  393. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  394. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  395. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  396. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  397. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
  398. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  399. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  400. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  401. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  402. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  403. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  404. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  405. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  406. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  407. data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  408. data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
  409. data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
  410. data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
  411. data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
  412. data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
  413. data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
  414. data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
  415. data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  416. data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
  417. data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
  418. data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
  419. data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
  420. data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
  421. data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  422. data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
  423. data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
  424. data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
  425. data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
  426. data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
  427. data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
  428. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
  429. data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
  430. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
  431. data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
  432. data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
  433. data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
  434. data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
  435. data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
  436. data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
  437. data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
  438. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
  439. data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
  440. data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
  441. data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
  442. data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
  443. data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
  444. data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
  445. data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
  446. data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
  447. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
  448. data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
  449. data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
  450. data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
  451. data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
  452. data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
  453. data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
  454. data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
  455. data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
  456. data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
  457. data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
  458. data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
  459. data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
  460. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
  461. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
  462. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
  463. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
  464. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
  465. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
  466. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
  467. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
  468. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
  469. data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
  470. data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
  471. data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
  472. data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
  473. data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
  474. data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
  475. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
  476. data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
  477. data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
  478. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
  479. data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
  480. data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
  481. data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
  482. data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
  483. data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
  484. data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
  485. data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
  486. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
  487. data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
  488. data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
  489. data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
  490. data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
  491. data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
  492. data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
  493. data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
  494. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
  495. data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
  496. data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
  497. data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
  498. data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
  499. data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
  500. data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
  501. data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
  502. data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
  503. data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
  504. data/vendor/ggml/src/ggml-feats.h +166 -0
  505. data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
  506. data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
  507. data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
  508. data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
  509. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
  510. data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
  511. data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
  512. data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
  513. data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
  514. data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
  515. data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
  516. data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
  517. data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
  518. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
  519. data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
  520. data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
  521. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
  522. data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
  523. data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
  524. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
  525. data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
  526. data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
  527. data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
  528. data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
  529. data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
  530. data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
  531. data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
  532. data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
  533. data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
  534. data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
  535. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
  536. data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
  537. data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
  538. data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
  539. data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
  540. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
  541. data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
  542. data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
  543. data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
  544. data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
  545. data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
  546. data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
  547. data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
  548. data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
  549. data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
  550. data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
  551. data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
  552. data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
  553. data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
  554. data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
  555. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
  556. data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
  557. data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
  558. data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
  559. data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
  560. data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
  561. data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
  562. data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
  563. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
  564. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
  565. data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
  566. data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
  567. data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
  568. data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
  569. data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
  570. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
  571. data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
  572. data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
  573. data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
  574. data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
  575. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
  576. data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
  577. data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
  578. data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
  579. data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
  580. data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
  581. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
  582. data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
  583. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
  584. data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
  585. data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
  586. data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
  587. data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
  588. data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
  589. data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
  590. data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
  591. data/vendor/ggml/src/ggml-impl.h +783 -0
  592. data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
  593. data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
  594. data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
  595. data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  596. data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
  597. data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
  598. data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
  599. data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
  600. data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
  601. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
  602. data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
  603. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
  604. data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
  605. data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
  606. data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
  607. data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
  608. data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
  609. data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
  610. data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
  611. data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
  612. data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
  613. data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
  614. data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
  615. data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
  616. data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
  617. data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
  618. data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
  619. data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
  620. data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
  621. data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
  622. data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
  623. data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
  624. data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
  625. data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
  626. data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
  627. data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
  628. data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
  629. data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
  630. data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
  631. data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
  632. data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
  633. data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
  634. data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
  635. data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
  636. data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
  637. data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
  638. data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
  639. data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  640. data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
  641. data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  642. data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
  643. data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
  644. data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
  645. data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
  646. data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
  647. data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
  648. data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
  649. data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  650. data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
  651. data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  652. data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
  653. data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
  654. data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
  655. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
  656. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
  657. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
  658. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
  659. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
  660. data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
  661. data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
  662. data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
  663. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
  664. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
  665. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
  666. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
  667. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
  668. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
  669. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
  670. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
  671. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
  672. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
  673. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
  674. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
  675. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
  676. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
  677. data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
  678. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
  679. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
  680. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
  681. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
  682. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
  683. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
  684. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
  685. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
  686. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
  687. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
  688. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
  689. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
  690. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
  691. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
  692. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
  693. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
  694. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
  695. data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
  696. data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
  697. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
  698. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
  699. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
  700. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
  701. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
  702. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
  703. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
  704. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
  705. data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
  706. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
  707. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
  708. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
  709. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
  710. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
  711. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
  712. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
  713. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
  714. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
  715. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
  716. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
  717. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
  718. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
  719. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
  720. data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
  721. data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
  722. data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
  723. data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
  724. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  725. data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  726. data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
  727. data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
  728. data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
  729. data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
  730. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
  731. data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
  732. data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
  733. data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
  734. data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
  735. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
  736. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
  737. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
  738. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
  739. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
  740. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
  741. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
  742. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
  743. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
  744. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
  745. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
  746. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
  747. data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
  748. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
  749. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
  750. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
  751. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
  752. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
  753. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  754. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  755. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
  756. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
  757. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
  758. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
  759. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  760. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  761. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  762. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  763. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  764. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  765. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  766. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  767. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  768. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  769. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  770. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  771. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  772. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
  773. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
  774. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  775. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  776. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  777. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  778. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  779. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
  780. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  781. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
  782. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  783. data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  784. data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  785. data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  786. data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  787. data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
  788. data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  789. data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  790. data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
  791. data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
  792. data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  793. data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  794. data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  795. data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  796. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  797. data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  798. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  799. data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  800. data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  801. data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  802. data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  803. data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  804. data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  805. data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
  806. data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  807. data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  808. data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  809. data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  810. data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
  811. data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
  812. data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
  813. data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
  814. data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
  815. data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
  816. data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
  817. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
  818. data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
  819. data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
  820. data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
  821. data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
  822. data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
  823. data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
  824. data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
  825. data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
  826. data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
  827. data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
  828. data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
  829. data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
  830. data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
  831. data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
  832. data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
  833. data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
  834. data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
  835. data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
  836. data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
  837. data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
  838. data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
  839. data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
  840. data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
  841. data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
  842. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
  843. data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
  844. data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
  845. data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
  846. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
  847. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
  848. data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
  849. data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
  850. data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
  851. data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
  852. data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
  853. data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
  854. data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
  855. data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
  856. data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
  857. data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
  858. data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
  859. data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
  860. data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
  861. data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
  862. data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
  863. data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
  864. data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
  865. data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
  866. data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
  867. data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
  868. data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
  869. data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
  870. data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
  871. data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
  872. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
  873. data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
  874. data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
  875. data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
  876. data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
  877. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
  878. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
  879. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
  880. data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
  881. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
  882. data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
  883. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
  884. data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
  885. data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
  886. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
  887. data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
  888. data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
  889. data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
  890. data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
  891. data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
  892. data/vendor/ggml/src/ggml-opt.cpp +1094 -0
  893. data/vendor/ggml/src/ggml-quants.c +5667 -0
  894. data/vendor/ggml/src/ggml-quants.h +115 -0
  895. data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
  896. data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
  897. data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
  898. data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
  899. data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
  900. data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
  901. data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
  902. data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
  903. data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
  904. data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
  905. data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
  906. data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
  907. data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
  908. data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
  909. data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
  910. data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
  911. data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
  912. data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
  913. data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
  914. data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
  915. data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
  916. data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
  917. data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
  918. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
  919. data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
  920. data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
  921. data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
  922. data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
  923. data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
  924. data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
  925. data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
  926. data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
  927. data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
  928. data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
  929. data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
  930. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
  931. data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
  932. data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
  933. data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
  934. data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
  935. data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
  936. data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
  937. data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
  938. data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  939. data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
  940. data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
  941. data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
  942. data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
  943. data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
  944. data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
  945. data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
  946. data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
  947. data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
  948. data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
  949. data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
  950. data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
  951. data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
  952. data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
  953. data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
  954. data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
  955. data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
  956. data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
  957. data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
  958. data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
  959. data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
  960. data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
  961. data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
  962. data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
  963. data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
  964. data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
  965. data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
  966. data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
  967. data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
  968. data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
  969. data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
  970. data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
  971. data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
  972. data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
  973. data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
  974. data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
  975. data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
  976. data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
  977. data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
  978. data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
  979. data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
  980. data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
  981. data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
  982. data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
  983. data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
  984. data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
  985. data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
  986. data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
  987. data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
  988. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
  989. data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
  990. data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
  991. data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
  992. data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
  993. data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
  994. data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
  995. data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  996. data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  997. data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
  998. data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
  999. data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
  1000. data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
  1001. data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
  1002. data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
  1003. data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
  1004. data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
  1005. data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
  1006. data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
  1007. data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  1008. data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  1009. data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  1010. data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  1011. data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  1012. data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  1013. data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  1014. data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  1015. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  1016. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  1017. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  1018. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  1019. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  1020. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  1021. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  1022. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  1023. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  1024. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  1025. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  1026. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  1027. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  1028. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  1029. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  1030. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  1031. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  1032. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  1033. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  1034. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  1035. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  1036. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  1037. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  1038. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  1039. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  1040. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  1041. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  1042. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  1043. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
  1044. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
  1045. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
  1046. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
  1047. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
  1048. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
  1049. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
  1050. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
  1051. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
  1052. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
  1053. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
  1054. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
  1055. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
  1056. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
  1057. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
  1058. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
  1059. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
  1060. data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
  1061. data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
  1062. data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
  1063. data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
  1064. data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  1065. data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
  1066. data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
  1067. data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
  1068. data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
  1069. data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
  1070. data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
  1071. data/vendor/ggml/src/ggml-threading.cpp +12 -0
  1072. data/vendor/ggml/src/ggml-threading.h +14 -0
  1073. data/vendor/ggml/src/ggml-version.h.in +4 -0
  1074. data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
  1075. data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
  1076. data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
  1077. data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
  1078. data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
  1079. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
  1080. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
  1081. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
  1082. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
  1083. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
  1084. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
  1085. data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
  1086. data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
  1087. data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
  1088. data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
  1089. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
  1090. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
  1091. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
  1092. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
  1093. data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
  1094. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
  1095. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
  1096. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
  1097. data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
  1098. data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
  1099. data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
  1100. data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
  1101. data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
  1102. data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
  1103. data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
  1104. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
  1105. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
  1106. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
  1107. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
  1108. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
  1109. data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
  1110. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
  1111. data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
  1112. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
  1113. data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
  1114. data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
  1115. data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
  1116. data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
  1117. data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  1118. data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
  1119. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
  1120. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
  1121. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
  1122. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
  1123. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  1124. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
  1125. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
  1126. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
  1127. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
  1128. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
  1129. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  1130. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
  1131. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  1132. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
  1133. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
  1134. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
  1135. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
  1136. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  1137. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
  1138. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
  1139. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
  1140. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  1141. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
  1142. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
  1143. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
  1144. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
  1145. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
  1146. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
  1147. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  1148. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
  1149. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
  1150. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
  1151. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  1152. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  1153. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  1154. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  1155. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
  1156. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
  1157. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
  1158. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  1159. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  1160. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  1161. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
  1162. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
  1163. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
  1164. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  1165. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  1166. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  1167. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  1168. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  1169. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  1170. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  1171. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  1172. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  1173. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
  1174. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
  1175. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
  1176. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  1177. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  1178. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
  1179. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
  1180. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
  1181. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
  1182. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
  1183. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
  1184. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
  1185. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
  1186. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
  1187. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
  1188. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
  1189. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
  1190. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
  1191. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
  1192. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
  1193. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
  1194. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
  1195. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
  1196. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
  1197. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
  1198. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
  1199. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
  1200. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
  1201. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
  1202. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
  1203. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
  1204. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
  1205. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
  1206. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
  1207. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
  1208. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
  1209. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
  1210. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  1211. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
  1212. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  1213. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  1214. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
  1215. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  1216. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  1217. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  1218. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  1219. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  1220. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  1221. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  1222. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  1223. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  1224. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  1225. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  1226. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
  1227. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  1228. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  1229. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  1230. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  1231. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  1232. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  1233. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  1234. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  1235. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
  1236. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  1237. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
  1238. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
  1239. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
  1240. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
  1241. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
  1242. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
  1243. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
  1244. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
  1245. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  1246. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  1247. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  1248. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  1249. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
  1250. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  1251. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
  1252. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
  1253. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  1254. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  1255. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  1256. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  1257. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  1258. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  1259. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  1260. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  1261. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
  1262. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
  1263. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
  1264. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
  1265. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
  1266. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
  1267. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
  1268. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
  1269. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  1270. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  1271. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
  1272. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
  1273. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
  1274. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
  1275. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
  1276. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
  1277. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
  1278. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
  1279. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
  1280. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
  1281. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  1282. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
  1283. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
  1284. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
  1285. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
  1286. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  1287. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
  1288. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
  1289. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
  1290. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
  1291. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
  1292. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
  1293. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
  1294. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
  1295. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
  1296. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
  1297. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
  1298. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  1299. data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  1300. data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
  1301. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
  1302. data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
  1303. data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
  1304. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
  1305. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
  1306. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
  1307. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
  1308. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
  1309. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
  1310. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
  1311. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
  1312. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
  1313. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
  1314. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
  1315. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
  1316. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
  1317. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
  1318. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
  1319. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
  1320. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
  1321. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
  1322. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
  1323. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
  1324. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
  1325. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
  1326. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
  1327. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
  1328. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
  1329. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
  1330. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
  1331. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
  1332. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
  1333. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
  1334. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
  1335. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
  1336. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
  1337. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
  1338. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
  1339. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
  1340. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
  1341. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
  1342. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
  1343. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
  1344. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
  1345. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
  1346. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
  1347. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
  1348. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
  1349. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
  1350. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
  1351. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
  1352. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
  1353. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
  1354. data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
  1355. data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  1356. data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
  1357. data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
  1358. data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
  1359. data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
  1360. data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
  1361. data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
  1362. data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
  1363. data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
  1364. data/vendor/ggml/src/ggml.c +8088 -0
  1365. data/vendor/ggml/src/ggml.cpp +26 -0
  1366. data/vendor/ggml/src/gguf.cpp +1706 -0
  1367. metadata +1459 -0
@@ -0,0 +1,2479 @@
1
+ #pragma clang diagnostic ignored "-Wunused-variable"
2
+ #pragma clang diagnostic ignored "-Wunused-function"
3
+ #pragma clang diagnostic ignored "-Wunused-but-set-variable"
4
+
5
+ #include <assert.h>
6
+ #include <HAP_compute_res.h>
7
+ #include <HAP_farf.h>
8
+ #include <HAP_perf.h>
9
+ #include <math.h>
10
+ #include <stdbool.h>
11
+ #include <stdatomic.h>
12
+ #include <stddef.h>
13
+ #include <stdint.h>
14
+ #include <string.h>
15
+
16
+ #include "hex-dma.h"
17
+ #include "hex-fastdiv.h"
18
+ #include "hex-profile.h"
19
+ #include "hmx-queue.h"
20
+ #include "hmx-utils.h"
21
+ #include "hvx-utils.h"
22
+ #include "hvx-dump.h"
23
+ #include "hvx-copy.h"
24
+ #include "hvx-reduce.h"
25
+ #include "hvx-flash-attn.h"
26
+ #include "htp-vtcm.h"
27
+ #include "work-queue.h"
28
+
29
+ #define GGML_COMMON_DECL_C
30
+ #include "ggml-common.h"
31
+ #include "htp-ctx.h"
32
+ #include "htp-ops.h"
33
+ #include "htp-tensor.h"
34
+ #include "hvx-quant.h"
35
+
36
+ #include "flash-attn-ops.h"
37
+ #include "hvx-fa-kernels.h"
38
+ #include "hmx-fa-kernels.h"
39
+
40
+ // Must be multiple of 32
41
+ #define FLASH_ATTN_BLOCK_SIZE (32 * 2)
42
+
43
+ struct htp_fa_context {
44
+ const struct htp_ops_context * octx;
45
+
46
+ struct fastdiv_values src0_div21;
47
+ struct fastdiv_values src0_div1;
48
+
49
+ struct fastdiv_values broadcast_rk2;
50
+ struct fastdiv_values broadcast_rk3;
51
+ struct fastdiv_values broadcast_rv2;
52
+ struct fastdiv_values broadcast_rv3;
53
+
54
+ struct fastdiv_values src3_div2;
55
+ struct fastdiv_values src3_div3;
56
+
57
+ float scale;
58
+ float max_bias;
59
+ __fp16 logit_softcap;
60
+
61
+ uint32_t n_head_log2;
62
+ float m0;
63
+ float m1;
64
+ __fp16 slopes[512];
65
+
66
+ uint32_t n_blocks;
67
+
68
+ size_t size_q_row_padded;
69
+ size_t size_k_row_padded;
70
+ size_t size_v_row_padded;
71
+
72
+ size_t size_k_block;
73
+ size_t size_v_block;
74
+ size_t size_m_block;
75
+
76
+ uint32_t qrows;
77
+ uint32_t qrows_per_thread;
78
+
79
+ bool is_q_fp32;
80
+
81
+ size_t size_q_block;
82
+ size_t size_vkq_acc;
83
+
84
+ uint8_t * spad_q;
85
+ uint8_t * spad_k;
86
+ uint8_t * spad_v;
87
+ uint8_t * spad_m;
88
+ uint8_t * spad_a;
89
+
90
+ const struct htp_tensor * k;
91
+ const struct htp_tensor * v;
92
+
93
+ uint64_t t_start;
94
+ };
95
+
96
+ struct hmx_fa_context {
97
+ const struct htp_ops_context * octx;
98
+ const struct htp_tensor * sinks; // attention sinks (src[4]), NULL if absent
99
+ const struct htp_tensor * k;
100
+ const struct htp_tensor * v;
101
+ bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2
102
+ uint32_t n_threads;
103
+
104
+ // Op parameters
105
+ __fp16 scale;
106
+ float max_bias;
107
+ __fp16 logit_softcap;
108
+ uint32_t n_head_log2;
109
+ float m0, m1;
110
+
111
+ // Dimensions
112
+ uint32_t DK, DV;
113
+ uint32_t n_kv; // kv_len
114
+ uint32_t n_kv_heads; // number of KV heads
115
+ uint32_t n_heads; // number of Q heads
116
+ uint32_t G; // GQA factor = n_heads / n_kv_heads
117
+ struct fastdiv_values div_G;
118
+ struct fastdiv_values src3_div2;
119
+ struct fastdiv_values src3_div3;
120
+ uint32_t n_kv_blocks;
121
+ uint32_t neq1; // Q token count
122
+
123
+ // Types
124
+ bool is_q_fp32;
125
+ bool is_dst_fp32;
126
+
127
+ // Dynamic block sizes
128
+ uint32_t Br; // Q tokens per block (before GQA expansion)
129
+ uint32_t Bc;
130
+ uint32_t g_br; // hex_align_up(G * Br, 32) - actual tile row dim
131
+
132
+ // VTCM buffers (allocated by vtcm_seq_alloc)
133
+ __fp16 * vtcm_q_dma; // Q DMA fetch buffer
134
+ __fp16 * vtcm_q_tiles; // Q tile format [g_br, D]
135
+ __fp16 * vtcm_o_tiles[2]; // O ping-pong [g_br, D]
136
+ __fp16 * vtcm_k_fp16[2]; // K DMA double-buffer [Bc, D]
137
+ __fp16 * vtcm_v_fp16[2]; // V DMA double-buffer [Bc, D]
138
+ __fp16 * vtcm_k_tiles[2]; // K tiles (transposed, double-buffered)
139
+ __fp16 * vtcm_v_tiles[2]; // V tiles (column-major, double-buffered)
140
+ __fp16 * vtcm_s_tiles[2]; // S = QK^T [g_br, Bc] (double-buffered)
141
+ __fp16 * vtcm_p_tiles[2]; // P = softmax(S) [g_br, Bc]
142
+ __fp16 * vtcm_d_tiles[2]; // Diagonal rescale, g_br/32 packed diagonal tiles (double-buffered)
143
+ __fp16 * vtcm_d_inv_l; // Diagonal rescale (1/l), same packed layout
144
+ HVX_Vector * vtcm_m_vec; // Row max [g_br]
145
+ HVX_Vector * vtcm_l_vec; // Row sum [g_br]
146
+ HVX_Vector * vtcm_s_rowmax; // Softmax intermediate [g_br]
147
+ HVX_Vector * vtcm_p_rowsum; // Softmax intermediate [g_br]
148
+ HVX_Vector * vtcm_row_bufs; // Per-thread softmax row scratch [n_threads][2][Bc/64]
149
+ uint8_t * vtcm_hmx_scales_id; // HMX output scales (identity)
150
+ uint8_t * vtcm_hmx_scales_qk; // HMX output scales (qk_scale)
151
+ __fp16 * vtcm_mask_buf; // VTCM mask buffer [Br * m_line], DMA'd per KV block
152
+ __fp16 * vtcm_slopes; // ALiBi slopes [g_br]
153
+ size_t row_buf_stride; // HVX vectors per row buffer (Bc/64)
154
+ size_t mask_buf_row_stride; // elements (__fp16) per row in mask buffer
155
+ size_t q_tile_bytes;
156
+ size_t o_tile_bytes;
157
+ size_t col_vec_bytes;
158
+ size_t d_tile_bytes;
159
+ bool mask_broadcast; // true when mask->ne[2] == 1 (head-independent, single 2D DMA)
160
+ dma_cache m_cache;
161
+ };
162
+
163
+ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) {
164
+ struct htp_fa_context * factx = (struct htp_fa_context *) data;
165
+ const struct htp_ops_context * octx = factx->octx;
166
+ const struct htp_tensor * q = octx->src[0];
167
+ const struct htp_tensor * k = octx->src[1];
168
+ const struct htp_tensor * v = octx->src[2];
169
+ const struct htp_tensor * mask = octx->src[3];
170
+ const struct htp_tensor * sinks = octx->src[4];
171
+ const struct htp_tensor * dst = octx->dst;
172
+
173
+ const uint32_t neq0 = q->ne[0];
174
+ const uint32_t neq1 = q->ne[1];
175
+ const uint32_t neq2 = q->ne[2];
176
+ const uint32_t neq3 = q->ne[3];
177
+
178
+ const uint32_t nek0 = k->ne[0];
179
+ const uint32_t nek1 = k->ne[1];
180
+ const uint32_t nek2 = k->ne[2];
181
+ const uint32_t nek3 = k->ne[3];
182
+
183
+ const uint32_t nev0 = v->ne[0];
184
+ const uint32_t nev1 = v->ne[1];
185
+ const uint32_t nev2 = v->ne[2];
186
+ const uint32_t nev3 = v->ne[3];
187
+
188
+ const uint32_t nbq1 = q->nb[1];
189
+ const uint32_t nbq2 = q->nb[2];
190
+ const uint32_t nbq3 = q->nb[3];
191
+
192
+ const uint32_t nbk1 = k->nb[1];
193
+ const uint32_t nbk2 = k->nb[2];
194
+ const uint32_t nbk3 = k->nb[3];
195
+
196
+ const uint32_t nbv1 = v->nb[1];
197
+ const uint32_t nbv2 = v->nb[2];
198
+ const uint32_t nbv3 = v->nb[3];
199
+
200
+ const uint32_t ne1 = dst->ne[1];
201
+ const uint32_t ne2 = dst->ne[2];
202
+ const uint32_t ne3 = dst->ne[3];
203
+
204
+ const uint32_t nb1 = dst->nb[1];
205
+ const uint32_t nb2 = dst->nb[2];
206
+ const uint32_t nb3 = dst->nb[3];
207
+
208
+ // total rows in q
209
+ const uint32_t nr = factx->qrows;
210
+ const uint32_t dr = factx->qrows_per_thread;
211
+ const uint32_t ir0 = dr * ith;
212
+ const uint32_t ir1 = MIN(ir0 + dr, nr);
213
+
214
+ if (ir0 >= ir1) return;
215
+
216
+ struct htp_thread_trace * tr = &octx->ctx->trace[ith];
217
+
218
+ dma_queue * dma = octx->ctx->dma[ith];
219
+
220
+ const uint32_t DK = nek0;
221
+ const uint32_t DV = nev0;
222
+
223
+ const size_t size_q_row = DK * ((q->type == HTP_TYPE_F32) ? 4 : 2);
224
+ const size_t size_k_row = htp_tensor_get_row_size(k->type, DK);
225
+ const size_t size_v_row = htp_tensor_get_row_size(v->type, DV);
226
+
227
+ // Scratchpad buffers for Q, K, V, Mask, and VKQ32 accumulator
228
+ uint8_t * spad_q = factx->spad_q + factx->size_q_block * ith;
229
+ uint8_t * spad_k = factx->spad_k + factx->size_k_block * 2 * ith;
230
+ uint8_t * spad_v = factx->spad_v + factx->size_v_block * 2 * ith;
231
+ uint8_t * spad_m = factx->spad_m + (mask ? factx->size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * ith;
232
+ uint8_t * spad_a = factx->spad_a + factx->size_vkq_acc * ith;
233
+
234
+ dma_cache m_cache;
235
+ dma_cache_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE);
236
+
237
+ for (uint32_t ir = ir0; ir < ir1; ++ir) {
238
+ const uint32_t iq3 = fastdiv(ir, &factx->src0_div21);
239
+ const uint32_t iq2 = fastdiv(ir - iq3*neq2*neq1, &factx->src0_div1);
240
+ const uint32_t iq1 = (ir - iq3*neq2*neq1 - iq2 * neq1);
241
+
242
+ const uint32_t ik3 = fastdiv(iq3, &factx->broadcast_rk3);
243
+ const uint32_t ik2 = fastdiv(iq2, &factx->broadcast_rk2);
244
+
245
+ const uint32_t iv3 = fastdiv(iq3, &factx->broadcast_rv3);
246
+ const uint32_t iv2 = fastdiv(iq2, &factx->broadcast_rv2);
247
+
248
+ const __fp16 * mp_base = NULL;
249
+ if (mask) {
250
+ const uint32_t im2 = fastmodulo(iq2, mask->ne[2], &factx->src3_div2);
251
+ const uint32_t im3 = fastmodulo(iq3, mask->ne[3], &factx->src3_div3);
252
+ mp_base = (const __fp16 *) ((const uint8_t *) mask->data + iq1*mask->nb[1] + im2*mask->nb[2] + im3*mask->nb[3]);
253
+ }
254
+
255
+ // Precalculate next row variables if there is a next row
256
+ bool has_next_ir = (ir + 1 < ir1);
257
+ uint32_t next_ik2 = 0, next_ik3 = 0, next_iv2 = 0, next_iv3 = 0;
258
+ const uint8_t * next_q_row_ptr = NULL;
259
+ const __fp16 * next_mp_base = NULL;
260
+
261
+ const uint8_t * next_k_src0 = NULL;
262
+ const uint8_t * next_v_src0 = NULL;
263
+ const uint8_t * next_m_src0 = NULL;
264
+ uint32_t next_block_size0 = 0;
265
+
266
+ const uint8_t * next_k_src1 = NULL;
267
+ const uint8_t * next_v_src1 = NULL;
268
+ const uint8_t * next_m_src1 = NULL;
269
+ uint32_t next_block_size1 = 0;
270
+
271
+ if (has_next_ir) {
272
+ const uint32_t next_ir = ir + 1;
273
+ const uint32_t next_iq3 = fastdiv(next_ir, &factx->src0_div21);
274
+ const uint32_t next_iq2 = fastdiv(next_ir - next_iq3*neq2*neq1, &factx->src0_div1);
275
+ const uint32_t next_iq1 = (next_ir - next_iq3*neq2*neq1 - next_iq2 * neq1);
276
+
277
+ next_ik3 = fastdiv(next_iq3, &factx->broadcast_rk3);
278
+ next_ik2 = fastdiv(next_iq2, &factx->broadcast_rk2);
279
+
280
+ next_iv3 = fastdiv(next_iq3, &factx->broadcast_rv3);
281
+ next_iv2 = fastdiv(next_iq2, &factx->broadcast_rv2);
282
+
283
+ next_q_row_ptr = (const uint8_t *) q->data + (next_iq1*nbq1 + next_iq2*nbq2 + next_iq3*nbq3);
284
+
285
+ if (mask) {
286
+ const uint32_t next_im2 = fastmodulo(next_iq2, mask->ne[2], &factx->src3_div2);
287
+ const uint32_t next_im3 = fastmodulo(next_iq3, mask->ne[3], &factx->src3_div3);
288
+ next_mp_base = (const __fp16 *) ((const uint8_t *) mask->data + next_iq1*mask->nb[1] + next_im2*mask->nb[2] + next_im3*mask->nb[3]);
289
+ }
290
+
291
+ // Precalculate next K/V block 0 source pointers
292
+ {
293
+ const uint32_t ic_start = 0;
294
+ next_block_size0 = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);
295
+ next_k_src0 = (const uint8_t *) k->data + (ic_start*nbk1 + next_ik2*nbk2 + next_ik3*nbk3);
296
+ next_v_src0 = (const uint8_t *) v->data + (ic_start*nbv1 + next_iv2*nbv2 + next_iv3*nbv3);
297
+ if (mask) {
298
+ next_m_src0 = (const uint8_t *) (next_mp_base + ic_start);
299
+ }
300
+ }
301
+
302
+ // Precalculate next K/V block 1 source pointers (if n_blocks > 1)
303
+ if (factx->n_blocks > 1) {
304
+ const uint32_t ic_start = 1 * FLASH_ATTN_BLOCK_SIZE;
305
+ next_block_size1 = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);
306
+ next_k_src1 = (const uint8_t *) k->data + (ic_start*nbk1 + next_ik2*nbk2 + next_ik3*nbk3);
307
+ next_v_src1 = (const uint8_t *) v->data + (ic_start*nbv1 + next_iv2*nbv2 + next_iv3*nbv3);
308
+ if (mask) {
309
+ next_m_src1 = (const uint8_t *) (next_mp_base + ic_start);
310
+ }
311
+ }
312
+ }
313
+
314
+ if (ir == ir0) {
315
+ // Fetch Q row
316
+ const uint8_t * q_row_ptr = (const uint8_t *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3);
317
+ dma_queue_push(dma, dma_make_ptr(spad_q, q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1);
318
+
319
+ // Prefetch first two blocks
320
+ for (uint32_t ib = 0; ib < MIN(factx->n_blocks, 2); ++ib) {
321
+ const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE;
322
+ const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);
323
+
324
+ // K
325
+ const uint8_t * k_src = (const uint8_t *) k->data + (ic_start*nbk1 + ik2*nbk2 + ik3*nbk3);
326
+ uint8_t * k_dst = spad_k + (ib % 2) * factx->size_k_block;
327
+ dma_queue_push(dma, dma_make_ptr(k_dst, k_src), factx->size_k_row_padded, nbk1, size_k_row, current_block_size);
328
+
329
+ // V
330
+ const uint8_t * v_src = (const uint8_t *) v->data + (ic_start*nbv1 + iv2*nbv2 + iv3*nbv3);
331
+ uint8_t * v_dst = spad_v + (ib % 2) * factx->size_v_block;
332
+ dma_queue_push(dma, dma_make_ptr(v_dst, v_src), factx->size_v_row_padded, nbv1, size_v_row, current_block_size);
333
+
334
+ // Mask
335
+ if (mask) {
336
+ const uint8_t * m_src = (const uint8_t *) (mp_base + ic_start);
337
+ // Mask is 1D contiguous for this row
338
+ dma_cache_push(dma, &m_cache, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1);
339
+ }
340
+ }
341
+ }
342
+
343
+ const uint32_t h = iq2; // head index
344
+ const __fp16 slope = factx->slopes[h];
345
+
346
+ HVX_Vector S_vec = hvx_vec_splat_f32(0.0f);
347
+ HVX_Vector M_vec = hvx_vec_splat_f32(HTP_FA_M_INITIAL_VAL);
348
+
349
+ // Clear accumulator
350
+ hvx_splat_f32_a(spad_a, 0, DV);
351
+ float * VKQ32 = (float *) (spad_a + 0);
352
+
353
+ uint8_t * q_ptr_vtcm = dma_queue_pop(dma).dst;
354
+ if (factx->is_q_fp32) {
355
+ hvx_copy_f16_f32_aa(q_ptr_vtcm, q_ptr_vtcm, DK); // inplace convert f32 to f16
356
+ }
357
+
358
+ const HVX_Vector slope_vec = hvx_vec_splat_f16(slope);
359
+ const HVX_Vector v_neg_inf = Q6_Vh_vsplat_R(0xfbff);
360
+ const HVX_Vector v_cap = (factx->logit_softcap != 0.0f) ? hvx_vec_splat_f16(factx->logit_softcap) : Q6_V_vzero();
361
+ const HVX_Vector vinf = Q6_Vh_vsplat_R(0xFC00);
362
+ const HVX_Vector vmin = Q6_Vh_vsplat_R(0xFBFF);
363
+ const HVX_Vector v_log2e = hvx_vec_splat_f16(EXP_LOG2E_F);
364
+ const uint32_t stride_v2 = factx->size_v_row_padded * 2;
365
+ for (uint32_t ib = 0; ib < factx->n_blocks; ++ib) {
366
+ const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE;
367
+ const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);
368
+
369
+ // Wait for DMA
370
+ uint8_t * k_base = dma_queue_pop(dma).dst; // K
371
+ uint8_t * v_base = dma_queue_pop(dma).dst; // V
372
+ __fp16 * m_base = mask ? dma_queue_pop(dma).dst : NULL; // M
373
+
374
+ if (factx->k->type == HTP_TYPE_Q8_0) {
375
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir);
376
+ for (uint32_t r = 0; r < current_block_size; ++r) {
377
+ __fp16 * row_k = (__fp16 *)(k_base + r * factx->size_k_row_padded);
378
+ hvx_dequantize_row_q8_0_f16(row_k, row_k, DK);
379
+ }
380
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir);
381
+ }
382
+ if (factx->v->type == HTP_TYPE_Q8_0) {
383
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir);
384
+ for (uint32_t r = 0; r < current_block_size; ++r) {
385
+ __fp16 * row_v = (__fp16 *)(v_base + r * factx->size_v_row_padded);
386
+ hvx_dequantize_row_q8_0_f16(row_v, row_v, DV);
387
+ }
388
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir);
389
+ }
390
+
391
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_QK, ir);
392
+
393
+ // Inner loop processing the block from VTCM
394
+ // 1. Compute scores (64 elements FP16)
395
+ HVX_Vector scores_f16 = Q6_V_vzero();
396
+ if (current_block_size > 0) {
397
+ HVX_Vector scores0 = hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base, factx->size_k_row_padded, DK, factx->scale);
398
+ HVX_Vector scores1 = (current_block_size > 32) ? hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base + 32 * factx->size_k_row_padded, factx->size_k_row_padded, DK, factx->scale) : Q6_V_vzero();
399
+ scores_f16 = hvx_vec_f32_to_f16(scores0, scores1);
400
+ }
401
+
402
+ // 2. Softcap (in FP16)
403
+ if (factx->logit_softcap != 0.0f) {
404
+ scores_f16 = hvx_vec_tanh_f16(scores_f16);
405
+ scores_f16 = hvx_vec_mul_f16_f16(scores_f16, v_cap);
406
+ }
407
+
408
+ HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(current_block_size * sizeof(__fp16));
409
+
410
+ // 3. Mask (in FP16)
411
+ if (mask) {
412
+ HVX_Vector m_vals_f16 = *(const HVX_UVector *) m_base;
413
+ HVX_VectorPred is_inf = Q6_Q_vcmp_eq_VhVh(m_vals_f16, vinf);
414
+ m_vals_f16 = Q6_V_vmux_QVV(is_inf, vmin, m_vals_f16);
415
+
416
+ HVX_Vector m_scaled = hvx_vec_mul_f16_f16(m_vals_f16, slope_vec);
417
+ scores_f16 = Q6_V_vmux_QVV(q_tail_keep, hvx_vec_add_f16_f16(scores_f16, m_scaled), v_neg_inf);
418
+ } else {
419
+ scores_f16 = Q6_V_vmux_QVV(q_tail_keep, scores_f16, v_neg_inf);
420
+ }
421
+
422
+ // Compute block max in FP16
423
+ HVX_Vector v_max_f16 = hvx_vec_reduce_max_f16(scores_f16);
424
+ HVX_Vector v_max = Q6_V_lo_W(hvx_vec_f16_to_f32(v_max_f16)); // splat block max in FP32
425
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_QK, ir);
426
+
427
+ if (ib + 1 == factx->n_blocks && has_next_ir) {
428
+ // Queue next row's Q row!
429
+ dma_queue_push(dma, dma_make_ptr(spad_q, next_q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1);
430
+
431
+ if (factx->n_blocks % 2 == 0) {
432
+ // Queue next row's block 0 (into buffer slot 0)
433
+ uint8_t * k_dst = spad_k + 0 * factx->size_k_block;
434
+ uint8_t * v_dst = spad_v + 0 * factx->size_v_block;
435
+
436
+ // K (block 0 of next row)
437
+ dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src0), factx->size_k_row_padded, nbk1, size_k_row, next_block_size0);
438
+
439
+ // V (block 0 of next row)
440
+ dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src0), factx->size_v_row_padded, nbv1, size_v_row, next_block_size0);
441
+
442
+ // Mask (block 0 of next row)
443
+ if (mask) {
444
+ dma_cache_push(dma, &m_cache, next_m_src0, next_block_size0 * 2, next_block_size0 * 2, next_block_size0 * 2, 1);
445
+ }
446
+ }
447
+ }
448
+
449
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, ir);
450
+ {
451
+ // 4. Online Softmax Update
452
+ HVX_Vector M_new_vec = Q6_Vsf_vmax_VsfVsf(v_max, M_vec);
453
+ HVX_Vector diff_vec = HVX_OP_SUB_F32(M_vec, M_new_vec);
454
+
455
+ HVX_Vector diff_f16 = hvx_vec_f32_to_f16(diff_vec, diff_vec);
456
+ HVX_Vector diff_base2 = hvx_vec_mul_f16_f16(diff_f16, v_log2e);
457
+ HVX_Vector ms_f16 = hvx_vec_exp2_f16(diff_base2);
458
+ HVX_Vector ms_vec = Q6_V_lo_W(hvx_vec_f16_to_f32(ms_f16));
459
+
460
+ M_vec = M_new_vec;
461
+
462
+ hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec);
463
+
464
+ // Compute P = exp2((S - M) * log2(e)) in FP16
465
+ HVX_Vector v_m_vec_f16 = hvx_vec_f32_to_f16(M_vec, M_vec);
466
+ HVX_Vector v_s_minus_m = Q6_Vqf16_vsub_VhfVhf(scores_f16, v_m_vec_f16);
467
+
468
+ HVX_Vector v_s_minus_m_base2 = hvx_vec_mul_f16_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m), v_log2e);
469
+
470
+ HVX_Vector P = hvx_vec_exp2_f16(v_s_minus_m_base2);
471
+ P = Q6_V_vmux_QVV(q_tail_keep, P, Q6_V_vzero());
472
+
473
+ // Convert P to FP32 to update the running sum S_vec
474
+ HVX_VectorPair P_pair = hvx_vec_f16_to_f32(P);
475
+ HVX_Vector P0 = Q6_V_lo_W(P_pair);
476
+ HVX_Vector P1 = Q6_V_hi_W(P_pair);
477
+ HVX_Vector p_sum_vec = hvx_vec_reduce_sum_f32(HVX_OP_ADD_F32(P0, P1));
478
+
479
+ S_vec = HVX_OP_ADD_F32(HVX_OP_MUL_F32(S_vec, ms_vec), p_sum_vec);
480
+
481
+ // 5. Accumulate V (F16 * F16 -> F32 accumulator)
482
+ const uint8_t * v_ptr = v_base;
483
+
484
+ for (uint32_t j = 0; j < current_block_size; j += 2) {
485
+ if (j + 1 == current_block_size) {
486
+ HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2));
487
+ hvx_mad_f32_f16_aa_vec(VKQ32, v_ptr, S0, DV);
488
+ break;
489
+ }
490
+
491
+ HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2));
492
+ HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2));
493
+
494
+ hvx_mad_f32_f16_aa_rx2_vec(VKQ32, v_ptr, v_ptr + factx->size_v_row_padded, S0, S1, DV);
495
+ v_ptr += stride_v2;
496
+ }
497
+ }
498
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, ir);
499
+
500
+ // Issue DMA for next+1 block (if exists)
501
+ if (ib + 2 < factx->n_blocks) {
502
+ const uint32_t next_ib = ib + 2;
503
+ const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE;
504
+ const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start);
505
+
506
+ // K
507
+ const uint8_t * k_src = (const uint8_t *) k->data + (next_ic_start*nbk1 + ik2*nbk2 + ik3*nbk3);
508
+ dma_queue_push(dma, dma_make_ptr(k_base, k_src), factx->size_k_row_padded, nbk1, size_k_row, next_block_size);
509
+
510
+ // V
511
+ const uint8_t * v_src = (const uint8_t *) v->data + (next_ic_start*nbv1 + iv2*nbv2 + iv3*nbv3);
512
+ dma_queue_push(dma, dma_make_ptr(v_base, v_src), factx->size_v_row_padded, nbv1, size_v_row, next_block_size);
513
+
514
+ // Mask
515
+ if (mask) {
516
+ const uint8_t * m_src = (const uint8_t *) (mp_base + next_ic_start);
517
+ dma_cache_push(dma, &m_cache, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1);
518
+ }
519
+ }
520
+ }
521
+
522
+ if (has_next_ir) {
523
+ if (factx->n_blocks % 2 == 0) {
524
+ // Queue next row's block 1 (into buffer slot 1, if n_blocks > 1)
525
+ if (factx->n_blocks > 1) {
526
+ uint8_t * k_dst = spad_k + 1 * factx->size_k_block;
527
+ uint8_t * v_dst = spad_v + 1 * factx->size_v_block;
528
+
529
+ // K (block 1 of next row)
530
+ dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src1), factx->size_k_row_padded, nbk1, size_k_row, next_block_size1);
531
+
532
+ // V (block 1 of next row)
533
+ dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src1), factx->size_v_row_padded, nbv1, size_v_row, next_block_size1);
534
+
535
+ // Mask (block 1 of next row)
536
+ if (mask) {
537
+ dma_cache_push(dma, &m_cache, next_m_src1, next_block_size1 * 2, next_block_size1 * 2, next_block_size1 * 2, 1);
538
+ }
539
+ }
540
+ } else {
541
+ // Queue next row's block 0 (into buffer slot 0)
542
+ {
543
+ uint8_t * k_dst = spad_k + 0 * factx->size_k_block;
544
+ uint8_t * v_dst = spad_v + 0 * factx->size_v_block;
545
+
546
+ // K (block 0 of next row)
547
+ dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src0), factx->size_k_row_padded, nbk1, size_k_row, next_block_size0);
548
+
549
+ // V (block 0 of next row)
550
+ dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src0), factx->size_v_row_padded, nbv1, size_v_row, next_block_size0);
551
+
552
+ // Mask (block 0 of next row)
553
+ if (mask) {
554
+ dma_cache_push(dma, &m_cache, next_m_src0, next_block_size0 * 2, next_block_size0 * 2, next_block_size0 * 2, 1);
555
+ }
556
+ }
557
+
558
+ // Queue next row's block 1 (into buffer slot 1, if n_blocks > 1)
559
+ if (factx->n_blocks > 1) {
560
+ uint8_t * k_dst = spad_k + 1 * factx->size_k_block;
561
+ uint8_t * v_dst = spad_v + 1 * factx->size_v_block;
562
+
563
+ // K (block 1 of next row)
564
+ dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src1), factx->size_k_row_padded, nbk1, size_k_row, next_block_size1);
565
+
566
+ // V (block 1 of next row)
567
+ dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src1), factx->size_v_row_padded, nbv1, size_v_row, next_block_size1);
568
+
569
+ // Mask (block 1 of next row)
570
+ if (mask) {
571
+ dma_cache_push(dma, &m_cache, next_m_src1, next_block_size1 * 2, next_block_size1 * 2, next_block_size1 * 2, 1);
572
+ }
573
+ }
574
+ }
575
+ }
576
+
577
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, ir);
578
+ // sinks
579
+ float M = hvx_vec_get_f32(M_vec);
580
+ float S = hvx_vec_get_f32(S_vec);
581
+
582
+ if (sinks) {
583
+ const float s = ((float *)((char *) sinks->data))[h];
584
+
585
+ float vs = 1.0f;
586
+
587
+ if (s > M) {
588
+ HVX_Vector diff_vec = hvx_vec_splat_f32(M - s);
589
+ HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec);
590
+ hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec);
591
+
592
+ float ms = hvx_vec_get_f32(ms_vec);
593
+ S = S * ms + vs;
594
+ } else {
595
+ HVX_Vector diff_vec = hvx_vec_splat_f32(s - M);
596
+ vs = hvx_vec_get_f32(hvx_vec_exp_f32(diff_vec));
597
+ S += vs;
598
+ }
599
+ }
600
+
601
+ const float S_inv = S == 0.0f ? 0.0f : 1.0f/S;
602
+ hvx_scale_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, S_inv);
603
+
604
+ // Store result
605
+ // dst indices
606
+ const uint32_t i1 = iq1;
607
+ const uint32_t i2 = iq2;
608
+ const uint32_t i3 = iq3;
609
+
610
+ // dst is permuted: [DV, n_heads, n_tokens, n_seq]
611
+ // head stride is nb[1], token stride is nb[2], batch stride is nb[3]
612
+ uint8_t * dst_ptr = (uint8_t *) dst->data + i2 * dst->nb[1] + i1 * dst->nb[2] + i3 * dst->nb[3];
613
+
614
+ if (dst->type == HTP_TYPE_F32) {
615
+ hvx_copy_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV);
616
+ } else if (dst->type == HTP_TYPE_F16) {
617
+ hvx_copy_f16_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV);
618
+ }
619
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, ir);
620
+ }
621
+ }
622
+
623
+ // ============================================================================
624
+ // HMX Phase args and thread logic
625
+ // ============================================================================
626
+
627
+ typedef struct {
628
+ struct hmx_fa_context * factx;
629
+ uint32_t kv_rows;
630
+ size_t src_stride;
631
+ void * curr_k;
632
+ uint32_t kv_start;
633
+ uint32_t rows_per_t;
634
+ size_t buf_idx;
635
+ } fa_k_int_args_t;
636
+
637
+ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) {
638
+ fa_k_int_args_t * args = (fa_k_int_args_t *) data;
639
+ struct hmx_fa_context * factx = args->factx;
640
+
641
+ const uint32_t total_rows = args->kv_rows;
642
+ const uint32_t rows_per_t = args->rows_per_t;
643
+ const uint32_t start = i * rows_per_t;
644
+ const uint32_t end = (uint32_t) hex_smin(start + rows_per_t, total_rows);
645
+
646
+ if (start >= total_rows) {
647
+ return;
648
+ }
649
+
650
+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
651
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start));
652
+ if (factx->k->type == HTP_TYPE_Q8_0) {
653
+ for (uint32_t r = start; r < end; ++r) {
654
+ __fp16 * row_k = (__fp16 *)((char *)args->curr_k + r * args->src_stride * sizeof(__fp16));
655
+ hvx_dequantize_row_q8_0_f16(row_k, row_k, factx->DK);
656
+ }
657
+ }
658
+ hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles[args->buf_idx], (const __fp16 *) args->curr_k, total_rows, factx->DK,
659
+ args->src_stride, start, end);
660
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start));
661
+ }
662
+
663
+ static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start, size_t buf_idx) {
664
+ work_queue_t wp = factx->octx->ctx->work_queue;
665
+ uint32_t n = 1;
666
+ if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) {
667
+ n = factx->n_threads;
668
+ }
669
+ uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2);
670
+ fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t, buf_idx };
671
+ if (n > 1) {
672
+ work_queue_run(wp, fa_k_interleave_thread, &args, n);
673
+ } else {
674
+ fa_k_interleave_thread(1, 0, &args);
675
+ }
676
+ }
677
+
678
+ typedef struct {
679
+ struct hmx_fa_context * factx;
680
+ uint32_t kv_rows;
681
+ size_t src_stride;
682
+ void * v_src;
683
+ void * v_tiles_dst;
684
+ size_t n_col_tiles;
685
+ uint32_t kv_start;
686
+ uint32_t rows_per_t;
687
+ } fa_v_int_args_t;
688
+
689
+ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) {
690
+ fa_v_int_args_t * args = (fa_v_int_args_t *) data;
691
+ struct hmx_fa_context * factx = args->factx;
692
+
693
+ const uint32_t total_rows = args->kv_rows;
694
+ const uint32_t rows_per_t = args->rows_per_t;
695
+ const uint32_t start = i * rows_per_t;
696
+ const uint32_t end = (uint32_t) hex_smin(start + rows_per_t, total_rows);
697
+
698
+ if (start >= total_rows) {
699
+ return;
700
+ }
701
+
702
+ __fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst;
703
+
704
+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
705
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start));
706
+ if (factx->v->type == HTP_TYPE_Q8_0) {
707
+ for (uint32_t r = start; r < end; ++r) {
708
+ __fp16 * row_v = (__fp16 *)((char *)args->v_src + r * args->src_stride * sizeof(__fp16));
709
+ hvx_dequantize_row_q8_0_f16(row_v, row_v, factx->DV);
710
+ }
711
+ }
712
+ hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV,
713
+ args->src_stride, (uint32_t) args->n_col_tiles, start, end);
714
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start));
715
+ }
716
+
717
+ static void fa_phase_v_interleave(struct hmx_fa_context * factx,
718
+ uint32_t kv_rows,
719
+ size_t src_stride,
720
+ void * v_src,
721
+ void * v_tiles_dst,
722
+ size_t n_col_tiles,
723
+ uint32_t kv_start) {
724
+ work_queue_t wp = factx->octx->ctx->work_queue;
725
+ uint32_t n = 1;
726
+ if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) {
727
+ n = factx->n_threads;
728
+ }
729
+ uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2);
730
+ fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t };
731
+ if (n > 1) {
732
+ work_queue_run(wp, fa_v_interleave_thread, &args, n);
733
+ } else {
734
+ fa_v_interleave_thread(1, 0, &args);
735
+ }
736
+ }
737
+
738
+ typedef struct {
739
+ struct hmx_fa_context * factx;
740
+ const struct htp_tensor * q;
741
+ uint32_t q_start;
742
+ uint32_t kv_head;
743
+ uint32_t ib3;
744
+ size_t n_rows_g;
745
+ size_t rows_per_t;
746
+ size_t n_rows_q;
747
+ bool q_transposed;
748
+ atomic_uint barrier;
749
+ } fa_q_load_args_t;
750
+
751
+ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) {
752
+ fa_q_load_args_t * args = (fa_q_load_args_t *) data;
753
+ struct hmx_fa_context * factx = args->factx;
754
+
755
+ const size_t n_rows_g = args->n_rows_g;
756
+ const size_t G = factx->G;
757
+ const size_t DK = factx->DK;
758
+
759
+ // Partition the padded Q rows (g_br) across threads.
760
+ // Keep start/end even so r and r+1 are always in the same thread's range.
761
+ const size_t rows_per_t = args->rows_per_t;
762
+ const size_t start = (size_t) i * rows_per_t;
763
+ const size_t end = hex_smin(start + rows_per_t, factx->g_br);
764
+
765
+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
766
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start));
767
+
768
+ // Parallel initialization of per-block state
769
+ {
770
+ const uint32_t g_br = factx->g_br;
771
+ const uint32_t DV = factx->DV;
772
+
773
+ const size_t col_vec_bytes = factx->col_vec_bytes;
774
+ const size_t d_tile_bytes = factx->d_tile_bytes;
775
+
776
+ // Initialize vtcm_l_vec & vtcm_m_vec
777
+ const size_t l_bytes_per_t = hex_align_up(col_vec_bytes / n, 128);
778
+ const size_t l_start = i * l_bytes_per_t;
779
+ const size_t l_end = hex_smin(l_start + l_bytes_per_t, col_vec_bytes);
780
+
781
+ const size_t m_bytes_per_t = hex_align_up(col_vec_bytes / n, 128);
782
+ const size_t m_start = i * m_bytes_per_t;
783
+ const size_t m_end = hex_smin(m_start + m_bytes_per_t, col_vec_bytes);
784
+
785
+ if (factx->sinks) {
786
+ const float * sinks_data = (const float *) (uintptr_t) factx->sinks->data;
787
+ float * m_vec = (float *) factx->vtcm_m_vec;
788
+ const size_t r_start = l_start / sizeof(float);
789
+ const size_t r_end = l_end / sizeof(float);
790
+ const float scale_factor = EXP_LOG2E_F;
791
+
792
+ const HVX_Vector v_scale = hvx_vec_splat_f32(scale_factor);
793
+
794
+ for (size_t r = r_start; r < r_end; r += 32) {
795
+ HVX_VectorAlias local_m;
796
+ for (size_t j = 0; j < 32; ++j) {
797
+ size_t curr_r = r + j;
798
+ if (curr_r < n_rows_g) {
799
+ const size_t h_idx = fastmodulo(curr_r, G, &factx->div_G);
800
+ const size_t head = args->kv_head * G + h_idx;
801
+ local_m.fp32[j] = sinks_data[head];
802
+ } else {
803
+ local_m.fp32[j] = HTP_FA_M_INITIAL_VAL;
804
+ }
805
+ }
806
+ HVX_Vector v_scaled = HVX_OP_MUL_F32(local_m.v, v_scale);
807
+ *(HVX_Vector *) (m_vec + r) = v_scaled;
808
+ }
809
+ if (l_start < col_vec_bytes) {
810
+ hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start);
811
+ }
812
+ } else {
813
+ if (l_start < col_vec_bytes) {
814
+ hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start);
815
+ }
816
+ if (m_start < col_vec_bytes) {
817
+ hvx_splat_f32_a((char *) factx->vtcm_m_vec + m_start, HTP_FA_M_INITIAL_VAL, (m_end - m_start) / sizeof(float));
818
+ }
819
+ }
820
+
821
+ // Zero the whole rescale region: vtcm_d_tiles[0], the optional vtcm_d_tiles[1]
822
+ // and vtcm_d_inv_l are equal-sized and allocated back to back, so one run covers
823
+ // them all. The scatter only ever writes the diagonal, ignore the rest.
824
+ const size_t d_bytes_per_t = hex_align_up(d_tile_bytes / n, 128);
825
+ const size_t d_start = i * d_bytes_per_t;
826
+ const size_t d_end = hex_smin(d_start + d_bytes_per_t, d_tile_bytes);
827
+ if (d_start < d_tile_bytes) {
828
+ hvx_splat_u8_a((char *) factx->vtcm_d_tiles[0] + d_start, 0, d_end - d_start);
829
+ }
830
+ }
831
+
832
+ if (start < factx->g_br) {
833
+ const struct htp_tensor * q = args->q;
834
+ const uint32_t q_start = args->q_start;
835
+ const uint32_t kv_head = args->kv_head;
836
+ const uint32_t ib3 = args->ib3;
837
+
838
+ assert(factx->DK == factx->DV);
839
+
840
+ const bool use_q_dma = (factx->vtcm_q_dma != NULL);
841
+
842
+ __fp16 * q_tiles = factx->vtcm_q_tiles;
843
+ if (use_q_dma) {
844
+ const size_t g_rows_end = hex_smin(end, n_rows_g);
845
+ const uint32_t d_limit = factx->is_q_fp32 ? DK / 32 : DK / 64;
846
+
847
+ uint8_t * q_flat = (uint8_t *) factx->vtcm_q_dma;
848
+ if (factx->is_q_fp32) {
849
+ switch (d_limit) {
850
+ case 2: hmx_fa_q_prep_fp32_d2(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break;
851
+ case 4: hmx_fa_q_prep_fp32_d4(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break;
852
+ default: hmx_fa_q_prep_fp32( q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, d_limit, args->q_transposed); break;
853
+ }
854
+ } else {
855
+ switch (d_limit) {
856
+ case 1: hmx_fa_q_prep_fp16_d1(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break;
857
+ case 2: hmx_fa_q_prep_fp16_d2(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break;
858
+ default: hmx_fa_q_prep_fp16( q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, d_limit, args->q_transposed); break;
859
+ }
860
+ }
861
+ } else {
862
+ // Fallback: direct-from-DDR/L2 path
863
+ hmx_fa_q_prep_fallback(q_tiles, q->data, q->nb[1], q->nb[2], q->nb[3],
864
+ q_start, kv_head, ib3, start, end, n_rows_g, G, DK, factx->is_q_fp32, &factx->div_G);
865
+ }
866
+ }
867
+
868
+ // Synchronize threads before zeroing out vtcm_o_tiles[0] to prevent race condition
869
+ if (n > 1) {
870
+ atomic_fetch_sub(&args->barrier, 1);
871
+ while (atomic_load(&args->barrier) > 0) {
872
+ // spin wait
873
+ }
874
+ }
875
+
876
+ // Zero out vtcm_o_tiles[0] as it was used as temp_q_vtcm
877
+ {
878
+ const uint32_t g_br = factx->g_br;
879
+ const uint32_t DV = factx->DV;
880
+ const size_t o_tile_bytes = factx->o_tile_bytes;
881
+ const size_t o_bytes_per_t = hex_align_up(o_tile_bytes / n, 128);
882
+ const size_t o_start = i * o_bytes_per_t;
883
+ const size_t o_end = hex_smin(o_start + o_bytes_per_t, o_tile_bytes);
884
+ if (o_start < o_tile_bytes) {
885
+ hvx_splat_u8_a((char *) factx->vtcm_o_tiles[0] + o_start, 0, o_end - o_start);
886
+ }
887
+ }
888
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start));
889
+ }
890
+
891
+ static void fa_phase_q_load(struct hmx_fa_context * factx,
892
+ const struct htp_tensor * q,
893
+ uint32_t q_start,
894
+ uint32_t kv_head,
895
+ uint32_t ib3,
896
+ size_t n_rows_g) {
897
+ work_queue_t wp = factx->octx->ctx->work_queue;
898
+ uint32_t n = 1;
899
+ if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) {
900
+ n = factx->n_threads;
901
+ }
902
+ size_t rows_per_t = hex_align_up(hmx_ceil_div(factx->g_br, n), 2);
903
+ const uint32_t n_rows_q = hex_smin(factx->Br, factx->neq1 - q_start);
904
+ fa_q_load_args_t args;
905
+ args.factx = factx;
906
+ args.q = q;
907
+ args.q_start = q_start;
908
+ args.kv_head = kv_head;
909
+ args.ib3 = ib3;
910
+ args.n_rows_g = n_rows_g;
911
+ args.rows_per_t = rows_per_t;
912
+ args.n_rows_q = n_rows_q;
913
+ args.q_transposed = q->nb[1] < q->nb[2];
914
+ atomic_init(&args.barrier, n);
915
+ if (n > 1) {
916
+ work_queue_run(wp, fa_q_load_thread, &args, n);
917
+ } else {
918
+ fa_q_load_thread(1, 0, &args);
919
+ }
920
+ }
921
+
922
+ typedef struct {
923
+ struct hmx_fa_context * factx;
924
+ const struct htp_tensor * dst;
925
+ const __fp16 * o_tile_src;
926
+ uint32_t q_start;
927
+ uint32_t kv_head;
928
+ uint32_t ib3;
929
+ size_t n_rows_g;
930
+ size_t rows_per_t;
931
+ } fa_o_store_args_t;
932
+
933
+ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) {
934
+ fa_o_store_args_t * args = (fa_o_store_args_t *) data;
935
+ struct hmx_fa_context * factx = args->factx;
936
+
937
+ const size_t n_rows_g = args->n_rows_g;
938
+ const size_t G = factx->G;
939
+ const size_t DV = factx->DV;
940
+
941
+ const size_t rows_per_t = args->rows_per_t;
942
+ const size_t start = (size_t) i * rows_per_t;
943
+ const size_t end = hex_smin(start + rows_per_t, n_rows_g);
944
+
945
+ if (start >= n_rows_g) {
946
+ return;
947
+ }
948
+
949
+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
950
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
951
+
952
+ const struct htp_tensor * dst = args->dst;
953
+ const __fp16 * o_tile_src = args->o_tile_src;
954
+ const uint32_t q_start = args->q_start;
955
+ const uint32_t kv_head = args->kv_head;
956
+ const uint32_t ib3 = args->ib3;
957
+
958
+ size_t q_idx = fastdiv(start, &factx->div_G);
959
+ size_t h_idx = fastmodulo(start, G, &factx->div_G);
960
+
961
+ for (size_t r = start; r < end; ++r) {
962
+ float * out = (float *) ((uint8_t *) dst->data + (kv_head * G + h_idx) * dst->nb[1] +
963
+ (q_start + q_idx) * dst->nb[2] + ib3 * dst->nb[3]);
964
+
965
+ size_t r0 = r / HMX_FP16_TILE_N_ROWS;
966
+ size_t r1 = r % HMX_FP16_TILE_N_ROWS;
967
+ const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV;
968
+
969
+ for (uint32_t d = 0; d < DV / 32; ++d) {
970
+ const HVX_Vector * in_tile = (const HVX_Vector *) (tile_row_base + d * HMX_FP16_TILE_N_ELMS);
971
+ HVX_VectorPair vp = hvx_vec_f16_to_f32_shuff(in_tile[r1 / 2]);
972
+ if (r1 % 2 == 0) {
973
+ *(HVX_UVector *) (out + d * 32) = Q6_V_lo_W(vp);
974
+ } else {
975
+ *(HVX_UVector *) (out + d * 32) = Q6_V_hi_W(vp);
976
+ }
977
+ }
978
+
979
+ h_idx++;
980
+ if (h_idx == G) {
981
+ h_idx = 0;
982
+ q_idx++;
983
+ }
984
+ }
985
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
986
+ }
987
+
988
+ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) {
989
+ fa_o_store_args_t * args = (fa_o_store_args_t *) data;
990
+ struct hmx_fa_context * factx = args->factx;
991
+
992
+ const size_t n_rows_g = args->n_rows_g;
993
+ const size_t rows_per_t = args->rows_per_t;
994
+ const size_t G = factx->G;
995
+ const size_t DV = factx->DV;
996
+ const size_t start = (size_t) i * rows_per_t;
997
+ const size_t end = hex_smin(start + rows_per_t, n_rows_g);
998
+
999
+ if (start >= n_rows_g) {
1000
+ return;
1001
+ }
1002
+
1003
+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
1004
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
1005
+
1006
+ const struct htp_tensor * dst = args->dst;
1007
+ const __fp16 * o_tile_src = args->o_tile_src;
1008
+ const uint32_t q_start = args->q_start;
1009
+ const uint32_t kv_head = args->kv_head;
1010
+ const uint32_t ib3 = args->ib3;
1011
+
1012
+ size_t q_idx = fastdiv(start, &factx->div_G);
1013
+ size_t h_idx = fastmodulo(start, G, &factx->div_G);
1014
+
1015
+ for (size_t r = start; r < end; ++r) {
1016
+ __fp16 * out = (__fp16 *) ((uint8_t *) dst->data + (kv_head * G + h_idx) * dst->nb[1] +
1017
+ (q_start + q_idx) * dst->nb[2] + ib3 * dst->nb[3]);
1018
+
1019
+ size_t r0 = r / HMX_FP16_TILE_N_ROWS;
1020
+ size_t r1 = r % HMX_FP16_TILE_N_ROWS;
1021
+ const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV;
1022
+
1023
+ for (uint32_t d = 0; d < DV / 64; ++d) {
1024
+ const __fp16 * in_dtile = tile_row_base + d * HMX_FP16_TILE_N_ELMS * 2;
1025
+ const HVX_Vector * pv_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2;
1026
+ const HVX_Vector * pv_in1 = pv_in0 + 16;
1027
+ HVX_VectorPair vp = Q6_W_vdeal_VVR(*pv_in1, *pv_in0, -2);
1028
+ if (r1 % 2 == 0) {
1029
+ *(HVX_UVector *) (out + d * 64) = Q6_V_lo_W(vp);
1030
+ } else {
1031
+ *(HVX_UVector *) (out + d * 64) = Q6_V_hi_W(vp);
1032
+ }
1033
+ }
1034
+
1035
+ h_idx++;
1036
+ if (h_idx == G) {
1037
+ h_idx = 0;
1038
+ q_idx++;
1039
+ }
1040
+ }
1041
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
1042
+ }
1043
+
1044
+ static void fa_phase_o_store(struct hmx_fa_context * factx,
1045
+ const struct htp_tensor * dst,
1046
+ const __fp16 * o_tile_src,
1047
+ uint32_t q_start,
1048
+ uint32_t kv_head,
1049
+ uint32_t ib3,
1050
+ size_t n_rows_g) {
1051
+ work_queue_t wp = factx->octx->ctx->work_queue;
1052
+ uint32_t n = 1;
1053
+ if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) {
1054
+ n = factx->n_threads;
1055
+ }
1056
+ size_t rows_per_t = hmx_ceil_div(n_rows_g, n);
1057
+ fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t };
1058
+ worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16;
1059
+ if (n > 1) {
1060
+ work_queue_run(wp, store_fn, &args, n);
1061
+ } else {
1062
+ store_fn(1, 0, &args);
1063
+ }
1064
+ }
1065
+
1066
+ typedef struct {
1067
+ struct hmx_fa_context * factx;
1068
+ size_t buf_idx;
1069
+ size_t kv_rows;
1070
+ size_t n_rows_g;
1071
+ size_t n_col_tiles;
1072
+ size_t n_tiles_per_bc;
1073
+ size_t n_row_tiles;
1074
+ size_t n_row_tiles_g_br;
1075
+ uint32_t Bc;
1076
+ uint32_t G;
1077
+ uint32_t kv_head;
1078
+ uint32_t kv_start;
1079
+ uint32_t q_start;
1080
+ uint32_t ib3;
1081
+ bool has_alibi; // true when max_bias != 0 (need slope * mask + add)
1082
+ __fp16 * slopes;
1083
+ const struct htp_tensor * mask;
1084
+ const __fp16 * mask_vtcm; // VTCM mask buffer base (NULL = DDR fallback)
1085
+ size_t mask_vtcm_row_stride; // elements (__fp16) per row in VTCM mask buffer
1086
+ struct fastdiv_values thread_div;
1087
+ } fa_softmax_args_t;
1088
+
1089
+ static inline void fa_softmax_impl(
1090
+ unsigned int n, unsigned int i, void * data,
1091
+ const bool has_mask,
1092
+ const bool mask_broadcast,
1093
+ const bool is_g1,
1094
+ const bool has_alibi,
1095
+ const bool has_softcap
1096
+ ) {
1097
+ fa_softmax_args_t * args = (fa_softmax_args_t *) data;
1098
+ struct hmx_fa_context * factx = args->factx;
1099
+
1100
+ const size_t n_rows_g = args->n_rows_g;
1101
+ const size_t kv_rows = args->kv_rows;
1102
+ const size_t Bc = args->Bc;
1103
+ const size_t G = args->G;
1104
+ const size_t n_tiles_per_bc = args->n_tiles_per_bc;
1105
+ const size_t n_row_vec_cnt = hmx_ceil_div(n_rows_g, 64);
1106
+ const uint32_t im3 = has_mask ? fastmodulo(args->ib3, args->mask->ne[3], &factx->src3_div3) : 0;
1107
+
1108
+ size_t vec_start = 0;
1109
+ size_t vec_end = n_row_vec_cnt;
1110
+ if (n > 1) {
1111
+ const size_t vecs_per_t = fastdiv(n_row_vec_cnt + n - 1, &args->thread_div);
1112
+ vec_start = i * vecs_per_t;
1113
+ vec_end = hex_smin(vec_start + vecs_per_t, n_row_vec_cnt);
1114
+ }
1115
+
1116
+ if (vec_start >= n_row_vec_cnt) {
1117
+ return;
1118
+ }
1119
+
1120
+ struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
1121
+ htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64));
1122
+
1123
+ // Per-thread row scratch: thread i uses bufs at offset i * 2 * stride
1124
+ const size_t row_buf_stride = factx->row_buf_stride;
1125
+ HVX_Vector * my_row_buf0 = factx->vtcm_row_bufs + i * 2 * row_buf_stride;
1126
+ HVX_Vector * my_row_buf1 = my_row_buf0 + row_buf_stride;
1127
+
1128
+ const HVX_Vector v_neg_inf = Q6_Vh_vsplat_R(0xfbff);
1129
+
1130
+ for (size_t r_vec_idx = vec_start; r_vec_idx < vec_end; ++r_vec_idx) {
1131
+ HVX_Vector rowmax_acc_v = v_neg_inf;
1132
+ HVX_Vector rowsum_acc_v = Q6_V_vzero();
1133
+ HVX_Vector m_prev_v0 = factx->vtcm_m_vec[r_vec_idx * 2 + 0];
1134
+ HVX_Vector m_prev_v1 = factx->vtcm_m_vec[r_vec_idx * 2 + 1];
1135
+
1136
+ HVX_Vector v_slopes = Q6_V_vzero();
1137
+ if (has_alibi) {
1138
+ v_slopes = hvx_vmem(args->slopes + r_vec_idx * 64);
1139
+ }
1140
+
1141
+ for (uint32_t r_vec_off = 0; r_vec_off < 64; r_vec_off += 2) {
1142
+ uint32_t r = r_vec_idx * 64 + r_vec_off;
1143
+ if (r >= hex_align_up(n_rows_g, 2)) {
1144
+ break;
1145
+ }
1146
+
1147
+ uint32_t r0 = r / HMX_FP16_TILE_N_ROWS;
1148
+ uint32_t r1 = r % HMX_FP16_TILE_N_ROWS;
1149
+
1150
+ const __fp16 * s_ld_base = factx->vtcm_s_tiles[args->buf_idx] + r0 * HMX_FP16_TILE_N_ROWS * Bc;
1151
+ __fp16 * p_st_base = factx->vtcm_p_tiles[args->buf_idx] + r0 * HMX_FP16_TILE_N_ROWS * Bc;
1152
+
1153
+ // Decode 2 rows from S tiles into per-thread row buffers
1154
+ if (has_softcap) {
1155
+ const HVX_Vector v_cap = hvx_vec_splat_f16(factx->logit_softcap);
1156
+ for (size_t c = 0; c < kv_rows; c += 64) {
1157
+ size_t ci = c / 64;
1158
+ const __fp16 * in_dtile = s_ld_base + ci * HMX_FP16_TILE_N_ELMS * 2;
1159
+ const HVX_Vector * pv_s_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2;
1160
+ const HVX_Vector * pv_s_in1 = pv_s_in0 + 16;
1161
+
1162
+ HVX_VectorPair vp_s_drow = Q6_W_vdeal_VVR(*pv_s_in1, *pv_s_in0, -2);
1163
+ HVX_Vector v_s_row0 = Q6_V_lo_W(vp_s_drow);
1164
+ HVX_Vector v_s_row1 = Q6_V_hi_W(vp_s_drow);
1165
+
1166
+ HVX_Vector t0 = hvx_vec_tanh_f16(v_s_row0);
1167
+ my_row_buf0[ci] = hvx_vec_mul_f16_f16(t0, v_cap);
1168
+
1169
+ HVX_Vector t1 = hvx_vec_tanh_f16(v_s_row1);
1170
+ my_row_buf1[ci] = hvx_vec_mul_f16_f16(t1, v_cap);
1171
+ }
1172
+ } else {
1173
+ size_t c = 0;
1174
+ for (; c + 64 < kv_rows; c += 128) {
1175
+ size_t ci0 = c / 64;
1176
+ size_t ci1 = ci0 + 1;
1177
+ const __fp16 * in_dtile0 = s_ld_base + ci0 * HMX_FP16_TILE_N_ELMS * 2;
1178
+ const __fp16 * in_dtile1 = s_ld_base + ci1 * HMX_FP16_TILE_N_ELMS * 2;
1179
+ const HVX_Vector * pv_s_in0_0 = ((const HVX_Vector *) in_dtile0) + r1 / 2;
1180
+ const HVX_Vector * pv_s_in1_0 = pv_s_in0_0 + 16;
1181
+ const HVX_Vector * pv_s_in0_1 = ((const HVX_Vector *) in_dtile1) + r1 / 2;
1182
+ const HVX_Vector * pv_s_in1_1 = pv_s_in0_1 + 16;
1183
+
1184
+ HVX_VectorPair vp_s_drow0 = Q6_W_vdeal_VVR(*pv_s_in1_0, *pv_s_in0_0, -2);
1185
+ my_row_buf0[ci0] = Q6_V_lo_W(vp_s_drow0);
1186
+ my_row_buf1[ci0] = Q6_V_hi_W(vp_s_drow0);
1187
+
1188
+ HVX_VectorPair vp_s_drow1 = Q6_W_vdeal_VVR(*pv_s_in1_1, *pv_s_in0_1, -2);
1189
+ my_row_buf0[ci1] = Q6_V_lo_W(vp_s_drow1);
1190
+ my_row_buf1[ci1] = Q6_V_hi_W(vp_s_drow1);
1191
+ }
1192
+ for (; c < kv_rows; c += 64) {
1193
+ size_t ci = c / 64;
1194
+ const __fp16 * in_dtile = s_ld_base + ci * HMX_FP16_TILE_N_ELMS * 2;
1195
+ const HVX_Vector * pv_s_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2;
1196
+ const HVX_Vector * pv_s_in1 = pv_s_in0 + 16;
1197
+
1198
+ HVX_VectorPair vp_s_drow = Q6_W_vdeal_VVR(*pv_s_in1, *pv_s_in0, -2);
1199
+ my_row_buf0[ci] = Q6_V_lo_W(vp_s_drow);
1200
+ my_row_buf1[ci] = Q6_V_hi_W(vp_s_drow);
1201
+ }
1202
+ }
1203
+
1204
+ // Apply mask & compute rowmax(S)
1205
+ HVX_Vector v_slope0 = Q6_V_vzero();
1206
+ HVX_Vector v_slope1 = Q6_V_vzero();
1207
+ if (has_alibi) {
1208
+ v_slope0 = hvx_vec_repl_f16(Q6_V_vror_VR(v_slopes, r_vec_off * 2));
1209
+ v_slope1 = (r + 1 < n_rows_g) ? hvx_vec_repl_f16(Q6_V_vror_VR(v_slopes, (r_vec_off + 1) * 2)) : Q6_V_vzero();
1210
+ }
1211
+
1212
+ const HVX_Vector v_threshold = Q6_Vh_vsplat_R(0xcc00); // fp16 -16.0
1213
+
1214
+ HVX_Vector v_s_rowmax0 = v_neg_inf;
1215
+ HVX_Vector v_s_rowmax1 = v_neg_inf;
1216
+ if (has_mask) {
1217
+ for (size_t c = 0; c < kv_rows; c += 64) {
1218
+ size_t ci = c / 64;
1219
+ const size_t ne = hex_smin(kv_rows - c, 64);
1220
+ HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(ne * sizeof(__fp16));
1221
+
1222
+ HVX_Vector v_mask0, v_mask1;
1223
+
1224
+ if (mask_broadcast) {
1225
+ if (is_g1) {
1226
+ const size_t qi0 = r + 0;
1227
+ v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + qi0 * args->mask_vtcm_row_stride + c);
1228
+ v_mask1 = v_neg_inf;
1229
+ if (r + 1 < n_rows_g) {
1230
+ const size_t qi1 = r + 1;
1231
+ v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + qi1 * args->mask_vtcm_row_stride + c);
1232
+ }
1233
+ } else {
1234
+ const size_t qi0 = fastdiv(r + 0, &factx->div_G);
1235
+ v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + qi0 * args->mask_vtcm_row_stride + c);
1236
+ v_mask1 = v_neg_inf;
1237
+ if (r + 1 < n_rows_g) {
1238
+ const size_t qi1 = fastdiv(r + 1, &factx->div_G);
1239
+ if (qi1 == qi0) {
1240
+ v_mask1 = v_mask0;
1241
+ } else {
1242
+ v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + qi1 * args->mask_vtcm_row_stride + c);
1243
+ }
1244
+ }
1245
+ }
1246
+ } else {
1247
+ // Head-dependent mask: pre-interleaved per row r.
1248
+ const size_t r0 = r + 0;
1249
+ v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + r0 * args->mask_vtcm_row_stride + c);
1250
+ v_mask1 = v_neg_inf;
1251
+ if (r + 1 < n_rows_g) {
1252
+ const size_t r1 = r + 1;
1253
+ v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + r1 * args->mask_vtcm_row_stride + c);
1254
+ }
1255
+ }
1256
+
1257
+ // Threshold: mask values below -16.0 are treated as -inf (causal mask).
1258
+ HVX_VectorPred q_keep0 = Q6_Q_and_QQ(Q6_Q_vcmp_gt_VhfVhf(v_mask0, v_threshold), q_tail_keep);
1259
+ HVX_VectorPred q_keep1 = Q6_Q_and_QQ(Q6_Q_vcmp_gt_VhfVhf(v_mask1, v_threshold), q_tail_keep);
1260
+
1261
+ // Scale mask values by log2(e) for base-2 calculations
1262
+ const HVX_Vector v_log2e = hvx_vec_splat_f16(EXP_LOG2E_F);
1263
+ HVX_Vector v_mask0_scaled = hvx_vec_mul_f16_f16(v_mask0, v_log2e);
1264
+ HVX_Vector v_mask1_scaled = hvx_vec_mul_f16_f16(v_mask1, v_log2e);
1265
+
1266
+ if (has_alibi) {
1267
+ HVX_Vector v_sm0 = hvx_vec_mul_f16_f16(v_mask0_scaled, v_slope0);
1268
+ HVX_Vector v_sm1 = hvx_vec_mul_f16_f16(v_mask1_scaled, v_slope1);
1269
+ my_row_buf0[ci] = Q6_V_vmux_QVV(q_keep0, hvx_vec_add_f16_f16(my_row_buf0[ci], v_sm0), v_neg_inf);
1270
+ my_row_buf1[ci] = Q6_V_vmux_QVV(q_keep1, hvx_vec_add_f16_f16(my_row_buf1[ci], v_sm1), v_neg_inf);
1271
+ } else {
1272
+ my_row_buf0[ci] = Q6_V_vmux_QVV(q_keep0, hvx_vec_add_f16_f16(my_row_buf0[ci], v_mask0_scaled), v_neg_inf);
1273
+ my_row_buf1[ci] = Q6_V_vmux_QVV(q_keep1, hvx_vec_add_f16_f16(my_row_buf1[ci], v_mask1_scaled), v_neg_inf);
1274
+ }
1275
+
1276
+ v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci]);
1277
+ v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci]);
1278
+ }
1279
+ } else {
1280
+ size_t c = 0;
1281
+ for (; c + 64 < kv_rows; c += 128) {
1282
+ size_t ci0 = c / 64;
1283
+ size_t ci1 = ci0 + 1;
1284
+ v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci0]);
1285
+ v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci0]);
1286
+ v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci1]);
1287
+ v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci1]);
1288
+ }
1289
+ for (; c < kv_rows; c += 64) {
1290
+ size_t ci = c / 64;
1291
+ const size_t ne = hex_smin(kv_rows - c, 64);
1292
+ HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(ne * sizeof(__fp16));
1293
+ if (ne < 64) {
1294
+ my_row_buf0[ci] = Q6_V_vmux_QVV(q_tail_keep, my_row_buf0[ci], v_neg_inf);
1295
+ my_row_buf1[ci] = Q6_V_vmux_QVV(q_tail_keep, my_row_buf1[ci], v_neg_inf);
1296
+ }
1297
+ v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci]);
1298
+ v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci]);
1299
+ }
1300
+ }
1301
+
1302
+ v_s_rowmax0 = hvx_vec_reduce_max_f16(v_s_rowmax0);
1303
+ v_s_rowmax1 = hvx_vec_reduce_max_f16(v_s_rowmax1);
1304
+
1305
+ // Splat m_prev[r], m_prev[r+1] from the float per-row accumulators and convert to fp16 vectors
1306
+ HVX_Vector v_m_prev0, v_m_prev1;
1307
+ if (r_vec_off < 32) {
1308
+ HVX_Vector v0 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v0, r_vec_off * 4));
1309
+ v_m_prev0 = hvx_vec_f32_to_f16(v0, v0);
1310
+ if (r + 1 < n_rows_g) {
1311
+ HVX_Vector v1 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v0, (r_vec_off + 1) * 4));
1312
+ v_m_prev1 = hvx_vec_f32_to_f16(v1, v1);
1313
+ } else {
1314
+ v_m_prev1 = Q6_V_vzero();
1315
+ }
1316
+ } else {
1317
+ HVX_Vector v0 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v1, (r_vec_off - 32) * 4));
1318
+ v_m_prev0 = hvx_vec_f32_to_f16(v0, v0);
1319
+ if (r + 1 < n_rows_g) {
1320
+ HVX_Vector v1 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v1, (r_vec_off + 1 - 32) * 4));
1321
+ v_m_prev1 = hvx_vec_f32_to_f16(v1, v1);
1322
+ } else {
1323
+ v_m_prev1 = Q6_V_vzero();
1324
+ }
1325
+ }
1326
+
1327
+ HVX_Vector v_dup_m0 = Q6_Vhf_vmax_VhfVhf(v_m_prev0, v_s_rowmax0);
1328
+ HVX_Vector v_dup_m1 = Q6_Vhf_vmax_VhfVhf(v_m_prev1, v_s_rowmax1);
1329
+
1330
+ // Insert row r, r+1 rowmax into rowmax_acc_v
1331
+ {
1332
+ HVX_VectorPred p_start = Q6_Q_vsetq_R(r_vec_off * 2);
1333
+ HVX_VectorPred p_mid = Q6_Q_vsetq_R((r_vec_off + 1) * 2);
1334
+ HVX_VectorPred p_end = Q6_Q_vsetq2_R((r_vec_off + 2) * 2);
1335
+ HVX_VectorPred p_lane0 = Q6_Q_and_QQn(p_mid, p_start);
1336
+ HVX_VectorPred p_lane1 = Q6_Q_and_QQn(p_end, p_mid);
1337
+ rowmax_acc_v = Q6_V_vmux_QVV(p_lane0, v_dup_m0, rowmax_acc_v);
1338
+ rowmax_acc_v = Q6_V_vmux_QVV(p_lane1, v_dup_m1, rowmax_acc_v);
1339
+ }
1340
+
1341
+ // Compute P = exp(S - m_new)
1342
+ const HVX_Vector v_zero = Q6_V_vzero();
1343
+ HVX_Vector v_p_rowsum0 = v_zero;
1344
+ HVX_Vector v_p_rowsum1 = v_zero;
1345
+
1346
+ size_t c = 0;
1347
+ for (; c + 64 < kv_rows; c += 128) {
1348
+ size_t ci0 = c / 64;
1349
+ size_t ci1 = ci0 + 1;
1350
+
1351
+ HVX_Vector v_s_minus_m0_0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci0], v_dup_m0);
1352
+ HVX_Vector v_s_minus_m1_0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci0], v_dup_m1);
1353
+ HVX_Vector v_s_minus_m0_1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci1], v_dup_m0);
1354
+ HVX_Vector v_s_minus_m1_1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci1], v_dup_m1);
1355
+
1356
+ HVX_Vector v_p_row0_hf_0 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m0_0));
1357
+ HVX_Vector v_p_row1_hf_0 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m1_0));
1358
+ HVX_Vector v_p_row0_hf_1 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m0_1));
1359
+ HVX_Vector v_p_row1_hf_1 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m1_1));
1360
+
1361
+ __fp16 * out_dtile0 = p_st_base + ci0 * HMX_FP16_TILE_N_ELMS * 2;
1362
+ __fp16 * out_dtile1 = p_st_base + ci1 * HMX_FP16_TILE_N_ELMS * 2;
1363
+ HVX_Vector * pv_p_out0_0 = ((HVX_Vector *) out_dtile0) + r1 / 2;
1364
+ HVX_Vector * pv_p_out1_0 = pv_p_out0_0 + 16;
1365
+ HVX_Vector * pv_p_out0_1 = ((HVX_Vector *) out_dtile1) + r1 / 2;
1366
+ HVX_Vector * pv_p_out1_1 = pv_p_out0_1 + 16;
1367
+
1368
+ HVX_VectorPair vp_p_dual0 = Q6_W_vshuff_VVR(v_p_row1_hf_0, v_p_row0_hf_0, -2);
1369
+ *pv_p_out0_0 = Q6_V_lo_W(vp_p_dual0);
1370
+ *pv_p_out1_0 = Q6_V_hi_W(vp_p_dual0);
1371
+
1372
+ HVX_VectorPair vp_p_dual1 = Q6_W_vshuff_VVR(v_p_row1_hf_1, v_p_row0_hf_1, -2);
1373
+ *pv_p_out0_1 = Q6_V_lo_W(vp_p_dual1);
1374
+ *pv_p_out1_1 = Q6_V_hi_W(vp_p_dual1);
1375
+
1376
+ HVX_VectorPair vp_p0_0 = hvx_vec_f16_to_f32_shuff(v_p_row0_hf_0);
1377
+ HVX_VectorPair vp_p1_0 = hvx_vec_f16_to_f32_shuff(v_p_row1_hf_0);
1378
+ HVX_VectorPair vp_p0_1 = hvx_vec_f16_to_f32_shuff(v_p_row0_hf_1);
1379
+ HVX_VectorPair vp_p1_1 = hvx_vec_f16_to_f32_shuff(v_p_row1_hf_1);
1380
+
1381
+ v_p_rowsum0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum0, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p0_0), Q6_V_hi_W(vp_p0_0)));
1382
+ v_p_rowsum0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum0, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p0_1), Q6_V_hi_W(vp_p0_1)));
1383
+ v_p_rowsum1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum1, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p1_0), Q6_V_hi_W(vp_p1_0)));
1384
+ v_p_rowsum1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum1, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p1_1), Q6_V_hi_W(vp_p1_1)));
1385
+ }
1386
+ for (size_t c_rem = c; c_rem < kv_rows; c_rem += 64) {
1387
+ size_t ci = c_rem / 64;
1388
+ HVX_Vector v_s_minus_m0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci], v_dup_m0);
1389
+ HVX_Vector v_s_minus_m1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci], v_dup_m1);
1390
+
1391
+ HVX_Vector v_p_row0_hf = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m0));
1392
+ HVX_Vector v_p_row1_hf = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m1));
1393
+ __fp16 * out_dtile = p_st_base + ci * HMX_FP16_TILE_N_ELMS * 2;
1394
+ HVX_Vector * pv_p_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
1395
+ HVX_Vector * pv_p_out1 = pv_p_out0 + 16;
1396
+
1397
+ HVX_VectorPair vp_p_dual = Q6_W_vshuff_VVR(v_p_row1_hf, v_p_row0_hf, -2);
1398
+ *pv_p_out0 = Q6_V_lo_W(vp_p_dual);
1399
+ *pv_p_out1 = Q6_V_hi_W(vp_p_dual);
1400
+
1401
+ HVX_VectorPair vp_p0 = hvx_vec_f16_to_f32_shuff(v_p_row0_hf);
1402
+ HVX_VectorPair vp_p1 = hvx_vec_f16_to_f32_shuff(v_p_row1_hf);
1403
+
1404
+ v_p_rowsum0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum0, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p0), Q6_V_hi_W(vp_p0)));
1405
+ v_p_rowsum1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum1, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p1), Q6_V_hi_W(vp_p1)));
1406
+ }
1407
+
1408
+ HVX_Vector rowsum0_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(v_p_rowsum0));
1409
+ HVX_Vector rowsum1_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(v_p_rowsum1));
1410
+ {
1411
+ HVX_Vector rv0_v = hvx_vec_f32_to_f16(rowsum0_sf, rowsum0_sf);
1412
+ HVX_Vector rv1_v = hvx_vec_f32_to_f16(rowsum1_sf, rowsum1_sf);
1413
+
1414
+ HVX_VectorPred p_start = Q6_Q_vsetq_R(r_vec_off * 2);
1415
+ HVX_VectorPred p_mid = Q6_Q_vsetq_R((r_vec_off + 1) * 2);
1416
+ HVX_VectorPred p_end = Q6_Q_vsetq2_R((r_vec_off + 2) * 2);
1417
+ HVX_VectorPred p_lane0 = Q6_Q_and_QQn(p_mid, p_start);
1418
+ HVX_VectorPred p_lane1 = Q6_Q_and_QQn(p_end, p_mid);
1419
+ rowsum_acc_v = Q6_V_vmux_QVV(p_lane0, rv0_v, rowsum_acc_v);
1420
+ rowsum_acc_v = Q6_V_vmux_QVV(p_lane1, rv1_v, rowsum_acc_v);
1421
+ }
1422
+ }
1423
+
1424
+ // Inline fa_ml_update_and_build_d for this vector (lock-free and in parallel)
1425
+ HVX_VectorPair rowmax_acc_pair = hvx_vec_f16_to_f32(rowmax_acc_v);
1426
+ HVX_Vector v_rowmax_acc_f32_0 = Q6_V_lo_W(rowmax_acc_pair);
1427
+ HVX_Vector v_rowmax_acc_f32_1 = Q6_V_hi_W(rowmax_acc_pair);
1428
+
1429
+ HVX_Vector v_m_curr0 = Q6_Vsf_vmax_VsfVsf(m_prev_v0, v_rowmax_acc_f32_0);
1430
+ HVX_Vector v_m_curr1 = Q6_Vsf_vmax_VsfVsf(m_prev_v1, v_rowmax_acc_f32_1);
1431
+
1432
+ HVX_Vector v_m_diff0 = HVX_OP_SUB_F32(m_prev_v0, v_m_curr0);
1433
+ HVX_Vector v_m_diff1 = HVX_OP_SUB_F32(m_prev_v1, v_m_curr1);
1434
+
1435
+ HVX_Vector v_m_diff_f16 = hvx_vec_f32_to_f16(v_m_diff0, v_m_diff1);
1436
+ HVX_Vector exp_m_diff_f16 = hvx_vec_exp2_f16(v_m_diff_f16);
1437
+
1438
+ HVX_VectorPair exp_m_diff_pair = hvx_vec_f16_to_f32(exp_m_diff_f16);
1439
+ HVX_Vector exp_m_diff0 = Q6_V_lo_W(exp_m_diff_pair);
1440
+ HVX_Vector exp_m_diff1 = Q6_V_hi_W(exp_m_diff_pair);
1441
+
1442
+ HVX_VectorPair rowsum_acc_pair = hvx_vec_f16_to_f32(rowsum_acc_v);
1443
+ HVX_Vector v_rowsum_acc_f32_0 = Q6_V_lo_W(rowsum_acc_pair);
1444
+ HVX_Vector v_rowsum_acc_f32_1 = Q6_V_hi_W(rowsum_acc_pair);
1445
+
1446
+ HVX_Vector v_l_curr0;
1447
+ HVX_Vector v_l_curr1;
1448
+ if (args->kv_start == 0 && factx->sinks != NULL) {
1449
+ // First KV block with sinks: m_prev holds the seeded sink value (not -inf),
1450
+ // so exp_m_diff = exp2(sink - m_curr) is the sink's contribution to the
1451
+ // denominator. l_prev is 0 here, so add exp_m_diff directly instead of
1452
+ // multiplying the (uninitialized) l_prev term.
1453
+ v_l_curr0 = HVX_OP_ADD_F32(exp_m_diff0, v_rowsum_acc_f32_0);
1454
+ v_l_curr1 = HVX_OP_ADD_F32(exp_m_diff1, v_rowsum_acc_f32_1);
1455
+ } else {
1456
+ HVX_Vector l_prev_v0 = factx->vtcm_l_vec[r_vec_idx * 2 + 0];
1457
+ HVX_Vector l_prev_v1 = factx->vtcm_l_vec[r_vec_idx * 2 + 1];
1458
+ v_l_curr0 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v0, exp_m_diff0), v_rowsum_acc_f32_0);
1459
+ v_l_curr1 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v1, exp_m_diff1), v_rowsum_acc_f32_1);
1460
+ }
1461
+
1462
+ factx->vtcm_m_vec[r_vec_idx * 2 + 0] = v_m_curr0;
1463
+ factx->vtcm_m_vec[r_vec_idx * 2 + 1] = v_m_curr1;
1464
+ factx->vtcm_l_vec[r_vec_idx * 2 + 0] = v_l_curr0;
1465
+ factx->vtcm_l_vec[r_vec_idx * 2 + 1] = v_l_curr1;
1466
+
1467
+ // Build diagonal tile D = diag(exp(m_diff))
1468
+ const HVX_Vector v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets;
1469
+ const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16));
1470
+ HVX_Vector v_exp_m_diff = exp_m_diff_f16;
1471
+
1472
+ __fp16 * const d_tiles_out = factx->vtcm_d_tiles[args->buf_idx];
1473
+
1474
+ size_t t0 = r_vec_idx * 2;
1475
+ if (t0 < args->n_row_tiles) {
1476
+ const HVX_Vector v_content = v_exp_m_diff;
1477
+ __fp16 * out_base = d_tiles_out + t0 * HMX_FP16_TILE_N_ELMS;
1478
+ Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content);
1479
+ }
1480
+
1481
+ size_t t1 = r_vec_idx * 2 + 1;
1482
+ if (t1 < args->n_row_tiles) {
1483
+ const HVX_Vector v_content = Q6_V_vror_VR(v_exp_m_diff, 64);
1484
+ __fp16 * out_base = d_tiles_out + t1 * HMX_FP16_TILE_N_ELMS;
1485
+ Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content);
1486
+ }
1487
+ }
1488
+ htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64));
1489
+ }
1490
+
1491
+ static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data) {
1492
+ fa_softmax_impl(n, i, data,
1493
+ /*has_mask=*/false,
1494
+ /*mask_broadcast=*/false,
1495
+ /*is_g1=*/false,
1496
+ /*has_alibi=*/false,
1497
+ /*has_softcap=*/false);
1498
+ }
1499
+
1500
+ static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i, void * data) {
1501
+ fa_softmax_impl(n, i, data,
1502
+ /*has_mask=*/true,
1503
+ /*mask_broadcast=*/true,
1504
+ /*is_g1=*/true,
1505
+ /*has_alibi=*/false,
1506
+ /*has_softcap=*/false);
1507
+ }
1508
+
1509
+ static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i, void * data) {
1510
+ fa_softmax_impl(n, i, data,
1511
+ /*has_mask=*/true,
1512
+ /*mask_broadcast=*/true,
1513
+ /*is_g1=*/false,
1514
+ /*has_alibi=*/false,
1515
+ /*has_softcap=*/false);
1516
+ }
1517
+
1518
+ static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) {
1519
+ fa_softmax_args_t * args = (fa_softmax_args_t *) data;
1520
+ struct hmx_fa_context * factx = args->factx;
1521
+
1522
+ const bool has_mask = (args->mask != NULL);
1523
+ const bool mask_broadcast = factx->mask_broadcast;
1524
+ const bool is_g1 = (args->G == 1);
1525
+ const bool has_alibi = args->has_alibi;
1526
+ const bool has_softcap = (factx->logit_softcap != 0.0f);
1527
+
1528
+ fa_softmax_impl(n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap);
1529
+ }
1530
+
1531
+ static __attribute__((noinline)) void fa_build_d_diag_inv_l(struct hmx_fa_context * factx,
1532
+ size_t n_row_tiles,
1533
+ size_t n_row_tiles_g_br) {
1534
+ const HVX_Vector v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets;
1535
+ const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16));
1536
+ const HVX_Vector one = hvx_vec_splat_f32(1.0f);
1537
+
1538
+ HVX_Vector v_content = Q6_V_vzero();
1539
+ for (size_t i = 0; i < n_row_tiles; ++i) {
1540
+ if ((i % 2) == 0) {
1541
+ HVX_Vector inv_lo = HVX_OP_MUL_F32(one, hvx_vec_inverse_f32(factx->vtcm_l_vec[i]));
1542
+ HVX_Vector inv_hi = (i + 1 < n_row_tiles) ? HVX_OP_MUL_F32(one, hvx_vec_inverse_f32(factx->vtcm_l_vec[i + 1])) : Q6_V_vzero();
1543
+ v_content = hvx_vec_f32_to_f16(inv_lo, inv_hi);
1544
+ } else {
1545
+ v_content = Q6_V_vror_VR(v_content, 64);
1546
+ }
1547
+
1548
+ __fp16 * out_base = factx->vtcm_d_inv_l + i * HMX_FP16_TILE_N_ELMS;
1549
+ Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content);
1550
+ }
1551
+ }
1552
+
1553
+ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx,
1554
+ fa_softmax_args_t * sargs,
1555
+ size_t n_row_tiles,
1556
+ size_t n_row_tiles_g_br) {
1557
+ work_queue_t wp = factx->octx->ctx->work_queue;
1558
+ const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64);
1559
+
1560
+ worker_callback_t softmax_fn = fa_softmax_thread;
1561
+ if (sargs->mask == NULL && factx->logit_softcap == 0.0f && !sargs->has_alibi) {
1562
+ softmax_fn = fa_softmax_thread_nomask;
1563
+ } else if (sargs->mask != NULL && factx->mask_broadcast && factx->logit_softcap == 0.0f && !sargs->has_alibi) {
1564
+ if (sargs->G == 1) {
1565
+ softmax_fn = fa_softmax_thread_mask_broadcast_g1;
1566
+ } else {
1567
+ softmax_fn = fa_softmax_thread_mask_broadcast_gn;
1568
+ }
1569
+ }
1570
+
1571
+ if (factx->n_threads > 1 && n_row_vec_cnt >= 2) {
1572
+ uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt);
1573
+ sargs->thread_div = init_fastdiv_values(n_use);
1574
+ work_queue_run(wp, softmax_fn, sargs, n_use);
1575
+ } else {
1576
+ softmax_fn(1, 0, sargs);
1577
+ }
1578
+ }
1579
+
1580
+ // ============================================================================
1581
+ // HMX job structs and worker functions
1582
+ // ============================================================================
1583
+
1584
+ typedef struct {
1585
+ const __fp16 * q_tiles;
1586
+ const __fp16 * k_tiles;
1587
+ __fp16 * s_tiles;
1588
+ size_t n_row_tiles;
1589
+ size_t n_col_tiles;
1590
+ size_t n_dot_tiles; // DK / 32
1591
+ size_t n_tiles_per_bc;
1592
+ uint8_t * hmx_scales;
1593
+ } hmx_fa_qk_job_t;
1594
+
1595
+ static void hmx_fa_qk_dot_worker(void * data) {
1596
+ hmx_fa_qk_job_t * job = (hmx_fa_qk_job_t *) data;
1597
+ const size_t n_row_tiles = job->n_row_tiles;
1598
+ const size_t n_col_tiles = job->n_col_tiles;
1599
+ const size_t n_dot_tiles = job->n_dot_tiles;
1600
+ const size_t n_tiles_per_bc = job->n_tiles_per_bc;
1601
+ const __fp16 * restrict q_tiles = job->q_tiles;
1602
+ const __fp16 * restrict k_tiles = job->k_tiles;
1603
+ __fp16 * restrict s_tiles = job->s_tiles;
1604
+ __builtin_assume(n_row_tiles > 0);
1605
+ __builtin_assume(n_col_tiles > 0);
1606
+ __builtin_assume(n_dot_tiles > 0);
1607
+
1608
+ asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales));
1609
+ const size_t dot_stride = n_dot_tiles * HMX_FP16_TILE_N_ELMS;
1610
+ for (size_t r = 0; r < n_row_tiles; ++r) {
1611
+ const __fp16 * row_tiles = q_tiles + r * dot_stride;
1612
+ const __fp16 * col_tiles = k_tiles;
1613
+ __fp16 * out_tile = s_tiles + r * n_tiles_per_bc * HMX_FP16_TILE_N_ELMS;
1614
+
1615
+ for (size_t c = 0; c < n_col_tiles; ++c) {
1616
+ hmx_fa_qk_dot_tile(row_tiles, col_tiles, out_tile, n_dot_tiles);
1617
+ col_tiles += dot_stride;
1618
+ out_tile += HMX_FP16_TILE_N_ELMS;
1619
+ }
1620
+ }
1621
+ }
1622
+
1623
+ typedef struct {
1624
+ __fp16 * o_curr;
1625
+ const __fp16 * o_prev;
1626
+ const __fp16 * p_tiles;
1627
+ const __fp16 * v_tiles;
1628
+ const __fp16 * d_tiles;
1629
+ uint8_t * hmx_scales;
1630
+ size_t n_row_tiles;
1631
+ size_t n_col_tiles;
1632
+ size_t n_row_tiles_g_br;
1633
+ size_t n_tiles_per_bc;
1634
+ size_t DV;
1635
+ } hmx_fa_o_update_job_t;
1636
+
1637
+ static void hmx_fa_o_update_worker(void * data) {
1638
+ hmx_fa_o_update_job_t * job = (hmx_fa_o_update_job_t *) data;
1639
+ const size_t n_row_tiles = job->n_row_tiles;
1640
+ const size_t n_col_tiles = job->n_col_tiles;
1641
+ const size_t n_row_tiles_g_br = job->n_row_tiles_g_br;
1642
+ const size_t n_tiles_per_bc = job->n_tiles_per_bc;
1643
+ const size_t DV_tiles = job->DV / 32;
1644
+ const __fp16 * restrict d_tiles = job->d_tiles;
1645
+ const __fp16 * restrict p_tiles = job->p_tiles;
1646
+ const __fp16 * restrict v_tiles = job->v_tiles;
1647
+ const __fp16 * restrict o_prev = job->o_prev;
1648
+ __fp16 * restrict o_curr = job->o_curr;
1649
+ __builtin_assume(n_row_tiles > 0);
1650
+ __builtin_assume(n_col_tiles > 0);
1651
+ __builtin_assume(DV_tiles > 0);
1652
+
1653
+ asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales));
1654
+ const size_t o_stride = n_row_tiles_g_br * HMX_FP16_TILE_N_ELMS;
1655
+ const size_t v_stride = n_tiles_per_bc * HMX_FP16_TILE_N_ELMS;
1656
+ for (size_t r = 0; r < n_row_tiles; ++r) {
1657
+ const __fp16 * d_diag = d_tiles + r * HMX_FP16_TILE_N_ELMS;
1658
+ const __fp16 * p_tile_in = p_tiles + (r * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS;
1659
+ const __fp16 * o_rc = o_prev + r * HMX_FP16_TILE_N_ELMS;
1660
+ const __fp16 * v_tile_in = v_tiles;
1661
+ __fp16 * o_tile_out = o_curr + r * HMX_FP16_TILE_N_ELMS;
1662
+
1663
+ for (size_t c = 0; c < DV_tiles; ++c) {
1664
+ hmx_fa_o_update_tile(d_diag, o_rc, p_tile_in, v_tile_in, o_tile_out, n_col_tiles);
1665
+ o_rc += o_stride;
1666
+ v_tile_in += v_stride;
1667
+ o_tile_out += o_stride;
1668
+ }
1669
+ }
1670
+ }
1671
+
1672
+ typedef struct {
1673
+ __fp16 * o_curr; // output (row-major tile layout)
1674
+ const __fp16 * o_prev; // input (column-major tile layout)
1675
+ const __fp16 * d_tiles; // diag(1/l) tiles
1676
+ uint8_t * hmx_scales;
1677
+ size_t n_row_tiles;
1678
+ size_t n_row_tiles_g_br;
1679
+ size_t DV;
1680
+ } hmx_fa_o_norm_job_t;
1681
+
1682
+ static void hmx_fa_o_norm_worker(void * data) {
1683
+ hmx_fa_o_norm_job_t * job = (hmx_fa_o_norm_job_t *) data;
1684
+ const size_t n_row_tiles = job->n_row_tiles;
1685
+ const size_t n_row_tiles_g_br = job->n_row_tiles_g_br;
1686
+ const size_t DV_tiles = job->DV / 32;
1687
+ const __fp16 * restrict d_tiles = job->d_tiles;
1688
+ const __fp16 * restrict o_prev = job->o_prev;
1689
+ __fp16 * restrict o_curr = job->o_curr;
1690
+ __builtin_assume(n_row_tiles > 0);
1691
+ __builtin_assume(DV_tiles > 0);
1692
+
1693
+ asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales));
1694
+ const size_t o_stride = n_row_tiles_g_br * HMX_FP16_TILE_N_ELMS;
1695
+ for (size_t r = 0; r < n_row_tiles; ++r) {
1696
+ const __fp16 * d_diag = d_tiles + r * HMX_FP16_TILE_N_ELMS;
1697
+ const __fp16 * o_rc = o_prev + r * HMX_FP16_TILE_N_ELMS;
1698
+ __fp16 * o_out = o_curr + r * DV_tiles * HMX_FP16_TILE_N_ELMS;
1699
+
1700
+ for (size_t c = 0; c < DV_tiles; ++c) {
1701
+ hmx_fa_o_norm_tile(d_diag, o_rc, o_out);
1702
+ o_rc += o_stride;
1703
+ o_out += HMX_FP16_TILE_N_ELMS;
1704
+ }
1705
+ }
1706
+ }
1707
+
1708
+ // Populate per-GQA-row ALiBi slopes for a given KV head.
1709
+ static __attribute__((noinline)) void fa_compute_slopes(
1710
+ const struct hmx_fa_context * factx,
1711
+ uint32_t kv_head,
1712
+ size_t n_rows_g) {
1713
+ __fp16 * slopes = factx->vtcm_slopes;
1714
+ if (factx->max_bias == 0.0f) {
1715
+ hvx_splat_f16_a(slopes, 1.0f, n_rows_g);
1716
+ return;
1717
+ }
1718
+
1719
+ const uint32_t G = factx->G;
1720
+ const uint32_t n_head_log2 = factx->n_head_log2;
1721
+ const float m0 = factx->m0;
1722
+ const float m1 = factx->m1;
1723
+
1724
+ __fp16 temp_slopes[512] __attribute__((aligned(128)));
1725
+ if (G <= 32) {
1726
+ // Fast path: Compute G unique slope values in vector registers
1727
+ HVX_Vector v_val = hvx_alibi_slopes(kv_head, G, n_head_log2, m0, m1);
1728
+
1729
+ __fp16 temp_slopes_aligned[64] __attribute__((aligned(128)));
1730
+ hvx_vmem(temp_slopes_aligned) = hvx_vec_f32_to_f16(v_val, Q6_V_vzero());
1731
+
1732
+ for (uint32_t i = 0; i < G; ++i) {
1733
+ temp_slopes[i] = temp_slopes_aligned[i];
1734
+ }
1735
+ } else {
1736
+ // Fallback path: G > 32 (rare configurations)
1737
+ for (uint32_t i = 0; i < G; ++i) {
1738
+ temp_slopes[i] = (__fp16)alibi_slope(kv_head * G + i, n_head_log2, m0, m1);
1739
+ }
1740
+ }
1741
+
1742
+ // Allocate stack buffer to avoid scalar writes to VTCM (which generates L2 misses)
1743
+ __fp16 local_slopes[n_rows_g] __attribute__((aligned(128)));
1744
+ for (size_t r = 0; r < n_rows_g; ++r) {
1745
+ local_slopes[r] = temp_slopes[fastmodulo(r, G, &factx->div_G)];
1746
+ }
1747
+
1748
+ // Copy to VTCM slopes using HVX block copy (both are aligned to 128 bytes)
1749
+ hvx_copy_f16_aa((uint8_t *)slopes, (const uint8_t *)local_slopes, n_rows_g);
1750
+ }
1751
+
1752
+ static void fa_push_mask_dma_gqa(
1753
+ dma_queue * dma,
1754
+ const struct htp_tensor * mask,
1755
+ uint32_t q_start,
1756
+ uint32_t im3,
1757
+ uint32_t kv_start,
1758
+ uint32_t kv_head,
1759
+ uint32_t G,
1760
+ uint32_t m_line_bytes,
1761
+ uint32_t kv_rows,
1762
+ uint32_t n_rows_q,
1763
+ struct hmx_fa_context * factx
1764
+ ) {
1765
+ for (uint32_t g = 0; g < G; ++g) {
1766
+ const uint32_t h_idx = kv_head * G + g;
1767
+ const uint32_t im2 = fastmodulo(h_idx, mask->ne[2], &factx->src3_div2);
1768
+ const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] +
1769
+ im2 * mask->nb[2] + im3 * mask->nb[3] + kv_start * sizeof(__fp16);
1770
+ uint8_t * ms_dst = (uint8_t *) factx->vtcm_mask_buf + g * m_line_bytes;
1771
+ dma_queue_push(dma, dma_make_ptr(ms_dst, ms_src), G * m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);
1772
+ }
1773
+ }
1774
+
1775
+ static void fa_pop_mask_dma_gqa(dma_queue * dma, uint32_t G) {
1776
+ for (uint32_t g = 0; g < G; ++g) {
1777
+ dma_queue_pop(dma);
1778
+ }
1779
+ }
1780
+
1781
+ static inline void fa_prefetch_block(dma_queue * dma, const struct htp_tensor * k, const struct htp_tensor * v, const struct htp_tensor * mask,
1782
+ uint32_t b, size_t Bc, size_t size_k_row_padded, size_t size_k_row, size_t size_v_row_padded, size_t size_v_row,
1783
+ uint32_t ik2, uint32_t ik3, uint32_t iv2, uint32_t iv3, uint32_t q_start, uint32_t im3, uint32_t kv_head, uint32_t G,
1784
+ size_t m_line_bytes, size_t n_rows_q, size_t nek1, size_t prefetch_buf, struct hmx_fa_context * factx) {
1785
+ const uint32_t prefetch_start = b * Bc;
1786
+ const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start);
1787
+ const uint8_t * k_prefetch_src = (const uint8_t *) k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3];
1788
+ dma_queue_push(dma, dma_make_ptr(factx->vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows);
1789
+ const uint8_t * v_prefetch_src = (const uint8_t *) v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3];
1790
+ dma_queue_push(dma, dma_make_ptr(factx->vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows);
1791
+
1792
+ if (mask) {
1793
+ if (__builtin_expect(factx->mask_broadcast, true)) {
1794
+ const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + prefetch_start * sizeof(__fp16);
1795
+ dma_cache_push(dma, &factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q);
1796
+ } else {
1797
+ fa_push_mask_dma_gqa(dma, mask, q_start, im3, prefetch_start, kv_head, G, m_line_bytes, prefetch_rows, n_rows_q, factx);
1798
+ }
1799
+ }
1800
+ }
1801
+
1802
+ // ============================================================================
1803
+ // Core HMX flash attention algorithm (GQA-merged)
1804
+ // ============================================================================
1805
+
1806
+ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
1807
+ struct htp_thread_trace * tr_hvx = &octx->ctx->trace[0];
1808
+ struct htp_thread_trace * tr_hmx = &octx->ctx->trace[HTP_MAX_NTHREADS];
1809
+ const struct htp_tensor * q = octx->src[0];
1810
+ const struct htp_tensor * k = octx->src[1];
1811
+ const struct htp_tensor * v = octx->src[2];
1812
+ const struct htp_tensor * mask = (octx->src[3] && octx->src[3]->data) ? octx->src[3] : NULL;
1813
+ const struct htp_tensor * dst = octx->dst;
1814
+
1815
+ struct htp_context * const ctx = octx->ctx;
1816
+
1817
+ if (!ctx->hmx_enabled) {
1818
+ return HTP_STATUS_NO_SUPPORT;
1819
+ }
1820
+
1821
+ // Dimensions
1822
+ const uint32_t neq0 = q->ne[0]; // head_dim (DK)
1823
+ const uint32_t neq1 = q->ne[1]; // n_tokens
1824
+ const uint32_t neq2 = q->ne[2]; // n_heads
1825
+ const uint32_t neq3 = q->ne[3]; // n_seqs
1826
+
1827
+ const uint32_t nek0 = k->ne[0]; // head_dim
1828
+ const uint32_t nek1 = k->ne[1]; // kv_len
1829
+
1830
+ const uint32_t nev0 = v->ne[0]; // head_dim (DV)
1831
+
1832
+ const uint32_t DK = neq0;
1833
+ const uint32_t DV = nev0;
1834
+
1835
+ // HMX requires head_dim to be multiple of 32
1836
+ if (DK % 32 != 0 || DV % 32 != 0) {
1837
+ return HTP_STATUS_NO_SUPPORT;
1838
+ }
1839
+
1840
+ const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params;
1841
+ const uint32_t n_kv_heads = k->ne[2];
1842
+
1843
+ // ======== Build context ========
1844
+ struct hmx_fa_context factx;
1845
+ memset(&factx, 0, sizeof(factx));
1846
+ factx.octx = octx;
1847
+ factx.sinks = octx->src[4]; // NULL if this op has no attention sinks
1848
+ factx.k = k;
1849
+ factx.v = v;
1850
+ factx.n_threads = kparams->n_threads;
1851
+ factx.DK = DK;
1852
+ factx.DV = DV;
1853
+ factx.n_kv = nek1;
1854
+ factx.n_kv_heads = n_kv_heads;
1855
+ factx.n_heads = neq2;
1856
+ factx.G = kparams->G;
1857
+ factx.div_G = kparams->u.hmx.div_G;
1858
+ factx.neq1 = neq1;
1859
+ factx.Br = kparams->Br;
1860
+ factx.Bc = kparams->Bc;
1861
+ factx.g_br = kparams->u.hmx.g_br;
1862
+ factx.n_kv_blocks = kparams->n_kv_blocks;
1863
+ factx.is_q_fp32 = (kparams->is_q_fp32 != 0);
1864
+ factx.is_dst_fp32 = (kparams->is_dst_fp32 != 0);
1865
+ factx.pipeline = (kparams->u.hmx.pipeline != 0);
1866
+ factx.mask_broadcast = (kparams->u.hmx.mask_broadcast != 0);
1867
+ if (mask) {
1868
+ factx.src3_div2 = kparams->src3_div2;
1869
+ factx.src3_div3 = kparams->src3_div3;
1870
+ }
1871
+
1872
+ if (kparams->logit_softcap == 0.0f) {
1873
+ factx.scale = (__fp16) (kparams->scale * EXP_LOG2E_F); // log2(e)
1874
+ } else {
1875
+ factx.scale = (__fp16) kparams->scale;
1876
+ }
1877
+ factx.max_bias = kparams->max_bias;
1878
+ factx.logit_softcap = (__fp16) (kparams->logit_softcap * EXP_LOG2E_F);
1879
+
1880
+ factx.n_head_log2 = kparams->n_head_log2;
1881
+ factx.m0 = kparams->m0;
1882
+ factx.m1 = kparams->m1;
1883
+
1884
+ const uint32_t Br = factx.Br;
1885
+ const uint32_t Bc = factx.Bc;
1886
+ const uint32_t g_br = factx.g_br;
1887
+ const bool pipeline = factx.pipeline;
1888
+ const uint32_t n_threads = factx.n_threads;
1889
+ const uint32_t G = factx.G;
1890
+
1891
+ // ======== VTCM allocation (GQA-aware) ========
1892
+ // K/V row sizes drive the DMA descriptors (not the VTCM layout) and are used
1893
+ // throughout the KV loop below.
1894
+ const size_t size_k_row = htp_tensor_get_row_size(k->type, DK);
1895
+ const size_t size_v_row = htp_tensor_get_row_size(v->type, DV);
1896
+ const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128);
1897
+ const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128);
1898
+
1899
+ // Build the VTCM layout once (shared with the host estimator) and place every
1900
+ // scratch buffer at its computed offset.
1901
+ struct hmx_fa_vtcm_layout L;
1902
+ hmx_fa_vtcm_layout_build(&L, G, DK, DV, Br, Bc, n_threads, pipeline, factx.is_q_fp32);
1903
+
1904
+ if (L.total_bytes > ctx->vtcm_size) {
1905
+ return HTP_STATUS_VTCM_TOO_SMALL;
1906
+ }
1907
+
1908
+ uint8_t * const base = ctx->vtcm_base;
1909
+
1910
+ factx.vtcm_q_dma = VTCM_LAYOUT_PTR(__fp16, base, L.off_q_dma);
1911
+ factx.vtcm_q_tiles = VTCM_LAYOUT_PTR(__fp16, base, L.off_q_tiles);
1912
+ factx.vtcm_o_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_o_tiles[0]);
1913
+ factx.vtcm_o_tiles[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_o_tiles[1]);
1914
+ factx.vtcm_k_fp16[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_fp16[0]);
1915
+ factx.vtcm_k_fp16[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_fp16[1]);
1916
+ factx.vtcm_v_fp16[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_fp16[0]);
1917
+ factx.vtcm_v_fp16[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_fp16[1]);
1918
+ factx.vtcm_k_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_tiles[0]);
1919
+ factx.vtcm_k_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_k_tiles[1], pipeline);
1920
+ factx.vtcm_v_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_tiles[0]);
1921
+ factx.vtcm_v_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_v_tiles[1], pipeline);
1922
+ factx.vtcm_s_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_s_tiles[0]);
1923
+ factx.vtcm_s_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_s_tiles[1], pipeline);
1924
+ factx.vtcm_p_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_p_tiles[0]);
1925
+ factx.vtcm_p_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_p_tiles[1], pipeline);
1926
+ factx.vtcm_d_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_d_tiles[0]);
1927
+ factx.vtcm_d_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_d_tiles[1], pipeline);
1928
+ factx.vtcm_d_inv_l = VTCM_LAYOUT_PTR(__fp16, base, L.off_d_inv_l);
1929
+ factx.vtcm_m_vec = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_m_vec);
1930
+ factx.vtcm_l_vec = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_l_vec);
1931
+ factx.vtcm_s_rowmax = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_s_rowmax);
1932
+ factx.vtcm_p_rowsum = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_p_rowsum);
1933
+ factx.vtcm_row_bufs = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_row_bufs);
1934
+ factx.row_buf_stride = L.row_buf_stride;
1935
+ factx.vtcm_hmx_scales_id = VTCM_LAYOUT_PTR(uint8_t, base, L.off_hmx_scales_id);
1936
+ factx.vtcm_hmx_scales_qk = VTCM_LAYOUT_PTR(uint8_t, base, L.off_hmx_scales_qk);
1937
+ factx.vtcm_mask_buf = VTCM_LAYOUT_PTR(__fp16, base, L.off_mask_buf);
1938
+ factx.mask_buf_row_stride = L.mask_buf_row_stride;
1939
+ factx.q_tile_bytes = L.q_tile_bytes;
1940
+ factx.o_tile_bytes = L.o_tile_bytes;
1941
+ factx.col_vec_bytes = L.col_vec_bytes;
1942
+ factx.d_tile_bytes = L.d_tile_bytes;
1943
+ factx.vtcm_slopes = VTCM_LAYOUT_PTR(__fp16, base, L.off_slopes);
1944
+
1945
+ const size_t m_line_bytes = L.m_line_bytes; // used by the mask DMAs in the KV loop
1946
+
1947
+ dma_cache_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE);
1948
+
1949
+ // ======== Initialize HMX output scales ========
1950
+ hmx_init_column_scales(factx.vtcm_hmx_scales_id, Q6_V_vsplat_R(0x3c00)); // 1.0
1951
+ hmx_init_column_scales(factx.vtcm_hmx_scales_qk, hvx_vec_splat_f16(factx.scale));
1952
+
1953
+ // ======== Skip compute if profiling ========
1954
+ if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
1955
+ return HTP_STATUS_OK;
1956
+ }
1957
+
1958
+ // ======== DMA setup ========
1959
+ dma_queue * const dma = ctx->dma[0];
1960
+
1961
+ const size_t n_row_tiles_g_br = g_br / HMX_FP16_TILE_N_ROWS;
1962
+ const size_t n_tiles_per_bc = Bc / HMX_FP16_TILE_N_COLS;
1963
+
1964
+ const size_t qo_element_size = factx.is_q_fp32 ? sizeof(float) : sizeof(__fp16);
1965
+
1966
+ const bool q_transposed = q->nb[1] < q->nb[2];
1967
+ const size_t q_src_stride = q_transposed ? q->nb[2] : q->nb[1];
1968
+ const size_t q_row_bytes_untransposed = factx.G * factx.DK * qo_element_size;
1969
+ const size_t q_row_bytes_trans_factor = factx.DK * qo_element_size;
1970
+ const uint32_t kv_rows0 = hex_smin(Bc, nek1);
1971
+
1972
+ // ======== Reusable job descriptors for pipeline ========
1973
+ hmx_fa_qk_job_t qk_job;
1974
+ hmx_fa_o_update_job_t ou_job;
1975
+ hmx_fa_o_norm_job_t on_job;
1976
+
1977
+ // ======== Main loop ========
1978
+ for (uint32_t ib3 = 0; ib3 < neq3; ++ib3) {
1979
+ const uint32_t im3 = mask ? fastmodulo(ib3, mask->ne[3], &factx.src3_div3) : 0;
1980
+ for (uint32_t q_start = 0; q_start < neq1; q_start += Br) {
1981
+ const uint32_t n_rows_q = hex_smin(Br, neq1 - q_start);
1982
+ const size_t n_rows_g = n_rows_q * G;
1983
+ const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS);
1984
+ const size_t n_row_tiles = g_br_actual / HMX_FP16_TILE_N_ROWS;
1985
+
1986
+ for (uint32_t kv_head = 0; kv_head < n_kv_heads; ++kv_head) {
1987
+ const uint32_t ik2 = kv_head;
1988
+ const uint32_t ik3 = fastdiv(ib3, &kparams->broadcast_rk3);
1989
+ const uint32_t iv2 = kv_head;
1990
+ const uint32_t iv3 = fastdiv(ib3, &kparams->broadcast_rv3);
1991
+
1992
+ // 1. Push Q and KV DMAs for the very first iteration.
1993
+ // Subsequent iterations are enqueued early at the end of the previous iteration.
1994
+ if (ib3 == 0 && q_start == 0 && kv_head == 0) {
1995
+ const uint8_t * q_ptr = (const uint8_t *) q->data;
1996
+ const size_t q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed;
1997
+ const size_t n_rows = q_transposed ? factx.G : n_rows_q;
1998
+ dma_queue_push(dma, dma_make_ptr(factx.vtcm_q_dma, q_ptr), q_row_bytes, hex_smax(q_src_stride, q_row_bytes), q_row_bytes, n_rows);
1999
+
2000
+ if (factx.n_kv_blocks > 0) {
2001
+ const uint8_t * k_src = (const uint8_t *) k->data + ik2 * k->nb[2] + ik3 * k->nb[3];
2002
+ dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0);
2003
+
2004
+ const uint8_t * v_src = (const uint8_t *) v->data + iv2 * v->nb[2] + iv3 * v->nb[3];
2005
+ dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0);
2006
+
2007
+ if (factx.pipeline && mask) {
2008
+ if (__builtin_expect(factx.mask_broadcast, true)) {
2009
+ const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + 0;
2010
+ dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), n_rows_q);
2011
+ } else {
2012
+ fa_push_mask_dma_gqa(dma, mask, q_start, im3, 0, kv_head, G, m_line_bytes, kv_rows0, n_rows_q, &factx);
2013
+ }
2014
+ }
2015
+ }
2016
+ }
2017
+
2018
+ // 2. Pop Q DMA (blocks until Q is loaded)
2019
+ dma_queue_pop(dma);
2020
+
2021
+ // ---- Load Q block & Initialize per-block state ----
2022
+ fa_phase_q_load(&factx, q, q_start, kv_head, ib3, n_rows_g);
2023
+
2024
+ __fp16 * o_tile_prev = factx.vtcm_o_tiles[0];
2025
+ __fp16 * o_tile_curr = factx.vtcm_o_tiles[1];
2026
+
2027
+ // ---- KV block loop with DMA double-buffering ----
2028
+ size_t buf_idx = 0;
2029
+
2030
+ htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) q_start);
2031
+ fa_compute_slopes(&factx, kv_head, n_rows_g);
2032
+ htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) q_start);
2033
+
2034
+ const size_t k_src_stride = size_k_row_padded / sizeof(__fp16);
2035
+ const size_t v_src_stride = size_v_row_padded / sizeof(__fp16);
2036
+
2037
+ hmx_queue_t hmx_q = ctx->hmx_queue;
2038
+
2039
+ if (factx.pipeline) {
2040
+ // Double-buffered job structs because HMX queue runs asynchronously
2041
+ hmx_fa_qk_job_t qk_job[2];
2042
+ hmx_fa_o_update_job_t ou_job[2];
2043
+
2044
+ // Prefetch block 1 early if there are multiple blocks
2045
+ if (factx.n_kv_blocks > 1) {
2046
+ fa_prefetch_block(dma, k, v, mask, 1, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row,
2047
+ ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, 1, &factx);
2048
+ }
2049
+
2050
+ // Prep and start QK-dot(0)
2051
+ void * curr_k0 = dma_queue_pop(dma).dst;
2052
+ fa_phase_k_interleave(&factx, kv_rows0, k_src_stride, curr_k0, 0, 0);
2053
+
2054
+ qk_job[0].q_tiles = factx.vtcm_q_tiles;
2055
+ qk_job[0].k_tiles = factx.vtcm_k_tiles[0];
2056
+ qk_job[0].s_tiles = factx.vtcm_s_tiles[0];
2057
+ qk_job[0].n_row_tiles = n_row_tiles;
2058
+ qk_job[0].n_col_tiles = hmx_ceil_div(kv_rows0, HMX_FP16_TILE_N_COLS);
2059
+ qk_job[0].n_dot_tiles = DK / 32;
2060
+ qk_job[0].n_tiles_per_bc = n_tiles_per_bc;
2061
+ qk_job[0].hmx_scales = factx.vtcm_hmx_scales_qk;
2062
+ hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job[0]));
2063
+
2064
+ for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) {
2065
+ const uint32_t kv_start = kv_blk * Bc;
2066
+ const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start);
2067
+ const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS);
2068
+
2069
+ // ---- 1. Pop and run V-prep for current block ----
2070
+ void * curr_v = dma_queue_pop(dma).dst;
2071
+ fa_phase_v_interleave(&factx, kv_rows, v_src_stride, curr_v, factx.vtcm_v_tiles[buf_idx], n_tiles_per_bc, kv_start);
2072
+
2073
+ // ---- 2. Pop and run mask-prep for current block ----
2074
+ __fp16 * current_mask_vtcm = NULL;
2075
+ if (mask) {
2076
+ if (__builtin_expect(factx.mask_broadcast, true)) {
2077
+ current_mask_vtcm = (__fp16 *) dma_queue_pop(dma).dst;
2078
+ } else {
2079
+ fa_pop_mask_dma_gqa(dma, G);
2080
+ current_mask_vtcm = factx.vtcm_mask_buf;
2081
+ }
2082
+ }
2083
+
2084
+ // ---- 3. Start HMX O update for block kv_blk - 1 (reads P[1 - buf_idx], V[1 - buf_idx], D) ----
2085
+ // O update relys on the previous block's P and V tiles.
2086
+ // O update MUST be pushed before the next block's QK-dot: hmx_queue_pop() retires the
2087
+ // oldest descriptor, so push order alone decides which pop waits for which job.
2088
+ // If OU went in after QK(i+1), the pop below would retire QK(i+1) and leave
2089
+ // OU(i-1) in flight into the next iteration, where V-prep overwrites V[prev_buf].
2090
+ if (kv_blk > 0) {
2091
+ const size_t prev_buf = 1 - buf_idx;
2092
+ ou_job[prev_buf].o_curr = o_tile_curr;
2093
+ ou_job[prev_buf].o_prev = o_tile_prev;
2094
+ ou_job[prev_buf].p_tiles = factx.vtcm_p_tiles[prev_buf];
2095
+ ou_job[prev_buf].v_tiles = factx.vtcm_v_tiles[prev_buf];
2096
+ ou_job[prev_buf].d_tiles = factx.vtcm_d_tiles[prev_buf];
2097
+ ou_job[prev_buf].hmx_scales = factx.vtcm_hmx_scales_id;
2098
+ ou_job[prev_buf].n_row_tiles = n_row_tiles;
2099
+ ou_job[prev_buf].n_col_tiles =
2100
+ hmx_ceil_div(hex_smin(Bc, nek1 - (kv_blk - 1) * Bc), HMX_FP16_TILE_N_COLS);
2101
+ ou_job[prev_buf].n_row_tiles_g_br = n_row_tiles_g_br;
2102
+ ou_job[prev_buf].n_tiles_per_bc = n_tiles_per_bc;
2103
+ ou_job[prev_buf].DV = DV;
2104
+ hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[prev_buf]));
2105
+ }
2106
+
2107
+ // ---- 4. Pop and run K-prep for next block & push next QK-dot ----
2108
+ if (kv_blk + 1 < factx.n_kv_blocks) {
2109
+ const uint32_t next_start = (kv_blk + 1) * Bc;
2110
+ const uint32_t next_rows = hex_smin(Bc, nek1 - next_start);
2111
+ const size_t next_buf = 1 - buf_idx;
2112
+
2113
+ void * next_k = dma_queue_pop(dma).dst;
2114
+ fa_phase_k_interleave(&factx, next_rows, k_src_stride, next_k, next_start, next_buf);
2115
+
2116
+ qk_job[next_buf].q_tiles = factx.vtcm_q_tiles;
2117
+ qk_job[next_buf].k_tiles = factx.vtcm_k_tiles[next_buf];
2118
+ qk_job[next_buf].s_tiles = factx.vtcm_s_tiles[next_buf];
2119
+ qk_job[next_buf].n_row_tiles = n_row_tiles;
2120
+ qk_job[next_buf].n_col_tiles = hmx_ceil_div(next_rows, HMX_FP16_TILE_N_COLS);
2121
+ qk_job[next_buf].n_dot_tiles = DK / 32;
2122
+ qk_job[next_buf].n_tiles_per_bc = n_tiles_per_bc;
2123
+ qk_job[next_buf].hmx_scales = factx.vtcm_hmx_scales_qk;
2124
+ hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job[next_buf]));
2125
+ }
2126
+
2127
+ // ---- 5. Wait for current block's QK-dot to finish ----
2128
+ hmx_queue_pop(hmx_q);
2129
+
2130
+ // ---- 6. Phase 2: softmax + build_D ----
2131
+ fa_softmax_args_t sargs;
2132
+ memset(&sargs, 0, sizeof(sargs));
2133
+ sargs.factx = &factx;
2134
+ sargs.buf_idx = buf_idx;
2135
+ sargs.kv_rows = kv_rows;
2136
+ sargs.n_rows_g = n_rows_g;
2137
+ sargs.n_col_tiles = n_col_tiles;
2138
+ sargs.n_tiles_per_bc = n_tiles_per_bc;
2139
+ sargs.n_row_tiles = n_row_tiles;
2140
+ sargs.n_row_tiles_g_br = n_row_tiles_g_br;
2141
+ sargs.Bc = Bc;
2142
+ sargs.G = G;
2143
+ sargs.kv_head = kv_head;
2144
+ sargs.kv_start = kv_start;
2145
+ sargs.q_start = q_start;
2146
+ sargs.ib3 = ib3;
2147
+ sargs.has_alibi = (factx.max_bias != 0.0f);
2148
+ sargs.mask = mask;
2149
+ sargs.mask_vtcm = current_mask_vtcm;
2150
+ sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride;
2151
+ sargs.slopes = factx.vtcm_slopes;
2152
+
2153
+ // Run Softmax on HVX (blocking call)
2154
+ fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br);
2155
+
2156
+ // Wait for HMX O update for block kv_blk - 1 to finish
2157
+ if (kv_blk > 0) {
2158
+ hmx_queue_pop(hmx_q);
2159
+ hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev);
2160
+ }
2161
+
2162
+ // Prefetch block kv_blk + 2
2163
+ if (kv_blk + 2 < factx.n_kv_blocks) {
2164
+ fa_prefetch_block(dma, k, v, mask, kv_blk + 2, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row,
2165
+ ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, buf_idx, &factx);
2166
+ }
2167
+
2168
+ buf_idx = 1 - buf_idx;
2169
+ }
2170
+
2171
+ // Epilogue
2172
+ if (factx.n_kv_blocks > 0) {
2173
+ const uint32_t last_blk = factx.n_kv_blocks - 1;
2174
+ const size_t last_cols = hmx_ceil_div(hex_smin(Bc, nek1 - last_blk * Bc), HMX_FP16_TILE_N_COLS);
2175
+ ou_job[0].o_curr = o_tile_curr;
2176
+ ou_job[0].o_prev = o_tile_prev;
2177
+ ou_job[0].p_tiles = factx.vtcm_p_tiles[1 - buf_idx];
2178
+ ou_job[0].v_tiles = factx.vtcm_v_tiles[1 - buf_idx];
2179
+ ou_job[0].d_tiles = factx.vtcm_d_tiles[1 - buf_idx];
2180
+ ou_job[0].hmx_scales = factx.vtcm_hmx_scales_id;
2181
+ ou_job[0].n_row_tiles = n_row_tiles;
2182
+ ou_job[0].n_col_tiles = last_cols;
2183
+ ou_job[0].n_row_tiles_g_br = n_row_tiles_g_br;
2184
+ ou_job[0].n_tiles_per_bc = n_tiles_per_bc;
2185
+ ou_job[0].DV = DV;
2186
+ hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[0]));
2187
+
2188
+ // Overlapped: run HVX build diag inv L while HMX is busy executing the update
2189
+ htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
2190
+ fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br);
2191
+ htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
2192
+ hmx_queue_pop(hmx_q);
2193
+
2194
+ hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev);
2195
+ }
2196
+
2197
+ } else {
2198
+ // Fallback path
2199
+ for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) {
2200
+ const uint32_t kv_start = kv_blk * Bc;
2201
+ const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start);
2202
+ const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS);
2203
+
2204
+ if (mask) {
2205
+ if (__builtin_expect(factx.mask_broadcast, true)) {
2206
+ const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16);
2207
+ dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);
2208
+ } else {
2209
+ fa_push_mask_dma_gqa(dma, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx);
2210
+ }
2211
+ }
2212
+
2213
+ if (kv_blk + 1 < factx.n_kv_blocks) {
2214
+ const uint32_t prefetch_start = (kv_blk + 1) * Bc;
2215
+ const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start);
2216
+ const size_t prefetch_buf = 1 - buf_idx;
2217
+ const uint8_t * k_prefetch_src = (const uint8_t *) k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3];
2218
+ dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows);
2219
+ const uint8_t * v_prefetch_src = (const uint8_t *) v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3];
2220
+ dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows);
2221
+ }
2222
+
2223
+ // Wait for current K DMA and interleave
2224
+ void * curr_k = dma_queue_pop(dma).dst;
2225
+ fa_phase_k_interleave(&factx, kv_rows, k_src_stride, curr_k, kv_start, 0);
2226
+
2227
+ {
2228
+ qk_job.q_tiles = factx.vtcm_q_tiles;
2229
+ qk_job.k_tiles = factx.vtcm_k_tiles[0];
2230
+ qk_job.s_tiles = factx.vtcm_s_tiles[0];
2231
+ qk_job.n_row_tiles = n_row_tiles;
2232
+ qk_job.n_col_tiles = n_col_tiles;
2233
+ qk_job.n_dot_tiles = (size_t) (DK / 32);
2234
+ qk_job.n_tiles_per_bc = n_tiles_per_bc;
2235
+ qk_job.hmx_scales = factx.vtcm_hmx_scales_qk;
2236
+
2237
+ hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job));
2238
+ hmx_queue_pop(ctx->hmx_queue);
2239
+ }
2240
+
2241
+ // Wait for current V DMA and interleave
2242
+ void * curr_v = dma_queue_pop(dma).dst;
2243
+ fa_phase_v_interleave(&factx, kv_rows, v_src_stride, curr_v, factx.vtcm_v_tiles[0], n_tiles_per_bc, kv_start);
2244
+
2245
+ // ---- Phase 3: softmax + build_D ----
2246
+ __fp16 * current_mask_vtcm = NULL;
2247
+ if (mask) {
2248
+ if (__builtin_expect(factx.mask_broadcast, true)) {
2249
+ current_mask_vtcm = (__fp16 *) dma_queue_pop(dma).dst;
2250
+ } else {
2251
+ fa_pop_mask_dma_gqa(dma, G);
2252
+ current_mask_vtcm = factx.vtcm_mask_buf;
2253
+ }
2254
+ }
2255
+
2256
+ fa_softmax_args_t sargs;
2257
+ memset(&sargs, 0, sizeof(sargs));
2258
+ sargs.factx = &factx;
2259
+ sargs.kv_rows = kv_rows;
2260
+ sargs.n_rows_g = n_rows_g;
2261
+ sargs.n_col_tiles = n_col_tiles;
2262
+ sargs.n_tiles_per_bc = n_tiles_per_bc;
2263
+ sargs.n_row_tiles = n_row_tiles;
2264
+ sargs.n_row_tiles_g_br = n_row_tiles_g_br;
2265
+ sargs.Bc = Bc;
2266
+ sargs.G = G;
2267
+ sargs.kv_head = kv_head;
2268
+ sargs.kv_start = kv_start;
2269
+ sargs.q_start = q_start;
2270
+ sargs.ib3 = ib3;
2271
+ sargs.has_alibi = (factx.max_bias != 0.0f);
2272
+ sargs.mask = mask;
2273
+ sargs.mask_vtcm = current_mask_vtcm;
2274
+ sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride;
2275
+ sargs.slopes = factx.vtcm_slopes;
2276
+ fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br);
2277
+
2278
+ {
2279
+ ou_job.o_curr = o_tile_curr;
2280
+ ou_job.o_prev = o_tile_prev;
2281
+ ou_job.p_tiles = factx.vtcm_p_tiles[0];
2282
+ ou_job.v_tiles = factx.vtcm_v_tiles[0];
2283
+ ou_job.d_tiles = factx.vtcm_d_tiles[0];
2284
+ ou_job.hmx_scales = factx.vtcm_hmx_scales_id;
2285
+ ou_job.n_row_tiles = n_row_tiles;
2286
+ ou_job.n_col_tiles = n_col_tiles;
2287
+ ou_job.n_row_tiles_g_br = n_row_tiles_g_br;
2288
+ ou_job.n_tiles_per_bc = n_tiles_per_bc;
2289
+ ou_job.DV = DV;
2290
+
2291
+ hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job));
2292
+ if (kv_blk + 1 == factx.n_kv_blocks) {
2293
+ // Overlapped: run HVX build diag inv L while HMX is busy executing the update
2294
+ htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
2295
+ fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br);
2296
+ htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
2297
+ }
2298
+ hmx_queue_pop(ctx->hmx_queue);
2299
+
2300
+ hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev);
2301
+ }
2302
+
2303
+ buf_idx = 1 - buf_idx;
2304
+ }
2305
+ }
2306
+
2307
+ // Enqueue DMAs for the next iteration early so they overlap with O-PROC
2308
+ uint32_t next_kv_head = kv_head + 1;
2309
+ uint32_t next_q_start = q_start;
2310
+ uint32_t next_ib3 = ib3;
2311
+ if (next_kv_head >= n_kv_heads) {
2312
+ next_kv_head = 0;
2313
+ next_q_start = q_start + Br;
2314
+ if (next_q_start >= neq1) {
2315
+ next_q_start = 0;
2316
+ next_ib3 = ib3 + 1;
2317
+ }
2318
+ }
2319
+ bool has_next = (next_ib3 < neq3);
2320
+
2321
+ if (has_next) {
2322
+ const uint32_t next_n_rows_q = hex_smin(Br, neq1 - next_q_start);
2323
+ const uint8_t * next_q_ptr = (const uint8_t *) q->data + next_q_start * q->nb[1] + (next_kv_head * factx.G) * q->nb[2] + next_ib3 * q->nb[3];
2324
+ const size_t next_q_row_bytes = q_transposed ? next_n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed;
2325
+ const size_t next_n_rows = q_transposed ? factx.G : next_n_rows_q;
2326
+ dma_queue_push(dma, dma_make_ptr(factx.vtcm_q_dma, next_q_ptr), next_q_row_bytes, hex_smax(q_src_stride, next_q_row_bytes), next_q_row_bytes, next_n_rows);
2327
+
2328
+ if (factx.n_kv_blocks > 0) {
2329
+ const uint32_t next_ik2 = next_kv_head;
2330
+ const uint32_t next_iv2 = next_kv_head;
2331
+ uint32_t next_ik3 = ik3;
2332
+ uint32_t next_iv3 = iv3;
2333
+ if (next_ib3 != ib3) {
2334
+ next_ik3 = fastdiv(next_ib3, &kparams->broadcast_rk3);
2335
+ next_iv3 = fastdiv(next_ib3, &kparams->broadcast_rv3);
2336
+ }
2337
+
2338
+ const uint8_t * next_k_src = (const uint8_t *) k->data + next_ik2 * k->nb[2] + next_ik3 * k->nb[3];
2339
+ dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[0], next_k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0);
2340
+
2341
+ const uint8_t * next_v_src = (const uint8_t *) v->data + next_iv2 * v->nb[2] + next_iv3 * v->nb[3];
2342
+ dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[0], next_v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0);
2343
+
2344
+ if (factx.pipeline && mask) {
2345
+ uint32_t next_im3 = im3;
2346
+ if (next_ib3 != ib3) {
2347
+ next_im3 = fastmodulo(next_ib3, mask->ne[3], &factx.src3_div3);
2348
+ }
2349
+ if (__builtin_expect(factx.mask_broadcast, true)) {
2350
+ const uint8_t * ms_src = (const uint8_t *) mask->data + next_q_start * mask->nb[1] + next_im3 * mask->nb[3] + 0;
2351
+ dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), next_n_rows_q);
2352
+ } else {
2353
+ fa_push_mask_dma_gqa(dma, mask, next_q_start, next_im3, 0, next_kv_head, G, m_line_bytes, kv_rows0, next_n_rows_q, &factx);
2354
+ }
2355
+ }
2356
+ }
2357
+ }
2358
+
2359
+ // ---- Final normalization ----
2360
+ {
2361
+ on_job.o_curr = o_tile_curr;
2362
+ on_job.o_prev = o_tile_prev;
2363
+ on_job.d_tiles = factx.vtcm_d_inv_l;
2364
+ on_job.hmx_scales = factx.vtcm_hmx_scales_id;
2365
+ on_job.n_row_tiles = n_row_tiles;
2366
+ on_job.n_row_tiles_g_br = n_row_tiles_g_br;
2367
+ on_job.DV = DV;
2368
+ hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_norm_worker, &on_job));
2369
+ hmx_queue_pop(ctx->hmx_queue);
2370
+ }
2371
+
2372
+ // ---- Store O block ----
2373
+ fa_phase_o_store(&factx, dst, o_tile_curr, q_start, kv_head, ib3, n_rows_g);
2374
+ }
2375
+ }
2376
+ }
2377
+
2378
+ return HTP_STATUS_OK;
2379
+ }
2380
+
2381
+ int op_flash_attn_ext(struct htp_ops_context * octx) {
2382
+ const struct htp_tensor * q = octx->src[0];
2383
+ const struct htp_tensor * k = octx->src[1];
2384
+ const struct htp_tensor * v = octx->src[2];
2385
+ const struct htp_tensor * mask = octx->src[3];
2386
+ const struct htp_tensor * dst = octx->dst;
2387
+
2388
+ // Check support
2389
+ if ((q->type != HTP_TYPE_F16 && q->type != HTP_TYPE_F32) ||
2390
+ (k->type != HTP_TYPE_F16 && k->type != HTP_TYPE_Q8_0) ||
2391
+ (v->type != HTP_TYPE_F16 && v->type != HTP_TYPE_Q8_0)) {
2392
+ return HTP_STATUS_NO_SUPPORT;
2393
+ }
2394
+
2395
+ const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params;
2396
+
2397
+ if (kparams->kernel_type == HTP_FA_KERNEL_UNSUPPORTED) {
2398
+ return HTP_STATUS_NO_SUPPORT;
2399
+ }
2400
+
2401
+ if (kparams->kernel_type == HTP_FA_KERNEL_HMX) {
2402
+ return hmx_flash_attn_ext(octx);
2403
+ }
2404
+
2405
+ struct htp_fa_context factx;
2406
+ factx.octx = octx;
2407
+ factx.k = k;
2408
+ factx.v = v;
2409
+
2410
+ factx.t_start = HAP_perf_get_qtimer_count();
2411
+
2412
+ factx.src0_div21 = kparams->u.hvx.src0_div21;
2413
+ factx.src0_div1 = kparams->u.hvx.src0_div1;
2414
+
2415
+ factx.broadcast_rk2 = kparams->broadcast_rk2;
2416
+ factx.broadcast_rk3 = kparams->broadcast_rk3;
2417
+ factx.broadcast_rv2 = kparams->broadcast_rv2;
2418
+ factx.broadcast_rv3 = kparams->broadcast_rv3;
2419
+
2420
+ if (mask) {
2421
+ factx.src3_div2 = kparams->src3_div2;
2422
+ factx.src3_div3 = kparams->src3_div3;
2423
+ }
2424
+
2425
+ factx.is_q_fp32 = (kparams->is_q_fp32 != 0);
2426
+ factx.size_q_row_padded = kparams->u.hvx.size_q_row_padded;
2427
+ factx.size_k_row_padded = kparams->u.hvx.size_k_row_padded;
2428
+ factx.size_v_row_padded = kparams->u.hvx.size_v_row_padded;
2429
+
2430
+ size_t size_q_block = factx.size_q_row_padded * 1; // single row for now
2431
+ factx.size_k_block = factx.size_k_row_padded * FLASH_ATTN_BLOCK_SIZE;
2432
+ factx.size_v_block = factx.size_v_row_padded * FLASH_ATTN_BLOCK_SIZE;
2433
+ factx.size_m_block = hex_round_up(FLASH_ATTN_BLOCK_SIZE * sizeof(__fp16), 128);
2434
+
2435
+ factx.n_blocks = kparams->n_kv_blocks;
2436
+
2437
+ factx.scale = kparams->scale;
2438
+ factx.max_bias = kparams->max_bias;
2439
+ factx.logit_softcap = (__fp16) kparams->logit_softcap;
2440
+
2441
+ factx.n_head_log2 = kparams->n_head_log2;
2442
+ factx.m0 = kparams->m0;
2443
+ factx.m1 = kparams->m1;
2444
+
2445
+ const uint32_t n_head = q->ne[2];
2446
+ if (n_head > 512) {
2447
+ return HTP_STATUS_NO_SUPPORT;
2448
+ }
2449
+ for (uint32_t h = 0; h < n_head; ++h) {
2450
+ factx.slopes[h] = (__fp16) ((kparams->max_bias > 0.0f) ? alibi_slope(h, factx.n_head_log2, factx.m0, factx.m1) : 1.0f);
2451
+ }
2452
+
2453
+ // total rows in q
2454
+ factx.qrows = kparams->qrows;
2455
+ factx.qrows_per_thread = kparams->qrows_per_thread;
2456
+
2457
+ size_t size_vkq_acc = hex_round_up(v->ne[0] * sizeof(float), 128); // VKQ32
2458
+
2459
+ factx.size_q_block = size_q_block;
2460
+ factx.size_vkq_acc = size_vkq_acc;
2461
+
2462
+ uint8_t * vtcm_cur = octx->ctx->vtcm_base;
2463
+
2464
+ factx.spad_q = vtcm_seq_alloc(&vtcm_cur, size_q_block * octx->n_threads);
2465
+ factx.spad_k = vtcm_seq_alloc(&vtcm_cur, factx.size_k_block * 2 * octx->n_threads);
2466
+ factx.spad_v = vtcm_seq_alloc(&vtcm_cur, factx.size_v_block * 2 * octx->n_threads);
2467
+ factx.spad_m = vtcm_seq_alloc(&vtcm_cur, (mask ? factx.size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * octx->n_threads);
2468
+ factx.spad_a = vtcm_seq_alloc(&vtcm_cur, size_vkq_acc * octx->n_threads);
2469
+
2470
+ if ((size_t) (vtcm_cur - octx->ctx->vtcm_base) > octx->ctx->vtcm_size) {
2471
+ return HTP_STATUS_VTCM_TOO_SMALL;
2472
+ }
2473
+
2474
+ if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {
2475
+ work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, octx->n_threads);
2476
+ }
2477
+
2478
+ return HTP_STATUS_OK;
2479
+ }