elelem-ggml 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/exe/elelem-ggml +8 -0
- data/ext/elelem/ggml/elelem.c +466 -0
- data/ext/elelem/ggml/extconf.rb +52 -0
- data/lib/elelem/ggml/chat_ml.rb +23 -0
- data/lib/elelem/ggml/gguf.rb +82 -0
- data/lib/elelem/ggml/provider.rb +63 -0
- data/lib/elelem/ggml/tokenizer.rb +95 -0
- data/lib/elelem/ggml/version.rb +7 -0
- data/lib/elelem/ggml.rb +37 -0
- data/vendor/ggml/AUTHORS +335 -0
- data/vendor/ggml/CMakeLists.txt +499 -0
- data/vendor/ggml/CONTRIBUTING.md +3 -0
- data/vendor/ggml/LICENSE +21 -0
- data/vendor/ggml/README.md +49 -0
- data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
- data/vendor/ggml/cmake/GitVars.cmake +22 -0
- data/vendor/ggml/cmake/common.cmake +50 -0
- data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
- data/vendor/ggml/ggml.pc.in +10 -0
- data/vendor/ggml/include/ggml-alloc.h +86 -0
- data/vendor/ggml/include/ggml-backend.h +437 -0
- data/vendor/ggml/include/ggml-blas.h +25 -0
- data/vendor/ggml/include/ggml-cann.h +123 -0
- data/vendor/ggml/include/ggml-cpp.h +39 -0
- data/vendor/ggml/include/ggml-cpu.h +152 -0
- data/vendor/ggml/include/ggml-cuda.h +47 -0
- data/vendor/ggml/include/ggml-et.h +28 -0
- data/vendor/ggml/include/ggml-hexagon.h +19 -0
- data/vendor/ggml/include/ggml-metal.h +61 -0
- data/vendor/ggml/include/ggml-opencl.h +26 -0
- data/vendor/ggml/include/ggml-openvino.h +37 -0
- data/vendor/ggml/include/ggml-opt.h +256 -0
- data/vendor/ggml/include/ggml-rpc.h +35 -0
- data/vendor/ggml/include/ggml-sycl.h +57 -0
- data/vendor/ggml/include/ggml-virtgpu.h +14 -0
- data/vendor/ggml/include/ggml-vulkan.h +29 -0
- data/vendor/ggml/include/ggml-webgpu.h +19 -0
- data/vendor/ggml/include/ggml-zdnn.h +17 -0
- data/vendor/ggml/include/ggml-zendnn.h +22 -0
- data/vendor/ggml/include/ggml.h +2963 -0
- data/vendor/ggml/include/gguf.h +211 -0
- data/vendor/ggml/requirements.txt +12 -0
- data/vendor/ggml/scripts/gen-authors.sh +9 -0
- data/vendor/ggml/scripts/make-release-checks.sh +96 -0
- data/vendor/ggml/scripts/make-release-desc.sh +69 -0
- data/vendor/ggml/scripts/make-release-summary.txt +33 -0
- data/vendor/ggml/scripts/release.sh +195 -0
- data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
- data/vendor/ggml/scripts/sync-llama.last +1 -0
- data/vendor/ggml/scripts/sync-llama.sh +21 -0
- data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
- data/vendor/ggml/scripts/sync-whisper.last +1 -0
- data/vendor/ggml/scripts/sync-whisper.sh +17 -0
- data/vendor/ggml/src/CMakeLists.txt +634 -0
- data/vendor/ggml/src/ggml-alloc.c +1249 -0
- data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
- data/vendor/ggml/src/ggml-backend-dl.h +45 -0
- data/vendor/ggml/src/ggml-backend-impl.h +291 -0
- data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
- data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
- data/vendor/ggml/src/ggml-backend.cpp +2517 -0
- data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
- data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
- data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
- data/vendor/ggml/src/ggml-cann/common.h +651 -0
- data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
- data/vendor/ggml/src/ggml-common.h +1911 -0
- data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
- data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
- data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
- data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
- data/vendor/ggml/src/ggml-cpu/common.h +95 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
- data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
- data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
- data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
- data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
- data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
- data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
- data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
- data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
- data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
- data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
- data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
- data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
- data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
- data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
- data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
- data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
- data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
- data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
- data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
- data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
- data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
- data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
- data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
- data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
- data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
- data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
- data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
- data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
- data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
- data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
- data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
- data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
- data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
- data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
- data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
- data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
- data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
- data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
- data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
- data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
- data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
- data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
- data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
- data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
- data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
- data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
- data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
- data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
- data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
- data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
- data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
- data/vendor/ggml/src/ggml-feats.h +166 -0
- data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
- data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
- data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
- data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
- data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
- data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
- data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
- data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
- data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
- data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
- data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
- data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
- data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
- data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
- data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
- data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
- data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
- data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
- data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
- data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
- data/vendor/ggml/src/ggml-impl.h +783 -0
- data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
- data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
- data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
- data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
- data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
- data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
- data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
- data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
- data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
- data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
- data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
- data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
- data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
- data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
- data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
- data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
- data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
- data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
- data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
- data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
- data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
- data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
- data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
- data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
- data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
- data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
- data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
- data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
- data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
- data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
- data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
- data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
- data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
- data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
- data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
- data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
- data/vendor/ggml/src/ggml-opt.cpp +1094 -0
- data/vendor/ggml/src/ggml-quants.c +5667 -0
- data/vendor/ggml/src/ggml-quants.h +115 -0
- data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
- data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
- data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
- data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
- data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
- data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
- data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
- data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
- data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
- data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
- data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
- data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
- data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
- data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
- data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
- data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
- data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
- data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
- data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
- data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
- data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
- data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
- data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
- data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
- data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
- data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
- data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
- data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
- data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
- data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
- data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
- data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
- data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
- data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
- data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
- data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
- data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
- data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
- data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
- data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
- data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
- data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
- data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
- data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
- data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
- data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
- data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
- data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
- data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
- data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/vendor/ggml/src/ggml-threading.cpp +12 -0
- data/vendor/ggml/src/ggml-threading.h +14 -0
- data/vendor/ggml/src/ggml-version.h.in +4 -0
- data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
- data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
- data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
- data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
- data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
- data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
- data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
- data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
- data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
- data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
- data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
- data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
- data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
- data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
- data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
- data/vendor/ggml/src/ggml.c +8088 -0
- data/vendor/ggml/src/ggml.cpp +26 -0
- data/vendor/ggml/src/gguf.cpp +1706 -0
- metadata +1459 -0
|
@@ -0,0 +1,2479 @@
|
|
|
1
|
+
#pragma clang diagnostic ignored "-Wunused-variable"
|
|
2
|
+
#pragma clang diagnostic ignored "-Wunused-function"
|
|
3
|
+
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
|
|
4
|
+
|
|
5
|
+
#include <assert.h>
|
|
6
|
+
#include <HAP_compute_res.h>
|
|
7
|
+
#include <HAP_farf.h>
|
|
8
|
+
#include <HAP_perf.h>
|
|
9
|
+
#include <math.h>
|
|
10
|
+
#include <stdbool.h>
|
|
11
|
+
#include <stdatomic.h>
|
|
12
|
+
#include <stddef.h>
|
|
13
|
+
#include <stdint.h>
|
|
14
|
+
#include <string.h>
|
|
15
|
+
|
|
16
|
+
#include "hex-dma.h"
|
|
17
|
+
#include "hex-fastdiv.h"
|
|
18
|
+
#include "hex-profile.h"
|
|
19
|
+
#include "hmx-queue.h"
|
|
20
|
+
#include "hmx-utils.h"
|
|
21
|
+
#include "hvx-utils.h"
|
|
22
|
+
#include "hvx-dump.h"
|
|
23
|
+
#include "hvx-copy.h"
|
|
24
|
+
#include "hvx-reduce.h"
|
|
25
|
+
#include "hvx-flash-attn.h"
|
|
26
|
+
#include "htp-vtcm.h"
|
|
27
|
+
#include "work-queue.h"
|
|
28
|
+
|
|
29
|
+
#define GGML_COMMON_DECL_C
|
|
30
|
+
#include "ggml-common.h"
|
|
31
|
+
#include "htp-ctx.h"
|
|
32
|
+
#include "htp-ops.h"
|
|
33
|
+
#include "htp-tensor.h"
|
|
34
|
+
#include "hvx-quant.h"
|
|
35
|
+
|
|
36
|
+
#include "flash-attn-ops.h"
|
|
37
|
+
#include "hvx-fa-kernels.h"
|
|
38
|
+
#include "hmx-fa-kernels.h"
|
|
39
|
+
|
|
40
|
+
// Must be multiple of 32
|
|
41
|
+
#define FLASH_ATTN_BLOCK_SIZE (32 * 2)
|
|
42
|
+
|
|
43
|
+
struct htp_fa_context {
|
|
44
|
+
const struct htp_ops_context * octx;
|
|
45
|
+
|
|
46
|
+
struct fastdiv_values src0_div21;
|
|
47
|
+
struct fastdiv_values src0_div1;
|
|
48
|
+
|
|
49
|
+
struct fastdiv_values broadcast_rk2;
|
|
50
|
+
struct fastdiv_values broadcast_rk3;
|
|
51
|
+
struct fastdiv_values broadcast_rv2;
|
|
52
|
+
struct fastdiv_values broadcast_rv3;
|
|
53
|
+
|
|
54
|
+
struct fastdiv_values src3_div2;
|
|
55
|
+
struct fastdiv_values src3_div3;
|
|
56
|
+
|
|
57
|
+
float scale;
|
|
58
|
+
float max_bias;
|
|
59
|
+
__fp16 logit_softcap;
|
|
60
|
+
|
|
61
|
+
uint32_t n_head_log2;
|
|
62
|
+
float m0;
|
|
63
|
+
float m1;
|
|
64
|
+
__fp16 slopes[512];
|
|
65
|
+
|
|
66
|
+
uint32_t n_blocks;
|
|
67
|
+
|
|
68
|
+
size_t size_q_row_padded;
|
|
69
|
+
size_t size_k_row_padded;
|
|
70
|
+
size_t size_v_row_padded;
|
|
71
|
+
|
|
72
|
+
size_t size_k_block;
|
|
73
|
+
size_t size_v_block;
|
|
74
|
+
size_t size_m_block;
|
|
75
|
+
|
|
76
|
+
uint32_t qrows;
|
|
77
|
+
uint32_t qrows_per_thread;
|
|
78
|
+
|
|
79
|
+
bool is_q_fp32;
|
|
80
|
+
|
|
81
|
+
size_t size_q_block;
|
|
82
|
+
size_t size_vkq_acc;
|
|
83
|
+
|
|
84
|
+
uint8_t * spad_q;
|
|
85
|
+
uint8_t * spad_k;
|
|
86
|
+
uint8_t * spad_v;
|
|
87
|
+
uint8_t * spad_m;
|
|
88
|
+
uint8_t * spad_a;
|
|
89
|
+
|
|
90
|
+
const struct htp_tensor * k;
|
|
91
|
+
const struct htp_tensor * v;
|
|
92
|
+
|
|
93
|
+
uint64_t t_start;
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
struct hmx_fa_context {
|
|
97
|
+
const struct htp_ops_context * octx;
|
|
98
|
+
const struct htp_tensor * sinks; // attention sinks (src[4]), NULL if absent
|
|
99
|
+
const struct htp_tensor * k;
|
|
100
|
+
const struct htp_tensor * v;
|
|
101
|
+
bool pipeline; // true when n_kv_blocks >= FA_MIN_KV_BLOCKS && n_threads >= 2
|
|
102
|
+
uint32_t n_threads;
|
|
103
|
+
|
|
104
|
+
// Op parameters
|
|
105
|
+
__fp16 scale;
|
|
106
|
+
float max_bias;
|
|
107
|
+
__fp16 logit_softcap;
|
|
108
|
+
uint32_t n_head_log2;
|
|
109
|
+
float m0, m1;
|
|
110
|
+
|
|
111
|
+
// Dimensions
|
|
112
|
+
uint32_t DK, DV;
|
|
113
|
+
uint32_t n_kv; // kv_len
|
|
114
|
+
uint32_t n_kv_heads; // number of KV heads
|
|
115
|
+
uint32_t n_heads; // number of Q heads
|
|
116
|
+
uint32_t G; // GQA factor = n_heads / n_kv_heads
|
|
117
|
+
struct fastdiv_values div_G;
|
|
118
|
+
struct fastdiv_values src3_div2;
|
|
119
|
+
struct fastdiv_values src3_div3;
|
|
120
|
+
uint32_t n_kv_blocks;
|
|
121
|
+
uint32_t neq1; // Q token count
|
|
122
|
+
|
|
123
|
+
// Types
|
|
124
|
+
bool is_q_fp32;
|
|
125
|
+
bool is_dst_fp32;
|
|
126
|
+
|
|
127
|
+
// Dynamic block sizes
|
|
128
|
+
uint32_t Br; // Q tokens per block (before GQA expansion)
|
|
129
|
+
uint32_t Bc;
|
|
130
|
+
uint32_t g_br; // hex_align_up(G * Br, 32) - actual tile row dim
|
|
131
|
+
|
|
132
|
+
// VTCM buffers (allocated by vtcm_seq_alloc)
|
|
133
|
+
__fp16 * vtcm_q_dma; // Q DMA fetch buffer
|
|
134
|
+
__fp16 * vtcm_q_tiles; // Q tile format [g_br, D]
|
|
135
|
+
__fp16 * vtcm_o_tiles[2]; // O ping-pong [g_br, D]
|
|
136
|
+
__fp16 * vtcm_k_fp16[2]; // K DMA double-buffer [Bc, D]
|
|
137
|
+
__fp16 * vtcm_v_fp16[2]; // V DMA double-buffer [Bc, D]
|
|
138
|
+
__fp16 * vtcm_k_tiles[2]; // K tiles (transposed, double-buffered)
|
|
139
|
+
__fp16 * vtcm_v_tiles[2]; // V tiles (column-major, double-buffered)
|
|
140
|
+
__fp16 * vtcm_s_tiles[2]; // S = QK^T [g_br, Bc] (double-buffered)
|
|
141
|
+
__fp16 * vtcm_p_tiles[2]; // P = softmax(S) [g_br, Bc]
|
|
142
|
+
__fp16 * vtcm_d_tiles[2]; // Diagonal rescale, g_br/32 packed diagonal tiles (double-buffered)
|
|
143
|
+
__fp16 * vtcm_d_inv_l; // Diagonal rescale (1/l), same packed layout
|
|
144
|
+
HVX_Vector * vtcm_m_vec; // Row max [g_br]
|
|
145
|
+
HVX_Vector * vtcm_l_vec; // Row sum [g_br]
|
|
146
|
+
HVX_Vector * vtcm_s_rowmax; // Softmax intermediate [g_br]
|
|
147
|
+
HVX_Vector * vtcm_p_rowsum; // Softmax intermediate [g_br]
|
|
148
|
+
HVX_Vector * vtcm_row_bufs; // Per-thread softmax row scratch [n_threads][2][Bc/64]
|
|
149
|
+
uint8_t * vtcm_hmx_scales_id; // HMX output scales (identity)
|
|
150
|
+
uint8_t * vtcm_hmx_scales_qk; // HMX output scales (qk_scale)
|
|
151
|
+
__fp16 * vtcm_mask_buf; // VTCM mask buffer [Br * m_line], DMA'd per KV block
|
|
152
|
+
__fp16 * vtcm_slopes; // ALiBi slopes [g_br]
|
|
153
|
+
size_t row_buf_stride; // HVX vectors per row buffer (Bc/64)
|
|
154
|
+
size_t mask_buf_row_stride; // elements (__fp16) per row in mask buffer
|
|
155
|
+
size_t q_tile_bytes;
|
|
156
|
+
size_t o_tile_bytes;
|
|
157
|
+
size_t col_vec_bytes;
|
|
158
|
+
size_t d_tile_bytes;
|
|
159
|
+
bool mask_broadcast; // true when mask->ne[2] == 1 (head-independent, single 2D DMA)
|
|
160
|
+
dma_cache m_cache;
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void * data) {
|
|
164
|
+
struct htp_fa_context * factx = (struct htp_fa_context *) data;
|
|
165
|
+
const struct htp_ops_context * octx = factx->octx;
|
|
166
|
+
const struct htp_tensor * q = octx->src[0];
|
|
167
|
+
const struct htp_tensor * k = octx->src[1];
|
|
168
|
+
const struct htp_tensor * v = octx->src[2];
|
|
169
|
+
const struct htp_tensor * mask = octx->src[3];
|
|
170
|
+
const struct htp_tensor * sinks = octx->src[4];
|
|
171
|
+
const struct htp_tensor * dst = octx->dst;
|
|
172
|
+
|
|
173
|
+
const uint32_t neq0 = q->ne[0];
|
|
174
|
+
const uint32_t neq1 = q->ne[1];
|
|
175
|
+
const uint32_t neq2 = q->ne[2];
|
|
176
|
+
const uint32_t neq3 = q->ne[3];
|
|
177
|
+
|
|
178
|
+
const uint32_t nek0 = k->ne[0];
|
|
179
|
+
const uint32_t nek1 = k->ne[1];
|
|
180
|
+
const uint32_t nek2 = k->ne[2];
|
|
181
|
+
const uint32_t nek3 = k->ne[3];
|
|
182
|
+
|
|
183
|
+
const uint32_t nev0 = v->ne[0];
|
|
184
|
+
const uint32_t nev1 = v->ne[1];
|
|
185
|
+
const uint32_t nev2 = v->ne[2];
|
|
186
|
+
const uint32_t nev3 = v->ne[3];
|
|
187
|
+
|
|
188
|
+
const uint32_t nbq1 = q->nb[1];
|
|
189
|
+
const uint32_t nbq2 = q->nb[2];
|
|
190
|
+
const uint32_t nbq3 = q->nb[3];
|
|
191
|
+
|
|
192
|
+
const uint32_t nbk1 = k->nb[1];
|
|
193
|
+
const uint32_t nbk2 = k->nb[2];
|
|
194
|
+
const uint32_t nbk3 = k->nb[3];
|
|
195
|
+
|
|
196
|
+
const uint32_t nbv1 = v->nb[1];
|
|
197
|
+
const uint32_t nbv2 = v->nb[2];
|
|
198
|
+
const uint32_t nbv3 = v->nb[3];
|
|
199
|
+
|
|
200
|
+
const uint32_t ne1 = dst->ne[1];
|
|
201
|
+
const uint32_t ne2 = dst->ne[2];
|
|
202
|
+
const uint32_t ne3 = dst->ne[3];
|
|
203
|
+
|
|
204
|
+
const uint32_t nb1 = dst->nb[1];
|
|
205
|
+
const uint32_t nb2 = dst->nb[2];
|
|
206
|
+
const uint32_t nb3 = dst->nb[3];
|
|
207
|
+
|
|
208
|
+
// total rows in q
|
|
209
|
+
const uint32_t nr = factx->qrows;
|
|
210
|
+
const uint32_t dr = factx->qrows_per_thread;
|
|
211
|
+
const uint32_t ir0 = dr * ith;
|
|
212
|
+
const uint32_t ir1 = MIN(ir0 + dr, nr);
|
|
213
|
+
|
|
214
|
+
if (ir0 >= ir1) return;
|
|
215
|
+
|
|
216
|
+
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
|
|
217
|
+
|
|
218
|
+
dma_queue * dma = octx->ctx->dma[ith];
|
|
219
|
+
|
|
220
|
+
const uint32_t DK = nek0;
|
|
221
|
+
const uint32_t DV = nev0;
|
|
222
|
+
|
|
223
|
+
const size_t size_q_row = DK * ((q->type == HTP_TYPE_F32) ? 4 : 2);
|
|
224
|
+
const size_t size_k_row = htp_tensor_get_row_size(k->type, DK);
|
|
225
|
+
const size_t size_v_row = htp_tensor_get_row_size(v->type, DV);
|
|
226
|
+
|
|
227
|
+
// Scratchpad buffers for Q, K, V, Mask, and VKQ32 accumulator
|
|
228
|
+
uint8_t * spad_q = factx->spad_q + factx->size_q_block * ith;
|
|
229
|
+
uint8_t * spad_k = factx->spad_k + factx->size_k_block * 2 * ith;
|
|
230
|
+
uint8_t * spad_v = factx->spad_v + factx->size_v_block * 2 * ith;
|
|
231
|
+
uint8_t * spad_m = factx->spad_m + (mask ? factx->size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * ith;
|
|
232
|
+
uint8_t * spad_a = factx->spad_a + factx->size_vkq_acc * ith;
|
|
233
|
+
|
|
234
|
+
dma_cache m_cache;
|
|
235
|
+
dma_cache_init(&m_cache, spad_m, factx->size_m_block, HVX_FA_DMA_CACHE_SIZE);
|
|
236
|
+
|
|
237
|
+
for (uint32_t ir = ir0; ir < ir1; ++ir) {
|
|
238
|
+
const uint32_t iq3 = fastdiv(ir, &factx->src0_div21);
|
|
239
|
+
const uint32_t iq2 = fastdiv(ir - iq3*neq2*neq1, &factx->src0_div1);
|
|
240
|
+
const uint32_t iq1 = (ir - iq3*neq2*neq1 - iq2 * neq1);
|
|
241
|
+
|
|
242
|
+
const uint32_t ik3 = fastdiv(iq3, &factx->broadcast_rk3);
|
|
243
|
+
const uint32_t ik2 = fastdiv(iq2, &factx->broadcast_rk2);
|
|
244
|
+
|
|
245
|
+
const uint32_t iv3 = fastdiv(iq3, &factx->broadcast_rv3);
|
|
246
|
+
const uint32_t iv2 = fastdiv(iq2, &factx->broadcast_rv2);
|
|
247
|
+
|
|
248
|
+
const __fp16 * mp_base = NULL;
|
|
249
|
+
if (mask) {
|
|
250
|
+
const uint32_t im2 = fastmodulo(iq2, mask->ne[2], &factx->src3_div2);
|
|
251
|
+
const uint32_t im3 = fastmodulo(iq3, mask->ne[3], &factx->src3_div3);
|
|
252
|
+
mp_base = (const __fp16 *) ((const uint8_t *) mask->data + iq1*mask->nb[1] + im2*mask->nb[2] + im3*mask->nb[3]);
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// Precalculate next row variables if there is a next row
|
|
256
|
+
bool has_next_ir = (ir + 1 < ir1);
|
|
257
|
+
uint32_t next_ik2 = 0, next_ik3 = 0, next_iv2 = 0, next_iv3 = 0;
|
|
258
|
+
const uint8_t * next_q_row_ptr = NULL;
|
|
259
|
+
const __fp16 * next_mp_base = NULL;
|
|
260
|
+
|
|
261
|
+
const uint8_t * next_k_src0 = NULL;
|
|
262
|
+
const uint8_t * next_v_src0 = NULL;
|
|
263
|
+
const uint8_t * next_m_src0 = NULL;
|
|
264
|
+
uint32_t next_block_size0 = 0;
|
|
265
|
+
|
|
266
|
+
const uint8_t * next_k_src1 = NULL;
|
|
267
|
+
const uint8_t * next_v_src1 = NULL;
|
|
268
|
+
const uint8_t * next_m_src1 = NULL;
|
|
269
|
+
uint32_t next_block_size1 = 0;
|
|
270
|
+
|
|
271
|
+
if (has_next_ir) {
|
|
272
|
+
const uint32_t next_ir = ir + 1;
|
|
273
|
+
const uint32_t next_iq3 = fastdiv(next_ir, &factx->src0_div21);
|
|
274
|
+
const uint32_t next_iq2 = fastdiv(next_ir - next_iq3*neq2*neq1, &factx->src0_div1);
|
|
275
|
+
const uint32_t next_iq1 = (next_ir - next_iq3*neq2*neq1 - next_iq2 * neq1);
|
|
276
|
+
|
|
277
|
+
next_ik3 = fastdiv(next_iq3, &factx->broadcast_rk3);
|
|
278
|
+
next_ik2 = fastdiv(next_iq2, &factx->broadcast_rk2);
|
|
279
|
+
|
|
280
|
+
next_iv3 = fastdiv(next_iq3, &factx->broadcast_rv3);
|
|
281
|
+
next_iv2 = fastdiv(next_iq2, &factx->broadcast_rv2);
|
|
282
|
+
|
|
283
|
+
next_q_row_ptr = (const uint8_t *) q->data + (next_iq1*nbq1 + next_iq2*nbq2 + next_iq3*nbq3);
|
|
284
|
+
|
|
285
|
+
if (mask) {
|
|
286
|
+
const uint32_t next_im2 = fastmodulo(next_iq2, mask->ne[2], &factx->src3_div2);
|
|
287
|
+
const uint32_t next_im3 = fastmodulo(next_iq3, mask->ne[3], &factx->src3_div3);
|
|
288
|
+
next_mp_base = (const __fp16 *) ((const uint8_t *) mask->data + next_iq1*mask->nb[1] + next_im2*mask->nb[2] + next_im3*mask->nb[3]);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// Precalculate next K/V block 0 source pointers
|
|
292
|
+
{
|
|
293
|
+
const uint32_t ic_start = 0;
|
|
294
|
+
next_block_size0 = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);
|
|
295
|
+
next_k_src0 = (const uint8_t *) k->data + (ic_start*nbk1 + next_ik2*nbk2 + next_ik3*nbk3);
|
|
296
|
+
next_v_src0 = (const uint8_t *) v->data + (ic_start*nbv1 + next_iv2*nbv2 + next_iv3*nbv3);
|
|
297
|
+
if (mask) {
|
|
298
|
+
next_m_src0 = (const uint8_t *) (next_mp_base + ic_start);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// Precalculate next K/V block 1 source pointers (if n_blocks > 1)
|
|
303
|
+
if (factx->n_blocks > 1) {
|
|
304
|
+
const uint32_t ic_start = 1 * FLASH_ATTN_BLOCK_SIZE;
|
|
305
|
+
next_block_size1 = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);
|
|
306
|
+
next_k_src1 = (const uint8_t *) k->data + (ic_start*nbk1 + next_ik2*nbk2 + next_ik3*nbk3);
|
|
307
|
+
next_v_src1 = (const uint8_t *) v->data + (ic_start*nbv1 + next_iv2*nbv2 + next_iv3*nbv3);
|
|
308
|
+
if (mask) {
|
|
309
|
+
next_m_src1 = (const uint8_t *) (next_mp_base + ic_start);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
if (ir == ir0) {
|
|
315
|
+
// Fetch Q row
|
|
316
|
+
const uint8_t * q_row_ptr = (const uint8_t *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3);
|
|
317
|
+
dma_queue_push(dma, dma_make_ptr(spad_q, q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1);
|
|
318
|
+
|
|
319
|
+
// Prefetch first two blocks
|
|
320
|
+
for (uint32_t ib = 0; ib < MIN(factx->n_blocks, 2); ++ib) {
|
|
321
|
+
const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE;
|
|
322
|
+
const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);
|
|
323
|
+
|
|
324
|
+
// K
|
|
325
|
+
const uint8_t * k_src = (const uint8_t *) k->data + (ic_start*nbk1 + ik2*nbk2 + ik3*nbk3);
|
|
326
|
+
uint8_t * k_dst = spad_k + (ib % 2) * factx->size_k_block;
|
|
327
|
+
dma_queue_push(dma, dma_make_ptr(k_dst, k_src), factx->size_k_row_padded, nbk1, size_k_row, current_block_size);
|
|
328
|
+
|
|
329
|
+
// V
|
|
330
|
+
const uint8_t * v_src = (const uint8_t *) v->data + (ic_start*nbv1 + iv2*nbv2 + iv3*nbv3);
|
|
331
|
+
uint8_t * v_dst = spad_v + (ib % 2) * factx->size_v_block;
|
|
332
|
+
dma_queue_push(dma, dma_make_ptr(v_dst, v_src), factx->size_v_row_padded, nbv1, size_v_row, current_block_size);
|
|
333
|
+
|
|
334
|
+
// Mask
|
|
335
|
+
if (mask) {
|
|
336
|
+
const uint8_t * m_src = (const uint8_t *) (mp_base + ic_start);
|
|
337
|
+
// Mask is 1D contiguous for this row
|
|
338
|
+
dma_cache_push(dma, &m_cache, m_src, current_block_size * 2, current_block_size * 2, current_block_size * 2, 1);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
const uint32_t h = iq2; // head index
|
|
344
|
+
const __fp16 slope = factx->slopes[h];
|
|
345
|
+
|
|
346
|
+
HVX_Vector S_vec = hvx_vec_splat_f32(0.0f);
|
|
347
|
+
HVX_Vector M_vec = hvx_vec_splat_f32(HTP_FA_M_INITIAL_VAL);
|
|
348
|
+
|
|
349
|
+
// Clear accumulator
|
|
350
|
+
hvx_splat_f32_a(spad_a, 0, DV);
|
|
351
|
+
float * VKQ32 = (float *) (spad_a + 0);
|
|
352
|
+
|
|
353
|
+
uint8_t * q_ptr_vtcm = dma_queue_pop(dma).dst;
|
|
354
|
+
if (factx->is_q_fp32) {
|
|
355
|
+
hvx_copy_f16_f32_aa(q_ptr_vtcm, q_ptr_vtcm, DK); // inplace convert f32 to f16
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const HVX_Vector slope_vec = hvx_vec_splat_f16(slope);
|
|
359
|
+
const HVX_Vector v_neg_inf = Q6_Vh_vsplat_R(0xfbff);
|
|
360
|
+
const HVX_Vector v_cap = (factx->logit_softcap != 0.0f) ? hvx_vec_splat_f16(factx->logit_softcap) : Q6_V_vzero();
|
|
361
|
+
const HVX_Vector vinf = Q6_Vh_vsplat_R(0xFC00);
|
|
362
|
+
const HVX_Vector vmin = Q6_Vh_vsplat_R(0xFBFF);
|
|
363
|
+
const HVX_Vector v_log2e = hvx_vec_splat_f16(EXP_LOG2E_F);
|
|
364
|
+
const uint32_t stride_v2 = factx->size_v_row_padded * 2;
|
|
365
|
+
for (uint32_t ib = 0; ib < factx->n_blocks; ++ib) {
|
|
366
|
+
const uint32_t ic_start = ib * FLASH_ATTN_BLOCK_SIZE;
|
|
367
|
+
const uint32_t current_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - ic_start);
|
|
368
|
+
|
|
369
|
+
// Wait for DMA
|
|
370
|
+
uint8_t * k_base = dma_queue_pop(dma).dst; // K
|
|
371
|
+
uint8_t * v_base = dma_queue_pop(dma).dst; // V
|
|
372
|
+
__fp16 * m_base = mask ? dma_queue_pop(dma).dst : NULL; // M
|
|
373
|
+
|
|
374
|
+
if (factx->k->type == HTP_TYPE_Q8_0) {
|
|
375
|
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir);
|
|
376
|
+
for (uint32_t r = 0; r < current_block_size; ++r) {
|
|
377
|
+
__fp16 * row_k = (__fp16 *)(k_base + r * factx->size_k_row_padded);
|
|
378
|
+
hvx_dequantize_row_q8_0_f16(row_k, row_k, DK);
|
|
379
|
+
}
|
|
380
|
+
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, ir);
|
|
381
|
+
}
|
|
382
|
+
if (factx->v->type == HTP_TYPE_Q8_0) {
|
|
383
|
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir);
|
|
384
|
+
for (uint32_t r = 0; r < current_block_size; ++r) {
|
|
385
|
+
__fp16 * row_v = (__fp16 *)(v_base + r * factx->size_v_row_padded);
|
|
386
|
+
hvx_dequantize_row_q8_0_f16(row_v, row_v, DV);
|
|
387
|
+
}
|
|
388
|
+
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, ir);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_QK, ir);
|
|
392
|
+
|
|
393
|
+
// Inner loop processing the block from VTCM
|
|
394
|
+
// 1. Compute scores (64 elements FP16)
|
|
395
|
+
HVX_Vector scores_f16 = Q6_V_vzero();
|
|
396
|
+
if (current_block_size > 0) {
|
|
397
|
+
HVX_Vector scores0 = hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base, factx->size_k_row_padded, DK, factx->scale);
|
|
398
|
+
HVX_Vector scores1 = (current_block_size > 32) ? hvx_dot_f16_f16_aa_rx32(q_ptr_vtcm, k_base + 32 * factx->size_k_row_padded, factx->size_k_row_padded, DK, factx->scale) : Q6_V_vzero();
|
|
399
|
+
scores_f16 = hvx_vec_f32_to_f16(scores0, scores1);
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
// 2. Softcap (in FP16)
|
|
403
|
+
if (factx->logit_softcap != 0.0f) {
|
|
404
|
+
scores_f16 = hvx_vec_tanh_f16(scores_f16);
|
|
405
|
+
scores_f16 = hvx_vec_mul_f16_f16(scores_f16, v_cap);
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(current_block_size * sizeof(__fp16));
|
|
409
|
+
|
|
410
|
+
// 3. Mask (in FP16)
|
|
411
|
+
if (mask) {
|
|
412
|
+
HVX_Vector m_vals_f16 = *(const HVX_UVector *) m_base;
|
|
413
|
+
HVX_VectorPred is_inf = Q6_Q_vcmp_eq_VhVh(m_vals_f16, vinf);
|
|
414
|
+
m_vals_f16 = Q6_V_vmux_QVV(is_inf, vmin, m_vals_f16);
|
|
415
|
+
|
|
416
|
+
HVX_Vector m_scaled = hvx_vec_mul_f16_f16(m_vals_f16, slope_vec);
|
|
417
|
+
scores_f16 = Q6_V_vmux_QVV(q_tail_keep, hvx_vec_add_f16_f16(scores_f16, m_scaled), v_neg_inf);
|
|
418
|
+
} else {
|
|
419
|
+
scores_f16 = Q6_V_vmux_QVV(q_tail_keep, scores_f16, v_neg_inf);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// Compute block max in FP16
|
|
423
|
+
HVX_Vector v_max_f16 = hvx_vec_reduce_max_f16(scores_f16);
|
|
424
|
+
HVX_Vector v_max = Q6_V_lo_W(hvx_vec_f16_to_f32(v_max_f16)); // splat block max in FP32
|
|
425
|
+
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_QK, ir);
|
|
426
|
+
|
|
427
|
+
if (ib + 1 == factx->n_blocks && has_next_ir) {
|
|
428
|
+
// Queue next row's Q row!
|
|
429
|
+
dma_queue_push(dma, dma_make_ptr(spad_q, next_q_row_ptr), factx->size_q_row_padded, nbq1, size_q_row, 1);
|
|
430
|
+
|
|
431
|
+
if (factx->n_blocks % 2 == 0) {
|
|
432
|
+
// Queue next row's block 0 (into buffer slot 0)
|
|
433
|
+
uint8_t * k_dst = spad_k + 0 * factx->size_k_block;
|
|
434
|
+
uint8_t * v_dst = spad_v + 0 * factx->size_v_block;
|
|
435
|
+
|
|
436
|
+
// K (block 0 of next row)
|
|
437
|
+
dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src0), factx->size_k_row_padded, nbk1, size_k_row, next_block_size0);
|
|
438
|
+
|
|
439
|
+
// V (block 0 of next row)
|
|
440
|
+
dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src0), factx->size_v_row_padded, nbv1, size_v_row, next_block_size0);
|
|
441
|
+
|
|
442
|
+
// Mask (block 0 of next row)
|
|
443
|
+
if (mask) {
|
|
444
|
+
dma_cache_push(dma, &m_cache, next_m_src0, next_block_size0 * 2, next_block_size0 * 2, next_block_size0 * 2, 1);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, ir);
|
|
450
|
+
{
|
|
451
|
+
// 4. Online Softmax Update
|
|
452
|
+
HVX_Vector M_new_vec = Q6_Vsf_vmax_VsfVsf(v_max, M_vec);
|
|
453
|
+
HVX_Vector diff_vec = HVX_OP_SUB_F32(M_vec, M_new_vec);
|
|
454
|
+
|
|
455
|
+
HVX_Vector diff_f16 = hvx_vec_f32_to_f16(diff_vec, diff_vec);
|
|
456
|
+
HVX_Vector diff_base2 = hvx_vec_mul_f16_f16(diff_f16, v_log2e);
|
|
457
|
+
HVX_Vector ms_f16 = hvx_vec_exp2_f16(diff_base2);
|
|
458
|
+
HVX_Vector ms_vec = Q6_V_lo_W(hvx_vec_f16_to_f32(ms_f16));
|
|
459
|
+
|
|
460
|
+
M_vec = M_new_vec;
|
|
461
|
+
|
|
462
|
+
hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec);
|
|
463
|
+
|
|
464
|
+
// Compute P = exp2((S - M) * log2(e)) in FP16
|
|
465
|
+
HVX_Vector v_m_vec_f16 = hvx_vec_f32_to_f16(M_vec, M_vec);
|
|
466
|
+
HVX_Vector v_s_minus_m = Q6_Vqf16_vsub_VhfVhf(scores_f16, v_m_vec_f16);
|
|
467
|
+
|
|
468
|
+
HVX_Vector v_s_minus_m_base2 = hvx_vec_mul_f16_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m), v_log2e);
|
|
469
|
+
|
|
470
|
+
HVX_Vector P = hvx_vec_exp2_f16(v_s_minus_m_base2);
|
|
471
|
+
P = Q6_V_vmux_QVV(q_tail_keep, P, Q6_V_vzero());
|
|
472
|
+
|
|
473
|
+
// Convert P to FP32 to update the running sum S_vec
|
|
474
|
+
HVX_VectorPair P_pair = hvx_vec_f16_to_f32(P);
|
|
475
|
+
HVX_Vector P0 = Q6_V_lo_W(P_pair);
|
|
476
|
+
HVX_Vector P1 = Q6_V_hi_W(P_pair);
|
|
477
|
+
HVX_Vector p_sum_vec = hvx_vec_reduce_sum_f32(HVX_OP_ADD_F32(P0, P1));
|
|
478
|
+
|
|
479
|
+
S_vec = HVX_OP_ADD_F32(HVX_OP_MUL_F32(S_vec, ms_vec), p_sum_vec);
|
|
480
|
+
|
|
481
|
+
// 5. Accumulate V (F16 * F16 -> F32 accumulator)
|
|
482
|
+
const uint8_t * v_ptr = v_base;
|
|
483
|
+
|
|
484
|
+
for (uint32_t j = 0; j < current_block_size; j += 2) {
|
|
485
|
+
if (j + 1 == current_block_size) {
|
|
486
|
+
HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2));
|
|
487
|
+
hvx_mad_f32_f16_aa_vec(VKQ32, v_ptr, S0, DV);
|
|
488
|
+
break;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
HVX_Vector S0 = hvx_vec_repl_f16(Q6_V_vror_VR(P, j * 2));
|
|
492
|
+
HVX_Vector S1 = hvx_vec_repl_f16(Q6_V_vror_VR(P, (j + 1) * 2));
|
|
493
|
+
|
|
494
|
+
hvx_mad_f32_f16_aa_rx2_vec(VKQ32, v_ptr, v_ptr + factx->size_v_row_padded, S0, S1, DV);
|
|
495
|
+
v_ptr += stride_v2;
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, ir);
|
|
499
|
+
|
|
500
|
+
// Issue DMA for next+1 block (if exists)
|
|
501
|
+
if (ib + 2 < factx->n_blocks) {
|
|
502
|
+
const uint32_t next_ib = ib + 2;
|
|
503
|
+
const uint32_t next_ic_start = next_ib * FLASH_ATTN_BLOCK_SIZE;
|
|
504
|
+
const uint32_t next_block_size = MIN(FLASH_ATTN_BLOCK_SIZE, nek1 - next_ic_start);
|
|
505
|
+
|
|
506
|
+
// K
|
|
507
|
+
const uint8_t * k_src = (const uint8_t *) k->data + (next_ic_start*nbk1 + ik2*nbk2 + ik3*nbk3);
|
|
508
|
+
dma_queue_push(dma, dma_make_ptr(k_base, k_src), factx->size_k_row_padded, nbk1, size_k_row, next_block_size);
|
|
509
|
+
|
|
510
|
+
// V
|
|
511
|
+
const uint8_t * v_src = (const uint8_t *) v->data + (next_ic_start*nbv1 + iv2*nbv2 + iv3*nbv3);
|
|
512
|
+
dma_queue_push(dma, dma_make_ptr(v_base, v_src), factx->size_v_row_padded, nbv1, size_v_row, next_block_size);
|
|
513
|
+
|
|
514
|
+
// Mask
|
|
515
|
+
if (mask) {
|
|
516
|
+
const uint8_t * m_src = (const uint8_t *) (mp_base + next_ic_start);
|
|
517
|
+
dma_cache_push(dma, &m_cache, m_src, next_block_size * 2, next_block_size * 2, next_block_size * 2, 1);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
if (has_next_ir) {
|
|
523
|
+
if (factx->n_blocks % 2 == 0) {
|
|
524
|
+
// Queue next row's block 1 (into buffer slot 1, if n_blocks > 1)
|
|
525
|
+
if (factx->n_blocks > 1) {
|
|
526
|
+
uint8_t * k_dst = spad_k + 1 * factx->size_k_block;
|
|
527
|
+
uint8_t * v_dst = spad_v + 1 * factx->size_v_block;
|
|
528
|
+
|
|
529
|
+
// K (block 1 of next row)
|
|
530
|
+
dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src1), factx->size_k_row_padded, nbk1, size_k_row, next_block_size1);
|
|
531
|
+
|
|
532
|
+
// V (block 1 of next row)
|
|
533
|
+
dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src1), factx->size_v_row_padded, nbv1, size_v_row, next_block_size1);
|
|
534
|
+
|
|
535
|
+
// Mask (block 1 of next row)
|
|
536
|
+
if (mask) {
|
|
537
|
+
dma_cache_push(dma, &m_cache, next_m_src1, next_block_size1 * 2, next_block_size1 * 2, next_block_size1 * 2, 1);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
} else {
|
|
541
|
+
// Queue next row's block 0 (into buffer slot 0)
|
|
542
|
+
{
|
|
543
|
+
uint8_t * k_dst = spad_k + 0 * factx->size_k_block;
|
|
544
|
+
uint8_t * v_dst = spad_v + 0 * factx->size_v_block;
|
|
545
|
+
|
|
546
|
+
// K (block 0 of next row)
|
|
547
|
+
dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src0), factx->size_k_row_padded, nbk1, size_k_row, next_block_size0);
|
|
548
|
+
|
|
549
|
+
// V (block 0 of next row)
|
|
550
|
+
dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src0), factx->size_v_row_padded, nbv1, size_v_row, next_block_size0);
|
|
551
|
+
|
|
552
|
+
// Mask (block 0 of next row)
|
|
553
|
+
if (mask) {
|
|
554
|
+
dma_cache_push(dma, &m_cache, next_m_src0, next_block_size0 * 2, next_block_size0 * 2, next_block_size0 * 2, 1);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// Queue next row's block 1 (into buffer slot 1, if n_blocks > 1)
|
|
559
|
+
if (factx->n_blocks > 1) {
|
|
560
|
+
uint8_t * k_dst = spad_k + 1 * factx->size_k_block;
|
|
561
|
+
uint8_t * v_dst = spad_v + 1 * factx->size_v_block;
|
|
562
|
+
|
|
563
|
+
// K (block 1 of next row)
|
|
564
|
+
dma_queue_push(dma, dma_make_ptr(k_dst, next_k_src1), factx->size_k_row_padded, nbk1, size_k_row, next_block_size1);
|
|
565
|
+
|
|
566
|
+
// V (block 1 of next row)
|
|
567
|
+
dma_queue_push(dma, dma_make_ptr(v_dst, next_v_src1), factx->size_v_row_padded, nbv1, size_v_row, next_block_size1);
|
|
568
|
+
|
|
569
|
+
// Mask (block 1 of next row)
|
|
570
|
+
if (mask) {
|
|
571
|
+
dma_cache_push(dma, &m_cache, next_m_src1, next_block_size1 * 2, next_block_size1 * 2, next_block_size1 * 2, 1);
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, ir);
|
|
578
|
+
// sinks
|
|
579
|
+
float M = hvx_vec_get_f32(M_vec);
|
|
580
|
+
float S = hvx_vec_get_f32(S_vec);
|
|
581
|
+
|
|
582
|
+
if (sinks) {
|
|
583
|
+
const float s = ((float *)((char *) sinks->data))[h];
|
|
584
|
+
|
|
585
|
+
float vs = 1.0f;
|
|
586
|
+
|
|
587
|
+
if (s > M) {
|
|
588
|
+
HVX_Vector diff_vec = hvx_vec_splat_f32(M - s);
|
|
589
|
+
HVX_Vector ms_vec = hvx_vec_exp_f32(diff_vec);
|
|
590
|
+
hvx_scale_vec_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, ms_vec);
|
|
591
|
+
|
|
592
|
+
float ms = hvx_vec_get_f32(ms_vec);
|
|
593
|
+
S = S * ms + vs;
|
|
594
|
+
} else {
|
|
595
|
+
HVX_Vector diff_vec = hvx_vec_splat_f32(s - M);
|
|
596
|
+
vs = hvx_vec_get_f32(hvx_vec_exp_f32(diff_vec));
|
|
597
|
+
S += vs;
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
const float S_inv = S == 0.0f ? 0.0f : 1.0f/S;
|
|
602
|
+
hvx_scale_f32_aa((uint8_t *) VKQ32, (const uint8_t *) VKQ32, DV, S_inv);
|
|
603
|
+
|
|
604
|
+
// Store result
|
|
605
|
+
// dst indices
|
|
606
|
+
const uint32_t i1 = iq1;
|
|
607
|
+
const uint32_t i2 = iq2;
|
|
608
|
+
const uint32_t i3 = iq3;
|
|
609
|
+
|
|
610
|
+
// dst is permuted: [DV, n_heads, n_tokens, n_seq]
|
|
611
|
+
// head stride is nb[1], token stride is nb[2], batch stride is nb[3]
|
|
612
|
+
uint8_t * dst_ptr = (uint8_t *) dst->data + i2 * dst->nb[1] + i1 * dst->nb[2] + i3 * dst->nb[3];
|
|
613
|
+
|
|
614
|
+
if (dst->type == HTP_TYPE_F32) {
|
|
615
|
+
hvx_copy_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV);
|
|
616
|
+
} else if (dst->type == HTP_TYPE_F16) {
|
|
617
|
+
hvx_copy_f16_f32_ua(dst_ptr, (uint8_t *) VKQ32, DV);
|
|
618
|
+
}
|
|
619
|
+
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, ir);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// ============================================================================
|
|
624
|
+
// HMX Phase args and thread logic
|
|
625
|
+
// ============================================================================
|
|
626
|
+
|
|
627
|
+
typedef struct {
|
|
628
|
+
struct hmx_fa_context * factx;
|
|
629
|
+
uint32_t kv_rows;
|
|
630
|
+
size_t src_stride;
|
|
631
|
+
void * curr_k;
|
|
632
|
+
uint32_t kv_start;
|
|
633
|
+
uint32_t rows_per_t;
|
|
634
|
+
size_t buf_idx;
|
|
635
|
+
} fa_k_int_args_t;
|
|
636
|
+
|
|
637
|
+
static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data) {
|
|
638
|
+
fa_k_int_args_t * args = (fa_k_int_args_t *) data;
|
|
639
|
+
struct hmx_fa_context * factx = args->factx;
|
|
640
|
+
|
|
641
|
+
const uint32_t total_rows = args->kv_rows;
|
|
642
|
+
const uint32_t rows_per_t = args->rows_per_t;
|
|
643
|
+
const uint32_t start = i * rows_per_t;
|
|
644
|
+
const uint32_t end = (uint32_t) hex_smin(start + rows_per_t, total_rows);
|
|
645
|
+
|
|
646
|
+
if (start >= total_rows) {
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
|
|
651
|
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start));
|
|
652
|
+
if (factx->k->type == HTP_TYPE_Q8_0) {
|
|
653
|
+
for (uint32_t r = start; r < end; ++r) {
|
|
654
|
+
__fp16 * row_k = (__fp16 *)((char *)args->curr_k + r * args->src_stride * sizeof(__fp16));
|
|
655
|
+
hvx_dequantize_row_q8_0_f16(row_k, row_k, factx->DK);
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles[args->buf_idx], (const __fp16 *) args->curr_k, total_rows, factx->DK,
|
|
659
|
+
args->src_stride, start, end);
|
|
660
|
+
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start));
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start, size_t buf_idx) {
|
|
664
|
+
work_queue_t wp = factx->octx->ctx->work_queue;
|
|
665
|
+
uint32_t n = 1;
|
|
666
|
+
if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) {
|
|
667
|
+
n = factx->n_threads;
|
|
668
|
+
}
|
|
669
|
+
uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2);
|
|
670
|
+
fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t, buf_idx };
|
|
671
|
+
if (n > 1) {
|
|
672
|
+
work_queue_run(wp, fa_k_interleave_thread, &args, n);
|
|
673
|
+
} else {
|
|
674
|
+
fa_k_interleave_thread(1, 0, &args);
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
typedef struct {
|
|
679
|
+
struct hmx_fa_context * factx;
|
|
680
|
+
uint32_t kv_rows;
|
|
681
|
+
size_t src_stride;
|
|
682
|
+
void * v_src;
|
|
683
|
+
void * v_tiles_dst;
|
|
684
|
+
size_t n_col_tiles;
|
|
685
|
+
uint32_t kv_start;
|
|
686
|
+
uint32_t rows_per_t;
|
|
687
|
+
} fa_v_int_args_t;
|
|
688
|
+
|
|
689
|
+
static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data) {
|
|
690
|
+
fa_v_int_args_t * args = (fa_v_int_args_t *) data;
|
|
691
|
+
struct hmx_fa_context * factx = args->factx;
|
|
692
|
+
|
|
693
|
+
const uint32_t total_rows = args->kv_rows;
|
|
694
|
+
const uint32_t rows_per_t = args->rows_per_t;
|
|
695
|
+
const uint32_t start = i * rows_per_t;
|
|
696
|
+
const uint32_t end = (uint32_t) hex_smin(start + rows_per_t, total_rows);
|
|
697
|
+
|
|
698
|
+
if (start >= total_rows) {
|
|
699
|
+
return;
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
__fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst;
|
|
703
|
+
|
|
704
|
+
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
|
|
705
|
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start));
|
|
706
|
+
if (factx->v->type == HTP_TYPE_Q8_0) {
|
|
707
|
+
for (uint32_t r = start; r < end; ++r) {
|
|
708
|
+
__fp16 * row_v = (__fp16 *)((char *)args->v_src + r * args->src_stride * sizeof(__fp16));
|
|
709
|
+
hvx_dequantize_row_q8_0_f16(row_v, row_v, factx->DV);
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV,
|
|
713
|
+
args->src_stride, (uint32_t) args->n_col_tiles, start, end);
|
|
714
|
+
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start));
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
static void fa_phase_v_interleave(struct hmx_fa_context * factx,
|
|
718
|
+
uint32_t kv_rows,
|
|
719
|
+
size_t src_stride,
|
|
720
|
+
void * v_src,
|
|
721
|
+
void * v_tiles_dst,
|
|
722
|
+
size_t n_col_tiles,
|
|
723
|
+
uint32_t kv_start) {
|
|
724
|
+
work_queue_t wp = factx->octx->ctx->work_queue;
|
|
725
|
+
uint32_t n = 1;
|
|
726
|
+
if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) {
|
|
727
|
+
n = factx->n_threads;
|
|
728
|
+
}
|
|
729
|
+
uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2);
|
|
730
|
+
fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t };
|
|
731
|
+
if (n > 1) {
|
|
732
|
+
work_queue_run(wp, fa_v_interleave_thread, &args, n);
|
|
733
|
+
} else {
|
|
734
|
+
fa_v_interleave_thread(1, 0, &args);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
typedef struct {
|
|
739
|
+
struct hmx_fa_context * factx;
|
|
740
|
+
const struct htp_tensor * q;
|
|
741
|
+
uint32_t q_start;
|
|
742
|
+
uint32_t kv_head;
|
|
743
|
+
uint32_t ib3;
|
|
744
|
+
size_t n_rows_g;
|
|
745
|
+
size_t rows_per_t;
|
|
746
|
+
size_t n_rows_q;
|
|
747
|
+
bool q_transposed;
|
|
748
|
+
atomic_uint barrier;
|
|
749
|
+
} fa_q_load_args_t;
|
|
750
|
+
|
|
751
|
+
static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) {
|
|
752
|
+
fa_q_load_args_t * args = (fa_q_load_args_t *) data;
|
|
753
|
+
struct hmx_fa_context * factx = args->factx;
|
|
754
|
+
|
|
755
|
+
const size_t n_rows_g = args->n_rows_g;
|
|
756
|
+
const size_t G = factx->G;
|
|
757
|
+
const size_t DK = factx->DK;
|
|
758
|
+
|
|
759
|
+
// Partition the padded Q rows (g_br) across threads.
|
|
760
|
+
// Keep start/end even so r and r+1 are always in the same thread's range.
|
|
761
|
+
const size_t rows_per_t = args->rows_per_t;
|
|
762
|
+
const size_t start = (size_t) i * rows_per_t;
|
|
763
|
+
const size_t end = hex_smin(start + rows_per_t, factx->g_br);
|
|
764
|
+
|
|
765
|
+
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
|
|
766
|
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start));
|
|
767
|
+
|
|
768
|
+
// Parallel initialization of per-block state
|
|
769
|
+
{
|
|
770
|
+
const uint32_t g_br = factx->g_br;
|
|
771
|
+
const uint32_t DV = factx->DV;
|
|
772
|
+
|
|
773
|
+
const size_t col_vec_bytes = factx->col_vec_bytes;
|
|
774
|
+
const size_t d_tile_bytes = factx->d_tile_bytes;
|
|
775
|
+
|
|
776
|
+
// Initialize vtcm_l_vec & vtcm_m_vec
|
|
777
|
+
const size_t l_bytes_per_t = hex_align_up(col_vec_bytes / n, 128);
|
|
778
|
+
const size_t l_start = i * l_bytes_per_t;
|
|
779
|
+
const size_t l_end = hex_smin(l_start + l_bytes_per_t, col_vec_bytes);
|
|
780
|
+
|
|
781
|
+
const size_t m_bytes_per_t = hex_align_up(col_vec_bytes / n, 128);
|
|
782
|
+
const size_t m_start = i * m_bytes_per_t;
|
|
783
|
+
const size_t m_end = hex_smin(m_start + m_bytes_per_t, col_vec_bytes);
|
|
784
|
+
|
|
785
|
+
if (factx->sinks) {
|
|
786
|
+
const float * sinks_data = (const float *) (uintptr_t) factx->sinks->data;
|
|
787
|
+
float * m_vec = (float *) factx->vtcm_m_vec;
|
|
788
|
+
const size_t r_start = l_start / sizeof(float);
|
|
789
|
+
const size_t r_end = l_end / sizeof(float);
|
|
790
|
+
const float scale_factor = EXP_LOG2E_F;
|
|
791
|
+
|
|
792
|
+
const HVX_Vector v_scale = hvx_vec_splat_f32(scale_factor);
|
|
793
|
+
|
|
794
|
+
for (size_t r = r_start; r < r_end; r += 32) {
|
|
795
|
+
HVX_VectorAlias local_m;
|
|
796
|
+
for (size_t j = 0; j < 32; ++j) {
|
|
797
|
+
size_t curr_r = r + j;
|
|
798
|
+
if (curr_r < n_rows_g) {
|
|
799
|
+
const size_t h_idx = fastmodulo(curr_r, G, &factx->div_G);
|
|
800
|
+
const size_t head = args->kv_head * G + h_idx;
|
|
801
|
+
local_m.fp32[j] = sinks_data[head];
|
|
802
|
+
} else {
|
|
803
|
+
local_m.fp32[j] = HTP_FA_M_INITIAL_VAL;
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
HVX_Vector v_scaled = HVX_OP_MUL_F32(local_m.v, v_scale);
|
|
807
|
+
*(HVX_Vector *) (m_vec + r) = v_scaled;
|
|
808
|
+
}
|
|
809
|
+
if (l_start < col_vec_bytes) {
|
|
810
|
+
hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start);
|
|
811
|
+
}
|
|
812
|
+
} else {
|
|
813
|
+
if (l_start < col_vec_bytes) {
|
|
814
|
+
hvx_splat_u8_a((char *) factx->vtcm_l_vec + l_start, 0, l_end - l_start);
|
|
815
|
+
}
|
|
816
|
+
if (m_start < col_vec_bytes) {
|
|
817
|
+
hvx_splat_f32_a((char *) factx->vtcm_m_vec + m_start, HTP_FA_M_INITIAL_VAL, (m_end - m_start) / sizeof(float));
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
// Zero the whole rescale region: vtcm_d_tiles[0], the optional vtcm_d_tiles[1]
|
|
822
|
+
// and vtcm_d_inv_l are equal-sized and allocated back to back, so one run covers
|
|
823
|
+
// them all. The scatter only ever writes the diagonal, ignore the rest.
|
|
824
|
+
const size_t d_bytes_per_t = hex_align_up(d_tile_bytes / n, 128);
|
|
825
|
+
const size_t d_start = i * d_bytes_per_t;
|
|
826
|
+
const size_t d_end = hex_smin(d_start + d_bytes_per_t, d_tile_bytes);
|
|
827
|
+
if (d_start < d_tile_bytes) {
|
|
828
|
+
hvx_splat_u8_a((char *) factx->vtcm_d_tiles[0] + d_start, 0, d_end - d_start);
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
if (start < factx->g_br) {
|
|
833
|
+
const struct htp_tensor * q = args->q;
|
|
834
|
+
const uint32_t q_start = args->q_start;
|
|
835
|
+
const uint32_t kv_head = args->kv_head;
|
|
836
|
+
const uint32_t ib3 = args->ib3;
|
|
837
|
+
|
|
838
|
+
assert(factx->DK == factx->DV);
|
|
839
|
+
|
|
840
|
+
const bool use_q_dma = (factx->vtcm_q_dma != NULL);
|
|
841
|
+
|
|
842
|
+
__fp16 * q_tiles = factx->vtcm_q_tiles;
|
|
843
|
+
if (use_q_dma) {
|
|
844
|
+
const size_t g_rows_end = hex_smin(end, n_rows_g);
|
|
845
|
+
const uint32_t d_limit = factx->is_q_fp32 ? DK / 32 : DK / 64;
|
|
846
|
+
|
|
847
|
+
uint8_t * q_flat = (uint8_t *) factx->vtcm_q_dma;
|
|
848
|
+
if (factx->is_q_fp32) {
|
|
849
|
+
switch (d_limit) {
|
|
850
|
+
case 2: hmx_fa_q_prep_fp32_d2(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break;
|
|
851
|
+
case 4: hmx_fa_q_prep_fp32_d4(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break;
|
|
852
|
+
default: hmx_fa_q_prep_fp32( q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, d_limit, args->q_transposed); break;
|
|
853
|
+
}
|
|
854
|
+
} else {
|
|
855
|
+
switch (d_limit) {
|
|
856
|
+
case 1: hmx_fa_q_prep_fp16_d1(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break;
|
|
857
|
+
case 2: hmx_fa_q_prep_fp16_d2(q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, args->q_transposed); break;
|
|
858
|
+
default: hmx_fa_q_prep_fp16( q_tiles, q_flat, start, end, g_rows_end, DK, G, args->n_rows_q, &factx->div_G, d_limit, args->q_transposed); break;
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
} else {
|
|
862
|
+
// Fallback: direct-from-DDR/L2 path
|
|
863
|
+
hmx_fa_q_prep_fallback(q_tiles, q->data, q->nb[1], q->nb[2], q->nb[3],
|
|
864
|
+
q_start, kv_head, ib3, start, end, n_rows_g, G, DK, factx->is_q_fp32, &factx->div_G);
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
// Synchronize threads before zeroing out vtcm_o_tiles[0] to prevent race condition
|
|
869
|
+
if (n > 1) {
|
|
870
|
+
atomic_fetch_sub(&args->barrier, 1);
|
|
871
|
+
while (atomic_load(&args->barrier) > 0) {
|
|
872
|
+
// spin wait
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
// Zero out vtcm_o_tiles[0] as it was used as temp_q_vtcm
|
|
877
|
+
{
|
|
878
|
+
const uint32_t g_br = factx->g_br;
|
|
879
|
+
const uint32_t DV = factx->DV;
|
|
880
|
+
const size_t o_tile_bytes = factx->o_tile_bytes;
|
|
881
|
+
const size_t o_bytes_per_t = hex_align_up(o_tile_bytes / n, 128);
|
|
882
|
+
const size_t o_start = i * o_bytes_per_t;
|
|
883
|
+
const size_t o_end = hex_smin(o_start + o_bytes_per_t, o_tile_bytes);
|
|
884
|
+
if (o_start < o_tile_bytes) {
|
|
885
|
+
hvx_splat_u8_a((char *) factx->vtcm_o_tiles[0] + o_start, 0, o_end - o_start);
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start));
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
static void fa_phase_q_load(struct hmx_fa_context * factx,
|
|
892
|
+
const struct htp_tensor * q,
|
|
893
|
+
uint32_t q_start,
|
|
894
|
+
uint32_t kv_head,
|
|
895
|
+
uint32_t ib3,
|
|
896
|
+
size_t n_rows_g) {
|
|
897
|
+
work_queue_t wp = factx->octx->ctx->work_queue;
|
|
898
|
+
uint32_t n = 1;
|
|
899
|
+
if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) {
|
|
900
|
+
n = factx->n_threads;
|
|
901
|
+
}
|
|
902
|
+
size_t rows_per_t = hex_align_up(hmx_ceil_div(factx->g_br, n), 2);
|
|
903
|
+
const uint32_t n_rows_q = hex_smin(factx->Br, factx->neq1 - q_start);
|
|
904
|
+
fa_q_load_args_t args;
|
|
905
|
+
args.factx = factx;
|
|
906
|
+
args.q = q;
|
|
907
|
+
args.q_start = q_start;
|
|
908
|
+
args.kv_head = kv_head;
|
|
909
|
+
args.ib3 = ib3;
|
|
910
|
+
args.n_rows_g = n_rows_g;
|
|
911
|
+
args.rows_per_t = rows_per_t;
|
|
912
|
+
args.n_rows_q = n_rows_q;
|
|
913
|
+
args.q_transposed = q->nb[1] < q->nb[2];
|
|
914
|
+
atomic_init(&args.barrier, n);
|
|
915
|
+
if (n > 1) {
|
|
916
|
+
work_queue_run(wp, fa_q_load_thread, &args, n);
|
|
917
|
+
} else {
|
|
918
|
+
fa_q_load_thread(1, 0, &args);
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
typedef struct {
|
|
923
|
+
struct hmx_fa_context * factx;
|
|
924
|
+
const struct htp_tensor * dst;
|
|
925
|
+
const __fp16 * o_tile_src;
|
|
926
|
+
uint32_t q_start;
|
|
927
|
+
uint32_t kv_head;
|
|
928
|
+
uint32_t ib3;
|
|
929
|
+
size_t n_rows_g;
|
|
930
|
+
size_t rows_per_t;
|
|
931
|
+
} fa_o_store_args_t;
|
|
932
|
+
|
|
933
|
+
static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) {
|
|
934
|
+
fa_o_store_args_t * args = (fa_o_store_args_t *) data;
|
|
935
|
+
struct hmx_fa_context * factx = args->factx;
|
|
936
|
+
|
|
937
|
+
const size_t n_rows_g = args->n_rows_g;
|
|
938
|
+
const size_t G = factx->G;
|
|
939
|
+
const size_t DV = factx->DV;
|
|
940
|
+
|
|
941
|
+
const size_t rows_per_t = args->rows_per_t;
|
|
942
|
+
const size_t start = (size_t) i * rows_per_t;
|
|
943
|
+
const size_t end = hex_smin(start + rows_per_t, n_rows_g);
|
|
944
|
+
|
|
945
|
+
if (start >= n_rows_g) {
|
|
946
|
+
return;
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
|
|
950
|
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
|
|
951
|
+
|
|
952
|
+
const struct htp_tensor * dst = args->dst;
|
|
953
|
+
const __fp16 * o_tile_src = args->o_tile_src;
|
|
954
|
+
const uint32_t q_start = args->q_start;
|
|
955
|
+
const uint32_t kv_head = args->kv_head;
|
|
956
|
+
const uint32_t ib3 = args->ib3;
|
|
957
|
+
|
|
958
|
+
size_t q_idx = fastdiv(start, &factx->div_G);
|
|
959
|
+
size_t h_idx = fastmodulo(start, G, &factx->div_G);
|
|
960
|
+
|
|
961
|
+
for (size_t r = start; r < end; ++r) {
|
|
962
|
+
float * out = (float *) ((uint8_t *) dst->data + (kv_head * G + h_idx) * dst->nb[1] +
|
|
963
|
+
(q_start + q_idx) * dst->nb[2] + ib3 * dst->nb[3]);
|
|
964
|
+
|
|
965
|
+
size_t r0 = r / HMX_FP16_TILE_N_ROWS;
|
|
966
|
+
size_t r1 = r % HMX_FP16_TILE_N_ROWS;
|
|
967
|
+
const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV;
|
|
968
|
+
|
|
969
|
+
for (uint32_t d = 0; d < DV / 32; ++d) {
|
|
970
|
+
const HVX_Vector * in_tile = (const HVX_Vector *) (tile_row_base + d * HMX_FP16_TILE_N_ELMS);
|
|
971
|
+
HVX_VectorPair vp = hvx_vec_f16_to_f32_shuff(in_tile[r1 / 2]);
|
|
972
|
+
if (r1 % 2 == 0) {
|
|
973
|
+
*(HVX_UVector *) (out + d * 32) = Q6_V_lo_W(vp);
|
|
974
|
+
} else {
|
|
975
|
+
*(HVX_UVector *) (out + d * 32) = Q6_V_hi_W(vp);
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
h_idx++;
|
|
980
|
+
if (h_idx == G) {
|
|
981
|
+
h_idx = 0;
|
|
982
|
+
q_idx++;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) {
|
|
989
|
+
fa_o_store_args_t * args = (fa_o_store_args_t *) data;
|
|
990
|
+
struct hmx_fa_context * factx = args->factx;
|
|
991
|
+
|
|
992
|
+
const size_t n_rows_g = args->n_rows_g;
|
|
993
|
+
const size_t rows_per_t = args->rows_per_t;
|
|
994
|
+
const size_t G = factx->G;
|
|
995
|
+
const size_t DV = factx->DV;
|
|
996
|
+
const size_t start = (size_t) i * rows_per_t;
|
|
997
|
+
const size_t end = hex_smin(start + rows_per_t, n_rows_g);
|
|
998
|
+
|
|
999
|
+
if (start >= n_rows_g) {
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
|
|
1004
|
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
|
|
1005
|
+
|
|
1006
|
+
const struct htp_tensor * dst = args->dst;
|
|
1007
|
+
const __fp16 * o_tile_src = args->o_tile_src;
|
|
1008
|
+
const uint32_t q_start = args->q_start;
|
|
1009
|
+
const uint32_t kv_head = args->kv_head;
|
|
1010
|
+
const uint32_t ib3 = args->ib3;
|
|
1011
|
+
|
|
1012
|
+
size_t q_idx = fastdiv(start, &factx->div_G);
|
|
1013
|
+
size_t h_idx = fastmodulo(start, G, &factx->div_G);
|
|
1014
|
+
|
|
1015
|
+
for (size_t r = start; r < end; ++r) {
|
|
1016
|
+
__fp16 * out = (__fp16 *) ((uint8_t *) dst->data + (kv_head * G + h_idx) * dst->nb[1] +
|
|
1017
|
+
(q_start + q_idx) * dst->nb[2] + ib3 * dst->nb[3]);
|
|
1018
|
+
|
|
1019
|
+
size_t r0 = r / HMX_FP16_TILE_N_ROWS;
|
|
1020
|
+
size_t r1 = r % HMX_FP16_TILE_N_ROWS;
|
|
1021
|
+
const __fp16 * tile_row_base = o_tile_src + r0 * HMX_FP16_TILE_N_ROWS * DV;
|
|
1022
|
+
|
|
1023
|
+
for (uint32_t d = 0; d < DV / 64; ++d) {
|
|
1024
|
+
const __fp16 * in_dtile = tile_row_base + d * HMX_FP16_TILE_N_ELMS * 2;
|
|
1025
|
+
const HVX_Vector * pv_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2;
|
|
1026
|
+
const HVX_Vector * pv_in1 = pv_in0 + 16;
|
|
1027
|
+
HVX_VectorPair vp = Q6_W_vdeal_VVR(*pv_in1, *pv_in0, -2);
|
|
1028
|
+
if (r1 % 2 == 0) {
|
|
1029
|
+
*(HVX_UVector *) (out + d * 64) = Q6_V_lo_W(vp);
|
|
1030
|
+
} else {
|
|
1031
|
+
*(HVX_UVector *) (out + d * 64) = Q6_V_hi_W(vp);
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
h_idx++;
|
|
1036
|
+
if (h_idx == G) {
|
|
1037
|
+
h_idx = 0;
|
|
1038
|
+
q_idx++;
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
static void fa_phase_o_store(struct hmx_fa_context * factx,
|
|
1045
|
+
const struct htp_tensor * dst,
|
|
1046
|
+
const __fp16 * o_tile_src,
|
|
1047
|
+
uint32_t q_start,
|
|
1048
|
+
uint32_t kv_head,
|
|
1049
|
+
uint32_t ib3,
|
|
1050
|
+
size_t n_rows_g) {
|
|
1051
|
+
work_queue_t wp = factx->octx->ctx->work_queue;
|
|
1052
|
+
uint32_t n = 1;
|
|
1053
|
+
if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) {
|
|
1054
|
+
n = factx->n_threads;
|
|
1055
|
+
}
|
|
1056
|
+
size_t rows_per_t = hmx_ceil_div(n_rows_g, n);
|
|
1057
|
+
fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t };
|
|
1058
|
+
worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16;
|
|
1059
|
+
if (n > 1) {
|
|
1060
|
+
work_queue_run(wp, store_fn, &args, n);
|
|
1061
|
+
} else {
|
|
1062
|
+
store_fn(1, 0, &args);
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
typedef struct {
|
|
1067
|
+
struct hmx_fa_context * factx;
|
|
1068
|
+
size_t buf_idx;
|
|
1069
|
+
size_t kv_rows;
|
|
1070
|
+
size_t n_rows_g;
|
|
1071
|
+
size_t n_col_tiles;
|
|
1072
|
+
size_t n_tiles_per_bc;
|
|
1073
|
+
size_t n_row_tiles;
|
|
1074
|
+
size_t n_row_tiles_g_br;
|
|
1075
|
+
uint32_t Bc;
|
|
1076
|
+
uint32_t G;
|
|
1077
|
+
uint32_t kv_head;
|
|
1078
|
+
uint32_t kv_start;
|
|
1079
|
+
uint32_t q_start;
|
|
1080
|
+
uint32_t ib3;
|
|
1081
|
+
bool has_alibi; // true when max_bias != 0 (need slope * mask + add)
|
|
1082
|
+
__fp16 * slopes;
|
|
1083
|
+
const struct htp_tensor * mask;
|
|
1084
|
+
const __fp16 * mask_vtcm; // VTCM mask buffer base (NULL = DDR fallback)
|
|
1085
|
+
size_t mask_vtcm_row_stride; // elements (__fp16) per row in VTCM mask buffer
|
|
1086
|
+
struct fastdiv_values thread_div;
|
|
1087
|
+
} fa_softmax_args_t;
|
|
1088
|
+
|
|
1089
|
+
static inline void fa_softmax_impl(
|
|
1090
|
+
unsigned int n, unsigned int i, void * data,
|
|
1091
|
+
const bool has_mask,
|
|
1092
|
+
const bool mask_broadcast,
|
|
1093
|
+
const bool is_g1,
|
|
1094
|
+
const bool has_alibi,
|
|
1095
|
+
const bool has_softcap
|
|
1096
|
+
) {
|
|
1097
|
+
fa_softmax_args_t * args = (fa_softmax_args_t *) data;
|
|
1098
|
+
struct hmx_fa_context * factx = args->factx;
|
|
1099
|
+
|
|
1100
|
+
const size_t n_rows_g = args->n_rows_g;
|
|
1101
|
+
const size_t kv_rows = args->kv_rows;
|
|
1102
|
+
const size_t Bc = args->Bc;
|
|
1103
|
+
const size_t G = args->G;
|
|
1104
|
+
const size_t n_tiles_per_bc = args->n_tiles_per_bc;
|
|
1105
|
+
const size_t n_row_vec_cnt = hmx_ceil_div(n_rows_g, 64);
|
|
1106
|
+
const uint32_t im3 = has_mask ? fastmodulo(args->ib3, args->mask->ne[3], &factx->src3_div3) : 0;
|
|
1107
|
+
|
|
1108
|
+
size_t vec_start = 0;
|
|
1109
|
+
size_t vec_end = n_row_vec_cnt;
|
|
1110
|
+
if (n > 1) {
|
|
1111
|
+
const size_t vecs_per_t = fastdiv(n_row_vec_cnt + n - 1, &args->thread_div);
|
|
1112
|
+
vec_start = i * vecs_per_t;
|
|
1113
|
+
vec_end = hex_smin(vec_start + vecs_per_t, n_row_vec_cnt);
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
if (vec_start >= n_row_vec_cnt) {
|
|
1117
|
+
return;
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
|
|
1121
|
+
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64));
|
|
1122
|
+
|
|
1123
|
+
// Per-thread row scratch: thread i uses bufs at offset i * 2 * stride
|
|
1124
|
+
const size_t row_buf_stride = factx->row_buf_stride;
|
|
1125
|
+
HVX_Vector * my_row_buf0 = factx->vtcm_row_bufs + i * 2 * row_buf_stride;
|
|
1126
|
+
HVX_Vector * my_row_buf1 = my_row_buf0 + row_buf_stride;
|
|
1127
|
+
|
|
1128
|
+
const HVX_Vector v_neg_inf = Q6_Vh_vsplat_R(0xfbff);
|
|
1129
|
+
|
|
1130
|
+
for (size_t r_vec_idx = vec_start; r_vec_idx < vec_end; ++r_vec_idx) {
|
|
1131
|
+
HVX_Vector rowmax_acc_v = v_neg_inf;
|
|
1132
|
+
HVX_Vector rowsum_acc_v = Q6_V_vzero();
|
|
1133
|
+
HVX_Vector m_prev_v0 = factx->vtcm_m_vec[r_vec_idx * 2 + 0];
|
|
1134
|
+
HVX_Vector m_prev_v1 = factx->vtcm_m_vec[r_vec_idx * 2 + 1];
|
|
1135
|
+
|
|
1136
|
+
HVX_Vector v_slopes = Q6_V_vzero();
|
|
1137
|
+
if (has_alibi) {
|
|
1138
|
+
v_slopes = hvx_vmem(args->slopes + r_vec_idx * 64);
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
for (uint32_t r_vec_off = 0; r_vec_off < 64; r_vec_off += 2) {
|
|
1142
|
+
uint32_t r = r_vec_idx * 64 + r_vec_off;
|
|
1143
|
+
if (r >= hex_align_up(n_rows_g, 2)) {
|
|
1144
|
+
break;
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
uint32_t r0 = r / HMX_FP16_TILE_N_ROWS;
|
|
1148
|
+
uint32_t r1 = r % HMX_FP16_TILE_N_ROWS;
|
|
1149
|
+
|
|
1150
|
+
const __fp16 * s_ld_base = factx->vtcm_s_tiles[args->buf_idx] + r0 * HMX_FP16_TILE_N_ROWS * Bc;
|
|
1151
|
+
__fp16 * p_st_base = factx->vtcm_p_tiles[args->buf_idx] + r0 * HMX_FP16_TILE_N_ROWS * Bc;
|
|
1152
|
+
|
|
1153
|
+
// Decode 2 rows from S tiles into per-thread row buffers
|
|
1154
|
+
if (has_softcap) {
|
|
1155
|
+
const HVX_Vector v_cap = hvx_vec_splat_f16(factx->logit_softcap);
|
|
1156
|
+
for (size_t c = 0; c < kv_rows; c += 64) {
|
|
1157
|
+
size_t ci = c / 64;
|
|
1158
|
+
const __fp16 * in_dtile = s_ld_base + ci * HMX_FP16_TILE_N_ELMS * 2;
|
|
1159
|
+
const HVX_Vector * pv_s_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2;
|
|
1160
|
+
const HVX_Vector * pv_s_in1 = pv_s_in0 + 16;
|
|
1161
|
+
|
|
1162
|
+
HVX_VectorPair vp_s_drow = Q6_W_vdeal_VVR(*pv_s_in1, *pv_s_in0, -2);
|
|
1163
|
+
HVX_Vector v_s_row0 = Q6_V_lo_W(vp_s_drow);
|
|
1164
|
+
HVX_Vector v_s_row1 = Q6_V_hi_W(vp_s_drow);
|
|
1165
|
+
|
|
1166
|
+
HVX_Vector t0 = hvx_vec_tanh_f16(v_s_row0);
|
|
1167
|
+
my_row_buf0[ci] = hvx_vec_mul_f16_f16(t0, v_cap);
|
|
1168
|
+
|
|
1169
|
+
HVX_Vector t1 = hvx_vec_tanh_f16(v_s_row1);
|
|
1170
|
+
my_row_buf1[ci] = hvx_vec_mul_f16_f16(t1, v_cap);
|
|
1171
|
+
}
|
|
1172
|
+
} else {
|
|
1173
|
+
size_t c = 0;
|
|
1174
|
+
for (; c + 64 < kv_rows; c += 128) {
|
|
1175
|
+
size_t ci0 = c / 64;
|
|
1176
|
+
size_t ci1 = ci0 + 1;
|
|
1177
|
+
const __fp16 * in_dtile0 = s_ld_base + ci0 * HMX_FP16_TILE_N_ELMS * 2;
|
|
1178
|
+
const __fp16 * in_dtile1 = s_ld_base + ci1 * HMX_FP16_TILE_N_ELMS * 2;
|
|
1179
|
+
const HVX_Vector * pv_s_in0_0 = ((const HVX_Vector *) in_dtile0) + r1 / 2;
|
|
1180
|
+
const HVX_Vector * pv_s_in1_0 = pv_s_in0_0 + 16;
|
|
1181
|
+
const HVX_Vector * pv_s_in0_1 = ((const HVX_Vector *) in_dtile1) + r1 / 2;
|
|
1182
|
+
const HVX_Vector * pv_s_in1_1 = pv_s_in0_1 + 16;
|
|
1183
|
+
|
|
1184
|
+
HVX_VectorPair vp_s_drow0 = Q6_W_vdeal_VVR(*pv_s_in1_0, *pv_s_in0_0, -2);
|
|
1185
|
+
my_row_buf0[ci0] = Q6_V_lo_W(vp_s_drow0);
|
|
1186
|
+
my_row_buf1[ci0] = Q6_V_hi_W(vp_s_drow0);
|
|
1187
|
+
|
|
1188
|
+
HVX_VectorPair vp_s_drow1 = Q6_W_vdeal_VVR(*pv_s_in1_1, *pv_s_in0_1, -2);
|
|
1189
|
+
my_row_buf0[ci1] = Q6_V_lo_W(vp_s_drow1);
|
|
1190
|
+
my_row_buf1[ci1] = Q6_V_hi_W(vp_s_drow1);
|
|
1191
|
+
}
|
|
1192
|
+
for (; c < kv_rows; c += 64) {
|
|
1193
|
+
size_t ci = c / 64;
|
|
1194
|
+
const __fp16 * in_dtile = s_ld_base + ci * HMX_FP16_TILE_N_ELMS * 2;
|
|
1195
|
+
const HVX_Vector * pv_s_in0 = ((const HVX_Vector *) in_dtile) + r1 / 2;
|
|
1196
|
+
const HVX_Vector * pv_s_in1 = pv_s_in0 + 16;
|
|
1197
|
+
|
|
1198
|
+
HVX_VectorPair vp_s_drow = Q6_W_vdeal_VVR(*pv_s_in1, *pv_s_in0, -2);
|
|
1199
|
+
my_row_buf0[ci] = Q6_V_lo_W(vp_s_drow);
|
|
1200
|
+
my_row_buf1[ci] = Q6_V_hi_W(vp_s_drow);
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
// Apply mask & compute rowmax(S)
|
|
1205
|
+
HVX_Vector v_slope0 = Q6_V_vzero();
|
|
1206
|
+
HVX_Vector v_slope1 = Q6_V_vzero();
|
|
1207
|
+
if (has_alibi) {
|
|
1208
|
+
v_slope0 = hvx_vec_repl_f16(Q6_V_vror_VR(v_slopes, r_vec_off * 2));
|
|
1209
|
+
v_slope1 = (r + 1 < n_rows_g) ? hvx_vec_repl_f16(Q6_V_vror_VR(v_slopes, (r_vec_off + 1) * 2)) : Q6_V_vzero();
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
const HVX_Vector v_threshold = Q6_Vh_vsplat_R(0xcc00); // fp16 -16.0
|
|
1213
|
+
|
|
1214
|
+
HVX_Vector v_s_rowmax0 = v_neg_inf;
|
|
1215
|
+
HVX_Vector v_s_rowmax1 = v_neg_inf;
|
|
1216
|
+
if (has_mask) {
|
|
1217
|
+
for (size_t c = 0; c < kv_rows; c += 64) {
|
|
1218
|
+
size_t ci = c / 64;
|
|
1219
|
+
const size_t ne = hex_smin(kv_rows - c, 64);
|
|
1220
|
+
HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(ne * sizeof(__fp16));
|
|
1221
|
+
|
|
1222
|
+
HVX_Vector v_mask0, v_mask1;
|
|
1223
|
+
|
|
1224
|
+
if (mask_broadcast) {
|
|
1225
|
+
if (is_g1) {
|
|
1226
|
+
const size_t qi0 = r + 0;
|
|
1227
|
+
v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + qi0 * args->mask_vtcm_row_stride + c);
|
|
1228
|
+
v_mask1 = v_neg_inf;
|
|
1229
|
+
if (r + 1 < n_rows_g) {
|
|
1230
|
+
const size_t qi1 = r + 1;
|
|
1231
|
+
v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + qi1 * args->mask_vtcm_row_stride + c);
|
|
1232
|
+
}
|
|
1233
|
+
} else {
|
|
1234
|
+
const size_t qi0 = fastdiv(r + 0, &factx->div_G);
|
|
1235
|
+
v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + qi0 * args->mask_vtcm_row_stride + c);
|
|
1236
|
+
v_mask1 = v_neg_inf;
|
|
1237
|
+
if (r + 1 < n_rows_g) {
|
|
1238
|
+
const size_t qi1 = fastdiv(r + 1, &factx->div_G);
|
|
1239
|
+
if (qi1 == qi0) {
|
|
1240
|
+
v_mask1 = v_mask0;
|
|
1241
|
+
} else {
|
|
1242
|
+
v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + qi1 * args->mask_vtcm_row_stride + c);
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
} else {
|
|
1247
|
+
// Head-dependent mask: pre-interleaved per row r.
|
|
1248
|
+
const size_t r0 = r + 0;
|
|
1249
|
+
v_mask0 = *(const HVX_Vector *) (args->mask_vtcm + r0 * args->mask_vtcm_row_stride + c);
|
|
1250
|
+
v_mask1 = v_neg_inf;
|
|
1251
|
+
if (r + 1 < n_rows_g) {
|
|
1252
|
+
const size_t r1 = r + 1;
|
|
1253
|
+
v_mask1 = *(const HVX_Vector *) (args->mask_vtcm + r1 * args->mask_vtcm_row_stride + c);
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
// Threshold: mask values below -16.0 are treated as -inf (causal mask).
|
|
1258
|
+
HVX_VectorPred q_keep0 = Q6_Q_and_QQ(Q6_Q_vcmp_gt_VhfVhf(v_mask0, v_threshold), q_tail_keep);
|
|
1259
|
+
HVX_VectorPred q_keep1 = Q6_Q_and_QQ(Q6_Q_vcmp_gt_VhfVhf(v_mask1, v_threshold), q_tail_keep);
|
|
1260
|
+
|
|
1261
|
+
// Scale mask values by log2(e) for base-2 calculations
|
|
1262
|
+
const HVX_Vector v_log2e = hvx_vec_splat_f16(EXP_LOG2E_F);
|
|
1263
|
+
HVX_Vector v_mask0_scaled = hvx_vec_mul_f16_f16(v_mask0, v_log2e);
|
|
1264
|
+
HVX_Vector v_mask1_scaled = hvx_vec_mul_f16_f16(v_mask1, v_log2e);
|
|
1265
|
+
|
|
1266
|
+
if (has_alibi) {
|
|
1267
|
+
HVX_Vector v_sm0 = hvx_vec_mul_f16_f16(v_mask0_scaled, v_slope0);
|
|
1268
|
+
HVX_Vector v_sm1 = hvx_vec_mul_f16_f16(v_mask1_scaled, v_slope1);
|
|
1269
|
+
my_row_buf0[ci] = Q6_V_vmux_QVV(q_keep0, hvx_vec_add_f16_f16(my_row_buf0[ci], v_sm0), v_neg_inf);
|
|
1270
|
+
my_row_buf1[ci] = Q6_V_vmux_QVV(q_keep1, hvx_vec_add_f16_f16(my_row_buf1[ci], v_sm1), v_neg_inf);
|
|
1271
|
+
} else {
|
|
1272
|
+
my_row_buf0[ci] = Q6_V_vmux_QVV(q_keep0, hvx_vec_add_f16_f16(my_row_buf0[ci], v_mask0_scaled), v_neg_inf);
|
|
1273
|
+
my_row_buf1[ci] = Q6_V_vmux_QVV(q_keep1, hvx_vec_add_f16_f16(my_row_buf1[ci], v_mask1_scaled), v_neg_inf);
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci]);
|
|
1277
|
+
v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci]);
|
|
1278
|
+
}
|
|
1279
|
+
} else {
|
|
1280
|
+
size_t c = 0;
|
|
1281
|
+
for (; c + 64 < kv_rows; c += 128) {
|
|
1282
|
+
size_t ci0 = c / 64;
|
|
1283
|
+
size_t ci1 = ci0 + 1;
|
|
1284
|
+
v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci0]);
|
|
1285
|
+
v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci0]);
|
|
1286
|
+
v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci1]);
|
|
1287
|
+
v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci1]);
|
|
1288
|
+
}
|
|
1289
|
+
for (; c < kv_rows; c += 64) {
|
|
1290
|
+
size_t ci = c / 64;
|
|
1291
|
+
const size_t ne = hex_smin(kv_rows - c, 64);
|
|
1292
|
+
HVX_VectorPred q_tail_keep = Q6_Q_vsetq2_R(ne * sizeof(__fp16));
|
|
1293
|
+
if (ne < 64) {
|
|
1294
|
+
my_row_buf0[ci] = Q6_V_vmux_QVV(q_tail_keep, my_row_buf0[ci], v_neg_inf);
|
|
1295
|
+
my_row_buf1[ci] = Q6_V_vmux_QVV(q_tail_keep, my_row_buf1[ci], v_neg_inf);
|
|
1296
|
+
}
|
|
1297
|
+
v_s_rowmax0 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax0, my_row_buf0[ci]);
|
|
1298
|
+
v_s_rowmax1 = Q6_Vhf_vmax_VhfVhf(v_s_rowmax1, my_row_buf1[ci]);
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
v_s_rowmax0 = hvx_vec_reduce_max_f16(v_s_rowmax0);
|
|
1303
|
+
v_s_rowmax1 = hvx_vec_reduce_max_f16(v_s_rowmax1);
|
|
1304
|
+
|
|
1305
|
+
// Splat m_prev[r], m_prev[r+1] from the float per-row accumulators and convert to fp16 vectors
|
|
1306
|
+
HVX_Vector v_m_prev0, v_m_prev1;
|
|
1307
|
+
if (r_vec_off < 32) {
|
|
1308
|
+
HVX_Vector v0 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v0, r_vec_off * 4));
|
|
1309
|
+
v_m_prev0 = hvx_vec_f32_to_f16(v0, v0);
|
|
1310
|
+
if (r + 1 < n_rows_g) {
|
|
1311
|
+
HVX_Vector v1 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v0, (r_vec_off + 1) * 4));
|
|
1312
|
+
v_m_prev1 = hvx_vec_f32_to_f16(v1, v1);
|
|
1313
|
+
} else {
|
|
1314
|
+
v_m_prev1 = Q6_V_vzero();
|
|
1315
|
+
}
|
|
1316
|
+
} else {
|
|
1317
|
+
HVX_Vector v0 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v1, (r_vec_off - 32) * 4));
|
|
1318
|
+
v_m_prev0 = hvx_vec_f32_to_f16(v0, v0);
|
|
1319
|
+
if (r + 1 < n_rows_g) {
|
|
1320
|
+
HVX_Vector v1 = hvx_vec_repl_f32(Q6_V_vror_VR(m_prev_v1, (r_vec_off + 1 - 32) * 4));
|
|
1321
|
+
v_m_prev1 = hvx_vec_f32_to_f16(v1, v1);
|
|
1322
|
+
} else {
|
|
1323
|
+
v_m_prev1 = Q6_V_vzero();
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
HVX_Vector v_dup_m0 = Q6_Vhf_vmax_VhfVhf(v_m_prev0, v_s_rowmax0);
|
|
1328
|
+
HVX_Vector v_dup_m1 = Q6_Vhf_vmax_VhfVhf(v_m_prev1, v_s_rowmax1);
|
|
1329
|
+
|
|
1330
|
+
// Insert row r, r+1 rowmax into rowmax_acc_v
|
|
1331
|
+
{
|
|
1332
|
+
HVX_VectorPred p_start = Q6_Q_vsetq_R(r_vec_off * 2);
|
|
1333
|
+
HVX_VectorPred p_mid = Q6_Q_vsetq_R((r_vec_off + 1) * 2);
|
|
1334
|
+
HVX_VectorPred p_end = Q6_Q_vsetq2_R((r_vec_off + 2) * 2);
|
|
1335
|
+
HVX_VectorPred p_lane0 = Q6_Q_and_QQn(p_mid, p_start);
|
|
1336
|
+
HVX_VectorPred p_lane1 = Q6_Q_and_QQn(p_end, p_mid);
|
|
1337
|
+
rowmax_acc_v = Q6_V_vmux_QVV(p_lane0, v_dup_m0, rowmax_acc_v);
|
|
1338
|
+
rowmax_acc_v = Q6_V_vmux_QVV(p_lane1, v_dup_m1, rowmax_acc_v);
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
// Compute P = exp(S - m_new)
|
|
1342
|
+
const HVX_Vector v_zero = Q6_V_vzero();
|
|
1343
|
+
HVX_Vector v_p_rowsum0 = v_zero;
|
|
1344
|
+
HVX_Vector v_p_rowsum1 = v_zero;
|
|
1345
|
+
|
|
1346
|
+
size_t c = 0;
|
|
1347
|
+
for (; c + 64 < kv_rows; c += 128) {
|
|
1348
|
+
size_t ci0 = c / 64;
|
|
1349
|
+
size_t ci1 = ci0 + 1;
|
|
1350
|
+
|
|
1351
|
+
HVX_Vector v_s_minus_m0_0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci0], v_dup_m0);
|
|
1352
|
+
HVX_Vector v_s_minus_m1_0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci0], v_dup_m1);
|
|
1353
|
+
HVX_Vector v_s_minus_m0_1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci1], v_dup_m0);
|
|
1354
|
+
HVX_Vector v_s_minus_m1_1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci1], v_dup_m1);
|
|
1355
|
+
|
|
1356
|
+
HVX_Vector v_p_row0_hf_0 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m0_0));
|
|
1357
|
+
HVX_Vector v_p_row1_hf_0 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m1_0));
|
|
1358
|
+
HVX_Vector v_p_row0_hf_1 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m0_1));
|
|
1359
|
+
HVX_Vector v_p_row1_hf_1 = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m1_1));
|
|
1360
|
+
|
|
1361
|
+
__fp16 * out_dtile0 = p_st_base + ci0 * HMX_FP16_TILE_N_ELMS * 2;
|
|
1362
|
+
__fp16 * out_dtile1 = p_st_base + ci1 * HMX_FP16_TILE_N_ELMS * 2;
|
|
1363
|
+
HVX_Vector * pv_p_out0_0 = ((HVX_Vector *) out_dtile0) + r1 / 2;
|
|
1364
|
+
HVX_Vector * pv_p_out1_0 = pv_p_out0_0 + 16;
|
|
1365
|
+
HVX_Vector * pv_p_out0_1 = ((HVX_Vector *) out_dtile1) + r1 / 2;
|
|
1366
|
+
HVX_Vector * pv_p_out1_1 = pv_p_out0_1 + 16;
|
|
1367
|
+
|
|
1368
|
+
HVX_VectorPair vp_p_dual0 = Q6_W_vshuff_VVR(v_p_row1_hf_0, v_p_row0_hf_0, -2);
|
|
1369
|
+
*pv_p_out0_0 = Q6_V_lo_W(vp_p_dual0);
|
|
1370
|
+
*pv_p_out1_0 = Q6_V_hi_W(vp_p_dual0);
|
|
1371
|
+
|
|
1372
|
+
HVX_VectorPair vp_p_dual1 = Q6_W_vshuff_VVR(v_p_row1_hf_1, v_p_row0_hf_1, -2);
|
|
1373
|
+
*pv_p_out0_1 = Q6_V_lo_W(vp_p_dual1);
|
|
1374
|
+
*pv_p_out1_1 = Q6_V_hi_W(vp_p_dual1);
|
|
1375
|
+
|
|
1376
|
+
HVX_VectorPair vp_p0_0 = hvx_vec_f16_to_f32_shuff(v_p_row0_hf_0);
|
|
1377
|
+
HVX_VectorPair vp_p1_0 = hvx_vec_f16_to_f32_shuff(v_p_row1_hf_0);
|
|
1378
|
+
HVX_VectorPair vp_p0_1 = hvx_vec_f16_to_f32_shuff(v_p_row0_hf_1);
|
|
1379
|
+
HVX_VectorPair vp_p1_1 = hvx_vec_f16_to_f32_shuff(v_p_row1_hf_1);
|
|
1380
|
+
|
|
1381
|
+
v_p_rowsum0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum0, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p0_0), Q6_V_hi_W(vp_p0_0)));
|
|
1382
|
+
v_p_rowsum0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum0, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p0_1), Q6_V_hi_W(vp_p0_1)));
|
|
1383
|
+
v_p_rowsum1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum1, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p1_0), Q6_V_hi_W(vp_p1_0)));
|
|
1384
|
+
v_p_rowsum1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum1, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p1_1), Q6_V_hi_W(vp_p1_1)));
|
|
1385
|
+
}
|
|
1386
|
+
for (size_t c_rem = c; c_rem < kv_rows; c_rem += 64) {
|
|
1387
|
+
size_t ci = c_rem / 64;
|
|
1388
|
+
HVX_Vector v_s_minus_m0 = Q6_Vqf16_vsub_VhfVhf(my_row_buf0[ci], v_dup_m0);
|
|
1389
|
+
HVX_Vector v_s_minus_m1 = Q6_Vqf16_vsub_VhfVhf(my_row_buf1[ci], v_dup_m1);
|
|
1390
|
+
|
|
1391
|
+
HVX_Vector v_p_row0_hf = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m0));
|
|
1392
|
+
HVX_Vector v_p_row1_hf = hvx_vec_exp2_f16(Q6_Vhf_equals_Vqf16(v_s_minus_m1));
|
|
1393
|
+
__fp16 * out_dtile = p_st_base + ci * HMX_FP16_TILE_N_ELMS * 2;
|
|
1394
|
+
HVX_Vector * pv_p_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
|
|
1395
|
+
HVX_Vector * pv_p_out1 = pv_p_out0 + 16;
|
|
1396
|
+
|
|
1397
|
+
HVX_VectorPair vp_p_dual = Q6_W_vshuff_VVR(v_p_row1_hf, v_p_row0_hf, -2);
|
|
1398
|
+
*pv_p_out0 = Q6_V_lo_W(vp_p_dual);
|
|
1399
|
+
*pv_p_out1 = Q6_V_hi_W(vp_p_dual);
|
|
1400
|
+
|
|
1401
|
+
HVX_VectorPair vp_p0 = hvx_vec_f16_to_f32_shuff(v_p_row0_hf);
|
|
1402
|
+
HVX_VectorPair vp_p1 = hvx_vec_f16_to_f32_shuff(v_p_row1_hf);
|
|
1403
|
+
|
|
1404
|
+
v_p_rowsum0 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum0, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p0), Q6_V_hi_W(vp_p0)));
|
|
1405
|
+
v_p_rowsum1 = Q6_Vqf32_vadd_Vqf32Vqf32(v_p_rowsum1, Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(vp_p1), Q6_V_hi_W(vp_p1)));
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
HVX_Vector rowsum0_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(v_p_rowsum0));
|
|
1409
|
+
HVX_Vector rowsum1_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(v_p_rowsum1));
|
|
1410
|
+
{
|
|
1411
|
+
HVX_Vector rv0_v = hvx_vec_f32_to_f16(rowsum0_sf, rowsum0_sf);
|
|
1412
|
+
HVX_Vector rv1_v = hvx_vec_f32_to_f16(rowsum1_sf, rowsum1_sf);
|
|
1413
|
+
|
|
1414
|
+
HVX_VectorPred p_start = Q6_Q_vsetq_R(r_vec_off * 2);
|
|
1415
|
+
HVX_VectorPred p_mid = Q6_Q_vsetq_R((r_vec_off + 1) * 2);
|
|
1416
|
+
HVX_VectorPred p_end = Q6_Q_vsetq2_R((r_vec_off + 2) * 2);
|
|
1417
|
+
HVX_VectorPred p_lane0 = Q6_Q_and_QQn(p_mid, p_start);
|
|
1418
|
+
HVX_VectorPred p_lane1 = Q6_Q_and_QQn(p_end, p_mid);
|
|
1419
|
+
rowsum_acc_v = Q6_V_vmux_QVV(p_lane0, rv0_v, rowsum_acc_v);
|
|
1420
|
+
rowsum_acc_v = Q6_V_vmux_QVV(p_lane1, rv1_v, rowsum_acc_v);
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
// Inline fa_ml_update_and_build_d for this vector (lock-free and in parallel)
|
|
1425
|
+
HVX_VectorPair rowmax_acc_pair = hvx_vec_f16_to_f32(rowmax_acc_v);
|
|
1426
|
+
HVX_Vector v_rowmax_acc_f32_0 = Q6_V_lo_W(rowmax_acc_pair);
|
|
1427
|
+
HVX_Vector v_rowmax_acc_f32_1 = Q6_V_hi_W(rowmax_acc_pair);
|
|
1428
|
+
|
|
1429
|
+
HVX_Vector v_m_curr0 = Q6_Vsf_vmax_VsfVsf(m_prev_v0, v_rowmax_acc_f32_0);
|
|
1430
|
+
HVX_Vector v_m_curr1 = Q6_Vsf_vmax_VsfVsf(m_prev_v1, v_rowmax_acc_f32_1);
|
|
1431
|
+
|
|
1432
|
+
HVX_Vector v_m_diff0 = HVX_OP_SUB_F32(m_prev_v0, v_m_curr0);
|
|
1433
|
+
HVX_Vector v_m_diff1 = HVX_OP_SUB_F32(m_prev_v1, v_m_curr1);
|
|
1434
|
+
|
|
1435
|
+
HVX_Vector v_m_diff_f16 = hvx_vec_f32_to_f16(v_m_diff0, v_m_diff1);
|
|
1436
|
+
HVX_Vector exp_m_diff_f16 = hvx_vec_exp2_f16(v_m_diff_f16);
|
|
1437
|
+
|
|
1438
|
+
HVX_VectorPair exp_m_diff_pair = hvx_vec_f16_to_f32(exp_m_diff_f16);
|
|
1439
|
+
HVX_Vector exp_m_diff0 = Q6_V_lo_W(exp_m_diff_pair);
|
|
1440
|
+
HVX_Vector exp_m_diff1 = Q6_V_hi_W(exp_m_diff_pair);
|
|
1441
|
+
|
|
1442
|
+
HVX_VectorPair rowsum_acc_pair = hvx_vec_f16_to_f32(rowsum_acc_v);
|
|
1443
|
+
HVX_Vector v_rowsum_acc_f32_0 = Q6_V_lo_W(rowsum_acc_pair);
|
|
1444
|
+
HVX_Vector v_rowsum_acc_f32_1 = Q6_V_hi_W(rowsum_acc_pair);
|
|
1445
|
+
|
|
1446
|
+
HVX_Vector v_l_curr0;
|
|
1447
|
+
HVX_Vector v_l_curr1;
|
|
1448
|
+
if (args->kv_start == 0 && factx->sinks != NULL) {
|
|
1449
|
+
// First KV block with sinks: m_prev holds the seeded sink value (not -inf),
|
|
1450
|
+
// so exp_m_diff = exp2(sink - m_curr) is the sink's contribution to the
|
|
1451
|
+
// denominator. l_prev is 0 here, so add exp_m_diff directly instead of
|
|
1452
|
+
// multiplying the (uninitialized) l_prev term.
|
|
1453
|
+
v_l_curr0 = HVX_OP_ADD_F32(exp_m_diff0, v_rowsum_acc_f32_0);
|
|
1454
|
+
v_l_curr1 = HVX_OP_ADD_F32(exp_m_diff1, v_rowsum_acc_f32_1);
|
|
1455
|
+
} else {
|
|
1456
|
+
HVX_Vector l_prev_v0 = factx->vtcm_l_vec[r_vec_idx * 2 + 0];
|
|
1457
|
+
HVX_Vector l_prev_v1 = factx->vtcm_l_vec[r_vec_idx * 2 + 1];
|
|
1458
|
+
v_l_curr0 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v0, exp_m_diff0), v_rowsum_acc_f32_0);
|
|
1459
|
+
v_l_curr1 = HVX_OP_ADD_F32(HVX_OP_MUL_F32(l_prev_v1, exp_m_diff1), v_rowsum_acc_f32_1);
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
factx->vtcm_m_vec[r_vec_idx * 2 + 0] = v_m_curr0;
|
|
1463
|
+
factx->vtcm_m_vec[r_vec_idx * 2 + 1] = v_m_curr1;
|
|
1464
|
+
factx->vtcm_l_vec[r_vec_idx * 2 + 0] = v_l_curr0;
|
|
1465
|
+
factx->vtcm_l_vec[r_vec_idx * 2 + 1] = v_l_curr1;
|
|
1466
|
+
|
|
1467
|
+
// Build diagonal tile D = diag(exp(m_diff))
|
|
1468
|
+
const HVX_Vector v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets;
|
|
1469
|
+
const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16));
|
|
1470
|
+
HVX_Vector v_exp_m_diff = exp_m_diff_f16;
|
|
1471
|
+
|
|
1472
|
+
__fp16 * const d_tiles_out = factx->vtcm_d_tiles[args->buf_idx];
|
|
1473
|
+
|
|
1474
|
+
size_t t0 = r_vec_idx * 2;
|
|
1475
|
+
if (t0 < args->n_row_tiles) {
|
|
1476
|
+
const HVX_Vector v_content = v_exp_m_diff;
|
|
1477
|
+
__fp16 * out_base = d_tiles_out + t0 * HMX_FP16_TILE_N_ELMS;
|
|
1478
|
+
Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content);
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
size_t t1 = r_vec_idx * 2 + 1;
|
|
1482
|
+
if (t1 < args->n_row_tiles) {
|
|
1483
|
+
const HVX_Vector v_content = Q6_V_vror_VR(v_exp_m_diff, 64);
|
|
1484
|
+
__fp16 * out_base = d_tiles_out + t1 * HMX_FP16_TILE_N_ELMS;
|
|
1485
|
+
Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content);
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64));
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
static void fa_softmax_thread_nomask(unsigned int n, unsigned int i, void * data) {
|
|
1492
|
+
fa_softmax_impl(n, i, data,
|
|
1493
|
+
/*has_mask=*/false,
|
|
1494
|
+
/*mask_broadcast=*/false,
|
|
1495
|
+
/*is_g1=*/false,
|
|
1496
|
+
/*has_alibi=*/false,
|
|
1497
|
+
/*has_softcap=*/false);
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
static void fa_softmax_thread_mask_broadcast_g1(unsigned int n, unsigned int i, void * data) {
|
|
1501
|
+
fa_softmax_impl(n, i, data,
|
|
1502
|
+
/*has_mask=*/true,
|
|
1503
|
+
/*mask_broadcast=*/true,
|
|
1504
|
+
/*is_g1=*/true,
|
|
1505
|
+
/*has_alibi=*/false,
|
|
1506
|
+
/*has_softcap=*/false);
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
static void fa_softmax_thread_mask_broadcast_gn(unsigned int n, unsigned int i, void * data) {
|
|
1510
|
+
fa_softmax_impl(n, i, data,
|
|
1511
|
+
/*has_mask=*/true,
|
|
1512
|
+
/*mask_broadcast=*/true,
|
|
1513
|
+
/*is_g1=*/false,
|
|
1514
|
+
/*has_alibi=*/false,
|
|
1515
|
+
/*has_softcap=*/false);
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
static void fa_softmax_thread(unsigned int n, unsigned int i, void * data) {
|
|
1519
|
+
fa_softmax_args_t * args = (fa_softmax_args_t *) data;
|
|
1520
|
+
struct hmx_fa_context * factx = args->factx;
|
|
1521
|
+
|
|
1522
|
+
const bool has_mask = (args->mask != NULL);
|
|
1523
|
+
const bool mask_broadcast = factx->mask_broadcast;
|
|
1524
|
+
const bool is_g1 = (args->G == 1);
|
|
1525
|
+
const bool has_alibi = args->has_alibi;
|
|
1526
|
+
const bool has_softcap = (factx->logit_softcap != 0.0f);
|
|
1527
|
+
|
|
1528
|
+
fa_softmax_impl(n, i, data, has_mask, mask_broadcast, is_g1, has_alibi, has_softcap);
|
|
1529
|
+
}
|
|
1530
|
+
|
|
1531
|
+
static __attribute__((noinline)) void fa_build_d_diag_inv_l(struct hmx_fa_context * factx,
|
|
1532
|
+
size_t n_row_tiles,
|
|
1533
|
+
size_t n_row_tiles_g_br) {
|
|
1534
|
+
const HVX_Vector v_offsets = *(const HVX_Vector *) d_tile_scatter_offsets;
|
|
1535
|
+
const HVX_VectorPred q_32_mask = Q6_Q_vsetq_R(32 * sizeof(__fp16));
|
|
1536
|
+
const HVX_Vector one = hvx_vec_splat_f32(1.0f);
|
|
1537
|
+
|
|
1538
|
+
HVX_Vector v_content = Q6_V_vzero();
|
|
1539
|
+
for (size_t i = 0; i < n_row_tiles; ++i) {
|
|
1540
|
+
if ((i % 2) == 0) {
|
|
1541
|
+
HVX_Vector inv_lo = HVX_OP_MUL_F32(one, hvx_vec_inverse_f32(factx->vtcm_l_vec[i]));
|
|
1542
|
+
HVX_Vector inv_hi = (i + 1 < n_row_tiles) ? HVX_OP_MUL_F32(one, hvx_vec_inverse_f32(factx->vtcm_l_vec[i + 1])) : Q6_V_vzero();
|
|
1543
|
+
v_content = hvx_vec_f32_to_f16(inv_lo, inv_hi);
|
|
1544
|
+
} else {
|
|
1545
|
+
v_content = Q6_V_vror_VR(v_content, 64);
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
__fp16 * out_base = factx->vtcm_d_inv_l + i * HMX_FP16_TILE_N_ELMS;
|
|
1549
|
+
Q6_vscatter_QRMVhV(q_32_mask, (size_t) out_base, HMX_FP16_TILE_SIZE - 1, v_offsets, v_content);
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1553
|
+
static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx,
|
|
1554
|
+
fa_softmax_args_t * sargs,
|
|
1555
|
+
size_t n_row_tiles,
|
|
1556
|
+
size_t n_row_tiles_g_br) {
|
|
1557
|
+
work_queue_t wp = factx->octx->ctx->work_queue;
|
|
1558
|
+
const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64);
|
|
1559
|
+
|
|
1560
|
+
worker_callback_t softmax_fn = fa_softmax_thread;
|
|
1561
|
+
if (sargs->mask == NULL && factx->logit_softcap == 0.0f && !sargs->has_alibi) {
|
|
1562
|
+
softmax_fn = fa_softmax_thread_nomask;
|
|
1563
|
+
} else if (sargs->mask != NULL && factx->mask_broadcast && factx->logit_softcap == 0.0f && !sargs->has_alibi) {
|
|
1564
|
+
if (sargs->G == 1) {
|
|
1565
|
+
softmax_fn = fa_softmax_thread_mask_broadcast_g1;
|
|
1566
|
+
} else {
|
|
1567
|
+
softmax_fn = fa_softmax_thread_mask_broadcast_gn;
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
if (factx->n_threads > 1 && n_row_vec_cnt >= 2) {
|
|
1572
|
+
uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt);
|
|
1573
|
+
sargs->thread_div = init_fastdiv_values(n_use);
|
|
1574
|
+
work_queue_run(wp, softmax_fn, sargs, n_use);
|
|
1575
|
+
} else {
|
|
1576
|
+
softmax_fn(1, 0, sargs);
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
// ============================================================================
|
|
1581
|
+
// HMX job structs and worker functions
|
|
1582
|
+
// ============================================================================
|
|
1583
|
+
|
|
1584
|
+
typedef struct {
|
|
1585
|
+
const __fp16 * q_tiles;
|
|
1586
|
+
const __fp16 * k_tiles;
|
|
1587
|
+
__fp16 * s_tiles;
|
|
1588
|
+
size_t n_row_tiles;
|
|
1589
|
+
size_t n_col_tiles;
|
|
1590
|
+
size_t n_dot_tiles; // DK / 32
|
|
1591
|
+
size_t n_tiles_per_bc;
|
|
1592
|
+
uint8_t * hmx_scales;
|
|
1593
|
+
} hmx_fa_qk_job_t;
|
|
1594
|
+
|
|
1595
|
+
static void hmx_fa_qk_dot_worker(void * data) {
|
|
1596
|
+
hmx_fa_qk_job_t * job = (hmx_fa_qk_job_t *) data;
|
|
1597
|
+
const size_t n_row_tiles = job->n_row_tiles;
|
|
1598
|
+
const size_t n_col_tiles = job->n_col_tiles;
|
|
1599
|
+
const size_t n_dot_tiles = job->n_dot_tiles;
|
|
1600
|
+
const size_t n_tiles_per_bc = job->n_tiles_per_bc;
|
|
1601
|
+
const __fp16 * restrict q_tiles = job->q_tiles;
|
|
1602
|
+
const __fp16 * restrict k_tiles = job->k_tiles;
|
|
1603
|
+
__fp16 * restrict s_tiles = job->s_tiles;
|
|
1604
|
+
__builtin_assume(n_row_tiles > 0);
|
|
1605
|
+
__builtin_assume(n_col_tiles > 0);
|
|
1606
|
+
__builtin_assume(n_dot_tiles > 0);
|
|
1607
|
+
|
|
1608
|
+
asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales));
|
|
1609
|
+
const size_t dot_stride = n_dot_tiles * HMX_FP16_TILE_N_ELMS;
|
|
1610
|
+
for (size_t r = 0; r < n_row_tiles; ++r) {
|
|
1611
|
+
const __fp16 * row_tiles = q_tiles + r * dot_stride;
|
|
1612
|
+
const __fp16 * col_tiles = k_tiles;
|
|
1613
|
+
__fp16 * out_tile = s_tiles + r * n_tiles_per_bc * HMX_FP16_TILE_N_ELMS;
|
|
1614
|
+
|
|
1615
|
+
for (size_t c = 0; c < n_col_tiles; ++c) {
|
|
1616
|
+
hmx_fa_qk_dot_tile(row_tiles, col_tiles, out_tile, n_dot_tiles);
|
|
1617
|
+
col_tiles += dot_stride;
|
|
1618
|
+
out_tile += HMX_FP16_TILE_N_ELMS;
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
typedef struct {
|
|
1624
|
+
__fp16 * o_curr;
|
|
1625
|
+
const __fp16 * o_prev;
|
|
1626
|
+
const __fp16 * p_tiles;
|
|
1627
|
+
const __fp16 * v_tiles;
|
|
1628
|
+
const __fp16 * d_tiles;
|
|
1629
|
+
uint8_t * hmx_scales;
|
|
1630
|
+
size_t n_row_tiles;
|
|
1631
|
+
size_t n_col_tiles;
|
|
1632
|
+
size_t n_row_tiles_g_br;
|
|
1633
|
+
size_t n_tiles_per_bc;
|
|
1634
|
+
size_t DV;
|
|
1635
|
+
} hmx_fa_o_update_job_t;
|
|
1636
|
+
|
|
1637
|
+
static void hmx_fa_o_update_worker(void * data) {
|
|
1638
|
+
hmx_fa_o_update_job_t * job = (hmx_fa_o_update_job_t *) data;
|
|
1639
|
+
const size_t n_row_tiles = job->n_row_tiles;
|
|
1640
|
+
const size_t n_col_tiles = job->n_col_tiles;
|
|
1641
|
+
const size_t n_row_tiles_g_br = job->n_row_tiles_g_br;
|
|
1642
|
+
const size_t n_tiles_per_bc = job->n_tiles_per_bc;
|
|
1643
|
+
const size_t DV_tiles = job->DV / 32;
|
|
1644
|
+
const __fp16 * restrict d_tiles = job->d_tiles;
|
|
1645
|
+
const __fp16 * restrict p_tiles = job->p_tiles;
|
|
1646
|
+
const __fp16 * restrict v_tiles = job->v_tiles;
|
|
1647
|
+
const __fp16 * restrict o_prev = job->o_prev;
|
|
1648
|
+
__fp16 * restrict o_curr = job->o_curr;
|
|
1649
|
+
__builtin_assume(n_row_tiles > 0);
|
|
1650
|
+
__builtin_assume(n_col_tiles > 0);
|
|
1651
|
+
__builtin_assume(DV_tiles > 0);
|
|
1652
|
+
|
|
1653
|
+
asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales));
|
|
1654
|
+
const size_t o_stride = n_row_tiles_g_br * HMX_FP16_TILE_N_ELMS;
|
|
1655
|
+
const size_t v_stride = n_tiles_per_bc * HMX_FP16_TILE_N_ELMS;
|
|
1656
|
+
for (size_t r = 0; r < n_row_tiles; ++r) {
|
|
1657
|
+
const __fp16 * d_diag = d_tiles + r * HMX_FP16_TILE_N_ELMS;
|
|
1658
|
+
const __fp16 * p_tile_in = p_tiles + (r * n_tiles_per_bc) * HMX_FP16_TILE_N_ELMS;
|
|
1659
|
+
const __fp16 * o_rc = o_prev + r * HMX_FP16_TILE_N_ELMS;
|
|
1660
|
+
const __fp16 * v_tile_in = v_tiles;
|
|
1661
|
+
__fp16 * o_tile_out = o_curr + r * HMX_FP16_TILE_N_ELMS;
|
|
1662
|
+
|
|
1663
|
+
for (size_t c = 0; c < DV_tiles; ++c) {
|
|
1664
|
+
hmx_fa_o_update_tile(d_diag, o_rc, p_tile_in, v_tile_in, o_tile_out, n_col_tiles);
|
|
1665
|
+
o_rc += o_stride;
|
|
1666
|
+
v_tile_in += v_stride;
|
|
1667
|
+
o_tile_out += o_stride;
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
typedef struct {
|
|
1673
|
+
__fp16 * o_curr; // output (row-major tile layout)
|
|
1674
|
+
const __fp16 * o_prev; // input (column-major tile layout)
|
|
1675
|
+
const __fp16 * d_tiles; // diag(1/l) tiles
|
|
1676
|
+
uint8_t * hmx_scales;
|
|
1677
|
+
size_t n_row_tiles;
|
|
1678
|
+
size_t n_row_tiles_g_br;
|
|
1679
|
+
size_t DV;
|
|
1680
|
+
} hmx_fa_o_norm_job_t;
|
|
1681
|
+
|
|
1682
|
+
static void hmx_fa_o_norm_worker(void * data) {
|
|
1683
|
+
hmx_fa_o_norm_job_t * job = (hmx_fa_o_norm_job_t *) data;
|
|
1684
|
+
const size_t n_row_tiles = job->n_row_tiles;
|
|
1685
|
+
const size_t n_row_tiles_g_br = job->n_row_tiles_g_br;
|
|
1686
|
+
const size_t DV_tiles = job->DV / 32;
|
|
1687
|
+
const __fp16 * restrict d_tiles = job->d_tiles;
|
|
1688
|
+
const __fp16 * restrict o_prev = job->o_prev;
|
|
1689
|
+
__fp16 * restrict o_curr = job->o_curr;
|
|
1690
|
+
__builtin_assume(n_row_tiles > 0);
|
|
1691
|
+
__builtin_assume(DV_tiles > 0);
|
|
1692
|
+
|
|
1693
|
+
asm volatile(HMX_SET_BIAS("%0") :: "r"((unsigned int)job->hmx_scales));
|
|
1694
|
+
const size_t o_stride = n_row_tiles_g_br * HMX_FP16_TILE_N_ELMS;
|
|
1695
|
+
for (size_t r = 0; r < n_row_tiles; ++r) {
|
|
1696
|
+
const __fp16 * d_diag = d_tiles + r * HMX_FP16_TILE_N_ELMS;
|
|
1697
|
+
const __fp16 * o_rc = o_prev + r * HMX_FP16_TILE_N_ELMS;
|
|
1698
|
+
__fp16 * o_out = o_curr + r * DV_tiles * HMX_FP16_TILE_N_ELMS;
|
|
1699
|
+
|
|
1700
|
+
for (size_t c = 0; c < DV_tiles; ++c) {
|
|
1701
|
+
hmx_fa_o_norm_tile(d_diag, o_rc, o_out);
|
|
1702
|
+
o_rc += o_stride;
|
|
1703
|
+
o_out += HMX_FP16_TILE_N_ELMS;
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
// Populate per-GQA-row ALiBi slopes for a given KV head.
|
|
1709
|
+
static __attribute__((noinline)) void fa_compute_slopes(
|
|
1710
|
+
const struct hmx_fa_context * factx,
|
|
1711
|
+
uint32_t kv_head,
|
|
1712
|
+
size_t n_rows_g) {
|
|
1713
|
+
__fp16 * slopes = factx->vtcm_slopes;
|
|
1714
|
+
if (factx->max_bias == 0.0f) {
|
|
1715
|
+
hvx_splat_f16_a(slopes, 1.0f, n_rows_g);
|
|
1716
|
+
return;
|
|
1717
|
+
}
|
|
1718
|
+
|
|
1719
|
+
const uint32_t G = factx->G;
|
|
1720
|
+
const uint32_t n_head_log2 = factx->n_head_log2;
|
|
1721
|
+
const float m0 = factx->m0;
|
|
1722
|
+
const float m1 = factx->m1;
|
|
1723
|
+
|
|
1724
|
+
__fp16 temp_slopes[512] __attribute__((aligned(128)));
|
|
1725
|
+
if (G <= 32) {
|
|
1726
|
+
// Fast path: Compute G unique slope values in vector registers
|
|
1727
|
+
HVX_Vector v_val = hvx_alibi_slopes(kv_head, G, n_head_log2, m0, m1);
|
|
1728
|
+
|
|
1729
|
+
__fp16 temp_slopes_aligned[64] __attribute__((aligned(128)));
|
|
1730
|
+
hvx_vmem(temp_slopes_aligned) = hvx_vec_f32_to_f16(v_val, Q6_V_vzero());
|
|
1731
|
+
|
|
1732
|
+
for (uint32_t i = 0; i < G; ++i) {
|
|
1733
|
+
temp_slopes[i] = temp_slopes_aligned[i];
|
|
1734
|
+
}
|
|
1735
|
+
} else {
|
|
1736
|
+
// Fallback path: G > 32 (rare configurations)
|
|
1737
|
+
for (uint32_t i = 0; i < G; ++i) {
|
|
1738
|
+
temp_slopes[i] = (__fp16)alibi_slope(kv_head * G + i, n_head_log2, m0, m1);
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1742
|
+
// Allocate stack buffer to avoid scalar writes to VTCM (which generates L2 misses)
|
|
1743
|
+
__fp16 local_slopes[n_rows_g] __attribute__((aligned(128)));
|
|
1744
|
+
for (size_t r = 0; r < n_rows_g; ++r) {
|
|
1745
|
+
local_slopes[r] = temp_slopes[fastmodulo(r, G, &factx->div_G)];
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
// Copy to VTCM slopes using HVX block copy (both are aligned to 128 bytes)
|
|
1749
|
+
hvx_copy_f16_aa((uint8_t *)slopes, (const uint8_t *)local_slopes, n_rows_g);
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1752
|
+
static void fa_push_mask_dma_gqa(
|
|
1753
|
+
dma_queue * dma,
|
|
1754
|
+
const struct htp_tensor * mask,
|
|
1755
|
+
uint32_t q_start,
|
|
1756
|
+
uint32_t im3,
|
|
1757
|
+
uint32_t kv_start,
|
|
1758
|
+
uint32_t kv_head,
|
|
1759
|
+
uint32_t G,
|
|
1760
|
+
uint32_t m_line_bytes,
|
|
1761
|
+
uint32_t kv_rows,
|
|
1762
|
+
uint32_t n_rows_q,
|
|
1763
|
+
struct hmx_fa_context * factx
|
|
1764
|
+
) {
|
|
1765
|
+
for (uint32_t g = 0; g < G; ++g) {
|
|
1766
|
+
const uint32_t h_idx = kv_head * G + g;
|
|
1767
|
+
const uint32_t im2 = fastmodulo(h_idx, mask->ne[2], &factx->src3_div2);
|
|
1768
|
+
const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] +
|
|
1769
|
+
im2 * mask->nb[2] + im3 * mask->nb[3] + kv_start * sizeof(__fp16);
|
|
1770
|
+
uint8_t * ms_dst = (uint8_t *) factx->vtcm_mask_buf + g * m_line_bytes;
|
|
1771
|
+
dma_queue_push(dma, dma_make_ptr(ms_dst, ms_src), G * m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
static void fa_pop_mask_dma_gqa(dma_queue * dma, uint32_t G) {
|
|
1776
|
+
for (uint32_t g = 0; g < G; ++g) {
|
|
1777
|
+
dma_queue_pop(dma);
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
static inline void fa_prefetch_block(dma_queue * dma, const struct htp_tensor * k, const struct htp_tensor * v, const struct htp_tensor * mask,
|
|
1782
|
+
uint32_t b, size_t Bc, size_t size_k_row_padded, size_t size_k_row, size_t size_v_row_padded, size_t size_v_row,
|
|
1783
|
+
uint32_t ik2, uint32_t ik3, uint32_t iv2, uint32_t iv3, uint32_t q_start, uint32_t im3, uint32_t kv_head, uint32_t G,
|
|
1784
|
+
size_t m_line_bytes, size_t n_rows_q, size_t nek1, size_t prefetch_buf, struct hmx_fa_context * factx) {
|
|
1785
|
+
const uint32_t prefetch_start = b * Bc;
|
|
1786
|
+
const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start);
|
|
1787
|
+
const uint8_t * k_prefetch_src = (const uint8_t *) k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3];
|
|
1788
|
+
dma_queue_push(dma, dma_make_ptr(factx->vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows);
|
|
1789
|
+
const uint8_t * v_prefetch_src = (const uint8_t *) v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3];
|
|
1790
|
+
dma_queue_push(dma, dma_make_ptr(factx->vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows);
|
|
1791
|
+
|
|
1792
|
+
if (mask) {
|
|
1793
|
+
if (__builtin_expect(factx->mask_broadcast, true)) {
|
|
1794
|
+
const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + prefetch_start * sizeof(__fp16);
|
|
1795
|
+
dma_cache_push(dma, &factx->m_cache, ms_src, m_line_bytes, mask->nb[1], prefetch_rows * sizeof(__fp16), n_rows_q);
|
|
1796
|
+
} else {
|
|
1797
|
+
fa_push_mask_dma_gqa(dma, mask, q_start, im3, prefetch_start, kv_head, G, m_line_bytes, prefetch_rows, n_rows_q, factx);
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
|
|
1802
|
+
// ============================================================================
|
|
1803
|
+
// Core HMX flash attention algorithm (GQA-merged)
|
|
1804
|
+
// ============================================================================
|
|
1805
|
+
|
|
1806
|
+
int hmx_flash_attn_ext(struct htp_ops_context * octx) {
|
|
1807
|
+
struct htp_thread_trace * tr_hvx = &octx->ctx->trace[0];
|
|
1808
|
+
struct htp_thread_trace * tr_hmx = &octx->ctx->trace[HTP_MAX_NTHREADS];
|
|
1809
|
+
const struct htp_tensor * q = octx->src[0];
|
|
1810
|
+
const struct htp_tensor * k = octx->src[1];
|
|
1811
|
+
const struct htp_tensor * v = octx->src[2];
|
|
1812
|
+
const struct htp_tensor * mask = (octx->src[3] && octx->src[3]->data) ? octx->src[3] : NULL;
|
|
1813
|
+
const struct htp_tensor * dst = octx->dst;
|
|
1814
|
+
|
|
1815
|
+
struct htp_context * const ctx = octx->ctx;
|
|
1816
|
+
|
|
1817
|
+
if (!ctx->hmx_enabled) {
|
|
1818
|
+
return HTP_STATUS_NO_SUPPORT;
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
// Dimensions
|
|
1822
|
+
const uint32_t neq0 = q->ne[0]; // head_dim (DK)
|
|
1823
|
+
const uint32_t neq1 = q->ne[1]; // n_tokens
|
|
1824
|
+
const uint32_t neq2 = q->ne[2]; // n_heads
|
|
1825
|
+
const uint32_t neq3 = q->ne[3]; // n_seqs
|
|
1826
|
+
|
|
1827
|
+
const uint32_t nek0 = k->ne[0]; // head_dim
|
|
1828
|
+
const uint32_t nek1 = k->ne[1]; // kv_len
|
|
1829
|
+
|
|
1830
|
+
const uint32_t nev0 = v->ne[0]; // head_dim (DV)
|
|
1831
|
+
|
|
1832
|
+
const uint32_t DK = neq0;
|
|
1833
|
+
const uint32_t DV = nev0;
|
|
1834
|
+
|
|
1835
|
+
// HMX requires head_dim to be multiple of 32
|
|
1836
|
+
if (DK % 32 != 0 || DV % 32 != 0) {
|
|
1837
|
+
return HTP_STATUS_NO_SUPPORT;
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params;
|
|
1841
|
+
const uint32_t n_kv_heads = k->ne[2];
|
|
1842
|
+
|
|
1843
|
+
// ======== Build context ========
|
|
1844
|
+
struct hmx_fa_context factx;
|
|
1845
|
+
memset(&factx, 0, sizeof(factx));
|
|
1846
|
+
factx.octx = octx;
|
|
1847
|
+
factx.sinks = octx->src[4]; // NULL if this op has no attention sinks
|
|
1848
|
+
factx.k = k;
|
|
1849
|
+
factx.v = v;
|
|
1850
|
+
factx.n_threads = kparams->n_threads;
|
|
1851
|
+
factx.DK = DK;
|
|
1852
|
+
factx.DV = DV;
|
|
1853
|
+
factx.n_kv = nek1;
|
|
1854
|
+
factx.n_kv_heads = n_kv_heads;
|
|
1855
|
+
factx.n_heads = neq2;
|
|
1856
|
+
factx.G = kparams->G;
|
|
1857
|
+
factx.div_G = kparams->u.hmx.div_G;
|
|
1858
|
+
factx.neq1 = neq1;
|
|
1859
|
+
factx.Br = kparams->Br;
|
|
1860
|
+
factx.Bc = kparams->Bc;
|
|
1861
|
+
factx.g_br = kparams->u.hmx.g_br;
|
|
1862
|
+
factx.n_kv_blocks = kparams->n_kv_blocks;
|
|
1863
|
+
factx.is_q_fp32 = (kparams->is_q_fp32 != 0);
|
|
1864
|
+
factx.is_dst_fp32 = (kparams->is_dst_fp32 != 0);
|
|
1865
|
+
factx.pipeline = (kparams->u.hmx.pipeline != 0);
|
|
1866
|
+
factx.mask_broadcast = (kparams->u.hmx.mask_broadcast != 0);
|
|
1867
|
+
if (mask) {
|
|
1868
|
+
factx.src3_div2 = kparams->src3_div2;
|
|
1869
|
+
factx.src3_div3 = kparams->src3_div3;
|
|
1870
|
+
}
|
|
1871
|
+
|
|
1872
|
+
if (kparams->logit_softcap == 0.0f) {
|
|
1873
|
+
factx.scale = (__fp16) (kparams->scale * EXP_LOG2E_F); // log2(e)
|
|
1874
|
+
} else {
|
|
1875
|
+
factx.scale = (__fp16) kparams->scale;
|
|
1876
|
+
}
|
|
1877
|
+
factx.max_bias = kparams->max_bias;
|
|
1878
|
+
factx.logit_softcap = (__fp16) (kparams->logit_softcap * EXP_LOG2E_F);
|
|
1879
|
+
|
|
1880
|
+
factx.n_head_log2 = kparams->n_head_log2;
|
|
1881
|
+
factx.m0 = kparams->m0;
|
|
1882
|
+
factx.m1 = kparams->m1;
|
|
1883
|
+
|
|
1884
|
+
const uint32_t Br = factx.Br;
|
|
1885
|
+
const uint32_t Bc = factx.Bc;
|
|
1886
|
+
const uint32_t g_br = factx.g_br;
|
|
1887
|
+
const bool pipeline = factx.pipeline;
|
|
1888
|
+
const uint32_t n_threads = factx.n_threads;
|
|
1889
|
+
const uint32_t G = factx.G;
|
|
1890
|
+
|
|
1891
|
+
// ======== VTCM allocation (GQA-aware) ========
|
|
1892
|
+
// K/V row sizes drive the DMA descriptors (not the VTCM layout) and are used
|
|
1893
|
+
// throughout the KV loop below.
|
|
1894
|
+
const size_t size_k_row = htp_tensor_get_row_size(k->type, DK);
|
|
1895
|
+
const size_t size_v_row = htp_tensor_get_row_size(v->type, DV);
|
|
1896
|
+
const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128);
|
|
1897
|
+
const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128);
|
|
1898
|
+
|
|
1899
|
+
// Build the VTCM layout once (shared with the host estimator) and place every
|
|
1900
|
+
// scratch buffer at its computed offset.
|
|
1901
|
+
struct hmx_fa_vtcm_layout L;
|
|
1902
|
+
hmx_fa_vtcm_layout_build(&L, G, DK, DV, Br, Bc, n_threads, pipeline, factx.is_q_fp32);
|
|
1903
|
+
|
|
1904
|
+
if (L.total_bytes > ctx->vtcm_size) {
|
|
1905
|
+
return HTP_STATUS_VTCM_TOO_SMALL;
|
|
1906
|
+
}
|
|
1907
|
+
|
|
1908
|
+
uint8_t * const base = ctx->vtcm_base;
|
|
1909
|
+
|
|
1910
|
+
factx.vtcm_q_dma = VTCM_LAYOUT_PTR(__fp16, base, L.off_q_dma);
|
|
1911
|
+
factx.vtcm_q_tiles = VTCM_LAYOUT_PTR(__fp16, base, L.off_q_tiles);
|
|
1912
|
+
factx.vtcm_o_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_o_tiles[0]);
|
|
1913
|
+
factx.vtcm_o_tiles[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_o_tiles[1]);
|
|
1914
|
+
factx.vtcm_k_fp16[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_fp16[0]);
|
|
1915
|
+
factx.vtcm_k_fp16[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_fp16[1]);
|
|
1916
|
+
factx.vtcm_v_fp16[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_fp16[0]);
|
|
1917
|
+
factx.vtcm_v_fp16[1] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_fp16[1]);
|
|
1918
|
+
factx.vtcm_k_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_k_tiles[0]);
|
|
1919
|
+
factx.vtcm_k_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_k_tiles[1], pipeline);
|
|
1920
|
+
factx.vtcm_v_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_v_tiles[0]);
|
|
1921
|
+
factx.vtcm_v_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_v_tiles[1], pipeline);
|
|
1922
|
+
factx.vtcm_s_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_s_tiles[0]);
|
|
1923
|
+
factx.vtcm_s_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_s_tiles[1], pipeline);
|
|
1924
|
+
factx.vtcm_p_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_p_tiles[0]);
|
|
1925
|
+
factx.vtcm_p_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_p_tiles[1], pipeline);
|
|
1926
|
+
factx.vtcm_d_tiles[0] = VTCM_LAYOUT_PTR(__fp16, base, L.off_d_tiles[0]);
|
|
1927
|
+
factx.vtcm_d_tiles[1] = VTCM_LAYOUT_PTR_OPTIONAL(__fp16, base, L.off_d_tiles[1], pipeline);
|
|
1928
|
+
factx.vtcm_d_inv_l = VTCM_LAYOUT_PTR(__fp16, base, L.off_d_inv_l);
|
|
1929
|
+
factx.vtcm_m_vec = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_m_vec);
|
|
1930
|
+
factx.vtcm_l_vec = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_l_vec);
|
|
1931
|
+
factx.vtcm_s_rowmax = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_s_rowmax);
|
|
1932
|
+
factx.vtcm_p_rowsum = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_p_rowsum);
|
|
1933
|
+
factx.vtcm_row_bufs = VTCM_LAYOUT_PTR(HVX_Vector, base, L.off_row_bufs);
|
|
1934
|
+
factx.row_buf_stride = L.row_buf_stride;
|
|
1935
|
+
factx.vtcm_hmx_scales_id = VTCM_LAYOUT_PTR(uint8_t, base, L.off_hmx_scales_id);
|
|
1936
|
+
factx.vtcm_hmx_scales_qk = VTCM_LAYOUT_PTR(uint8_t, base, L.off_hmx_scales_qk);
|
|
1937
|
+
factx.vtcm_mask_buf = VTCM_LAYOUT_PTR(__fp16, base, L.off_mask_buf);
|
|
1938
|
+
factx.mask_buf_row_stride = L.mask_buf_row_stride;
|
|
1939
|
+
factx.q_tile_bytes = L.q_tile_bytes;
|
|
1940
|
+
factx.o_tile_bytes = L.o_tile_bytes;
|
|
1941
|
+
factx.col_vec_bytes = L.col_vec_bytes;
|
|
1942
|
+
factx.d_tile_bytes = L.d_tile_bytes;
|
|
1943
|
+
factx.vtcm_slopes = VTCM_LAYOUT_PTR(__fp16, base, L.off_slopes);
|
|
1944
|
+
|
|
1945
|
+
const size_t m_line_bytes = L.m_line_bytes; // used by the mask DMAs in the KV loop
|
|
1946
|
+
|
|
1947
|
+
dma_cache_init(&factx.m_cache, (uint8_t *) factx.vtcm_mask_buf, L.m_buf_slot_bytes, HMX_FA_DMA_CACHE_SIZE);
|
|
1948
|
+
|
|
1949
|
+
// ======== Initialize HMX output scales ========
|
|
1950
|
+
hmx_init_column_scales(factx.vtcm_hmx_scales_id, Q6_V_vsplat_R(0x3c00)); // 1.0
|
|
1951
|
+
hmx_init_column_scales(factx.vtcm_hmx_scales_qk, hvx_vec_splat_f16(factx.scale));
|
|
1952
|
+
|
|
1953
|
+
// ======== Skip compute if profiling ========
|
|
1954
|
+
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
|
|
1955
|
+
return HTP_STATUS_OK;
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1958
|
+
// ======== DMA setup ========
|
|
1959
|
+
dma_queue * const dma = ctx->dma[0];
|
|
1960
|
+
|
|
1961
|
+
const size_t n_row_tiles_g_br = g_br / HMX_FP16_TILE_N_ROWS;
|
|
1962
|
+
const size_t n_tiles_per_bc = Bc / HMX_FP16_TILE_N_COLS;
|
|
1963
|
+
|
|
1964
|
+
const size_t qo_element_size = factx.is_q_fp32 ? sizeof(float) : sizeof(__fp16);
|
|
1965
|
+
|
|
1966
|
+
const bool q_transposed = q->nb[1] < q->nb[2];
|
|
1967
|
+
const size_t q_src_stride = q_transposed ? q->nb[2] : q->nb[1];
|
|
1968
|
+
const size_t q_row_bytes_untransposed = factx.G * factx.DK * qo_element_size;
|
|
1969
|
+
const size_t q_row_bytes_trans_factor = factx.DK * qo_element_size;
|
|
1970
|
+
const uint32_t kv_rows0 = hex_smin(Bc, nek1);
|
|
1971
|
+
|
|
1972
|
+
// ======== Reusable job descriptors for pipeline ========
|
|
1973
|
+
hmx_fa_qk_job_t qk_job;
|
|
1974
|
+
hmx_fa_o_update_job_t ou_job;
|
|
1975
|
+
hmx_fa_o_norm_job_t on_job;
|
|
1976
|
+
|
|
1977
|
+
// ======== Main loop ========
|
|
1978
|
+
for (uint32_t ib3 = 0; ib3 < neq3; ++ib3) {
|
|
1979
|
+
const uint32_t im3 = mask ? fastmodulo(ib3, mask->ne[3], &factx.src3_div3) : 0;
|
|
1980
|
+
for (uint32_t q_start = 0; q_start < neq1; q_start += Br) {
|
|
1981
|
+
const uint32_t n_rows_q = hex_smin(Br, neq1 - q_start);
|
|
1982
|
+
const size_t n_rows_g = n_rows_q * G;
|
|
1983
|
+
const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS);
|
|
1984
|
+
const size_t n_row_tiles = g_br_actual / HMX_FP16_TILE_N_ROWS;
|
|
1985
|
+
|
|
1986
|
+
for (uint32_t kv_head = 0; kv_head < n_kv_heads; ++kv_head) {
|
|
1987
|
+
const uint32_t ik2 = kv_head;
|
|
1988
|
+
const uint32_t ik3 = fastdiv(ib3, &kparams->broadcast_rk3);
|
|
1989
|
+
const uint32_t iv2 = kv_head;
|
|
1990
|
+
const uint32_t iv3 = fastdiv(ib3, &kparams->broadcast_rv3);
|
|
1991
|
+
|
|
1992
|
+
// 1. Push Q and KV DMAs for the very first iteration.
|
|
1993
|
+
// Subsequent iterations are enqueued early at the end of the previous iteration.
|
|
1994
|
+
if (ib3 == 0 && q_start == 0 && kv_head == 0) {
|
|
1995
|
+
const uint8_t * q_ptr = (const uint8_t *) q->data;
|
|
1996
|
+
const size_t q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed;
|
|
1997
|
+
const size_t n_rows = q_transposed ? factx.G : n_rows_q;
|
|
1998
|
+
dma_queue_push(dma, dma_make_ptr(factx.vtcm_q_dma, q_ptr), q_row_bytes, hex_smax(q_src_stride, q_row_bytes), q_row_bytes, n_rows);
|
|
1999
|
+
|
|
2000
|
+
if (factx.n_kv_blocks > 0) {
|
|
2001
|
+
const uint8_t * k_src = (const uint8_t *) k->data + ik2 * k->nb[2] + ik3 * k->nb[3];
|
|
2002
|
+
dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[0], k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0);
|
|
2003
|
+
|
|
2004
|
+
const uint8_t * v_src = (const uint8_t *) v->data + iv2 * v->nb[2] + iv3 * v->nb[3];
|
|
2005
|
+
dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[0], v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0);
|
|
2006
|
+
|
|
2007
|
+
if (factx.pipeline && mask) {
|
|
2008
|
+
if (__builtin_expect(factx.mask_broadcast, true)) {
|
|
2009
|
+
const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + 0;
|
|
2010
|
+
dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), n_rows_q);
|
|
2011
|
+
} else {
|
|
2012
|
+
fa_push_mask_dma_gqa(dma, mask, q_start, im3, 0, kv_head, G, m_line_bytes, kv_rows0, n_rows_q, &factx);
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
|
|
2018
|
+
// 2. Pop Q DMA (blocks until Q is loaded)
|
|
2019
|
+
dma_queue_pop(dma);
|
|
2020
|
+
|
|
2021
|
+
// ---- Load Q block & Initialize per-block state ----
|
|
2022
|
+
fa_phase_q_load(&factx, q, q_start, kv_head, ib3, n_rows_g);
|
|
2023
|
+
|
|
2024
|
+
__fp16 * o_tile_prev = factx.vtcm_o_tiles[0];
|
|
2025
|
+
__fp16 * o_tile_curr = factx.vtcm_o_tiles[1];
|
|
2026
|
+
|
|
2027
|
+
// ---- KV block loop with DMA double-buffering ----
|
|
2028
|
+
size_t buf_idx = 0;
|
|
2029
|
+
|
|
2030
|
+
htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) q_start);
|
|
2031
|
+
fa_compute_slopes(&factx, kv_head, n_rows_g);
|
|
2032
|
+
htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) q_start);
|
|
2033
|
+
|
|
2034
|
+
const size_t k_src_stride = size_k_row_padded / sizeof(__fp16);
|
|
2035
|
+
const size_t v_src_stride = size_v_row_padded / sizeof(__fp16);
|
|
2036
|
+
|
|
2037
|
+
hmx_queue_t hmx_q = ctx->hmx_queue;
|
|
2038
|
+
|
|
2039
|
+
if (factx.pipeline) {
|
|
2040
|
+
// Double-buffered job structs because HMX queue runs asynchronously
|
|
2041
|
+
hmx_fa_qk_job_t qk_job[2];
|
|
2042
|
+
hmx_fa_o_update_job_t ou_job[2];
|
|
2043
|
+
|
|
2044
|
+
// Prefetch block 1 early if there are multiple blocks
|
|
2045
|
+
if (factx.n_kv_blocks > 1) {
|
|
2046
|
+
fa_prefetch_block(dma, k, v, mask, 1, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row,
|
|
2047
|
+
ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, 1, &factx);
|
|
2048
|
+
}
|
|
2049
|
+
|
|
2050
|
+
// Prep and start QK-dot(0)
|
|
2051
|
+
void * curr_k0 = dma_queue_pop(dma).dst;
|
|
2052
|
+
fa_phase_k_interleave(&factx, kv_rows0, k_src_stride, curr_k0, 0, 0);
|
|
2053
|
+
|
|
2054
|
+
qk_job[0].q_tiles = factx.vtcm_q_tiles;
|
|
2055
|
+
qk_job[0].k_tiles = factx.vtcm_k_tiles[0];
|
|
2056
|
+
qk_job[0].s_tiles = factx.vtcm_s_tiles[0];
|
|
2057
|
+
qk_job[0].n_row_tiles = n_row_tiles;
|
|
2058
|
+
qk_job[0].n_col_tiles = hmx_ceil_div(kv_rows0, HMX_FP16_TILE_N_COLS);
|
|
2059
|
+
qk_job[0].n_dot_tiles = DK / 32;
|
|
2060
|
+
qk_job[0].n_tiles_per_bc = n_tiles_per_bc;
|
|
2061
|
+
qk_job[0].hmx_scales = factx.vtcm_hmx_scales_qk;
|
|
2062
|
+
hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job[0]));
|
|
2063
|
+
|
|
2064
|
+
for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) {
|
|
2065
|
+
const uint32_t kv_start = kv_blk * Bc;
|
|
2066
|
+
const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start);
|
|
2067
|
+
const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS);
|
|
2068
|
+
|
|
2069
|
+
// ---- 1. Pop and run V-prep for current block ----
|
|
2070
|
+
void * curr_v = dma_queue_pop(dma).dst;
|
|
2071
|
+
fa_phase_v_interleave(&factx, kv_rows, v_src_stride, curr_v, factx.vtcm_v_tiles[buf_idx], n_tiles_per_bc, kv_start);
|
|
2072
|
+
|
|
2073
|
+
// ---- 2. Pop and run mask-prep for current block ----
|
|
2074
|
+
__fp16 * current_mask_vtcm = NULL;
|
|
2075
|
+
if (mask) {
|
|
2076
|
+
if (__builtin_expect(factx.mask_broadcast, true)) {
|
|
2077
|
+
current_mask_vtcm = (__fp16 *) dma_queue_pop(dma).dst;
|
|
2078
|
+
} else {
|
|
2079
|
+
fa_pop_mask_dma_gqa(dma, G);
|
|
2080
|
+
current_mask_vtcm = factx.vtcm_mask_buf;
|
|
2081
|
+
}
|
|
2082
|
+
}
|
|
2083
|
+
|
|
2084
|
+
// ---- 3. Start HMX O update for block kv_blk - 1 (reads P[1 - buf_idx], V[1 - buf_idx], D) ----
|
|
2085
|
+
// O update relys on the previous block's P and V tiles.
|
|
2086
|
+
// O update MUST be pushed before the next block's QK-dot: hmx_queue_pop() retires the
|
|
2087
|
+
// oldest descriptor, so push order alone decides which pop waits for which job.
|
|
2088
|
+
// If OU went in after QK(i+1), the pop below would retire QK(i+1) and leave
|
|
2089
|
+
// OU(i-1) in flight into the next iteration, where V-prep overwrites V[prev_buf].
|
|
2090
|
+
if (kv_blk > 0) {
|
|
2091
|
+
const size_t prev_buf = 1 - buf_idx;
|
|
2092
|
+
ou_job[prev_buf].o_curr = o_tile_curr;
|
|
2093
|
+
ou_job[prev_buf].o_prev = o_tile_prev;
|
|
2094
|
+
ou_job[prev_buf].p_tiles = factx.vtcm_p_tiles[prev_buf];
|
|
2095
|
+
ou_job[prev_buf].v_tiles = factx.vtcm_v_tiles[prev_buf];
|
|
2096
|
+
ou_job[prev_buf].d_tiles = factx.vtcm_d_tiles[prev_buf];
|
|
2097
|
+
ou_job[prev_buf].hmx_scales = factx.vtcm_hmx_scales_id;
|
|
2098
|
+
ou_job[prev_buf].n_row_tiles = n_row_tiles;
|
|
2099
|
+
ou_job[prev_buf].n_col_tiles =
|
|
2100
|
+
hmx_ceil_div(hex_smin(Bc, nek1 - (kv_blk - 1) * Bc), HMX_FP16_TILE_N_COLS);
|
|
2101
|
+
ou_job[prev_buf].n_row_tiles_g_br = n_row_tiles_g_br;
|
|
2102
|
+
ou_job[prev_buf].n_tiles_per_bc = n_tiles_per_bc;
|
|
2103
|
+
ou_job[prev_buf].DV = DV;
|
|
2104
|
+
hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[prev_buf]));
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
// ---- 4. Pop and run K-prep for next block & push next QK-dot ----
|
|
2108
|
+
if (kv_blk + 1 < factx.n_kv_blocks) {
|
|
2109
|
+
const uint32_t next_start = (kv_blk + 1) * Bc;
|
|
2110
|
+
const uint32_t next_rows = hex_smin(Bc, nek1 - next_start);
|
|
2111
|
+
const size_t next_buf = 1 - buf_idx;
|
|
2112
|
+
|
|
2113
|
+
void * next_k = dma_queue_pop(dma).dst;
|
|
2114
|
+
fa_phase_k_interleave(&factx, next_rows, k_src_stride, next_k, next_start, next_buf);
|
|
2115
|
+
|
|
2116
|
+
qk_job[next_buf].q_tiles = factx.vtcm_q_tiles;
|
|
2117
|
+
qk_job[next_buf].k_tiles = factx.vtcm_k_tiles[next_buf];
|
|
2118
|
+
qk_job[next_buf].s_tiles = factx.vtcm_s_tiles[next_buf];
|
|
2119
|
+
qk_job[next_buf].n_row_tiles = n_row_tiles;
|
|
2120
|
+
qk_job[next_buf].n_col_tiles = hmx_ceil_div(next_rows, HMX_FP16_TILE_N_COLS);
|
|
2121
|
+
qk_job[next_buf].n_dot_tiles = DK / 32;
|
|
2122
|
+
qk_job[next_buf].n_tiles_per_bc = n_tiles_per_bc;
|
|
2123
|
+
qk_job[next_buf].hmx_scales = factx.vtcm_hmx_scales_qk;
|
|
2124
|
+
hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job[next_buf]));
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
// ---- 5. Wait for current block's QK-dot to finish ----
|
|
2128
|
+
hmx_queue_pop(hmx_q);
|
|
2129
|
+
|
|
2130
|
+
// ---- 6. Phase 2: softmax + build_D ----
|
|
2131
|
+
fa_softmax_args_t sargs;
|
|
2132
|
+
memset(&sargs, 0, sizeof(sargs));
|
|
2133
|
+
sargs.factx = &factx;
|
|
2134
|
+
sargs.buf_idx = buf_idx;
|
|
2135
|
+
sargs.kv_rows = kv_rows;
|
|
2136
|
+
sargs.n_rows_g = n_rows_g;
|
|
2137
|
+
sargs.n_col_tiles = n_col_tiles;
|
|
2138
|
+
sargs.n_tiles_per_bc = n_tiles_per_bc;
|
|
2139
|
+
sargs.n_row_tiles = n_row_tiles;
|
|
2140
|
+
sargs.n_row_tiles_g_br = n_row_tiles_g_br;
|
|
2141
|
+
sargs.Bc = Bc;
|
|
2142
|
+
sargs.G = G;
|
|
2143
|
+
sargs.kv_head = kv_head;
|
|
2144
|
+
sargs.kv_start = kv_start;
|
|
2145
|
+
sargs.q_start = q_start;
|
|
2146
|
+
sargs.ib3 = ib3;
|
|
2147
|
+
sargs.has_alibi = (factx.max_bias != 0.0f);
|
|
2148
|
+
sargs.mask = mask;
|
|
2149
|
+
sargs.mask_vtcm = current_mask_vtcm;
|
|
2150
|
+
sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride;
|
|
2151
|
+
sargs.slopes = factx.vtcm_slopes;
|
|
2152
|
+
|
|
2153
|
+
// Run Softmax on HVX (blocking call)
|
|
2154
|
+
fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br);
|
|
2155
|
+
|
|
2156
|
+
// Wait for HMX O update for block kv_blk - 1 to finish
|
|
2157
|
+
if (kv_blk > 0) {
|
|
2158
|
+
hmx_queue_pop(hmx_q);
|
|
2159
|
+
hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev);
|
|
2160
|
+
}
|
|
2161
|
+
|
|
2162
|
+
// Prefetch block kv_blk + 2
|
|
2163
|
+
if (kv_blk + 2 < factx.n_kv_blocks) {
|
|
2164
|
+
fa_prefetch_block(dma, k, v, mask, kv_blk + 2, Bc, size_k_row_padded, size_k_row, size_v_row_padded, size_v_row,
|
|
2165
|
+
ik2, ik3, iv2, iv3, q_start, im3, kv_head, G, m_line_bytes, n_rows_q, nek1, buf_idx, &factx);
|
|
2166
|
+
}
|
|
2167
|
+
|
|
2168
|
+
buf_idx = 1 - buf_idx;
|
|
2169
|
+
}
|
|
2170
|
+
|
|
2171
|
+
// Epilogue
|
|
2172
|
+
if (factx.n_kv_blocks > 0) {
|
|
2173
|
+
const uint32_t last_blk = factx.n_kv_blocks - 1;
|
|
2174
|
+
const size_t last_cols = hmx_ceil_div(hex_smin(Bc, nek1 - last_blk * Bc), HMX_FP16_TILE_N_COLS);
|
|
2175
|
+
ou_job[0].o_curr = o_tile_curr;
|
|
2176
|
+
ou_job[0].o_prev = o_tile_prev;
|
|
2177
|
+
ou_job[0].p_tiles = factx.vtcm_p_tiles[1 - buf_idx];
|
|
2178
|
+
ou_job[0].v_tiles = factx.vtcm_v_tiles[1 - buf_idx];
|
|
2179
|
+
ou_job[0].d_tiles = factx.vtcm_d_tiles[1 - buf_idx];
|
|
2180
|
+
ou_job[0].hmx_scales = factx.vtcm_hmx_scales_id;
|
|
2181
|
+
ou_job[0].n_row_tiles = n_row_tiles;
|
|
2182
|
+
ou_job[0].n_col_tiles = last_cols;
|
|
2183
|
+
ou_job[0].n_row_tiles_g_br = n_row_tiles_g_br;
|
|
2184
|
+
ou_job[0].n_tiles_per_bc = n_tiles_per_bc;
|
|
2185
|
+
ou_job[0].DV = DV;
|
|
2186
|
+
hmx_queue_push(hmx_q, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job[0]));
|
|
2187
|
+
|
|
2188
|
+
// Overlapped: run HVX build diag inv L while HMX is busy executing the update
|
|
2189
|
+
htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
|
|
2190
|
+
fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br);
|
|
2191
|
+
htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
|
|
2192
|
+
hmx_queue_pop(hmx_q);
|
|
2193
|
+
|
|
2194
|
+
hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev);
|
|
2195
|
+
}
|
|
2196
|
+
|
|
2197
|
+
} else {
|
|
2198
|
+
// Fallback path
|
|
2199
|
+
for (uint32_t kv_blk = 0; kv_blk < factx.n_kv_blocks; ++kv_blk) {
|
|
2200
|
+
const uint32_t kv_start = kv_blk * Bc;
|
|
2201
|
+
const uint32_t kv_rows = hex_smin(Bc, nek1 - kv_start);
|
|
2202
|
+
const size_t n_col_tiles = hmx_ceil_div(kv_rows, HMX_FP16_TILE_N_COLS);
|
|
2203
|
+
|
|
2204
|
+
if (mask) {
|
|
2205
|
+
if (__builtin_expect(factx.mask_broadcast, true)) {
|
|
2206
|
+
const uint8_t * ms_src = (const uint8_t *) mask->data + q_start * mask->nb[1] + im3 * mask->nb[3] + kv_start * sizeof(__fp16);
|
|
2207
|
+
dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows * sizeof(__fp16), n_rows_q);
|
|
2208
|
+
} else {
|
|
2209
|
+
fa_push_mask_dma_gqa(dma, mask, q_start, im3, kv_start, kv_head, G, m_line_bytes, kv_rows, n_rows_q, &factx);
|
|
2210
|
+
}
|
|
2211
|
+
}
|
|
2212
|
+
|
|
2213
|
+
if (kv_blk + 1 < factx.n_kv_blocks) {
|
|
2214
|
+
const uint32_t prefetch_start = (kv_blk + 1) * Bc;
|
|
2215
|
+
const uint32_t prefetch_rows = hex_smin(Bc, nek1 - prefetch_start);
|
|
2216
|
+
const size_t prefetch_buf = 1 - buf_idx;
|
|
2217
|
+
const uint8_t * k_prefetch_src = (const uint8_t *) k->data + prefetch_start * k->nb[1] + ik2 * k->nb[2] + ik3 * k->nb[3];
|
|
2218
|
+
dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[prefetch_buf], k_prefetch_src), size_k_row_padded, k->nb[1], size_k_row, prefetch_rows);
|
|
2219
|
+
const uint8_t * v_prefetch_src = (const uint8_t *) v->data + prefetch_start * v->nb[1] + iv2 * v->nb[2] + iv3 * v->nb[3];
|
|
2220
|
+
dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[prefetch_buf], v_prefetch_src), size_v_row_padded, v->nb[1], size_v_row, prefetch_rows);
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2223
|
+
// Wait for current K DMA and interleave
|
|
2224
|
+
void * curr_k = dma_queue_pop(dma).dst;
|
|
2225
|
+
fa_phase_k_interleave(&factx, kv_rows, k_src_stride, curr_k, kv_start, 0);
|
|
2226
|
+
|
|
2227
|
+
{
|
|
2228
|
+
qk_job.q_tiles = factx.vtcm_q_tiles;
|
|
2229
|
+
qk_job.k_tiles = factx.vtcm_k_tiles[0];
|
|
2230
|
+
qk_job.s_tiles = factx.vtcm_s_tiles[0];
|
|
2231
|
+
qk_job.n_row_tiles = n_row_tiles;
|
|
2232
|
+
qk_job.n_col_tiles = n_col_tiles;
|
|
2233
|
+
qk_job.n_dot_tiles = (size_t) (DK / 32);
|
|
2234
|
+
qk_job.n_tiles_per_bc = n_tiles_per_bc;
|
|
2235
|
+
qk_job.hmx_scales = factx.vtcm_hmx_scales_qk;
|
|
2236
|
+
|
|
2237
|
+
hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_qk_dot_worker, &qk_job));
|
|
2238
|
+
hmx_queue_pop(ctx->hmx_queue);
|
|
2239
|
+
}
|
|
2240
|
+
|
|
2241
|
+
// Wait for current V DMA and interleave
|
|
2242
|
+
void * curr_v = dma_queue_pop(dma).dst;
|
|
2243
|
+
fa_phase_v_interleave(&factx, kv_rows, v_src_stride, curr_v, factx.vtcm_v_tiles[0], n_tiles_per_bc, kv_start);
|
|
2244
|
+
|
|
2245
|
+
// ---- Phase 3: softmax + build_D ----
|
|
2246
|
+
__fp16 * current_mask_vtcm = NULL;
|
|
2247
|
+
if (mask) {
|
|
2248
|
+
if (__builtin_expect(factx.mask_broadcast, true)) {
|
|
2249
|
+
current_mask_vtcm = (__fp16 *) dma_queue_pop(dma).dst;
|
|
2250
|
+
} else {
|
|
2251
|
+
fa_pop_mask_dma_gqa(dma, G);
|
|
2252
|
+
current_mask_vtcm = factx.vtcm_mask_buf;
|
|
2253
|
+
}
|
|
2254
|
+
}
|
|
2255
|
+
|
|
2256
|
+
fa_softmax_args_t sargs;
|
|
2257
|
+
memset(&sargs, 0, sizeof(sargs));
|
|
2258
|
+
sargs.factx = &factx;
|
|
2259
|
+
sargs.kv_rows = kv_rows;
|
|
2260
|
+
sargs.n_rows_g = n_rows_g;
|
|
2261
|
+
sargs.n_col_tiles = n_col_tiles;
|
|
2262
|
+
sargs.n_tiles_per_bc = n_tiles_per_bc;
|
|
2263
|
+
sargs.n_row_tiles = n_row_tiles;
|
|
2264
|
+
sargs.n_row_tiles_g_br = n_row_tiles_g_br;
|
|
2265
|
+
sargs.Bc = Bc;
|
|
2266
|
+
sargs.G = G;
|
|
2267
|
+
sargs.kv_head = kv_head;
|
|
2268
|
+
sargs.kv_start = kv_start;
|
|
2269
|
+
sargs.q_start = q_start;
|
|
2270
|
+
sargs.ib3 = ib3;
|
|
2271
|
+
sargs.has_alibi = (factx.max_bias != 0.0f);
|
|
2272
|
+
sargs.mask = mask;
|
|
2273
|
+
sargs.mask_vtcm = current_mask_vtcm;
|
|
2274
|
+
sargs.mask_vtcm_row_stride = factx.mask_buf_row_stride;
|
|
2275
|
+
sargs.slopes = factx.vtcm_slopes;
|
|
2276
|
+
fa_phase_softmax_and_build_d(&factx, &sargs, n_row_tiles, n_row_tiles_g_br);
|
|
2277
|
+
|
|
2278
|
+
{
|
|
2279
|
+
ou_job.o_curr = o_tile_curr;
|
|
2280
|
+
ou_job.o_prev = o_tile_prev;
|
|
2281
|
+
ou_job.p_tiles = factx.vtcm_p_tiles[0];
|
|
2282
|
+
ou_job.v_tiles = factx.vtcm_v_tiles[0];
|
|
2283
|
+
ou_job.d_tiles = factx.vtcm_d_tiles[0];
|
|
2284
|
+
ou_job.hmx_scales = factx.vtcm_hmx_scales_id;
|
|
2285
|
+
ou_job.n_row_tiles = n_row_tiles;
|
|
2286
|
+
ou_job.n_col_tiles = n_col_tiles;
|
|
2287
|
+
ou_job.n_row_tiles_g_br = n_row_tiles_g_br;
|
|
2288
|
+
ou_job.n_tiles_per_bc = n_tiles_per_bc;
|
|
2289
|
+
ou_job.DV = DV;
|
|
2290
|
+
|
|
2291
|
+
hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_update_worker, &ou_job));
|
|
2292
|
+
if (kv_blk + 1 == factx.n_kv_blocks) {
|
|
2293
|
+
// Overlapped: run HVX build diag inv L while HMX is busy executing the update
|
|
2294
|
+
htp_trace_event_start(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
|
|
2295
|
+
fa_build_d_diag_inv_l(&factx, n_row_tiles, n_row_tiles_g_br);
|
|
2296
|
+
htp_trace_event_stop(tr_hvx, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) q_start);
|
|
2297
|
+
}
|
|
2298
|
+
hmx_queue_pop(ctx->hmx_queue);
|
|
2299
|
+
|
|
2300
|
+
hex_swap_ptr((void **) &o_tile_curr, (void **) &o_tile_prev);
|
|
2301
|
+
}
|
|
2302
|
+
|
|
2303
|
+
buf_idx = 1 - buf_idx;
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
2306
|
+
|
|
2307
|
+
// Enqueue DMAs for the next iteration early so they overlap with O-PROC
|
|
2308
|
+
uint32_t next_kv_head = kv_head + 1;
|
|
2309
|
+
uint32_t next_q_start = q_start;
|
|
2310
|
+
uint32_t next_ib3 = ib3;
|
|
2311
|
+
if (next_kv_head >= n_kv_heads) {
|
|
2312
|
+
next_kv_head = 0;
|
|
2313
|
+
next_q_start = q_start + Br;
|
|
2314
|
+
if (next_q_start >= neq1) {
|
|
2315
|
+
next_q_start = 0;
|
|
2316
|
+
next_ib3 = ib3 + 1;
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
bool has_next = (next_ib3 < neq3);
|
|
2320
|
+
|
|
2321
|
+
if (has_next) {
|
|
2322
|
+
const uint32_t next_n_rows_q = hex_smin(Br, neq1 - next_q_start);
|
|
2323
|
+
const uint8_t * next_q_ptr = (const uint8_t *) q->data + next_q_start * q->nb[1] + (next_kv_head * factx.G) * q->nb[2] + next_ib3 * q->nb[3];
|
|
2324
|
+
const size_t next_q_row_bytes = q_transposed ? next_n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed;
|
|
2325
|
+
const size_t next_n_rows = q_transposed ? factx.G : next_n_rows_q;
|
|
2326
|
+
dma_queue_push(dma, dma_make_ptr(factx.vtcm_q_dma, next_q_ptr), next_q_row_bytes, hex_smax(q_src_stride, next_q_row_bytes), next_q_row_bytes, next_n_rows);
|
|
2327
|
+
|
|
2328
|
+
if (factx.n_kv_blocks > 0) {
|
|
2329
|
+
const uint32_t next_ik2 = next_kv_head;
|
|
2330
|
+
const uint32_t next_iv2 = next_kv_head;
|
|
2331
|
+
uint32_t next_ik3 = ik3;
|
|
2332
|
+
uint32_t next_iv3 = iv3;
|
|
2333
|
+
if (next_ib3 != ib3) {
|
|
2334
|
+
next_ik3 = fastdiv(next_ib3, &kparams->broadcast_rk3);
|
|
2335
|
+
next_iv3 = fastdiv(next_ib3, &kparams->broadcast_rv3);
|
|
2336
|
+
}
|
|
2337
|
+
|
|
2338
|
+
const uint8_t * next_k_src = (const uint8_t *) k->data + next_ik2 * k->nb[2] + next_ik3 * k->nb[3];
|
|
2339
|
+
dma_queue_push(dma, dma_make_ptr(factx.vtcm_k_fp16[0], next_k_src), size_k_row_padded, k->nb[1], size_k_row, kv_rows0);
|
|
2340
|
+
|
|
2341
|
+
const uint8_t * next_v_src = (const uint8_t *) v->data + next_iv2 * v->nb[2] + next_iv3 * v->nb[3];
|
|
2342
|
+
dma_queue_push(dma, dma_make_ptr(factx.vtcm_v_fp16[0], next_v_src), size_v_row_padded, v->nb[1], size_v_row, kv_rows0);
|
|
2343
|
+
|
|
2344
|
+
if (factx.pipeline && mask) {
|
|
2345
|
+
uint32_t next_im3 = im3;
|
|
2346
|
+
if (next_ib3 != ib3) {
|
|
2347
|
+
next_im3 = fastmodulo(next_ib3, mask->ne[3], &factx.src3_div3);
|
|
2348
|
+
}
|
|
2349
|
+
if (__builtin_expect(factx.mask_broadcast, true)) {
|
|
2350
|
+
const uint8_t * ms_src = (const uint8_t *) mask->data + next_q_start * mask->nb[1] + next_im3 * mask->nb[3] + 0;
|
|
2351
|
+
dma_cache_push(dma, &factx.m_cache, ms_src, m_line_bytes, mask->nb[1], kv_rows0 * sizeof(__fp16), next_n_rows_q);
|
|
2352
|
+
} else {
|
|
2353
|
+
fa_push_mask_dma_gqa(dma, mask, next_q_start, next_im3, 0, next_kv_head, G, m_line_bytes, kv_rows0, next_n_rows_q, &factx);
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
}
|
|
2357
|
+
}
|
|
2358
|
+
|
|
2359
|
+
// ---- Final normalization ----
|
|
2360
|
+
{
|
|
2361
|
+
on_job.o_curr = o_tile_curr;
|
|
2362
|
+
on_job.o_prev = o_tile_prev;
|
|
2363
|
+
on_job.d_tiles = factx.vtcm_d_inv_l;
|
|
2364
|
+
on_job.hmx_scales = factx.vtcm_hmx_scales_id;
|
|
2365
|
+
on_job.n_row_tiles = n_row_tiles;
|
|
2366
|
+
on_job.n_row_tiles_g_br = n_row_tiles_g_br;
|
|
2367
|
+
on_job.DV = DV;
|
|
2368
|
+
hmx_queue_push(ctx->hmx_queue, hmx_queue_make_desc(hmx_fa_o_norm_worker, &on_job));
|
|
2369
|
+
hmx_queue_pop(ctx->hmx_queue);
|
|
2370
|
+
}
|
|
2371
|
+
|
|
2372
|
+
// ---- Store O block ----
|
|
2373
|
+
fa_phase_o_store(&factx, dst, o_tile_curr, q_start, kv_head, ib3, n_rows_g);
|
|
2374
|
+
}
|
|
2375
|
+
}
|
|
2376
|
+
}
|
|
2377
|
+
|
|
2378
|
+
return HTP_STATUS_OK;
|
|
2379
|
+
}
|
|
2380
|
+
|
|
2381
|
+
int op_flash_attn_ext(struct htp_ops_context * octx) {
|
|
2382
|
+
const struct htp_tensor * q = octx->src[0];
|
|
2383
|
+
const struct htp_tensor * k = octx->src[1];
|
|
2384
|
+
const struct htp_tensor * v = octx->src[2];
|
|
2385
|
+
const struct htp_tensor * mask = octx->src[3];
|
|
2386
|
+
const struct htp_tensor * dst = octx->dst;
|
|
2387
|
+
|
|
2388
|
+
// Check support
|
|
2389
|
+
if ((q->type != HTP_TYPE_F16 && q->type != HTP_TYPE_F32) ||
|
|
2390
|
+
(k->type != HTP_TYPE_F16 && k->type != HTP_TYPE_Q8_0) ||
|
|
2391
|
+
(v->type != HTP_TYPE_F16 && v->type != HTP_TYPE_Q8_0)) {
|
|
2392
|
+
return HTP_STATUS_NO_SUPPORT;
|
|
2393
|
+
}
|
|
2394
|
+
|
|
2395
|
+
const struct htp_fa_kernel_params * kparams = (const struct htp_fa_kernel_params *) octx->kernel_params;
|
|
2396
|
+
|
|
2397
|
+
if (kparams->kernel_type == HTP_FA_KERNEL_UNSUPPORTED) {
|
|
2398
|
+
return HTP_STATUS_NO_SUPPORT;
|
|
2399
|
+
}
|
|
2400
|
+
|
|
2401
|
+
if (kparams->kernel_type == HTP_FA_KERNEL_HMX) {
|
|
2402
|
+
return hmx_flash_attn_ext(octx);
|
|
2403
|
+
}
|
|
2404
|
+
|
|
2405
|
+
struct htp_fa_context factx;
|
|
2406
|
+
factx.octx = octx;
|
|
2407
|
+
factx.k = k;
|
|
2408
|
+
factx.v = v;
|
|
2409
|
+
|
|
2410
|
+
factx.t_start = HAP_perf_get_qtimer_count();
|
|
2411
|
+
|
|
2412
|
+
factx.src0_div21 = kparams->u.hvx.src0_div21;
|
|
2413
|
+
factx.src0_div1 = kparams->u.hvx.src0_div1;
|
|
2414
|
+
|
|
2415
|
+
factx.broadcast_rk2 = kparams->broadcast_rk2;
|
|
2416
|
+
factx.broadcast_rk3 = kparams->broadcast_rk3;
|
|
2417
|
+
factx.broadcast_rv2 = kparams->broadcast_rv2;
|
|
2418
|
+
factx.broadcast_rv3 = kparams->broadcast_rv3;
|
|
2419
|
+
|
|
2420
|
+
if (mask) {
|
|
2421
|
+
factx.src3_div2 = kparams->src3_div2;
|
|
2422
|
+
factx.src3_div3 = kparams->src3_div3;
|
|
2423
|
+
}
|
|
2424
|
+
|
|
2425
|
+
factx.is_q_fp32 = (kparams->is_q_fp32 != 0);
|
|
2426
|
+
factx.size_q_row_padded = kparams->u.hvx.size_q_row_padded;
|
|
2427
|
+
factx.size_k_row_padded = kparams->u.hvx.size_k_row_padded;
|
|
2428
|
+
factx.size_v_row_padded = kparams->u.hvx.size_v_row_padded;
|
|
2429
|
+
|
|
2430
|
+
size_t size_q_block = factx.size_q_row_padded * 1; // single row for now
|
|
2431
|
+
factx.size_k_block = factx.size_k_row_padded * FLASH_ATTN_BLOCK_SIZE;
|
|
2432
|
+
factx.size_v_block = factx.size_v_row_padded * FLASH_ATTN_BLOCK_SIZE;
|
|
2433
|
+
factx.size_m_block = hex_round_up(FLASH_ATTN_BLOCK_SIZE * sizeof(__fp16), 128);
|
|
2434
|
+
|
|
2435
|
+
factx.n_blocks = kparams->n_kv_blocks;
|
|
2436
|
+
|
|
2437
|
+
factx.scale = kparams->scale;
|
|
2438
|
+
factx.max_bias = kparams->max_bias;
|
|
2439
|
+
factx.logit_softcap = (__fp16) kparams->logit_softcap;
|
|
2440
|
+
|
|
2441
|
+
factx.n_head_log2 = kparams->n_head_log2;
|
|
2442
|
+
factx.m0 = kparams->m0;
|
|
2443
|
+
factx.m1 = kparams->m1;
|
|
2444
|
+
|
|
2445
|
+
const uint32_t n_head = q->ne[2];
|
|
2446
|
+
if (n_head > 512) {
|
|
2447
|
+
return HTP_STATUS_NO_SUPPORT;
|
|
2448
|
+
}
|
|
2449
|
+
for (uint32_t h = 0; h < n_head; ++h) {
|
|
2450
|
+
factx.slopes[h] = (__fp16) ((kparams->max_bias > 0.0f) ? alibi_slope(h, factx.n_head_log2, factx.m0, factx.m1) : 1.0f);
|
|
2451
|
+
}
|
|
2452
|
+
|
|
2453
|
+
// total rows in q
|
|
2454
|
+
factx.qrows = kparams->qrows;
|
|
2455
|
+
factx.qrows_per_thread = kparams->qrows_per_thread;
|
|
2456
|
+
|
|
2457
|
+
size_t size_vkq_acc = hex_round_up(v->ne[0] * sizeof(float), 128); // VKQ32
|
|
2458
|
+
|
|
2459
|
+
factx.size_q_block = size_q_block;
|
|
2460
|
+
factx.size_vkq_acc = size_vkq_acc;
|
|
2461
|
+
|
|
2462
|
+
uint8_t * vtcm_cur = octx->ctx->vtcm_base;
|
|
2463
|
+
|
|
2464
|
+
factx.spad_q = vtcm_seq_alloc(&vtcm_cur, size_q_block * octx->n_threads);
|
|
2465
|
+
factx.spad_k = vtcm_seq_alloc(&vtcm_cur, factx.size_k_block * 2 * octx->n_threads);
|
|
2466
|
+
factx.spad_v = vtcm_seq_alloc(&vtcm_cur, factx.size_v_block * 2 * octx->n_threads);
|
|
2467
|
+
factx.spad_m = vtcm_seq_alloc(&vtcm_cur, (mask ? factx.size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * octx->n_threads);
|
|
2468
|
+
factx.spad_a = vtcm_seq_alloc(&vtcm_cur, size_vkq_acc * octx->n_threads);
|
|
2469
|
+
|
|
2470
|
+
if ((size_t) (vtcm_cur - octx->ctx->vtcm_base) > octx->ctx->vtcm_size) {
|
|
2471
|
+
return HTP_STATUS_VTCM_TOO_SMALL;
|
|
2472
|
+
}
|
|
2473
|
+
|
|
2474
|
+
if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {
|
|
2475
|
+
work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, octx->n_threads);
|
|
2476
|
+
}
|
|
2477
|
+
|
|
2478
|
+
return HTP_STATUS_OK;
|
|
2479
|
+
}
|