elelem-ggml 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/exe/elelem-ggml +8 -0
- data/ext/elelem/ggml/elelem.c +466 -0
- data/ext/elelem/ggml/extconf.rb +52 -0
- data/lib/elelem/ggml/chat_ml.rb +23 -0
- data/lib/elelem/ggml/gguf.rb +82 -0
- data/lib/elelem/ggml/provider.rb +63 -0
- data/lib/elelem/ggml/tokenizer.rb +95 -0
- data/lib/elelem/ggml/version.rb +7 -0
- data/lib/elelem/ggml.rb +37 -0
- data/vendor/ggml/AUTHORS +335 -0
- data/vendor/ggml/CMakeLists.txt +499 -0
- data/vendor/ggml/CONTRIBUTING.md +3 -0
- data/vendor/ggml/LICENSE +21 -0
- data/vendor/ggml/README.md +49 -0
- data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
- data/vendor/ggml/cmake/GitVars.cmake +22 -0
- data/vendor/ggml/cmake/common.cmake +50 -0
- data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
- data/vendor/ggml/ggml.pc.in +10 -0
- data/vendor/ggml/include/ggml-alloc.h +86 -0
- data/vendor/ggml/include/ggml-backend.h +437 -0
- data/vendor/ggml/include/ggml-blas.h +25 -0
- data/vendor/ggml/include/ggml-cann.h +123 -0
- data/vendor/ggml/include/ggml-cpp.h +39 -0
- data/vendor/ggml/include/ggml-cpu.h +152 -0
- data/vendor/ggml/include/ggml-cuda.h +47 -0
- data/vendor/ggml/include/ggml-et.h +28 -0
- data/vendor/ggml/include/ggml-hexagon.h +19 -0
- data/vendor/ggml/include/ggml-metal.h +61 -0
- data/vendor/ggml/include/ggml-opencl.h +26 -0
- data/vendor/ggml/include/ggml-openvino.h +37 -0
- data/vendor/ggml/include/ggml-opt.h +256 -0
- data/vendor/ggml/include/ggml-rpc.h +35 -0
- data/vendor/ggml/include/ggml-sycl.h +57 -0
- data/vendor/ggml/include/ggml-virtgpu.h +14 -0
- data/vendor/ggml/include/ggml-vulkan.h +29 -0
- data/vendor/ggml/include/ggml-webgpu.h +19 -0
- data/vendor/ggml/include/ggml-zdnn.h +17 -0
- data/vendor/ggml/include/ggml-zendnn.h +22 -0
- data/vendor/ggml/include/ggml.h +2963 -0
- data/vendor/ggml/include/gguf.h +211 -0
- data/vendor/ggml/requirements.txt +12 -0
- data/vendor/ggml/scripts/gen-authors.sh +9 -0
- data/vendor/ggml/scripts/make-release-checks.sh +96 -0
- data/vendor/ggml/scripts/make-release-desc.sh +69 -0
- data/vendor/ggml/scripts/make-release-summary.txt +33 -0
- data/vendor/ggml/scripts/release.sh +195 -0
- data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
- data/vendor/ggml/scripts/sync-llama.last +1 -0
- data/vendor/ggml/scripts/sync-llama.sh +21 -0
- data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
- data/vendor/ggml/scripts/sync-whisper.last +1 -0
- data/vendor/ggml/scripts/sync-whisper.sh +17 -0
- data/vendor/ggml/src/CMakeLists.txt +634 -0
- data/vendor/ggml/src/ggml-alloc.c +1249 -0
- data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
- data/vendor/ggml/src/ggml-backend-dl.h +45 -0
- data/vendor/ggml/src/ggml-backend-impl.h +291 -0
- data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
- data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
- data/vendor/ggml/src/ggml-backend.cpp +2517 -0
- data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
- data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
- data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
- data/vendor/ggml/src/ggml-cann/common.h +651 -0
- data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
- data/vendor/ggml/src/ggml-common.h +1911 -0
- data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
- data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
- data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
- data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
- data/vendor/ggml/src/ggml-cpu/common.h +95 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
- data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
- data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
- data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
- data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
- data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
- data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
- data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
- data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
- data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
- data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
- data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
- data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
- data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
- data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
- data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
- data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
- data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
- data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
- data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
- data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
- data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
- data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
- data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
- data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
- data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
- data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
- data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
- data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
- data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
- data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
- data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
- data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
- data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
- data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
- data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
- data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
- data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
- data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
- data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
- data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
- data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
- data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
- data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
- data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
- data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
- data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
- data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
- data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
- data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
- data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
- data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
- data/vendor/ggml/src/ggml-feats.h +166 -0
- data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
- data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
- data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
- data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
- data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
- data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
- data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
- data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
- data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
- data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
- data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
- data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
- data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
- data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
- data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
- data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
- data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
- data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
- data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
- data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
- data/vendor/ggml/src/ggml-impl.h +783 -0
- data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
- data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
- data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
- data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
- data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
- data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
- data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
- data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
- data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
- data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
- data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
- data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
- data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
- data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
- data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
- data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
- data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
- data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
- data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
- data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
- data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
- data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
- data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
- data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
- data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
- data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
- data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
- data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
- data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
- data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
- data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
- data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
- data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
- data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
- data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
- data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
- data/vendor/ggml/src/ggml-opt.cpp +1094 -0
- data/vendor/ggml/src/ggml-quants.c +5667 -0
- data/vendor/ggml/src/ggml-quants.h +115 -0
- data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
- data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
- data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
- data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
- data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
- data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
- data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
- data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
- data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
- data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
- data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
- data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
- data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
- data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
- data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
- data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
- data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
- data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
- data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
- data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
- data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
- data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
- data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
- data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
- data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
- data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
- data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
- data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
- data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
- data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
- data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
- data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
- data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
- data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
- data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
- data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
- data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
- data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
- data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
- data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
- data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
- data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
- data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
- data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
- data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
- data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
- data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
- data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
- data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
- data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/vendor/ggml/src/ggml-threading.cpp +12 -0
- data/vendor/ggml/src/ggml-threading.h +14 -0
- data/vendor/ggml/src/ggml-version.h.in +4 -0
- data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
- data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
- data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
- data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
- data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
- data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
- data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
- data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
- data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
- data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
- data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
- data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
- data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
- data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
- data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
- data/vendor/ggml/src/ggml.c +8088 -0
- data/vendor/ggml/src/ggml.cpp +26 -0
- data/vendor/ggml/src/gguf.cpp +1706 -0
- metadata +1459 -0
|
@@ -0,0 +1,1504 @@
|
|
|
1
|
+
#include "mmvq.cuh"
|
|
2
|
+
#include "quantize.cuh"
|
|
3
|
+
#include "unary.cuh"
|
|
4
|
+
#include "vecdotq.cuh"
|
|
5
|
+
|
|
6
|
+
#include <cstdint>
|
|
7
|
+
#include <type_traits>
|
|
8
|
+
|
|
9
|
+
typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs);
|
|
10
|
+
|
|
11
|
+
static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) {
|
|
12
|
+
switch (type) {
|
|
13
|
+
case GGML_TYPE_Q1_0: return vec_dot_q1_0_q8_1;
|
|
14
|
+
case GGML_TYPE_Q2_0: return vec_dot_q2_0_q8_1;
|
|
15
|
+
case GGML_TYPE_Q4_0: return vec_dot_q4_0_q8_1;
|
|
16
|
+
case GGML_TYPE_Q4_1: return vec_dot_q4_1_q8_1;
|
|
17
|
+
case GGML_TYPE_Q5_0: return vec_dot_q5_0_q8_1;
|
|
18
|
+
case GGML_TYPE_Q5_1: return vec_dot_q5_1_q8_1;
|
|
19
|
+
case GGML_TYPE_Q8_0: return vec_dot_q8_0_q8_1;
|
|
20
|
+
case GGML_TYPE_MXFP4: return vec_dot_mxfp4_q8_1;
|
|
21
|
+
case GGML_TYPE_NVFP4: return vec_dot_nvfp4_q8_1;
|
|
22
|
+
case GGML_TYPE_Q2_K: return vec_dot_q2_K_q8_1;
|
|
23
|
+
case GGML_TYPE_Q3_K: return vec_dot_q3_K_q8_1;
|
|
24
|
+
case GGML_TYPE_Q4_K: return vec_dot_q4_K_q8_1;
|
|
25
|
+
case GGML_TYPE_Q5_K: return vec_dot_q5_K_q8_1;
|
|
26
|
+
case GGML_TYPE_Q6_K: return vec_dot_q6_K_q8_1;
|
|
27
|
+
case GGML_TYPE_IQ2_XXS: return vec_dot_iq2_xxs_q8_1;
|
|
28
|
+
case GGML_TYPE_IQ2_XS: return vec_dot_iq2_xs_q8_1;
|
|
29
|
+
case GGML_TYPE_IQ2_S: return vec_dot_iq2_s_q8_1;
|
|
30
|
+
case GGML_TYPE_IQ3_XXS: return vec_dot_iq3_xxs_q8_1;
|
|
31
|
+
case GGML_TYPE_IQ1_S: return vec_dot_iq1_s_q8_1;
|
|
32
|
+
case GGML_TYPE_IQ1_M: return vec_dot_iq1_m_q8_1;
|
|
33
|
+
case GGML_TYPE_IQ4_NL: return vec_dot_iq4_nl_q8_1;
|
|
34
|
+
case GGML_TYPE_IQ4_XS: return vec_dot_iq4_xs_q8_1;
|
|
35
|
+
case GGML_TYPE_IQ3_S: return vec_dot_iq3_s_q8_1;
|
|
36
|
+
default: return nullptr;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) {
|
|
41
|
+
switch (type) {
|
|
42
|
+
case GGML_TYPE_Q1_0: return VDR_Q1_0_Q8_1_MMVQ;
|
|
43
|
+
case GGML_TYPE_Q2_0: return VDR_Q2_0_Q8_1_MMVQ;
|
|
44
|
+
case GGML_TYPE_Q4_0: return VDR_Q4_0_Q8_1_MMVQ;
|
|
45
|
+
case GGML_TYPE_Q4_1: return VDR_Q4_1_Q8_1_MMVQ;
|
|
46
|
+
case GGML_TYPE_Q5_0: return VDR_Q5_0_Q8_1_MMVQ;
|
|
47
|
+
case GGML_TYPE_Q5_1: return VDR_Q5_1_Q8_1_MMVQ;
|
|
48
|
+
case GGML_TYPE_Q8_0: return VDR_Q8_0_Q8_1_MMVQ;
|
|
49
|
+
case GGML_TYPE_MXFP4: return VDR_MXFP4_Q8_1_MMVQ;
|
|
50
|
+
case GGML_TYPE_NVFP4: return VDR_NVFP4_Q8_1_MMVQ;
|
|
51
|
+
case GGML_TYPE_Q2_K: return VDR_Q2_K_Q8_1_MMVQ;
|
|
52
|
+
case GGML_TYPE_Q3_K: return VDR_Q3_K_Q8_1_MMVQ;
|
|
53
|
+
case GGML_TYPE_Q4_K: return VDR_Q4_K_Q8_1_MMVQ;
|
|
54
|
+
case GGML_TYPE_Q5_K: return VDR_Q5_K_Q8_1_MMVQ;
|
|
55
|
+
case GGML_TYPE_Q6_K: return VDR_Q6_K_Q8_1_MMVQ;
|
|
56
|
+
case GGML_TYPE_IQ2_XXS: return VDR_IQ2_XXS_Q8_1_MMVQ;
|
|
57
|
+
case GGML_TYPE_IQ2_XS: return VDR_IQ2_XS_Q8_1_MMVQ;
|
|
58
|
+
case GGML_TYPE_IQ2_S: return VDR_IQ2_S_Q8_1_MMVQ;
|
|
59
|
+
case GGML_TYPE_IQ3_XXS: return VDR_IQ3_XXS_Q8_1_MMVQ;
|
|
60
|
+
case GGML_TYPE_IQ3_S: return VDR_IQ3_S_Q8_1_MMVQ;
|
|
61
|
+
case GGML_TYPE_IQ4_NL: return VDR_IQ4_NL_Q8_1_MMVQ;
|
|
62
|
+
case GGML_TYPE_IQ4_XS: return VDR_IQ4_XS_Q8_1_MMVQ;
|
|
63
|
+
default: return 1;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
enum mmvq_parameter_table_id {
|
|
68
|
+
MMVQ_PARAMETERS_GENERIC = 0,
|
|
69
|
+
MMVQ_PARAMETERS_TURING,
|
|
70
|
+
MMVQ_PARAMETERS_GCN,
|
|
71
|
+
MMVQ_PARAMETERS_RDNA2,
|
|
72
|
+
MMVQ_PARAMETERS_RDNA3_0,
|
|
73
|
+
MMVQ_PARAMETERS_RDNA4,
|
|
74
|
+
MMVQ_PARAMETERS_GB10
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
|
|
78
|
+
#if defined(RDNA4)
|
|
79
|
+
return MMVQ_PARAMETERS_RDNA4;
|
|
80
|
+
#elif defined(RDNA3_0)
|
|
81
|
+
return MMVQ_PARAMETERS_RDNA3_0;
|
|
82
|
+
#elif defined(RDNA2) || defined(RDNA3_5)
|
|
83
|
+
return MMVQ_PARAMETERS_RDNA2;
|
|
84
|
+
#elif defined(GCN) || defined(CDNA)
|
|
85
|
+
return MMVQ_PARAMETERS_GCN;
|
|
86
|
+
#elif defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING && __CUDA_ARCH__ < GGML_CUDA_CC_AMPERE
|
|
87
|
+
return MMVQ_PARAMETERS_TURING;
|
|
88
|
+
#elif defined(__CUDA_ARCH__) && __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
|
89
|
+
return MMVQ_PARAMETERS_GB10;
|
|
90
|
+
#else
|
|
91
|
+
return MMVQ_PARAMETERS_GENERIC;
|
|
92
|
+
#endif
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
|
|
96
|
+
if (GGML_CUDA_CC_IS_RDNA4(cc)) {
|
|
97
|
+
return MMVQ_PARAMETERS_RDNA4;
|
|
98
|
+
}
|
|
99
|
+
if (GGML_CUDA_CC_IS_RDNA3_0(cc)) {
|
|
100
|
+
return MMVQ_PARAMETERS_RDNA3_0;
|
|
101
|
+
}
|
|
102
|
+
if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) {
|
|
103
|
+
return MMVQ_PARAMETERS_RDNA2;
|
|
104
|
+
}
|
|
105
|
+
if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) {
|
|
106
|
+
return MMVQ_PARAMETERS_GCN;
|
|
107
|
+
}
|
|
108
|
+
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_TURING && ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_AMPERE) {
|
|
109
|
+
return MMVQ_PARAMETERS_TURING;
|
|
110
|
+
}
|
|
111
|
+
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_DGX_SPARK) {
|
|
112
|
+
return MMVQ_PARAMETERS_GB10;
|
|
113
|
+
}
|
|
114
|
+
return MMVQ_PARAMETERS_GENERIC;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Per-architecture maximum batch size for which MMVQ should be used for MUL_MAT_ID.
|
|
118
|
+
// Returns a value <= MMVQ_MAX_BATCH_SIZE. Default is MMVQ_MAX_BATCH_SIZE.
|
|
119
|
+
// Check https://github.com/ggml-org/llama.cpp/pull/20905#issuecomment-4145835627 for details
|
|
120
|
+
|
|
121
|
+
static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_pascal_older(ggml_type type) {
|
|
122
|
+
switch (type) {
|
|
123
|
+
case GGML_TYPE_IQ1_S: return 6;
|
|
124
|
+
case GGML_TYPE_IQ1_M: return 6;
|
|
125
|
+
case GGML_TYPE_IQ2_S: return 4;
|
|
126
|
+
case GGML_TYPE_IQ2_XS: return 5;
|
|
127
|
+
case GGML_TYPE_IQ2_XXS: return 5;
|
|
128
|
+
case GGML_TYPE_IQ3_S: return 4;
|
|
129
|
+
case GGML_TYPE_IQ3_XXS: return 4;
|
|
130
|
+
case GGML_TYPE_IQ4_NL: return 6;
|
|
131
|
+
case GGML_TYPE_IQ4_XS: return 5;
|
|
132
|
+
case GGML_TYPE_MXFP4: return 4;
|
|
133
|
+
case GGML_TYPE_NVFP4: return 4;
|
|
134
|
+
case GGML_TYPE_Q2_K: return 4;
|
|
135
|
+
case GGML_TYPE_Q3_K: return 4;
|
|
136
|
+
case GGML_TYPE_Q4_0: return 6;
|
|
137
|
+
case GGML_TYPE_Q4_1: return 6;
|
|
138
|
+
case GGML_TYPE_Q4_K: return 5;
|
|
139
|
+
case GGML_TYPE_Q5_0: return 6;
|
|
140
|
+
case GGML_TYPE_Q5_1: return 6;
|
|
141
|
+
case GGML_TYPE_Q5_K: return 5;
|
|
142
|
+
case GGML_TYPE_Q6_K: return 4;
|
|
143
|
+
case GGML_TYPE_Q8_0: return 4;
|
|
144
|
+
default: return MMVQ_MAX_BATCH_SIZE;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_turing_plus(ggml_type type) {
|
|
149
|
+
switch (type) {
|
|
150
|
+
case GGML_TYPE_IQ2_S: return 7;
|
|
151
|
+
case GGML_TYPE_IQ3_S: return 6;
|
|
152
|
+
case GGML_TYPE_IQ3_XXS: return 7;
|
|
153
|
+
case GGML_TYPE_MXFP4: return 7;
|
|
154
|
+
case GGML_TYPE_NVFP4: return 8;
|
|
155
|
+
case GGML_TYPE_Q2_K: return 7;
|
|
156
|
+
case GGML_TYPE_Q3_K: return 5;
|
|
157
|
+
default: return MMVQ_MAX_BATCH_SIZE;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_gcn(ggml_type type) {
|
|
162
|
+
switch (type) {
|
|
163
|
+
case GGML_TYPE_IQ1_S: return 5;
|
|
164
|
+
case GGML_TYPE_IQ1_M: return 5;
|
|
165
|
+
case GGML_TYPE_IQ2_S: return 4;
|
|
166
|
+
case GGML_TYPE_IQ2_XS: return 4;
|
|
167
|
+
case GGML_TYPE_IQ2_XXS: return 4;
|
|
168
|
+
case GGML_TYPE_IQ3_S: return 4;
|
|
169
|
+
case GGML_TYPE_IQ3_XXS: return 4;
|
|
170
|
+
case GGML_TYPE_IQ4_NL: return 6;
|
|
171
|
+
case GGML_TYPE_IQ4_XS: return 4;
|
|
172
|
+
case GGML_TYPE_Q2_K: return 4;
|
|
173
|
+
case GGML_TYPE_Q3_K: return 4;
|
|
174
|
+
case GGML_TYPE_Q4_0: return 5;
|
|
175
|
+
case GGML_TYPE_Q4_1: return 5;
|
|
176
|
+
case GGML_TYPE_Q4_K: return 4;
|
|
177
|
+
case GGML_TYPE_Q5_K: return 4;
|
|
178
|
+
case GGML_TYPE_Q6_K: return 4;
|
|
179
|
+
case GGML_TYPE_Q8_0: return 4;
|
|
180
|
+
default: return MMVQ_MAX_BATCH_SIZE;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_cdna(ggml_type type) {
|
|
185
|
+
switch (type) {
|
|
186
|
+
case GGML_TYPE_IQ2_S: return 5;
|
|
187
|
+
case GGML_TYPE_IQ2_XS: return 5;
|
|
188
|
+
case GGML_TYPE_IQ2_XXS: return 5;
|
|
189
|
+
case GGML_TYPE_IQ3_S: return 4;
|
|
190
|
+
case GGML_TYPE_IQ3_XXS: return 5;
|
|
191
|
+
default: return MMVQ_MAX_BATCH_SIZE;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna1_rdna2(ggml_type type) {
|
|
196
|
+
switch (type) {
|
|
197
|
+
case GGML_TYPE_IQ2_S: return 4;
|
|
198
|
+
case GGML_TYPE_IQ2_XS: return 4;
|
|
199
|
+
case GGML_TYPE_IQ2_XXS: return 4;
|
|
200
|
+
case GGML_TYPE_IQ3_S: return 4;
|
|
201
|
+
case GGML_TYPE_IQ3_XXS: return 4;
|
|
202
|
+
case GGML_TYPE_Q2_K: return 7;
|
|
203
|
+
case GGML_TYPE_Q3_K: return 4;
|
|
204
|
+
case GGML_TYPE_Q4_K: return 5;
|
|
205
|
+
case GGML_TYPE_Q5_K: return 6;
|
|
206
|
+
case GGML_TYPE_Q6_K: return 5;
|
|
207
|
+
default: return MMVQ_MAX_BATCH_SIZE;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna3(ggml_type type) {
|
|
212
|
+
switch (type) {
|
|
213
|
+
case GGML_TYPE_IQ1_S: return 6;
|
|
214
|
+
case GGML_TYPE_IQ1_M: return 6;
|
|
215
|
+
case GGML_TYPE_IQ2_S: return 4;
|
|
216
|
+
case GGML_TYPE_IQ2_XS: return 4;
|
|
217
|
+
case GGML_TYPE_IQ2_XXS: return 4;
|
|
218
|
+
case GGML_TYPE_IQ3_S: return 4;
|
|
219
|
+
case GGML_TYPE_IQ3_XXS: return 4;
|
|
220
|
+
case GGML_TYPE_IQ4_NL: return 6;
|
|
221
|
+
case GGML_TYPE_IQ4_XS: return 6;
|
|
222
|
+
case GGML_TYPE_Q4_K: return 4;
|
|
223
|
+
case GGML_TYPE_Q5_K: return 4;
|
|
224
|
+
case GGML_TYPE_Q6_K: return 4;
|
|
225
|
+
default: return MMVQ_MAX_BATCH_SIZE;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna4(ggml_type type) {
|
|
230
|
+
switch (type) {
|
|
231
|
+
case GGML_TYPE_IQ1_S: return 7;
|
|
232
|
+
case GGML_TYPE_IQ1_M: return 7;
|
|
233
|
+
case GGML_TYPE_IQ2_S: return 4;
|
|
234
|
+
case GGML_TYPE_IQ2_XS: return 4;
|
|
235
|
+
case GGML_TYPE_IQ2_XXS: return 4;
|
|
236
|
+
case GGML_TYPE_IQ3_S: return 4;
|
|
237
|
+
case GGML_TYPE_IQ3_XXS: return 4;
|
|
238
|
+
case GGML_TYPE_IQ4_NL: return 7;
|
|
239
|
+
case GGML_TYPE_IQ4_XS: return 5;
|
|
240
|
+
case GGML_TYPE_MXFP4: return 5;
|
|
241
|
+
case GGML_TYPE_NVFP4: return 5;
|
|
242
|
+
case GGML_TYPE_Q3_K: return 4;
|
|
243
|
+
case GGML_TYPE_Q4_0: return 7;
|
|
244
|
+
case GGML_TYPE_Q4_1: return 7;
|
|
245
|
+
case GGML_TYPE_Q4_K: return 4;
|
|
246
|
+
case GGML_TYPE_Q5_0: return 7;
|
|
247
|
+
case GGML_TYPE_Q5_1: return 7;
|
|
248
|
+
case GGML_TYPE_Q5_K: return 5;
|
|
249
|
+
case GGML_TYPE_Q6_K: return 5;
|
|
250
|
+
case GGML_TYPE_Q8_0: return 7;
|
|
251
|
+
default: return MMVQ_MAX_BATCH_SIZE;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// Host function: returns the max batch size for the current arch+type at runtime.
|
|
256
|
+
int get_mmvq_mmid_max_batch(ggml_type type, int cc) {
|
|
257
|
+
// NVIDIA: Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID.
|
|
258
|
+
if (GGML_CUDA_CC_IS_NVIDIA(cc)) {
|
|
259
|
+
if (cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) {
|
|
260
|
+
return MMVQ_MAX_BATCH_SIZE;
|
|
261
|
+
}
|
|
262
|
+
if (cc >= GGML_CUDA_CC_TURING) {
|
|
263
|
+
return get_mmvq_mmid_max_batch_turing_plus(type);
|
|
264
|
+
}
|
|
265
|
+
return get_mmvq_mmid_max_batch_pascal_older(type);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// AMD
|
|
269
|
+
if (GGML_CUDA_CC_IS_AMD(cc)) {
|
|
270
|
+
if (GGML_CUDA_CC_IS_RDNA4(cc)) {
|
|
271
|
+
return get_mmvq_mmid_max_batch_rdna4(type);
|
|
272
|
+
}
|
|
273
|
+
if (GGML_CUDA_CC_IS_RDNA3(cc)) {
|
|
274
|
+
return get_mmvq_mmid_max_batch_rdna3(type);
|
|
275
|
+
}
|
|
276
|
+
if (GGML_CUDA_CC_IS_RDNA1(cc) || GGML_CUDA_CC_IS_RDNA2(cc)) {
|
|
277
|
+
return get_mmvq_mmid_max_batch_rdna1_rdna2(type);
|
|
278
|
+
}
|
|
279
|
+
if (GGML_CUDA_CC_IS_CDNA(cc)) {
|
|
280
|
+
return get_mmvq_mmid_max_batch_cdna(type);
|
|
281
|
+
}
|
|
282
|
+
if (GGML_CUDA_CC_IS_GCN(cc)) {
|
|
283
|
+
return get_mmvq_mmid_max_batch_gcn(type);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
return MMVQ_MAX_BATCH_SIZE;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
|
|
290
|
+
if (!ggml_is_quantized(type)) {
|
|
291
|
+
return false;
|
|
292
|
+
}
|
|
293
|
+
// k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner.
|
|
294
|
+
// Only list quant-types MMQ supports, others would fall back to cuBLAS.
|
|
295
|
+
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) {
|
|
296
|
+
switch (type) { // tuned on RTX 4090
|
|
297
|
+
case GGML_TYPE_Q2_K:
|
|
298
|
+
return ne11 <= 4;
|
|
299
|
+
case GGML_TYPE_Q3_K:
|
|
300
|
+
return ne11 <= 6;
|
|
301
|
+
case GGML_TYPE_Q4_K:
|
|
302
|
+
case GGML_TYPE_Q5_K:
|
|
303
|
+
return ne11 <= 7;
|
|
304
|
+
default:
|
|
305
|
+
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) {
|
|
309
|
+
switch (type) { // tuned on RTX 5090
|
|
310
|
+
case GGML_TYPE_Q2_K:
|
|
311
|
+
case GGML_TYPE_Q3_K:
|
|
312
|
+
case GGML_TYPE_Q4_K:
|
|
313
|
+
case GGML_TYPE_Q5_K:
|
|
314
|
+
return ne11 <= 5;
|
|
315
|
+
case GGML_TYPE_Q6_K:
|
|
316
|
+
return ne11 <= 7;
|
|
317
|
+
default:
|
|
318
|
+
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
|
|
322
|
+
switch (type) { // tuned on DGX Spark GB10
|
|
323
|
+
case GGML_TYPE_Q2_K:
|
|
324
|
+
return ne11 <= 6;
|
|
325
|
+
default:
|
|
326
|
+
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ORIN) {
|
|
330
|
+
switch (type) { // tuned for Jetson Orin
|
|
331
|
+
case GGML_TYPE_Q2_K:
|
|
332
|
+
case GGML_TYPE_Q3_K:
|
|
333
|
+
case GGML_TYPE_Q4_K:
|
|
334
|
+
case GGML_TYPE_Q5_K:
|
|
335
|
+
case GGML_TYPE_Q6_K:
|
|
336
|
+
return ne11 <= 1;
|
|
337
|
+
default:
|
|
338
|
+
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
if (GGML_CUDA_CC_IS_CDNA(cc)) {
|
|
342
|
+
if (GGML_CUDA_CC_IS_CDNA1(cc)) {
|
|
343
|
+
switch (type) {
|
|
344
|
+
case GGML_TYPE_Q4_0:
|
|
345
|
+
case GGML_TYPE_Q4_1:
|
|
346
|
+
return ne11 <= 7;
|
|
347
|
+
case GGML_TYPE_Q5_1:
|
|
348
|
+
return ne11 <= 7;
|
|
349
|
+
case GGML_TYPE_Q8_0:
|
|
350
|
+
return ne11 <= 6;
|
|
351
|
+
case GGML_TYPE_Q2_K:
|
|
352
|
+
return ne11 <= 4;
|
|
353
|
+
case GGML_TYPE_Q3_K:
|
|
354
|
+
return ne11 <= 3;
|
|
355
|
+
case GGML_TYPE_Q4_K:
|
|
356
|
+
return ne11 <= 2;
|
|
357
|
+
case GGML_TYPE_Q5_K:
|
|
358
|
+
return ne11 <= 3;
|
|
359
|
+
case GGML_TYPE_Q6_K:
|
|
360
|
+
return ne11 <= 4;
|
|
361
|
+
case GGML_TYPE_IQ1_S:
|
|
362
|
+
return ne11 <= 5;
|
|
363
|
+
case GGML_TYPE_IQ2_XXS:
|
|
364
|
+
case GGML_TYPE_IQ3_S:
|
|
365
|
+
case GGML_TYPE_IQ4_XS:
|
|
366
|
+
return ne11 <= 6;
|
|
367
|
+
default:
|
|
368
|
+
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
switch (type) { // tuned for CDNA2
|
|
372
|
+
case GGML_TYPE_Q2_K:
|
|
373
|
+
return ne11 <= 5;
|
|
374
|
+
case GGML_TYPE_Q3_K:
|
|
375
|
+
case GGML_TYPE_Q4_K:
|
|
376
|
+
case GGML_TYPE_Q5_K:
|
|
377
|
+
return ne11 <= 3;
|
|
378
|
+
case GGML_TYPE_Q6_K:
|
|
379
|
+
return ne11 <= 5;
|
|
380
|
+
default:
|
|
381
|
+
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
// Device constexpr: returns the max batch size for the current arch+type at compile time.
|
|
388
|
+
template <ggml_type type>
|
|
389
|
+
static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() {
|
|
390
|
+
#if defined(RDNA4)
|
|
391
|
+
return get_mmvq_mmid_max_batch_rdna4(type);
|
|
392
|
+
#elif defined(RDNA3)
|
|
393
|
+
return get_mmvq_mmid_max_batch_rdna3(type);
|
|
394
|
+
#elif defined(RDNA2) || defined(RDNA1)
|
|
395
|
+
return get_mmvq_mmid_max_batch_rdna1_rdna2(type);
|
|
396
|
+
#elif defined(CDNA)
|
|
397
|
+
return get_mmvq_mmid_max_batch_cdna(type);
|
|
398
|
+
#elif defined(GCN)
|
|
399
|
+
return get_mmvq_mmid_max_batch_gcn(type);
|
|
400
|
+
#elif defined(__CUDA_ARCH__) && (__CUDA_ARCH__ == GGML_CUDA_CC_VOLTA || __CUDA_ARCH__ >= GGML_CUDA_CC_ADA_LOVELACE)
|
|
401
|
+
return MMVQ_MAX_BATCH_SIZE;
|
|
402
|
+
#elif defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING
|
|
403
|
+
return get_mmvq_mmid_max_batch_turing_plus(type);
|
|
404
|
+
#else
|
|
405
|
+
return get_mmvq_mmid_max_batch_pascal_older(type);
|
|
406
|
+
#endif
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_dst, mmvq_parameter_table_id table_id, bool small_k = false, bool halve_iters = false) {
|
|
410
|
+
if (table_id == MMVQ_PARAMETERS_GENERIC) {
|
|
411
|
+
switch (ncols_dst) {
|
|
412
|
+
case 1:
|
|
413
|
+
case 2:
|
|
414
|
+
case 3:
|
|
415
|
+
case 4:
|
|
416
|
+
return 4;
|
|
417
|
+
case 5:
|
|
418
|
+
case 6:
|
|
419
|
+
case 7:
|
|
420
|
+
case 8:
|
|
421
|
+
return 2;
|
|
422
|
+
default:
|
|
423
|
+
return 1;
|
|
424
|
+
}
|
|
425
|
+
} else if (table_id == MMVQ_PARAMETERS_GCN) {
|
|
426
|
+
switch (ncols_dst) {
|
|
427
|
+
case 1:
|
|
428
|
+
case 2:
|
|
429
|
+
case 3:
|
|
430
|
+
case 4:
|
|
431
|
+
return 2;
|
|
432
|
+
case 5:
|
|
433
|
+
case 6:
|
|
434
|
+
case 7:
|
|
435
|
+
case 8:
|
|
436
|
+
default:
|
|
437
|
+
return 1;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
if (table_id == MMVQ_PARAMETERS_RDNA4) {
|
|
441
|
+
// nwarps=8 benefits types with simple vec_dot on RDNA4 (ncols_dst=1).
|
|
442
|
+
// Types with complex vec_dot (Q3_K, IQ2_*, IQ3_*) regress due to register
|
|
443
|
+
// pressure and lookup table contention at higher thread counts.
|
|
444
|
+
if (ncols_dst == 1) {
|
|
445
|
+
switch (type) {
|
|
446
|
+
case GGML_TYPE_Q4_0:
|
|
447
|
+
case GGML_TYPE_Q4_1:
|
|
448
|
+
case GGML_TYPE_Q5_0:
|
|
449
|
+
case GGML_TYPE_Q5_1:
|
|
450
|
+
case GGML_TYPE_Q8_0:
|
|
451
|
+
case GGML_TYPE_Q2_K:
|
|
452
|
+
case GGML_TYPE_Q4_K:
|
|
453
|
+
case GGML_TYPE_Q5_K:
|
|
454
|
+
case GGML_TYPE_Q6_K:
|
|
455
|
+
case GGML_TYPE_IQ4_NL:
|
|
456
|
+
case GGML_TYPE_IQ4_XS:
|
|
457
|
+
return 8;
|
|
458
|
+
default:
|
|
459
|
+
return 1;
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
return 1;
|
|
463
|
+
}
|
|
464
|
+
if (table_id == MMVQ_PARAMETERS_RDNA3_0) {
|
|
465
|
+
// RDNA3 (W7900): stricter whitelist than RDNA4.
|
|
466
|
+
// Q2_K / Q5_K / IQ4_XS regress in full quant sweeps.
|
|
467
|
+
if (ncols_dst == 1) {
|
|
468
|
+
switch (type) {
|
|
469
|
+
case GGML_TYPE_Q4_0:
|
|
470
|
+
case GGML_TYPE_Q4_1:
|
|
471
|
+
case GGML_TYPE_Q5_0:
|
|
472
|
+
case GGML_TYPE_Q5_1:
|
|
473
|
+
case GGML_TYPE_Q8_0:
|
|
474
|
+
return 8;
|
|
475
|
+
case GGML_TYPE_Q6_K:
|
|
476
|
+
return 2;
|
|
477
|
+
case GGML_TYPE_IQ4_NL:
|
|
478
|
+
return 8;
|
|
479
|
+
default:
|
|
480
|
+
return 1;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
return 1;
|
|
484
|
+
}
|
|
485
|
+
if (table_id == MMVQ_PARAMETERS_TURING) {
|
|
486
|
+
if (ncols_dst == 1) {
|
|
487
|
+
switch (type) {
|
|
488
|
+
case GGML_TYPE_Q2_K:
|
|
489
|
+
case GGML_TYPE_Q3_K:
|
|
490
|
+
case GGML_TYPE_Q4_K:
|
|
491
|
+
case GGML_TYPE_Q5_K:
|
|
492
|
+
case GGML_TYPE_Q6_K:
|
|
493
|
+
return 2;
|
|
494
|
+
default:
|
|
495
|
+
return 4;
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
switch (ncols_dst) {
|
|
499
|
+
case 2:
|
|
500
|
+
case 3:
|
|
501
|
+
case 4:
|
|
502
|
+
return 4;
|
|
503
|
+
case 5:
|
|
504
|
+
case 6:
|
|
505
|
+
case 7:
|
|
506
|
+
case 8:
|
|
507
|
+
return 2;
|
|
508
|
+
default:
|
|
509
|
+
return 1;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
if (table_id == MMVQ_PARAMETERS_GB10) {
|
|
513
|
+
const int generic = calc_nwarps(type, ncols_dst, MMVQ_PARAMETERS_GENERIC);
|
|
514
|
+
// Only worth the wider block when it actually retires the K loop in half the trips (Observation)
|
|
515
|
+
if (ncols_dst == 1 && !small_k && halve_iters) {
|
|
516
|
+
switch (type) {
|
|
517
|
+
case GGML_TYPE_Q4_0:
|
|
518
|
+
case GGML_TYPE_Q4_1:
|
|
519
|
+
case GGML_TYPE_Q5_0:
|
|
520
|
+
case GGML_TYPE_Q5_1:
|
|
521
|
+
case GGML_TYPE_Q8_0:
|
|
522
|
+
case GGML_TYPE_Q4_K:
|
|
523
|
+
case GGML_TYPE_Q5_K:
|
|
524
|
+
case GGML_TYPE_Q6_K:
|
|
525
|
+
case GGML_TYPE_IQ4_NL:
|
|
526
|
+
return 2 * generic;
|
|
527
|
+
default:
|
|
528
|
+
break;
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
return generic;
|
|
532
|
+
}
|
|
533
|
+
return 1;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) {
|
|
537
|
+
if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN || table_id == MMVQ_PARAMETERS_TURING || table_id == MMVQ_PARAMETERS_GB10) {
|
|
538
|
+
switch (ncols_dst) {
|
|
539
|
+
case 1:
|
|
540
|
+
return small_k ? nwarps : 1;
|
|
541
|
+
case 2:
|
|
542
|
+
case 3:
|
|
543
|
+
case 4:
|
|
544
|
+
case 5:
|
|
545
|
+
case 6:
|
|
546
|
+
case 7:
|
|
547
|
+
case 8:
|
|
548
|
+
return 2;
|
|
549
|
+
default:
|
|
550
|
+
return 1;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
return 1;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
template <ggml_type type, int ncols_dst, bool has_fusion, bool small_k = false, bool halve_iters = false>
|
|
557
|
+
__launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id(), small_k, halve_iters)*ggml_cuda_get_physical_warp_size(), 1)
|
|
558
|
+
static __global__ void mul_mat_vec_q(
|
|
559
|
+
const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr,
|
|
560
|
+
const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,
|
|
561
|
+
const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,
|
|
562
|
+
const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,
|
|
563
|
+
const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst,
|
|
564
|
+
const uint32_t ids_stride) {
|
|
565
|
+
const void * GGML_CUDA_RESTRICT vx = vx_ptr;
|
|
566
|
+
const void * GGML_CUDA_RESTRICT vy = vy_ptr;
|
|
567
|
+
const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
|
|
568
|
+
float * GGML_CUDA_RESTRICT dst = dst_ptr;
|
|
569
|
+
|
|
570
|
+
constexpr int qk = ggml_cuda_type_traits<type>::qk;
|
|
571
|
+
constexpr int qi = ggml_cuda_type_traits<type>::qi;
|
|
572
|
+
constexpr int vdr = get_vdr_mmvq(type);
|
|
573
|
+
constexpr mmvq_parameter_table_id table_id = get_device_table_id();
|
|
574
|
+
constexpr int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters);
|
|
575
|
+
constexpr int rows_per_cuda_block = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps);
|
|
576
|
+
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
|
577
|
+
|
|
578
|
+
constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);
|
|
579
|
+
|
|
580
|
+
const int tid = warp_size*threadIdx.y + threadIdx.x;
|
|
581
|
+
const int row0 = rows_per_cuda_block*blockIdx.x;
|
|
582
|
+
const int blocks_per_row_x = ncols_x / qk;
|
|
583
|
+
constexpr int blocks_per_iter = vdr * nwarps*warp_size / qi;
|
|
584
|
+
|
|
585
|
+
const uint32_t channel_dst = blockIdx.y;
|
|
586
|
+
|
|
587
|
+
uint32_t channel_x;
|
|
588
|
+
uint32_t channel_y;
|
|
589
|
+
uint32_t sample_dst;
|
|
590
|
+
|
|
591
|
+
ggml_cuda_pdl_sync();
|
|
592
|
+
channel_x = ncols_dst == 1 && ids ? ids[channel_dst] : fastdiv(channel_dst, channel_ratio);
|
|
593
|
+
channel_y = ncols_dst == 1 && ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst;
|
|
594
|
+
sample_dst = blockIdx.z;
|
|
595
|
+
|
|
596
|
+
const uint32_t sample_x = fastdiv(sample_dst, sample_ratio);
|
|
597
|
+
const uint32_t sample_y = sample_dst;
|
|
598
|
+
|
|
599
|
+
bool use_gate = false;
|
|
600
|
+
bool use_bias = false;
|
|
601
|
+
bool use_gate_bias = false;
|
|
602
|
+
bool use_scale = false;
|
|
603
|
+
bool use_gate_scale = false;
|
|
604
|
+
[[maybe_unused]] const void * vgate = nullptr;
|
|
605
|
+
const float * x_bias = nullptr;
|
|
606
|
+
const float * gate_bias = nullptr;
|
|
607
|
+
const float * x_scale = nullptr;
|
|
608
|
+
const float * gate_scale = nullptr;
|
|
609
|
+
ggml_glu_op active_glu;
|
|
610
|
+
float glu_limit = 0.0f;
|
|
611
|
+
|
|
612
|
+
if constexpr (has_fusion) {
|
|
613
|
+
use_gate = fusion.gate != nullptr;
|
|
614
|
+
use_bias = fusion.x_bias != nullptr;
|
|
615
|
+
use_gate_bias = fusion.gate_bias != nullptr && use_gate;
|
|
616
|
+
vgate = fusion.gate;
|
|
617
|
+
x_bias = (const float *) fusion.x_bias;
|
|
618
|
+
gate_bias = (const float *) fusion.gate_bias;
|
|
619
|
+
active_glu = fusion.glu_op;
|
|
620
|
+
glu_limit = fusion.glu_limit;
|
|
621
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
622
|
+
use_scale = fusion.x_scale != nullptr;
|
|
623
|
+
use_gate_scale = fusion.gate_scale != nullptr && use_gate;
|
|
624
|
+
x_scale = (const float *) fusion.x_scale;
|
|
625
|
+
gate_scale = (const float *) fusion.gate_scale;
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
[[maybe_unused]] float x_biases[ncols_dst] = { 0.0f };
|
|
631
|
+
[[maybe_unused]] float gate_biases[ncols_dst] = { 0.0f };
|
|
632
|
+
[[maybe_unused]] float x_scales = 1.0f;
|
|
633
|
+
[[maybe_unused]] float gate_scales = 1.0f;
|
|
634
|
+
if constexpr (has_fusion) {
|
|
635
|
+
// 1. Hide latency by prefetching bias, gates and scales here
|
|
636
|
+
// 2. load only on threads that won't die after partial sum calculation
|
|
637
|
+
const uint32_t channel_bias = ids ? channel_x : channel_dst;
|
|
638
|
+
if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 &&
|
|
639
|
+
(rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) {
|
|
640
|
+
if (use_bias) {
|
|
641
|
+
x_bias = x_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0;
|
|
642
|
+
#pragma unroll
|
|
643
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
644
|
+
x_biases[j] = x_bias[j * stride_col_dst + threadIdx.x];
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
if (use_gate_bias) {
|
|
648
|
+
gate_bias = gate_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0;
|
|
649
|
+
#pragma unroll
|
|
650
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
651
|
+
gate_biases[j] = gate_bias[j * stride_col_dst + threadIdx.x];
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
655
|
+
if (use_scale) {
|
|
656
|
+
x_scales = x_scale[ids ? channel_x : 0];
|
|
657
|
+
}
|
|
658
|
+
if (use_gate_scale) {
|
|
659
|
+
gate_scales = gate_scale[ids ? channel_x : 0];
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
// partial sum for each thread
|
|
666
|
+
float tmp[ncols_dst][rows_per_cuda_block] = {{0.0f}};
|
|
667
|
+
float tmp_gate[ncols_dst][rows_per_cuda_block] = {{0.0f}};
|
|
668
|
+
|
|
669
|
+
const block_q8_1 * y = ((const block_q8_1 *) vy) + sample_y*stride_sample_y + channel_y*stride_channel_y;
|
|
670
|
+
const int kbx_offset = sample_x*stride_sample_x + channel_x*stride_channel_x + row0*stride_row_x;
|
|
671
|
+
|
|
672
|
+
for (int kbx = tid / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {
|
|
673
|
+
const int kby = kbx * (qk/QK8_1); // y block index that aligns with kbx
|
|
674
|
+
|
|
675
|
+
// x block quant index when casting the quants to int
|
|
676
|
+
const int kqs = vdr * (tid % (qi/vdr));
|
|
677
|
+
|
|
678
|
+
#pragma unroll
|
|
679
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
680
|
+
#pragma unroll
|
|
681
|
+
for (int i = 0; i < rows_per_cuda_block; ++i) {
|
|
682
|
+
tmp[j][i] += vec_dot_q_cuda(
|
|
683
|
+
vx, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);
|
|
684
|
+
if constexpr (has_fusion) {
|
|
685
|
+
if (use_gate) {
|
|
686
|
+
tmp_gate[j][i] += vec_dot_q_cuda(
|
|
687
|
+
vgate, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
__shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size];
|
|
695
|
+
[[maybe_unused]] __shared__ float tmp_shared_gate[(has_fusion && (nwarps-1 > 0)) ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size];
|
|
696
|
+
|
|
697
|
+
if (threadIdx.y > 0) {
|
|
698
|
+
#pragma unroll
|
|
699
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
700
|
+
#pragma unroll
|
|
701
|
+
for (int i = 0; i < rows_per_cuda_block; ++i) {
|
|
702
|
+
tmp_shared[threadIdx.y-1][j][i][threadIdx.x] = tmp[j][i];
|
|
703
|
+
if constexpr (has_fusion) {
|
|
704
|
+
if (use_gate) {
|
|
705
|
+
tmp_shared_gate[threadIdx.y-1][j][i][threadIdx.x] = tmp_gate[j][i];
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
__syncthreads();
|
|
712
|
+
if (threadIdx.y > 0) {
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
dst += sample_dst*stride_sample_dst + channel_dst*stride_channel_dst + row0;
|
|
717
|
+
|
|
718
|
+
// sum up partial sums and write back result
|
|
719
|
+
#pragma unroll
|
|
720
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
721
|
+
#pragma unroll
|
|
722
|
+
for (int i = 0; i < rows_per_cuda_block; ++i) {
|
|
723
|
+
#pragma unroll
|
|
724
|
+
for (int l = 0; l < nwarps-1; ++l) {
|
|
725
|
+
tmp[j][i] += tmp_shared[l][j][i][threadIdx.x];
|
|
726
|
+
if constexpr (has_fusion) {
|
|
727
|
+
if (use_gate) {
|
|
728
|
+
tmp_gate[j][i] += tmp_shared_gate[l][j][i][threadIdx.x];
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
tmp[j][i] = warp_reduce_sum<warp_size>(tmp[j][i]);
|
|
733
|
+
if constexpr (has_fusion) {
|
|
734
|
+
if (use_gate) {
|
|
735
|
+
tmp_gate[j][i] = warp_reduce_sum<warp_size>(tmp_gate[j][i]);
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
if (threadIdx.x == i && (rows_per_cuda_block == 1 || uint32_t(row0 + i) < stride_col_dst)) {
|
|
740
|
+
float result = tmp[j][i];
|
|
741
|
+
if constexpr (has_fusion) {
|
|
742
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
743
|
+
result *= x_scales;
|
|
744
|
+
}
|
|
745
|
+
result += x_biases[j];
|
|
746
|
+
if (use_gate) {
|
|
747
|
+
float gate_value = tmp_gate[j][i];
|
|
748
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
749
|
+
gate_value *= gate_scales;
|
|
750
|
+
}
|
|
751
|
+
gate_value += gate_biases[j];
|
|
752
|
+
switch (active_glu) {
|
|
753
|
+
case GGML_GLU_OP_SWIGLU:
|
|
754
|
+
result *= ggml_cuda_op_silu_single(gate_value);
|
|
755
|
+
break;
|
|
756
|
+
case GGML_GLU_OP_GEGLU:
|
|
757
|
+
result *= ggml_cuda_op_gelu_single(gate_value);
|
|
758
|
+
break;
|
|
759
|
+
case GGML_GLU_OP_SWIGLU_OAI:
|
|
760
|
+
result = ggml_cuda_op_swiglu_oai_single(gate_value, result);
|
|
761
|
+
break;
|
|
762
|
+
case GGML_GLU_OP_SWIGLU_CLAMP:
|
|
763
|
+
result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit);
|
|
764
|
+
break;
|
|
765
|
+
default:
|
|
766
|
+
result = result * gate_value;
|
|
767
|
+
break;
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
dst[j*stride_col_dst + i] = result;
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
if constexpr (!has_fusion) {
|
|
777
|
+
GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, active_glu, glu_limit, gate_bias, x_bias, x_scale, gate_scale, tmp_gate);
|
|
778
|
+
}
|
|
779
|
+
if constexpr (type != GGML_TYPE_NVFP4) {
|
|
780
|
+
GGML_UNUSED_VARS(use_scale, use_gate_scale, x_scale, gate_scale, x_scales, gate_scales);
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
// Dedicated MoE multi-token kernel.
|
|
785
|
+
// Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst)
|
|
786
|
+
// Block: (warp_size, ncols_dst) - each warp handles one token independently.
|
|
787
|
+
// No shared memory reduction needed since each warp works alone.
|
|
788
|
+
template <ggml_type type, int c_rows_per_block, bool has_fusion = false>
|
|
789
|
+
__launch_bounds__(get_mmvq_mmid_max_batch_for_device<type>()*ggml_cuda_get_physical_warp_size(), 1)
|
|
790
|
+
static __global__ void mul_mat_vec_q_moe(
|
|
791
|
+
const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion,
|
|
792
|
+
float * dst_ptr,
|
|
793
|
+
const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
|
|
794
|
+
const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst,
|
|
795
|
+
const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,
|
|
796
|
+
const uint32_t ncols_dst, const uint32_t ids_stride) {
|
|
797
|
+
const void * GGML_CUDA_RESTRICT vx = vx_ptr;
|
|
798
|
+
const void * GGML_CUDA_RESTRICT vy = vy_ptr;
|
|
799
|
+
const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
|
|
800
|
+
float * GGML_CUDA_RESTRICT dst = dst_ptr;
|
|
801
|
+
|
|
802
|
+
constexpr int qk = ggml_cuda_type_traits<type>::qk;
|
|
803
|
+
constexpr int qi = ggml_cuda_type_traits<type>::qi;
|
|
804
|
+
constexpr int vdr = get_vdr_mmvq(type);
|
|
805
|
+
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
|
806
|
+
|
|
807
|
+
constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);
|
|
808
|
+
|
|
809
|
+
// fuse gate, bias, scales, and glu_op into the up projection
|
|
810
|
+
bool use_gate = false;
|
|
811
|
+
const void * vgate = nullptr;
|
|
812
|
+
const float * x_bias = nullptr;
|
|
813
|
+
const float * gate_bias = nullptr;
|
|
814
|
+
const float * x_scale = nullptr;
|
|
815
|
+
const float * gate_scale = nullptr;
|
|
816
|
+
ggml_glu_op active_glu = GGML_GLU_OP_SWIGLU;
|
|
817
|
+
float glu_limit = 0.0f;
|
|
818
|
+
|
|
819
|
+
if constexpr (has_fusion) {
|
|
820
|
+
use_gate = fusion.gate != nullptr;
|
|
821
|
+
vgate = fusion.gate;
|
|
822
|
+
x_bias = (const float *) fusion.x_bias;
|
|
823
|
+
gate_bias = (const float *) fusion.gate_bias;
|
|
824
|
+
active_glu = fusion.glu_op;
|
|
825
|
+
glu_limit = fusion.glu_limit;
|
|
826
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
827
|
+
x_scale = (const float *) fusion.x_scale;
|
|
828
|
+
gate_scale = (const float *) fusion.gate_scale;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
const uint32_t token_idx = threadIdx.y;
|
|
833
|
+
const int row0 = c_rows_per_block*blockIdx.x;
|
|
834
|
+
const int blocks_per_row_x = ncols_x / qk;
|
|
835
|
+
constexpr int blocks_per_iter = vdr * warp_size / qi;
|
|
836
|
+
|
|
837
|
+
const uint32_t channel_dst = blockIdx.y;
|
|
838
|
+
|
|
839
|
+
if (token_idx >= ncols_dst) {
|
|
840
|
+
return;
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
ggml_cuda_pdl_sync();
|
|
844
|
+
const uint32_t channel_x = ids[channel_dst + token_idx * ids_stride];
|
|
845
|
+
const uint32_t channel_y = fastmodulo(channel_dst, nchannels_y);
|
|
846
|
+
|
|
847
|
+
const block_q8_1 * y = ((const block_q8_1 *) vy) + channel_y*stride_channel_y + token_idx*stride_col_y;
|
|
848
|
+
const int kbx_offset = channel_x*stride_channel_x + row0*stride_row_x;
|
|
849
|
+
|
|
850
|
+
// partial sum for each thread
|
|
851
|
+
float tmp[c_rows_per_block] = {0.0f};
|
|
852
|
+
float tmp_gate[c_rows_per_block] = {0.0f};
|
|
853
|
+
|
|
854
|
+
for (int kbx = threadIdx.x / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {
|
|
855
|
+
const int kby = kbx * (qk/QK8_1);
|
|
856
|
+
const int kqs = vdr * (threadIdx.x % (qi/vdr));
|
|
857
|
+
|
|
858
|
+
#pragma unroll
|
|
859
|
+
for (int i = 0; i < c_rows_per_block; ++i) {
|
|
860
|
+
tmp[i] += vec_dot_q_cuda(vx, &y[kby], kbx_offset + i*stride_row_x + kbx, kqs);
|
|
861
|
+
if constexpr (has_fusion) {
|
|
862
|
+
if (use_gate) {
|
|
863
|
+
tmp_gate[i] += vec_dot_q_cuda(vgate, &y[kby], kbx_offset + i*stride_row_x + kbx, kqs);
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
ggml_cuda_pdl_lc();
|
|
870
|
+
|
|
871
|
+
// Warp-level reduction only - no shared memory needed
|
|
872
|
+
#pragma unroll
|
|
873
|
+
for (int i = 0; i < c_rows_per_block; ++i) {
|
|
874
|
+
tmp[i] = warp_reduce_sum<warp_size>(tmp[i]);
|
|
875
|
+
if constexpr (has_fusion) {
|
|
876
|
+
if (use_gate) {
|
|
877
|
+
tmp_gate[i] = warp_reduce_sum<warp_size>(tmp_gate[i]);
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
// Write results
|
|
883
|
+
if (threadIdx.x < c_rows_per_block && (c_rows_per_block == 1 || uint32_t(row0 + threadIdx.x) < nrows_x)) {
|
|
884
|
+
float result = tmp[threadIdx.x];
|
|
885
|
+
if constexpr (has_fusion) {
|
|
886
|
+
const uint32_t bias_idx = channel_x*stride_channel_dst + row0 + threadIdx.x;
|
|
887
|
+
|
|
888
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
889
|
+
if (x_scale) {
|
|
890
|
+
result *= x_scale[channel_x];
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
if (x_bias) {
|
|
894
|
+
result += x_bias[bias_idx];
|
|
895
|
+
}
|
|
896
|
+
if (use_gate) {
|
|
897
|
+
float gate_value = tmp_gate[threadIdx.x];
|
|
898
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
899
|
+
if (gate_scale) {
|
|
900
|
+
gate_value *= gate_scale[channel_x];
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
if (gate_bias) {
|
|
904
|
+
gate_value += gate_bias[bias_idx];
|
|
905
|
+
}
|
|
906
|
+
switch (active_glu) {
|
|
907
|
+
case GGML_GLU_OP_SWIGLU:
|
|
908
|
+
result *= ggml_cuda_op_silu_single(gate_value);
|
|
909
|
+
break;
|
|
910
|
+
case GGML_GLU_OP_GEGLU:
|
|
911
|
+
result *= ggml_cuda_op_gelu_single(gate_value);
|
|
912
|
+
break;
|
|
913
|
+
case GGML_GLU_OP_SWIGLU_OAI:
|
|
914
|
+
result = ggml_cuda_op_swiglu_oai_single(gate_value, result);
|
|
915
|
+
break;
|
|
916
|
+
case GGML_GLU_OP_SWIGLU_CLAMP:
|
|
917
|
+
result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit);
|
|
918
|
+
break;
|
|
919
|
+
default:
|
|
920
|
+
result = result * gate_value;
|
|
921
|
+
break;
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = result;
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
if constexpr (!has_fusion) {
|
|
929
|
+
GGML_UNUSED_VARS(use_gate, tmp_gate, vgate, x_bias, gate_bias, active_glu, glu_limit, x_scale, gate_scale);
|
|
930
|
+
} else if constexpr (type != GGML_TYPE_NVFP4) {
|
|
931
|
+
GGML_UNUSED_VARS(x_scale, gate_scale);
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
template<ggml_type type>
|
|
936
|
+
static std::pair<dim3, dim3> calc_launch_params(
|
|
937
|
+
const int ncols_dst, const int nrows_x, const int nchannels_dst, const int nsamples_or_ntokens,
|
|
938
|
+
const int warp_size, const mmvq_parameter_table_id table_id, const bool small_k = false, const bool halve_iters = false) {
|
|
939
|
+
const int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters);
|
|
940
|
+
const int rpb = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps);
|
|
941
|
+
const int64_t nblocks = (nrows_x + rpb - 1) / rpb;
|
|
942
|
+
const dim3 block_nums(nblocks, nchannels_dst, nsamples_or_ntokens);
|
|
943
|
+
const dim3 block_dims(warp_size, nwarps, 1);
|
|
944
|
+
return {block_nums, block_dims};
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
template<ggml_type type, int c_ncols_dst, bool small_k = false, bool halve_iters = false>
|
|
948
|
+
static void mul_mat_vec_q_switch_fusion(
|
|
949
|
+
const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
|
|
950
|
+
const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,
|
|
951
|
+
const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,
|
|
952
|
+
const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,
|
|
953
|
+
const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst,
|
|
954
|
+
const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared,
|
|
955
|
+
const uint32_t ids_stride, cudaStream_t stream) {
|
|
956
|
+
|
|
957
|
+
const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr ||
|
|
958
|
+
fusion.x_scale != nullptr || fusion.gate_scale != nullptr;
|
|
959
|
+
if constexpr (c_ncols_dst == 1) {
|
|
960
|
+
if (has_fusion) {
|
|
961
|
+
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);
|
|
962
|
+
ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, true, small_k, halve_iters>, launch_params,
|
|
963
|
+
vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,
|
|
964
|
+
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
965
|
+
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);
|
|
966
|
+
return;
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
GGML_ASSERT(!has_fusion && "fusion only supported for ncols_dst=1");
|
|
971
|
+
|
|
972
|
+
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);
|
|
973
|
+
ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, false, small_k, halve_iters>, launch_params,
|
|
974
|
+
vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,
|
|
975
|
+
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
976
|
+
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
template <ggml_type type>
|
|
980
|
+
static void mul_mat_vec_q_moe_launch(
|
|
981
|
+
const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
|
|
982
|
+
const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
|
|
983
|
+
const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst,
|
|
984
|
+
const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,
|
|
985
|
+
const uint32_t ncols_dst, const uint32_t ids_stride,
|
|
986
|
+
const int warp_size, const int nchannels_dst, cudaStream_t stream) {
|
|
987
|
+
|
|
988
|
+
constexpr int rows_per_block = 2; // 2 gives best perf based on tuning
|
|
989
|
+
const int64_t nblocks_rows = (nrows_x + rows_per_block - 1) / rows_per_block;
|
|
990
|
+
const dim3 block_nums(nblocks_rows, nchannels_dst);
|
|
991
|
+
const dim3 block_dims(warp_size, ncols_dst);
|
|
992
|
+
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
|
|
993
|
+
|
|
994
|
+
const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr ||
|
|
995
|
+
fusion.x_scale != nullptr || fusion.gate_scale != nullptr;
|
|
996
|
+
|
|
997
|
+
if (has_fusion) {
|
|
998
|
+
ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, true>, launch_params,
|
|
999
|
+
vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,
|
|
1000
|
+
stride_row_x, stride_col_y, stride_col_dst,
|
|
1001
|
+
stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1002
|
+
ncols_dst, ids_stride);
|
|
1003
|
+
} else {
|
|
1004
|
+
ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, false>, launch_params,
|
|
1005
|
+
vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,
|
|
1006
|
+
stride_row_x, stride_col_y, stride_col_dst,
|
|
1007
|
+
stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1008
|
+
ncols_dst, ids_stride);
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
template <ggml_type type>
|
|
1013
|
+
static void mul_mat_vec_q_switch_ncols_dst(
|
|
1014
|
+
const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
|
|
1015
|
+
const int ncols_x, const int nrows_x, const int ncols_dst,
|
|
1016
|
+
const int stride_row_x, const int stride_col_y, const int stride_col_dst,
|
|
1017
|
+
const int nchannels_x, const int nchannels_y, const int nchannels_dst,
|
|
1018
|
+
const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
|
|
1019
|
+
const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,
|
|
1020
|
+
const int ids_stride, cudaStream_t stream) {
|
|
1021
|
+
|
|
1022
|
+
GGML_ASSERT(ncols_x % ggml_blck_size(type) == 0);
|
|
1023
|
+
GGML_ASSERT(ncols_dst <= MMVQ_MAX_BATCH_SIZE);
|
|
1024
|
+
|
|
1025
|
+
const uint3 nchannels_y_fd = ids ? init_fastdiv_values(nchannels_y) : make_uint3(0, 0, 0);
|
|
1026
|
+
const uint3 channel_ratio_fd = ids ? make_uint3(0, 0, 0) : init_fastdiv_values(nchannels_dst / nchannels_x);
|
|
1027
|
+
const uint3 sample_ratio_fd = init_fastdiv_values(nsamples_dst / nsamples_x);
|
|
1028
|
+
|
|
1029
|
+
const int device = ggml_cuda_get_device();
|
|
1030
|
+
const int cc = ggml_cuda_info().devices[device].cc;
|
|
1031
|
+
const int warp_size = ggml_cuda_info().devices[device].warp_size;
|
|
1032
|
+
const mmvq_parameter_table_id table_id = get_device_table_id(cc);
|
|
1033
|
+
|
|
1034
|
+
const bool has_ids = ids != nullptr;
|
|
1035
|
+
|
|
1036
|
+
// How the K loop divides up at the baseline block width, both decisions below use these.
|
|
1037
|
+
constexpr int qk = ggml_cuda_type_traits<type>::qk;
|
|
1038
|
+
constexpr int qi = ggml_cuda_type_traits<type>::qi;
|
|
1039
|
+
constexpr int vdr = get_vdr_mmvq(type);
|
|
1040
|
+
const int blocks_per_row_x = ncols_x / qk;
|
|
1041
|
+
const int blocks_per_iter_1warp = vdr * warp_size / qi;
|
|
1042
|
+
|
|
1043
|
+
const auto should_use_small_k = [&](int c_ncols_dst) {
|
|
1044
|
+
// When K is small, increase rows_per_block to match nwarps so each warp has more work to do
|
|
1045
|
+
// Trigger when the full thread block covers all K blocks in a single loop iteration and few threads remain idle.
|
|
1046
|
+
const int nwarps = calc_nwarps(type, c_ncols_dst, table_id);
|
|
1047
|
+
bool use = nwarps > 1 && blocks_per_row_x < nwarps * blocks_per_iter_1warp;
|
|
1048
|
+
|
|
1049
|
+
constexpr std::array<ggml_type, 2> iq_slow_turing = {
|
|
1050
|
+
GGML_TYPE_IQ3_XXS,
|
|
1051
|
+
GGML_TYPE_IQ3_S,
|
|
1052
|
+
};
|
|
1053
|
+
constexpr std::array<ggml_type, 8> iq_slow_other = {
|
|
1054
|
+
GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS,
|
|
1055
|
+
GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS,
|
|
1056
|
+
};
|
|
1057
|
+
constexpr std::array<ggml_type, 3> slow_pascal = {
|
|
1058
|
+
GGML_TYPE_IQ3_S,
|
|
1059
|
+
GGML_TYPE_Q2_K,
|
|
1060
|
+
GGML_TYPE_Q3_K,
|
|
1061
|
+
};
|
|
1062
|
+
|
|
1063
|
+
const bool is_nvidia_turing_plus = GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_TURING;
|
|
1064
|
+
const bool is_nvidia_pascal_older = GGML_CUDA_CC_IS_NVIDIA(cc) && cc < GGML_CUDA_CC_VOLTA;
|
|
1065
|
+
|
|
1066
|
+
if (is_nvidia_turing_plus) {
|
|
1067
|
+
if (ncols_dst == 1 &&
|
|
1068
|
+
std::find(iq_slow_turing.begin(), iq_slow_turing.end(), type) != iq_slow_turing.end()) {
|
|
1069
|
+
use = false;
|
|
1070
|
+
}
|
|
1071
|
+
} else if ((ncols_dst == 1 && std::find(iq_slow_other.begin(), iq_slow_other.end(), type) != iq_slow_other.end()) ||
|
|
1072
|
+
(is_nvidia_pascal_older && std::find(slow_pascal.begin(), slow_pascal.end(), type) != slow_pascal.end()) ||
|
|
1073
|
+
GGML_CUDA_CC_IS_RDNA(cc)) {
|
|
1074
|
+
use = false;
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
return use;
|
|
1078
|
+
};
|
|
1079
|
+
|
|
1080
|
+
// Whether doubling nwarps pays off on the ncols_dst == 1 path, where K sets the K loop trip count.
|
|
1081
|
+
const auto should_halve_iters = [&] {
|
|
1082
|
+
if (table_id != MMVQ_PARAMETERS_GB10) {
|
|
1083
|
+
return false;
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
// Expert rows are gathered per token, so a wider block adds reduction work without reuse.
|
|
1087
|
+
if (has_ids) {
|
|
1088
|
+
return false;
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
const int blocks_per_iter = calc_nwarps(type, 1, table_id) * blocks_per_iter_1warp;
|
|
1092
|
+
const int iters = (blocks_per_row_x + blocks_per_iter - 1) / blocks_per_iter;
|
|
1093
|
+
const int iters_wide = (blocks_per_row_x + blocks_per_iter * 2 - 1) / (blocks_per_iter * 2);
|
|
1094
|
+
|
|
1095
|
+
// An odd trip count leaves half the wider block idle for its last iteration, that tail is
|
|
1096
|
+
// only affordable once the loop is long enough to dilute it to an eighth of the work (observation).
|
|
1097
|
+
const int idle = iters_wide * 2 - iters;
|
|
1098
|
+
|
|
1099
|
+
return idle * 8 <= iters_wide * 2;
|
|
1100
|
+
};
|
|
1101
|
+
|
|
1102
|
+
if (has_ids && ncols_dst > 1) {
|
|
1103
|
+
// Multi-token MUL_MAT_ID path - dedicated MoE kernel
|
|
1104
|
+
mul_mat_vec_q_moe_launch<type>(
|
|
1105
|
+
vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, nrows_x,
|
|
1106
|
+
stride_row_x, stride_col_y, stride_col_dst,
|
|
1107
|
+
stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1108
|
+
ncols_dst, ids_stride, warp_size, nchannels_dst, stream);
|
|
1109
|
+
return;
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
switch (ncols_dst) {
|
|
1113
|
+
case 1: {
|
|
1114
|
+
// static, else MSVC lambda capture breaks the constexpr uses below
|
|
1115
|
+
static constexpr int c_ncols_dst = 1;
|
|
1116
|
+
|
|
1117
|
+
// Tag types keep the flags compile-time, so __launch_bounds__ matches what is launched.
|
|
1118
|
+
const auto launch = [&](auto small_k_tag, auto halve_iters_tag) {
|
|
1119
|
+
constexpr bool c_small_k = decltype(small_k_tag)::value;
|
|
1120
|
+
// Types the table does not promote would compile a second, identical kernel.
|
|
1121
|
+
constexpr bool c_promoted =
|
|
1122
|
+
calc_nwarps(type, c_ncols_dst, MMVQ_PARAMETERS_GB10, false, true) !=
|
|
1123
|
+
calc_nwarps(type, c_ncols_dst, MMVQ_PARAMETERS_GB10, false, false);
|
|
1124
|
+
|
|
1125
|
+
constexpr bool c_halve_iters = decltype(halve_iters_tag)::value && c_promoted;
|
|
1126
|
+
|
|
1127
|
+
const std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst,
|
|
1128
|
+
nsamples_dst, warp_size, table_id, c_small_k, c_halve_iters);
|
|
1129
|
+
mul_mat_vec_q_switch_fusion<type, c_ncols_dst, c_small_k, c_halve_iters>(
|
|
1130
|
+
vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
|
|
1131
|
+
channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd,
|
|
1132
|
+
stride_sample_x, stride_sample_y, stride_sample_dst, dims.first, dims.second, 0, ids_stride,
|
|
1133
|
+
stream);
|
|
1134
|
+
};
|
|
1135
|
+
|
|
1136
|
+
if (should_use_small_k(c_ncols_dst)) {
|
|
1137
|
+
launch(std::true_type{}, std::false_type{});
|
|
1138
|
+
} else if (should_halve_iters()) {
|
|
1139
|
+
launch(std::false_type{}, std::true_type{});
|
|
1140
|
+
} else {
|
|
1141
|
+
launch(std::false_type{}, std::false_type{});
|
|
1142
|
+
}
|
|
1143
|
+
} break;
|
|
1144
|
+
case 2: {
|
|
1145
|
+
constexpr int c_ncols_dst = 2;
|
|
1146
|
+
std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
|
|
1147
|
+
mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
|
|
1148
|
+
channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1149
|
+
sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
|
|
1150
|
+
dims.first, dims.second, 0, ids_stride, stream);
|
|
1151
|
+
} break;
|
|
1152
|
+
case 3: {
|
|
1153
|
+
constexpr int c_ncols_dst = 3;
|
|
1154
|
+
std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
|
|
1155
|
+
mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
|
|
1156
|
+
channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1157
|
+
sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
|
|
1158
|
+
dims.first, dims.second, 0, ids_stride, stream);
|
|
1159
|
+
} break;
|
|
1160
|
+
case 4: {
|
|
1161
|
+
constexpr int c_ncols_dst = 4;
|
|
1162
|
+
std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
|
|
1163
|
+
mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
|
|
1164
|
+
channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1165
|
+
sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
|
|
1166
|
+
dims.first, dims.second, 0, ids_stride, stream);
|
|
1167
|
+
} break;
|
|
1168
|
+
case 5: {
|
|
1169
|
+
constexpr int c_ncols_dst = 5;
|
|
1170
|
+
std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
|
|
1171
|
+
mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
|
|
1172
|
+
channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1173
|
+
sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
|
|
1174
|
+
dims.first, dims.second, 0, ids_stride, stream);
|
|
1175
|
+
} break;
|
|
1176
|
+
case 6: {
|
|
1177
|
+
constexpr int c_ncols_dst = 6;
|
|
1178
|
+
std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
|
|
1179
|
+
mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
|
|
1180
|
+
channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1181
|
+
sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
|
|
1182
|
+
dims.first, dims.second, 0, ids_stride, stream);
|
|
1183
|
+
} break;
|
|
1184
|
+
case 7: {
|
|
1185
|
+
constexpr int c_ncols_dst = 7;
|
|
1186
|
+
std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
|
|
1187
|
+
mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
|
|
1188
|
+
channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1189
|
+
sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
|
|
1190
|
+
dims.first, dims.second, 0, ids_stride, stream);
|
|
1191
|
+
} break;
|
|
1192
|
+
case 8: {
|
|
1193
|
+
constexpr int c_ncols_dst = 8;
|
|
1194
|
+
std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
|
|
1195
|
+
mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
|
|
1196
|
+
channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1197
|
+
sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,
|
|
1198
|
+
dims.first, dims.second, 0, ids_stride, stream);
|
|
1199
|
+
} break;
|
|
1200
|
+
default:
|
|
1201
|
+
GGML_ABORT("fatal error");
|
|
1202
|
+
break;
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
static void mul_mat_vec_q_switch_type(
|
|
1206
|
+
const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
|
|
1207
|
+
const int ncols_x, const int nrows_x, const int ncols_dst,
|
|
1208
|
+
const int stride_row_x, const int stride_col_y, const int stride_col_dst,
|
|
1209
|
+
const int nchannels_x, const int nchannels_y, const int nchannels_dst,
|
|
1210
|
+
const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
|
|
1211
|
+
const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,
|
|
1212
|
+
const int ids_stride, cudaStream_t stream) {
|
|
1213
|
+
switch (type_x) {
|
|
1214
|
+
case GGML_TYPE_Q1_0:
|
|
1215
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q1_0>
|
|
1216
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1217
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1218
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1219
|
+
break;
|
|
1220
|
+
case GGML_TYPE_Q2_0:
|
|
1221
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q2_0>
|
|
1222
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1223
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1224
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1225
|
+
break;
|
|
1226
|
+
case GGML_TYPE_Q4_0:
|
|
1227
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q4_0>
|
|
1228
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1229
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1230
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1231
|
+
break;
|
|
1232
|
+
case GGML_TYPE_Q4_1:
|
|
1233
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q4_1>
|
|
1234
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1235
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1236
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1237
|
+
break;
|
|
1238
|
+
case GGML_TYPE_Q5_0:
|
|
1239
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q5_0>
|
|
1240
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1241
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1242
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1243
|
+
break;
|
|
1244
|
+
case GGML_TYPE_Q5_1:
|
|
1245
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q5_1>
|
|
1246
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1247
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1248
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1249
|
+
break;
|
|
1250
|
+
case GGML_TYPE_Q8_0:
|
|
1251
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q8_0>
|
|
1252
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1253
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1254
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1255
|
+
break;
|
|
1256
|
+
case GGML_TYPE_MXFP4:
|
|
1257
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_MXFP4>
|
|
1258
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1259
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1260
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1261
|
+
break;
|
|
1262
|
+
case GGML_TYPE_NVFP4:
|
|
1263
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_NVFP4>
|
|
1264
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1265
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1266
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1267
|
+
break;
|
|
1268
|
+
case GGML_TYPE_Q2_K:
|
|
1269
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q2_K>
|
|
1270
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1271
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1272
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1273
|
+
break;
|
|
1274
|
+
case GGML_TYPE_Q3_K:
|
|
1275
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q3_K>
|
|
1276
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1277
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1278
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1279
|
+
break;
|
|
1280
|
+
case GGML_TYPE_Q4_K:
|
|
1281
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q4_K>
|
|
1282
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1283
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1284
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1285
|
+
break;
|
|
1286
|
+
case GGML_TYPE_Q5_K:
|
|
1287
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q5_K>
|
|
1288
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1289
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1290
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1291
|
+
break;
|
|
1292
|
+
case GGML_TYPE_Q6_K:
|
|
1293
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q6_K>
|
|
1294
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1295
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1296
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1297
|
+
break;
|
|
1298
|
+
case GGML_TYPE_IQ2_XXS:
|
|
1299
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ2_XXS>
|
|
1300
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1301
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1302
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1303
|
+
break;
|
|
1304
|
+
case GGML_TYPE_IQ2_XS:
|
|
1305
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ2_XS>
|
|
1306
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1307
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1308
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1309
|
+
break;
|
|
1310
|
+
case GGML_TYPE_IQ2_S:
|
|
1311
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ2_S>
|
|
1312
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1313
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1314
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1315
|
+
break;
|
|
1316
|
+
case GGML_TYPE_IQ3_XXS:
|
|
1317
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ3_XXS>
|
|
1318
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1319
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1320
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1321
|
+
break;
|
|
1322
|
+
case GGML_TYPE_IQ1_S:
|
|
1323
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ1_S>
|
|
1324
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1325
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1326
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1327
|
+
break;
|
|
1328
|
+
case GGML_TYPE_IQ1_M:
|
|
1329
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ1_M>
|
|
1330
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1331
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1332
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1333
|
+
break;
|
|
1334
|
+
case GGML_TYPE_IQ4_NL:
|
|
1335
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ4_NL>
|
|
1336
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1337
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1338
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1339
|
+
break;
|
|
1340
|
+
case GGML_TYPE_IQ4_XS:
|
|
1341
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ4_XS>
|
|
1342
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1343
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1344
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1345
|
+
break;
|
|
1346
|
+
case GGML_TYPE_IQ3_S:
|
|
1347
|
+
mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ3_S>
|
|
1348
|
+
(vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,
|
|
1349
|
+
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
|
1350
|
+
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
|
|
1351
|
+
break;
|
|
1352
|
+
default:
|
|
1353
|
+
GGML_ABORT("fatal error");
|
|
1354
|
+
break;
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
void ggml_cuda_mul_mat_vec_q(
|
|
1359
|
+
ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
|
|
1360
|
+
const ggml_cuda_mm_fusion_args_host * fusion) {
|
|
1361
|
+
GGML_ASSERT( src1->type == GGML_TYPE_F32);
|
|
1362
|
+
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
|
1363
|
+
GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.
|
|
1364
|
+
|
|
1365
|
+
GGML_TENSOR_BINARY_OP_LOCALS;
|
|
1366
|
+
|
|
1367
|
+
cudaStream_t stream = ctx.stream();
|
|
1368
|
+
|
|
1369
|
+
const size_t ts_src0 = ggml_type_size(src0->type);
|
|
1370
|
+
const size_t ts_src1 = ggml_type_size(src1->type);
|
|
1371
|
+
const size_t ts_dst = ggml_type_size(dst->type);
|
|
1372
|
+
|
|
1373
|
+
GGML_ASSERT( nb00 == ts_src0);
|
|
1374
|
+
GGML_ASSERT( nb10 == ts_src1);
|
|
1375
|
+
GGML_ASSERT( nb0 == ts_dst);
|
|
1376
|
+
GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));
|
|
1377
|
+
|
|
1378
|
+
GGML_ASSERT(!ids || ne12 <= MMVQ_MAX_BATCH_SIZE);
|
|
1379
|
+
|
|
1380
|
+
const float * src1_d = (const float *) src1->data;
|
|
1381
|
+
const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr;
|
|
1382
|
+
float * dst_d = (float *) dst->data;
|
|
1383
|
+
|
|
1384
|
+
ggml_cuda_mm_fusion_args_device fusion_local{};
|
|
1385
|
+
|
|
1386
|
+
if (fusion) {
|
|
1387
|
+
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
|
|
1388
|
+
GGML_ASSERT( !ids || dst->ne[2] <= get_mmvq_mmid_max_batch(src0->type, cc));
|
|
1389
|
+
GGML_ASSERT( ids || dst->ne[1] == 1);
|
|
1390
|
+
// Scale fusion is only allowed for NVFP4 currently as the cost of checking this at run-time in the prologue is
|
|
1391
|
+
// non-negligible for some models such as gpt-oss-20b
|
|
1392
|
+
GGML_ASSERT((fusion->x_scale == nullptr && fusion->gate_scale == nullptr) || src0->type == GGML_TYPE_NVFP4);
|
|
1393
|
+
|
|
1394
|
+
if (fusion->x_bias) {
|
|
1395
|
+
GGML_ASSERT(fusion->x_bias->type == GGML_TYPE_F32);
|
|
1396
|
+
GGML_ASSERT(fusion->x_bias->ne[0] == dst->ne[0]);
|
|
1397
|
+
GGML_ASSERT(!ids || fusion->x_bias->ne[1] == src0->ne[2]);
|
|
1398
|
+
fusion_local.x_bias = fusion->x_bias->data;
|
|
1399
|
+
}
|
|
1400
|
+
if (fusion->gate) {
|
|
1401
|
+
GGML_ASSERT(fusion->gate->type == src0->type && ggml_are_same_stride(fusion->gate, src0));
|
|
1402
|
+
fusion_local.gate = fusion->gate->data;
|
|
1403
|
+
}
|
|
1404
|
+
if (fusion->gate_bias) {
|
|
1405
|
+
GGML_ASSERT(fusion->gate_bias->type == GGML_TYPE_F32);
|
|
1406
|
+
GGML_ASSERT(fusion->gate_bias->ne[0] == dst->ne[0]);
|
|
1407
|
+
GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]);
|
|
1408
|
+
fusion_local.gate_bias = fusion->gate_bias->data;
|
|
1409
|
+
}
|
|
1410
|
+
if (fusion->x_scale) {
|
|
1411
|
+
GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32);
|
|
1412
|
+
GGML_ASSERT(ggml_is_contiguous(fusion->x_scale));
|
|
1413
|
+
GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1));
|
|
1414
|
+
fusion_local.x_scale = fusion->x_scale->data;
|
|
1415
|
+
}
|
|
1416
|
+
if (fusion->gate_scale) {
|
|
1417
|
+
GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32);
|
|
1418
|
+
GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale));
|
|
1419
|
+
GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1));
|
|
1420
|
+
fusion_local.gate_scale = fusion->gate_scale->data;
|
|
1421
|
+
}
|
|
1422
|
+
fusion_local.glu_op = fusion->glu_op;
|
|
1423
|
+
fusion_local.glu_limit = fusion->glu_limit;
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
// If src0 is a temporary compute buffer, clear any potential padding.
|
|
1427
|
+
if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
|
|
1428
|
+
const size_t size_data = ggml_nbytes(src0);
|
|
1429
|
+
const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0);
|
|
1430
|
+
if (size_alloc > size_data) {
|
|
1431
|
+
GGML_ASSERT(ggml_is_contiguously_allocated(src0));
|
|
1432
|
+
GGML_ASSERT(!src0->view_src);
|
|
1433
|
+
CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream));
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);
|
|
1438
|
+
ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);
|
|
1439
|
+
{
|
|
1440
|
+
const int64_t s11 = src1->nb[1] / ts_src1;
|
|
1441
|
+
const int64_t s12 = src1->nb[2] / ts_src1;
|
|
1442
|
+
const int64_t s13 = src1->nb[3] / ts_src1;
|
|
1443
|
+
quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
const int64_t s01 = src0->nb[1] / ts_src0;
|
|
1447
|
+
const int64_t s11 = ne10_padded / QK8_1;
|
|
1448
|
+
const int64_t s1 = dst->nb[1] / ts_dst;
|
|
1449
|
+
const int64_t s02 = src0->nb[2] / ts_src0;
|
|
1450
|
+
const int64_t s2 = dst->nb[2] / ts_dst;
|
|
1451
|
+
const int64_t s03 = src0->nb[3] / ts_src0;
|
|
1452
|
+
const int64_t s3 = dst->nb[3] / ts_dst;
|
|
1453
|
+
|
|
1454
|
+
const int64_t s12 = ne11*s11;
|
|
1455
|
+
const int64_t s13 = ne12*s12;
|
|
1456
|
+
|
|
1457
|
+
// For MUL_MAT_ID the memory layout is different than for MUL_MAT:
|
|
1458
|
+
const int64_t ncols_dst = ids ? ne2 : ne1;
|
|
1459
|
+
const int64_t nchannels_y = ids ? ne11 : ne12;
|
|
1460
|
+
const int64_t nchannels_dst = ids ? ne1 : ne2;
|
|
1461
|
+
const int64_t stride_col_dst = ids ? s2 : s1;
|
|
1462
|
+
const int64_t stride_col_y = ids ? s12 : s11;
|
|
1463
|
+
const int64_t stride_channel_dst = ids ? s1 : s2;
|
|
1464
|
+
const int64_t stride_channel_y = ids ? s11 : s12;
|
|
1465
|
+
|
|
1466
|
+
const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;
|
|
1467
|
+
|
|
1468
|
+
mul_mat_vec_q_switch_type(
|
|
1469
|
+
src0->data, src0->type, src1_q8_1.get(), ids_d, fusion_local, dst_d, ne00,
|
|
1470
|
+
ne01, ncols_dst, s01, stride_col_y, stride_col_dst,
|
|
1471
|
+
ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
|
|
1472
|
+
ne03, ne3, s03, s13, s3, ids_stride, stream);
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
void ggml_cuda_op_mul_mat_vec_q(
|
|
1476
|
+
ggml_backend_cuda_context & ctx,
|
|
1477
|
+
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
|
|
1478
|
+
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
|
|
1479
|
+
const int64_t src1_padded_row_size, cudaStream_t stream) {
|
|
1480
|
+
|
|
1481
|
+
const int64_t ne00 = src0->ne[0];
|
|
1482
|
+
const int64_t row_diff = row_high - row_low;
|
|
1483
|
+
|
|
1484
|
+
const int64_t ne10 = src1->ne[0];
|
|
1485
|
+
GGML_ASSERT(ne10 % QK8_1 == 0);
|
|
1486
|
+
|
|
1487
|
+
const int64_t ne0 = dst->ne[0];
|
|
1488
|
+
|
|
1489
|
+
int id = ggml_cuda_get_device();
|
|
1490
|
+
|
|
1491
|
+
// the main device has a larger memory buffer to hold the results from all GPUs
|
|
1492
|
+
// nrows_dst == nrows of the matrix that the kernel writes into
|
|
1493
|
+
const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff;
|
|
1494
|
+
|
|
1495
|
+
const int stride_row_x = ne00 / ggml_blck_size(src0->type);
|
|
1496
|
+
const int stride_col_y = src1_padded_row_size / QK8_1;
|
|
1497
|
+
|
|
1498
|
+
ggml_cuda_mm_fusion_args_device fusion_local{};
|
|
1499
|
+
mul_mat_vec_q_switch_type(
|
|
1500
|
+
src0_dd_i, src0->type, src1_ddq_i, nullptr, fusion_local, dst_dd_i, ne00, row_diff, src1_ncols, stride_row_x, stride_col_y, nrows_dst,
|
|
1501
|
+
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, stream);
|
|
1502
|
+
|
|
1503
|
+
GGML_UNUSED_VARS(src1, dst, src1_ddf_i, src1_ncols, src1_padded_row_size);
|
|
1504
|
+
}
|