elelem-ggml 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/exe/elelem-ggml +8 -0
- data/ext/elelem/ggml/elelem.c +466 -0
- data/ext/elelem/ggml/extconf.rb +52 -0
- data/lib/elelem/ggml/chat_ml.rb +23 -0
- data/lib/elelem/ggml/gguf.rb +82 -0
- data/lib/elelem/ggml/provider.rb +63 -0
- data/lib/elelem/ggml/tokenizer.rb +95 -0
- data/lib/elelem/ggml/version.rb +7 -0
- data/lib/elelem/ggml.rb +37 -0
- data/vendor/ggml/AUTHORS +335 -0
- data/vendor/ggml/CMakeLists.txt +499 -0
- data/vendor/ggml/CONTRIBUTING.md +3 -0
- data/vendor/ggml/LICENSE +21 -0
- data/vendor/ggml/README.md +49 -0
- data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
- data/vendor/ggml/cmake/GitVars.cmake +22 -0
- data/vendor/ggml/cmake/common.cmake +50 -0
- data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
- data/vendor/ggml/ggml.pc.in +10 -0
- data/vendor/ggml/include/ggml-alloc.h +86 -0
- data/vendor/ggml/include/ggml-backend.h +437 -0
- data/vendor/ggml/include/ggml-blas.h +25 -0
- data/vendor/ggml/include/ggml-cann.h +123 -0
- data/vendor/ggml/include/ggml-cpp.h +39 -0
- data/vendor/ggml/include/ggml-cpu.h +152 -0
- data/vendor/ggml/include/ggml-cuda.h +47 -0
- data/vendor/ggml/include/ggml-et.h +28 -0
- data/vendor/ggml/include/ggml-hexagon.h +19 -0
- data/vendor/ggml/include/ggml-metal.h +61 -0
- data/vendor/ggml/include/ggml-opencl.h +26 -0
- data/vendor/ggml/include/ggml-openvino.h +37 -0
- data/vendor/ggml/include/ggml-opt.h +256 -0
- data/vendor/ggml/include/ggml-rpc.h +35 -0
- data/vendor/ggml/include/ggml-sycl.h +57 -0
- data/vendor/ggml/include/ggml-virtgpu.h +14 -0
- data/vendor/ggml/include/ggml-vulkan.h +29 -0
- data/vendor/ggml/include/ggml-webgpu.h +19 -0
- data/vendor/ggml/include/ggml-zdnn.h +17 -0
- data/vendor/ggml/include/ggml-zendnn.h +22 -0
- data/vendor/ggml/include/ggml.h +2963 -0
- data/vendor/ggml/include/gguf.h +211 -0
- data/vendor/ggml/requirements.txt +12 -0
- data/vendor/ggml/scripts/gen-authors.sh +9 -0
- data/vendor/ggml/scripts/make-release-checks.sh +96 -0
- data/vendor/ggml/scripts/make-release-desc.sh +69 -0
- data/vendor/ggml/scripts/make-release-summary.txt +33 -0
- data/vendor/ggml/scripts/release.sh +195 -0
- data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
- data/vendor/ggml/scripts/sync-llama.last +1 -0
- data/vendor/ggml/scripts/sync-llama.sh +21 -0
- data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
- data/vendor/ggml/scripts/sync-whisper.last +1 -0
- data/vendor/ggml/scripts/sync-whisper.sh +17 -0
- data/vendor/ggml/src/CMakeLists.txt +634 -0
- data/vendor/ggml/src/ggml-alloc.c +1249 -0
- data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
- data/vendor/ggml/src/ggml-backend-dl.h +45 -0
- data/vendor/ggml/src/ggml-backend-impl.h +291 -0
- data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
- data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
- data/vendor/ggml/src/ggml-backend.cpp +2517 -0
- data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
- data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
- data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
- data/vendor/ggml/src/ggml-cann/common.h +651 -0
- data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
- data/vendor/ggml/src/ggml-common.h +1911 -0
- data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
- data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
- data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
- data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
- data/vendor/ggml/src/ggml-cpu/common.h +95 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
- data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
- data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
- data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
- data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
- data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
- data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
- data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
- data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
- data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
- data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
- data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
- data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
- data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
- data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
- data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
- data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
- data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
- data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
- data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
- data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
- data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
- data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
- data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
- data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
- data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
- data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
- data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
- data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
- data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
- data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
- data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
- data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
- data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
- data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
- data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
- data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
- data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
- data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
- data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
- data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
- data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
- data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
- data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
- data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
- data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
- data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
- data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
- data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
- data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
- data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
- data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
- data/vendor/ggml/src/ggml-feats.h +166 -0
- data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
- data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
- data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
- data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
- data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
- data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
- data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
- data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
- data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
- data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
- data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
- data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
- data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
- data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
- data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
- data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
- data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
- data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
- data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
- data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
- data/vendor/ggml/src/ggml-impl.h +783 -0
- data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
- data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
- data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
- data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
- data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
- data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
- data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
- data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
- data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
- data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
- data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
- data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
- data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
- data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
- data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
- data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
- data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
- data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
- data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
- data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
- data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
- data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
- data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
- data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
- data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
- data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
- data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
- data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
- data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
- data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
- data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
- data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
- data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
- data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
- data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
- data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
- data/vendor/ggml/src/ggml-opt.cpp +1094 -0
- data/vendor/ggml/src/ggml-quants.c +5667 -0
- data/vendor/ggml/src/ggml-quants.h +115 -0
- data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
- data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
- data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
- data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
- data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
- data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
- data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
- data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
- data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
- data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
- data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
- data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
- data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
- data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
- data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
- data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
- data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
- data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
- data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
- data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
- data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
- data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
- data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
- data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
- data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
- data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
- data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
- data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
- data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
- data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
- data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
- data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
- data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
- data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
- data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
- data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
- data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
- data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
- data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
- data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
- data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
- data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
- data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
- data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
- data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
- data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
- data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
- data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
- data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
- data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/vendor/ggml/src/ggml-threading.cpp +12 -0
- data/vendor/ggml/src/ggml-threading.h +14 -0
- data/vendor/ggml/src/ggml-version.h.in +4 -0
- data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
- data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
- data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
- data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
- data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
- data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
- data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
- data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
- data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
- data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
- data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
- data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
- data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
- data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
- data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
- data/vendor/ggml/src/ggml.c +8088 -0
- data/vendor/ggml/src/ggml.cpp +26 -0
- data/vendor/ggml/src/gguf.cpp +1706 -0
- metadata +1459 -0
|
@@ -0,0 +1,1598 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include "common.cuh"
|
|
4
|
+
|
|
5
|
+
#include <climits>
|
|
6
|
+
#include <cstdint>
|
|
7
|
+
|
|
8
|
+
#define MMQ_DP4A_MAX_BATCH_SIZE 64 // Max. batch size to use for dp4a MMQ kernels when FP16 tensor cores are available.
|
|
9
|
+
#define MMQ_ITER_K 256
|
|
10
|
+
#define MMQ_ITER_K_FP4 512
|
|
11
|
+
#define MMQ_NWARPS 8
|
|
12
|
+
|
|
13
|
+
typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);
|
|
14
|
+
typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);
|
|
15
|
+
typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,
|
|
16
|
+
float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max);
|
|
17
|
+
|
|
18
|
+
enum mmq_q8_1_ds_layout {
|
|
19
|
+
MMQ_Q8_1_DS_LAYOUT_D4,
|
|
20
|
+
MMQ_Q8_1_DS_LAYOUT_DS4,
|
|
21
|
+
MMQ_Q8_1_DS_LAYOUT_D2S6,
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
static constexpr int QK8_1_MMQ = 4*QK8_1;
|
|
25
|
+
static constexpr int QK_FP4_MMQ = 2*QK8_1_MMQ;
|
|
26
|
+
|
|
27
|
+
struct block_q8_1_mmq {
|
|
28
|
+
// The y float data is converted to a data layout that can simply be copied to shared memory as a contiguous block.
|
|
29
|
+
// The y float data is first grouped as blocks of 128 values.
|
|
30
|
+
// These blocks are then treated as individual data values and transposed.
|
|
31
|
+
//
|
|
32
|
+
// To avoid shared memory bank conflicts each block is padded with 16 bytes.
|
|
33
|
+
// This padding is also used to store block scales/partial sums.
|
|
34
|
+
// The scales multiplied with the quantized data are equal to the unquantized values.
|
|
35
|
+
// The partial sums are obtained by summing up a subgroup of the contained values (prior to quantization)
|
|
36
|
+
// and are only needed for performance reasons.
|
|
37
|
+
//
|
|
38
|
+
// The exact data stored depends on the x data type.
|
|
39
|
+
union {
|
|
40
|
+
float d4[4]; // 1 32 bit scale per 32 values, stored as d0,d1,d2,d3
|
|
41
|
+
half2 ds4[4]; // 1 16 bit scale + 1 16 bit partial sum per 32 values, stored as d0,s0,d1,s1,d2,s2,d3,s3
|
|
42
|
+
half d2s6[8]; // 1 16 bit scale per 64 values + 1 16 bit partial sum per 16 values for the first 96 values,
|
|
43
|
+
// stored as d0,d1,s1,s2,s3,s4,s5
|
|
44
|
+
};
|
|
45
|
+
int8_t qs[QK8_1_MMQ];
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
// this struct is used for fp4 data types (currently only used for Blackwell)
|
|
49
|
+
// mxfp4 has block size 32, each int32 of d4 contains 2 e8m0 scales in the lower 16 bits
|
|
50
|
+
// nvfp4 has block size 16, each int32 of d4 contains 4 ue4m3 scales
|
|
51
|
+
struct block_fp4_mmq {
|
|
52
|
+
uint32_t d4[4];
|
|
53
|
+
int8_t qs[QK_FP4_MMQ / 2];
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
static_assert(sizeof(block_q8_1_mmq) == QK8_1_MMQ + 4*sizeof(half2), "Unexpected block_q8_1_mmq size");
|
|
57
|
+
static_assert(sizeof(block_q8_1_mmq) == 4*sizeof(block_q8_1), "Unexpected block_q8_1_mmq size");
|
|
58
|
+
static_assert(sizeof(block_fp4_mmq) == sizeof(block_q8_1_mmq), "Unexpected block_fp4_mmq size");
|
|
59
|
+
|
|
60
|
+
static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) {
|
|
61
|
+
switch (type_x) {
|
|
62
|
+
case GGML_TYPE_Q1_0:
|
|
63
|
+
case GGML_TYPE_Q2_0:
|
|
64
|
+
return MMQ_Q8_1_DS_LAYOUT_D4;
|
|
65
|
+
case GGML_TYPE_Q4_0:
|
|
66
|
+
case GGML_TYPE_Q4_1:
|
|
67
|
+
return MMQ_Q8_1_DS_LAYOUT_DS4;
|
|
68
|
+
case GGML_TYPE_Q5_0:
|
|
69
|
+
return MMQ_Q8_1_DS_LAYOUT_D4;
|
|
70
|
+
case GGML_TYPE_Q5_1:
|
|
71
|
+
return MMQ_Q8_1_DS_LAYOUT_DS4;
|
|
72
|
+
case GGML_TYPE_Q8_0:
|
|
73
|
+
return MMQ_Q8_1_DS_LAYOUT_D4;
|
|
74
|
+
case GGML_TYPE_MXFP4:
|
|
75
|
+
return MMQ_Q8_1_DS_LAYOUT_D4;
|
|
76
|
+
case GGML_TYPE_NVFP4:
|
|
77
|
+
return MMQ_Q8_1_DS_LAYOUT_D4;
|
|
78
|
+
case GGML_TYPE_Q2_K:
|
|
79
|
+
return MMQ_Q8_1_DS_LAYOUT_D2S6;
|
|
80
|
+
case GGML_TYPE_Q3_K:
|
|
81
|
+
return MMQ_Q8_1_DS_LAYOUT_D4;
|
|
82
|
+
case GGML_TYPE_Q4_K:
|
|
83
|
+
case GGML_TYPE_Q5_K:
|
|
84
|
+
return MMQ_Q8_1_DS_LAYOUT_DS4;
|
|
85
|
+
case GGML_TYPE_Q6_K:
|
|
86
|
+
case GGML_TYPE_IQ2_XXS:
|
|
87
|
+
case GGML_TYPE_IQ2_XS:
|
|
88
|
+
case GGML_TYPE_IQ2_S:
|
|
89
|
+
case GGML_TYPE_IQ3_XXS:
|
|
90
|
+
case GGML_TYPE_IQ3_S:
|
|
91
|
+
return MMQ_Q8_1_DS_LAYOUT_D4;
|
|
92
|
+
case GGML_TYPE_IQ1_S:
|
|
93
|
+
return MMQ_Q8_1_DS_LAYOUT_DS4;
|
|
94
|
+
case GGML_TYPE_IQ4_XS:
|
|
95
|
+
case GGML_TYPE_IQ4_NL:
|
|
96
|
+
return MMQ_Q8_1_DS_LAYOUT_D4;
|
|
97
|
+
default:
|
|
98
|
+
GGML_ABORT("fatal error");
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
struct tile_x_sizes {
|
|
104
|
+
int qs;
|
|
105
|
+
int dm;
|
|
106
|
+
int sc;
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
// Decouple shared memory tile sizes from WARP_SIZE to allow for different warp sizes.
|
|
110
|
+
// The K dimension of the tiles has either,
|
|
111
|
+
// 1*MMQ_TILE_NE_K==32 (always for TILE_Y_K) or 2*MMQ_TILE_NE_K==64 (typically for TILE_X_K),
|
|
112
|
+
// 32 bit elements for the quantized data (does not include scales).
|
|
113
|
+
// In other words, the size of the quantized data in the K dimension is a multiple of MMQ_TILE_NE_K.
|
|
114
|
+
// The final tile size in K direction is padded to avoid shared memory bank conflicts,
|
|
115
|
+
// in terms of 32 bit elements that means K % 2 == 1 for dp4a or K % 8 == 4 for mma.
|
|
116
|
+
#define MMQ_TILE_NE_K 32
|
|
117
|
+
|
|
118
|
+
// block_q8_1_mmq has (128 8-bit ints == 32 32-bit ints + 4 32-bit scales)
|
|
119
|
+
#define MMQ_TILE_Y_K (MMQ_TILE_NE_K + MMQ_TILE_NE_K / QI8_1)
|
|
120
|
+
#define MMQ_TILE_Y_FP4_K MMQ_TILE_Y_K
|
|
121
|
+
|
|
122
|
+
enum ggml_cuda_mmq_sram_layout {
|
|
123
|
+
GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0,
|
|
124
|
+
GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1,
|
|
125
|
+
GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K,
|
|
126
|
+
GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K,
|
|
127
|
+
GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K,
|
|
128
|
+
GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, // MXFP4 and NVFP4 on Blackwell.
|
|
129
|
+
GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, // Generic NVFP4
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
static constexpr __host__ __device__ int ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_sram_layout sram_layout) {
|
|
133
|
+
switch (sram_layout) {
|
|
134
|
+
case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0:
|
|
135
|
+
return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4;
|
|
136
|
+
case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1:
|
|
137
|
+
return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_1 + 4;
|
|
138
|
+
case GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K:
|
|
139
|
+
return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K + 4;
|
|
140
|
+
case GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K:
|
|
141
|
+
return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4;
|
|
142
|
+
case GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K:
|
|
143
|
+
return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/QI6_K + MMQ_TILE_NE_K/8 + 7;
|
|
144
|
+
case GGML_CUDA_MMQ_SRAM_LAYOUT_FP4:
|
|
145
|
+
return 2*MMQ_TILE_NE_K + 8 + 4;
|
|
146
|
+
case GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4:
|
|
147
|
+
return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4;
|
|
148
|
+
default:
|
|
149
|
+
return -1;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0) % 8 == 4, "Wrong padding.");
|
|
154
|
+
static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1) % 8 == 4, "Wrong padding.");
|
|
155
|
+
static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K) % 8 == 4, "Wrong padding.");
|
|
156
|
+
static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K) % 8 == 4, "Wrong padding.");
|
|
157
|
+
static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K) % 8 == 4, "Wrong padding.");
|
|
158
|
+
static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) % 8 == 4, "Wrong padding.");
|
|
159
|
+
static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4) % 8 == 4, "Wrong padding.");
|
|
160
|
+
|
|
161
|
+
static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) == ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1), "Wrong tile size for MXFP4");
|
|
162
|
+
|
|
163
|
+
// Config options for the MMQ kernel.
|
|
164
|
+
// Should not affect results, only speed/register pressure/shared memory use.
|
|
165
|
+
struct ggml_cuda_mmq_config {
|
|
166
|
+
ggml_type type; // src0->type
|
|
167
|
+
int nthreads; // Number of threads per CUDA block.
|
|
168
|
+
int occupancy; // Targeted occupancy for the MMA kernel.
|
|
169
|
+
int I; // SRAM tile width in src0->ne[1]/dst->ne[0] direction.
|
|
170
|
+
int J; // SRAM tile width in src1->ne[1]/dst->ne[1] direction.
|
|
171
|
+
ggml_cuda_mmq_sram_layout sram_layout; // SRAM tile length in src0->ne[0]/src1->ne[0] direction (physical 32 bit elements).
|
|
172
|
+
int K_vram; // VRAM tile length in src0->ne[0]/src1->ne[0] direction (logical elements).
|
|
173
|
+
bool stream_k; // Whether or not to use stream-k decomposition.
|
|
174
|
+
bool fallback; // Whether a fallback for out-of-bounds check in src0->ne[1] direction is needed.
|
|
175
|
+
|
|
176
|
+
constexpr __host__ __device__ ggml_cuda_mmq_config(
|
|
177
|
+
ggml_type type, int nthreads, int occupancy, int I, int J, ggml_cuda_mmq_sram_layout sram_layout, int K_vram, bool stream_k, bool fallback) :
|
|
178
|
+
type(type), nthreads(nthreads), occupancy(occupancy), I(I), J(J), sram_layout(sram_layout), K_vram(K_vram), stream_k(stream_k), fallback(fallback) {}
|
|
179
|
+
|
|
180
|
+
constexpr __device__ int rows_per_warp() const {
|
|
181
|
+
#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
|
|
182
|
+
return 16;
|
|
183
|
+
#else
|
|
184
|
+
return J >= 48 && J % 16 == 0 ? 32 : 16;
|
|
185
|
+
#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// TODO transition all combinations of GPUs and quantizations to the MMA data layout.
|
|
189
|
+
__host__ int use_mma_data_layout(const int cc) const {
|
|
190
|
+
if (amd_mfma_available(cc) || amd_wmma_available(cc) || turing_mma_available(cc)) {
|
|
191
|
+
return true;
|
|
192
|
+
}
|
|
193
|
+
return false;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
constexpr __device__ bool use_mma_data_layout() const {
|
|
197
|
+
#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)
|
|
198
|
+
return true;
|
|
199
|
+
#else
|
|
200
|
+
return false;
|
|
201
|
+
#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
#define CASE(type_, nthreads_, occupancy_, I_, J_, sram_layout_, K_vram_, stream_k_, fallback_) \
|
|
207
|
+
if (type == (type_) && J == (J_) && fallback == (fallback_)) { \
|
|
208
|
+
static_assert((nthreads_) % 32 == 0 && (nthreads_) <= 512, "bad nthreads"); \
|
|
209
|
+
static_assert( (occupancy_) <= 8, "bad occupancy"); \
|
|
210
|
+
static_assert((I_) % 32 == 0, "bad I"); \
|
|
211
|
+
static_assert((J_) % 8 == 0, "bad J"); \
|
|
212
|
+
static_assert((K_vram_) % 256 == 0, "bad K_vram"); \
|
|
213
|
+
return ggml_cuda_mmq_config((type_), (nthreads_), (occupancy_), (I_), (J_), (sram_layout_), (K_vram_), (stream_k_), (fallback_)); \
|
|
214
|
+
} \
|
|
215
|
+
|
|
216
|
+
#include "mmq-config-pascal-older.cuh"
|
|
217
|
+
#include "mmq-config-pascal-dp4a.cuh"
|
|
218
|
+
#include "mmq-config-ampere.cuh"
|
|
219
|
+
#include "mmq-config-blackwell.cuh"
|
|
220
|
+
|
|
221
|
+
#include "mmq-config-cdna.cuh"
|
|
222
|
+
#include "mmq-config-rdna2.cuh"
|
|
223
|
+
#include "mmq-config-rdna3.cuh"
|
|
224
|
+
#include "mmq-config-rdna3-5.cuh"
|
|
225
|
+
#include "mmq-config-rdna4.cuh"
|
|
226
|
+
|
|
227
|
+
#undef CASE
|
|
228
|
+
|
|
229
|
+
static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc) {
|
|
230
|
+
if (GGML_CUDA_CC_IS_AMD(cc)) {
|
|
231
|
+
if (GGML_CUDA_CC_IS_CDNA(cc)) {
|
|
232
|
+
return ggml_cuda_mmq_get_config_cdna(type, J, fallback);
|
|
233
|
+
}
|
|
234
|
+
if (GGML_CUDA_CC_IS_RDNA4(cc)) {
|
|
235
|
+
return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);
|
|
236
|
+
}
|
|
237
|
+
if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {
|
|
238
|
+
return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
|
|
239
|
+
}
|
|
240
|
+
if (GGML_CUDA_CC_IS_RDNA3(cc)) { // covers RDNA 3.0
|
|
241
|
+
return ggml_cuda_mmq_get_config_rdna3(type, J, fallback);
|
|
242
|
+
}
|
|
243
|
+
return ggml_cuda_mmq_get_config_rdna2(type, J, fallback);
|
|
244
|
+
}
|
|
245
|
+
if (blackwell_mma_available(cc)) {
|
|
246
|
+
return ggml_cuda_mmq_get_config_blackwell(type, J, fallback);
|
|
247
|
+
}
|
|
248
|
+
if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) {
|
|
249
|
+
return ggml_cuda_mmq_get_config_ampere(type, J, fallback);
|
|
250
|
+
}
|
|
251
|
+
if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_DP4A) {
|
|
252
|
+
return ggml_cuda_mmq_get_config_pascal_dp4a(type, J, fallback);
|
|
253
|
+
}
|
|
254
|
+
return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback) {
|
|
258
|
+
#ifdef GGML_USE_HIP
|
|
259
|
+
#ifdef CDNA
|
|
260
|
+
return ggml_cuda_mmq_get_config_cdna(type, J, fallback);
|
|
261
|
+
#elif defined(RDNA4)
|
|
262
|
+
return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);
|
|
263
|
+
#elif defined(RDNA3_5)
|
|
264
|
+
return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);
|
|
265
|
+
#elif defined(RDNA3)
|
|
266
|
+
return ggml_cuda_mmq_get_config_rdna3(type, J, fallback);
|
|
267
|
+
#else
|
|
268
|
+
return ggml_cuda_mmq_get_config_rdna2(type, J, fallback);
|
|
269
|
+
#endif // CDNA
|
|
270
|
+
#else
|
|
271
|
+
#ifdef BLACKWELL_MMA_AVAILABLE
|
|
272
|
+
return ggml_cuda_mmq_get_config_blackwell(type, J, fallback);
|
|
273
|
+
#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA
|
|
274
|
+
return ggml_cuda_mmq_get_config_ampere(type, J, fallback);
|
|
275
|
+
#elif __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A
|
|
276
|
+
return ggml_cuda_mmq_get_config_pascal_dp4a(type, J, fallback);
|
|
277
|
+
#else
|
|
278
|
+
return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback);
|
|
279
|
+
#endif // BLACKWELL_MMA_AVAILABLE
|
|
280
|
+
#endif // GGML_USE_HIP
|
|
281
|
+
GGML_UNUSED_VARS(type, J, fallback);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
static __host__ int ggml_cuda_mmq_get_type(const ggml_type type, const int J, const bool fallback, const int cc) {
|
|
285
|
+
return ggml_cuda_mmq_get_config(type, J, fallback, cc).type;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
static constexpr __device__ int ggml_cuda_mmq_get_type(ggml_type type, int J, bool fallback) {
|
|
289
|
+
return ggml_cuda_mmq_get_config(type, J, fallback).type;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
static __host__ int ggml_cuda_mmq_get_nthreads(const ggml_type type, const int J, const bool fallback, const int cc) {
|
|
293
|
+
return ggml_cuda_mmq_get_config(type, J, fallback, cc).nthreads;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
static constexpr __device__ int ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback) {
|
|
297
|
+
return ggml_cuda_mmq_get_config(type, J, fallback).nthreads;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
static __host__ int ggml_cuda_mmq_get_occupancy(const ggml_type type, const int J, const bool fallback, const int cc) {
|
|
301
|
+
return ggml_cuda_mmq_get_config(type, J, fallback, cc).occupancy;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
static constexpr __device__ int ggml_cuda_mmq_get_occupancy(ggml_type type, int J, bool fallback) {
|
|
305
|
+
return ggml_cuda_mmq_get_config(type, J, fallback).occupancy;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
static __host__ int ggml_cuda_mmq_get_I(const ggml_type type, const int J, const bool fallback, const int cc) {
|
|
309
|
+
return ggml_cuda_mmq_get_config(type, J, fallback, cc).I;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
static constexpr __device__ int ggml_cuda_mmq_get_I(ggml_type type, int J, bool fallback) {
|
|
313
|
+
return ggml_cuda_mmq_get_config(type, J, fallback).I;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
static __host__ int ggml_cuda_mmq_get_J(const ggml_type type, const int J, const bool fallback, const int cc) {
|
|
317
|
+
return ggml_cuda_mmq_get_config(type, J, fallback, cc).J;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
static constexpr __device__ int ggml_cuda_mmq_get_J(ggml_type type, int J, bool fallback) {
|
|
321
|
+
return ggml_cuda_mmq_get_config(type, J, fallback).J;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
static __host__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(const ggml_type type, const int J, const bool fallback, const int cc) {
|
|
325
|
+
return ggml_cuda_mmq_get_config(type, J, fallback, cc).sram_layout;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
static constexpr __device__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(ggml_type type, int J, bool fallback) {
|
|
329
|
+
return ggml_cuda_mmq_get_config(type, J, fallback).sram_layout;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
static __host__ int ggml_cuda_mmq_get_K_vram(const ggml_type type, const int J, const bool fallback, const int cc) {
|
|
333
|
+
return ggml_cuda_mmq_get_config(type, J, fallback, cc).K_vram;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
static constexpr __device__ int ggml_cuda_mmq_get_K_vram(ggml_type type, int J, bool fallback) {
|
|
337
|
+
return ggml_cuda_mmq_get_config(type, J, fallback).K_vram;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
static __host__ bool ggml_cuda_mmq_get_stream_k(const ggml_type type, const int J, const bool fallback, const int cc) {
|
|
341
|
+
return ggml_cuda_mmq_get_config(type, J, fallback, cc).stream_k;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
static constexpr __device__ bool ggml_cuda_mmq_get_stream_k(ggml_type type, int J, bool fallback) {
|
|
345
|
+
return ggml_cuda_mmq_get_config(type, J, fallback).stream_k;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
static __host__ int ggml_cuda_mmq_get_fallback(const ggml_type type, const int J, const bool fallback, const int cc) {
|
|
349
|
+
return ggml_cuda_mmq_get_config(type, J, fallback, cc).fallback;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
static constexpr __device__ int ggml_cuda_mmq_get_fallback(ggml_type type, int J, bool fallback) {
|
|
353
|
+
return ggml_cuda_mmq_get_config(type, J, fallback).fallback;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// ---------------------------------------------------------------------------------------------
|
|
357
|
+
|
|
358
|
+
static __host__ int ggml_cuda_mmq_get_sram_stride(const ggml_type type, const int J, const bool fallback, const int cc) {
|
|
359
|
+
return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, cc));
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
static constexpr __device__ int ggml_cuda_mmq_get_sram_stride(ggml_type type, int J, bool fallback) {
|
|
363
|
+
return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback));
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
static __host__ int ggml_cuda_mmq_get_J_max(const ggml_type type, const bool fallback, const int cc, const int64_t ne11) {
|
|
367
|
+
int ret = std::min(ne11, int64_t(512));
|
|
368
|
+
ret -= ret % 8;
|
|
369
|
+
for (;ret > 0; ret -= 8) {
|
|
370
|
+
if (ggml_cuda_mmq_get_config(type, ret, fallback, cc).type != GGML_TYPE_COUNT) {
|
|
371
|
+
return ret;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
return ret;
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type, int J, bool fallback) {
|
|
378
|
+
return ggml_cuda_mmq_get_config(type, J, fallback).rows_per_warp();
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
#define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_0 + I/QI4_0, 0}
|
|
382
|
+
#define MMQ_DP4A_TXS_Q4_1 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_1 + I/QI4_1, 0}
|
|
383
|
+
#define MMQ_DP4A_TXS_Q8_0 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_0 + I/(QI8_0/2), 0}
|
|
384
|
+
#define MMQ_DP4A_TXS_Q8_0_16 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*4/QI8_0 + I/(QI8_0/4), 0}
|
|
385
|
+
#define MMQ_DP4A_TXS_Q8_1 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_1 + I/(QI8_1/2), 0}
|
|
386
|
+
#define MMQ_DP4A_TXS_Q2_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K + I, 0}
|
|
387
|
+
#define MMQ_DP4A_TXS_Q3_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I, I*MMQ_TILE_NE_K/8 + I/8}
|
|
388
|
+
#define MMQ_DP4A_TXS_Q4_K tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_K, I*MMQ_TILE_NE_K/8 + I/8}
|
|
389
|
+
#define MMQ_DP4A_TXS_Q5_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI5_K + I/QI5_K, I*MMQ_TILE_NE_K/8 + I/8}
|
|
390
|
+
#define MMQ_DP4A_TXS_Q6_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI6_K + I/QI6_K, I*MMQ_TILE_NE_K/8 + I/8}
|
|
391
|
+
|
|
392
|
+
static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int I) {
|
|
393
|
+
switch (type) {
|
|
394
|
+
case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0;
|
|
395
|
+
case GGML_TYPE_Q2_0: return MMQ_DP4A_TXS_Q8_0;
|
|
396
|
+
case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0;
|
|
397
|
+
case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1;
|
|
398
|
+
case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0;
|
|
399
|
+
case GGML_TYPE_Q5_1: return MMQ_DP4A_TXS_Q8_1;
|
|
400
|
+
case GGML_TYPE_Q8_0: return MMQ_DP4A_TXS_Q8_0;
|
|
401
|
+
case GGML_TYPE_MXFP4: return MMQ_DP4A_TXS_Q8_1;
|
|
402
|
+
case GGML_TYPE_NVFP4: return MMQ_DP4A_TXS_Q8_0_16;
|
|
403
|
+
case GGML_TYPE_Q2_K: return MMQ_DP4A_TXS_Q2_K;
|
|
404
|
+
case GGML_TYPE_Q3_K: return MMQ_DP4A_TXS_Q3_K;
|
|
405
|
+
case GGML_TYPE_Q4_K: return MMQ_DP4A_TXS_Q4_K;
|
|
406
|
+
case GGML_TYPE_Q5_K: return MMQ_DP4A_TXS_Q5_K;
|
|
407
|
+
case GGML_TYPE_Q6_K: return MMQ_DP4A_TXS_Q6_K;
|
|
408
|
+
case GGML_TYPE_IQ2_XXS: return MMQ_DP4A_TXS_Q8_0;
|
|
409
|
+
case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16;
|
|
410
|
+
case GGML_TYPE_IQ2_S: return MMQ_DP4A_TXS_Q8_0_16;
|
|
411
|
+
case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0;
|
|
412
|
+
case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0;
|
|
413
|
+
case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0;
|
|
414
|
+
case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0;
|
|
415
|
+
case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0;
|
|
416
|
+
default: return tile_x_sizes{0, 0, 0};
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// FIXME temporary until all combinations of data types and GPUs can use the MMA data layout
|
|
421
|
+
static __host__ int ggml_cuda_mmq_get_nbytes_shared_x(const ggml_cuda_mmq_config & config, const int cc) {
|
|
422
|
+
if (config.use_mma_data_layout(cc)) {
|
|
423
|
+
return config.I * ggml_cuda_mmq_get_sram_stride(config.sram_layout) * 4;
|
|
424
|
+
}
|
|
425
|
+
const tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(config.type, config.I);
|
|
426
|
+
return (txs.qs + txs.dm + txs.sc) * 4;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// ------------------------------------------------------------
|
|
430
|
+
|
|
431
|
+
#include "mmq-load-tiles.cuh"
|
|
432
|
+
#include "mmq-vec-dot.cuh"
|
|
433
|
+
|
|
434
|
+
template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a(
|
|
435
|
+
const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst,
|
|
436
|
+
const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
|
|
437
|
+
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
|
438
|
+
constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;
|
|
439
|
+
constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);
|
|
440
|
+
|
|
441
|
+
const bool y_scale_used = y_scale != nullptr;
|
|
442
|
+
|
|
443
|
+
#pragma unroll
|
|
444
|
+
for (int j0 = 0; j0 < J; j0 += nwarps) {
|
|
445
|
+
const int j = j0 + threadIdx.y;
|
|
446
|
+
|
|
447
|
+
if (j > j_max) {
|
|
448
|
+
return;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
#pragma unroll
|
|
452
|
+
for (int i0 = 0; i0 < I; i0 += warp_size) {
|
|
453
|
+
const int i = i0 + threadIdx.x;
|
|
454
|
+
|
|
455
|
+
if (fallback && i > i_max) {
|
|
456
|
+
continue;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
460
|
+
if (y_scale_used) {
|
|
461
|
+
dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
|
|
462
|
+
} else {
|
|
463
|
+
dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
|
|
464
|
+
}
|
|
465
|
+
} else {
|
|
466
|
+
dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];
|
|
467
|
+
GGML_UNUSED(y_scale_used);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
template<ggml_type type, int J, bool fallback>
|
|
474
|
+
static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(
|
|
475
|
+
const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst,
|
|
476
|
+
const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {
|
|
477
|
+
|
|
478
|
+
#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
|
|
479
|
+
typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;
|
|
480
|
+
#else
|
|
481
|
+
typedef tile<16, 8, int> tile_C;
|
|
482
|
+
#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)
|
|
483
|
+
|
|
484
|
+
constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);
|
|
485
|
+
constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
|
|
486
|
+
|
|
487
|
+
const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I);
|
|
488
|
+
|
|
489
|
+
const bool y_scale_used = y_scale != nullptr;
|
|
490
|
+
|
|
491
|
+
#pragma unroll
|
|
492
|
+
for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
|
|
493
|
+
#pragma unroll
|
|
494
|
+
for (int n = 0; n < ntx; ++n) {
|
|
495
|
+
#pragma unroll
|
|
496
|
+
for (int l = 0; l < tile_C::ne; ++l) {
|
|
497
|
+
const int j = j0 + (threadIdx.y % ntx) * tile_C::J + tile_C::get_j(l);
|
|
498
|
+
|
|
499
|
+
if (j > j_max) {
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
const int i = i0 + n*tile_C::I + tile_C::get_i(l);
|
|
504
|
+
|
|
505
|
+
if (fallback && i > i_max) {
|
|
506
|
+
continue;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
510
|
+
if (y_scale_used) {
|
|
511
|
+
dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l];
|
|
512
|
+
} else {
|
|
513
|
+
dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l];
|
|
514
|
+
}
|
|
515
|
+
} else {
|
|
516
|
+
dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l];
|
|
517
|
+
GGML_UNUSED(y_scale_used);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// -------------------------------------------------------------------------------------------------------------------------------------
|
|
525
|
+
|
|
526
|
+
// TODO remove this struct and use ggml_cuda_mmq_sram_layout instead.
|
|
527
|
+
struct ggml_cuda_mmq_util_funcs {
|
|
528
|
+
int vdr;
|
|
529
|
+
ggml_cuda_mmq_load_tiles_t load_tiles;
|
|
530
|
+
ggml_cuda_mmq_vec_dot_t vec_dot;
|
|
531
|
+
ggml_cuda_mmq_write_back_t write_back;
|
|
532
|
+
|
|
533
|
+
constexpr __host__ __device__ ggml_cuda_mmq_util_funcs(
|
|
534
|
+
int vdr, ggml_cuda_mmq_load_tiles_t load_tiles, ggml_cuda_mmq_vec_dot_t vec_dot, ggml_cuda_mmq_write_back_t write_back) :
|
|
535
|
+
vdr(vdr), load_tiles(load_tiles), vec_dot(vec_dot), write_back(write_back) {}
|
|
536
|
+
};
|
|
537
|
+
|
|
538
|
+
template <ggml_type type, int J, bool fallback>
|
|
539
|
+
static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcs() {
|
|
540
|
+
if (!ggml_cuda_mmq_get_config(type, J, fallback).use_mma_data_layout()) {
|
|
541
|
+
switch (type) {
|
|
542
|
+
case GGML_TYPE_Q1_0:
|
|
543
|
+
return ggml_cuda_mmq_util_funcs(
|
|
544
|
+
VDR_Q1_0_Q8_1_MMQ,
|
|
545
|
+
ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,
|
|
546
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
|
547
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
548
|
+
case GGML_TYPE_Q2_0:
|
|
549
|
+
return ggml_cuda_mmq_util_funcs(
|
|
550
|
+
VDR_Q2_0_Q8_1_MMQ,
|
|
551
|
+
ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>,
|
|
552
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
|
553
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
554
|
+
case GGML_TYPE_Q4_0:
|
|
555
|
+
return ggml_cuda_mmq_util_funcs(
|
|
556
|
+
VDR_Q4_0_Q8_1_MMQ,
|
|
557
|
+
ggml_cuda_mmq_load_tiles_q4_0<type, J, fallback>,
|
|
558
|
+
ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a<type, J, fallback>,
|
|
559
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
560
|
+
case GGML_TYPE_Q4_1:
|
|
561
|
+
return ggml_cuda_mmq_util_funcs(
|
|
562
|
+
VDR_Q4_1_Q8_1_MMQ,
|
|
563
|
+
ggml_cuda_mmq_load_tiles_q4_1<type, J, fallback>,
|
|
564
|
+
ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a<type, J, fallback>,
|
|
565
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
566
|
+
case GGML_TYPE_Q5_0:
|
|
567
|
+
return ggml_cuda_mmq_util_funcs(
|
|
568
|
+
VDR_Q5_0_Q8_1_MMQ,
|
|
569
|
+
ggml_cuda_mmq_load_tiles_q5_0<type, J, fallback>,
|
|
570
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
|
571
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
572
|
+
case GGML_TYPE_Q5_1:
|
|
573
|
+
return ggml_cuda_mmq_util_funcs(
|
|
574
|
+
VDR_Q5_1_Q8_1_MMQ,
|
|
575
|
+
ggml_cuda_mmq_load_tiles_q5_1<type, J, fallback>,
|
|
576
|
+
ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,
|
|
577
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
578
|
+
case GGML_TYPE_Q8_0:
|
|
579
|
+
return ggml_cuda_mmq_util_funcs(
|
|
580
|
+
VDR_Q8_0_Q8_1_MMQ,
|
|
581
|
+
ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>,
|
|
582
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
|
583
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
584
|
+
// ---------------------------------------------------------------------------------------------
|
|
585
|
+
case GGML_TYPE_Q2_K:
|
|
586
|
+
return ggml_cuda_mmq_util_funcs(
|
|
587
|
+
VDR_Q2_K_Q8_1_MMQ,
|
|
588
|
+
ggml_cuda_mmq_load_tiles_q2_K<type, J, fallback>,
|
|
589
|
+
ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a<type, J, fallback>,
|
|
590
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
591
|
+
case GGML_TYPE_Q3_K:
|
|
592
|
+
return ggml_cuda_mmq_util_funcs(
|
|
593
|
+
VDR_Q3_K_Q8_1_MMQ,
|
|
594
|
+
ggml_cuda_mmq_load_tiles_q3_K<type, J, fallback>,
|
|
595
|
+
ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a<type, J, fallback>,
|
|
596
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
597
|
+
case GGML_TYPE_Q4_K:
|
|
598
|
+
return ggml_cuda_mmq_util_funcs(
|
|
599
|
+
VDR_Q4_K_Q8_1_MMQ,
|
|
600
|
+
ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>,
|
|
601
|
+
ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a<type, J, fallback>,
|
|
602
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
603
|
+
case GGML_TYPE_Q5_K:
|
|
604
|
+
return ggml_cuda_mmq_util_funcs(
|
|
605
|
+
VDR_Q5_K_Q8_1_MMQ,
|
|
606
|
+
ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>,
|
|
607
|
+
ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a<type, J, fallback>,
|
|
608
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
609
|
+
case GGML_TYPE_Q6_K:
|
|
610
|
+
return ggml_cuda_mmq_util_funcs(
|
|
611
|
+
VDR_Q6_K_Q8_1_MMQ,
|
|
612
|
+
ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>,
|
|
613
|
+
ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a<type, J, fallback>,
|
|
614
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
615
|
+
// ---------------------------------------------------------------------------------------------
|
|
616
|
+
case GGML_TYPE_IQ1_S:
|
|
617
|
+
return ggml_cuda_mmq_util_funcs(
|
|
618
|
+
VDR_IQ1_S_Q8_1_MMQ,
|
|
619
|
+
ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,
|
|
620
|
+
ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,
|
|
621
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
622
|
+
case GGML_TYPE_IQ2_XXS:
|
|
623
|
+
return ggml_cuda_mmq_util_funcs(
|
|
624
|
+
VDR_IQ2_XXS_Q8_1_MMQ,
|
|
625
|
+
ggml_cuda_mmq_load_tiles_iq2_xxs<type, J, fallback>,
|
|
626
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
|
627
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
628
|
+
case GGML_TYPE_IQ2_XS:
|
|
629
|
+
return ggml_cuda_mmq_util_funcs(
|
|
630
|
+
VDR_IQ2_XS_Q8_1_MMQ,
|
|
631
|
+
ggml_cuda_mmq_load_tiles_iq2_xs<type, J, fallback>,
|
|
632
|
+
ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,
|
|
633
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
634
|
+
case GGML_TYPE_IQ2_S:
|
|
635
|
+
return ggml_cuda_mmq_util_funcs(
|
|
636
|
+
VDR_IQ2_S_Q8_1_MMQ,
|
|
637
|
+
ggml_cuda_mmq_load_tiles_iq2_s<type, J, fallback>,
|
|
638
|
+
ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,
|
|
639
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
640
|
+
case GGML_TYPE_IQ3_XXS:
|
|
641
|
+
return ggml_cuda_mmq_util_funcs(
|
|
642
|
+
VDR_IQ3_XXS_Q8_1_MMQ,
|
|
643
|
+
ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,
|
|
644
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
|
645
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
646
|
+
case GGML_TYPE_IQ3_S:
|
|
647
|
+
return ggml_cuda_mmq_util_funcs(
|
|
648
|
+
VDR_IQ3_S_Q8_1_MMQ,
|
|
649
|
+
ggml_cuda_mmq_load_tiles_iq3_s<type, J, fallback>,
|
|
650
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
|
651
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
652
|
+
case GGML_TYPE_IQ4_XS:
|
|
653
|
+
return ggml_cuda_mmq_util_funcs(
|
|
654
|
+
VDR_IQ4_XS_Q8_1_MMQ,
|
|
655
|
+
ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>,
|
|
656
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
|
657
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
658
|
+
case GGML_TYPE_IQ4_NL:
|
|
659
|
+
return ggml_cuda_mmq_util_funcs(
|
|
660
|
+
VDR_IQ4_NL_Q8_1_MMQ,
|
|
661
|
+
ggml_cuda_mmq_load_tiles_iq4_nl<type, J, fallback>,
|
|
662
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
|
663
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
664
|
+
// ---------------------------------------------------------------------------------------------
|
|
665
|
+
case GGML_TYPE_MXFP4:
|
|
666
|
+
return ggml_cuda_mmq_util_funcs(
|
|
667
|
+
VDR_MXFP4_Q8_1_MMQ,
|
|
668
|
+
ggml_cuda_mmq_load_tiles_mxfp4<type, J, fallback>,
|
|
669
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,
|
|
670
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
671
|
+
case GGML_TYPE_NVFP4:
|
|
672
|
+
return ggml_cuda_mmq_util_funcs(
|
|
673
|
+
VDR_NVFP4_Q8_1_MMQ,
|
|
674
|
+
ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback>,
|
|
675
|
+
ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,
|
|
676
|
+
ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);
|
|
677
|
+
default:
|
|
678
|
+
return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr);
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// ---------------------------------------------------------------------------------------------
|
|
683
|
+
|
|
684
|
+
#ifdef BLACKWELL_MMA_AVAILABLE
|
|
685
|
+
switch (type) {
|
|
686
|
+
case GGML_TYPE_MXFP4:
|
|
687
|
+
return ggml_cuda_mmq_util_funcs(
|
|
688
|
+
-1,
|
|
689
|
+
ggml_cuda_mmq_load_tiles_mxfp4_fp4<type, J, fallback>,
|
|
690
|
+
ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>,
|
|
691
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
692
|
+
case GGML_TYPE_NVFP4:
|
|
693
|
+
return ggml_cuda_mmq_util_funcs(
|
|
694
|
+
-1,
|
|
695
|
+
ggml_cuda_mmq_load_tiles_nvfp4_nvfp4<type, J, fallback>,
|
|
696
|
+
ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>,
|
|
697
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
698
|
+
default:
|
|
699
|
+
break;
|
|
700
|
+
}
|
|
701
|
+
#endif // BLACKWELL_MMA_AVAILABLE
|
|
702
|
+
|
|
703
|
+
// ---------------------------------------------------------------------------------------------
|
|
704
|
+
|
|
705
|
+
switch (type) {
|
|
706
|
+
case GGML_TYPE_Q1_0:
|
|
707
|
+
return ggml_cuda_mmq_util_funcs(
|
|
708
|
+
-1,
|
|
709
|
+
ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,
|
|
710
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
|
711
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
712
|
+
case GGML_TYPE_Q2_0:
|
|
713
|
+
return ggml_cuda_mmq_util_funcs(
|
|
714
|
+
-1,
|
|
715
|
+
ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>,
|
|
716
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
|
717
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
718
|
+
case GGML_TYPE_Q4_0:
|
|
719
|
+
return ggml_cuda_mmq_util_funcs(
|
|
720
|
+
-1,
|
|
721
|
+
ggml_cuda_mmq_load_tiles_q4_0<type, J, fallback>,
|
|
722
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_DS4>,
|
|
723
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
724
|
+
case GGML_TYPE_Q4_1:
|
|
725
|
+
return ggml_cuda_mmq_util_funcs(
|
|
726
|
+
-1,
|
|
727
|
+
ggml_cuda_mmq_load_tiles_q4_1<type, J, fallback>,
|
|
728
|
+
ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
|
|
729
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
730
|
+
case GGML_TYPE_Q5_0:
|
|
731
|
+
return ggml_cuda_mmq_util_funcs(
|
|
732
|
+
-1,
|
|
733
|
+
ggml_cuda_mmq_load_tiles_q5_0<type, J, fallback>,
|
|
734
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
|
735
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
736
|
+
case GGML_TYPE_Q5_1:
|
|
737
|
+
return ggml_cuda_mmq_util_funcs(
|
|
738
|
+
-1,
|
|
739
|
+
ggml_cuda_mmq_load_tiles_q5_1<type, J, fallback>,
|
|
740
|
+
ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
|
|
741
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
742
|
+
case GGML_TYPE_Q8_0:
|
|
743
|
+
return ggml_cuda_mmq_util_funcs(
|
|
744
|
+
-1,
|
|
745
|
+
ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>,
|
|
746
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
|
747
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
748
|
+
// ---------------------------------------------------------------------------------------------
|
|
749
|
+
case GGML_TYPE_Q2_K:
|
|
750
|
+
return ggml_cuda_mmq_util_funcs(
|
|
751
|
+
-1,
|
|
752
|
+
ggml_cuda_mmq_load_tiles_q2_K<type, J, fallback>,
|
|
753
|
+
ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma<type, J, fallback>,
|
|
754
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
755
|
+
case GGML_TYPE_Q3_K:
|
|
756
|
+
return ggml_cuda_mmq_util_funcs(
|
|
757
|
+
-1,
|
|
758
|
+
ggml_cuda_mmq_load_tiles_q3_K<type, J, fallback>,
|
|
759
|
+
ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>,
|
|
760
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
761
|
+
case GGML_TYPE_Q4_K:
|
|
762
|
+
return ggml_cuda_mmq_util_funcs(
|
|
763
|
+
-1,
|
|
764
|
+
ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>,
|
|
765
|
+
ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
|
|
766
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
767
|
+
case GGML_TYPE_Q5_K:
|
|
768
|
+
return ggml_cuda_mmq_util_funcs(
|
|
769
|
+
-1,
|
|
770
|
+
ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>,
|
|
771
|
+
ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
|
|
772
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
773
|
+
case GGML_TYPE_Q6_K:
|
|
774
|
+
return ggml_cuda_mmq_util_funcs(
|
|
775
|
+
-1,
|
|
776
|
+
ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>,
|
|
777
|
+
ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma<type, J, fallback>,
|
|
778
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
779
|
+
// ---------------------------------------------------------------------------------------------
|
|
780
|
+
case GGML_TYPE_IQ1_S:
|
|
781
|
+
return ggml_cuda_mmq_util_funcs(
|
|
782
|
+
-1,
|
|
783
|
+
ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,
|
|
784
|
+
ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,
|
|
785
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
786
|
+
case GGML_TYPE_IQ2_XXS:
|
|
787
|
+
return ggml_cuda_mmq_util_funcs(
|
|
788
|
+
-1,
|
|
789
|
+
ggml_cuda_mmq_load_tiles_iq2_xxs<type, J, fallback>,
|
|
790
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
|
791
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
792
|
+
case GGML_TYPE_IQ2_XS:
|
|
793
|
+
return ggml_cuda_mmq_util_funcs(
|
|
794
|
+
-1,
|
|
795
|
+
ggml_cuda_mmq_load_tiles_iq2_xs<type, J, fallback>,
|
|
796
|
+
ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>,
|
|
797
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
798
|
+
case GGML_TYPE_IQ2_S:
|
|
799
|
+
return ggml_cuda_mmq_util_funcs(
|
|
800
|
+
-1,
|
|
801
|
+
ggml_cuda_mmq_load_tiles_iq2_s<type, J, fallback>,
|
|
802
|
+
ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>,
|
|
803
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
804
|
+
case GGML_TYPE_IQ3_XXS:
|
|
805
|
+
return ggml_cuda_mmq_util_funcs(
|
|
806
|
+
-1,
|
|
807
|
+
ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,
|
|
808
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
|
809
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
810
|
+
case GGML_TYPE_IQ3_S:
|
|
811
|
+
return ggml_cuda_mmq_util_funcs(
|
|
812
|
+
-1,
|
|
813
|
+
ggml_cuda_mmq_load_tiles_iq3_s<type, J, fallback>,
|
|
814
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
|
815
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
816
|
+
case GGML_TYPE_IQ4_XS:
|
|
817
|
+
return ggml_cuda_mmq_util_funcs(
|
|
818
|
+
-1,
|
|
819
|
+
ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>,
|
|
820
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
|
821
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
822
|
+
case GGML_TYPE_IQ4_NL:
|
|
823
|
+
return ggml_cuda_mmq_util_funcs(
|
|
824
|
+
-1,
|
|
825
|
+
ggml_cuda_mmq_load_tiles_iq4_nl<type, J, fallback>,
|
|
826
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
|
827
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
828
|
+
// ---------------------------------------------------------------------------------------------
|
|
829
|
+
case GGML_TYPE_MXFP4:
|
|
830
|
+
return ggml_cuda_mmq_util_funcs(
|
|
831
|
+
-1,
|
|
832
|
+
ggml_cuda_mmq_load_tiles_mxfp4<type, J, fallback>,
|
|
833
|
+
ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,
|
|
834
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
835
|
+
case GGML_TYPE_NVFP4:
|
|
836
|
+
return ggml_cuda_mmq_util_funcs(
|
|
837
|
+
-1,
|
|
838
|
+
ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback>,
|
|
839
|
+
ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>,
|
|
840
|
+
ggml_cuda_mmq_write_back_mma<type, J, fallback>);
|
|
841
|
+
default:
|
|
842
|
+
return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr);
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
template <ggml_type type, int J, bool fallback>
|
|
847
|
+
static constexpr __device__ int ggml_cuda_mmq_get_vdr() {
|
|
848
|
+
return ggml_cuda_mmq_get_util_funcs<type, J, fallback>().vdr;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
template <ggml_type type, int J, bool fallback>
|
|
852
|
+
static constexpr __device__ ggml_cuda_mmq_load_tiles_t ggml_cuda_mmq_get_load_tiles() {
|
|
853
|
+
return ggml_cuda_mmq_get_util_funcs<type, J, fallback>().load_tiles;
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
template <ggml_type type, int J, bool fallback>
|
|
857
|
+
static constexpr __device__ ggml_cuda_mmq_vec_dot_t ggml_cuda_mmq_get_vec_dot() {
|
|
858
|
+
return ggml_cuda_mmq_get_util_funcs<type, J, fallback>().vec_dot;
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
template <ggml_type type, int J, bool fallback>
|
|
862
|
+
static constexpr __device__ ggml_cuda_mmq_write_back_t ggml_cuda_mmq_get_write_back() {
|
|
863
|
+
return ggml_cuda_mmq_get_util_funcs<type, J, fallback>().write_back;
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
// ---------------------------------------------------------------------------------------------
|
|
867
|
+
|
|
868
|
+
template <ggml_type type, int J, bool fallback, bool fixup>
|
|
869
|
+
static __device__ __forceinline__ void mul_mat_q_process_tile(
|
|
870
|
+
const char * __restrict__ x, const int offset_x, const int * __restrict__ y,
|
|
871
|
+
const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup,
|
|
872
|
+
const float * __restrict__ y_scale,
|
|
873
|
+
const int stride_row_x, const int ncols_y, const int stride_col_dst,
|
|
874
|
+
const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) {
|
|
875
|
+
|
|
876
|
+
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
|
877
|
+
constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;
|
|
878
|
+
constexpr int qk = ggml_cuda_type_traits<type>::qk;
|
|
879
|
+
constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);
|
|
880
|
+
constexpr ggml_cuda_mmq_load_tiles_t load_tiles = ggml_cuda_mmq_get_load_tiles<type, J, fallback>();
|
|
881
|
+
constexpr ggml_cuda_mmq_vec_dot_t vec_dot = ggml_cuda_mmq_get_vec_dot<type, J, fallback>();
|
|
882
|
+
constexpr ggml_cuda_mmq_write_back_t write_back = ggml_cuda_mmq_get_write_back<type, J, fallback>();
|
|
883
|
+
|
|
884
|
+
extern __shared__ int data_mul_mat_q[];
|
|
885
|
+
int * tile_y = data_mul_mat_q + J;
|
|
886
|
+
int * tile_x = tile_y + GGML_PAD(J*MMQ_TILE_Y_K, nwarps*warp_size);
|
|
887
|
+
|
|
888
|
+
#if defined(BLACKWELL_MMA_AVAILABLE)
|
|
889
|
+
// FP4 tile stores 8 blocks
|
|
890
|
+
constexpr int ne_block = (type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4) ? QK_FP4_MMQ : QK8_1_MMQ;
|
|
891
|
+
#else
|
|
892
|
+
constexpr int ne_block = QK8_1_MMQ;
|
|
893
|
+
#endif // defined(BLACKWELL_MMA_AVAILABLE)
|
|
894
|
+
|
|
895
|
+
constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback);
|
|
896
|
+
constexpr int blocks_per_iter = ITER_K / qk;
|
|
897
|
+
|
|
898
|
+
float sum[J*I / (nwarps*warp_size)] = {0.0f};
|
|
899
|
+
|
|
900
|
+
constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int);
|
|
901
|
+
|
|
902
|
+
for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) {
|
|
903
|
+
load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x);
|
|
904
|
+
{
|
|
905
|
+
const int * by0 = y + ncols_y * (kb0 * qk / ne_block) * sz;
|
|
906
|
+
#pragma unroll
|
|
907
|
+
for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {
|
|
908
|
+
int l = l0 + threadIdx.y*warp_size + threadIdx.x;
|
|
909
|
+
|
|
910
|
+
tile_y[l] = by0[l];
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
__syncthreads();
|
|
915
|
+
|
|
916
|
+
vec_dot(tile_x, tile_y, sum, 0);
|
|
917
|
+
|
|
918
|
+
__syncthreads();
|
|
919
|
+
|
|
920
|
+
{
|
|
921
|
+
const int * by0 = y + ncols_y * ((kb0 * qk / ne_block) * sz + sz);
|
|
922
|
+
#pragma unroll
|
|
923
|
+
for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {
|
|
924
|
+
int l = l0 + threadIdx.y*warp_size + threadIdx.x;
|
|
925
|
+
|
|
926
|
+
tile_y[l] = by0[l];
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
__syncthreads();
|
|
931
|
+
|
|
932
|
+
vec_dot(tile_x, tile_y, sum, MMQ_TILE_NE_K);
|
|
933
|
+
|
|
934
|
+
__syncthreads();
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
if (fixup) {
|
|
938
|
+
write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J);
|
|
939
|
+
} else {
|
|
940
|
+
write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
|
|
945
|
+
// The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598
|
|
946
|
+
|
|
947
|
+
template <ggml_type type, int J, bool fallback>
|
|
948
|
+
__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback), ggml_cuda_mmq_get_occupancy(type, J, fallback))
|
|
949
|
+
static __global__ void mul_mat_q(
|
|
950
|
+
const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,
|
|
951
|
+
const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, float * __restrict__ tmp_fixup,
|
|
952
|
+
const float * __restrict__ y_scale,
|
|
953
|
+
const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst,
|
|
954
|
+
const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
|
|
955
|
+
const uint3 sample_ratio, const uint3 nsamples_y, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,
|
|
956
|
+
const uint3 ntx) {
|
|
957
|
+
|
|
958
|
+
// Skip unused template specializations for faster compilation:
|
|
959
|
+
if (ggml_cuda_mmq_get_config(type, J, fallback).type == GGML_TYPE_COUNT) {
|
|
960
|
+
NO_DEVICE_CODE;
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
|
965
|
+
constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;
|
|
966
|
+
constexpr int qk = ggml_cuda_type_traits<type>::qk;
|
|
967
|
+
constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);
|
|
968
|
+
|
|
969
|
+
const uint32_t nty = (nrows_x + I - 1) / I; // Number of tiles y
|
|
970
|
+
|
|
971
|
+
// Initialize the ids for writing back data with just the index.
|
|
972
|
+
// For regular matrix multiplications this is never changed.
|
|
973
|
+
// For MoE the correct indices are loaded from ids_dst.
|
|
974
|
+
extern __shared__ int ids_dst_shared[]; // Stored at beginning of shared memory.
|
|
975
|
+
#pragma unroll
|
|
976
|
+
for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {
|
|
977
|
+
const int j = j0 + threadIdx.y*warp_size + threadIdx.x;
|
|
978
|
+
|
|
979
|
+
if (j0 + nwarps*warp_size > J && j >= J) {
|
|
980
|
+
break;
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
ids_dst_shared[j] = j;
|
|
984
|
+
}
|
|
985
|
+
__syncthreads();
|
|
986
|
+
|
|
987
|
+
if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback)) {
|
|
988
|
+
const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
|
|
989
|
+
const int wt = tmp2.x;
|
|
990
|
+
const int zt = tmp2.y;
|
|
991
|
+
const int jt = blockIdx.y;
|
|
992
|
+
const int it = blockIdx.x;
|
|
993
|
+
|
|
994
|
+
// Defaults for regular matrix multiplication:
|
|
995
|
+
int col_low = 0;
|
|
996
|
+
int col_high = ncols_dst;
|
|
997
|
+
int col_diff = ncols_dst;
|
|
998
|
+
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
|
999
|
+
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
|
1000
|
+
int offset_y_scale;
|
|
1001
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
1002
|
+
offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
|
1003
|
+
} else {
|
|
1004
|
+
GGML_UNUSED(offset_y_scale);
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
if (ids_dst) {
|
|
1008
|
+
col_low = expert_bounds[zt + 0];
|
|
1009
|
+
col_high = expert_bounds[zt + 1];
|
|
1010
|
+
col_diff = col_high - col_low;
|
|
1011
|
+
|
|
1012
|
+
offset_y = 0;
|
|
1013
|
+
offset_dst = 0;
|
|
1014
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
1015
|
+
offset_y_scale = 0;
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
if (jt*J >= col_diff) {
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
// __syncthreads(); // There is no previous tile that could cause a race condition.
|
|
1023
|
+
#pragma unroll
|
|
1024
|
+
for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {
|
|
1025
|
+
const int j = j0 + threadIdx.y*warp_size + threadIdx.x;
|
|
1026
|
+
|
|
1027
|
+
if (j0 + nwarps*warp_size > J && j >= J) {
|
|
1028
|
+
break;
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
ids_dst_shared[j] = ids_dst[col_low + jt*J + j];
|
|
1032
|
+
}
|
|
1033
|
+
__syncthreads();
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
offset_y += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int));
|
|
1037
|
+
offset_dst += it*I;
|
|
1038
|
+
const float * y_scale_tile = nullptr;
|
|
1039
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
1040
|
+
offset_y_scale += col_low + jt*J;
|
|
1041
|
+
y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr;
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
const int tile_x_max_i = nrows_x - it*I - 1;
|
|
1045
|
+
const int tile_y_max_j = col_diff - jt*J - 1;
|
|
1046
|
+
|
|
1047
|
+
const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;
|
|
1048
|
+
|
|
1049
|
+
constexpr bool fixup = false;
|
|
1050
|
+
mul_mat_q_process_tile<type, J, fallback, fixup>
|
|
1051
|
+
(x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
|
|
1052
|
+
stride_row_x, ncols_y, stride_col_dst,
|
|
1053
|
+
tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z);
|
|
1054
|
+
return;
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback);
|
|
1058
|
+
constexpr int blocks_per_iter = ITER_K / qk;
|
|
1059
|
+
|
|
1060
|
+
// kbc == k block continuous, current index in continuous ijk space.
|
|
1061
|
+
int kbc = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;
|
|
1062
|
+
int kbc_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;
|
|
1063
|
+
|
|
1064
|
+
kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter;
|
|
1065
|
+
kbc_stop -= fastmodulo(kbc_stop, blocks_per_ne00) % blocks_per_iter;
|
|
1066
|
+
|
|
1067
|
+
// kb0 == k index when doing the matrix multiplication for an output tile.
|
|
1068
|
+
int kb0_start = fastmodulo(kbc, blocks_per_ne00);
|
|
1069
|
+
int kb0_stop = min(blocks_per_ne00.z, uint32_t(kb0_start + kbc_stop - kbc));
|
|
1070
|
+
while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) {
|
|
1071
|
+
int tmp = fastdiv(kbc, blocks_per_ne00);
|
|
1072
|
+
uint2 tmp2 = fast_div_modulo(tmp, ntx);
|
|
1073
|
+
const int jt = tmp2.y;
|
|
1074
|
+
tmp = tmp2.x;
|
|
1075
|
+
tmp2 = fast_div_modulo(tmp, nchannels_y);
|
|
1076
|
+
const int zt = tmp2.y;
|
|
1077
|
+
tmp = tmp2.x;
|
|
1078
|
+
tmp2 = fast_div_modulo(tmp, nsamples_y);
|
|
1079
|
+
const int wt = tmp2.y;
|
|
1080
|
+
const int it = tmp2.x;
|
|
1081
|
+
|
|
1082
|
+
// Defaults for regular matrix multiplication:
|
|
1083
|
+
int col_low = 0;
|
|
1084
|
+
int col_high = ncols_dst;
|
|
1085
|
+
int col_diff = ncols_dst;
|
|
1086
|
+
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
|
1087
|
+
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
|
1088
|
+
int offset_y_scale;
|
|
1089
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
1090
|
+
offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
|
1091
|
+
} else {
|
|
1092
|
+
GGML_UNUSED(offset_y_scale);
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
if (ids_dst) {
|
|
1096
|
+
col_low = expert_bounds[zt + 0];
|
|
1097
|
+
col_high = expert_bounds[zt + 1];
|
|
1098
|
+
col_diff = col_high - col_low;
|
|
1099
|
+
|
|
1100
|
+
offset_y = 0;
|
|
1101
|
+
offset_dst = 0;
|
|
1102
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
1103
|
+
offset_y_scale = 0;
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
if (jt*J >= col_diff) {
|
|
1107
|
+
kbc += blocks_per_ne00.z;
|
|
1108
|
+
kbc -= fastmodulo(kbc, blocks_per_ne00);
|
|
1109
|
+
|
|
1110
|
+
kb0_start = 0;
|
|
1111
|
+
kb0_stop = min(blocks_per_ne00.z, uint32_t(kbc_stop - kbc));
|
|
1112
|
+
|
|
1113
|
+
continue;
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
__syncthreads();
|
|
1117
|
+
#pragma unroll
|
|
1118
|
+
for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {
|
|
1119
|
+
const int j = j0 + threadIdx.y*warp_size + threadIdx.x;
|
|
1120
|
+
|
|
1121
|
+
if (j0 + nwarps*warp_size > J && j >= J) {
|
|
1122
|
+
break;
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
ids_dst_shared[j] = ids_dst[col_low + jt*J + j];
|
|
1126
|
+
}
|
|
1127
|
+
__syncthreads();
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int));
|
|
1131
|
+
offset_dst += it*I;
|
|
1132
|
+
const float * y_scale_tile = nullptr;
|
|
1133
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
1134
|
+
offset_y_scale += col_low + jt * J;
|
|
1135
|
+
y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr;
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
const int tile_x_max_i = nrows_x - it*I - 1;
|
|
1139
|
+
const int tile_y_max_j = col_diff - jt*J - 1;
|
|
1140
|
+
|
|
1141
|
+
const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;
|
|
1142
|
+
|
|
1143
|
+
constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
|
|
1144
|
+
mul_mat_q_process_tile<type, J, fallback, fixup>
|
|
1145
|
+
(x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
|
|
1146
|
+
stride_row_x, ncols_y, stride_col_dst,
|
|
1147
|
+
tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);
|
|
1148
|
+
|
|
1149
|
+
kbc += blocks_per_ne00.z;
|
|
1150
|
+
kbc -= fastmodulo(kbc, blocks_per_ne00);
|
|
1151
|
+
|
|
1152
|
+
kb0_start = 0;
|
|
1153
|
+
kb0_stop = min(blocks_per_ne00.z, uint32_t(kbc_stop - kbc));
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
if (kbc >= kbc_stop) {
|
|
1157
|
+
return;
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
int tmp = fastdiv(kbc, blocks_per_ne00);
|
|
1161
|
+
uint2 tmp2 = fast_div_modulo(tmp, ntx);
|
|
1162
|
+
const int jt = tmp2.y;
|
|
1163
|
+
tmp = tmp2.x;
|
|
1164
|
+
tmp2 = fast_div_modulo(tmp, nchannels_y);
|
|
1165
|
+
const int zt = tmp2.y;
|
|
1166
|
+
tmp = tmp2.x;
|
|
1167
|
+
tmp2 = fast_div_modulo(tmp, nsamples_y);
|
|
1168
|
+
const int wt = tmp2.y;
|
|
1169
|
+
const int it = tmp2.x;
|
|
1170
|
+
|
|
1171
|
+
// Defaults for regular matrix multiplication:
|
|
1172
|
+
int col_low = 0;
|
|
1173
|
+
int col_high = ncols_dst;
|
|
1174
|
+
int col_diff = ncols_dst;
|
|
1175
|
+
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
|
1176
|
+
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
|
1177
|
+
int offset_y_scale;
|
|
1178
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
1179
|
+
offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
|
1180
|
+
} else {
|
|
1181
|
+
GGML_UNUSED(offset_y_scale);
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
if (ids_dst) {
|
|
1185
|
+
col_low = expert_bounds[zt + 0];
|
|
1186
|
+
col_high = expert_bounds[zt + 1];
|
|
1187
|
+
col_diff = col_high - col_low;
|
|
1188
|
+
|
|
1189
|
+
offset_y = 0;
|
|
1190
|
+
offset_dst = 0;
|
|
1191
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
1192
|
+
offset_y_scale = 0;
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
if (jt*J >= col_diff) {
|
|
1196
|
+
return;
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
// The memory layout for the fixup buffer is always contiguous, therefore reset ids:
|
|
1200
|
+
__syncthreads();
|
|
1201
|
+
#pragma unroll
|
|
1202
|
+
for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {
|
|
1203
|
+
const int j = j0 + threadIdx.y*warp_size + threadIdx.x;
|
|
1204
|
+
|
|
1205
|
+
if (j0 + nwarps*warp_size > J && j >= J) {
|
|
1206
|
+
break;
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
ids_dst_shared[j] = j;
|
|
1210
|
+
}
|
|
1211
|
+
__syncthreads();
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int));
|
|
1215
|
+
offset_dst += it*I;
|
|
1216
|
+
const float * y_scale_tile = nullptr;
|
|
1217
|
+
if constexpr (type == GGML_TYPE_NVFP4) {
|
|
1218
|
+
offset_y_scale += col_low + jt * J;
|
|
1219
|
+
y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr;
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
const int tile_x_max_i = nrows_x - it*I - 1;
|
|
1223
|
+
const int tile_y_max_j = col_diff - jt*J - 1;
|
|
1224
|
+
|
|
1225
|
+
const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;
|
|
1226
|
+
|
|
1227
|
+
constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.
|
|
1228
|
+
mul_mat_q_process_tile<type, J, fallback, fixup>
|
|
1229
|
+
(x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,
|
|
1230
|
+
stride_row_x, ncols_y, stride_col_dst,
|
|
1231
|
+
tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
template <ggml_type type, int J, bool fallback>
|
|
1235
|
+
__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback)/2, 1)
|
|
1236
|
+
static __global__ void mul_mat_q_stream_k_fixup(
|
|
1237
|
+
const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst,
|
|
1238
|
+
float * __restrict__ tmp_last_tile, const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst,
|
|
1239
|
+
const int stride_col_dst, const uint3 nchannels_y, const int stride_channel_dst, const uint3 nsamples_y,
|
|
1240
|
+
const int stride_sample_dst, const uint3 ntx) {
|
|
1241
|
+
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
|
1242
|
+
constexpr int nwarps = (ggml_cuda_mmq_get_nthreads(type, J, fallback) / 2) / warp_size;
|
|
1243
|
+
constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);
|
|
1244
|
+
constexpr int qk = ggml_cuda_type_traits<type>::qk;
|
|
1245
|
+
constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback);
|
|
1246
|
+
constexpr int blocks_per_iter = ITER_K / qk;
|
|
1247
|
+
|
|
1248
|
+
float sum[J / nwarps] = {0.0f};
|
|
1249
|
+
const int i = blockIdx.y*warp_size + threadIdx.x;
|
|
1250
|
+
|
|
1251
|
+
const int nty = (nrows_x + I - 1) / I;
|
|
1252
|
+
|
|
1253
|
+
const int bidx0 = blockIdx.x;
|
|
1254
|
+
|
|
1255
|
+
// kbc == k block continuous, current index in continuous ijk space.
|
|
1256
|
+
int kbc0 = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;
|
|
1257
|
+
int kbc0_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;
|
|
1258
|
+
|
|
1259
|
+
kbc0 -= fastmodulo(kbc0, blocks_per_ne00) % blocks_per_iter;
|
|
1260
|
+
kbc0_stop -= fastmodulo(kbc0_stop, blocks_per_ne00) % blocks_per_iter;
|
|
1261
|
+
|
|
1262
|
+
const bool did_not_have_any_data = kbc0 == kbc0_stop;
|
|
1263
|
+
const bool wrote_beginning_of_tile = fastmodulo(kbc0, blocks_per_ne00) == 0;
|
|
1264
|
+
const bool did_not_write_last = fastdiv(kbc0, blocks_per_ne00) == fastdiv(kbc0_stop, blocks_per_ne00) && fastmodulo(kbc0_stop, blocks_per_ne00) != 0;
|
|
1265
|
+
if (did_not_have_any_data || wrote_beginning_of_tile || did_not_write_last) {
|
|
1266
|
+
return;
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
bool any_fixup = false;
|
|
1270
|
+
|
|
1271
|
+
// Iterate over previous blocks and sum up partial sums written to fixup buffer.
|
|
1272
|
+
// All CUDA blocks that get here must have a previous block that needs a fixup.
|
|
1273
|
+
int bidx = bidx0 - 1;
|
|
1274
|
+
int kbc_stop = kbc0;
|
|
1275
|
+
while(true) {
|
|
1276
|
+
int kbc = int64_t(bidx)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;
|
|
1277
|
+
kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter;
|
|
1278
|
+
|
|
1279
|
+
if (kbc == kbc_stop) { // Did not have any data.
|
|
1280
|
+
bidx--;
|
|
1281
|
+
kbc_stop = kbc;
|
|
1282
|
+
continue;
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1285
|
+
any_fixup = true;
|
|
1286
|
+
|
|
1287
|
+
|
|
1288
|
+
#pragma unroll
|
|
1289
|
+
for (int j0 = 0; j0 < J; j0 += nwarps) {
|
|
1290
|
+
const int j = j0 + threadIdx.y;
|
|
1291
|
+
|
|
1292
|
+
sum[j0/nwarps] += tmp_last_tile[bidx*(J*I) + j*I + i];
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
// If this block started in a previous tile we are done and don't need to combine additional partial results.
|
|
1296
|
+
if (fastmodulo(kbc, blocks_per_ne00) == 0 || fastdiv(kbc, blocks_per_ne00) < fastdiv(kbc0, blocks_per_ne00)) {
|
|
1297
|
+
break;
|
|
1298
|
+
}
|
|
1299
|
+
bidx--;
|
|
1300
|
+
kbc_stop = kbc;
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
if (!any_fixup) {
|
|
1304
|
+
return;
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
int tmp = fastdiv(kbc0, blocks_per_ne00);
|
|
1308
|
+
uint2 tmp2 = fast_div_modulo(tmp, ntx);
|
|
1309
|
+
const int jt = tmp2.y;
|
|
1310
|
+
tmp = tmp2.x;
|
|
1311
|
+
tmp2 = fast_div_modulo(tmp, nchannels_y);
|
|
1312
|
+
const int zt = tmp2.y;
|
|
1313
|
+
tmp = tmp2.x;
|
|
1314
|
+
tmp2 = fast_div_modulo(tmp, nsamples_y);
|
|
1315
|
+
const int wt = tmp2.y;
|
|
1316
|
+
const int it = tmp2.x;
|
|
1317
|
+
|
|
1318
|
+
if (!ids_dst) {
|
|
1319
|
+
const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst + it*I;
|
|
1320
|
+
dst += offset_dst;
|
|
1321
|
+
|
|
1322
|
+
const int i_max = nrows_x - it*I - 1;
|
|
1323
|
+
const int j_max = ncols_dst - jt*J - 1;
|
|
1324
|
+
if (fallback && i > i_max) {
|
|
1325
|
+
return;
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
#pragma unroll
|
|
1329
|
+
for (int j0 = 0; j0 < J; j0 += nwarps) {
|
|
1330
|
+
const int j = j0 + threadIdx.y;
|
|
1331
|
+
|
|
1332
|
+
if (j > j_max) {
|
|
1333
|
+
return;
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
dst[j*stride_col_dst + i] += sum[j0/nwarps];
|
|
1337
|
+
}
|
|
1338
|
+
return;
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
__shared__ int ids_dst_shared[J];
|
|
1342
|
+
const int col_low = expert_bounds[zt + 0];
|
|
1343
|
+
const int col_high = expert_bounds[zt + 1];
|
|
1344
|
+
const int col_diff = col_high - col_low;
|
|
1345
|
+
|
|
1346
|
+
for (int j = threadIdx.y*warp_size + threadIdx.x; j < J; j += nwarps*warp_size) {
|
|
1347
|
+
ids_dst_shared[j] = ids_dst[col_low + jt*J + j];
|
|
1348
|
+
}
|
|
1349
|
+
__syncthreads();
|
|
1350
|
+
|
|
1351
|
+
const int offset_dst = it*I;
|
|
1352
|
+
dst += offset_dst;
|
|
1353
|
+
|
|
1354
|
+
const int i_max = nrows_x - it*I - 1;
|
|
1355
|
+
const int j_max = col_diff - jt*J - 1;
|
|
1356
|
+
if (fallback && i > i_max) {
|
|
1357
|
+
return;
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
#pragma unroll
|
|
1361
|
+
for (int j0 = 0; j0 < J; j0 += nwarps) {
|
|
1362
|
+
const int j = j0 + threadIdx.y;
|
|
1363
|
+
|
|
1364
|
+
if (j > j_max) {
|
|
1365
|
+
return;
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
dst[ids_dst_shared[j]*stride_col_dst + i] += sum[j0/nwarps];
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
struct mmq_args {
|
|
1373
|
+
const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; float * dst;
|
|
1374
|
+
const float * y_scale;
|
|
1375
|
+
int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst;
|
|
1376
|
+
int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst;
|
|
1377
|
+
int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst;
|
|
1378
|
+
int64_t ncols_max;
|
|
1379
|
+
};
|
|
1380
|
+
|
|
1381
|
+
static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) {
|
|
1382
|
+
const size_t nbs_ids = config.J*sizeof(int);
|
|
1383
|
+
const size_t nbs_x = ggml_cuda_mmq_get_nbytes_shared_x(config, cc);
|
|
1384
|
+
const size_t nbs_y = config.J * (sizeof(block_q8_1_mmq));
|
|
1385
|
+
return nbs_ids + nbs_x + GGML_PAD(nbs_y, config.nthreads*sizeof(int));
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
template <ggml_type type, int J, bool fallback>
|
|
1389
|
+
static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
|
|
1390
|
+
const int id = ggml_cuda_get_device();
|
|
1391
|
+
const int cc = ggml_cuda_info().devices[id].cc;
|
|
1392
|
+
const int nsm = ggml_cuda_info().devices[id].nsm;
|
|
1393
|
+
const int warp_size = ggml_cuda_info().devices[id].warp_size;
|
|
1394
|
+
|
|
1395
|
+
const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc);
|
|
1396
|
+
GGML_ASSERT(config.nthreads % warp_size == 0);
|
|
1397
|
+
const int nwarps = config.nthreads / warp_size;
|
|
1398
|
+
const int nbytes_shared = mmq_get_nbytes_shared(config, cc);
|
|
1399
|
+
|
|
1400
|
+
const dim3 block_dims(warp_size, nwarps, 1);
|
|
1401
|
+
|
|
1402
|
+
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false>), nbytes_shared);
|
|
1403
|
+
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, true>), nbytes_shared);
|
|
1404
|
+
|
|
1405
|
+
const int nty = (args.nrows_x + config.I - 1) / config.I;
|
|
1406
|
+
const int ntx = (args.ncols_max + config.J - 1) / config.J;
|
|
1407
|
+
const int ntzw = args.nchannels_y * args.nsamples_y;
|
|
1408
|
+
const dim3 block_nums_xy_tiling(nty, ntx, ntzw);
|
|
1409
|
+
|
|
1410
|
+
GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0);
|
|
1411
|
+
GGML_ASSERT(args.nsamples_y % args.nsamples_x == 0);
|
|
1412
|
+
const int channel_ratio = args.nchannels_y / args.nchannels_x;
|
|
1413
|
+
const int sample_ratio = args.nsamples_y / args.nsamples_x;
|
|
1414
|
+
|
|
1415
|
+
const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits<type>::qk);
|
|
1416
|
+
const uint3 ntx_fd = init_fastdiv_values(ntx);
|
|
1417
|
+
const uint3 nchannels_y_fd = init_fastdiv_values(args.nchannels_y);
|
|
1418
|
+
const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y);
|
|
1419
|
+
const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio);
|
|
1420
|
+
const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio);
|
|
1421
|
+
|
|
1422
|
+
if (!ggml_cuda_mmq_get_stream_k(type, J, fallback, cc)) {
|
|
1423
|
+
mul_mat_q<type, J, fallback><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
|
|
1424
|
+
(args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale,
|
|
1425
|
+
blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
|
|
1426
|
+
channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
|
|
1427
|
+
sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
|
|
1428
|
+
ntx_fd);
|
|
1429
|
+
return;
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
// For the stream-k kernel it is possible to run it with tiling by setting the number of CUDA blocks equal to the number of tiles.
|
|
1433
|
+
// This is worthwhile if the efficiency of tiling is high and skipping the fixup kernel is more important.
|
|
1434
|
+
const int ntiles_dst = ntx * nty * ntzw;
|
|
1435
|
+
const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;
|
|
1436
|
+
const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);
|
|
1437
|
+
const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm, 1, 1);
|
|
1438
|
+
|
|
1439
|
+
GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.
|
|
1440
|
+
|
|
1441
|
+
const bool fixup_needed = ntiles_dst % block_nums_stream_k.x != 0;
|
|
1442
|
+
|
|
1443
|
+
ggml_cuda_pool & pool = ctx.pool(id);
|
|
1444
|
+
ggml_cuda_pool_alloc<float> tmp_fixup(pool);
|
|
1445
|
+
if (fixup_needed) {
|
|
1446
|
+
tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I);
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
const dim3 block_nums_fixup(block_nums_stream_k.x, config.I/warp_size, 1);
|
|
1450
|
+
const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);
|
|
1451
|
+
|
|
1452
|
+
mul_mat_q<type, J, fallback><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>
|
|
1453
|
+
(args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.y_scale,
|
|
1454
|
+
blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
|
|
1455
|
+
channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
|
|
1456
|
+
sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
|
|
1457
|
+
ntx_fd);
|
|
1458
|
+
|
|
1459
|
+
if (!fixup_needed) {
|
|
1460
|
+
return;
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1463
|
+
CUDA_CHECK(cudaGetLastError());
|
|
1464
|
+
mul_mat_q_stream_k_fixup<type, J, fallback><<<block_nums_fixup, block_dims_fixup, 0, stream>>>
|
|
1465
|
+
(args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,
|
|
1466
|
+
args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,
|
|
1467
|
+
ntx_fd);
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
template <ggml_type type, bool fallback>
|
|
1471
|
+
void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
|
|
1472
|
+
const int id = ggml_cuda_get_device();
|
|
1473
|
+
const int cc = ggml_cuda_info().devices[id].cc;
|
|
1474
|
+
const size_t smpbo = ggml_cuda_info().devices[id].smpbo;
|
|
1475
|
+
|
|
1476
|
+
int J_best = 0;
|
|
1477
|
+
int ntiles_J_best = INT_MAX;
|
|
1478
|
+
|
|
1479
|
+
for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) {
|
|
1480
|
+
const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc);
|
|
1481
|
+
if (config.type == GGML_TYPE_COUNT) {
|
|
1482
|
+
continue;
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
if (mmq_get_nbytes_shared(config, cc) > smpbo) {
|
|
1486
|
+
continue;
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1489
|
+
const int ntiles_x = (args.ncols_max + config.J - 1) / config.J;
|
|
1490
|
+
|
|
1491
|
+
if (ntiles_x < ntiles_J_best) {
|
|
1492
|
+
J_best = J;
|
|
1493
|
+
ntiles_J_best = ntiles_x;
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
switch (J_best) {
|
|
1498
|
+
case 8:
|
|
1499
|
+
launch_mul_mat_q<type, 8, fallback>(ctx, args, stream);
|
|
1500
|
+
break;
|
|
1501
|
+
case 16:
|
|
1502
|
+
launch_mul_mat_q<type, 16, fallback>(ctx, args, stream);
|
|
1503
|
+
break;
|
|
1504
|
+
case 24:
|
|
1505
|
+
launch_mul_mat_q<type, 24, fallback>(ctx, args, stream);
|
|
1506
|
+
break;
|
|
1507
|
+
case 32:
|
|
1508
|
+
launch_mul_mat_q<type, 32, fallback>(ctx, args, stream);
|
|
1509
|
+
break;
|
|
1510
|
+
case 40:
|
|
1511
|
+
launch_mul_mat_q<type, 40, fallback>(ctx, args, stream);
|
|
1512
|
+
break;
|
|
1513
|
+
case 48:
|
|
1514
|
+
launch_mul_mat_q<type, 48, fallback>(ctx, args, stream);
|
|
1515
|
+
break;
|
|
1516
|
+
case 56:
|
|
1517
|
+
launch_mul_mat_q<type, 56, fallback>(ctx, args, stream);
|
|
1518
|
+
break;
|
|
1519
|
+
case 64:
|
|
1520
|
+
launch_mul_mat_q<type, 64, fallback>(ctx, args, stream);
|
|
1521
|
+
break;
|
|
1522
|
+
case 72:
|
|
1523
|
+
launch_mul_mat_q<type, 72, fallback>(ctx, args, stream);
|
|
1524
|
+
break;
|
|
1525
|
+
case 80:
|
|
1526
|
+
launch_mul_mat_q<type, 80, fallback>(ctx, args, stream);
|
|
1527
|
+
break;
|
|
1528
|
+
case 88:
|
|
1529
|
+
launch_mul_mat_q<type, 88, fallback>(ctx, args, stream);
|
|
1530
|
+
break;
|
|
1531
|
+
case 96:
|
|
1532
|
+
launch_mul_mat_q<type, 96, fallback>(ctx, args, stream);
|
|
1533
|
+
break;
|
|
1534
|
+
case 104:
|
|
1535
|
+
launch_mul_mat_q<type, 104, fallback>(ctx, args, stream);
|
|
1536
|
+
break;
|
|
1537
|
+
case 112:
|
|
1538
|
+
launch_mul_mat_q<type, 112, fallback>(ctx, args, stream);
|
|
1539
|
+
break;
|
|
1540
|
+
case 120:
|
|
1541
|
+
launch_mul_mat_q<type, 120, fallback>(ctx, args, stream);
|
|
1542
|
+
break;
|
|
1543
|
+
case 128:
|
|
1544
|
+
launch_mul_mat_q<type, 128, fallback>(ctx, args, stream);
|
|
1545
|
+
break;
|
|
1546
|
+
default:
|
|
1547
|
+
fprintf(stderr, "J_best=%d\n", J_best);
|
|
1548
|
+
GGML_ABORT("fatal error");
|
|
1549
|
+
break;
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1553
|
+
template <ggml_type type>
|
|
1554
|
+
void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
|
|
1555
|
+
if (args.nrows_x % 128 == 0) {
|
|
1556
|
+
constexpr bool fallback = false;
|
|
1557
|
+
mul_mat_q_switch_J<type, fallback>(ctx, args, stream);
|
|
1558
|
+
} else {
|
|
1559
|
+
constexpr bool fallback = true;
|
|
1560
|
+
mul_mat_q_switch_J<type, fallback>(ctx, args, stream);
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
#define DECL_MMQ_CASE(type) \
|
|
1565
|
+
template void mul_mat_q_case<type>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \
|
|
1566
|
+
|
|
1567
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q1_0);
|
|
1568
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q2_0);
|
|
1569
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q4_0);
|
|
1570
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q4_1);
|
|
1571
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q5_0);
|
|
1572
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q5_1);
|
|
1573
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q8_0);
|
|
1574
|
+
// -----------------------------------------
|
|
1575
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q2_K);
|
|
1576
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);
|
|
1577
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);
|
|
1578
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);
|
|
1579
|
+
extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);
|
|
1580
|
+
// -----------------------------------------
|
|
1581
|
+
extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
|
|
1582
|
+
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
|
|
1583
|
+
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
|
|
1584
|
+
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);
|
|
1585
|
+
extern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);
|
|
1586
|
+
extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);
|
|
1587
|
+
extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);
|
|
1588
|
+
extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);
|
|
1589
|
+
// -----------------------------------------
|
|
1590
|
+
extern DECL_MMQ_CASE(GGML_TYPE_MXFP4);
|
|
1591
|
+
extern DECL_MMQ_CASE(GGML_TYPE_NVFP4);
|
|
1592
|
+
|
|
1593
|
+
// -------------------------------------------------------------------------------------------------------------------------
|
|
1594
|
+
|
|
1595
|
+
void ggml_cuda_mul_mat_q(
|
|
1596
|
+
ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);
|
|
1597
|
+
|
|
1598
|
+
bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts);
|