elelem-ggml 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/exe/elelem-ggml +8 -0
- data/ext/elelem/ggml/elelem.c +466 -0
- data/ext/elelem/ggml/extconf.rb +52 -0
- data/lib/elelem/ggml/chat_ml.rb +23 -0
- data/lib/elelem/ggml/gguf.rb +82 -0
- data/lib/elelem/ggml/provider.rb +63 -0
- data/lib/elelem/ggml/tokenizer.rb +95 -0
- data/lib/elelem/ggml/version.rb +7 -0
- data/lib/elelem/ggml.rb +37 -0
- data/vendor/ggml/AUTHORS +335 -0
- data/vendor/ggml/CMakeLists.txt +499 -0
- data/vendor/ggml/CONTRIBUTING.md +3 -0
- data/vendor/ggml/LICENSE +21 -0
- data/vendor/ggml/README.md +49 -0
- data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
- data/vendor/ggml/cmake/GitVars.cmake +22 -0
- data/vendor/ggml/cmake/common.cmake +50 -0
- data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
- data/vendor/ggml/ggml.pc.in +10 -0
- data/vendor/ggml/include/ggml-alloc.h +86 -0
- data/vendor/ggml/include/ggml-backend.h +437 -0
- data/vendor/ggml/include/ggml-blas.h +25 -0
- data/vendor/ggml/include/ggml-cann.h +123 -0
- data/vendor/ggml/include/ggml-cpp.h +39 -0
- data/vendor/ggml/include/ggml-cpu.h +152 -0
- data/vendor/ggml/include/ggml-cuda.h +47 -0
- data/vendor/ggml/include/ggml-et.h +28 -0
- data/vendor/ggml/include/ggml-hexagon.h +19 -0
- data/vendor/ggml/include/ggml-metal.h +61 -0
- data/vendor/ggml/include/ggml-opencl.h +26 -0
- data/vendor/ggml/include/ggml-openvino.h +37 -0
- data/vendor/ggml/include/ggml-opt.h +256 -0
- data/vendor/ggml/include/ggml-rpc.h +35 -0
- data/vendor/ggml/include/ggml-sycl.h +57 -0
- data/vendor/ggml/include/ggml-virtgpu.h +14 -0
- data/vendor/ggml/include/ggml-vulkan.h +29 -0
- data/vendor/ggml/include/ggml-webgpu.h +19 -0
- data/vendor/ggml/include/ggml-zdnn.h +17 -0
- data/vendor/ggml/include/ggml-zendnn.h +22 -0
- data/vendor/ggml/include/ggml.h +2963 -0
- data/vendor/ggml/include/gguf.h +211 -0
- data/vendor/ggml/requirements.txt +12 -0
- data/vendor/ggml/scripts/gen-authors.sh +9 -0
- data/vendor/ggml/scripts/make-release-checks.sh +96 -0
- data/vendor/ggml/scripts/make-release-desc.sh +69 -0
- data/vendor/ggml/scripts/make-release-summary.txt +33 -0
- data/vendor/ggml/scripts/release.sh +195 -0
- data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
- data/vendor/ggml/scripts/sync-llama.last +1 -0
- data/vendor/ggml/scripts/sync-llama.sh +21 -0
- data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
- data/vendor/ggml/scripts/sync-whisper.last +1 -0
- data/vendor/ggml/scripts/sync-whisper.sh +17 -0
- data/vendor/ggml/src/CMakeLists.txt +634 -0
- data/vendor/ggml/src/ggml-alloc.c +1249 -0
- data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
- data/vendor/ggml/src/ggml-backend-dl.h +45 -0
- data/vendor/ggml/src/ggml-backend-impl.h +291 -0
- data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
- data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
- data/vendor/ggml/src/ggml-backend.cpp +2517 -0
- data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
- data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
- data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
- data/vendor/ggml/src/ggml-cann/common.h +651 -0
- data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
- data/vendor/ggml/src/ggml-common.h +1911 -0
- data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
- data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
- data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
- data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
- data/vendor/ggml/src/ggml-cpu/common.h +95 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
- data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
- data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
- data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
- data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
- data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
- data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
- data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
- data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
- data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
- data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
- data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
- data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
- data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
- data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
- data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
- data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
- data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
- data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
- data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
- data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
- data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
- data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
- data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
- data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
- data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
- data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
- data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
- data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
- data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
- data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
- data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
- data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
- data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
- data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
- data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
- data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
- data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
- data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
- data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
- data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
- data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
- data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
- data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
- data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
- data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
- data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
- data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
- data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
- data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
- data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
- data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
- data/vendor/ggml/src/ggml-feats.h +166 -0
- data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
- data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
- data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
- data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
- data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
- data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
- data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
- data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
- data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
- data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
- data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
- data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
- data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
- data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
- data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
- data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
- data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
- data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
- data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
- data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
- data/vendor/ggml/src/ggml-impl.h +783 -0
- data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
- data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
- data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
- data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
- data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
- data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
- data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
- data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
- data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
- data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
- data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
- data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
- data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
- data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
- data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
- data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
- data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
- data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
- data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
- data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
- data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
- data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
- data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
- data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
- data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
- data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
- data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
- data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
- data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
- data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
- data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
- data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
- data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
- data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
- data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
- data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
- data/vendor/ggml/src/ggml-opt.cpp +1094 -0
- data/vendor/ggml/src/ggml-quants.c +5667 -0
- data/vendor/ggml/src/ggml-quants.h +115 -0
- data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
- data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
- data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
- data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
- data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
- data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
- data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
- data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
- data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
- data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
- data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
- data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
- data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
- data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
- data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
- data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
- data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
- data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
- data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
- data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
- data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
- data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
- data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
- data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
- data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
- data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
- data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
- data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
- data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
- data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
- data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
- data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
- data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
- data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
- data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
- data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
- data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
- data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
- data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
- data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
- data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
- data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
- data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
- data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
- data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
- data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
- data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
- data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
- data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
- data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/vendor/ggml/src/ggml-threading.cpp +12 -0
- data/vendor/ggml/src/ggml-threading.h +14 -0
- data/vendor/ggml/src/ggml-version.h.in +4 -0
- data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
- data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
- data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
- data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
- data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
- data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
- data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
- data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
- data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
- data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
- data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
- data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
- data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
- data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
- data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
- data/vendor/ggml/src/ggml.c +8088 -0
- data/vendor/ggml/src/ggml.cpp +26 -0
- data/vendor/ggml/src/gguf.cpp +1706 -0
- metadata +1459 -0
|
@@ -0,0 +1,3033 @@
|
|
|
1
|
+
#include "mmvq.hpp"
|
|
2
|
+
|
|
3
|
+
#include "ggml.h"
|
|
4
|
+
#include "common.hpp"
|
|
5
|
+
#include "element_wise.hpp"
|
|
6
|
+
#include "quants.hpp"
|
|
7
|
+
#include "vecdotq.hpp"
|
|
8
|
+
|
|
9
|
+
// Minimum weight-row count at which the Q4_K multi-column MMVQ kernel handles two output rows per
|
|
10
|
+
// subgroup (rows_per_sg == 2) instead of one, when ncols_dst == 2.
|
|
11
|
+
//
|
|
12
|
+
// Pairing rows lets a subgroup load each activation block once and apply it to two rows, at the cost
|
|
13
|
+
// of halving the number of subgroups in the launch. With only two destination columns there is too
|
|
14
|
+
// little work per row to hide that loss of parallelism, so pairing only pays off once there are
|
|
15
|
+
// enough rows to keep the device occupied. This is a measured performance crossover, not a
|
|
16
|
+
// correctness or hardware limit - both variants compute the same result for any nrows.
|
|
17
|
+
//
|
|
18
|
+
// Derived on Intel Arc Pro B70 with `test-backend-ops perf -o MUL_MAT` (Q4_K, ncols_dst == 2),
|
|
19
|
+
// sweeping nrows over 5120..6912 at ncols 17408 and 19968: one row per subgroup was up to 9% faster
|
|
20
|
+
// below the crossover, two rows per subgroup 8-15% faster above it, and the crossover fell inside
|
|
21
|
+
// (6144, 6272] for both ncols with no measurable ncols dependence. A later 32-row granularity sweep
|
|
22
|
+
// narrowed it to (6144, 6176], so 6272 is a conservative gate rather than the exact crossover.
|
|
23
|
+
// ncols_dst >= 3 amortizes the activation loads over more columns and is faster with two rows at
|
|
24
|
+
// every row count, so it does not consult this threshold.
|
|
25
|
+
static constexpr int Q4_K_MMVQ_ROW_PAIR_MIN_NROWS = 6272;
|
|
26
|
+
|
|
27
|
+
template <typename reorder_vec_dot_q_sycl>
|
|
28
|
+
static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,
|
|
29
|
+
const int ncols, const int nrows, const sycl::nd_item<3> & nd_item) {
|
|
30
|
+
using block_type = ggml_sycl_reordered::block_q_t<reorder_vec_dot_q_sycl::gtype>;
|
|
31
|
+
using block_traits = typename block_type::traits;
|
|
32
|
+
|
|
33
|
+
const auto sg = nd_item.get_sub_group();
|
|
34
|
+
const int sg_range = sg.get_group_linear_range();
|
|
35
|
+
const int workgroup_id = nd_item.get_group_linear_id();
|
|
36
|
+
const int sg_id = sg.get_group_linear_id();
|
|
37
|
+
const int row = workgroup_id * sg_range + sg_id;
|
|
38
|
+
|
|
39
|
+
if (row >= nrows) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const int blocks_per_row = ncols / block_traits::qk;
|
|
44
|
+
constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);
|
|
45
|
+
constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;
|
|
46
|
+
const int nblocks = nrows * (ncols / block_traits::qk);
|
|
47
|
+
|
|
48
|
+
static_assert(blocks_per_subgroup > 0);
|
|
49
|
+
static_assert(block_elements_per_subgroup > 0);
|
|
50
|
+
|
|
51
|
+
float partial_sum = 0.0f;
|
|
52
|
+
for (int i = sg.get_local_linear_id() / block_elements_per_subgroup; i < blocks_per_row; i += blocks_per_subgroup) {
|
|
53
|
+
const int ibx = row * blocks_per_row + i; // x block index
|
|
54
|
+
|
|
55
|
+
const auto bx_offset = block_type::get_block_offset(ibx, nblocks);
|
|
56
|
+
const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);
|
|
57
|
+
// Y block index that aligns with ibx
|
|
58
|
+
const int iby = i * block_type::block_to_q8_1_ratio();
|
|
59
|
+
const int8_t* q8_1_quant_ptr = (const int8_t*)vy + iby * QK8_1;
|
|
60
|
+
const sycl::half2* q8_1_ds_ptr = (const sycl::half2*)((const char*)vy + ncols + iby * sizeof(sycl::half2));
|
|
61
|
+
|
|
62
|
+
#pragma unroll
|
|
63
|
+
for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {
|
|
64
|
+
// x block quant index when casting the quants to int
|
|
65
|
+
const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
|
|
66
|
+
|
|
67
|
+
partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
auto sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum, std::plus<>());
|
|
72
|
+
|
|
73
|
+
if (sg.leader()) {
|
|
74
|
+
dst[row] = sum;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// With has_fusion, `vgate` is a second weight matrix sharing vx's shape, stride and reorder
|
|
79
|
+
// layout: one pass computes both row dot products and the epilogue writes glu(gate, up).
|
|
80
|
+
template <typename reorder_vec_dot_q_sycl, int ncols_dst, bool has_fusion = false, int rows_per_sg = 1>
|
|
81
|
+
static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void * __restrict__ vgate,
|
|
82
|
+
const void * __restrict__ vy, float * __restrict__ dst, const int ncols,
|
|
83
|
+
const int nrows, const int stride_col_y_bytes, const int stride_col_dst,
|
|
84
|
+
const ggml_glu_op glu_op, const sycl::nd_item<3> & nd_item) {
|
|
85
|
+
using block_type = ggml_sycl_reordered::block_q_t<reorder_vec_dot_q_sycl::gtype>;
|
|
86
|
+
using block_traits = typename block_type::traits;
|
|
87
|
+
|
|
88
|
+
const auto sg = nd_item.get_sub_group();
|
|
89
|
+
const int sg_range = sg.get_group_linear_range();
|
|
90
|
+
const int workgroup_id = nd_item.get_group_linear_id();
|
|
91
|
+
const int sg_id = sg.get_group_linear_id();
|
|
92
|
+
const int row0 = (workgroup_id * sg_range + sg_id) * rows_per_sg;
|
|
93
|
+
|
|
94
|
+
// row is sub-group uniform, so this retires whole sub-groups and the collectives below
|
|
95
|
+
// stay convergent
|
|
96
|
+
if (row0 >= nrows) {
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
static_assert(rows_per_sg == 1 ||
|
|
101
|
+
reorder_vec_dot_shared_activations<reorder_vec_dot_q_sycl::gtype>::value);
|
|
102
|
+
|
|
103
|
+
const int blocks_per_row = ncols / block_traits::qk;
|
|
104
|
+
constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);
|
|
105
|
+
constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;
|
|
106
|
+
const int nblocks = nrows * (ncols / block_traits::qk);
|
|
107
|
+
|
|
108
|
+
static_assert(blocks_per_subgroup > 0);
|
|
109
|
+
static_assert(block_elements_per_subgroup > 0);
|
|
110
|
+
|
|
111
|
+
float partial_sum[ncols_dst][rows_per_sg] = {};
|
|
112
|
+
// sized 1 rather than 0 when unused: zero-length arrays are not standard C++, and the
|
|
113
|
+
// array is dead and eliminated in that case
|
|
114
|
+
[[maybe_unused]] float partial_gate[has_fusion ? ncols_dst : 1][has_fusion ? rows_per_sg : 1] = {};
|
|
115
|
+
for (int i = sg.get_local_linear_id() / block_elements_per_subgroup; i < blocks_per_row; i += blocks_per_subgroup) {
|
|
116
|
+
const int iby = i * block_type::block_to_q8_1_ratio();
|
|
117
|
+
|
|
118
|
+
#pragma unroll
|
|
119
|
+
for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {
|
|
120
|
+
const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
|
|
121
|
+
|
|
122
|
+
if constexpr (rows_per_sg > 1) {
|
|
123
|
+
typename reorder_vec_dot_q_sycl::weights wx[rows_per_sg];
|
|
124
|
+
[[maybe_unused]] typename reorder_vec_dot_q_sycl::weights wg[rows_per_sg];
|
|
125
|
+
#pragma unroll
|
|
126
|
+
for (int r = 0; r < rows_per_sg; ++r) {
|
|
127
|
+
const int row = sycl::min(row0 + r, nrows - 1);
|
|
128
|
+
const int ibx = row * blocks_per_row + i;
|
|
129
|
+
const auto bx_offset = block_type::get_block_offset(ibx, nblocks);
|
|
130
|
+
const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);
|
|
131
|
+
wx[r] = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs);
|
|
132
|
+
if constexpr (has_fusion) {
|
|
133
|
+
wg[r] = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
#pragma unroll
|
|
137
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
138
|
+
const char * vy_j = (const char *) vy + j * stride_col_y_bytes;
|
|
139
|
+
const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;
|
|
140
|
+
const sycl::half2 * q8_1_ds_ptr =
|
|
141
|
+
(const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));
|
|
142
|
+
const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs);
|
|
143
|
+
#pragma unroll
|
|
144
|
+
for (int r = 0; r < rows_per_sg; ++r) {
|
|
145
|
+
partial_sum[j][r] += reorder_vec_dot_q_sycl::apply(wx[r], a);
|
|
146
|
+
if constexpr (has_fusion) {
|
|
147
|
+
partial_gate[j][r] += reorder_vec_dot_q_sycl::apply(wg[r], a);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
} else if constexpr (reorder_vec_dot_shared_weights<reorder_vec_dot_q_sycl::gtype>::value) {
|
|
152
|
+
const int ibx = row0 * blocks_per_row + i;
|
|
153
|
+
const auto bx_offset = block_type::get_block_offset(ibx, nblocks);
|
|
154
|
+
const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);
|
|
155
|
+
const auto wx = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs);
|
|
156
|
+
if constexpr (has_fusion) {
|
|
157
|
+
const auto wg = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs);
|
|
158
|
+
|
|
159
|
+
#pragma unroll
|
|
160
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
161
|
+
const char * vy_j = (const char *) vy + j * stride_col_y_bytes;
|
|
162
|
+
const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;
|
|
163
|
+
const sycl::half2 * q8_1_ds_ptr =
|
|
164
|
+
(const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));
|
|
165
|
+
|
|
166
|
+
// up and gate share the activation, so load it once and apply it twice
|
|
167
|
+
const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs);
|
|
168
|
+
|
|
169
|
+
partial_sum[j][0] += reorder_vec_dot_q_sycl::apply(wx, a);
|
|
170
|
+
partial_gate[j][0] += reorder_vec_dot_q_sycl::apply(wg, a);
|
|
171
|
+
}
|
|
172
|
+
} else {
|
|
173
|
+
#pragma unroll
|
|
174
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
175
|
+
const char * vy_j = (const char *) vy + j * stride_col_y_bytes;
|
|
176
|
+
const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;
|
|
177
|
+
const sycl::half2 * q8_1_ds_ptr =
|
|
178
|
+
(const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));
|
|
179
|
+
|
|
180
|
+
partial_sum[j][0] += reorder_vec_dot_q_sycl::dot(wx, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
} else {
|
|
184
|
+
const int ibx = row0 * blocks_per_row + i;
|
|
185
|
+
const auto bx_offset = block_type::get_block_offset(ibx, nblocks);
|
|
186
|
+
const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);
|
|
187
|
+
#pragma unroll
|
|
188
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
189
|
+
const char * vy_j = (const char *) vy + j * stride_col_y_bytes;
|
|
190
|
+
const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1;
|
|
191
|
+
const sycl::half2 * q8_1_ds_ptr =
|
|
192
|
+
(const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2));
|
|
193
|
+
|
|
194
|
+
partial_sum[j][0] +=
|
|
195
|
+
reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
|
|
196
|
+
|
|
197
|
+
if constexpr (has_fusion) {
|
|
198
|
+
partial_gate[j][0] +=
|
|
199
|
+
reorder_vec_dot_q_sycl()(vgate, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
#pragma unroll
|
|
207
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
208
|
+
#pragma unroll
|
|
209
|
+
for (int r = 0; r < rows_per_sg; ++r) {
|
|
210
|
+
float sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum[j][r], std::plus<>());
|
|
211
|
+
|
|
212
|
+
if constexpr (has_fusion) {
|
|
213
|
+
const float gate = sycl::reduce_over_group(nd_item.get_sub_group(), partial_gate[j][r], std::plus<>());
|
|
214
|
+
|
|
215
|
+
// uniform across the launch; the launcher only instantiates SWIGLU and GEGLU
|
|
216
|
+
sum *= glu_op == GGML_GLU_OP_SWIGLU ? op_silu(gate) : op_gelu(gate);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
if (sg.leader() && row0 + r < nrows) {
|
|
220
|
+
dst[j * stride_col_dst + row0 + r] = sum;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl>
|
|
227
|
+
static void mul_mat_vec_q(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst,
|
|
228
|
+
const int ncols, const int nrows, const sycl::nd_item<3> & item_ct1) {
|
|
229
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1);
|
|
230
|
+
|
|
231
|
+
if (row >= nrows) {
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const int blocks_per_row = ncols / qk;
|
|
236
|
+
constexpr int blocks_per_warp = (vdr * WARP_SIZE + qi - 1) / qi; // Ensuring blocks_per_warp > 0
|
|
237
|
+
|
|
238
|
+
assert(blocks_per_warp > 0);
|
|
239
|
+
|
|
240
|
+
// partial sum for each thread
|
|
241
|
+
float tmp = 0.0f;
|
|
242
|
+
|
|
243
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
244
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
245
|
+
|
|
246
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) {
|
|
247
|
+
const int ibx = row * blocks_per_row + i; // x block index
|
|
248
|
+
|
|
249
|
+
const int iby = i * (qk / QK8_1); // y block index that aligns with ibx
|
|
250
|
+
|
|
251
|
+
for (size_t elem = 0; elem < qi / vdr; elem += WARP_SIZE) {
|
|
252
|
+
const int iqs = elem + vdr * (item_ct1.get_local_id(2) %
|
|
253
|
+
(qi / vdr)); // x block quant index when casting the quants to int
|
|
254
|
+
|
|
255
|
+
tmp += vec_dot_q_sycl(&x[ibx], &y[iby], iqs);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// sum up partial sums and write back result
|
|
260
|
+
#pragma unroll
|
|
261
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
262
|
+
tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
266
|
+
dst[row] = tmp;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
template <int qk, int qi, typename block_q_t, int vdr,
|
|
271
|
+
vec_dot_q_sycl_t vec_dot_q_sycl, int ncols_dst>
|
|
272
|
+
static void mul_mat_vec_q_ncols(
|
|
273
|
+
const void * __restrict__ vx,
|
|
274
|
+
const void * __restrict__ vy,
|
|
275
|
+
float * __restrict__ dst,
|
|
276
|
+
const int ncols,
|
|
277
|
+
const int nrows,
|
|
278
|
+
const int stride_col_y,
|
|
279
|
+
const int stride_col_dst,
|
|
280
|
+
const sycl::nd_item<3> & item_ct1) {
|
|
281
|
+
|
|
282
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1)
|
|
283
|
+
+ item_ct1.get_local_id(1);
|
|
284
|
+
|
|
285
|
+
if (row >= nrows) {
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
const int blocks_per_row = ncols / qk;
|
|
290
|
+
constexpr int blocks_per_warp = (vdr * WARP_SIZE + qi - 1) / qi;
|
|
291
|
+
|
|
292
|
+
// partial sums: one per output column
|
|
293
|
+
float tmp[ncols_dst] = {0.0f};
|
|
294
|
+
|
|
295
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
296
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
297
|
+
|
|
298
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr);
|
|
299
|
+
i < blocks_per_row;
|
|
300
|
+
i += blocks_per_warp) {
|
|
301
|
+
|
|
302
|
+
const int ibx = row * blocks_per_row + i;
|
|
303
|
+
const int iby = i * (qk / QK8_1);
|
|
304
|
+
|
|
305
|
+
// read weight block once, dot against all columns
|
|
306
|
+
for (size_t elem = 0; elem < qi / vdr; elem += WARP_SIZE) {
|
|
307
|
+
const int iqs = elem + vdr * (item_ct1.get_local_id(2) % (qi / vdr));
|
|
308
|
+
|
|
309
|
+
#pragma unroll
|
|
310
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
311
|
+
tmp[j] += vec_dot_q_sycl(&x[ibx], &y[j * stride_col_y + iby], iqs);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// reduce within subgroup
|
|
317
|
+
#pragma unroll
|
|
318
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
319
|
+
#pragma unroll
|
|
320
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
321
|
+
tmp[j] += dpct::permute_sub_group_by_xor(
|
|
322
|
+
item_ct1.get_sub_group(), tmp[j], mask);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
327
|
+
#pragma unroll
|
|
328
|
+
for (int j = 0; j < ncols_dst; ++j) {
|
|
329
|
+
dst[j * stride_col_dst + row] = tmp[j];
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
template <int qk, int qi, typename block_q_t, int vdr>
|
|
335
|
+
static void mul_mat_vec_q_iq2_xxs_q8_1(const void *__restrict__ vx,
|
|
336
|
+
const void *__restrict__ vy,
|
|
337
|
+
float *__restrict__ dst, const int ncols,
|
|
338
|
+
const int nrows,
|
|
339
|
+
const sycl::nd_item<3> &item_ct1) {
|
|
340
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
|
|
341
|
+
item_ct1.get_local_id(1);
|
|
342
|
+
|
|
343
|
+
if (row >= nrows) {
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
const int blocks_per_row = ncols / qk;
|
|
348
|
+
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
|
349
|
+
assert(blocks_per_warp>0);
|
|
350
|
+
|
|
351
|
+
// partial sum for each thread
|
|
352
|
+
float tmp = 0.0f;
|
|
353
|
+
|
|
354
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
355
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
356
|
+
|
|
357
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
|
|
358
|
+
i += blocks_per_warp) {
|
|
359
|
+
const int ibx = row*blocks_per_row + i; // x block index
|
|
360
|
+
|
|
361
|
+
const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
|
|
362
|
+
|
|
363
|
+
const int iqs =
|
|
364
|
+
vdr *
|
|
365
|
+
(item_ct1.get_local_id(2) %
|
|
366
|
+
(qi / vdr)); // x block quant index when casting the quants to int
|
|
367
|
+
|
|
368
|
+
tmp += vec_dot_iq2_xxs_q8_1(&x[ibx], &y[iby], iqs, iq2xxs_grid, ksigns_iq2xs, kmask_iq2xs);
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// sum up partial sums and write back result
|
|
372
|
+
#pragma unroll
|
|
373
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
374
|
+
tmp +=
|
|
375
|
+
dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
379
|
+
dst[row] = tmp;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
template <int qk, int qi, typename block_q_t, int vdr>
|
|
384
|
+
static void mul_mat_vec_q_iq2_xs_q8_1(const void *__restrict__ vx,
|
|
385
|
+
const void *__restrict__ vy,
|
|
386
|
+
float *__restrict__ dst, const int ncols,
|
|
387
|
+
const int nrows,
|
|
388
|
+
const sycl::nd_item<3> &item_ct1) {
|
|
389
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
|
|
390
|
+
item_ct1.get_local_id(1);
|
|
391
|
+
|
|
392
|
+
if (row >= nrows) {
|
|
393
|
+
return;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
const int blocks_per_row = ncols / qk;
|
|
397
|
+
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
|
398
|
+
assert(blocks_per_warp>0);
|
|
399
|
+
// partial sum for each thread
|
|
400
|
+
float tmp = 0.0f;
|
|
401
|
+
|
|
402
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
403
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
404
|
+
|
|
405
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
|
|
406
|
+
i += blocks_per_warp) {
|
|
407
|
+
const int ibx = row*blocks_per_row + i; // x block index
|
|
408
|
+
|
|
409
|
+
const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
|
|
410
|
+
|
|
411
|
+
const int iqs =
|
|
412
|
+
vdr *
|
|
413
|
+
(item_ct1.get_local_id(2) %
|
|
414
|
+
(qi / vdr)); // x block quant index when casting the quants to int
|
|
415
|
+
|
|
416
|
+
tmp += vec_dot_iq2_xs_q8_1(&x[ibx], &y[iby], iqs, iq2xs_grid, ksigns64);
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// sum up partial sums and write back result
|
|
420
|
+
#pragma unroll
|
|
421
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
422
|
+
tmp +=
|
|
423
|
+
dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
427
|
+
dst[row] = tmp;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
template <int qk, int qi, typename block_q_t, int vdr>
|
|
432
|
+
static void mul_mat_vec_q_iq2_s_q8_1(const void *__restrict__ vx,
|
|
433
|
+
const void *__restrict__ vy,
|
|
434
|
+
float *__restrict__ dst, const int ncols,
|
|
435
|
+
const int nrows,
|
|
436
|
+
const sycl::nd_item<3> &item_ct1) {
|
|
437
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
|
|
438
|
+
item_ct1.get_local_id(1);
|
|
439
|
+
|
|
440
|
+
if (row >= nrows) {
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
const int blocks_per_row = ncols / qk;
|
|
445
|
+
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
|
446
|
+
assert(blocks_per_warp>0);
|
|
447
|
+
// partial sum for each thread
|
|
448
|
+
float tmp = 0.0f;
|
|
449
|
+
|
|
450
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
451
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
452
|
+
|
|
453
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
|
|
454
|
+
i += blocks_per_warp) {
|
|
455
|
+
const int ibx = row*blocks_per_row + i; // x block index
|
|
456
|
+
|
|
457
|
+
const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
|
|
458
|
+
|
|
459
|
+
const int iqs =
|
|
460
|
+
vdr *
|
|
461
|
+
(item_ct1.get_local_id(2) %
|
|
462
|
+
(qi / vdr)); // x block quant index when casting the quants to int
|
|
463
|
+
|
|
464
|
+
tmp += vec_dot_iq2_s_q8_1(&x[ibx], &y[iby], iqs);
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
// sum up partial sums and write back result
|
|
468
|
+
#pragma unroll
|
|
469
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
470
|
+
tmp +=
|
|
471
|
+
dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
475
|
+
dst[row] = tmp;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
template <int qk, int qi, typename block_q_t, int vdr>
|
|
480
|
+
static void mul_mat_vec_q_iq3_xxs_q8_1(const void *__restrict__ vx,
|
|
481
|
+
const void *__restrict__ vy,
|
|
482
|
+
float *__restrict__ dst, const int ncols,
|
|
483
|
+
const int nrows,
|
|
484
|
+
const sycl::nd_item<3> &item_ct1) {
|
|
485
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
|
|
486
|
+
item_ct1.get_local_id(1);
|
|
487
|
+
|
|
488
|
+
if (row >= nrows) {
|
|
489
|
+
return;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
const int blocks_per_row = ncols / qk;
|
|
493
|
+
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
|
494
|
+
assert(blocks_per_warp>0);
|
|
495
|
+
// partial sum for each thread
|
|
496
|
+
float tmp = 0.0f;
|
|
497
|
+
|
|
498
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
499
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
500
|
+
|
|
501
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
|
|
502
|
+
i += blocks_per_warp) {
|
|
503
|
+
const int ibx = row*blocks_per_row + i; // x block index
|
|
504
|
+
|
|
505
|
+
const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
|
|
506
|
+
|
|
507
|
+
const int iqs =
|
|
508
|
+
vdr *
|
|
509
|
+
(item_ct1.get_local_id(2) %
|
|
510
|
+
(qi / vdr)); // x block quant index when casting the quants to int
|
|
511
|
+
|
|
512
|
+
tmp += vec_dot_iq3_xxs_q8_1(&x[ibx], &y[iby], iqs, iq3xxs_grid, ksigns64);
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
// sum up partial sums and write back result
|
|
516
|
+
#pragma unroll
|
|
517
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
518
|
+
tmp +=
|
|
519
|
+
dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
523
|
+
dst[row] = tmp;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
template <int qk, int qi, typename block_q_t, int vdr>
|
|
528
|
+
static void mul_mat_vec_q_iq3_s_q8_1(const void *__restrict__ vx,
|
|
529
|
+
const void *__restrict__ vy,
|
|
530
|
+
float *__restrict__ dst, const int ncols,
|
|
531
|
+
const int nrows,
|
|
532
|
+
const sycl::nd_item<3> &item_ct1) {
|
|
533
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
|
|
534
|
+
item_ct1.get_local_id(1);
|
|
535
|
+
|
|
536
|
+
if (row >= nrows) {
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
const int blocks_per_row = ncols / qk;
|
|
541
|
+
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
|
542
|
+
assert(blocks_per_warp>0);
|
|
543
|
+
// partial sum for each thread
|
|
544
|
+
float tmp = 0.0f;
|
|
545
|
+
|
|
546
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
547
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
548
|
+
|
|
549
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
|
|
550
|
+
i += blocks_per_warp) {
|
|
551
|
+
const int ibx = row*blocks_per_row + i; // x block index
|
|
552
|
+
|
|
553
|
+
const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
|
|
554
|
+
|
|
555
|
+
const int iqs =
|
|
556
|
+
vdr *
|
|
557
|
+
(item_ct1.get_local_id(2) %
|
|
558
|
+
(qi / vdr)); // x block quant index when casting the quants to int
|
|
559
|
+
|
|
560
|
+
tmp += vec_dot_iq3_s_q8_1(&x[ibx], &y[iby], iqs, iq3s_grid);
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
// sum up partial sums and write back result
|
|
564
|
+
#pragma unroll
|
|
565
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
566
|
+
tmp +=
|
|
567
|
+
dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
571
|
+
dst[row] = tmp;
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
template <int qk, int qi, typename block_q_t, int vdr>
|
|
576
|
+
static void mul_mat_vec_q_iq1_s_q8_1(const void *__restrict__ vx,
|
|
577
|
+
const void *__restrict__ vy,
|
|
578
|
+
float *__restrict__ dst, const int ncols,
|
|
579
|
+
const int nrows,
|
|
580
|
+
const sycl::nd_item<3> &item_ct1) {
|
|
581
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
|
|
582
|
+
item_ct1.get_local_id(1);
|
|
583
|
+
|
|
584
|
+
if (row >= nrows) {
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
const int blocks_per_row = ncols / qk;
|
|
589
|
+
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
|
590
|
+
assert(blocks_per_warp>0);
|
|
591
|
+
// partial sum for each thread
|
|
592
|
+
float tmp = 0.0f;
|
|
593
|
+
|
|
594
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
595
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
596
|
+
|
|
597
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
|
|
598
|
+
i += blocks_per_warp) {
|
|
599
|
+
const int ibx = row*blocks_per_row + i; // x block index
|
|
600
|
+
|
|
601
|
+
const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
|
|
602
|
+
|
|
603
|
+
const int iqs =
|
|
604
|
+
vdr *
|
|
605
|
+
(item_ct1.get_local_id(2) %
|
|
606
|
+
(qi / vdr)); // x block quant index when casting the quants to int
|
|
607
|
+
|
|
608
|
+
tmp += vec_dot_iq1_s_q8_1(&x[ibx], &y[iby], iqs, iq1s_grid_gpu);
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
// sum up partial sums and write back result
|
|
612
|
+
#pragma unroll
|
|
613
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
614
|
+
tmp +=
|
|
615
|
+
dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
619
|
+
dst[row] = tmp;
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
template <int qk, int qi, typename block_q_t, int vdr>
|
|
624
|
+
static void mul_mat_vec_q_iq1_m_q8_1(const void *__restrict__ vx,
|
|
625
|
+
const void *__restrict__ vy,
|
|
626
|
+
float *__restrict__ dst, const int ncols,
|
|
627
|
+
const int nrows,
|
|
628
|
+
const sycl::nd_item<3> &item_ct1) {
|
|
629
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
|
|
630
|
+
item_ct1.get_local_id(1);
|
|
631
|
+
|
|
632
|
+
if (row >= nrows) {
|
|
633
|
+
return;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
const int blocks_per_row = ncols / qk;
|
|
637
|
+
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
|
638
|
+
assert(blocks_per_warp>0);
|
|
639
|
+
// partial sum for each thread
|
|
640
|
+
float tmp = 0.0f;
|
|
641
|
+
|
|
642
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
643
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
644
|
+
|
|
645
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
|
|
646
|
+
i += blocks_per_warp) {
|
|
647
|
+
const int ibx = row*blocks_per_row + i; // x block index
|
|
648
|
+
|
|
649
|
+
const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
|
|
650
|
+
|
|
651
|
+
const int iqs =
|
|
652
|
+
vdr *
|
|
653
|
+
(item_ct1.get_local_id(2) %
|
|
654
|
+
(qi / vdr)); // x block quant index when casting the quants to int
|
|
655
|
+
|
|
656
|
+
tmp += vec_dot_iq1_m_q8_1(&x[ibx], &y[iby], iqs);
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
// sum up partial sums and write back result
|
|
660
|
+
#pragma unroll
|
|
661
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
662
|
+
tmp +=
|
|
663
|
+
dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
667
|
+
dst[row] = tmp;
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
template <int qk, int qi, typename block_q_t, int vdr>
|
|
672
|
+
static void mul_mat_vec_q_iq4_nl_q8_1(const void *__restrict__ vx,
|
|
673
|
+
const void *__restrict__ vy,
|
|
674
|
+
float *__restrict__ dst, const int ncols,
|
|
675
|
+
const int nrows,
|
|
676
|
+
const sycl::nd_item<3> &item_ct1) {
|
|
677
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
|
|
678
|
+
item_ct1.get_local_id(1);
|
|
679
|
+
|
|
680
|
+
if (row >= nrows) {
|
|
681
|
+
return;
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
const int blocks_per_row = ncols / qk;
|
|
685
|
+
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
|
686
|
+
assert(blocks_per_warp>0);
|
|
687
|
+
// partial sum for each thread
|
|
688
|
+
float tmp = 0.0f;
|
|
689
|
+
|
|
690
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
691
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
692
|
+
|
|
693
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
|
|
694
|
+
i += blocks_per_warp) {
|
|
695
|
+
const int ibx = row*blocks_per_row + i; // x block index
|
|
696
|
+
|
|
697
|
+
const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
|
|
698
|
+
|
|
699
|
+
const int iqs =
|
|
700
|
+
vdr *
|
|
701
|
+
(item_ct1.get_local_id(2) %
|
|
702
|
+
(qi / vdr)); // x block quant index when casting the quants to int
|
|
703
|
+
|
|
704
|
+
tmp += vec_dot_iq4_nl_q8_1(&x[ibx], &y[iby], iqs);
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
// sum up partial sums and write back result
|
|
708
|
+
#pragma unroll
|
|
709
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
710
|
+
tmp +=
|
|
711
|
+
dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
715
|
+
dst[row] = tmp;
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
template <int qk, int qi, typename block_q_t, int vdr>
|
|
721
|
+
static void mul_mat_vec_q_iq4_xs_q8_1(const void *__restrict__ vx,
|
|
722
|
+
const void *__restrict__ vy,
|
|
723
|
+
float *__restrict__ dst, const int ncols,
|
|
724
|
+
const int nrows,
|
|
725
|
+
const sycl::nd_item<3> &item_ct1) {
|
|
726
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) +
|
|
727
|
+
item_ct1.get_local_id(1);
|
|
728
|
+
|
|
729
|
+
if (row >= nrows) {
|
|
730
|
+
return;
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
const int blocks_per_row = ncols / qk;
|
|
734
|
+
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
|
735
|
+
assert(blocks_per_warp>0);
|
|
736
|
+
// partial sum for each thread
|
|
737
|
+
float tmp = 0.0f;
|
|
738
|
+
|
|
739
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
740
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
741
|
+
|
|
742
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row;
|
|
743
|
+
i += blocks_per_warp) {
|
|
744
|
+
const int ibx = row*blocks_per_row + i; // x block index
|
|
745
|
+
|
|
746
|
+
const int iby = i * (qk/QK8_1); // y block index that aligns with ibx
|
|
747
|
+
|
|
748
|
+
const int iqs =
|
|
749
|
+
vdr *
|
|
750
|
+
(item_ct1.get_local_id(2) %
|
|
751
|
+
(qi / vdr)); // x block quant index when casting the quants to int
|
|
752
|
+
|
|
753
|
+
tmp += vec_dot_iq4_xs_q8_1(&x[ibx], &y[iby], iqs);
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
// sum up partial sums and write back result
|
|
757
|
+
#pragma unroll
|
|
758
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
759
|
+
tmp +=
|
|
760
|
+
dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
764
|
+
dst[row] = tmp;
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
static void reorder_mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
|
|
769
|
+
const int nrows, dpct::queue_ptr stream) {
|
|
770
|
+
GGML_ASSERT(ncols % QK4_0 == 0);
|
|
771
|
+
// Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
|
|
772
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
773
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
774
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
775
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
776
|
+
|
|
777
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
778
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
779
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
780
|
+
mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_0>>(vx, vy, dst, ncols, nrows,
|
|
781
|
+
nd_item);
|
|
782
|
+
});
|
|
783
|
+
});
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
template <int ncols_dst>
|
|
787
|
+
static void reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols(
|
|
788
|
+
const void * vx, const void * vy, float * dst,
|
|
789
|
+
const int ncols, const int nrows,
|
|
790
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
791
|
+
dpct::queue_ptr stream) {
|
|
792
|
+
GGML_ASSERT(ncols % QK4_0 == 0);
|
|
793
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
794
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
795
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
796
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
797
|
+
|
|
798
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
799
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
800
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
801
|
+
mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_0>, ncols_dst>(
|
|
802
|
+
vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
|
|
803
|
+
/*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
|
|
804
|
+
});
|
|
805
|
+
});
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
static void reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(
|
|
809
|
+
const void * vx, const void * vy, float * dst,
|
|
810
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
811
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
812
|
+
dpct::queue_ptr stream) {
|
|
813
|
+
switch (ncols_dst) {
|
|
814
|
+
case 1: reorder_mul_mat_vec_q4_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
815
|
+
case 2: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
816
|
+
case 3: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
817
|
+
case 4: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
818
|
+
case 5: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
819
|
+
case 6: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
820
|
+
case 7: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
821
|
+
case 8: reorder_mul_mat_vec_q4_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
822
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q4_0 reorder multi-col MMVQ", ncols_dst);
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
static void mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows,
|
|
827
|
+
dpct::queue_ptr stream) {
|
|
828
|
+
GGML_ASSERT(ncols % QK4_0 == 0);
|
|
829
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
830
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
831
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
832
|
+
|
|
833
|
+
{
|
|
834
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
835
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
836
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
837
|
+
mul_mat_vec_q<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>(
|
|
838
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
839
|
+
});
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
template <int ncols_dst>
|
|
845
|
+
static void mul_mat_vec_q4_0_q8_1_sycl_ncols(
|
|
846
|
+
const void * vx, const void * vy, float * dst,
|
|
847
|
+
const int ncols, const int nrows,
|
|
848
|
+
const int stride_col_y, const int stride_col_dst,
|
|
849
|
+
dpct::queue_ptr stream) {
|
|
850
|
+
GGML_ASSERT(ncols % QK4_0 == 0);
|
|
851
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
852
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
853
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
854
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
855
|
+
cgh.parallel_for(
|
|
856
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
857
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
858
|
+
mul_mat_vec_q_ncols<QK4_0, QI4_0, block_q4_0,
|
|
859
|
+
VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1, ncols_dst>(
|
|
860
|
+
vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
|
|
861
|
+
});
|
|
862
|
+
});
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
static void mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(
|
|
866
|
+
const void * vx, const void * vy, float * dst,
|
|
867
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
868
|
+
const int stride_col_y, const int stride_col_dst,
|
|
869
|
+
dpct::queue_ptr stream) {
|
|
870
|
+
switch (ncols_dst) {
|
|
871
|
+
case 1: mul_mat_vec_q4_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
872
|
+
case 2: mul_mat_vec_q4_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
873
|
+
case 3: mul_mat_vec_q4_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
874
|
+
case 4: mul_mat_vec_q4_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
875
|
+
case 5: mul_mat_vec_q4_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
876
|
+
case 6: mul_mat_vec_q4_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
877
|
+
case 7: mul_mat_vec_q4_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
878
|
+
case 8: mul_mat_vec_q4_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
879
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q4_0 multi-col MMVQ", ncols_dst);
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
static void mul_mat_vec_q4_1_q8_1_sycl(const void *vx, const void *vy,
|
|
884
|
+
float *dst, const int ncols,
|
|
885
|
+
const int nrows,
|
|
886
|
+
dpct::queue_ptr stream) {
|
|
887
|
+
GGML_ASSERT(ncols % QK4_1 == 0);
|
|
888
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
889
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
890
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
891
|
+
{
|
|
892
|
+
|
|
893
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
894
|
+
|
|
895
|
+
cgh.parallel_for(
|
|
896
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
897
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
898
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
899
|
+
mul_mat_vec_q<QK4_0, QI4_1, block_q4_1,
|
|
900
|
+
VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>(
|
|
901
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
902
|
+
});
|
|
903
|
+
});
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
template <int ncols_dst>
|
|
908
|
+
static void mul_mat_vec_q4_1_q8_1_sycl_ncols(
|
|
909
|
+
const void * vx, const void * vy, float * dst,
|
|
910
|
+
const int ncols, const int nrows,
|
|
911
|
+
const int stride_col_y, const int stride_col_dst,
|
|
912
|
+
dpct::queue_ptr stream) {
|
|
913
|
+
GGML_ASSERT(ncols % QK4_1 == 0);
|
|
914
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
915
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
916
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
917
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
918
|
+
cgh.parallel_for(
|
|
919
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
920
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
921
|
+
mul_mat_vec_q_ncols<QK4_0, QI4_1, block_q4_1,
|
|
922
|
+
VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1, ncols_dst>(
|
|
923
|
+
vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
|
|
924
|
+
});
|
|
925
|
+
});
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
static void mul_mat_vec_q4_1_q8_1_sycl_switch_ncols(
|
|
929
|
+
const void * vx, const void * vy, float * dst,
|
|
930
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
931
|
+
const int stride_col_y, const int stride_col_dst,
|
|
932
|
+
dpct::queue_ptr stream) {
|
|
933
|
+
switch (ncols_dst) {
|
|
934
|
+
case 1: mul_mat_vec_q4_1_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
935
|
+
case 2: mul_mat_vec_q4_1_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
936
|
+
case 3: mul_mat_vec_q4_1_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
937
|
+
case 4: mul_mat_vec_q4_1_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
938
|
+
case 5: mul_mat_vec_q4_1_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
939
|
+
case 6: mul_mat_vec_q4_1_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
940
|
+
case 7: mul_mat_vec_q4_1_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
941
|
+
case 8: mul_mat_vec_q4_1_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
942
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q4_1 multi-col MMVQ", ncols_dst);
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
static void mul_mat_vec_mxfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows,
|
|
947
|
+
dpct::queue_ptr stream) {
|
|
948
|
+
GGML_ASSERT(ncols % QK_MXFP4 == 0);
|
|
949
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
950
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
951
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
952
|
+
|
|
953
|
+
{
|
|
954
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
955
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
956
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
957
|
+
mul_mat_vec_q<QK_MXFP4, QI_MXFP4, block_mxfp4, VDR_MXFP4_Q8_1_MMVQ, vec_dot_mxfp4_q8_1>(
|
|
958
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
959
|
+
});
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
template <int ncols_dst>
|
|
965
|
+
static void mul_mat_vec_mxfp4_q8_1_sycl_ncols(
|
|
966
|
+
const void * vx, const void * vy, float * dst,
|
|
967
|
+
const int ncols, const int nrows,
|
|
968
|
+
const int stride_col_y, const int stride_col_dst,
|
|
969
|
+
dpct::queue_ptr stream) {
|
|
970
|
+
GGML_ASSERT(ncols % QK_MXFP4 == 0);
|
|
971
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
972
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
973
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
974
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
975
|
+
cgh.parallel_for(
|
|
976
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
977
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
978
|
+
mul_mat_vec_q_ncols<QK_MXFP4, QI_MXFP4, block_mxfp4,
|
|
979
|
+
VDR_MXFP4_Q8_1_MMVQ, vec_dot_mxfp4_q8_1, ncols_dst>(
|
|
980
|
+
vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
|
|
981
|
+
});
|
|
982
|
+
});
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
static void mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols(
|
|
986
|
+
const void * vx, const void * vy, float * dst,
|
|
987
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
988
|
+
const int stride_col_y, const int stride_col_dst,
|
|
989
|
+
dpct::queue_ptr stream) {
|
|
990
|
+
switch (ncols_dst) {
|
|
991
|
+
case 1: mul_mat_vec_mxfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
992
|
+
case 2: mul_mat_vec_mxfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
993
|
+
case 3: mul_mat_vec_mxfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
994
|
+
case 4: mul_mat_vec_mxfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
995
|
+
case 5: mul_mat_vec_mxfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
996
|
+
case 6: mul_mat_vec_mxfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
997
|
+
case 7: mul_mat_vec_mxfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
998
|
+
case 8: mul_mat_vec_mxfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
999
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for MXFP4 multi-col MMVQ", ncols_dst);
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
static void mul_mat_vec_nvfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows,
|
|
1004
|
+
dpct::queue_ptr stream) {
|
|
1005
|
+
GGML_ASSERT(ncols % QK_NVFP4 == 0);
|
|
1006
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1007
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1008
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1009
|
+
|
|
1010
|
+
{
|
|
1011
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1012
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1013
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1014
|
+
mul_mat_vec_q<QK_NVFP4, QI_NVFP4, block_nvfp4, VDR_NVFP4_Q8_1_MMVQ, vec_dot_nvfp4_q8_1>(
|
|
1015
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
1016
|
+
});
|
|
1017
|
+
});
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
template <int ncols_dst>
|
|
1022
|
+
static void mul_mat_vec_nvfp4_q8_1_sycl_ncols(
|
|
1023
|
+
const void * vx, const void * vy, float * dst,
|
|
1024
|
+
const int ncols, const int nrows,
|
|
1025
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1026
|
+
dpct::queue_ptr stream) {
|
|
1027
|
+
GGML_ASSERT(ncols % QK_NVFP4 == 0);
|
|
1028
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1029
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1030
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1031
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1032
|
+
cgh.parallel_for(
|
|
1033
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1034
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1035
|
+
mul_mat_vec_q_ncols<QK_NVFP4, QI_NVFP4, block_nvfp4,
|
|
1036
|
+
VDR_NVFP4_Q8_1_MMVQ, vec_dot_nvfp4_q8_1, ncols_dst>(
|
|
1037
|
+
vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
|
|
1038
|
+
});
|
|
1039
|
+
});
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
static void mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols(
|
|
1043
|
+
const void * vx, const void * vy, float * dst,
|
|
1044
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1045
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1046
|
+
dpct::queue_ptr stream) {
|
|
1047
|
+
switch (ncols_dst) {
|
|
1048
|
+
case 1: mul_mat_vec_nvfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1049
|
+
case 2: mul_mat_vec_nvfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1050
|
+
case 3: mul_mat_vec_nvfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1051
|
+
case 4: mul_mat_vec_nvfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1052
|
+
case 5: mul_mat_vec_nvfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1053
|
+
case 6: mul_mat_vec_nvfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1054
|
+
case 7: mul_mat_vec_nvfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1055
|
+
case 8: mul_mat_vec_nvfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1056
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for NVFP4 multi-col MMVQ", ncols_dst);
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
static void mul_mat_vec_q5_0_q8_1_sycl(const void *vx, const void *vy,
|
|
1061
|
+
float *dst, const int ncols,
|
|
1062
|
+
const int nrows,
|
|
1063
|
+
dpct::queue_ptr stream) {
|
|
1064
|
+
GGML_ASSERT(ncols % QK5_0 == 0);
|
|
1065
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1066
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1067
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1068
|
+
{
|
|
1069
|
+
|
|
1070
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
1071
|
+
|
|
1072
|
+
cgh.parallel_for(
|
|
1073
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1074
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
1075
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1076
|
+
mul_mat_vec_q<QK5_0, QI5_0, block_q5_0,
|
|
1077
|
+
VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>(
|
|
1078
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
1079
|
+
});
|
|
1080
|
+
});
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
template <int ncols_dst>
|
|
1085
|
+
static void mul_mat_vec_q5_0_q8_1_sycl_ncols(
|
|
1086
|
+
const void * vx, const void * vy, float * dst,
|
|
1087
|
+
const int ncols, const int nrows,
|
|
1088
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1089
|
+
dpct::queue_ptr stream) {
|
|
1090
|
+
GGML_ASSERT(ncols % QK5_0 == 0);
|
|
1091
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1092
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1093
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1094
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1095
|
+
cgh.parallel_for(
|
|
1096
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1097
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1098
|
+
mul_mat_vec_q_ncols<QK5_0, QI5_0, block_q5_0,
|
|
1099
|
+
VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1, ncols_dst>(
|
|
1100
|
+
vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
|
|
1101
|
+
});
|
|
1102
|
+
});
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
static void mul_mat_vec_q5_0_q8_1_sycl_switch_ncols(
|
|
1106
|
+
const void * vx, const void * vy, float * dst,
|
|
1107
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1108
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1109
|
+
dpct::queue_ptr stream) {
|
|
1110
|
+
switch (ncols_dst) {
|
|
1111
|
+
case 1: mul_mat_vec_q5_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1112
|
+
case 2: mul_mat_vec_q5_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1113
|
+
case 3: mul_mat_vec_q5_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1114
|
+
case 4: mul_mat_vec_q5_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1115
|
+
case 5: mul_mat_vec_q5_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1116
|
+
case 6: mul_mat_vec_q5_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1117
|
+
case 7: mul_mat_vec_q5_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1118
|
+
case 8: mul_mat_vec_q5_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1119
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q5_0 multi-col MMVQ", ncols_dst);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
static void mul_mat_vec_q5_1_q8_1_sycl(const void *vx, const void *vy,
|
|
1124
|
+
float *dst, const int ncols,
|
|
1125
|
+
const int nrows,
|
|
1126
|
+
dpct::queue_ptr stream) {
|
|
1127
|
+
GGML_ASSERT(ncols % QK5_1 == 0);
|
|
1128
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1129
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1130
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1131
|
+
{
|
|
1132
|
+
|
|
1133
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
1134
|
+
|
|
1135
|
+
cgh.parallel_for(
|
|
1136
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1137
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
1138
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1139
|
+
mul_mat_vec_q<QK5_1, QI5_1, block_q5_1,
|
|
1140
|
+
VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>(
|
|
1141
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
1142
|
+
});
|
|
1143
|
+
});
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
template <int ncols_dst>
|
|
1148
|
+
static void mul_mat_vec_q5_1_q8_1_sycl_ncols(
|
|
1149
|
+
const void * vx, const void * vy, float * dst,
|
|
1150
|
+
const int ncols, const int nrows,
|
|
1151
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1152
|
+
dpct::queue_ptr stream) {
|
|
1153
|
+
GGML_ASSERT(ncols % QK5_1 == 0);
|
|
1154
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1155
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1156
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1157
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1158
|
+
cgh.parallel_for(
|
|
1159
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1160
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1161
|
+
mul_mat_vec_q_ncols<QK5_1, QI5_1, block_q5_1,
|
|
1162
|
+
VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1, ncols_dst>(
|
|
1163
|
+
vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
|
|
1164
|
+
});
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
static void mul_mat_vec_q5_1_q8_1_sycl_switch_ncols(
|
|
1169
|
+
const void * vx, const void * vy, float * dst,
|
|
1170
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1171
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1172
|
+
dpct::queue_ptr stream) {
|
|
1173
|
+
switch (ncols_dst) {
|
|
1174
|
+
case 1: mul_mat_vec_q5_1_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1175
|
+
case 2: mul_mat_vec_q5_1_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1176
|
+
case 3: mul_mat_vec_q5_1_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1177
|
+
case 4: mul_mat_vec_q5_1_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1178
|
+
case 5: mul_mat_vec_q5_1_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1179
|
+
case 6: mul_mat_vec_q5_1_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1180
|
+
case 7: mul_mat_vec_q5_1_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1181
|
+
case 8: mul_mat_vec_q5_1_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1182
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q5_1 multi-col MMVQ", ncols_dst);
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
static void reorder_mul_mat_vec_q8_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
|
|
1187
|
+
const int nrows, dpct::queue_ptr stream) {
|
|
1188
|
+
GGML_ASSERT(ncols % QK8_0 == 0);
|
|
1189
|
+
// Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
|
|
1190
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1191
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
1192
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1193
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1194
|
+
|
|
1195
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1196
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1197
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1198
|
+
mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0>>(vx, vy, dst, ncols, nrows,
|
|
1199
|
+
nd_item);
|
|
1200
|
+
});
|
|
1201
|
+
});
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
template <int ncols_dst>
|
|
1205
|
+
static void reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols(
|
|
1206
|
+
const void * vx, const void * vy, float * dst,
|
|
1207
|
+
const int ncols, const int nrows,
|
|
1208
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1209
|
+
dpct::queue_ptr stream) {
|
|
1210
|
+
GGML_ASSERT(ncols % QK8_0 == 0);
|
|
1211
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1212
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
1213
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1214
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1215
|
+
|
|
1216
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1217
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1218
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1219
|
+
mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0>, ncols_dst>(
|
|
1220
|
+
vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
|
|
1221
|
+
/*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
|
|
1222
|
+
});
|
|
1223
|
+
});
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
static void reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(
|
|
1227
|
+
const void * vx, const void * vy, float * dst,
|
|
1228
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1229
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1230
|
+
dpct::queue_ptr stream) {
|
|
1231
|
+
switch (ncols_dst) {
|
|
1232
|
+
case 1: reorder_mul_mat_vec_q8_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1233
|
+
case 2: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1234
|
+
case 3: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1235
|
+
case 4: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1236
|
+
case 5: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1237
|
+
case 6: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1238
|
+
case 7: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1239
|
+
case 8: reorder_mul_mat_vec_q8_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1240
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q8_0 reorder multi-col MMVQ", ncols_dst);
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy,
|
|
1245
|
+
float *dst, const int ncols,
|
|
1246
|
+
const int nrows,
|
|
1247
|
+
dpct::queue_ptr stream) {
|
|
1248
|
+
GGML_ASSERT(ncols % QK8_0 == 0);
|
|
1249
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1250
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1251
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1252
|
+
{
|
|
1253
|
+
|
|
1254
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
1255
|
+
|
|
1256
|
+
cgh.parallel_for(
|
|
1257
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1258
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
1259
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1260
|
+
mul_mat_vec_q<QK8_0, QI8_0, block_q8_0,
|
|
1261
|
+
VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>(
|
|
1262
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
1263
|
+
});
|
|
1264
|
+
});
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
template <int ncols_dst>
|
|
1269
|
+
static void mul_mat_vec_q8_0_q8_1_sycl_ncols(
|
|
1270
|
+
const void * vx, const void * vy, float * dst,
|
|
1271
|
+
const int ncols, const int nrows,
|
|
1272
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1273
|
+
dpct::queue_ptr stream) {
|
|
1274
|
+
GGML_ASSERT(ncols % QK8_0 == 0);
|
|
1275
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1276
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1277
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1278
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1279
|
+
cgh.parallel_for(
|
|
1280
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1281
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1282
|
+
mul_mat_vec_q_ncols<QK8_0, QI8_0, block_q8_0,
|
|
1283
|
+
VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1, ncols_dst>(
|
|
1284
|
+
vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
|
|
1285
|
+
});
|
|
1286
|
+
});
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
static void mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(
|
|
1290
|
+
const void * vx, const void * vy, float * dst,
|
|
1291
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1292
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1293
|
+
dpct::queue_ptr stream) {
|
|
1294
|
+
switch (ncols_dst) {
|
|
1295
|
+
case 1: mul_mat_vec_q8_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1296
|
+
case 2: mul_mat_vec_q8_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1297
|
+
case 3: mul_mat_vec_q8_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1298
|
+
case 4: mul_mat_vec_q8_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1299
|
+
case 5: mul_mat_vec_q8_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1300
|
+
case 6: mul_mat_vec_q8_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1301
|
+
case 7: mul_mat_vec_q8_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1302
|
+
case 8: mul_mat_vec_q8_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1303
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q8_0 multi-col MMVQ", ncols_dst);
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
static void mul_mat_vec_q1_0_q8_1_sycl(const void * vx, const void * vy,
|
|
1308
|
+
float * dst, const int ncols,
|
|
1309
|
+
const int nrows,
|
|
1310
|
+
dpct::queue_ptr stream) {
|
|
1311
|
+
GGML_ASSERT(ncols % QK1_0 == 0);
|
|
1312
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1313
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1314
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1315
|
+
|
|
1316
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1317
|
+
cgh.parallel_for(
|
|
1318
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1319
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1320
|
+
mul_mat_vec_q<QK1_0, QI1_0, block_q1_0,
|
|
1321
|
+
VDR_Q1_0_Q8_1_MMVQ, vec_dot_q1_0_q8_1>(
|
|
1322
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
1323
|
+
});
|
|
1324
|
+
});
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
template <int ncols_dst>
|
|
1328
|
+
static void mul_mat_vec_q1_0_q8_1_sycl_ncols(
|
|
1329
|
+
const void * vx, const void * vy, float * dst,
|
|
1330
|
+
const int ncols, const int nrows,
|
|
1331
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1332
|
+
dpct::queue_ptr stream) {
|
|
1333
|
+
GGML_ASSERT(ncols % QK1_0 == 0);
|
|
1334
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1335
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1336
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1337
|
+
|
|
1338
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1339
|
+
cgh.parallel_for(
|
|
1340
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1341
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1342
|
+
mul_mat_vec_q_ncols<QK1_0, QI1_0, block_q1_0,
|
|
1343
|
+
VDR_Q1_0_Q8_1_MMVQ, vec_dot_q1_0_q8_1, ncols_dst>(
|
|
1344
|
+
vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
|
|
1345
|
+
});
|
|
1346
|
+
});
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
static void mul_mat_vec_q1_0_q8_1_sycl_switch_ncols(
|
|
1350
|
+
const void * vx, const void * vy, float * dst,
|
|
1351
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1352
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1353
|
+
dpct::queue_ptr stream) {
|
|
1354
|
+
switch (ncols_dst) {
|
|
1355
|
+
case 1: mul_mat_vec_q1_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1356
|
+
case 2: mul_mat_vec_q1_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1357
|
+
case 3: mul_mat_vec_q1_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1358
|
+
case 4: mul_mat_vec_q1_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1359
|
+
case 5: mul_mat_vec_q1_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1360
|
+
case 6: mul_mat_vec_q1_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1361
|
+
case 7: mul_mat_vec_q1_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1362
|
+
case 8: mul_mat_vec_q1_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1363
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q1_0 multi-col MMVQ", ncols_dst);
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
static void mul_mat_vec_q2_0_q8_1_sycl(const void * vx, const void * vy,
|
|
1368
|
+
float * dst, const int ncols,
|
|
1369
|
+
const int nrows,
|
|
1370
|
+
dpct::queue_ptr stream) {
|
|
1371
|
+
GGML_ASSERT(ncols % QK2_0 == 0);
|
|
1372
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1373
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1374
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1375
|
+
|
|
1376
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1377
|
+
cgh.parallel_for(
|
|
1378
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1379
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1380
|
+
mul_mat_vec_q<QK2_0, QI2_0, block_q2_0,
|
|
1381
|
+
VDR_Q2_0_Q8_1_MMVQ, vec_dot_q2_0_q8_1>(
|
|
1382
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
1383
|
+
});
|
|
1384
|
+
});
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
template <int ncols_dst>
|
|
1388
|
+
static void mul_mat_vec_q2_0_q8_1_sycl_ncols(
|
|
1389
|
+
const void * vx, const void * vy, float * dst,
|
|
1390
|
+
const int ncols, const int nrows,
|
|
1391
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1392
|
+
dpct::queue_ptr stream) {
|
|
1393
|
+
GGML_ASSERT(ncols % QK2_0 == 0);
|
|
1394
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1395
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1396
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1397
|
+
|
|
1398
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1399
|
+
cgh.parallel_for(
|
|
1400
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1401
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1402
|
+
mul_mat_vec_q_ncols<QK2_0, QI2_0, block_q2_0,
|
|
1403
|
+
VDR_Q2_0_Q8_1_MMVQ, vec_dot_q2_0_q8_1, ncols_dst>(
|
|
1404
|
+
vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
|
|
1405
|
+
});
|
|
1406
|
+
});
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
static void mul_mat_vec_q2_0_q8_1_sycl_switch_ncols(
|
|
1410
|
+
const void * vx, const void * vy, float * dst,
|
|
1411
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1412
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1413
|
+
dpct::queue_ptr stream) {
|
|
1414
|
+
switch (ncols_dst) {
|
|
1415
|
+
case 1: mul_mat_vec_q2_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1416
|
+
case 2: mul_mat_vec_q2_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1417
|
+
case 3: mul_mat_vec_q2_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1418
|
+
case 4: mul_mat_vec_q2_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1419
|
+
case 5: mul_mat_vec_q2_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1420
|
+
case 6: mul_mat_vec_q2_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1421
|
+
case 7: mul_mat_vec_q2_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1422
|
+
case 8: mul_mat_vec_q2_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1423
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q2_0 multi-col MMVQ", ncols_dst);
|
|
1424
|
+
}
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy,
|
|
1428
|
+
float *dst, const int ncols,
|
|
1429
|
+
const int nrows,
|
|
1430
|
+
dpct::queue_ptr stream) {
|
|
1431
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1432
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1433
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1434
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1435
|
+
{
|
|
1436
|
+
|
|
1437
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
1438
|
+
|
|
1439
|
+
cgh.parallel_for(
|
|
1440
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1441
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
1442
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1443
|
+
mul_mat_vec_q<QK_K, QI2_K, block_q2_K,
|
|
1444
|
+
VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>(
|
|
1445
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
1446
|
+
});
|
|
1447
|
+
});
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
template <int ncols_dst>
|
|
1452
|
+
static void mul_mat_vec_q2_K_q8_1_sycl_ncols(
|
|
1453
|
+
const void * vx, const void * vy, float * dst,
|
|
1454
|
+
const int ncols, const int nrows,
|
|
1455
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1456
|
+
dpct::queue_ptr stream) {
|
|
1457
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1458
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1459
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1460
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1461
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1462
|
+
cgh.parallel_for(
|
|
1463
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1464
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1465
|
+
mul_mat_vec_q_ncols<QK_K, QI2_K, block_q2_K,
|
|
1466
|
+
VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1, ncols_dst>(
|
|
1467
|
+
vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
|
|
1468
|
+
});
|
|
1469
|
+
});
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
static void mul_mat_vec_q2_K_q8_1_sycl_switch_ncols(
|
|
1473
|
+
const void * vx, const void * vy, float * dst,
|
|
1474
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1475
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1476
|
+
dpct::queue_ptr stream) {
|
|
1477
|
+
switch (ncols_dst) {
|
|
1478
|
+
case 1: mul_mat_vec_q2_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1479
|
+
case 2: mul_mat_vec_q2_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1480
|
+
case 3: mul_mat_vec_q2_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1481
|
+
case 4: mul_mat_vec_q2_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1482
|
+
case 5: mul_mat_vec_q2_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1483
|
+
case 6: mul_mat_vec_q2_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1484
|
+
case 7: mul_mat_vec_q2_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1485
|
+
case 8: mul_mat_vec_q2_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1486
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q2_K multi-col MMVQ", ncols_dst);
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
static void reorder_mul_mat_vec_q2_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
|
|
1491
|
+
const int nrows, dpct::queue_ptr stream) {
|
|
1492
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1493
|
+
|
|
1494
|
+
// Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
|
|
1495
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1496
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
1497
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1498
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1499
|
+
|
|
1500
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1501
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1502
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1503
|
+
mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q2_K>>(vx, vy, dst, ncols, nrows,
|
|
1504
|
+
nd_item);
|
|
1505
|
+
});
|
|
1506
|
+
});
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
template <int ncols_dst>
|
|
1510
|
+
static void reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols(
|
|
1511
|
+
const void * vx, const void * vy, float * dst,
|
|
1512
|
+
const int ncols, const int nrows,
|
|
1513
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1514
|
+
dpct::queue_ptr stream) {
|
|
1515
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1516
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1517
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
1518
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1519
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1520
|
+
|
|
1521
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1522
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1523
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1524
|
+
mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q2_K>, ncols_dst>(
|
|
1525
|
+
vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
|
|
1526
|
+
/*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
|
|
1527
|
+
});
|
|
1528
|
+
});
|
|
1529
|
+
}
|
|
1530
|
+
|
|
1531
|
+
static void reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols(
|
|
1532
|
+
const void * vx, const void * vy, float * dst,
|
|
1533
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1534
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1535
|
+
dpct::queue_ptr stream) {
|
|
1536
|
+
switch (ncols_dst) {
|
|
1537
|
+
case 1: reorder_mul_mat_vec_q2_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1538
|
+
case 2: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1539
|
+
case 3: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1540
|
+
case 4: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1541
|
+
case 5: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1542
|
+
case 6: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1543
|
+
case 7: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1544
|
+
case 8: reorder_mul_mat_vec_q2_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1545
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q2_K reorder multi-col MMVQ", ncols_dst);
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
static void mul_mat_vec_q3_K_q8_1_sycl(const void *vx, const void *vy,
|
|
1550
|
+
float *dst, const int ncols,
|
|
1551
|
+
const int nrows,
|
|
1552
|
+
dpct::queue_ptr stream) {
|
|
1553
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1554
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1555
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1556
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1557
|
+
{
|
|
1558
|
+
|
|
1559
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
1560
|
+
|
|
1561
|
+
cgh.parallel_for(
|
|
1562
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1563
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
1564
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1565
|
+
mul_mat_vec_q<QK_K, QI3_K, block_q3_K,
|
|
1566
|
+
VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>(
|
|
1567
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
1568
|
+
});
|
|
1569
|
+
});
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
static void reorder_mul_mat_vec_q3_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
|
|
1574
|
+
const int nrows, dpct::queue_ptr stream) {
|
|
1575
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1576
|
+
|
|
1577
|
+
// Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
|
|
1578
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1579
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
1580
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1581
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1582
|
+
|
|
1583
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1584
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1585
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1586
|
+
mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q3_K>>(vx, vy, dst, ncols, nrows,
|
|
1587
|
+
nd_item);
|
|
1588
|
+
});
|
|
1589
|
+
});
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
template <int ncols_dst>
|
|
1593
|
+
static void reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols(
|
|
1594
|
+
const void * vx, const void * vy, float * dst,
|
|
1595
|
+
const int ncols, const int nrows,
|
|
1596
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1597
|
+
dpct::queue_ptr stream) {
|
|
1598
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1599
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1600
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
1601
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1602
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1603
|
+
|
|
1604
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1605
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1606
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1607
|
+
mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q3_K>, ncols_dst>(
|
|
1608
|
+
vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
|
|
1609
|
+
/*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
|
|
1610
|
+
});
|
|
1611
|
+
});
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
static void reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols(
|
|
1615
|
+
const void * vx, const void * vy, float * dst,
|
|
1616
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1617
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1618
|
+
dpct::queue_ptr stream) {
|
|
1619
|
+
switch (ncols_dst) {
|
|
1620
|
+
case 1: reorder_mul_mat_vec_q3_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1621
|
+
case 2: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1622
|
+
case 3: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1623
|
+
case 4: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1624
|
+
case 5: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1625
|
+
case 6: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1626
|
+
case 7: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1627
|
+
case 8: reorder_mul_mat_vec_q3_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1628
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q3_K reorder multi-col MMVQ", ncols_dst);
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
template <int ncols_dst>
|
|
1633
|
+
static void mul_mat_vec_q3_K_q8_1_sycl_ncols(
|
|
1634
|
+
const void * vx, const void * vy, float * dst,
|
|
1635
|
+
const int ncols, const int nrows,
|
|
1636
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1637
|
+
dpct::queue_ptr stream) {
|
|
1638
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1639
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1640
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1641
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1642
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1643
|
+
cgh.parallel_for(
|
|
1644
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1645
|
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1646
|
+
mul_mat_vec_q_ncols<QK_K, QI3_K, block_q3_K,
|
|
1647
|
+
VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1, ncols_dst>(
|
|
1648
|
+
vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1);
|
|
1649
|
+
});
|
|
1650
|
+
});
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
static void mul_mat_vec_q3_K_q8_1_sycl_switch_ncols(
|
|
1654
|
+
const void * vx, const void * vy, float * dst,
|
|
1655
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1656
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1657
|
+
dpct::queue_ptr stream) {
|
|
1658
|
+
switch (ncols_dst) {
|
|
1659
|
+
case 1: mul_mat_vec_q3_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1660
|
+
case 2: mul_mat_vec_q3_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1661
|
+
case 3: mul_mat_vec_q3_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1662
|
+
case 4: mul_mat_vec_q3_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1663
|
+
case 5: mul_mat_vec_q3_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1664
|
+
case 6: mul_mat_vec_q3_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1665
|
+
case 7: mul_mat_vec_q3_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1666
|
+
case 8: mul_mat_vec_q3_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1667
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q3_K multi-col MMVQ", ncols_dst);
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
|
|
1672
|
+
static void mul_mat_vec_q4_K_q8_1_sycl(const void *vx, const void *vy,
|
|
1673
|
+
float *dst, const int ncols,
|
|
1674
|
+
const int nrows,
|
|
1675
|
+
dpct::queue_ptr stream) {
|
|
1676
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1677
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1678
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1679
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1680
|
+
{
|
|
1681
|
+
|
|
1682
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
1683
|
+
|
|
1684
|
+
cgh.parallel_for(
|
|
1685
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1686
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
1687
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1688
|
+
mul_mat_vec_q<QK_K, QI4_K, block_q4_K,
|
|
1689
|
+
VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>(
|
|
1690
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
1691
|
+
});
|
|
1692
|
+
});
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
template <int ncols_dst>
|
|
1697
|
+
static void mul_mat_vec_q4_K_q8_1_sycl_ncols(
|
|
1698
|
+
const void * vx, const void * vy, float * dst,
|
|
1699
|
+
const int ncols, const int nrows,
|
|
1700
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1701
|
+
dpct::queue_ptr stream) {
|
|
1702
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1703
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1704
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1705
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1706
|
+
|
|
1707
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1708
|
+
cgh.parallel_for(
|
|
1709
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1710
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
1711
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1712
|
+
mul_mat_vec_q_ncols<QK_K, QI4_K, block_q4_K,
|
|
1713
|
+
VDR_Q4_K_Q8_1_MMVQ,
|
|
1714
|
+
vec_dot_q4_K_q8_1,
|
|
1715
|
+
ncols_dst>(
|
|
1716
|
+
vx, vy, dst, ncols, nrows,
|
|
1717
|
+
stride_col_y, stride_col_dst, item_ct1);
|
|
1718
|
+
});
|
|
1719
|
+
});
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
static void mul_mat_vec_q4_K_q8_1_sycl_switch_ncols(
|
|
1723
|
+
const void * vx, const void * vy, float * dst,
|
|
1724
|
+
const int ncols, const int nrows,
|
|
1725
|
+
const int ncols_dst,
|
|
1726
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1727
|
+
dpct::queue_ptr stream) {
|
|
1728
|
+
switch (ncols_dst) {
|
|
1729
|
+
case 1: mul_mat_vec_q4_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1730
|
+
case 2: mul_mat_vec_q4_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1731
|
+
case 3: mul_mat_vec_q4_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1732
|
+
case 4: mul_mat_vec_q4_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1733
|
+
case 5: mul_mat_vec_q4_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1734
|
+
case 6: mul_mat_vec_q4_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1735
|
+
case 7: mul_mat_vec_q4_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1736
|
+
case 8: mul_mat_vec_q4_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1737
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q4_K multi-col MMVQ", ncols_dst);
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
static void reorder_mul_mat_vec_q4_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
|
|
1742
|
+
const int nrows, dpct::queue_ptr stream) {
|
|
1743
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1744
|
+
|
|
1745
|
+
// Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
|
|
1746
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1747
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
1748
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1749
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1750
|
+
|
|
1751
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1752
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1753
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1754
|
+
mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>>(vx, vy, dst, ncols,
|
|
1755
|
+
nrows, nd_item);
|
|
1756
|
+
});
|
|
1757
|
+
});
|
|
1758
|
+
}
|
|
1759
|
+
|
|
1760
|
+
template <int ncols_dst, int rows_per_sg>
|
|
1761
|
+
static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl(
|
|
1762
|
+
const void * vx, const void * vy, float * dst,
|
|
1763
|
+
const int ncols, const int nrows,
|
|
1764
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1765
|
+
dpct::queue_ptr stream) {
|
|
1766
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1767
|
+
|
|
1768
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1769
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg);
|
|
1770
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1771
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1772
|
+
|
|
1773
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1774
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1775
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1776
|
+
mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>, ncols_dst,
|
|
1777
|
+
/*has_fusion=*/ false, rows_per_sg>(
|
|
1778
|
+
vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
|
|
1779
|
+
/*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
|
|
1780
|
+
});
|
|
1781
|
+
});
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
template <int ncols_dst>
|
|
1785
|
+
static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols(
|
|
1786
|
+
const void * vx, const void * vy, float * dst,
|
|
1787
|
+
const int ncols, const int nrows,
|
|
1788
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1789
|
+
dpct::queue_ptr stream) {
|
|
1790
|
+
constexpr int rows_per_sg = ncols_dst >= 3 && ncols_dst <= 4 ? 2 : 1;
|
|
1791
|
+
reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<ncols_dst, rows_per_sg>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream);
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1794
|
+
static void reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols(
|
|
1795
|
+
const void * vx, const void * vy, float * dst,
|
|
1796
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1797
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1798
|
+
dpct::queue_ptr stream) {
|
|
1799
|
+
switch (ncols_dst) {
|
|
1800
|
+
case 1: reorder_mul_mat_vec_q4_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1801
|
+
case 2:
|
|
1802
|
+
if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) {
|
|
1803
|
+
reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream);
|
|
1804
|
+
} else {
|
|
1805
|
+
reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 1>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream);
|
|
1806
|
+
}
|
|
1807
|
+
break;
|
|
1808
|
+
case 3: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1809
|
+
case 4: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1810
|
+
case 5: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1811
|
+
case 6: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1812
|
+
case 7: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1813
|
+
case 8: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1814
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q4_K reorder multi-col MMVQ", ncols_dst);
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
static void mul_mat_vec_q5_K_q8_1_sycl(const void *vx, const void *vy,
|
|
1819
|
+
float *dst, const int ncols,
|
|
1820
|
+
const int nrows,
|
|
1821
|
+
dpct::queue_ptr stream) {
|
|
1822
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1823
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1824
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1825
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1826
|
+
{
|
|
1827
|
+
|
|
1828
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
1829
|
+
|
|
1830
|
+
cgh.parallel_for(
|
|
1831
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1832
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
1833
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1834
|
+
mul_mat_vec_q<QK_K, QI5_K, block_q5_K,
|
|
1835
|
+
VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>(
|
|
1836
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
1837
|
+
});
|
|
1838
|
+
});
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
template <int ncols_dst>
|
|
1843
|
+
static void mul_mat_vec_q5_K_q8_1_sycl_ncols(
|
|
1844
|
+
const void * vx, const void * vy, float * dst,
|
|
1845
|
+
const int ncols, const int nrows,
|
|
1846
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1847
|
+
dpct::queue_ptr stream) {
|
|
1848
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1849
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
1850
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1851
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
1852
|
+
|
|
1853
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1854
|
+
cgh.parallel_for(
|
|
1855
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1856
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
1857
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1858
|
+
mul_mat_vec_q_ncols<QK_K, QI5_K, block_q5_K,
|
|
1859
|
+
VDR_Q5_K_Q8_1_MMVQ,
|
|
1860
|
+
vec_dot_q5_K_q8_1,
|
|
1861
|
+
ncols_dst>(
|
|
1862
|
+
vx, vy, dst, ncols, nrows,
|
|
1863
|
+
stride_col_y, stride_col_dst, item_ct1);
|
|
1864
|
+
});
|
|
1865
|
+
});
|
|
1866
|
+
}
|
|
1867
|
+
|
|
1868
|
+
static void mul_mat_vec_q5_K_q8_1_sycl_switch_ncols(
|
|
1869
|
+
const void * vx, const void * vy, float * dst,
|
|
1870
|
+
const int ncols, const int nrows,
|
|
1871
|
+
const int ncols_dst,
|
|
1872
|
+
const int stride_col_y, const int stride_col_dst,
|
|
1873
|
+
dpct::queue_ptr stream) {
|
|
1874
|
+
switch (ncols_dst) {
|
|
1875
|
+
case 1: mul_mat_vec_q5_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1876
|
+
case 2: mul_mat_vec_q5_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1877
|
+
case 3: mul_mat_vec_q5_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1878
|
+
case 4: mul_mat_vec_q5_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1879
|
+
case 5: mul_mat_vec_q5_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1880
|
+
case 6: mul_mat_vec_q5_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1881
|
+
case 7: mul_mat_vec_q5_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1882
|
+
case 8: mul_mat_vec_q5_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
1883
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q5_K multi-col MMVQ", ncols_dst);
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
static void reorder_mul_mat_vec_q5_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
|
|
1888
|
+
const int nrows, dpct::queue_ptr stream) {
|
|
1889
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1890
|
+
|
|
1891
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1892
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
1893
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1894
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1895
|
+
|
|
1896
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1897
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1898
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1899
|
+
mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K>>(vx, vy, dst, ncols,
|
|
1900
|
+
nrows, nd_item);
|
|
1901
|
+
});
|
|
1902
|
+
});
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
template <int ncols_dst>
|
|
1906
|
+
static void reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols(
|
|
1907
|
+
const void * vx, const void * vy, float * dst,
|
|
1908
|
+
const int ncols, const int nrows,
|
|
1909
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1910
|
+
dpct::queue_ptr stream) {
|
|
1911
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1912
|
+
|
|
1913
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1914
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
1915
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1916
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1917
|
+
|
|
1918
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1919
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1920
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1921
|
+
mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K>, ncols_dst>(
|
|
1922
|
+
vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
|
|
1923
|
+
/*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
|
|
1924
|
+
});
|
|
1925
|
+
});
|
|
1926
|
+
}
|
|
1927
|
+
|
|
1928
|
+
static void reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols(
|
|
1929
|
+
const void * vx, const void * vy, float * dst,
|
|
1930
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1931
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1932
|
+
dpct::queue_ptr stream) {
|
|
1933
|
+
switch (ncols_dst) {
|
|
1934
|
+
case 1: reorder_mul_mat_vec_q5_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1935
|
+
case 2: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1936
|
+
case 3: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1937
|
+
case 4: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1938
|
+
case 5: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1939
|
+
case 6: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1940
|
+
case 7: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1941
|
+
case 8: reorder_mul_mat_vec_q5_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1942
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q5_K reorder multi-col MMVQ", ncols_dst);
|
|
1943
|
+
}
|
|
1944
|
+
}
|
|
1945
|
+
|
|
1946
|
+
static void reorder_mul_mat_vec_q6_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
|
|
1947
|
+
const int nrows, dpct::queue_ptr stream) {
|
|
1948
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1949
|
+
// Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel.
|
|
1950
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1951
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
1952
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1953
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1954
|
+
|
|
1955
|
+
|
|
1956
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1957
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1958
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1959
|
+
mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q6_K>>(vx, vy, dst, ncols, nrows,
|
|
1960
|
+
nd_item);
|
|
1961
|
+
});
|
|
1962
|
+
});
|
|
1963
|
+
}
|
|
1964
|
+
|
|
1965
|
+
template <int ncols_dst>
|
|
1966
|
+
static void reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols(
|
|
1967
|
+
const void * vx, const void * vy, float * dst,
|
|
1968
|
+
const int ncols, const int nrows,
|
|
1969
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1970
|
+
dpct::queue_ptr stream) {
|
|
1971
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
1972
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
1973
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
|
|
1974
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
1975
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
1976
|
+
|
|
1977
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
1978
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
1979
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
1980
|
+
mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_Q6_K>, ncols_dst>(
|
|
1981
|
+
vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
|
|
1982
|
+
/*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
|
|
1983
|
+
});
|
|
1984
|
+
});
|
|
1985
|
+
}
|
|
1986
|
+
|
|
1987
|
+
static void reorder_mul_mat_vec_q6_k_q8_1_sycl_switch_ncols(
|
|
1988
|
+
const void * vx, const void * vy, float * dst,
|
|
1989
|
+
const int ncols, const int nrows, const int ncols_dst,
|
|
1990
|
+
const int stride_col_y_bytes, const int stride_col_dst,
|
|
1991
|
+
dpct::queue_ptr stream) {
|
|
1992
|
+
switch (ncols_dst) {
|
|
1993
|
+
case 1: reorder_mul_mat_vec_q6_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
1994
|
+
case 2: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1995
|
+
case 3: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1996
|
+
case 4: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1997
|
+
case 5: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1998
|
+
case 6: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
1999
|
+
case 7: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
2000
|
+
case 8: reorder_mul_mat_vec_q6_k_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
|
|
2001
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q6_K reorder multi-col MMVQ", ncols_dst);
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
|
|
2005
|
+
static void mul_mat_vec_q6_K_q8_1_sycl(const void *vx, const void *vy,
|
|
2006
|
+
float *dst, const int ncols,
|
|
2007
|
+
const int nrows,
|
|
2008
|
+
dpct::queue_ptr stream) {
|
|
2009
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2010
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2011
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2012
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2013
|
+
{
|
|
2014
|
+
|
|
2015
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
2016
|
+
|
|
2017
|
+
cgh.parallel_for(
|
|
2018
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2019
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2020
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2021
|
+
mul_mat_vec_q<QK_K, QI6_K, block_q6_K,
|
|
2022
|
+
VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>(
|
|
2023
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
2024
|
+
});
|
|
2025
|
+
});
|
|
2026
|
+
}
|
|
2027
|
+
}
|
|
2028
|
+
|
|
2029
|
+
template <int ncols_dst>
|
|
2030
|
+
static void mul_mat_vec_q6_K_q8_1_sycl_ncols(
|
|
2031
|
+
const void * vx, const void * vy, float * dst,
|
|
2032
|
+
const int ncols, const int nrows,
|
|
2033
|
+
const int stride_col_y, const int stride_col_dst,
|
|
2034
|
+
dpct::queue_ptr stream) {
|
|
2035
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2036
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2037
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2038
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2039
|
+
|
|
2040
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
2041
|
+
cgh.parallel_for(
|
|
2042
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2043
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2044
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2045
|
+
mul_mat_vec_q_ncols<QK_K, QI6_K, block_q6_K,
|
|
2046
|
+
VDR_Q6_K_Q8_1_MMVQ,
|
|
2047
|
+
vec_dot_q6_K_q8_1,
|
|
2048
|
+
ncols_dst>(
|
|
2049
|
+
vx, vy, dst, ncols, nrows,
|
|
2050
|
+
stride_col_y, stride_col_dst, item_ct1);
|
|
2051
|
+
});
|
|
2052
|
+
});
|
|
2053
|
+
}
|
|
2054
|
+
|
|
2055
|
+
static void mul_mat_vec_q6_K_q8_1_sycl_switch_ncols(
|
|
2056
|
+
const void * vx, const void * vy, float * dst,
|
|
2057
|
+
const int ncols, const int nrows,
|
|
2058
|
+
const int ncols_dst,
|
|
2059
|
+
const int stride_col_y, const int stride_col_dst,
|
|
2060
|
+
dpct::queue_ptr stream) {
|
|
2061
|
+
switch (ncols_dst) {
|
|
2062
|
+
case 1: mul_mat_vec_q6_K_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
2063
|
+
case 2: mul_mat_vec_q6_K_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2064
|
+
case 3: mul_mat_vec_q6_K_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2065
|
+
case 4: mul_mat_vec_q6_K_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2066
|
+
case 5: mul_mat_vec_q6_K_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2067
|
+
case 6: mul_mat_vec_q6_K_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2068
|
+
case 7: mul_mat_vec_q6_K_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2069
|
+
case 8: mul_mat_vec_q6_K_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2070
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for Q6_K multi-col MMVQ", ncols_dst);
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
|
|
2075
|
+
static void mul_mat_vec_iq2_xxs_q8_1_sycl(const void *vx, const void *vy,
|
|
2076
|
+
float *dst, const int ncols,
|
|
2077
|
+
const int nrows,
|
|
2078
|
+
dpct::queue_ptr stream) {
|
|
2079
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2080
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2081
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2082
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2083
|
+
{
|
|
2084
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
2085
|
+
cgh.parallel_for(
|
|
2086
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2087
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2088
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2089
|
+
mul_mat_vec_q_iq2_xxs_q8_1<QK_K, QI2_XXS/2, block_iq2_xxs, 1>(
|
|
2090
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
2091
|
+
});
|
|
2092
|
+
});
|
|
2093
|
+
}
|
|
2094
|
+
}
|
|
2095
|
+
|
|
2096
|
+
static void mul_mat_vec_iq2_xs_q8_1_sycl(const void *vx, const void *vy,
|
|
2097
|
+
float *dst, const int ncols,
|
|
2098
|
+
const int nrows,
|
|
2099
|
+
dpct::queue_ptr stream) {
|
|
2100
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2101
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2102
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2103
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2104
|
+
{
|
|
2105
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
2106
|
+
cgh.parallel_for(
|
|
2107
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2108
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2109
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2110
|
+
mul_mat_vec_q_iq2_xs_q8_1<QK_K, QI2_XS/2, block_iq2_xs, 1>(
|
|
2111
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
2112
|
+
});
|
|
2113
|
+
});
|
|
2114
|
+
}
|
|
2115
|
+
}
|
|
2116
|
+
|
|
2117
|
+
static void mul_mat_vec_iq2_s_q8_1_sycl(const void *vx, const void *vy,
|
|
2118
|
+
float *dst, const int ncols,
|
|
2119
|
+
const int nrows,
|
|
2120
|
+
dpct::queue_ptr stream) {
|
|
2121
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2122
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2123
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2124
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2125
|
+
{
|
|
2126
|
+
|
|
2127
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
2128
|
+
cgh.parallel_for(
|
|
2129
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2130
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2131
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2132
|
+
mul_mat_vec_q_iq2_s_q8_1<QK_K, QI2_S/2, block_iq2_s, 1>(
|
|
2133
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
2134
|
+
});
|
|
2135
|
+
});
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
|
|
2139
|
+
static void mul_mat_vec_iq3_xxs_q8_1_sycl(const void *vx, const void *vy,
|
|
2140
|
+
float *dst, const int ncols,
|
|
2141
|
+
const int nrows,
|
|
2142
|
+
dpct::queue_ptr stream) {
|
|
2143
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2144
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2145
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2146
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2147
|
+
{
|
|
2148
|
+
|
|
2149
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
2150
|
+
cgh.parallel_for(
|
|
2151
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2152
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2153
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2154
|
+
mul_mat_vec_q_iq3_xxs_q8_1<QK_K, QI3_XXS/2, block_iq3_xxs, 1>(
|
|
2155
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
2156
|
+
});
|
|
2157
|
+
});
|
|
2158
|
+
}
|
|
2159
|
+
}
|
|
2160
|
+
|
|
2161
|
+
static void mul_mat_vec_iq3_s_q8_1_sycl(const void *vx, const void *vy,
|
|
2162
|
+
float *dst, const int ncols,
|
|
2163
|
+
const int nrows,
|
|
2164
|
+
dpct::queue_ptr stream) {
|
|
2165
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2166
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2167
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2168
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2169
|
+
{
|
|
2170
|
+
|
|
2171
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
2172
|
+
cgh.parallel_for(
|
|
2173
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2174
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2175
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2176
|
+
mul_mat_vec_q_iq3_s_q8_1<QK_K, QI3_S/2, block_iq3_s, 1>(
|
|
2177
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
2178
|
+
});
|
|
2179
|
+
});
|
|
2180
|
+
}
|
|
2181
|
+
}
|
|
2182
|
+
|
|
2183
|
+
static void mul_mat_vec_iq1_s_q8_1_sycl(const void *vx, const void *vy,
|
|
2184
|
+
float *dst, const int ncols,
|
|
2185
|
+
const int nrows,
|
|
2186
|
+
dpct::queue_ptr stream) {
|
|
2187
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2188
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2189
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2190
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2191
|
+
{
|
|
2192
|
+
|
|
2193
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
2194
|
+
cgh.parallel_for(
|
|
2195
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2196
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2197
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2198
|
+
mul_mat_vec_q_iq1_s_q8_1<QK_K, QI1_S, block_iq1_s, 1>(
|
|
2199
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
2200
|
+
});
|
|
2201
|
+
});
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
|
|
2205
|
+
static void mul_mat_vec_iq1_m_q8_1_sycl(const void *vx, const void *vy,
|
|
2206
|
+
float *dst, const int ncols,
|
|
2207
|
+
const int nrows,
|
|
2208
|
+
dpct::queue_ptr stream) {
|
|
2209
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2210
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2211
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2212
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2213
|
+
{
|
|
2214
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
2215
|
+
cgh.parallel_for(
|
|
2216
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2217
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2218
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2219
|
+
mul_mat_vec_q_iq1_m_q8_1<QK_K, QI1_S, block_iq1_m, 1>(
|
|
2220
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
2221
|
+
});
|
|
2222
|
+
});
|
|
2223
|
+
}
|
|
2224
|
+
}
|
|
2225
|
+
|
|
2226
|
+
static void mul_mat_vec_iq4_nl_q8_1_sycl(const void *vx, const void *vy,
|
|
2227
|
+
float *dst, const int ncols,
|
|
2228
|
+
const int nrows,
|
|
2229
|
+
dpct::queue_ptr stream) {
|
|
2230
|
+
GGML_ASSERT(ncols % QK4_NL == 0);
|
|
2231
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2232
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2233
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2234
|
+
{
|
|
2235
|
+
|
|
2236
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
2237
|
+
cgh.parallel_for(
|
|
2238
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2239
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2240
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2241
|
+
mul_mat_vec_q_iq4_nl_q8_1<QK4_NL, QI4_NL, block_iq4_nl, 2>(
|
|
2242
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
2243
|
+
});
|
|
2244
|
+
});
|
|
2245
|
+
}
|
|
2246
|
+
}
|
|
2247
|
+
|
|
2248
|
+
static void mul_mat_vec_iq4_xs_q8_1_sycl(const void *vx, const void *vy,
|
|
2249
|
+
float *dst, const int ncols,
|
|
2250
|
+
const int nrows,
|
|
2251
|
+
dpct::queue_ptr stream) {
|
|
2252
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2253
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2254
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2255
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2256
|
+
{
|
|
2257
|
+
|
|
2258
|
+
stream->submit([&](sycl::handler &cgh) {
|
|
2259
|
+
cgh.parallel_for(
|
|
2260
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2261
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2262
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2263
|
+
mul_mat_vec_q_iq4_xs_q8_1<QK_K, QI4_XS/4, block_iq4_xs, 1>(
|
|
2264
|
+
vx, vy, dst, ncols, nrows, item_ct1);
|
|
2265
|
+
});
|
|
2266
|
+
});
|
|
2267
|
+
}
|
|
2268
|
+
}
|
|
2269
|
+
|
|
2270
|
+
template <int ncols_dst>
|
|
2271
|
+
static void mul_mat_vec_iq4_xs_q8_1_sycl_ncols(
|
|
2272
|
+
const void * vx, const void * vy, float * dst,
|
|
2273
|
+
const int ncols, const int nrows,
|
|
2274
|
+
const int stride_col_y, const int stride_col_dst,
|
|
2275
|
+
dpct::queue_ptr stream) {
|
|
2276
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2277
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2278
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2279
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2280
|
+
|
|
2281
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
2282
|
+
cgh.parallel_for(
|
|
2283
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2284
|
+
[=](sycl::nd_item<3> item_ct1)
|
|
2285
|
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2286
|
+
mul_mat_vec_q_ncols<QK_K, QI4_XS/4, block_iq4_xs,
|
|
2287
|
+
1,
|
|
2288
|
+
vec_dot_iq4_xs_q8_1,
|
|
2289
|
+
ncols_dst>(
|
|
2290
|
+
vx, vy, dst, ncols, nrows,
|
|
2291
|
+
stride_col_y, stride_col_dst, item_ct1);
|
|
2292
|
+
});
|
|
2293
|
+
});
|
|
2294
|
+
}
|
|
2295
|
+
|
|
2296
|
+
static void mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols(
|
|
2297
|
+
const void * vx, const void * vy, float * dst,
|
|
2298
|
+
const int ncols, const int nrows,
|
|
2299
|
+
const int ncols_dst,
|
|
2300
|
+
const int stride_col_y, const int stride_col_dst,
|
|
2301
|
+
dpct::queue_ptr stream) {
|
|
2302
|
+
switch (ncols_dst) {
|
|
2303
|
+
case 1: mul_mat_vec_iq4_xs_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
|
|
2304
|
+
case 2: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2305
|
+
case 3: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2306
|
+
case 4: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2307
|
+
case 5: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2308
|
+
case 6: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2309
|
+
case 7: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2310
|
+
case 8: mul_mat_vec_iq4_xs_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break;
|
|
2311
|
+
default: GGML_ABORT("unsupported ncols_dst=%d for IQ4_XS multi-col MMVQ", ncols_dst);
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2314
|
+
|
|
2315
|
+
void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1,
|
|
2316
|
+
ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
|
|
2317
|
+
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low,
|
|
2318
|
+
const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_col_size,
|
|
2319
|
+
const dpct::queue_ptr & stream) {
|
|
2320
|
+
const int64_t ne10 = src1->ne[0];
|
|
2321
|
+
GGML_ASSERT(ne10 % QK8_1 == 0);
|
|
2322
|
+
|
|
2323
|
+
const int64_t ne00 = src0->ne[0];
|
|
2324
|
+
const int64_t row_diff = row_high - row_low;
|
|
2325
|
+
|
|
2326
|
+
int id;
|
|
2327
|
+
SYCL_CHECK(CHECK_TRY_ERROR(id = get_current_device_id()));
|
|
2328
|
+
const size_t q8_1_ts = sizeof(block_q8_1);
|
|
2329
|
+
const size_t q8_1_bs = QK8_1;
|
|
2330
|
+
// the main device has a larger memory buffer to hold the results from all GPUs
|
|
2331
|
+
// nrows_dst == nrows of the matrix that the kernel writes into
|
|
2332
|
+
|
|
2333
|
+
for (int i = 0; i < src1_ncols; i++) {
|
|
2334
|
+
const size_t src1_ddq_i_offset = i * src1_padded_col_size * q8_1_ts / q8_1_bs;
|
|
2335
|
+
const char * src1_ddq_i_bs = src1_ddq_i + src1_ddq_i_offset;
|
|
2336
|
+
float * dst_dd_i_bs = dst_dd_i + i * dst->ne[0];
|
|
2337
|
+
switch (src0->type) {
|
|
2338
|
+
case GGML_TYPE_Q4_0:
|
|
2339
|
+
if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
|
|
2340
|
+
((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
|
|
2341
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2342
|
+
const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
|
|
2343
|
+
const int stride_col_dst = dst->ne[0];
|
|
2344
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2345
|
+
reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(
|
|
2346
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2347
|
+
src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
|
|
2348
|
+
return;
|
|
2349
|
+
} else {
|
|
2350
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_0_q8_1_sycl\n");
|
|
2351
|
+
reorder_mul_mat_vec_q4_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2352
|
+
}
|
|
2353
|
+
} else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2354
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2355
|
+
const int stride_col_dst = dst->ne[0];
|
|
2356
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2357
|
+
mul_mat_vec_q4_0_q8_1_sycl_switch_ncols(
|
|
2358
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2359
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2360
|
+
return;
|
|
2361
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2362
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_0_q8_1_sycl\n");
|
|
2363
|
+
mul_mat_vec_q4_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2364
|
+
}
|
|
2365
|
+
break;
|
|
2366
|
+
case GGML_TYPE_Q4_1:
|
|
2367
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2368
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2369
|
+
const int stride_col_dst = dst->ne[0];
|
|
2370
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_1_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2371
|
+
mul_mat_vec_q4_1_q8_1_sycl_switch_ncols(
|
|
2372
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2373
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2374
|
+
return;
|
|
2375
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2376
|
+
mul_mat_vec_q4_1_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2377
|
+
}
|
|
2378
|
+
break;
|
|
2379
|
+
case GGML_TYPE_Q5_0:
|
|
2380
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2381
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2382
|
+
const int stride_col_dst = dst->ne[0];
|
|
2383
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2384
|
+
mul_mat_vec_q5_0_q8_1_sycl_switch_ncols(
|
|
2385
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2386
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2387
|
+
return;
|
|
2388
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2389
|
+
mul_mat_vec_q5_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2390
|
+
}
|
|
2391
|
+
break;
|
|
2392
|
+
case GGML_TYPE_Q5_1:
|
|
2393
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2394
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2395
|
+
const int stride_col_dst = dst->ne[0];
|
|
2396
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_1_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2397
|
+
mul_mat_vec_q5_1_q8_1_sycl_switch_ncols(
|
|
2398
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2399
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2400
|
+
return;
|
|
2401
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2402
|
+
mul_mat_vec_q5_1_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2403
|
+
}
|
|
2404
|
+
break;
|
|
2405
|
+
case GGML_TYPE_Q8_0:
|
|
2406
|
+
if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
|
|
2407
|
+
((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
|
|
2408
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2409
|
+
const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
|
|
2410
|
+
const int stride_col_dst = dst->ne[0];
|
|
2411
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2412
|
+
reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(
|
|
2413
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2414
|
+
src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
|
|
2415
|
+
return;
|
|
2416
|
+
} else {
|
|
2417
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q8_0_q8_1_sycl\n");
|
|
2418
|
+
reorder_mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2419
|
+
}
|
|
2420
|
+
} else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2421
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2422
|
+
const int stride_col_dst = dst->ne[0];
|
|
2423
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q8_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2424
|
+
mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(
|
|
2425
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2426
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2427
|
+
return;
|
|
2428
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2429
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q8_0_q8_1_sycl\n");
|
|
2430
|
+
mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2431
|
+
}
|
|
2432
|
+
break;
|
|
2433
|
+
case GGML_TYPE_Q1_0:
|
|
2434
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2435
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2436
|
+
const int stride_col_dst = dst->ne[0];
|
|
2437
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q1_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2438
|
+
mul_mat_vec_q1_0_q8_1_sycl_switch_ncols(
|
|
2439
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2440
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2441
|
+
return;
|
|
2442
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2443
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q1_0_q8_1_sycl\n");
|
|
2444
|
+
mul_mat_vec_q1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2445
|
+
}
|
|
2446
|
+
break;
|
|
2447
|
+
case GGML_TYPE_Q2_0:
|
|
2448
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2449
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2450
|
+
const int stride_col_dst = dst->ne[0];
|
|
2451
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2452
|
+
mul_mat_vec_q2_0_q8_1_sycl_switch_ncols(
|
|
2453
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2454
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2455
|
+
return;
|
|
2456
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2457
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_0_q8_1_sycl\n");
|
|
2458
|
+
mul_mat_vec_q2_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2459
|
+
}
|
|
2460
|
+
break;
|
|
2461
|
+
case GGML_TYPE_Q2_K:
|
|
2462
|
+
if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
|
|
2463
|
+
((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
|
|
2464
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2465
|
+
const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
|
|
2466
|
+
const int stride_col_dst = dst->ne[0];
|
|
2467
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2468
|
+
reorder_mul_mat_vec_q2_k_q8_1_sycl_switch_ncols(
|
|
2469
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2470
|
+
src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
|
|
2471
|
+
return;
|
|
2472
|
+
} else {
|
|
2473
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q2_k_q8_1_sycl\n");
|
|
2474
|
+
reorder_mul_mat_vec_q2_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2475
|
+
}
|
|
2476
|
+
} else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2477
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2478
|
+
const int stride_col_dst = dst->ne[0];
|
|
2479
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2480
|
+
mul_mat_vec_q2_K_q8_1_sycl_switch_ncols(
|
|
2481
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2482
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2483
|
+
return;
|
|
2484
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2485
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_K_q8_1_sycl\n");
|
|
2486
|
+
mul_mat_vec_q2_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2487
|
+
}
|
|
2488
|
+
break;
|
|
2489
|
+
case GGML_TYPE_Q3_K:
|
|
2490
|
+
if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
|
|
2491
|
+
((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
|
|
2492
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2493
|
+
const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
|
|
2494
|
+
const int stride_col_dst = dst->ne[0];
|
|
2495
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2496
|
+
reorder_mul_mat_vec_q3_k_q8_1_sycl_switch_ncols(
|
|
2497
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2498
|
+
src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
|
|
2499
|
+
return;
|
|
2500
|
+
} else {
|
|
2501
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q3_k_q8_1_sycl\n");
|
|
2502
|
+
reorder_mul_mat_vec_q3_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2503
|
+
}
|
|
2504
|
+
} else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2505
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2506
|
+
const int stride_col_dst = dst->ne[0];
|
|
2507
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q3_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2508
|
+
mul_mat_vec_q3_K_q8_1_sycl_switch_ncols(
|
|
2509
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2510
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2511
|
+
return;
|
|
2512
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2513
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q3_K_q8_1_sycl\n");
|
|
2514
|
+
mul_mat_vec_q3_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2515
|
+
}
|
|
2516
|
+
break;
|
|
2517
|
+
case GGML_TYPE_Q4_K:
|
|
2518
|
+
if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
|
|
2519
|
+
((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
|
|
2520
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2521
|
+
const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
|
|
2522
|
+
const int stride_col_dst = dst->ne[0];
|
|
2523
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2524
|
+
reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols(
|
|
2525
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2526
|
+
src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
|
|
2527
|
+
return;
|
|
2528
|
+
} else {
|
|
2529
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q4_k_q8_1_sycl\n");
|
|
2530
|
+
reorder_mul_mat_vec_q4_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2531
|
+
}
|
|
2532
|
+
} else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2533
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2534
|
+
const int stride_col_dst = dst->ne[0];
|
|
2535
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2536
|
+
mul_mat_vec_q4_K_q8_1_sycl_switch_ncols(
|
|
2537
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2538
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2539
|
+
return;
|
|
2540
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2541
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q4_K_q8_1_sycl\n");
|
|
2542
|
+
mul_mat_vec_q4_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2543
|
+
}
|
|
2544
|
+
break;
|
|
2545
|
+
case GGML_TYPE_Q5_K:
|
|
2546
|
+
if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
|
|
2547
|
+
((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
|
|
2548
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2549
|
+
const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
|
|
2550
|
+
const int stride_col_dst = dst->ne[0];
|
|
2551
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2552
|
+
reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols(
|
|
2553
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2554
|
+
src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
|
|
2555
|
+
return;
|
|
2556
|
+
} else {
|
|
2557
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q5_k_q8_1_sycl\n");
|
|
2558
|
+
reorder_mul_mat_vec_q5_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2559
|
+
}
|
|
2560
|
+
} else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2561
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2562
|
+
const int stride_col_dst = dst->ne[0];
|
|
2563
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2564
|
+
mul_mat_vec_q5_K_q8_1_sycl_switch_ncols(
|
|
2565
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2566
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2567
|
+
return;
|
|
2568
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2569
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q5_K_q8_1_sycl\n");
|
|
2570
|
+
mul_mat_vec_q5_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2571
|
+
}
|
|
2572
|
+
break;
|
|
2573
|
+
case GGML_TYPE_Q6_K:
|
|
2574
|
+
if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
|
|
2575
|
+
((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
|
|
2576
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2577
|
+
const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
|
|
2578
|
+
const int stride_col_dst = dst->ne[0];
|
|
2579
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q6_k_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2580
|
+
reorder_mul_mat_vec_q6_k_q8_1_sycl_switch_ncols(
|
|
2581
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2582
|
+
src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
|
|
2583
|
+
return;
|
|
2584
|
+
} else {
|
|
2585
|
+
GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_q6_k_q8_1_sycl\n");
|
|
2586
|
+
reorder_mul_mat_vec_q6_k_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2587
|
+
}
|
|
2588
|
+
} else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2589
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2590
|
+
const int stride_col_dst = dst->ne[0];
|
|
2591
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q6_K_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2592
|
+
mul_mat_vec_q6_K_q8_1_sycl_switch_ncols(
|
|
2593
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2594
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2595
|
+
return;
|
|
2596
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2597
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_q6_k_q8_1_sycl\n");
|
|
2598
|
+
mul_mat_vec_q6_K_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2599
|
+
}
|
|
2600
|
+
break;
|
|
2601
|
+
case GGML_TYPE_IQ1_S:
|
|
2602
|
+
mul_mat_vec_iq1_s_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2603
|
+
break;
|
|
2604
|
+
case GGML_TYPE_IQ1_M:
|
|
2605
|
+
mul_mat_vec_iq1_m_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2606
|
+
break;
|
|
2607
|
+
case GGML_TYPE_IQ2_XXS:
|
|
2608
|
+
mul_mat_vec_iq2_xxs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2609
|
+
break;
|
|
2610
|
+
case GGML_TYPE_IQ2_XS:
|
|
2611
|
+
mul_mat_vec_iq2_xs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2612
|
+
break;
|
|
2613
|
+
case GGML_TYPE_IQ2_S:
|
|
2614
|
+
mul_mat_vec_iq2_s_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2615
|
+
break;
|
|
2616
|
+
case GGML_TYPE_IQ3_XXS:
|
|
2617
|
+
mul_mat_vec_iq3_xxs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2618
|
+
break;
|
|
2619
|
+
case GGML_TYPE_IQ3_S:
|
|
2620
|
+
mul_mat_vec_iq3_s_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2621
|
+
break;
|
|
2622
|
+
case GGML_TYPE_IQ4_NL:
|
|
2623
|
+
mul_mat_vec_iq4_nl_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2624
|
+
break;
|
|
2625
|
+
case GGML_TYPE_IQ4_XS:
|
|
2626
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2627
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2628
|
+
const int stride_col_dst = dst->ne[0];
|
|
2629
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2630
|
+
mul_mat_vec_iq4_xs_q8_1_sycl_switch_ncols(
|
|
2631
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2632
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2633
|
+
return;
|
|
2634
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2635
|
+
mul_mat_vec_iq4_xs_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2636
|
+
}
|
|
2637
|
+
break;
|
|
2638
|
+
case GGML_TYPE_MXFP4:
|
|
2639
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2640
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2641
|
+
const int stride_col_dst = dst->ne[0];
|
|
2642
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2643
|
+
mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols(
|
|
2644
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2645
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2646
|
+
return;
|
|
2647
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2648
|
+
mul_mat_vec_mxfp4_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2649
|
+
}
|
|
2650
|
+
break;
|
|
2651
|
+
case GGML_TYPE_NVFP4:
|
|
2652
|
+
if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
|
|
2653
|
+
const int stride_col_y = src1_padded_col_size / QK8_1;
|
|
2654
|
+
const int stride_col_dst = dst->ne[0];
|
|
2655
|
+
GGML_SYCL_DEBUG("Calling mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
|
|
2656
|
+
mul_mat_vec_nvfp4_q8_1_sycl_switch_ncols(
|
|
2657
|
+
src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
|
|
2658
|
+
src1_ncols, stride_col_y, stride_col_dst, stream);
|
|
2659
|
+
return;
|
|
2660
|
+
} else if (i == 0 || src1_ncols == 1) {
|
|
2661
|
+
mul_mat_vec_nvfp4_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
|
|
2662
|
+
}
|
|
2663
|
+
break;
|
|
2664
|
+
default:
|
|
2665
|
+
GGML_ABORT("fatal error: unsupport src0 data type %s\n", ggml_type_name(src0->type));
|
|
2666
|
+
}
|
|
2667
|
+
}
|
|
2668
|
+
GGML_UNUSED(src1);
|
|
2669
|
+
GGML_UNUSED(dst);
|
|
2670
|
+
GGML_UNUSED(src1_ddf_i);
|
|
2671
|
+
GGML_UNUSED(ctx);
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2674
|
+
// src1_row_stride: 0 for shared src1 (gate/up proj), else per-expert stride (down proj).
|
|
2675
|
+
template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl>
|
|
2676
|
+
static void mul_mat_vec_q_moe(
|
|
2677
|
+
const void * __restrict__ vx_base, const void * __restrict__ vy_base,
|
|
2678
|
+
float * __restrict__ dst_base, const int32_t * __restrict__ ids_dev,
|
|
2679
|
+
const int ncols, const int nrows,
|
|
2680
|
+
const size_t expert_weight_stride, const size_t dst_row_stride,
|
|
2681
|
+
const size_t src1_row_stride,
|
|
2682
|
+
const sycl::nd_item<3> & item_ct1) {
|
|
2683
|
+
|
|
2684
|
+
const int expert_idx = item_ct1.get_group(1);
|
|
2685
|
+
const int i02 = ids_dev[expert_idx];
|
|
2686
|
+
|
|
2687
|
+
const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride;
|
|
2688
|
+
const char * vy = (const char *) vy_base + (size_t) expert_idx * src1_row_stride;
|
|
2689
|
+
float * dst = (float *) ((char *) dst_base + (size_t) expert_idx * dst_row_stride);
|
|
2690
|
+
|
|
2691
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1);
|
|
2692
|
+
|
|
2693
|
+
if (row >= nrows) {
|
|
2694
|
+
return;
|
|
2695
|
+
}
|
|
2696
|
+
|
|
2697
|
+
const int blocks_per_row = ncols / qk;
|
|
2698
|
+
constexpr int blocks_per_warp = (vdr * WARP_SIZE + qi - 1) / qi;
|
|
2699
|
+
|
|
2700
|
+
float tmp = 0.0f;
|
|
2701
|
+
|
|
2702
|
+
const block_q_t * x = (const block_q_t *) vx;
|
|
2703
|
+
const block_q8_1 * y = (const block_q8_1 *) vy;
|
|
2704
|
+
|
|
2705
|
+
for (int i = item_ct1.get_local_id(2) / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) {
|
|
2706
|
+
const int ibx = row * blocks_per_row + i;
|
|
2707
|
+
const int iby = i * (qk / QK8_1);
|
|
2708
|
+
|
|
2709
|
+
for (size_t elem = 0; elem < qi / vdr; elem += WARP_SIZE) {
|
|
2710
|
+
const int iqs = elem + vdr * (item_ct1.get_local_id(2) % (qi / vdr));
|
|
2711
|
+
tmp += vec_dot_q_sycl(&x[ibx], &y[iby], iqs);
|
|
2712
|
+
}
|
|
2713
|
+
}
|
|
2714
|
+
|
|
2715
|
+
#pragma unroll
|
|
2716
|
+
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
|
2717
|
+
tmp += dpct::permute_sub_group_by_xor(item_ct1.get_sub_group(), tmp, mask);
|
|
2718
|
+
}
|
|
2719
|
+
|
|
2720
|
+
if (item_ct1.get_local_id(2) == 0) {
|
|
2721
|
+
dst[row] = tmp;
|
|
2722
|
+
}
|
|
2723
|
+
}
|
|
2724
|
+
|
|
2725
|
+
template <int qk, int qi, typename block_q_t, int vdr, vec_dot_q_sycl_t vec_dot_q_sycl>
|
|
2726
|
+
static void launch_mul_mat_vec_q_moe(
|
|
2727
|
+
const void * vx_base, const void * vy, const int32_t * ids_dev,
|
|
2728
|
+
float * dst_base, const int ncols, const int nrows, const int n_experts_used,
|
|
2729
|
+
const size_t expert_weight_stride, const size_t dst_row_stride,
|
|
2730
|
+
const size_t src1_row_stride,
|
|
2731
|
+
dpct::queue_ptr stream) {
|
|
2732
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2733
|
+
const sycl::range<3> block_nums(1, (unsigned) n_experts_used, (unsigned) block_num_y);
|
|
2734
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2735
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
2736
|
+
cgh.parallel_for(
|
|
2737
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2738
|
+
[=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2739
|
+
mul_mat_vec_q_moe<qk, qi, block_q_t, vdr, vec_dot_q_sycl>(
|
|
2740
|
+
vx_base, vy, dst_base, ids_dev, ncols, nrows,
|
|
2741
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, item);
|
|
2742
|
+
});
|
|
2743
|
+
});
|
|
2744
|
+
}
|
|
2745
|
+
|
|
2746
|
+
bool ggml_sycl_mul_mat_vec_q_id(
|
|
2747
|
+
enum ggml_type src0_type,
|
|
2748
|
+
const void * vx_base,
|
|
2749
|
+
const void * vy,
|
|
2750
|
+
const int32_t * ids_dev,
|
|
2751
|
+
float * dst_base,
|
|
2752
|
+
int ncols,
|
|
2753
|
+
int nrows,
|
|
2754
|
+
int n_experts_used,
|
|
2755
|
+
size_t expert_weight_stride,
|
|
2756
|
+
size_t dst_row_stride,
|
|
2757
|
+
size_t src1_row_stride,
|
|
2758
|
+
dpct::queue_ptr stream) {
|
|
2759
|
+
switch (src0_type) {
|
|
2760
|
+
case GGML_TYPE_Q4_0:
|
|
2761
|
+
launch_mul_mat_vec_q_moe<QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>(
|
|
2762
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2763
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2764
|
+
return true;
|
|
2765
|
+
case GGML_TYPE_Q4_1:
|
|
2766
|
+
launch_mul_mat_vec_q_moe<QK4_1, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>(
|
|
2767
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2768
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2769
|
+
return true;
|
|
2770
|
+
case GGML_TYPE_Q5_0:
|
|
2771
|
+
launch_mul_mat_vec_q_moe<QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>(
|
|
2772
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2773
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2774
|
+
return true;
|
|
2775
|
+
case GGML_TYPE_Q5_1:
|
|
2776
|
+
launch_mul_mat_vec_q_moe<QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>(
|
|
2777
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2778
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2779
|
+
return true;
|
|
2780
|
+
case GGML_TYPE_Q8_0:
|
|
2781
|
+
launch_mul_mat_vec_q_moe<QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>(
|
|
2782
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2783
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2784
|
+
return true;
|
|
2785
|
+
case GGML_TYPE_Q2_0:
|
|
2786
|
+
launch_mul_mat_vec_q_moe<QK2_0, QI2_0, block_q2_0, VDR_Q2_0_Q8_1_MMVQ, vec_dot_q2_0_q8_1>(
|
|
2787
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2788
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2789
|
+
return true;
|
|
2790
|
+
case GGML_TYPE_Q2_K:
|
|
2791
|
+
launch_mul_mat_vec_q_moe<QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>(
|
|
2792
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2793
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2794
|
+
return true;
|
|
2795
|
+
case GGML_TYPE_Q3_K:
|
|
2796
|
+
launch_mul_mat_vec_q_moe<QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>(
|
|
2797
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2798
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2799
|
+
return true;
|
|
2800
|
+
case GGML_TYPE_Q4_K:
|
|
2801
|
+
launch_mul_mat_vec_q_moe<QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>(
|
|
2802
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2803
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2804
|
+
return true;
|
|
2805
|
+
case GGML_TYPE_Q5_K:
|
|
2806
|
+
launch_mul_mat_vec_q_moe<QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>(
|
|
2807
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2808
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2809
|
+
return true;
|
|
2810
|
+
case GGML_TYPE_Q6_K:
|
|
2811
|
+
launch_mul_mat_vec_q_moe<QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>(
|
|
2812
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2813
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2814
|
+
return true;
|
|
2815
|
+
case GGML_TYPE_MXFP4:
|
|
2816
|
+
launch_mul_mat_vec_q_moe<QK_MXFP4, QI_MXFP4, block_mxfp4, VDR_MXFP4_Q8_1_MMVQ, vec_dot_mxfp4_q8_1>(
|
|
2817
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2818
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2819
|
+
return true;
|
|
2820
|
+
case GGML_TYPE_NVFP4:
|
|
2821
|
+
launch_mul_mat_vec_q_moe<QK_NVFP4, QI_NVFP4, block_nvfp4, VDR_NVFP4_Q8_1_MMVQ, vec_dot_nvfp4_q8_1>(
|
|
2822
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2823
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2824
|
+
return true;
|
|
2825
|
+
default:
|
|
2826
|
+
return false;
|
|
2827
|
+
}
|
|
2828
|
+
}
|
|
2829
|
+
|
|
2830
|
+
// Reorder (SoA) MoE expert GEMV: MoE expert/row/lane indexing (from mul_mat_vec_q_moe) with the
|
|
2831
|
+
// dense-reorder per-block reads (from mul_mat_vec_q_reorder). Each expert slice in vx_base is a
|
|
2832
|
+
// self-contained SoA, so nblocks = nrows*(ncols/qk) per expert and the constant expert stride holds.
|
|
2833
|
+
template <typename reorder_vec_dot_q_sycl>
|
|
2834
|
+
static void mul_mat_vec_q_moe_reorder(
|
|
2835
|
+
const void * __restrict__ vx_base, const void * __restrict__ vy_base,
|
|
2836
|
+
float * __restrict__ dst_base, const int32_t * __restrict__ ids_dev,
|
|
2837
|
+
const int ncols, const int nrows,
|
|
2838
|
+
const size_t expert_weight_stride, const size_t dst_row_stride,
|
|
2839
|
+
const size_t src1_row_stride,
|
|
2840
|
+
const sycl::nd_item<3> & item_ct1) {
|
|
2841
|
+
using block_type = ggml_sycl_reordered::block_q_t<reorder_vec_dot_q_sycl::gtype>;
|
|
2842
|
+
using block_traits = typename block_type::traits;
|
|
2843
|
+
|
|
2844
|
+
const int expert_idx = item_ct1.get_group(1);
|
|
2845
|
+
const int i02 = ids_dev[expert_idx];
|
|
2846
|
+
|
|
2847
|
+
const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride;
|
|
2848
|
+
const char * vy = (const char *) vy_base + (size_t) expert_idx * src1_row_stride;
|
|
2849
|
+
float * dst = (float *) ((char *) dst_base + (size_t) expert_idx * dst_row_stride);
|
|
2850
|
+
|
|
2851
|
+
const int row = item_ct1.get_group(2) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1);
|
|
2852
|
+
if (row >= nrows) {
|
|
2853
|
+
return;
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
const auto sg = item_ct1.get_sub_group();
|
|
2857
|
+
|
|
2858
|
+
const int blocks_per_row = ncols / block_traits::qk;
|
|
2859
|
+
constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);
|
|
2860
|
+
constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;
|
|
2861
|
+
const int nblocks = nrows * (ncols / block_traits::qk);
|
|
2862
|
+
|
|
2863
|
+
static_assert(blocks_per_subgroup > 0);
|
|
2864
|
+
static_assert(block_elements_per_subgroup > 0);
|
|
2865
|
+
|
|
2866
|
+
float partial_sum = 0.0f;
|
|
2867
|
+
for (int i = sg.get_local_linear_id() / block_elements_per_subgroup; i < blocks_per_row; i += blocks_per_subgroup) {
|
|
2868
|
+
const int ibx = row * blocks_per_row + i;
|
|
2869
|
+
|
|
2870
|
+
const auto bx_offset = block_type::get_block_offset(ibx, nblocks);
|
|
2871
|
+
const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx);
|
|
2872
|
+
|
|
2873
|
+
const int iby = i * block_type::block_to_q8_1_ratio();
|
|
2874
|
+
const int8_t * q8_1_quant_ptr = (const int8_t *) vy + iby * QK8_1;
|
|
2875
|
+
const sycl::half2 * q8_1_ds_ptr = (const sycl::half2 *) ((const char *) vy + ncols + iby * sizeof(sycl::half2));
|
|
2876
|
+
|
|
2877
|
+
#pragma unroll
|
|
2878
|
+
for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {
|
|
2879
|
+
const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
|
|
2880
|
+
partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
|
|
2881
|
+
}
|
|
2882
|
+
}
|
|
2883
|
+
|
|
2884
|
+
auto sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());
|
|
2885
|
+
if (sg.leader()) {
|
|
2886
|
+
dst[row] = sum;
|
|
2887
|
+
}
|
|
2888
|
+
}
|
|
2889
|
+
|
|
2890
|
+
template <typename reorder_vec_dot_q_sycl>
|
|
2891
|
+
static void launch_mul_mat_vec_q_moe_reorder(
|
|
2892
|
+
const void * vx_base, const void * vy, const int32_t * ids_dev,
|
|
2893
|
+
float * dst_base, const int ncols, const int nrows, const int n_experts_used,
|
|
2894
|
+
const size_t expert_weight_stride, const size_t dst_row_stride,
|
|
2895
|
+
const size_t src1_row_stride,
|
|
2896
|
+
dpct::queue_ptr stream) {
|
|
2897
|
+
const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y;
|
|
2898
|
+
const sycl::range<3> block_nums(1, (unsigned) n_experts_used, (unsigned) block_num_y);
|
|
2899
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE);
|
|
2900
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
2901
|
+
cgh.parallel_for(
|
|
2902
|
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2903
|
+
[=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2904
|
+
mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl>(
|
|
2905
|
+
vx_base, vy, dst_base, ids_dev, ncols, nrows,
|
|
2906
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, item);
|
|
2907
|
+
});
|
|
2908
|
+
});
|
|
2909
|
+
}
|
|
2910
|
+
|
|
2911
|
+
bool ggml_sycl_mul_mat_vec_q_id_reorder(
|
|
2912
|
+
enum ggml_type src0_type,
|
|
2913
|
+
const void * vx_base,
|
|
2914
|
+
const void * vy,
|
|
2915
|
+
const int32_t * ids_dev,
|
|
2916
|
+
float * dst_base,
|
|
2917
|
+
int ncols,
|
|
2918
|
+
int nrows,
|
|
2919
|
+
int n_experts_used,
|
|
2920
|
+
size_t expert_weight_stride,
|
|
2921
|
+
size_t dst_row_stride,
|
|
2922
|
+
size_t src1_row_stride,
|
|
2923
|
+
dpct::queue_ptr stream) {
|
|
2924
|
+
switch (src0_type) {
|
|
2925
|
+
case GGML_TYPE_Q4_K:
|
|
2926
|
+
launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>>(
|
|
2927
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2928
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2929
|
+
return true;
|
|
2930
|
+
case GGML_TYPE_Q5_K:
|
|
2931
|
+
launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q5_K>>(
|
|
2932
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2933
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2934
|
+
return true;
|
|
2935
|
+
case GGML_TYPE_Q6_K:
|
|
2936
|
+
launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q6_K>>(
|
|
2937
|
+
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
|
2938
|
+
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
|
2939
|
+
return true;
|
|
2940
|
+
default:
|
|
2941
|
+
return false;
|
|
2942
|
+
}
|
|
2943
|
+
}
|
|
2944
|
+
|
|
2945
|
+
template <typename reorder_vec_dot_q_sycl, int ncols_dst, int rows_per_sg>
|
|
2946
|
+
static void launch_mul_mat_vec_q_reorder_glu_impl(const void * vx, const void * vgate, const void * vy, float * dst,
|
|
2947
|
+
const int ncols, const int nrows, const int stride_col_y_bytes,
|
|
2948
|
+
const int stride_col_dst, const ggml_glu_op glu_op,
|
|
2949
|
+
dpct::queue_ptr stream) {
|
|
2950
|
+
GGML_ASSERT(ncols % QK_K == 0);
|
|
2951
|
+
|
|
2952
|
+
constexpr size_t num_subgroups = WARP_SIZE;
|
|
2953
|
+
|
|
2954
|
+
const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg);
|
|
2955
|
+
const sycl::range<3> block_nums(1, 1, block_num_y);
|
|
2956
|
+
const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
|
|
2957
|
+
|
|
2958
|
+
stream->submit([&](sycl::handler & cgh) {
|
|
2959
|
+
cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
|
2960
|
+
[=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
|
2961
|
+
mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl, ncols_dst, /*has_fusion=*/ true,
|
|
2962
|
+
rows_per_sg>(
|
|
2963
|
+
vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op,
|
|
2964
|
+
nd_item);
|
|
2965
|
+
});
|
|
2966
|
+
});
|
|
2967
|
+
}
|
|
2968
|
+
|
|
2969
|
+
template <typename reorder_vec_dot_q_sycl, int ncols_dst>
|
|
2970
|
+
static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate, const void * vy, float * dst,
|
|
2971
|
+
const int ncols, const int nrows, const int stride_col_y_bytes,
|
|
2972
|
+
const int stride_col_dst, const ggml_glu_op glu_op,
|
|
2973
|
+
dpct::queue_ptr stream) {
|
|
2974
|
+
constexpr int rows_per_sg =
|
|
2975
|
+
reorder_vec_dot_shared_activations<reorder_vec_dot_q_sycl::gtype>::value && ncols_dst >= 3 && ncols_dst <= 4
|
|
2976
|
+
? 2
|
|
2977
|
+
: 1;
|
|
2978
|
+
launch_mul_mat_vec_q_reorder_glu_impl<reorder_vec_dot_q_sycl, ncols_dst, rows_per_sg>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream);
|
|
2979
|
+
}
|
|
2980
|
+
|
|
2981
|
+
bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx,
|
|
2982
|
+
const void * vgate, const void * vy, float * dst, int ncols, int nrows,
|
|
2983
|
+
int ncols_dst, int stride_col_y_bytes, int stride_col_dst,
|
|
2984
|
+
dpct::queue_ptr stream) {
|
|
2985
|
+
if (src0_type != GGML_TYPE_Q4_K) {
|
|
2986
|
+
return false;
|
|
2987
|
+
}
|
|
2988
|
+
if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) {
|
|
2989
|
+
return false;
|
|
2990
|
+
}
|
|
2991
|
+
|
|
2992
|
+
using vec_dot = reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>;
|
|
2993
|
+
|
|
2994
|
+
switch (ncols_dst) {
|
|
2995
|
+
case 1:
|
|
2996
|
+
launch_mul_mat_vec_q_reorder_glu<vec_dot, 1>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
|
|
2997
|
+
stride_col_dst, glu_op, stream);
|
|
2998
|
+
return true;
|
|
2999
|
+
case 2:
|
|
3000
|
+
if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) {
|
|
3001
|
+
launch_mul_mat_vec_q_reorder_glu_impl<vec_dot, 2, 2>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream);
|
|
3002
|
+
} else {
|
|
3003
|
+
launch_mul_mat_vec_q_reorder_glu_impl<vec_dot, 2, 1>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream);
|
|
3004
|
+
}
|
|
3005
|
+
return true;
|
|
3006
|
+
case 3:
|
|
3007
|
+
launch_mul_mat_vec_q_reorder_glu<vec_dot, 3>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
|
|
3008
|
+
stride_col_dst, glu_op, stream);
|
|
3009
|
+
return true;
|
|
3010
|
+
case 4:
|
|
3011
|
+
launch_mul_mat_vec_q_reorder_glu<vec_dot, 4>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
|
|
3012
|
+
stride_col_dst, glu_op, stream);
|
|
3013
|
+
return true;
|
|
3014
|
+
case 5:
|
|
3015
|
+
launch_mul_mat_vec_q_reorder_glu<vec_dot, 5>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
|
|
3016
|
+
stride_col_dst, glu_op, stream);
|
|
3017
|
+
return true;
|
|
3018
|
+
case 6:
|
|
3019
|
+
launch_mul_mat_vec_q_reorder_glu<vec_dot, 6>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
|
|
3020
|
+
stride_col_dst, glu_op, stream);
|
|
3021
|
+
return true;
|
|
3022
|
+
case 7:
|
|
3023
|
+
launch_mul_mat_vec_q_reorder_glu<vec_dot, 7>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
|
|
3024
|
+
stride_col_dst, glu_op, stream);
|
|
3025
|
+
return true;
|
|
3026
|
+
case 8:
|
|
3027
|
+
launch_mul_mat_vec_q_reorder_glu<vec_dot, 8>(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes,
|
|
3028
|
+
stride_col_dst, glu_op, stream);
|
|
3029
|
+
return true;
|
|
3030
|
+
default:
|
|
3031
|
+
return false;
|
|
3032
|
+
}
|
|
3033
|
+
}
|