elelem-ggml 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/exe/elelem-ggml +8 -0
- data/ext/elelem/ggml/elelem.c +466 -0
- data/ext/elelem/ggml/extconf.rb +52 -0
- data/lib/elelem/ggml/chat_ml.rb +23 -0
- data/lib/elelem/ggml/gguf.rb +82 -0
- data/lib/elelem/ggml/provider.rb +63 -0
- data/lib/elelem/ggml/tokenizer.rb +95 -0
- data/lib/elelem/ggml/version.rb +7 -0
- data/lib/elelem/ggml.rb +37 -0
- data/vendor/ggml/AUTHORS +335 -0
- data/vendor/ggml/CMakeLists.txt +499 -0
- data/vendor/ggml/CONTRIBUTING.md +3 -0
- data/vendor/ggml/LICENSE +21 -0
- data/vendor/ggml/README.md +49 -0
- data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
- data/vendor/ggml/cmake/GitVars.cmake +22 -0
- data/vendor/ggml/cmake/common.cmake +50 -0
- data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
- data/vendor/ggml/ggml.pc.in +10 -0
- data/vendor/ggml/include/ggml-alloc.h +86 -0
- data/vendor/ggml/include/ggml-backend.h +437 -0
- data/vendor/ggml/include/ggml-blas.h +25 -0
- data/vendor/ggml/include/ggml-cann.h +123 -0
- data/vendor/ggml/include/ggml-cpp.h +39 -0
- data/vendor/ggml/include/ggml-cpu.h +152 -0
- data/vendor/ggml/include/ggml-cuda.h +47 -0
- data/vendor/ggml/include/ggml-et.h +28 -0
- data/vendor/ggml/include/ggml-hexagon.h +19 -0
- data/vendor/ggml/include/ggml-metal.h +61 -0
- data/vendor/ggml/include/ggml-opencl.h +26 -0
- data/vendor/ggml/include/ggml-openvino.h +37 -0
- data/vendor/ggml/include/ggml-opt.h +256 -0
- data/vendor/ggml/include/ggml-rpc.h +35 -0
- data/vendor/ggml/include/ggml-sycl.h +57 -0
- data/vendor/ggml/include/ggml-virtgpu.h +14 -0
- data/vendor/ggml/include/ggml-vulkan.h +29 -0
- data/vendor/ggml/include/ggml-webgpu.h +19 -0
- data/vendor/ggml/include/ggml-zdnn.h +17 -0
- data/vendor/ggml/include/ggml-zendnn.h +22 -0
- data/vendor/ggml/include/ggml.h +2963 -0
- data/vendor/ggml/include/gguf.h +211 -0
- data/vendor/ggml/requirements.txt +12 -0
- data/vendor/ggml/scripts/gen-authors.sh +9 -0
- data/vendor/ggml/scripts/make-release-checks.sh +96 -0
- data/vendor/ggml/scripts/make-release-desc.sh +69 -0
- data/vendor/ggml/scripts/make-release-summary.txt +33 -0
- data/vendor/ggml/scripts/release.sh +195 -0
- data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
- data/vendor/ggml/scripts/sync-llama.last +1 -0
- data/vendor/ggml/scripts/sync-llama.sh +21 -0
- data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
- data/vendor/ggml/scripts/sync-whisper.last +1 -0
- data/vendor/ggml/scripts/sync-whisper.sh +17 -0
- data/vendor/ggml/src/CMakeLists.txt +634 -0
- data/vendor/ggml/src/ggml-alloc.c +1249 -0
- data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
- data/vendor/ggml/src/ggml-backend-dl.h +45 -0
- data/vendor/ggml/src/ggml-backend-impl.h +291 -0
- data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
- data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
- data/vendor/ggml/src/ggml-backend.cpp +2517 -0
- data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
- data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
- data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
- data/vendor/ggml/src/ggml-cann/common.h +651 -0
- data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
- data/vendor/ggml/src/ggml-common.h +1911 -0
- data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
- data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
- data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
- data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
- data/vendor/ggml/src/ggml-cpu/common.h +95 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
- data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
- data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
- data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
- data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
- data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
- data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
- data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
- data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
- data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
- data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
- data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
- data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
- data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
- data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
- data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
- data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
- data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
- data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
- data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
- data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
- data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
- data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
- data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
- data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
- data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
- data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
- data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
- data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
- data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
- data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
- data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
- data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
- data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
- data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
- data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
- data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
- data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
- data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
- data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
- data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
- data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
- data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
- data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
- data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
- data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
- data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
- data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
- data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
- data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
- data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
- data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
- data/vendor/ggml/src/ggml-feats.h +166 -0
- data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
- data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
- data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
- data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
- data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
- data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
- data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
- data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
- data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
- data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
- data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
- data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
- data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
- data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
- data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
- data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
- data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
- data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
- data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
- data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
- data/vendor/ggml/src/ggml-impl.h +783 -0
- data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
- data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
- data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
- data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
- data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
- data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
- data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
- data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
- data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
- data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
- data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
- data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
- data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
- data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
- data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
- data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
- data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
- data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
- data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
- data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
- data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
- data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
- data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
- data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
- data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
- data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
- data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
- data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
- data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
- data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
- data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
- data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
- data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
- data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
- data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
- data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
- data/vendor/ggml/src/ggml-opt.cpp +1094 -0
- data/vendor/ggml/src/ggml-quants.c +5667 -0
- data/vendor/ggml/src/ggml-quants.h +115 -0
- data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
- data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
- data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
- data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
- data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
- data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
- data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
- data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
- data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
- data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
- data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
- data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
- data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
- data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
- data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
- data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
- data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
- data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
- data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
- data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
- data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
- data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
- data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
- data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
- data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
- data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
- data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
- data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
- data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
- data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
- data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
- data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
- data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
- data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
- data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
- data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
- data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
- data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
- data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
- data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
- data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
- data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
- data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
- data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
- data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
- data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
- data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
- data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
- data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
- data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/vendor/ggml/src/ggml-threading.cpp +12 -0
- data/vendor/ggml/src/ggml-threading.h +14 -0
- data/vendor/ggml/src/ggml-version.h.in +4 -0
- data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
- data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
- data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
- data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
- data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
- data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
- data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
- data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
- data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
- data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
- data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
- data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
- data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
- data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
- data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
- data/vendor/ggml/src/ggml.c +8088 -0
- data/vendor/ggml/src/ggml.cpp +26 -0
- data/vendor/ggml/src/gguf.cpp +1706 -0
- metadata +1459 -0
|
@@ -0,0 +1,2663 @@
|
|
|
1
|
+
//------------------------------------------------------------------------------
|
|
2
|
+
// This file is contains kernels for data conversion.
|
|
3
|
+
// These kernels are used when loading the model, so its performance is less
|
|
4
|
+
// important.
|
|
5
|
+
//------------------------------------------------------------------------------
|
|
6
|
+
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
|
7
|
+
|
|
8
|
+
#ifdef cl_intel_required_subgroup_size
|
|
9
|
+
#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
|
|
10
|
+
#define INTEL_GPU 1
|
|
11
|
+
#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
|
|
12
|
+
#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
|
|
13
|
+
#elif defined(cl_qcom_reqd_sub_group_size)
|
|
14
|
+
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
|
|
15
|
+
#define ADRENO_GPU 1
|
|
16
|
+
#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
|
|
17
|
+
#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
|
|
18
|
+
#endif
|
|
19
|
+
|
|
20
|
+
#define QK4_0 32
|
|
21
|
+
#define QR4_0 2
|
|
22
|
+
#define QK4_1 32
|
|
23
|
+
#define QR4_1 2
|
|
24
|
+
#define QK5_0 32
|
|
25
|
+
#define QR5_0 2
|
|
26
|
+
#define QK5_1 32
|
|
27
|
+
#define QR5_1 2
|
|
28
|
+
#define QK8_0 32
|
|
29
|
+
#define QR8_0 1
|
|
30
|
+
#define QK1_0 128
|
|
31
|
+
#define QR1_0 1
|
|
32
|
+
#define QK_K 256
|
|
33
|
+
#define K_SCALE_SIZE (3 * QK_K / 64)
|
|
34
|
+
#define K_QUANTS_PER_ITERATION 2
|
|
35
|
+
|
|
36
|
+
typedef char int8_t;
|
|
37
|
+
typedef uchar uint8_t;
|
|
38
|
+
typedef short int16_t;
|
|
39
|
+
typedef ushort uint16_t;
|
|
40
|
+
typedef int int32_t;
|
|
41
|
+
typedef uint uint32_t;
|
|
42
|
+
|
|
43
|
+
//------------------------------------------------------------------------------
|
|
44
|
+
// block_q1_0
|
|
45
|
+
//------------------------------------------------------------------------------
|
|
46
|
+
typedef struct {
|
|
47
|
+
half d; // delta
|
|
48
|
+
uchar qs[QK1_0/8]; // 1-bit signs (16 bytes)
|
|
49
|
+
} block_q1_0;
|
|
50
|
+
|
|
51
|
+
//------------------------------------------------------------------------------
|
|
52
|
+
// block_q4_0
|
|
53
|
+
//------------------------------------------------------------------------------
|
|
54
|
+
struct block_q4_0
|
|
55
|
+
{
|
|
56
|
+
half d;
|
|
57
|
+
uint8_t qs[QK4_0 / 2];
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
//------------------------------------------------------------------------------
|
|
61
|
+
// block_q4_1
|
|
62
|
+
//------------------------------------------------------------------------------
|
|
63
|
+
struct block_q4_1 {
|
|
64
|
+
half d; // delta
|
|
65
|
+
half m; // min
|
|
66
|
+
uchar qs[QK4_1 / 2]; // nibbles / quants
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
//------------------------------------------------------------------------------
|
|
70
|
+
// block_q5_0
|
|
71
|
+
//------------------------------------------------------------------------------
|
|
72
|
+
struct block_q5_0 {
|
|
73
|
+
half d; // delta
|
|
74
|
+
uchar qh[4]; // 5-th bit of quants
|
|
75
|
+
uchar qs[QK5_0 / 2]; // nibbles / quants
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
//------------------------------------------------------------------------------
|
|
79
|
+
// block_q5_1
|
|
80
|
+
//------------------------------------------------------------------------------
|
|
81
|
+
struct block_q5_1 {
|
|
82
|
+
half d; // delta
|
|
83
|
+
half m; // min
|
|
84
|
+
uchar qh[4]; // 5-th bit of quants
|
|
85
|
+
uchar qs[QK5_1 / 2]; // nibbles / quants
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
//------------------------------------------------------------------------------
|
|
89
|
+
// block_q4_k
|
|
90
|
+
//------------------------------------------------------------------------------
|
|
91
|
+
struct block_q4_K {
|
|
92
|
+
half d; // delta
|
|
93
|
+
half dm; // min
|
|
94
|
+
uchar s[K_SCALE_SIZE];
|
|
95
|
+
uchar q[QK_K / 2]; // nibbles / quants
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
//------------------------------------------------------------------------------
|
|
99
|
+
// block_q5_k
|
|
100
|
+
//------------------------------------------------------------------------------
|
|
101
|
+
struct block_q5_K {
|
|
102
|
+
half d; // delta
|
|
103
|
+
half dm; // min
|
|
104
|
+
uchar s[K_SCALE_SIZE];
|
|
105
|
+
uchar qh[QK_K / 8];
|
|
106
|
+
uchar qs[QK_K / 2]; // nibbles / quants
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
//------------------------------------------------------------------------------
|
|
110
|
+
// block_q6_K
|
|
111
|
+
//------------------------------------------------------------------------------
|
|
112
|
+
struct block_q6_K {
|
|
113
|
+
uint8_t ql[QK_K/2]; // quants, lower 4 bits
|
|
114
|
+
uint8_t qh[QK_K/4]; // quants, upper 2 bits
|
|
115
|
+
int8_t scales[QK_K/16]; // scales, quantized with 8 bits
|
|
116
|
+
half d; // super-block scale
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
//------------------------------------------------------------------------------
|
|
120
|
+
// block_iq4_nl
|
|
121
|
+
//------------------------------------------------------------------------------
|
|
122
|
+
#define QK4_NL 32
|
|
123
|
+
|
|
124
|
+
struct block_iq4_nl
|
|
125
|
+
{
|
|
126
|
+
half d;
|
|
127
|
+
uint8_t qs[QK4_NL / 2];
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
//------------------------------------------------------------------------------
|
|
131
|
+
// bf16 to f16
|
|
132
|
+
//------------------------------------------------------------------------------
|
|
133
|
+
kernel void kernel_convert_bf16_to_f16(
|
|
134
|
+
global const ushort * src,
|
|
135
|
+
global half * dst,
|
|
136
|
+
ulong off_dst,
|
|
137
|
+
ulong n
|
|
138
|
+
) {
|
|
139
|
+
uint i = get_global_id(0);
|
|
140
|
+
if (i >= n) {
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
dst[i + off_dst] = (half) as_float((uint) src[i] << 16);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
//------------------------------------------------------------------------------
|
|
148
|
+
// f16 to bf16
|
|
149
|
+
//------------------------------------------------------------------------------
|
|
150
|
+
kernel void kernel_convert_f16_to_bf16(
|
|
151
|
+
global const half * src,
|
|
152
|
+
ulong off_src,
|
|
153
|
+
global ushort * dst,
|
|
154
|
+
ulong n
|
|
155
|
+
) {
|
|
156
|
+
uint i = get_global_id(0);
|
|
157
|
+
if (i >= n) {
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
float f = (float) src[i + off_src];
|
|
162
|
+
uint bits = as_uint(f);
|
|
163
|
+
if ((bits & 0x7fffffffu) > 0x7f800000u) {
|
|
164
|
+
// nan to quiet nan
|
|
165
|
+
dst[i] = (ushort)((bits >> 16) | 0x40u);
|
|
166
|
+
} else {
|
|
167
|
+
uint rounded = bits + 0x7fffu + ((bits >> 16) & 1u);
|
|
168
|
+
dst[i] = (ushort)(rounded >> 16);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
//------------------------------------------------------------------------------
|
|
173
|
+
// kernel_convert_block_q1_0
|
|
174
|
+
// Convert block_q1_0 (AOS) to 2 separate arrays (SOA): quant bytes + scales.
|
|
175
|
+
// q1_0 bits are stored in natural order (bit j of byte i -> weight 8*i + j)
|
|
176
|
+
//------------------------------------------------------------------------------
|
|
177
|
+
kernel void kernel_convert_block_q1_0(
|
|
178
|
+
global block_q1_0 * src0,
|
|
179
|
+
global uchar * dst_q,
|
|
180
|
+
global half * dst_d
|
|
181
|
+
) {
|
|
182
|
+
global block_q1_0 * b = (global block_q1_0 *) src0 + get_global_id(0);
|
|
183
|
+
global uchar * q = (global uchar *) dst_q + (QK1_0/8)*get_global_id(0);
|
|
184
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
185
|
+
|
|
186
|
+
*d = b->d;
|
|
187
|
+
|
|
188
|
+
for (int i = 0; i < QK1_0/8; ++i) {
|
|
189
|
+
q[i] = b->qs[i];
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
kernel void kernel_restore_block_q1_0(
|
|
194
|
+
global uchar * src_q,
|
|
195
|
+
global half * src_d,
|
|
196
|
+
global block_q1_0 * dst
|
|
197
|
+
) {
|
|
198
|
+
global block_q1_0 * b = (global block_q1_0 *) dst + get_global_id(0);
|
|
199
|
+
global uchar * q = (global uchar *) src_q + (QK1_0/8)*get_global_id(0);
|
|
200
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
201
|
+
|
|
202
|
+
b->d = *d;
|
|
203
|
+
for (int i = 0; i < QK1_0/8; ++i) {
|
|
204
|
+
b->qs[i] = q[i];
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
//------------------------------------------------------------------------------
|
|
209
|
+
// kernel_convert_block_q4_0
|
|
210
|
+
// Convert the block_q4_0 format to 2 separate arrays (AOS -> SOA).
|
|
211
|
+
// This kernel does not deshuffle the bits.
|
|
212
|
+
//------------------------------------------------------------------------------
|
|
213
|
+
kernel void kernel_convert_block_q4_0(
|
|
214
|
+
global struct block_q4_0 * src0,
|
|
215
|
+
global uchar * dst_q,
|
|
216
|
+
global half * dst_d
|
|
217
|
+
) {
|
|
218
|
+
global struct block_q4_0 * b = (global struct block_q4_0 *) src0 + get_global_id(0);
|
|
219
|
+
global uchar * q = (global uchar *) dst_q + QK4_0/2*get_global_id(0);
|
|
220
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
221
|
+
|
|
222
|
+
*d = b->d;
|
|
223
|
+
|
|
224
|
+
for (int i = 0; i < QK4_0/2; ++i) {
|
|
225
|
+
q[i] = b->qs[i];
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
kernel void kernel_restore_block_q4_0(
|
|
230
|
+
global uchar * src_q,
|
|
231
|
+
global half * src_d,
|
|
232
|
+
global struct block_q4_0 * dst
|
|
233
|
+
) {
|
|
234
|
+
global struct block_q4_0 * b = (global struct block_q4_0 *) dst + get_global_id(0);
|
|
235
|
+
global uchar * q = (global uchar *) src_q + QK4_0/2*get_global_id(0);
|
|
236
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
237
|
+
|
|
238
|
+
b->d = *d;
|
|
239
|
+
for (int i = 0; i < QK4_0/2; ++i) {
|
|
240
|
+
b->qs[i] = q[i];
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
//------------------------------------------------------------------------------
|
|
245
|
+
// kernel_convert_block_q4_0_noshuffle
|
|
246
|
+
// Flatten q4_0 weights and unshuffle the bits
|
|
247
|
+
//------------------------------------------------------------------------------
|
|
248
|
+
|
|
249
|
+
kernel void kernel_convert_block_q4_0_noshuffle(
|
|
250
|
+
global struct block_q4_0 * src0,
|
|
251
|
+
global uchar * dst_q,
|
|
252
|
+
global half * dst_d
|
|
253
|
+
) {
|
|
254
|
+
global struct block_q4_0 * b = (global struct block_q4_0 *) src0 + get_global_id(0);
|
|
255
|
+
global uchar * q = (global uchar *) dst_q + QK4_0/2*get_global_id(0);
|
|
256
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
257
|
+
|
|
258
|
+
*d = b->d;
|
|
259
|
+
for (int i = 0; i < QK4_0/4; ++i) {
|
|
260
|
+
uchar x0 = b->qs[2*i + 0];
|
|
261
|
+
uchar x1 = b->qs[2*i + 1];
|
|
262
|
+
|
|
263
|
+
q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
264
|
+
q[i + QK4_0/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
265
|
+
|
|
266
|
+
#ifdef ADRENO_GPU
|
|
267
|
+
// Workaround for adreno - must have the following printf statement for
|
|
268
|
+
// the kernel to work properly. Otherwise it produces incorrect result.
|
|
269
|
+
// convert_uchar above also seems necessary.
|
|
270
|
+
// Compare against a large number so that it does not print anything.
|
|
271
|
+
// get_sub_group_local_id() also works.
|
|
272
|
+
if (get_global_id(0) == 65536*4096) {
|
|
273
|
+
printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));
|
|
274
|
+
}
|
|
275
|
+
#endif
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
kernel void kernel_restore_block_q4_0_noshuffle(
|
|
280
|
+
global uchar * src_q,
|
|
281
|
+
global half * src_d,
|
|
282
|
+
global struct block_q4_0 * dst,
|
|
283
|
+
uchar mask_0F,
|
|
284
|
+
uchar mask_F0
|
|
285
|
+
) {
|
|
286
|
+
global struct block_q4_0 * b = (global struct block_q4_0 *) dst + get_global_id(0);
|
|
287
|
+
global uchar * q = (global uchar *) src_q + QK4_0/2*get_global_id(0);
|
|
288
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
289
|
+
|
|
290
|
+
b->d = *d;
|
|
291
|
+
for (int i = 0; i < QK4_0/4; ++i) {
|
|
292
|
+
uchar x0 = q[i + 0 ] ;
|
|
293
|
+
uchar x1 = q[i + QK4_0/4];
|
|
294
|
+
|
|
295
|
+
b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));
|
|
296
|
+
b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
kernel void kernel_convert_block_q4_0_trans4_ns(
|
|
301
|
+
global struct block_q4_0 * src0,
|
|
302
|
+
__global uint * dst_q,
|
|
303
|
+
__global half * dst_d,
|
|
304
|
+
uint ne00,
|
|
305
|
+
uint ne01
|
|
306
|
+
) {
|
|
307
|
+
uint i00 = get_global_id(1);
|
|
308
|
+
uint i01 = get_global_id(0);
|
|
309
|
+
uint i02 = get_global_id(2);
|
|
310
|
+
|
|
311
|
+
if (i01 >= ne01) {
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
uint ne00_blk = ne00 / QK4_0;
|
|
316
|
+
uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
317
|
+
uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
318
|
+
|
|
319
|
+
global struct block_q4_0 * b = src0 + src_blk_offset;
|
|
320
|
+
dst_d[dst_blk_offset] = b->d;
|
|
321
|
+
|
|
322
|
+
// extract quantization and unshuffle
|
|
323
|
+
ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0];
|
|
324
|
+
|
|
325
|
+
ushort8 post_block = (ushort8)(0);
|
|
326
|
+
|
|
327
|
+
uchar * pre_block_ptr = (uchar *)(&pre_block);
|
|
328
|
+
uchar * post_block_ptr = (uchar *)(&post_block);
|
|
329
|
+
|
|
330
|
+
for (int i = 0; i < QK4_0 / 4; ++i) {
|
|
331
|
+
uchar x0 = pre_block_ptr[2*i + 0];
|
|
332
|
+
uchar x1 = pre_block_ptr[2*i + 1];
|
|
333
|
+
|
|
334
|
+
post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
335
|
+
post_block_ptr[i + QK4_0 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
uint4 q_block = as_uint4(post_block);
|
|
339
|
+
|
|
340
|
+
uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
|
|
341
|
+
dst_q[offset] = q_block.x;
|
|
342
|
+
dst_q[offset + ne01] = q_block.y;
|
|
343
|
+
dst_q[offset + ne01 * 2] = q_block.z;
|
|
344
|
+
dst_q[offset + ne01 * 3] = q_block.w;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
kernel void kernel_restore_block_q4_0_trans4_ns(
|
|
348
|
+
__global uint * src_q,
|
|
349
|
+
__global half * src_d,
|
|
350
|
+
__global struct block_q4_0 * dst0,
|
|
351
|
+
uint ne00,
|
|
352
|
+
uint ne01
|
|
353
|
+
) {
|
|
354
|
+
uint i00 = get_global_id(1);
|
|
355
|
+
uint i01 = get_global_id(0);
|
|
356
|
+
uint i02 = get_global_id(2);
|
|
357
|
+
|
|
358
|
+
if (i01 >= ne01) {
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
uint ne00_blk = ne00 / QK4_0;
|
|
363
|
+
uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
364
|
+
uint src_d_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
365
|
+
|
|
366
|
+
__global struct block_q4_0 * b = dst0 + dst_blk_offset;
|
|
367
|
+
b->d = src_d[src_d_offset];
|
|
368
|
+
|
|
369
|
+
// collect transposed quantization parts for a block
|
|
370
|
+
uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
|
|
371
|
+
uint4 q_block;
|
|
372
|
+
q_block.x = src_q[src_q_offset];
|
|
373
|
+
q_block.y = src_q[src_q_offset + ne01];
|
|
374
|
+
q_block.z = src_q[src_q_offset + ne01 * 2];
|
|
375
|
+
q_block.w = src_q[src_q_offset + ne01 * 3];
|
|
376
|
+
|
|
377
|
+
ushort8 post_block = as_ushort8(q_block);
|
|
378
|
+
ushort8 pre_block = (ushort8)(0);
|
|
379
|
+
|
|
380
|
+
uchar * pre_block_ptr = (uchar *)(&pre_block);
|
|
381
|
+
uchar * post_block_ptr = (uchar *)(&post_block);
|
|
382
|
+
|
|
383
|
+
for (int i = 0; i < QK4_0 / 4; ++i) {
|
|
384
|
+
uchar x0 = post_block_ptr[i + 0];
|
|
385
|
+
uchar x1 = post_block_ptr[i + QK4_0 / 4];
|
|
386
|
+
|
|
387
|
+
pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
388
|
+
pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
((__global ushort8 *)(&(b->qs[0])))[0] = pre_block;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
//------------------------------------------------------------------------------
|
|
395
|
+
// kernel_convert_block_q4_1
|
|
396
|
+
// Convert the block_q4_1 format to 2 separate arrays (AOS -> SOA).
|
|
397
|
+
// This kernel does not deshuffle the bits.
|
|
398
|
+
//------------------------------------------------------------------------------
|
|
399
|
+
kernel void kernel_convert_block_q4_1(
|
|
400
|
+
global struct block_q4_1 * src0,
|
|
401
|
+
global uchar * dst_q,
|
|
402
|
+
global half * dst_d,
|
|
403
|
+
global half * dst_m
|
|
404
|
+
) {
|
|
405
|
+
global struct block_q4_1 * b = (global struct block_q4_1 *) src0 + get_global_id(0);
|
|
406
|
+
global uchar * q = (global uchar *) dst_q + QK4_1/2*get_global_id(0);
|
|
407
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
408
|
+
global half * m = (global half *) dst_m + get_global_id(0);
|
|
409
|
+
|
|
410
|
+
*d = b->d;
|
|
411
|
+
*m = b->m;
|
|
412
|
+
|
|
413
|
+
for (int i = 0; i < QK4_1/2; ++i) {
|
|
414
|
+
q[i] = b->qs[i];
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
kernel void kernel_restore_block_q4_1(
|
|
419
|
+
global uchar * src_q,
|
|
420
|
+
global half * src_d,
|
|
421
|
+
global half * src_m,
|
|
422
|
+
global struct block_q4_1 * dst
|
|
423
|
+
) {
|
|
424
|
+
global struct block_q4_1 * b = (global struct block_q4_1 *) dst + get_global_id(0);
|
|
425
|
+
global uchar * q = (global uchar *) src_q + QK4_1/2*get_global_id(0);
|
|
426
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
427
|
+
global half * m = (global half *) src_m + get_global_id(0);
|
|
428
|
+
|
|
429
|
+
b->d = *d;
|
|
430
|
+
b->m = *m;
|
|
431
|
+
for (int i = 0; i < QK4_1/2; ++i) {
|
|
432
|
+
b->qs[i] = q[i];
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
kernel void kernel_convert_block_q4_1_noshuffle(
|
|
437
|
+
global struct block_q4_1 * src0,
|
|
438
|
+
global uchar * dst_q,
|
|
439
|
+
global half * dst_d,
|
|
440
|
+
global half * dst_m
|
|
441
|
+
) {
|
|
442
|
+
global struct block_q4_1 * b = (global struct block_q4_1 *) src0 + get_global_id(0);
|
|
443
|
+
global uchar * q = (global uchar *) dst_q + QK4_1/2*get_global_id(0);
|
|
444
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
445
|
+
global half * m = (global half *) dst_m + get_global_id(0);
|
|
446
|
+
|
|
447
|
+
*d = b->d;
|
|
448
|
+
*m = b->m;
|
|
449
|
+
for (int i = 0; i < QK4_1/4; ++i) {
|
|
450
|
+
uchar x0 = b->qs[2*i + 0];
|
|
451
|
+
uchar x1 = b->qs[2*i + 1];
|
|
452
|
+
|
|
453
|
+
q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
454
|
+
q[i + QK4_1/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
455
|
+
|
|
456
|
+
#ifdef ADRENO_GPU
|
|
457
|
+
if (get_global_id(0) == 65536*4096) {
|
|
458
|
+
printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));
|
|
459
|
+
}
|
|
460
|
+
#endif
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
kernel void kernel_restore_block_q4_1_noshuffle(
|
|
465
|
+
global uchar * src_q,
|
|
466
|
+
global half * src_d,
|
|
467
|
+
global half * src_m,
|
|
468
|
+
global struct block_q4_1 * dst,
|
|
469
|
+
uchar mask_0F,
|
|
470
|
+
uchar mask_F0
|
|
471
|
+
) {
|
|
472
|
+
global struct block_q4_1 * b = (global struct block_q4_1 *) dst + get_global_id(0);
|
|
473
|
+
global uchar * q = (global uchar *) src_q + QK4_1/2*get_global_id(0);
|
|
474
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
475
|
+
global half * m = (global half *) src_m + get_global_id(0);
|
|
476
|
+
|
|
477
|
+
b->d = *d;
|
|
478
|
+
b->m = *m;
|
|
479
|
+
for (int i = 0; i < QK4_1/4; ++i) {
|
|
480
|
+
uchar x0 = q[i + 0 ] ;
|
|
481
|
+
uchar x1 = q[i + QK4_1/4];
|
|
482
|
+
|
|
483
|
+
b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));
|
|
484
|
+
b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
kernel void kernel_convert_block_q4_1_trans4_ns(
|
|
489
|
+
__global struct block_q4_1 * src0,
|
|
490
|
+
__global uint * dst_q,
|
|
491
|
+
__global half * dst_d,
|
|
492
|
+
__global half * dst_m,
|
|
493
|
+
uint ne00,
|
|
494
|
+
uint ne01
|
|
495
|
+
) {
|
|
496
|
+
uint i00 = get_global_id(1);
|
|
497
|
+
uint i01 = get_global_id(0);
|
|
498
|
+
uint i02 = get_global_id(2);
|
|
499
|
+
|
|
500
|
+
if (i01 >= ne01) {
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
uint ne00_blk = ne00 / QK4_1;
|
|
505
|
+
uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
506
|
+
uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
507
|
+
|
|
508
|
+
global struct block_q4_1 * b = src0 + src_blk_offset;
|
|
509
|
+
dst_d[dst_blk_offset] = b->d;
|
|
510
|
+
dst_m[dst_blk_offset] = b->m;
|
|
511
|
+
|
|
512
|
+
// extract quantization and unshuffle
|
|
513
|
+
ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0];
|
|
514
|
+
|
|
515
|
+
ushort8 post_block = (ushort8)(0);
|
|
516
|
+
|
|
517
|
+
uchar * pre_block_ptr = (uchar *)(&pre_block);
|
|
518
|
+
uchar * post_block_ptr = (uchar *)(&post_block);
|
|
519
|
+
|
|
520
|
+
for (int i = 0; i < QK4_1 / 4; ++i) {
|
|
521
|
+
uchar x0 = pre_block_ptr[2*i + 0];
|
|
522
|
+
uchar x1 = pre_block_ptr[2*i + 1];
|
|
523
|
+
|
|
524
|
+
post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
525
|
+
post_block_ptr[i + QK4_1 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
uint4 q_block = as_uint4(post_block);
|
|
529
|
+
|
|
530
|
+
uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
|
|
531
|
+
dst_q[offset] = q_block.x;
|
|
532
|
+
dst_q[offset + ne01] = q_block.y;
|
|
533
|
+
dst_q[offset + ne01 * 2] = q_block.z;
|
|
534
|
+
dst_q[offset + ne01 * 3] = q_block.w;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
kernel void kernel_restore_block_q4_1_trans4_ns(
|
|
538
|
+
__global uint * src_q,
|
|
539
|
+
__global half * src_d,
|
|
540
|
+
__global half * src_m,
|
|
541
|
+
__global struct block_q4_1 * dst0,
|
|
542
|
+
uint ne00,
|
|
543
|
+
uint ne01
|
|
544
|
+
) {
|
|
545
|
+
int i00 = get_global_id(1);
|
|
546
|
+
uint i01 = get_global_id(0);
|
|
547
|
+
uint i02 = get_global_id(2);
|
|
548
|
+
|
|
549
|
+
if (i01 >= ne01) {
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
uint ne00_blk = ne00 / QK4_1;
|
|
554
|
+
uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
555
|
+
uint src_dm_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
556
|
+
|
|
557
|
+
__global struct block_q4_1 * b = dst0 + dst_blk_offset;
|
|
558
|
+
b->d = src_d[src_dm_offset];
|
|
559
|
+
b->m = src_m[src_dm_offset];
|
|
560
|
+
|
|
561
|
+
// collect transposed quantization parts for a block
|
|
562
|
+
uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
|
|
563
|
+
uint4 q_block;
|
|
564
|
+
q_block.x = src_q[src_q_offset];
|
|
565
|
+
q_block.y = src_q[src_q_offset + ne01];
|
|
566
|
+
q_block.z = src_q[src_q_offset + ne01 * 2];
|
|
567
|
+
q_block.w = src_q[src_q_offset + ne01 * 3];
|
|
568
|
+
|
|
569
|
+
ushort8 post_block = as_ushort8(q_block);
|
|
570
|
+
ushort8 pre_block = (ushort8)(0);
|
|
571
|
+
|
|
572
|
+
uchar * pre_block_ptr = (uchar *)(&pre_block);
|
|
573
|
+
uchar * post_block_ptr = (uchar *)(&post_block);
|
|
574
|
+
|
|
575
|
+
for (int i = 0; i < QK4_0 / 4; ++i) {
|
|
576
|
+
uchar x0 = post_block_ptr[i + 0];
|
|
577
|
+
uchar x1 = post_block_ptr[i + QK4_0 / 4];
|
|
578
|
+
|
|
579
|
+
pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
580
|
+
pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
((__global ushort8 *)(&(b->qs[0])))[0] = pre_block;
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
//------------------------------------------------------------------------------
|
|
587
|
+
// kernel_convert_block_q5_0
|
|
588
|
+
// Convert the block_q5_0 format to 3 separate arrays (AOS -> SOA).
|
|
589
|
+
// This kernel does not deshuffle the bits.
|
|
590
|
+
//------------------------------------------------------------------------------
|
|
591
|
+
kernel void kernel_convert_block_q5_0(
|
|
592
|
+
global struct block_q5_0 * src0,
|
|
593
|
+
global uchar * dst_qs,
|
|
594
|
+
global uint * dst_qh,
|
|
595
|
+
global half * dst_d,
|
|
596
|
+
ulong n_blk
|
|
597
|
+
) {
|
|
598
|
+
if (get_global_id(0) >= n_blk) {
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
global struct block_q5_0 * b = (global struct block_q5_0 *) src0 + get_global_id(0);
|
|
603
|
+
global uchar * qs = (global uchar *) dst_qs + (QK5_0/2)*get_global_id(0);
|
|
604
|
+
global uint * qh = (global uint *) dst_qh + get_global_id(0);
|
|
605
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
606
|
+
|
|
607
|
+
*d = b->d;
|
|
608
|
+
*qh = *((global uint *)(b->qh));
|
|
609
|
+
|
|
610
|
+
for (int i = 0; i < QK5_0/2; ++i) {
|
|
611
|
+
qs[i] = b->qs[i];
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
kernel void kernel_restore_block_q5_0(
|
|
616
|
+
global uchar * src_qs,
|
|
617
|
+
global uint * src_qh,
|
|
618
|
+
global half * src_d,
|
|
619
|
+
global struct block_q5_0 * dst
|
|
620
|
+
) {
|
|
621
|
+
global struct block_q5_0 * b = (global struct block_q5_0 *) dst + get_global_id(0);
|
|
622
|
+
global uchar * qs = (global uchar *) src_qs + (QK5_0/2)*get_global_id(0);
|
|
623
|
+
global uint * qh = (global uint *) src_qh + get_global_id(0);
|
|
624
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
625
|
+
|
|
626
|
+
b->d = *d;
|
|
627
|
+
*((global uint *)(b->qh)) = *qh;
|
|
628
|
+
for (int i = 0; i < QK5_0/2; ++i) {
|
|
629
|
+
b->qs[i] = qs[i];
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
kernel void kernel_convert_block_q5_0_noshuffle(
|
|
634
|
+
global struct block_q5_0 * src0,
|
|
635
|
+
global uchar * dst_q,
|
|
636
|
+
global uint * dst_qh,
|
|
637
|
+
global half * dst_d
|
|
638
|
+
) {
|
|
639
|
+
global struct block_q5_0 * b = (global struct block_q5_0 *) src0 + get_global_id(0);
|
|
640
|
+
global uchar * q = (global uchar *) dst_q + QK5_0/2*get_global_id(0);
|
|
641
|
+
global uint * qh = (global uint *) dst_qh + get_global_id(0);
|
|
642
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
643
|
+
|
|
644
|
+
*d = b->d;
|
|
645
|
+
*qh = *((global uint *)(b->qh));
|
|
646
|
+
|
|
647
|
+
for (int i = 0; i < QK5_0/4; ++i) {
|
|
648
|
+
uchar x0 = b->qs[2*i + 0];
|
|
649
|
+
uchar x1 = b->qs[2*i + 1];
|
|
650
|
+
|
|
651
|
+
q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
652
|
+
q[i + QK5_0/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
653
|
+
|
|
654
|
+
#ifdef ADRENO_GPU
|
|
655
|
+
if (get_global_id(0) == 65536*4096) {
|
|
656
|
+
printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));
|
|
657
|
+
}
|
|
658
|
+
#endif
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
kernel void kernel_restore_block_q5_0_noshuffle(
|
|
663
|
+
global uchar * src_q,
|
|
664
|
+
global uint * src_qh,
|
|
665
|
+
global half * src_d,
|
|
666
|
+
global struct block_q5_0 * dst,
|
|
667
|
+
uchar mask_0F,
|
|
668
|
+
uchar mask_F0
|
|
669
|
+
) {
|
|
670
|
+
global struct block_q5_0 * b = (global struct block_q5_0 *) dst + get_global_id(0);
|
|
671
|
+
global uchar * q = (global uchar *) src_q + QK5_0/2*get_global_id(0);
|
|
672
|
+
global uint * qh = (global uint *) src_qh + get_global_id(0);
|
|
673
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
674
|
+
|
|
675
|
+
b->d = *d;
|
|
676
|
+
*((global uint *)(b->qh)) = *qh;
|
|
677
|
+
|
|
678
|
+
for (int i = 0; i < QK5_0/4; ++i) {
|
|
679
|
+
uchar x0 = q[i + 0 ];
|
|
680
|
+
uchar x1 = q[i + QK5_0/4];
|
|
681
|
+
|
|
682
|
+
b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));
|
|
683
|
+
b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
kernel void kernel_convert_block_q5_0_trans4_ns(
|
|
688
|
+
__global struct block_q5_0 * src0,
|
|
689
|
+
__global uint * dst_qs,
|
|
690
|
+
__global uint * dst_qh,
|
|
691
|
+
__global half * dst_d,
|
|
692
|
+
uint ne00,
|
|
693
|
+
uint ne01
|
|
694
|
+
) {
|
|
695
|
+
uint i00 = get_global_id(1);
|
|
696
|
+
uint i01 = get_global_id(0);
|
|
697
|
+
uint i02 = get_global_id(2);
|
|
698
|
+
|
|
699
|
+
if (i01 >= ne01) {
|
|
700
|
+
return;
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
uint ne00_blk = ne00 / QK5_0;
|
|
704
|
+
uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
705
|
+
uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
706
|
+
|
|
707
|
+
global struct block_q5_0 * b = src0 + src_blk_offset;
|
|
708
|
+
dst_d[dst_blk_offset] = b->d;
|
|
709
|
+
|
|
710
|
+
dst_qh[dst_blk_offset] = ((global uint *)(&(b->qh[0])))[0];
|
|
711
|
+
|
|
712
|
+
// extract quantization and unshuffle
|
|
713
|
+
ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0];
|
|
714
|
+
ushort8 post_block = (ushort8)(0);
|
|
715
|
+
|
|
716
|
+
uchar * pre_block_ptr = (uchar *)(&pre_block);
|
|
717
|
+
uchar * post_block_ptr = (uchar *)(&post_block);
|
|
718
|
+
|
|
719
|
+
for (int i = 0; i < QK5_0 / 4; ++i) {
|
|
720
|
+
uchar x0 = pre_block_ptr[2*i + 0];
|
|
721
|
+
uchar x1 = pre_block_ptr[2*i + 1];
|
|
722
|
+
|
|
723
|
+
post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
724
|
+
post_block_ptr[i + QK5_0 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
uint4 q_block = as_uint4(post_block);
|
|
728
|
+
|
|
729
|
+
uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
|
|
730
|
+
dst_qs[offset] = q_block.x;
|
|
731
|
+
dst_qs[offset + ne01] = q_block.y;
|
|
732
|
+
dst_qs[offset + ne01 * 2] = q_block.z;
|
|
733
|
+
dst_qs[offset + ne01 * 3] = q_block.w;
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
kernel void kernel_restore_block_q5_0_trans4_ns(
|
|
737
|
+
__global uint * src_qs,
|
|
738
|
+
__global uint * src_qh,
|
|
739
|
+
__global half * src_d,
|
|
740
|
+
__global struct block_q5_0 * dst0,
|
|
741
|
+
uint ne00,
|
|
742
|
+
uint ne01
|
|
743
|
+
) {
|
|
744
|
+
int i00 = get_global_id(1);
|
|
745
|
+
uint i01 = get_global_id(0);
|
|
746
|
+
uint i02 = get_global_id(2);
|
|
747
|
+
|
|
748
|
+
if (i01 >= ne01) {
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
uint ne00_blk = ne00 / QK5_0;
|
|
753
|
+
uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
754
|
+
uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
755
|
+
|
|
756
|
+
__global struct block_q5_0 * b = dst0 + dst_blk_offset;
|
|
757
|
+
b->d = src_d[src_blk_offset];
|
|
758
|
+
|
|
759
|
+
((__global uint *)(&(b->qh[0])))[0] = src_qh[src_blk_offset];
|
|
760
|
+
|
|
761
|
+
// collect transposed quantization parts for a block
|
|
762
|
+
uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
|
|
763
|
+
uint4 q_block;
|
|
764
|
+
q_block.x = src_qs[src_q_offset];
|
|
765
|
+
q_block.y = src_qs[src_q_offset + ne01];
|
|
766
|
+
q_block.z = src_qs[src_q_offset + ne01 * 2];
|
|
767
|
+
q_block.w = src_qs[src_q_offset + ne01 * 3];
|
|
768
|
+
|
|
769
|
+
ushort8 post_block = as_ushort8(q_block);
|
|
770
|
+
ushort8 pre_block = (ushort8)(0);
|
|
771
|
+
|
|
772
|
+
uchar * pre_block_ptr = (uchar *)(&pre_block);
|
|
773
|
+
uchar * post_block_ptr = (uchar *)(&post_block);
|
|
774
|
+
|
|
775
|
+
for (int i = 0; i < QK5_0 / 4; ++i) {
|
|
776
|
+
uchar x0 = post_block_ptr[i + 0];
|
|
777
|
+
uchar x1 = post_block_ptr[i + QK5_0 / 4];
|
|
778
|
+
|
|
779
|
+
pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
780
|
+
pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
((__global ushort8 *)(&(b->qs[0])))[0] = pre_block;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
//------------------------------------------------------------------------------
|
|
787
|
+
// kernel_convert_block_q5_1
|
|
788
|
+
// Convert the block_q5_1 format to 4 separate arrays (AOS -> SOA).
|
|
789
|
+
// This kernel does not deshuffle the bits.
|
|
790
|
+
//------------------------------------------------------------------------------
|
|
791
|
+
kernel void kernel_convert_block_q5_1(
|
|
792
|
+
global struct block_q5_1 * src0,
|
|
793
|
+
global uchar * dst_qs,
|
|
794
|
+
global uint * dst_qh,
|
|
795
|
+
global half * dst_d,
|
|
796
|
+
global half * dst_m,
|
|
797
|
+
ulong n_blk
|
|
798
|
+
) {
|
|
799
|
+
if (get_global_id(0) >= n_blk) {
|
|
800
|
+
return;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
global struct block_q5_1 * b = (global struct block_q5_1 *) src0 + get_global_id(0);
|
|
804
|
+
global uchar * qs = (global uchar *) dst_qs + (QK5_1/2)*get_global_id(0);
|
|
805
|
+
global uint * qh = (global uint *) dst_qh + get_global_id(0);
|
|
806
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
807
|
+
global half * m = (global half *) dst_m + get_global_id(0);
|
|
808
|
+
|
|
809
|
+
*d = b->d;
|
|
810
|
+
*m = b->m;
|
|
811
|
+
*qh = *((global uint *)(b->qh));
|
|
812
|
+
|
|
813
|
+
for (int i = 0; i < QK5_1/2; ++i) {
|
|
814
|
+
qs[i] = b->qs[i];
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
kernel void kernel_restore_block_q5_1(
|
|
819
|
+
global uchar * src_qs,
|
|
820
|
+
global uint * src_qh,
|
|
821
|
+
global half * src_d,
|
|
822
|
+
global half * src_m,
|
|
823
|
+
global struct block_q5_1 * dst
|
|
824
|
+
) {
|
|
825
|
+
global struct block_q5_1 * b = (global struct block_q5_1 *) dst + get_global_id(0);
|
|
826
|
+
global uchar * qs = (global uchar *) src_qs + (QK5_1/2)*get_global_id(0);
|
|
827
|
+
global uint * qh = (global uint *) src_qh + get_global_id(0);
|
|
828
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
829
|
+
global half * m = (global half *) src_m + get_global_id(0);
|
|
830
|
+
|
|
831
|
+
b->d = *d;
|
|
832
|
+
b->m = *m;
|
|
833
|
+
*((global uint *)(b->qh)) = *qh;
|
|
834
|
+
for (int i = 0; i < QK5_1/2; ++i) {
|
|
835
|
+
b->qs[i] = qs[i];
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
kernel void kernel_convert_block_q5_1_noshuffle(
|
|
840
|
+
global struct block_q5_1 * src0,
|
|
841
|
+
global uchar * dst_q,
|
|
842
|
+
global uint * dst_qh,
|
|
843
|
+
global half * dst_d,
|
|
844
|
+
global half * dst_m
|
|
845
|
+
) {
|
|
846
|
+
global struct block_q5_1 * b = (global struct block_q5_1 *) src0 + get_global_id(0);
|
|
847
|
+
global uchar * q = (global uchar *) dst_q + QK5_1/2*get_global_id(0);
|
|
848
|
+
global uint * qh = (global uint *) dst_qh + get_global_id(0);
|
|
849
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
850
|
+
global half * m = (global half *) dst_m + get_global_id(0);
|
|
851
|
+
|
|
852
|
+
*d = b->d;
|
|
853
|
+
*m = b->m;
|
|
854
|
+
*qh = *((global uint *)(b->qh));
|
|
855
|
+
|
|
856
|
+
for (int i = 0; i < QK5_1/4; ++i) {
|
|
857
|
+
uchar x0 = b->qs[2*i + 0];
|
|
858
|
+
uchar x1 = b->qs[2*i + 1];
|
|
859
|
+
|
|
860
|
+
q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
861
|
+
q[i + QK5_1/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
862
|
+
|
|
863
|
+
#ifdef ADRENO_GPU
|
|
864
|
+
if (get_global_id(0) == 65536*4096) {
|
|
865
|
+
printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));
|
|
866
|
+
}
|
|
867
|
+
#endif
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
kernel void kernel_restore_block_q5_1_noshuffle(
|
|
872
|
+
global uchar * src_q,
|
|
873
|
+
global uint * src_qh,
|
|
874
|
+
global half * src_d,
|
|
875
|
+
global half * src_m,
|
|
876
|
+
global struct block_q5_1 * dst,
|
|
877
|
+
uchar mask_0F,
|
|
878
|
+
uchar mask_F0
|
|
879
|
+
) {
|
|
880
|
+
global struct block_q5_1 * b = (global struct block_q5_1 *) dst + get_global_id(0);
|
|
881
|
+
global uchar * q = (global uchar *) src_q + QK5_1/2*get_global_id(0);
|
|
882
|
+
global uint * qh = (global uint *) src_qh + get_global_id(0);
|
|
883
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
884
|
+
global half * m = (global half *) src_m + get_global_id(0);
|
|
885
|
+
|
|
886
|
+
b->d = *d;
|
|
887
|
+
b->m = *m;
|
|
888
|
+
*((global uint *)(b->qh)) = *qh;
|
|
889
|
+
|
|
890
|
+
for (int i = 0; i < QK5_1/4; ++i) {
|
|
891
|
+
uchar x0 = q[i + 0 ];
|
|
892
|
+
uchar x1 = q[i + QK5_1/4];
|
|
893
|
+
|
|
894
|
+
b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));
|
|
895
|
+
b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
kernel void kernel_convert_block_q5_1_trans4_ns(
|
|
900
|
+
__global struct block_q5_1 * src0,
|
|
901
|
+
__global uint * dst_qs,
|
|
902
|
+
__global uint * dst_qh,
|
|
903
|
+
__global half * dst_d,
|
|
904
|
+
__global half * dst_m,
|
|
905
|
+
uint ne00,
|
|
906
|
+
uint ne01
|
|
907
|
+
) {
|
|
908
|
+
uint i00 = get_global_id(1);
|
|
909
|
+
uint i01 = get_global_id(0);
|
|
910
|
+
uint i02 = get_global_id(2);
|
|
911
|
+
|
|
912
|
+
if (i01 >= ne01) {
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
uint ne00_blk = ne00 / QK5_1;
|
|
917
|
+
uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
918
|
+
uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
919
|
+
|
|
920
|
+
global struct block_q5_1 * b = src0 + src_blk_offset;
|
|
921
|
+
dst_d[dst_blk_offset] = b->d;
|
|
922
|
+
dst_m[dst_blk_offset] = b->m;
|
|
923
|
+
|
|
924
|
+
dst_qh[dst_blk_offset] = ((global uint *)(&(b->qh[0])))[0];
|
|
925
|
+
|
|
926
|
+
// extract quantization and unshuffle
|
|
927
|
+
ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0];
|
|
928
|
+
ushort8 post_block = (ushort8)(0);
|
|
929
|
+
|
|
930
|
+
uchar * pre_block_ptr = (uchar *)(&pre_block);
|
|
931
|
+
uchar * post_block_ptr = (uchar *)(&post_block);
|
|
932
|
+
|
|
933
|
+
for (int i = 0; i < QK5_1 / 4; ++i) {
|
|
934
|
+
uchar x0 = pre_block_ptr[2*i + 0];
|
|
935
|
+
uchar x1 = pre_block_ptr[2*i + 1];
|
|
936
|
+
|
|
937
|
+
post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
938
|
+
post_block_ptr[i + QK5_1 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
uint4 q_block = as_uint4(post_block);
|
|
942
|
+
|
|
943
|
+
uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
|
|
944
|
+
dst_qs[offset] = q_block.x;
|
|
945
|
+
dst_qs[offset + ne01] = q_block.y;
|
|
946
|
+
dst_qs[offset + ne01 * 2] = q_block.z;
|
|
947
|
+
dst_qs[offset + ne01 * 3] = q_block.w;
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
kernel void kernel_restore_block_q5_1_trans4_ns(
|
|
951
|
+
__global uint * src_qs,
|
|
952
|
+
__global uint * src_qh,
|
|
953
|
+
__global half * src_d,
|
|
954
|
+
__global half * src_m,
|
|
955
|
+
__global struct block_q5_1 * dst0,
|
|
956
|
+
uint ne00,
|
|
957
|
+
uint ne01
|
|
958
|
+
) {
|
|
959
|
+
int i00 = get_global_id(1);
|
|
960
|
+
uint i01 = get_global_id(0);
|
|
961
|
+
uint i02 = get_global_id(2);
|
|
962
|
+
|
|
963
|
+
if (i01 >= ne01) {
|
|
964
|
+
return;
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
uint ne00_blk = ne00 / QK5_1;
|
|
968
|
+
uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
969
|
+
uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
970
|
+
|
|
971
|
+
__global struct block_q5_1 * b = dst0 + dst_blk_offset;
|
|
972
|
+
b->d = src_d[src_blk_offset];
|
|
973
|
+
b->m = src_m[src_blk_offset];
|
|
974
|
+
|
|
975
|
+
((__global uint *)(&(b->qh[0])))[0] = src_qh[src_blk_offset];
|
|
976
|
+
|
|
977
|
+
// collect transposed quantization parts for a block
|
|
978
|
+
uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
|
|
979
|
+
uint4 q_block;
|
|
980
|
+
q_block.x = src_qs[src_q_offset];
|
|
981
|
+
q_block.y = src_qs[src_q_offset + ne01];
|
|
982
|
+
q_block.z = src_qs[src_q_offset + ne01 * 2];
|
|
983
|
+
q_block.w = src_qs[src_q_offset + ne01 * 3];
|
|
984
|
+
|
|
985
|
+
ushort8 post_block = as_ushort8(q_block);
|
|
986
|
+
ushort8 pre_block = (ushort8)(0);
|
|
987
|
+
|
|
988
|
+
uchar * pre_block_ptr = (uchar *)(&pre_block);
|
|
989
|
+
uchar * post_block_ptr = (uchar *)(&post_block);
|
|
990
|
+
|
|
991
|
+
for (int i = 0; i < QK5_1 / 4; ++i) {
|
|
992
|
+
uchar x0 = post_block_ptr[i + 0];
|
|
993
|
+
uchar x1 = post_block_ptr[i + QK5_1 / 4];
|
|
994
|
+
|
|
995
|
+
pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
996
|
+
pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
997
|
+
}
|
|
998
|
+
((__global ushort8 *)(&(b->qs[0])))[0] = pre_block;
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
kernel void kernel_convert_block_q4_k_trans4_ns(
|
|
1002
|
+
__global struct block_q4_K * src0,
|
|
1003
|
+
__global uint * dst_q,
|
|
1004
|
+
__global half * dst_d,
|
|
1005
|
+
__global half * dst_dm,
|
|
1006
|
+
__global uchar * dst_s,
|
|
1007
|
+
uint ne00,
|
|
1008
|
+
uint ne01,
|
|
1009
|
+
uchar mask_0F,
|
|
1010
|
+
uchar mask_F0
|
|
1011
|
+
) {
|
|
1012
|
+
uint i00 = get_global_id(1);
|
|
1013
|
+
uint i01 = get_global_id(0);
|
|
1014
|
+
uint i02 = get_global_id(2);
|
|
1015
|
+
|
|
1016
|
+
if (i01 >= ne01) {
|
|
1017
|
+
return;
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
uint ne00_blk = ne00 / QK_K;
|
|
1021
|
+
uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1022
|
+
uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
1023
|
+
|
|
1024
|
+
__global struct block_q4_K * b = src0 + src_blk_offset;
|
|
1025
|
+
|
|
1026
|
+
dst_d [dst_blk_offset] = b->d;
|
|
1027
|
+
dst_dm[dst_blk_offset] = b->dm;
|
|
1028
|
+
|
|
1029
|
+
uint4 qv[8];
|
|
1030
|
+
uchar * qv_bytes = (uchar *)qv;
|
|
1031
|
+
for (int i = 0; i < QK_K / 64; ++i) {
|
|
1032
|
+
for (int j = 0; j < 16; ++j) {
|
|
1033
|
+
uchar x0 = b->q[i*32 + 2*j];
|
|
1034
|
+
uchar x1 = b->q[i*32 + 2*j + 1];
|
|
1035
|
+
|
|
1036
|
+
qv_bytes[i*32 + j ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
|
|
1037
|
+
qv_bytes[i*32 + j + 16] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
|
|
1042
|
+
#pragma unroll
|
|
1043
|
+
for (int p = 0; p < 8; ++p) {
|
|
1044
|
+
uint4 v = qv[p];
|
|
1045
|
+
dst_q[base + (p * 4 + 0) * ne01] = v.x;
|
|
1046
|
+
dst_q[base + (p * 4 + 1) * ne01] = v.y;
|
|
1047
|
+
dst_q[base + (p * 4 + 2) * ne01] = v.z;
|
|
1048
|
+
dst_q[base + (p * 4 + 3) * ne01] = v.w;
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
__global uchar * s_dst = dst_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE;
|
|
1052
|
+
#pragma unroll
|
|
1053
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
1054
|
+
s_dst[i] = b->s[i];
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
kernel void kernel_restore_block_q4_k_trans4_ns(
|
|
1059
|
+
__global uint * src_q,
|
|
1060
|
+
__global half * src_d,
|
|
1061
|
+
__global half * src_dm,
|
|
1062
|
+
__global uchar * src_s,
|
|
1063
|
+
__global struct block_q4_K * dst0,
|
|
1064
|
+
uint ne00,
|
|
1065
|
+
uint ne01,
|
|
1066
|
+
uchar mask_0F,
|
|
1067
|
+
uchar mask_F0
|
|
1068
|
+
) {
|
|
1069
|
+
uint i00 = get_global_id(1); // block index along K
|
|
1070
|
+
uint i01 = get_global_id(0); // row index
|
|
1071
|
+
uint i02 = get_global_id(2); // batch index
|
|
1072
|
+
|
|
1073
|
+
if (i01 >= ne01) {
|
|
1074
|
+
return;
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
uint ne00_blk = ne00 / QK_K;
|
|
1078
|
+
|
|
1079
|
+
uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
1080
|
+
uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1081
|
+
|
|
1082
|
+
__global struct block_q4_K * b = dst0 + dst_blk_offset;
|
|
1083
|
+
|
|
1084
|
+
b->d = src_d[src_blk_offset];
|
|
1085
|
+
b->dm = src_dm[src_blk_offset];
|
|
1086
|
+
|
|
1087
|
+
__global uchar * s_src = src_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE;
|
|
1088
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
1089
|
+
b->s[i] = s_src[i];
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
|
|
1093
|
+
|
|
1094
|
+
uint4 qv[8];
|
|
1095
|
+
for (int p = 0; p < 8; ++p) {
|
|
1096
|
+
qv[p].x = src_q[base + (p * 4 + 0) * ne01];
|
|
1097
|
+
qv[p].y = src_q[base + (p * 4 + 1) * ne01];
|
|
1098
|
+
qv[p].z = src_q[base + (p * 4 + 2) * ne01];
|
|
1099
|
+
qv[p].w = src_q[base + (p * 4 + 3) * ne01];
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
uchar * qv_bytes = (uchar *)qv;
|
|
1103
|
+
for (int i = 0; i < QK_K / 64; ++i) {
|
|
1104
|
+
for (int j = 0; j < 16; ++j) {
|
|
1105
|
+
uchar lo = qv_bytes[i*32 + j];
|
|
1106
|
+
uchar hi = qv_bytes[i*32 + j + 16];
|
|
1107
|
+
b->q[i*32 + 2*j] = convert_uchar((lo & mask_0F) | ((hi & mask_0F) << 4));
|
|
1108
|
+
b->q[i*32 + 2*j + 1] = convert_uchar(((lo & mask_F0) >> 4) | (hi & mask_F0));
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
//------------------------------------------------------------------------------
|
|
1114
|
+
// kernel_convert_block_q4_k_tiled_ns
|
|
1115
|
+
//
|
|
1116
|
+
// Tiled-wide layout for the long-vocab q4_K lm_head/embed GEMV (decode path).
|
|
1117
|
+
// Mirror of kernel_convert_block_q6_k_tiled_ns: recovers each weight's 4-bit
|
|
1118
|
+
// code in CANONICAL ggml element order (e in [0,256)) and re-packs into 32 uints
|
|
1119
|
+
// (8 codes/uint), stored TILED by 64 output rows so the matching GEMV
|
|
1120
|
+
// (gemv_noshuffle_q4_k_f32_tiled) coalesces every weight load. The 12-byte
|
|
1121
|
+
// packed scale block `s` and d/dm are stored per (row, K-block) tiled; the GEMV
|
|
1122
|
+
// re-derives the 8 (scale,min) pairs via get_scale_min_k4, exactly like the o4
|
|
1123
|
+
// kernel. Both ends owned here -> correct by construction vs the reference q4_K
|
|
1124
|
+
// dequant. Requires ne01 % 64 == 0 (gated host-side). Buffer sizes identical to
|
|
1125
|
+
// the trans4_ns layout.
|
|
1126
|
+
//
|
|
1127
|
+
// q uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk+sb)*8 + g)*64 + rit
|
|
1128
|
+
// s (12 bytes) of (r, sb): idx = (rt*ne00_blk+sb)*64 + rit, *12
|
|
1129
|
+
// d/dm (half) of (r, sb): idx = (rt*ne00_blk+sb)*64 + rit
|
|
1130
|
+
// where rt = r/64, rit = r%64.
|
|
1131
|
+
//------------------------------------------------------------------------------
|
|
1132
|
+
kernel void kernel_convert_block_q4_k_tiled_ns(
|
|
1133
|
+
__global struct block_q4_K * src0,
|
|
1134
|
+
__global uint * dst_q, // 32 uints / superblock (4-bit codes, 8 codes/uint)
|
|
1135
|
+
__global half * dst_d, // 1 half / superblock
|
|
1136
|
+
__global half * dst_dm, // 1 half / superblock
|
|
1137
|
+
__global uchar * dst_s, // K_SCALE_SIZE (12) bytes / superblock
|
|
1138
|
+
uint ne00,
|
|
1139
|
+
uint ne01
|
|
1140
|
+
) {
|
|
1141
|
+
uint i00 = get_global_id(1); // K-block index (superblock along ne00)
|
|
1142
|
+
uint i01 = get_global_id(0); // output row index (along ne01)
|
|
1143
|
+
uint i02 = get_global_id(2); // batch
|
|
1144
|
+
|
|
1145
|
+
uint ne00_blk = ne00 / QK_K;
|
|
1146
|
+
|
|
1147
|
+
uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1148
|
+
__global struct block_q4_K * b = src0 + src_blk_offset;
|
|
1149
|
+
|
|
1150
|
+
uint rt = i01 / 64;
|
|
1151
|
+
uint rit = i01 % 64;
|
|
1152
|
+
uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00;
|
|
1153
|
+
|
|
1154
|
+
// --- recover canonical 4-bit codes in e-order, pack 8 codes/uint ---
|
|
1155
|
+
uint qw[32] = {0};
|
|
1156
|
+
for (uint e = 0; e < 256; ++e) {
|
|
1157
|
+
uint g = e >> 6; // group 0..3 (q advances 32 bytes/group)
|
|
1158
|
+
uint within = e & 63u;
|
|
1159
|
+
uint hlf = within >> 5; // 0 = low nibble, 1 = high nibble
|
|
1160
|
+
uint l = within & 31u; // 0..31
|
|
1161
|
+
uchar byte = b->q[g * 32u + l];
|
|
1162
|
+
uint code = (hlf == 0u) ? (uint)(byte & 0x0F) : (uint)(byte >> 4);
|
|
1163
|
+
qw[e >> 3] |= code << ((e & 7u) * 4u);
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
for (uint gr = 0; gr < 8; ++gr) {
|
|
1167
|
+
uint base = (tile_blk * 8u + gr) * 64u + rit; // uint4 index
|
|
1168
|
+
dst_q[base * 4u + 0u] = qw[gr * 4u + 0u];
|
|
1169
|
+
dst_q[base * 4u + 1u] = qw[gr * 4u + 1u];
|
|
1170
|
+
dst_q[base * 4u + 2u] = qw[gr * 4u + 2u];
|
|
1171
|
+
dst_q[base * 4u + 3u] = qw[gr * 4u + 3u];
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
// packed scales (12 bytes), tiled per (row, block)
|
|
1175
|
+
__global uchar * s_dst = dst_s + (tile_blk * 64u + rit) * K_SCALE_SIZE;
|
|
1176
|
+
#pragma unroll
|
|
1177
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
1178
|
+
s_dst[i] = b->s[i];
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
dst_d [tile_blk * 64u + rit] = b->d;
|
|
1182
|
+
dst_dm[tile_blk * 64u + rit] = b->dm;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
kernel void kernel_convert_block_q5_k_trans4_ns(
|
|
1186
|
+
__global struct block_q5_K * src0,
|
|
1187
|
+
__global uint * dst_qs,
|
|
1188
|
+
__global uint * dst_qh,
|
|
1189
|
+
__global half * dst_d,
|
|
1190
|
+
__global half * dst_dm,
|
|
1191
|
+
__global uchar * dst_s,
|
|
1192
|
+
uint ne00,
|
|
1193
|
+
uint ne01,
|
|
1194
|
+
uchar mask_0F,
|
|
1195
|
+
uchar mask_F0
|
|
1196
|
+
) {
|
|
1197
|
+
uint i00 = get_global_id(1);
|
|
1198
|
+
uint i01 = get_global_id(0);
|
|
1199
|
+
uint i02 = get_global_id(2);
|
|
1200
|
+
|
|
1201
|
+
if (i01 >= ne01) {
|
|
1202
|
+
return;
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
uint ne00_blk = ne00 / QK_K;
|
|
1206
|
+
uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1207
|
+
uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
1208
|
+
|
|
1209
|
+
__global struct block_q5_K * b = src0 + src_blk_offset;
|
|
1210
|
+
|
|
1211
|
+
dst_d [dst_blk_offset] = b->d;
|
|
1212
|
+
dst_dm[dst_blk_offset] = b->dm;
|
|
1213
|
+
|
|
1214
|
+
for (int k = 0; k < 8; k++) {
|
|
1215
|
+
uchar b0 = 0, b1 = 0, b2 = 0, b3 = 0;
|
|
1216
|
+
for (int bit = 0; bit < 8; bit++) {
|
|
1217
|
+
b0 |= (uchar)(((b->qh[bit] >> k) & 1) << bit);
|
|
1218
|
+
b1 |= (uchar)(((b->qh[8 + bit] >> k) & 1) << bit);
|
|
1219
|
+
b2 |= (uchar)(((b->qh[16 + bit] >> k) & 1) << bit);
|
|
1220
|
+
b3 |= (uchar)(((b->qh[24 + bit] >> k) & 1) << bit);
|
|
1221
|
+
}
|
|
1222
|
+
uint packed = (uint)b0 | ((uint)b1 << 8) | ((uint)b2 << 16) | ((uint)b3 << 24);
|
|
1223
|
+
dst_qh[i01 + (i00 * 8 + k) * ne01 + i02 * ne00_blk * 8 * ne01] = packed;
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
uint4 qv[8];
|
|
1227
|
+
uchar * qv_bytes = (uchar *)qv;
|
|
1228
|
+
for (int i = 0; i < QK_K / 64; ++i) {
|
|
1229
|
+
for (int j = 0; j < 16; ++j) {
|
|
1230
|
+
uchar x0 = b->qs[i*32 + 2*j];
|
|
1231
|
+
uchar x1 = b->qs[i*32 + 2*j + 1];
|
|
1232
|
+
|
|
1233
|
+
qv_bytes[i*32 + j ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
|
|
1234
|
+
qv_bytes[i*32 + j + 16] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
|
|
1239
|
+
#pragma unroll
|
|
1240
|
+
for (int p = 0; p < 8; ++p) {
|
|
1241
|
+
uint4 v = qv[p];
|
|
1242
|
+
dst_qs[base + (p * 4 + 0) * ne01] = v.x;
|
|
1243
|
+
dst_qs[base + (p * 4 + 1) * ne01] = v.y;
|
|
1244
|
+
dst_qs[base + (p * 4 + 2) * ne01] = v.z;
|
|
1245
|
+
dst_qs[base + (p * 4 + 3) * ne01] = v.w;
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
__global uchar * s_dst = dst_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE;
|
|
1249
|
+
#pragma unroll
|
|
1250
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
1251
|
+
s_dst[i] = b->s[i];
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
kernel void kernel_restore_block_q5_k_trans4_ns(
|
|
1256
|
+
__global uint * src_qs,
|
|
1257
|
+
__global uint * src_qh,
|
|
1258
|
+
__global half * src_d,
|
|
1259
|
+
__global half * src_dm,
|
|
1260
|
+
__global uchar * src_s,
|
|
1261
|
+
__global struct block_q5_K * dst0,
|
|
1262
|
+
uint ne00,
|
|
1263
|
+
uint ne01,
|
|
1264
|
+
uchar mask_0F,
|
|
1265
|
+
uchar mask_F0
|
|
1266
|
+
) {
|
|
1267
|
+
uint i00 = get_global_id(1); // block index along K
|
|
1268
|
+
uint i01 = get_global_id(0); // row index
|
|
1269
|
+
uint i02 = get_global_id(2); // batch index
|
|
1270
|
+
|
|
1271
|
+
if (i01 >= ne01) {
|
|
1272
|
+
return;
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
uint ne00_blk = ne00 / QK_K;
|
|
1276
|
+
|
|
1277
|
+
uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
1278
|
+
uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1279
|
+
|
|
1280
|
+
__global struct block_q5_K * b = dst0 + dst_blk_offset;
|
|
1281
|
+
|
|
1282
|
+
b->d = src_d[src_blk_offset];
|
|
1283
|
+
b->dm = src_dm[src_blk_offset];
|
|
1284
|
+
|
|
1285
|
+
for (int j = 0; j < 32; j++) b->qh[j] = 0;
|
|
1286
|
+
for (int k = 0; k < 8; k++) {
|
|
1287
|
+
uint packed = src_qh[i01 + (i00 * 8 + k) * ne01 + i02 * ne00_blk * 8 * ne01];
|
|
1288
|
+
uchar b0 = (uchar)(packed & 0xFF);
|
|
1289
|
+
uchar b1 = (uchar)((packed >> 8) & 0xFF);
|
|
1290
|
+
uchar b2 = (uchar)((packed >> 16) & 0xFF);
|
|
1291
|
+
uchar b3 = (uchar)((packed >> 24) & 0xFF);
|
|
1292
|
+
for (int bit = 0; bit < 8; bit++) {
|
|
1293
|
+
b->qh[bit] |= (uchar)(((b0 >> bit) & 1) << k);
|
|
1294
|
+
b->qh[8 + bit] |= (uchar)(((b1 >> bit) & 1) << k);
|
|
1295
|
+
b->qh[16 + bit] |= (uchar)(((b2 >> bit) & 1) << k);
|
|
1296
|
+
b->qh[24 + bit] |= (uchar)(((b3 >> bit) & 1) << k);
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
__global uchar * s_src = src_s + (i02 * ne01 + i01) * ne00_blk * K_SCALE_SIZE + i00 * K_SCALE_SIZE;
|
|
1301
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
1302
|
+
b->s[i] = s_src[i];
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
uint base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
|
|
1306
|
+
|
|
1307
|
+
uint4 qv[8];
|
|
1308
|
+
for (int p = 0; p < 8; ++p) {
|
|
1309
|
+
qv[p].x = src_qs[base + (p * 4 + 0) * ne01];
|
|
1310
|
+
qv[p].y = src_qs[base + (p * 4 + 1) * ne01];
|
|
1311
|
+
qv[p].z = src_qs[base + (p * 4 + 2) * ne01];
|
|
1312
|
+
qv[p].w = src_qs[base + (p * 4 + 3) * ne01];
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1315
|
+
uchar * qv_bytes = (uchar *)qv;
|
|
1316
|
+
for (int i = 0; i < QK_K / 64; ++i) {
|
|
1317
|
+
for (int j = 0; j < 16; ++j) {
|
|
1318
|
+
uchar lo = qv_bytes[i*32 + j];
|
|
1319
|
+
uchar hi = qv_bytes[i*32 + j + 16];
|
|
1320
|
+
b->qs[i*32 + 2*j] = convert_uchar((lo & mask_0F) | ((hi & mask_0F) << 4));
|
|
1321
|
+
b->qs[i*32 + 2*j + 1] = convert_uchar(((lo & mask_F0) >> 4) | (hi & mask_F0));
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
kernel void kernel_convert_block_q6_k_trans4_ns(
|
|
1327
|
+
__global struct block_q6_K * src0,
|
|
1328
|
+
__global uint * dst_ql,
|
|
1329
|
+
__global uint * dst_qh,
|
|
1330
|
+
__global half * dst_d,
|
|
1331
|
+
__global char * dst_s,
|
|
1332
|
+
uint ne00,
|
|
1333
|
+
uint ne01,
|
|
1334
|
+
uchar mask_0F,
|
|
1335
|
+
uchar mask_F0
|
|
1336
|
+
) {
|
|
1337
|
+
uint i00 = get_global_id(1);
|
|
1338
|
+
uint i01 = get_global_id(0);
|
|
1339
|
+
uint i02 = get_global_id(2);
|
|
1340
|
+
|
|
1341
|
+
if (i01 >= ne01) {
|
|
1342
|
+
return;
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
uint ne00_blk = ne00 / QK_K;
|
|
1346
|
+
|
|
1347
|
+
uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1348
|
+
uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
1349
|
+
|
|
1350
|
+
__global struct block_q6_K * b = src0 + src_blk_offset;
|
|
1351
|
+
|
|
1352
|
+
dst_d[dst_blk_offset] = b->d;
|
|
1353
|
+
|
|
1354
|
+
uint4 qlv[8];
|
|
1355
|
+
uchar * qlv_bytes = (uchar *)qlv;
|
|
1356
|
+
for (int i = 0; i < 2; ++i) {
|
|
1357
|
+
for (int j = 0; j < 16; ++j) {
|
|
1358
|
+
uchar x0 = b->ql[i*64 + 2*j];
|
|
1359
|
+
uchar x1 = b->ql[i*64 + 2*j + 1];
|
|
1360
|
+
uchar x2 = b->ql[i*64 + 32 + 2*j];
|
|
1361
|
+
uchar x3 = b->ql[i*64 + 32 + 2*j + 1];
|
|
1362
|
+
qlv_bytes[i*64 + j ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
|
|
1363
|
+
qlv_bytes[i*64 + j + 16] = convert_uchar(x2 & mask_0F) | convert_uchar((x3 & mask_0F) << 4);
|
|
1364
|
+
qlv_bytes[i*64 + j + 32] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
|
|
1365
|
+
qlv_bytes[i*64 + j + 48] = convert_uchar((x2 & mask_F0) >> 4) | convert_uchar(x3 & mask_F0);
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
uint ql_base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
|
|
1370
|
+
|
|
1371
|
+
#pragma unroll
|
|
1372
|
+
for (int p = 0; p < 8; ++p) {
|
|
1373
|
+
uint4 v = qlv[p];
|
|
1374
|
+
dst_ql[ql_base + (p * 4 + 0) * ne01] = v.x;
|
|
1375
|
+
dst_ql[ql_base + (p * 4 + 1) * ne01] = v.y;
|
|
1376
|
+
dst_ql[ql_base + (p * 4 + 2) * ne01] = v.z;
|
|
1377
|
+
dst_ql[ql_base + (p * 4 + 3) * ne01] = v.w;
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
uint qhv[16] = {0};
|
|
1381
|
+
|
|
1382
|
+
for (int n = 0; n < 2; ++n) {
|
|
1383
|
+
for (int l = 0; l < 32; ++l) {
|
|
1384
|
+
uchar h = b->qh[n*32 + l];
|
|
1385
|
+
int u = l / 16;
|
|
1386
|
+
int bit_pos = (l % 16) * 2;
|
|
1387
|
+
qhv[(n*4 + 0)*2 + u] |= ((uint)((h >> 0) & 0x03)) << bit_pos;
|
|
1388
|
+
qhv[(n*4 + 1)*2 + u] |= ((uint)((h >> 2) & 0x03)) << bit_pos;
|
|
1389
|
+
qhv[(n*4 + 2)*2 + u] |= ((uint)((h >> 4) & 0x03)) << bit_pos;
|
|
1390
|
+
qhv[(n*4 + 3)*2 + u] |= ((uint)((h >> 6) & 0x03)) << bit_pos;
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
uint qh_base = i02 * ne00_blk * ne01 * 16 + i00 * ne01 * 16 + i01;
|
|
1395
|
+
|
|
1396
|
+
for (int p = 0; p < 16; ++p) {
|
|
1397
|
+
dst_qh[qh_base + p * ne01] = qhv[p];
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
__global char * s_dst = dst_s + (i02 * ne01 + i01) * ne00_blk * 16 + i00 * 16;
|
|
1401
|
+
#pragma unroll
|
|
1402
|
+
for (int i = 0; i < 16; ++i) {
|
|
1403
|
+
s_dst[i] = b->scales[i];
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
kernel void kernel_restore_block_q6_k_trans4_ns(
|
|
1408
|
+
__global uint * src_ql,
|
|
1409
|
+
__global uint * src_qh,
|
|
1410
|
+
__global half * src_d,
|
|
1411
|
+
__global char * src_s,
|
|
1412
|
+
__global struct block_q6_K * dst0,
|
|
1413
|
+
uint ne00,
|
|
1414
|
+
uint ne01,
|
|
1415
|
+
uchar mask_0F,
|
|
1416
|
+
uchar mask_F0
|
|
1417
|
+
) {
|
|
1418
|
+
uint i00 = get_global_id(1); // block index along K
|
|
1419
|
+
uint i01 = get_global_id(0); // row index
|
|
1420
|
+
uint i02 = get_global_id(2); // batch index
|
|
1421
|
+
|
|
1422
|
+
if (i01 >= ne01) {
|
|
1423
|
+
return;
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
uint ne00_blk = ne00 / QK_K;
|
|
1427
|
+
|
|
1428
|
+
uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
1429
|
+
uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1430
|
+
|
|
1431
|
+
__global struct block_q6_K * b = dst0 + dst_blk_offset;
|
|
1432
|
+
|
|
1433
|
+
b->d = src_d[src_blk_offset];
|
|
1434
|
+
|
|
1435
|
+
uint ql_base = i02 * ne00_blk * ne01 * 32 + i00 * ne01 * 32 + i01;
|
|
1436
|
+
uint4 qlv[8];
|
|
1437
|
+
for (int p = 0; p < 8; ++p) {
|
|
1438
|
+
qlv[p].x = src_ql[ql_base + (p * 4 + 0) * ne01];
|
|
1439
|
+
qlv[p].y = src_ql[ql_base + (p * 4 + 1) * ne01];
|
|
1440
|
+
qlv[p].z = src_ql[ql_base + (p * 4 + 2) * ne01];
|
|
1441
|
+
qlv[p].w = src_ql[ql_base + (p * 4 + 3) * ne01];
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
uchar * qlv_bytes = (uchar *)qlv;
|
|
1445
|
+
for (int i = 0; i < 2; ++i) {
|
|
1446
|
+
for (int j = 0; j < 16; ++j) {
|
|
1447
|
+
uchar lo_02 = qlv_bytes[i*64 + j];
|
|
1448
|
+
uchar lo_13 = qlv_bytes[i*64 + j + 16];
|
|
1449
|
+
uchar hi_02 = qlv_bytes[i*64 + j + 32];
|
|
1450
|
+
uchar hi_13 = qlv_bytes[i*64 + j + 48];
|
|
1451
|
+
b->ql[i*64 + 2*j] = convert_uchar((lo_02 & mask_0F) | ((hi_02 & mask_0F) << 4));
|
|
1452
|
+
b->ql[i*64 + 2*j + 1] = convert_uchar(((lo_02 & mask_F0) >> 4) | (hi_02 & mask_F0));
|
|
1453
|
+
b->ql[i*64 + 32 + 2*j] = convert_uchar((lo_13 & mask_0F) | ((hi_13 & mask_0F) << 4));
|
|
1454
|
+
b->ql[i*64 + 32 + 2*j + 1] = convert_uchar(((lo_13 & mask_F0) >> 4) | (hi_13 & mask_F0));
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
uint qh_base = i02 * ne00_blk * ne01 * 16 + i00 * ne01 * 16 + i01;
|
|
1459
|
+
uint qhv[16];
|
|
1460
|
+
for (int p = 0; p < 16; ++p) {
|
|
1461
|
+
qhv[p] = src_qh[qh_base + p * ne01];
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
for (int n = 0; n < 2; ++n) {
|
|
1465
|
+
for (int l = 0; l < 32; ++l) {
|
|
1466
|
+
int u = l / 16;
|
|
1467
|
+
int bit_pos = (l % 16) * 2;
|
|
1468
|
+
uchar v0 = (uchar)((qhv[(n*4 + 0)*2 + u] >> bit_pos) & 0x03);
|
|
1469
|
+
uchar v1 = (uchar)((qhv[(n*4 + 1)*2 + u] >> bit_pos) & 0x03);
|
|
1470
|
+
uchar v2 = (uchar)((qhv[(n*4 + 2)*2 + u] >> bit_pos) & 0x03);
|
|
1471
|
+
uchar v3 = (uchar)((qhv[(n*4 + 3)*2 + u] >> bit_pos) & 0x03);
|
|
1472
|
+
b->qh[n*32 + l] = v0 | (v1 << 2) | (v2 << 4) | (v3 << 6);
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
__global char * s_src = src_s + (i02 * ne01 + i01) * ne00_blk * 16 + i00 * 16;
|
|
1477
|
+
for (int i = 0; i < 16; ++i) {
|
|
1478
|
+
b->scales[i] = s_src[i];
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
//------------------------------------------------------------------------------
|
|
1483
|
+
// block_mxfp4
|
|
1484
|
+
//------------------------------------------------------------------------------
|
|
1485
|
+
#define QK_MXFP4 32
|
|
1486
|
+
struct block_mxfp4 {
|
|
1487
|
+
uchar e; // E8M0
|
|
1488
|
+
uchar qs[QK_MXFP4 / 2];
|
|
1489
|
+
};
|
|
1490
|
+
|
|
1491
|
+
//------------------------------------------------------------------------------
|
|
1492
|
+
// kernel_convert_block_mxfp4
|
|
1493
|
+
// Convert the block_mxfp4 format to 2 separate arrays (AOS -> SOA).
|
|
1494
|
+
// This kernel does not deshuffle the bits.
|
|
1495
|
+
//------------------------------------------------------------------------------
|
|
1496
|
+
kernel void kernel_convert_block_mxfp4(
|
|
1497
|
+
global struct block_mxfp4 * src0,
|
|
1498
|
+
global uchar * dst_q,
|
|
1499
|
+
global uchar * dst_e
|
|
1500
|
+
) {
|
|
1501
|
+
global struct block_mxfp4 * b = (global struct block_mxfp4 *) src0 + get_global_id(0);
|
|
1502
|
+
global uchar * q = (global uchar *) dst_q + QK_MXFP4 / 2 * get_global_id(0);
|
|
1503
|
+
global uchar * e = (global uchar *) dst_e + get_global_id(0);
|
|
1504
|
+
|
|
1505
|
+
*e = b->e;
|
|
1506
|
+
|
|
1507
|
+
for (int i = 0; i < QK_MXFP4 / 2; ++i) {
|
|
1508
|
+
q[i] = b->qs[i];
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
|
|
1512
|
+
kernel void kernel_convert_block_mxfp4_trans(
|
|
1513
|
+
global struct block_mxfp4 * src0,
|
|
1514
|
+
__global uint4 * dst_q,
|
|
1515
|
+
__global uchar * dst_e,
|
|
1516
|
+
uint ne00,
|
|
1517
|
+
uint ne01
|
|
1518
|
+
) {
|
|
1519
|
+
int i00 = get_global_id(1);
|
|
1520
|
+
uint i01 = get_global_id(0);
|
|
1521
|
+
uint i02 = get_global_id(2);
|
|
1522
|
+
|
|
1523
|
+
uint ne00_blk = ne00 / QK_MXFP4;
|
|
1524
|
+
uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1525
|
+
uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
1526
|
+
|
|
1527
|
+
global struct block_mxfp4 * b = src0 + src_blk_offset;
|
|
1528
|
+
|
|
1529
|
+
dst_q[dst_blk_offset] = ((global uint4 *)(&(b->qs[0])))[0];
|
|
1530
|
+
dst_e[dst_blk_offset] = b->e;
|
|
1531
|
+
}
|
|
1532
|
+
|
|
1533
|
+
kernel void kernel_restore_block_mxfp4(
|
|
1534
|
+
global uchar * src_q,
|
|
1535
|
+
global half * src_e,
|
|
1536
|
+
global struct block_mxfp4 * dst
|
|
1537
|
+
) {
|
|
1538
|
+
global struct block_mxfp4 * b = (global struct block_mxfp4 *) dst + get_global_id(0);
|
|
1539
|
+
global uchar * q = (global uchar *) src_q + QK_MXFP4 / 2 * get_global_id(0);
|
|
1540
|
+
global uchar * e = (global uchar *) src_e + get_global_id(0);
|
|
1541
|
+
|
|
1542
|
+
b->e = *e;
|
|
1543
|
+
for (int i = 0; i < QK_MXFP4 / 2; ++i) {
|
|
1544
|
+
b->qs[i] = q[i];
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
kernel void kernel_restore_block_mxfp4_trans(
|
|
1549
|
+
__global uint4 * src_q,
|
|
1550
|
+
__global uchar * src_e,
|
|
1551
|
+
global struct block_mxfp4 * dst,
|
|
1552
|
+
uint ne00,
|
|
1553
|
+
uint ne01
|
|
1554
|
+
) {
|
|
1555
|
+
int i00 = get_global_id(1);
|
|
1556
|
+
uint i01 = get_global_id(0);
|
|
1557
|
+
uint i02 = get_global_id(2);
|
|
1558
|
+
|
|
1559
|
+
uint ne00_blk = ne00 / QK_MXFP4;
|
|
1560
|
+
uint src_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
1561
|
+
uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1562
|
+
|
|
1563
|
+
global struct block_mxfp4 * b = dst + dst_blk_offset;
|
|
1564
|
+
|
|
1565
|
+
((global uint4 *)(&(b->qs[0])))[0] = src_q[src_blk_offset];
|
|
1566
|
+
b->e = src_e[src_blk_offset];
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
//------------------------------------------------------------------------------
|
|
1570
|
+
// kernel_convert_block_q6_k_tiled_ns
|
|
1571
|
+
//
|
|
1572
|
+
// Tiled-wide layout for the long-vocab q6_K lm_head/embed GEMV (decode path).
|
|
1573
|
+
// Unlike *_trans4_ns (which mirrors the bit-interleave the legacy 2-output GEMV
|
|
1574
|
+
// consumes), this kernel is correct-by-construction against the CANONICAL ggml
|
|
1575
|
+
// q6_K dequant: it recovers each weight's 6-bit code in element order e in
|
|
1576
|
+
// [0,256), then re-packs low-4-bits into 32 uints (8 codes/uint) and high-2-bits
|
|
1577
|
+
// into 16 uints (16 codes/uint). The matching GEMV (gemv_noshuffle_q6_k_f32_tiled)
|
|
1578
|
+
// unpacks the same order, so both ends are owned here.
|
|
1579
|
+
//
|
|
1580
|
+
// Storage is TILED by 64 output rows so the GEMV's 64-thread tile coalesces:
|
|
1581
|
+
// ql uint4 granule g of (row r, K-block sb): idx = ((rt*ne00_blk + sb)*8 + g)*64 + rit
|
|
1582
|
+
// qh uint4 granule g: idx = ((rt*ne00_blk + sb)*4 + g)*64 + rit
|
|
1583
|
+
// scales (char16) of (r, sb): idx = (rt*ne00_blk + sb)*64 + rit
|
|
1584
|
+
// d (half) of (r, sb): idx = (rt*ne00_blk + sb)*64 + rit
|
|
1585
|
+
// where rt = r/64, rit = r%64. Requires ne01 % 64 == 0 (gated host-side).
|
|
1586
|
+
// Buffer sizes are byte-identical to the trans4_ns layout.
|
|
1587
|
+
//------------------------------------------------------------------------------
|
|
1588
|
+
kernel void kernel_convert_block_q6_k_tiled_ns(
|
|
1589
|
+
__global struct block_q6_K * src0,
|
|
1590
|
+
__global uint * dst_ql, // 32 uints / superblock (low 4 bits, 8 codes/uint)
|
|
1591
|
+
__global uint * dst_qh, // 16 uints / superblock (high 2 bits, 16 codes/uint)
|
|
1592
|
+
__global half * dst_d, // 1 half / superblock
|
|
1593
|
+
__global char * dst_s, // 16 chars/ superblock
|
|
1594
|
+
uint ne00,
|
|
1595
|
+
uint ne01
|
|
1596
|
+
) {
|
|
1597
|
+
uint i00 = get_global_id(1); // K-block index (superblock along ne00)
|
|
1598
|
+
uint i01 = get_global_id(0); // output row index (along ne01)
|
|
1599
|
+
uint i02 = get_global_id(2); // batch
|
|
1600
|
+
|
|
1601
|
+
uint ne00_blk = ne00 / QK_K;
|
|
1602
|
+
|
|
1603
|
+
// Source block: row-major over (i02, i01, i00).
|
|
1604
|
+
uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1605
|
+
__global struct block_q6_K * b = src0 + src_blk_offset;
|
|
1606
|
+
|
|
1607
|
+
uint rt = i01 / 64;
|
|
1608
|
+
uint rit = i01 % 64;
|
|
1609
|
+
uint tile_blk = (i02 * (ne01 / 64) + rt) * ne00_blk + i00; // tile-major (row-tile, K-block)
|
|
1610
|
+
|
|
1611
|
+
// --- recover canonical 6-bit codes, pack into ql (4b) + qh (2b) in e-order ---
|
|
1612
|
+
// 32 ql-uints (8 low-nibbles each) + 16 qh-uints (16 2-bit slots each).
|
|
1613
|
+
uint qlw[32] = {0};
|
|
1614
|
+
uint qhw[16] = {0};
|
|
1615
|
+
|
|
1616
|
+
for (uint e = 0; e < 256; ++e) {
|
|
1617
|
+
uint n = (e >= 128) ? 1u : 0u; // which 128-half
|
|
1618
|
+
uint within = e - n * 128u;
|
|
1619
|
+
uint q = within / 32u; // quadrant 0..3
|
|
1620
|
+
uint l = within % 32u; // 0..31
|
|
1621
|
+
|
|
1622
|
+
uint off_ql = n * 64u; // raw ql byte base for this half
|
|
1623
|
+
uint off_qh = n * 32u; // raw qh byte base for this half
|
|
1624
|
+
|
|
1625
|
+
uchar low4;
|
|
1626
|
+
uchar qlb0 = b->ql[off_ql + l];
|
|
1627
|
+
uchar qlb1 = b->ql[off_ql + l + 32];
|
|
1628
|
+
if (q == 0) low4 = qlb0 & 0x0F;
|
|
1629
|
+
else if (q == 1) low4 = qlb1 & 0x0F;
|
|
1630
|
+
else if (q == 2) low4 = (qlb0 >> 4) & 0x0F;
|
|
1631
|
+
else low4 = (qlb1 >> 4) & 0x0F;
|
|
1632
|
+
|
|
1633
|
+
uchar hi2 = (b->qh[off_qh + l] >> (q * 2u)) & 0x03;
|
|
1634
|
+
|
|
1635
|
+
// pack low4 (e-order): uint e/8, nibble (e%8)
|
|
1636
|
+
qlw[e >> 3] |= ((uint)low4) << ((e & 7u) * 4u);
|
|
1637
|
+
// pack hi2 (e-order): uint e/16, 2-bit slot (e%16)
|
|
1638
|
+
qhw[e >> 4] |= ((uint)hi2) << ((e & 15u) * 2u);
|
|
1639
|
+
}
|
|
1640
|
+
|
|
1641
|
+
// --- write tiled ---
|
|
1642
|
+
for (uint g = 0; g < 8; ++g) {
|
|
1643
|
+
uint base = (tile_blk * 8u + g) * 64u + rit; // uint4 index
|
|
1644
|
+
dst_ql[base * 4u + 0u] = qlw[g * 4u + 0u];
|
|
1645
|
+
dst_ql[base * 4u + 1u] = qlw[g * 4u + 1u];
|
|
1646
|
+
dst_ql[base * 4u + 2u] = qlw[g * 4u + 2u];
|
|
1647
|
+
dst_ql[base * 4u + 3u] = qlw[g * 4u + 3u];
|
|
1648
|
+
}
|
|
1649
|
+
for (uint g = 0; g < 4; ++g) {
|
|
1650
|
+
uint base = (tile_blk * 4u + g) * 64u + rit; // uint4 index
|
|
1651
|
+
dst_qh[base * 4u + 0u] = qhw[g * 4u + 0u];
|
|
1652
|
+
dst_qh[base * 4u + 1u] = qhw[g * 4u + 1u];
|
|
1653
|
+
dst_qh[base * 4u + 2u] = qhw[g * 4u + 2u];
|
|
1654
|
+
dst_qh[base * 4u + 3u] = qhw[g * 4u + 3u];
|
|
1655
|
+
}
|
|
1656
|
+
|
|
1657
|
+
// scales: 16 chars contiguous per (row, block), tiled
|
|
1658
|
+
__global char * s_dst = dst_s + (tile_blk * 64u + rit) * 16u;
|
|
1659
|
+
#pragma unroll
|
|
1660
|
+
for (int i = 0; i < 16; ++i) {
|
|
1661
|
+
s_dst[i] = b->scales[i];
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
// super-block scale
|
|
1665
|
+
dst_d[tile_blk * 64u + rit] = b->d;
|
|
1666
|
+
}
|
|
1667
|
+
|
|
1668
|
+
kernel void kernel_convert_block_mxfp4_trans4_ns(
|
|
1669
|
+
global struct block_mxfp4 * src0,
|
|
1670
|
+
__global uint * dst_q,
|
|
1671
|
+
__global uchar * dst_e,
|
|
1672
|
+
uint ne00,
|
|
1673
|
+
uint ne01
|
|
1674
|
+
) {
|
|
1675
|
+
uint i00 = get_global_id(1);
|
|
1676
|
+
uint i01 = get_global_id(0);
|
|
1677
|
+
uint i02 = get_global_id(2);
|
|
1678
|
+
|
|
1679
|
+
if (i01 >= ne01) {
|
|
1680
|
+
return;
|
|
1681
|
+
}
|
|
1682
|
+
|
|
1683
|
+
uint ne00_blk = ne00 / QK_MXFP4;
|
|
1684
|
+
uint src_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1685
|
+
uint dst_blk_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
1686
|
+
|
|
1687
|
+
global struct block_mxfp4 * b = src0 + src_blk_offset;
|
|
1688
|
+
dst_e[dst_blk_offset] = b->e;
|
|
1689
|
+
|
|
1690
|
+
// extract quantization and unshuffle
|
|
1691
|
+
ushort8 pre_block = ((global ushort8 *)(&(b->qs[0])))[0];
|
|
1692
|
+
|
|
1693
|
+
ushort8 post_block = (ushort8)(0);
|
|
1694
|
+
|
|
1695
|
+
uchar * pre_block_ptr = (uchar *)(&pre_block);
|
|
1696
|
+
uchar * post_block_ptr = (uchar *)(&post_block);
|
|
1697
|
+
|
|
1698
|
+
for (int i = 0; i < QK_MXFP4 / 4; ++i) {
|
|
1699
|
+
uchar x0 = pre_block_ptr[2*i + 0];
|
|
1700
|
+
uchar x1 = pre_block_ptr[2*i + 1];
|
|
1701
|
+
|
|
1702
|
+
post_block_ptr[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
1703
|
+
post_block_ptr[i + QK_MXFP4 / 4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1706
|
+
uint4 q_block = as_uint4(post_block);
|
|
1707
|
+
|
|
1708
|
+
uint offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
|
|
1709
|
+
dst_q[offset] = q_block.x;
|
|
1710
|
+
dst_q[offset + ne01] = q_block.y;
|
|
1711
|
+
dst_q[offset + ne01 * 2] = q_block.z;
|
|
1712
|
+
dst_q[offset + ne01 * 3] = q_block.w;
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
kernel void kernel_restore_block_mxfp4_trans4_ns(
|
|
1716
|
+
__global uint * src_q,
|
|
1717
|
+
__global uchar * src_e,
|
|
1718
|
+
__global struct block_mxfp4 * dst0,
|
|
1719
|
+
uint ne00,
|
|
1720
|
+
uint ne01
|
|
1721
|
+
) {
|
|
1722
|
+
uint i00 = get_global_id(1);
|
|
1723
|
+
uint i01 = get_global_id(0);
|
|
1724
|
+
uint i02 = get_global_id(2);
|
|
1725
|
+
|
|
1726
|
+
if (i01 >= ne01) {
|
|
1727
|
+
return;
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1730
|
+
uint ne00_blk = ne00 / QK_MXFP4;
|
|
1731
|
+
uint dst_blk_offset = i00 + i01 * ne00_blk + i02 * ne00_blk * ne01;
|
|
1732
|
+
uint src_d_offset = i01 + i00 * ne01 + i02 * ne00_blk * ne01;
|
|
1733
|
+
|
|
1734
|
+
__global struct block_mxfp4 * b = dst0 + dst_blk_offset;
|
|
1735
|
+
b->e = src_e[src_d_offset];
|
|
1736
|
+
|
|
1737
|
+
// collect transposed quantization parts for a block
|
|
1738
|
+
uint src_q_offset = i02 * ne00_blk * ne01 * 4 + i00 * ne01 * 4 + i01;
|
|
1739
|
+
uint4 q_block;
|
|
1740
|
+
q_block.x = src_q[src_q_offset];
|
|
1741
|
+
q_block.y = src_q[src_q_offset + ne01];
|
|
1742
|
+
q_block.z = src_q[src_q_offset + ne01 * 2];
|
|
1743
|
+
q_block.w = src_q[src_q_offset + ne01 * 3];
|
|
1744
|
+
|
|
1745
|
+
ushort8 post_block = as_ushort8(q_block);
|
|
1746
|
+
ushort8 pre_block = (ushort8)(0);
|
|
1747
|
+
|
|
1748
|
+
uchar * pre_block_ptr = (uchar *)(&pre_block);
|
|
1749
|
+
uchar * post_block_ptr = (uchar *)(&post_block);
|
|
1750
|
+
|
|
1751
|
+
for (int i = 0; i < QK_MXFP4 / 4; ++i) {
|
|
1752
|
+
uchar x0 = post_block_ptr[i + 0];
|
|
1753
|
+
uchar x1 = post_block_ptr[i + QK_MXFP4 / 4];
|
|
1754
|
+
|
|
1755
|
+
pre_block_ptr[2 * i + 0] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);
|
|
1756
|
+
pre_block_ptr[2 * i + 1] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1759
|
+
((__global ushort8 *)(&(b->qs[0])))[0] = pre_block;
|
|
1760
|
+
}
|
|
1761
|
+
|
|
1762
|
+
|
|
1763
|
+
//------------------------------------------------------------------------------
|
|
1764
|
+
// block_q8_0
|
|
1765
|
+
//------------------------------------------------------------------------------
|
|
1766
|
+
typedef struct {
|
|
1767
|
+
half d; // delta
|
|
1768
|
+
char qs[QK8_0]; // quants
|
|
1769
|
+
} block_q8_0;
|
|
1770
|
+
|
|
1771
|
+
kernel void kernel_convert_block_q8_0(
|
|
1772
|
+
global block_q8_0 * src0,
|
|
1773
|
+
global uchar * dst_q,
|
|
1774
|
+
global half * dst_d
|
|
1775
|
+
) {
|
|
1776
|
+
global block_q8_0 * b = (global block_q8_0 *) src0 + get_global_id(0);
|
|
1777
|
+
global uchar * q = (global uchar *) dst_q + QK8_0*get_global_id(0);
|
|
1778
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
1779
|
+
|
|
1780
|
+
*d = b->d;
|
|
1781
|
+
|
|
1782
|
+
for (int i = 0; i < QK8_0; ++i) {
|
|
1783
|
+
q[i] = b->qs[i];
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
|
|
1787
|
+
kernel void kernel_restore_block_q8_0(
|
|
1788
|
+
global uchar * src_q,
|
|
1789
|
+
global half * src_d,
|
|
1790
|
+
global block_q8_0 * dst
|
|
1791
|
+
) {
|
|
1792
|
+
global block_q8_0 * b = (global block_q8_0 *) dst + get_global_id(0);
|
|
1793
|
+
global uchar * q = (global uchar *) src_q + QK8_0*get_global_id(0);
|
|
1794
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
1795
|
+
|
|
1796
|
+
b->d = *d;
|
|
1797
|
+
for (int i = 0; i < QK8_0; ++i) {
|
|
1798
|
+
b->qs[i] = q[i];
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
|
|
1802
|
+
// View-aware AoS q8_0 -> f32 dequant (f32/f32 FA path).
|
|
1803
|
+
kernel void kernel_dequant_q8_0_f32_view_aos(
|
|
1804
|
+
global char * src,
|
|
1805
|
+
ulong src_offset,
|
|
1806
|
+
ulong src_nb1,
|
|
1807
|
+
ulong src_nb2,
|
|
1808
|
+
ulong src_nb3,
|
|
1809
|
+
int nblk0,
|
|
1810
|
+
int ne1,
|
|
1811
|
+
int ne2,
|
|
1812
|
+
int ne3,
|
|
1813
|
+
global float * dst
|
|
1814
|
+
) {
|
|
1815
|
+
int blk_i0 = get_global_id(0);
|
|
1816
|
+
int i1 = get_global_id(1);
|
|
1817
|
+
int batch = get_global_id(2);
|
|
1818
|
+
|
|
1819
|
+
if (blk_i0 >= nblk0) return;
|
|
1820
|
+
if (i1 >= ne1) return;
|
|
1821
|
+
|
|
1822
|
+
int i2 = batch % ne2;
|
|
1823
|
+
int i3 = batch / ne2;
|
|
1824
|
+
if (i3 >= ne3) return;
|
|
1825
|
+
|
|
1826
|
+
global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK8_0);
|
|
1827
|
+
float d = vload_half(0, (global half *)block);
|
|
1828
|
+
global char * qs = block + 2;
|
|
1829
|
+
|
|
1830
|
+
ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0;
|
|
1831
|
+
global float * out = dst + (dst_row_base + blk_i0) * QK8_0;
|
|
1832
|
+
|
|
1833
|
+
for (int i = 0; i < QK8_0; ++i) {
|
|
1834
|
+
out[i] = d * (float)qs[i];
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
// View-aware AoS q8_0 -> f16 dequant. Rows tight, batch strides may be gapped.
|
|
1839
|
+
kernel void kernel_dequant_q8_0_f16_view_aos(
|
|
1840
|
+
global char * src,
|
|
1841
|
+
ulong src_offset,
|
|
1842
|
+
ulong src_nb1,
|
|
1843
|
+
ulong src_nb2,
|
|
1844
|
+
ulong src_nb3,
|
|
1845
|
+
int nblk0,
|
|
1846
|
+
int ne1,
|
|
1847
|
+
int ne2,
|
|
1848
|
+
int ne3,
|
|
1849
|
+
global half * dst
|
|
1850
|
+
) {
|
|
1851
|
+
int blk_i0 = get_global_id(0);
|
|
1852
|
+
int i1 = get_global_id(1);
|
|
1853
|
+
int batch = get_global_id(2);
|
|
1854
|
+
|
|
1855
|
+
if (blk_i0 >= nblk0) return;
|
|
1856
|
+
if (i1 >= ne1) return;
|
|
1857
|
+
|
|
1858
|
+
int i2 = batch % ne2;
|
|
1859
|
+
int i3 = batch / ne2;
|
|
1860
|
+
if (i3 >= ne3) return;
|
|
1861
|
+
|
|
1862
|
+
global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK8_0);
|
|
1863
|
+
float d = vload_half(0, (global half *)block);
|
|
1864
|
+
global char * qs = block + 2;
|
|
1865
|
+
|
|
1866
|
+
ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0;
|
|
1867
|
+
global half * out = dst + (dst_row_base + blk_i0) * QK8_0;
|
|
1868
|
+
|
|
1869
|
+
for (int i = 0; i < QK8_0; ++i) {
|
|
1870
|
+
out[i] = (half)(d * (float)qs[i]);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
|
|
1874
|
+
// View-aware AoS q4_0 -> f32 dequant (mirrors the q8_0 view variant).
|
|
1875
|
+
kernel void kernel_dequant_q4_0_f32_view_aos(
|
|
1876
|
+
global char * src,
|
|
1877
|
+
ulong src_offset,
|
|
1878
|
+
ulong src_nb1,
|
|
1879
|
+
ulong src_nb2,
|
|
1880
|
+
ulong src_nb3,
|
|
1881
|
+
int nblk0,
|
|
1882
|
+
int ne1,
|
|
1883
|
+
int ne2,
|
|
1884
|
+
int ne3,
|
|
1885
|
+
global float * dst
|
|
1886
|
+
) {
|
|
1887
|
+
int blk_i0 = get_global_id(0);
|
|
1888
|
+
int i1 = get_global_id(1);
|
|
1889
|
+
int batch = get_global_id(2);
|
|
1890
|
+
|
|
1891
|
+
if (blk_i0 >= nblk0) return;
|
|
1892
|
+
if (i1 >= ne1) return;
|
|
1893
|
+
|
|
1894
|
+
int i2 = batch % ne2;
|
|
1895
|
+
int i3 = batch / ne2;
|
|
1896
|
+
if (i3 >= ne3) return;
|
|
1897
|
+
|
|
1898
|
+
global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK4_0/2);
|
|
1899
|
+
float d = vload_half(0, (global half *)block);
|
|
1900
|
+
global uchar * qs = (global uchar *)(block + 2);
|
|
1901
|
+
|
|
1902
|
+
ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0;
|
|
1903
|
+
global float * out = dst + (dst_row_base + blk_i0) * QK4_0;
|
|
1904
|
+
|
|
1905
|
+
for (int i = 0; i < QK4_0/2; ++i) {
|
|
1906
|
+
uchar byte = qs[i];
|
|
1907
|
+
int q0 = (int)(byte & 0x0F) - 8;
|
|
1908
|
+
int q1 = (int)(byte >> 4) - 8;
|
|
1909
|
+
out[i] = d * (float)q0;
|
|
1910
|
+
out[i + QK4_0/2] = d * (float)q1;
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
|
|
1914
|
+
// View-aware AoS q4_0 -> f16 dequant (mirrors the q8_0 view variant).
|
|
1915
|
+
kernel void kernel_dequant_q4_0_f16_view_aos(
|
|
1916
|
+
global char * src,
|
|
1917
|
+
ulong src_offset,
|
|
1918
|
+
ulong src_nb1,
|
|
1919
|
+
ulong src_nb2,
|
|
1920
|
+
ulong src_nb3,
|
|
1921
|
+
int nblk0,
|
|
1922
|
+
int ne1,
|
|
1923
|
+
int ne2,
|
|
1924
|
+
int ne3,
|
|
1925
|
+
global half * dst
|
|
1926
|
+
) {
|
|
1927
|
+
int blk_i0 = get_global_id(0);
|
|
1928
|
+
int i1 = get_global_id(1);
|
|
1929
|
+
int batch = get_global_id(2);
|
|
1930
|
+
|
|
1931
|
+
if (blk_i0 >= nblk0) return;
|
|
1932
|
+
if (i1 >= ne1) return;
|
|
1933
|
+
|
|
1934
|
+
int i2 = batch % ne2;
|
|
1935
|
+
int i3 = batch / ne2;
|
|
1936
|
+
if (i3 >= ne3) return;
|
|
1937
|
+
|
|
1938
|
+
global char * block = src + src_offset + (ulong)i3*src_nb3 + (ulong)i2*src_nb2 + (ulong)i1*src_nb1 + (ulong)blk_i0 * (2 + QK4_0/2);
|
|
1939
|
+
float d = vload_half(0, (global half *)block);
|
|
1940
|
+
global uchar * qs = (global uchar *)(block + 2);
|
|
1941
|
+
|
|
1942
|
+
ulong dst_row_base = ((ulong)i3 * ne2 * ne1 + (ulong)i2 * ne1 + (ulong)i1) * nblk0;
|
|
1943
|
+
global half * out = dst + (dst_row_base + blk_i0) * QK4_0;
|
|
1944
|
+
|
|
1945
|
+
for (int i = 0; i < QK4_0/2; ++i) {
|
|
1946
|
+
uchar byte = qs[i];
|
|
1947
|
+
int q0 = (int)(byte & 0x0F) - 8;
|
|
1948
|
+
int q1 = (int)(byte >> 4) - 8;
|
|
1949
|
+
out[i] = (half)(d * (float)q0);
|
|
1950
|
+
out[i + QK4_0/2] = (half)(d * (float)q1);
|
|
1951
|
+
}
|
|
1952
|
+
}
|
|
1953
|
+
|
|
1954
|
+
kernel void kernel_restore_block_q8_0_trans(
|
|
1955
|
+
global uchar * src_q,
|
|
1956
|
+
global half * src_d,
|
|
1957
|
+
global block_q8_0 * dst,
|
|
1958
|
+
uint ne00,
|
|
1959
|
+
uint ne01
|
|
1960
|
+
){
|
|
1961
|
+
uint num_blk_per_row = ne00 / QK8_0;
|
|
1962
|
+
|
|
1963
|
+
global block_q8_0 * b = (global block_q8_0 *) dst + get_global_id(0) * num_blk_per_row;
|
|
1964
|
+
global uchar * q = (global uchar *) src_q + get_global_id(0) * 4; // 4 8-bit packed
|
|
1965
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
1966
|
+
|
|
1967
|
+
for (uint blk = 0; blk < num_blk_per_row; blk++) {
|
|
1968
|
+
b->d = *d;
|
|
1969
|
+
|
|
1970
|
+
for (uint i = 0; i < QK8_0; i+=4) {
|
|
1971
|
+
b->qs[i] = q[0];
|
|
1972
|
+
b->qs[i+1] = q[1];
|
|
1973
|
+
b->qs[i+2] = q[2];
|
|
1974
|
+
b->qs[i+3] = q[3];
|
|
1975
|
+
|
|
1976
|
+
q += 4 * ne01; // M stride
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
d += ne01;
|
|
1980
|
+
|
|
1981
|
+
b++;
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
|
|
1985
|
+
//------------------------------------------------------------------------------
|
|
1986
|
+
// kernel_convert_block_q4_K
|
|
1987
|
+
// Convert the block_q4_K format to 4 separate arrays (AOS -> SOA).
|
|
1988
|
+
// This kernel does not deshuffle the bits.
|
|
1989
|
+
// Each thread processes a super block.
|
|
1990
|
+
// Mask args are just to keep the signature consistent with the no-shuffle
|
|
1991
|
+
// version and they are not used in this kernel.
|
|
1992
|
+
//------------------------------------------------------------------------------
|
|
1993
|
+
kernel void kernel_convert_block_q4_K(
|
|
1994
|
+
global struct block_q4_K * src0,
|
|
1995
|
+
global uchar * dst_q,
|
|
1996
|
+
global uchar * dst_s,
|
|
1997
|
+
global half * dst_d,
|
|
1998
|
+
global half * dst_dm,
|
|
1999
|
+
uchar mask_0F,
|
|
2000
|
+
uchar mask_F0
|
|
2001
|
+
) {
|
|
2002
|
+
global struct block_q4_K * b = (global struct block_q4_K *) src0 + get_global_id(0);
|
|
2003
|
+
global uchar * q = (global uchar *) dst_q + QK_K/2*get_global_id(0);
|
|
2004
|
+
global uchar * s = (global uchar *) dst_s + K_SCALE_SIZE*get_global_id(0);
|
|
2005
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
2006
|
+
global half * dm = (global half *) dst_dm + get_global_id(0);
|
|
2007
|
+
|
|
2008
|
+
*d = b->d;
|
|
2009
|
+
*dm = b->dm;
|
|
2010
|
+
|
|
2011
|
+
for (int i = 0; i < QK_K/2; ++i) {
|
|
2012
|
+
q[i] = b->q[i];
|
|
2013
|
+
}
|
|
2014
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
2015
|
+
s[i] = b->s[i];
|
|
2016
|
+
}
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
// Restore block_q4_K from flattened arrays.
|
|
2020
|
+
// Each thread processes a super block.
|
|
2021
|
+
// Mask args are just to keep the signature consistent with the no-shuffle ones.
|
|
2022
|
+
kernel void kernel_restore_block_q4_K(
|
|
2023
|
+
global uchar * src_q,
|
|
2024
|
+
global uchar * src_s,
|
|
2025
|
+
global half * src_d,
|
|
2026
|
+
global half * src_dm,
|
|
2027
|
+
global struct block_q4_K * dst,
|
|
2028
|
+
uchar mask_0F,
|
|
2029
|
+
uchar mask_F0
|
|
2030
|
+
) {
|
|
2031
|
+
global struct block_q4_K * b = (global struct block_q4_K *) dst + get_global_id(0);
|
|
2032
|
+
global uchar * q = (global uchar *) src_q + QK_K/2*get_global_id(0);
|
|
2033
|
+
global uchar * s = (global uchar *) src_s + K_SCALE_SIZE*get_global_id(0);
|
|
2034
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
2035
|
+
global half * dm = (global half *) src_dm + get_global_id(0);
|
|
2036
|
+
|
|
2037
|
+
b->d = *d;
|
|
2038
|
+
b->dm = *dm;
|
|
2039
|
+
|
|
2040
|
+
for (int i = 0; i < QK_K/2; ++i) {
|
|
2041
|
+
b->q[i] = q[i];
|
|
2042
|
+
}
|
|
2043
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
2044
|
+
b->s[i] = s[i];
|
|
2045
|
+
}
|
|
2046
|
+
}
|
|
2047
|
+
|
|
2048
|
+
kernel void kernel_convert_block_q4_K_noshuffle(
|
|
2049
|
+
global struct block_q4_K * src0,
|
|
2050
|
+
global uchar * dst_q,
|
|
2051
|
+
global uchar * dst_s,
|
|
2052
|
+
global half * dst_d,
|
|
2053
|
+
global half * dst_dm,
|
|
2054
|
+
uchar mask_0F,
|
|
2055
|
+
uchar mask_F0
|
|
2056
|
+
) {
|
|
2057
|
+
global struct block_q4_K * b = (global struct block_q4_K *) src0 + get_global_id(0);
|
|
2058
|
+
global uchar * q = (global uchar *) dst_q + QK_K/2 * get_global_id(0);
|
|
2059
|
+
global uchar * s = (global uchar *) dst_s + K_SCALE_SIZE * get_global_id(0);
|
|
2060
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
2061
|
+
global half * dm = (global half *) dst_dm + get_global_id(0);
|
|
2062
|
+
|
|
2063
|
+
*d = b->d;
|
|
2064
|
+
*dm = b->dm;
|
|
2065
|
+
|
|
2066
|
+
for (int i = 0; i < QK_K / 64; ++i) {
|
|
2067
|
+
for (int j = 0; j < 16; ++j) {
|
|
2068
|
+
uchar x0 = b->q[i*32 + 2*j];
|
|
2069
|
+
uchar x1 = b->q[i*32 + 2*j + 1];
|
|
2070
|
+
q[i*32 + j] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
|
|
2071
|
+
q[i*32 + j + 16] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
|
|
2072
|
+
}
|
|
2073
|
+
}
|
|
2074
|
+
|
|
2075
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
2076
|
+
s[i] = b->s[i];
|
|
2077
|
+
}
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2080
|
+
kernel void kernel_restore_block_q4_K_noshuffle(
|
|
2081
|
+
global uchar * src_q,
|
|
2082
|
+
global uchar * src_s,
|
|
2083
|
+
global half * src_d,
|
|
2084
|
+
global half * src_dm,
|
|
2085
|
+
global struct block_q4_K * dst,
|
|
2086
|
+
uchar mask_0F,
|
|
2087
|
+
uchar mask_F0
|
|
2088
|
+
) {
|
|
2089
|
+
global struct block_q4_K * b = (global struct block_q4_K *) dst + get_global_id(0);
|
|
2090
|
+
global uchar * q = (global uchar *) src_q + QK_K/2 * get_global_id(0);
|
|
2091
|
+
global uchar * s = (global uchar *) src_s + K_SCALE_SIZE * get_global_id(0);
|
|
2092
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
2093
|
+
global half * dm = (global half *) src_dm + get_global_id(0);
|
|
2094
|
+
|
|
2095
|
+
b->d = *d;
|
|
2096
|
+
b->dm = *dm;
|
|
2097
|
+
|
|
2098
|
+
for (int i = 0; i < QK_K / 64; ++i) {
|
|
2099
|
+
for (int j = 0; j < 16; ++j) {
|
|
2100
|
+
uchar lo = q[i*32 + j];
|
|
2101
|
+
uchar hi = q[i*32 + j + 16];
|
|
2102
|
+
b->q[i*32 + 2*j] = convert_uchar((lo & mask_0F) | ((hi & mask_0F) << 4));
|
|
2103
|
+
b->q[i*32 + 2*j + 1] = convert_uchar(((lo & mask_F0) >> 4) | (hi & mask_F0));
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
2108
|
+
b->s[i] = s[i];
|
|
2109
|
+
}
|
|
2110
|
+
}
|
|
2111
|
+
|
|
2112
|
+
//------------------------------------------------------------------------------
|
|
2113
|
+
// kernel_convert_block_q5_K
|
|
2114
|
+
// Convert the block_q5_K format to 5 separate arrays (AOS -> SOA).
|
|
2115
|
+
// Each thread processes a super block.
|
|
2116
|
+
//------------------------------------------------------------------------------
|
|
2117
|
+
kernel void kernel_convert_block_q5_K(
|
|
2118
|
+
global struct block_q5_K * src0,
|
|
2119
|
+
global uchar * dst_q,
|
|
2120
|
+
global uchar * dst_qh,
|
|
2121
|
+
global uchar * dst_s,
|
|
2122
|
+
global half * dst_d,
|
|
2123
|
+
global half * dst_dm,
|
|
2124
|
+
uchar mask_0F,
|
|
2125
|
+
uchar mask_F0
|
|
2126
|
+
) {
|
|
2127
|
+
global struct block_q5_K * b = (global struct block_q5_K *) src0 + get_global_id(0);
|
|
2128
|
+
global uchar * q = (global uchar *) dst_q + QK_K/2*get_global_id(0);
|
|
2129
|
+
global uchar * qh = (global uchar *) dst_qh + QK_K/8*get_global_id(0);
|
|
2130
|
+
global uchar * s = (global uchar *) dst_s + K_SCALE_SIZE*get_global_id(0);
|
|
2131
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
2132
|
+
global half * dm = (global half *) dst_dm + get_global_id(0);
|
|
2133
|
+
|
|
2134
|
+
*d = b->d;
|
|
2135
|
+
*dm = b->dm;
|
|
2136
|
+
|
|
2137
|
+
for (int i = 0; i < QK_K/2; ++i) {
|
|
2138
|
+
q[i] = b->qs[i];
|
|
2139
|
+
}
|
|
2140
|
+
for (int i = 0; i < QK_K/8; ++i) {
|
|
2141
|
+
qh[i] = b->qh[i];
|
|
2142
|
+
}
|
|
2143
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
2144
|
+
s[i] = b->s[i];
|
|
2145
|
+
}
|
|
2146
|
+
}
|
|
2147
|
+
|
|
2148
|
+
// Restore block_q5_K from flattened arrays.
|
|
2149
|
+
// Each thread processes a super block.
|
|
2150
|
+
kernel void kernel_restore_block_q5_K(
|
|
2151
|
+
global uchar * src_q,
|
|
2152
|
+
global uchar * src_qh,
|
|
2153
|
+
global uchar * src_s,
|
|
2154
|
+
global half * src_d,
|
|
2155
|
+
global half * src_dm,
|
|
2156
|
+
global struct block_q5_K * dst,
|
|
2157
|
+
uchar mask_0F,
|
|
2158
|
+
uchar mask_F0
|
|
2159
|
+
) {
|
|
2160
|
+
global struct block_q5_K * b = (global struct block_q5_K *) dst + get_global_id(0);
|
|
2161
|
+
global uchar * q = (global uchar *) src_q + QK_K/2*get_global_id(0);
|
|
2162
|
+
global uchar * qh = (global uchar *) src_qh + QK_K/8*get_global_id(0);
|
|
2163
|
+
global uchar * s = (global uchar *) src_s + K_SCALE_SIZE*get_global_id(0);
|
|
2164
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
2165
|
+
global half * dm = (global half *) src_dm + get_global_id(0);
|
|
2166
|
+
|
|
2167
|
+
b->d = *d;
|
|
2168
|
+
b->dm = *dm;
|
|
2169
|
+
|
|
2170
|
+
for (int i = 0; i < QK_K/2; ++i) {
|
|
2171
|
+
b->qs[i] = q[i];
|
|
2172
|
+
}
|
|
2173
|
+
for (int i = 0; i < QK_K/8; ++i) {
|
|
2174
|
+
b->qh[i] = qh[i];
|
|
2175
|
+
}
|
|
2176
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
2177
|
+
b->s[i] = s[i];
|
|
2178
|
+
}
|
|
2179
|
+
}
|
|
2180
|
+
|
|
2181
|
+
kernel void kernel_convert_block_q5_K_noshuffle(
|
|
2182
|
+
global struct block_q5_K * src0,
|
|
2183
|
+
global uchar * dst_q,
|
|
2184
|
+
global uchar * dst_qh,
|
|
2185
|
+
global uchar * dst_s,
|
|
2186
|
+
global half * dst_d,
|
|
2187
|
+
global half * dst_dm,
|
|
2188
|
+
uchar mask_0F,
|
|
2189
|
+
uchar mask_F0
|
|
2190
|
+
) {
|
|
2191
|
+
global struct block_q5_K * b = (global struct block_q5_K *) src0 + get_global_id(0);
|
|
2192
|
+
global uchar * q = (global uchar *) dst_q + QK_K/2 * get_global_id(0);
|
|
2193
|
+
global uchar * qh = (global uchar *) dst_qh + QK_K/8 * get_global_id(0);
|
|
2194
|
+
global uchar * s = (global uchar *) dst_s + K_SCALE_SIZE * get_global_id(0);
|
|
2195
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
2196
|
+
global half * dm = (global half *) dst_dm + get_global_id(0);
|
|
2197
|
+
|
|
2198
|
+
*d = b->d;
|
|
2199
|
+
*dm = b->dm;
|
|
2200
|
+
|
|
2201
|
+
for (int i = 0; i < QK_K / 64; ++i) {
|
|
2202
|
+
for (int j = 0; j < 16; ++j) {
|
|
2203
|
+
uchar x0 = b->qs[i*32 + 2*j];
|
|
2204
|
+
uchar x1 = b->qs[i*32 + 2*j + 1];
|
|
2205
|
+
q[i*32 + j] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
|
|
2206
|
+
q[i*32 + j + 16] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
|
|
2207
|
+
}
|
|
2208
|
+
}
|
|
2209
|
+
|
|
2210
|
+
for (int l = 0; l < QK_K/8; ++l) {
|
|
2211
|
+
uchar x0 = 0;
|
|
2212
|
+
for (int i = 0; i < 8; ++i) {
|
|
2213
|
+
x0 |= ((b->qh[(l%4)*8+i] >> (l/4)) & 0x01) << i;
|
|
2214
|
+
}
|
|
2215
|
+
qh[l] = x0;
|
|
2216
|
+
}
|
|
2217
|
+
|
|
2218
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
2219
|
+
s[i] = b->s[i];
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2223
|
+
kernel void kernel_restore_block_q5_K_noshuffle(
|
|
2224
|
+
global uchar * src_q,
|
|
2225
|
+
global uchar * src_qh,
|
|
2226
|
+
global uchar * src_s,
|
|
2227
|
+
global half * src_d,
|
|
2228
|
+
global half * src_dm,
|
|
2229
|
+
global struct block_q5_K * dst,
|
|
2230
|
+
uchar mask_0F,
|
|
2231
|
+
uchar mask_F0
|
|
2232
|
+
) {
|
|
2233
|
+
global struct block_q5_K * b = (global struct block_q5_K *) dst + get_global_id(0);
|
|
2234
|
+
global uchar * q = (global uchar *) src_q + QK_K/2 * get_global_id(0);
|
|
2235
|
+
global uchar * qh = (global uchar *) src_qh + QK_K/8 * get_global_id(0);
|
|
2236
|
+
global uchar * s = (global uchar *) src_s + K_SCALE_SIZE * get_global_id(0);
|
|
2237
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
2238
|
+
global half * dm = (global half *) src_dm + get_global_id(0);
|
|
2239
|
+
|
|
2240
|
+
b->d = *d;
|
|
2241
|
+
b->dm = *dm;
|
|
2242
|
+
|
|
2243
|
+
for (int i = 0; i < QK_K / 64; ++i) {
|
|
2244
|
+
for (int j = 0; j < 16; ++j) {
|
|
2245
|
+
uchar lo = q[i*32 + j];
|
|
2246
|
+
uchar hi = q[i*32 + j + 16];
|
|
2247
|
+
b->qs[i*32 + 2*j] = convert_uchar((lo & mask_0F) | ((hi & mask_0F) << 4));
|
|
2248
|
+
b->qs[i*32 + 2*j + 1] = convert_uchar(((lo & mask_F0) >> 4) | (hi & mask_F0));
|
|
2249
|
+
}
|
|
2250
|
+
}
|
|
2251
|
+
|
|
2252
|
+
for (int g = 0; g < 4; ++g) {
|
|
2253
|
+
for (int i = 0; i < 8; ++i) {
|
|
2254
|
+
uchar x0 = 0;
|
|
2255
|
+
for (int k = 0; k < 8; ++k) {
|
|
2256
|
+
x0 |= ((qh[4*k+g] >> i) & 0x01) << k;
|
|
2257
|
+
}
|
|
2258
|
+
b->qh[g*8+i] = x0;
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
for (int i = 0; i < K_SCALE_SIZE; ++i) {
|
|
2263
|
+
b->s[i] = s[i];
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
2266
|
+
|
|
2267
|
+
//------------------------------------------------------------------------------
|
|
2268
|
+
// kernel_convert_block_q6_K
|
|
2269
|
+
// Convert the block_q6_K format to 3 separate arrays (AOS -> SOA).
|
|
2270
|
+
// This kernel does not deshuffle the bits.
|
|
2271
|
+
// Each thread processes a super block.
|
|
2272
|
+
//------------------------------------------------------------------------------
|
|
2273
|
+
kernel void kernel_convert_block_q6_K(
|
|
2274
|
+
global struct block_q6_K * src0,
|
|
2275
|
+
global uchar * dst_ql,
|
|
2276
|
+
global uchar * dst_qh,
|
|
2277
|
+
global char * dst_s,
|
|
2278
|
+
global half * dst_d,
|
|
2279
|
+
uchar mask_lsb_8,
|
|
2280
|
+
ulong n_blk
|
|
2281
|
+
) {
|
|
2282
|
+
if (get_global_id(0) >= n_blk) {
|
|
2283
|
+
return;
|
|
2284
|
+
}
|
|
2285
|
+
global struct block_q6_K * b = (global struct block_q6_K *) src0 + get_global_id(0);
|
|
2286
|
+
global uchar * ql = (global uchar *) dst_ql + QK_K/2*get_global_id(0);
|
|
2287
|
+
global uchar * qh = (global uchar *) dst_qh + QK_K/4*get_global_id(0);
|
|
2288
|
+
global char * s = (global char *) dst_s + QK_K/16*get_global_id(0);
|
|
2289
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
2290
|
+
|
|
2291
|
+
*d = b->d;
|
|
2292
|
+
|
|
2293
|
+
for (int i = 0; i < QK_K/2; ++i) {
|
|
2294
|
+
ql[i] = b->ql[i];
|
|
2295
|
+
}
|
|
2296
|
+
for (int i = 0; i < QK_K/4; ++i) {
|
|
2297
|
+
qh[i] = b->qh[i];
|
|
2298
|
+
}
|
|
2299
|
+
for (int i = 0; i < QK_K/16; ++i) {
|
|
2300
|
+
s[i] = b->scales[i];
|
|
2301
|
+
}
|
|
2302
|
+
}
|
|
2303
|
+
|
|
2304
|
+
// Restore block_q6_K from flattened arrays.
|
|
2305
|
+
// Each thread processes a super block.
|
|
2306
|
+
kernel void kernel_restore_block_q6_K(
|
|
2307
|
+
global uchar * dst_ql,
|
|
2308
|
+
global uchar * dst_qh,
|
|
2309
|
+
global char * dst_s,
|
|
2310
|
+
global half * dst_d,
|
|
2311
|
+
global struct block_q6_K * dst,
|
|
2312
|
+
uchar mask_lsb_8,
|
|
2313
|
+
ulong n_blk
|
|
2314
|
+
) {
|
|
2315
|
+
if (get_global_id(0) >= n_blk) {
|
|
2316
|
+
return;
|
|
2317
|
+
}
|
|
2318
|
+
global struct block_q6_K * b = (global struct block_q6_K *) dst + get_global_id(0);
|
|
2319
|
+
global uchar * ql = (global uchar *) dst_ql + QK_K/2*get_global_id(0);
|
|
2320
|
+
global uchar * qh = (global uchar *) dst_qh + QK_K/4*get_global_id(0);
|
|
2321
|
+
global char * s = (global char *) dst_s + QK_K/16*get_global_id(0);
|
|
2322
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
2323
|
+
|
|
2324
|
+
b->d = *d;
|
|
2325
|
+
|
|
2326
|
+
for (int i = 0; i < QK_K/2; ++i) {
|
|
2327
|
+
b->ql[i] = ql[i];
|
|
2328
|
+
}
|
|
2329
|
+
for (int i = 0; i < QK_K/4; ++i) {
|
|
2330
|
+
b->qh[i] = qh[i];
|
|
2331
|
+
}
|
|
2332
|
+
for (int i = 0; i < QK_K/16; ++i) {
|
|
2333
|
+
b->scales[i] = s[i];
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
|
|
2337
|
+
kernel void kernel_convert_block_q6_K_noshuffle(
|
|
2338
|
+
global struct block_q6_K * src0,
|
|
2339
|
+
global uchar * dst_ql,
|
|
2340
|
+
global uchar * dst_qh,
|
|
2341
|
+
global char * dst_s,
|
|
2342
|
+
global half * dst_d,
|
|
2343
|
+
uchar mask_lsb_8,
|
|
2344
|
+
ulong n_blk
|
|
2345
|
+
) {
|
|
2346
|
+
if (get_global_id(0) >= n_blk) {
|
|
2347
|
+
return;
|
|
2348
|
+
}
|
|
2349
|
+
global struct block_q6_K * b = (global struct block_q6_K *) src0 + get_global_id(0);
|
|
2350
|
+
global uchar * ql = (global uchar *) dst_ql + QK_K/2*get_global_id(0);
|
|
2351
|
+
global uchar * qh = (global uchar *) dst_qh + QK_K/4*get_global_id(0);
|
|
2352
|
+
global char * s = (global char *) dst_s + QK_K/16*get_global_id(0);
|
|
2353
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
2354
|
+
|
|
2355
|
+
*d = b->d;
|
|
2356
|
+
|
|
2357
|
+
for (int i = 0; i < QK_K/2/4; ++i) {
|
|
2358
|
+
uchar x0 = b->ql[i*2 + 0] & mask_lsb_8;
|
|
2359
|
+
uchar x1 = b->ql[i*2 + 1] & mask_lsb_8;
|
|
2360
|
+
ql[i + 0] = (x0 & 0x0F) | ((x1 & 0x0F) << 4);
|
|
2361
|
+
ql[i + 32] = ((x0 & 0xF0) >> 4) | (x1 & 0xF0);
|
|
2362
|
+
|
|
2363
|
+
uchar x2 = b->ql[i*2 + 0 + 64] & mask_lsb_8;
|
|
2364
|
+
uchar x3 = b->ql[i*2 + 1 + 64] & mask_lsb_8;
|
|
2365
|
+
ql[i + 64] = (x2 & 0x0F) | ((x3 & 0x0F) << 4);
|
|
2366
|
+
ql[i + 96] = ((x2 & 0xF0) >> 4) | (x3 & 0xF0);
|
|
2367
|
+
}
|
|
2368
|
+
|
|
2369
|
+
for (int i = 0; i < QK_K/4/8; ++i) {
|
|
2370
|
+
uchar x0 = b->qh[i*4 + 0] & mask_lsb_8;
|
|
2371
|
+
uchar x1 = b->qh[i*4 + 1] & mask_lsb_8;
|
|
2372
|
+
uchar x2 = b->qh[i*4 + 2] & mask_lsb_8;
|
|
2373
|
+
uchar x3 = b->qh[i*4 + 3] & mask_lsb_8;
|
|
2374
|
+
qh[i + 0] = (x0 & 0x03) | ((x1 & 0x03) << 2) | ((x2 & 0x03) << 4) | ((x3 & 0x03) << 6);
|
|
2375
|
+
qh[i + 8] = ((x0 & 0x0C) >> 2) | (x1 & 0x0C) | ((x2 & 0x0C) << 2) | ((x3 & 0x0C) << 4);
|
|
2376
|
+
qh[i + 16] = ((x0 & 0x30) >> 4) | ((x1 & 0x30) >> 2) | (x2 & 0x30) | ((x3 & 0x30) << 2);
|
|
2377
|
+
qh[i + 24] = ((x0 & 0xC0) >> 6) | ((x1 & 0xC0) >> 4) | ((x2 & 0xC0) >> 2) | (x3 & 0xC0);
|
|
2378
|
+
|
|
2379
|
+
uchar x4 = b->qh[i*4 + 0 + 32] & mask_lsb_8;
|
|
2380
|
+
uchar x5 = b->qh[i*4 + 1 + 32] & mask_lsb_8;
|
|
2381
|
+
uchar x6 = b->qh[i*4 + 2 + 32] & mask_lsb_8;
|
|
2382
|
+
uchar x7 = b->qh[i*4 + 3 + 32] & mask_lsb_8;
|
|
2383
|
+
qh[i + 32] = (x4 & 0x03) | ((x5 & 0x03) << 2) | ((x6 & 0x03) << 4) | ((x7 & 0x03) << 6);
|
|
2384
|
+
qh[i + 40] = ((x4 & 0x0C) >> 2) | (x5 & 0x0C) | ((x6 & 0x0C) << 2) | ((x7 & 0x0C) << 4);
|
|
2385
|
+
qh[i + 48] = ((x4 & 0x30) >> 4) | ((x5 & 0x30) >> 2) | (x6 & 0x30) | ((x7 & 0x30) << 2);
|
|
2386
|
+
qh[i + 56] = ((x4 & 0xC0) >> 6) | ((x5 & 0xC0) >> 4) | ((x6 & 0xC0) >> 2) | (x7 & 0xC0);
|
|
2387
|
+
}
|
|
2388
|
+
|
|
2389
|
+
for (int i = 0; i < QK_K/16; ++i) {
|
|
2390
|
+
s[i] = b->scales[i];
|
|
2391
|
+
}
|
|
2392
|
+
}
|
|
2393
|
+
|
|
2394
|
+
kernel void kernel_restore_block_q6_K_noshuffle(
|
|
2395
|
+
global uchar * src_ql,
|
|
2396
|
+
global uchar * src_qh,
|
|
2397
|
+
global char * src_s,
|
|
2398
|
+
global half * src_d,
|
|
2399
|
+
global struct block_q6_K * dst,
|
|
2400
|
+
uchar mask_lsb_8,
|
|
2401
|
+
ulong n_blk
|
|
2402
|
+
) {
|
|
2403
|
+
if (get_global_id(0) >= n_blk) {
|
|
2404
|
+
return;
|
|
2405
|
+
}
|
|
2406
|
+
global struct block_q6_K * b = (global struct block_q6_K *) dst + get_global_id(0);
|
|
2407
|
+
global uchar * ql = (global uchar *) src_ql + QK_K/2*get_global_id(0);
|
|
2408
|
+
global uchar * qh = (global uchar *) src_qh + QK_K/4*get_global_id(0);
|
|
2409
|
+
global char * s = (global char *) src_s + QK_K/16*get_global_id(0);
|
|
2410
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
2411
|
+
|
|
2412
|
+
b->d = *d;
|
|
2413
|
+
|
|
2414
|
+
for (int i = 0; i < QK_K/2/4; ++i) {
|
|
2415
|
+
uchar x0 = ql[i + 0] & mask_lsb_8;
|
|
2416
|
+
uchar x1 = ql[i + 32] & mask_lsb_8;
|
|
2417
|
+
b->ql[i*2 + 0] = (x0 & 0x0F) | ((x1 & 0x0F) << 4);
|
|
2418
|
+
b->ql[i*2 + 1] = ((x0 & 0xF0) >> 4) | (x1 & 0xF0);
|
|
2419
|
+
|
|
2420
|
+
uchar x2 = ql[i + 64] & mask_lsb_8;
|
|
2421
|
+
uchar x3 = ql[i + 96] & mask_lsb_8;
|
|
2422
|
+
b->ql[i*2 + 0 + 64] = (x2 & 0x0F) | ((x3 & 0x0F) << 4);
|
|
2423
|
+
b->ql[i*2 + 1 + 64] = ((x2 & 0xF0) >> 4) | (x3 & 0xF0);
|
|
2424
|
+
}
|
|
2425
|
+
|
|
2426
|
+
for (int i = 0; i < QK_K/4/8; ++i) {
|
|
2427
|
+
uchar x0 = qh[i + 0] & mask_lsb_8;
|
|
2428
|
+
uchar x1 = qh[i + 8] & mask_lsb_8;
|
|
2429
|
+
uchar x2 = qh[i + 16] & mask_lsb_8;
|
|
2430
|
+
uchar x3 = qh[i + 24] & mask_lsb_8;
|
|
2431
|
+
b->qh[i*4 + 0] = (x0 & 0x03) | ((x1 & 0x03) << 2) | ((x2 & 0x03) << 4) | ((x3 & 0x03) << 6);
|
|
2432
|
+
b->qh[i*4 + 1] = ((x0 & 0x0C) >> 2) | (x1 & 0x0C) | ((x2 & 0x0C) << 2) | ((x3 & 0x0C) << 4);
|
|
2433
|
+
b->qh[i*4 + 2] = ((x0 & 0x30) >> 4) | ((x1 & 0x30) >> 2) | (x2 & 0x30) | ((x3 & 0x30) << 2);
|
|
2434
|
+
b->qh[i*4 + 3] = ((x0 & 0xC0) >> 6) | ((x1 & 0xC0) >> 4) | ((x2 & 0xC0) >> 2) | (x3 & 0xC0);
|
|
2435
|
+
|
|
2436
|
+
uchar x4 = qh[i + 0 + 32] & mask_lsb_8;
|
|
2437
|
+
uchar x5 = qh[i + 8 + 32] & mask_lsb_8;
|
|
2438
|
+
uchar x6 = qh[i + 16 + 32] & mask_lsb_8;
|
|
2439
|
+
uchar x7 = qh[i + 24 + 32] & mask_lsb_8;
|
|
2440
|
+
b->qh[i*4 + 0 + 32] = (x4 & 0x03) | ((x5 & 0x03) << 2) | ((x6 & 0x03) << 4) | ((x7 & 0x03) << 6);
|
|
2441
|
+
b->qh[i*4 + 1 + 32] = ((x4 & 0x0C) >> 2) | (x5 & 0x0C) | ((x6 & 0x0C) << 2) | ((x7 & 0x0C) << 4);
|
|
2442
|
+
b->qh[i*4 + 2 + 32] = ((x4 & 0x30) >> 4) | ((x5 & 0x30) >> 2) | (x6 & 0x30) | ((x7 & 0x30) << 2);
|
|
2443
|
+
b->qh[i*4 + 3 + 32] = ((x4 & 0xC0) >> 6) | ((x5 & 0xC0) >> 4) | ((x6 & 0xC0) >> 2) | (x7 & 0xC0);
|
|
2444
|
+
}
|
|
2445
|
+
|
|
2446
|
+
for (int i = 0; i < QK_K/16; ++i) {
|
|
2447
|
+
b->scales[i] = s[i];
|
|
2448
|
+
}
|
|
2449
|
+
}
|
|
2450
|
+
|
|
2451
|
+
//------------------------------------------------------------------------------
|
|
2452
|
+
// kernel_convert_block_iq4_nl
|
|
2453
|
+
// Convert the block_iq4_nl format to 2 separate arrays (AOS -> SOA).
|
|
2454
|
+
//------------------------------------------------------------------------------
|
|
2455
|
+
kernel void kernel_convert_block_iq4_nl(
|
|
2456
|
+
global struct block_iq4_nl * src0,
|
|
2457
|
+
global uchar * dst_q,
|
|
2458
|
+
global half * dst_d,
|
|
2459
|
+
uchar mask_0F,
|
|
2460
|
+
uchar mask_F0,
|
|
2461
|
+
ulong n_blk
|
|
2462
|
+
) {
|
|
2463
|
+
if (get_global_id(0) >= n_blk) {
|
|
2464
|
+
return;
|
|
2465
|
+
}
|
|
2466
|
+
global struct block_iq4_nl * b = (global struct block_iq4_nl *) src0 + get_global_id(0);
|
|
2467
|
+
global uchar * q = (global uchar *) dst_q + QK4_NL/2*get_global_id(0);
|
|
2468
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
2469
|
+
|
|
2470
|
+
*d = b->d;
|
|
2471
|
+
|
|
2472
|
+
for (int i = 0; i < QK4_NL/2; ++i) {
|
|
2473
|
+
q[i] = b->qs[i];
|
|
2474
|
+
}
|
|
2475
|
+
}
|
|
2476
|
+
|
|
2477
|
+
kernel void kernel_restore_block_iq4_nl(
|
|
2478
|
+
global uchar * src_q,
|
|
2479
|
+
global half * src_d,
|
|
2480
|
+
global struct block_iq4_nl * dst,
|
|
2481
|
+
ulong n_blk
|
|
2482
|
+
) {
|
|
2483
|
+
if (get_global_id(0) >= n_blk) {
|
|
2484
|
+
return;
|
|
2485
|
+
}
|
|
2486
|
+
global struct block_iq4_nl * b = (global struct block_iq4_nl *) dst + get_global_id(0);
|
|
2487
|
+
global uchar * q = (global uchar *) src_q + QK4_NL/2*get_global_id(0);
|
|
2488
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
2489
|
+
|
|
2490
|
+
b->d = *d;
|
|
2491
|
+
|
|
2492
|
+
for (int i = 0; i < QK4_NL/2; ++i) {
|
|
2493
|
+
b->qs[i] = q[i];
|
|
2494
|
+
}
|
|
2495
|
+
}
|
|
2496
|
+
|
|
2497
|
+
kernel void kernel_convert_block_iq4_nl_noshuffle(
|
|
2498
|
+
global struct block_iq4_nl * src0,
|
|
2499
|
+
global uchar * dst_q,
|
|
2500
|
+
global half * dst_d,
|
|
2501
|
+
uchar mask_0F,
|
|
2502
|
+
uchar mask_F0,
|
|
2503
|
+
ulong n_blk
|
|
2504
|
+
) {
|
|
2505
|
+
if (get_global_id(0) >= n_blk) {
|
|
2506
|
+
return;
|
|
2507
|
+
}
|
|
2508
|
+
global struct block_iq4_nl * b = (global struct block_iq4_nl *) src0 + get_global_id(0);
|
|
2509
|
+
global uchar * q = (global uchar *) dst_q + QK4_NL/2*get_global_id(0);
|
|
2510
|
+
global half * d = (global half *) dst_d + get_global_id(0);
|
|
2511
|
+
|
|
2512
|
+
*d = b->d;
|
|
2513
|
+
for (int i = 0; i < QK4_NL/4; ++i) {
|
|
2514
|
+
uchar x0 = b->qs[2*i + 0];
|
|
2515
|
+
uchar x1 = b->qs[2*i + 1];
|
|
2516
|
+
|
|
2517
|
+
q[i + 0 ] = convert_uchar(x0 & mask_0F) | convert_uchar((x1 & mask_0F) << 4);
|
|
2518
|
+
q[i + QK4_NL/4] = convert_uchar((x0 & mask_F0) >> 4) | convert_uchar(x1 & mask_F0);
|
|
2519
|
+
}
|
|
2520
|
+
}
|
|
2521
|
+
|
|
2522
|
+
kernel void kernel_restore_block_iq4_nl_noshuffle(
|
|
2523
|
+
global uchar * src_q,
|
|
2524
|
+
global half * src_d,
|
|
2525
|
+
global struct block_iq4_nl * dst,
|
|
2526
|
+
uchar mask_0F,
|
|
2527
|
+
uchar mask_F0,
|
|
2528
|
+
ulong n_blk
|
|
2529
|
+
) {
|
|
2530
|
+
if (get_global_id(0) >= n_blk) {
|
|
2531
|
+
return;
|
|
2532
|
+
}
|
|
2533
|
+
global struct block_iq4_nl * b = (global struct block_iq4_nl *) dst + get_global_id(0);
|
|
2534
|
+
global uchar * q = (global uchar *) src_q + QK4_NL/2*get_global_id(0);
|
|
2535
|
+
global half * d = (global half *) src_d + get_global_id(0);
|
|
2536
|
+
|
|
2537
|
+
b->d = *d;
|
|
2538
|
+
for (int i = 0; i < QK4_NL/4; ++i) {
|
|
2539
|
+
uchar x0 = q[i + 0 ];
|
|
2540
|
+
uchar x1 = q[i + QK4_NL/4];
|
|
2541
|
+
|
|
2542
|
+
b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));
|
|
2543
|
+
b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
|
|
2547
|
+
// ---------------------------------------------------------------------------
|
|
2548
|
+
// kernel_moe_expand_scale_q8_0
|
|
2549
|
+
//
|
|
2550
|
+
// Expand the q8_0 per-32-block scale d (one half/block, [expert][row][block]) into
|
|
2551
|
+
// the UNIFORM scale[16] format the generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a,
|
|
2552
|
+
// MOE_QT=80) consumes: 16 f16 per 256-superblock (per-16-element segment), where the
|
|
2553
|
+
// two segments of each 32-block share the block's d. q8_0 is symmetric -> no min
|
|
2554
|
+
// buffer (the GEMM runs with has_min=0). The int8 weight codes are reused verbatim
|
|
2555
|
+
// from the existing flat q8_0 weight buffer (extra0_q8_0->q), so only the scale is
|
|
2556
|
+
// rebuilt here. One work-item per (row, superblock, expert).
|
|
2557
|
+
// ---------------------------------------------------------------------------
|
|
2558
|
+
kernel void kernel_moe_expand_scale_q8_0(
|
|
2559
|
+
global const half * src_d, // [expert][row][block], one scale per 32-block
|
|
2560
|
+
global half * dst_scale, // [expert][row][block][2] (FLAT per-32-block)
|
|
2561
|
+
int ne00,
|
|
2562
|
+
int ne01
|
|
2563
|
+
) {
|
|
2564
|
+
int row = get_global_id(0);
|
|
2565
|
+
int blk = get_global_id(1); // 32-block index along K
|
|
2566
|
+
int e = get_global_id(2);
|
|
2567
|
+
if (row >= ne01) { return; }
|
|
2568
|
+
|
|
2569
|
+
long nb = ne00 / 32; // 32-blocks per row (K only needs % 32 == 0)
|
|
2570
|
+
half d = src_d[((long)e*ne01 + row)*nb + blk];
|
|
2571
|
+
long b = (((long)e*ne01 + row)*nb + blk) * 2;
|
|
2572
|
+
dst_scale[b + 0] = d;
|
|
2573
|
+
dst_scale[b + 1] = d;
|
|
2574
|
+
}
|
|
2575
|
+
|
|
2576
|
+
// ---------------------------------------------------------------------------
|
|
2577
|
+
// kernel_moe_expand_scale_q5_0
|
|
2578
|
+
//
|
|
2579
|
+
// q5_0 = symmetric, value = d*(code-16), code = nibble | (hi<<4) in 0..31. The
|
|
2580
|
+
// generic dp4a MoE GEMM keeps the unsigned code and centers via the min term:
|
|
2581
|
+
// scale*dp4a(code,a) - min*sum(a), scale = d, min = d*16.
|
|
2582
|
+
// Reads the existing q5_0 d ([expert][block][row], one half/32-block, from the
|
|
2583
|
+
// trans4 convert) and writes the FLAT per-32-block uniform scale[2]/min[1] in
|
|
2584
|
+
// [expert][row][block] order (a transpose). One work-item per (row, block, expert).
|
|
2585
|
+
// ---------------------------------------------------------------------------
|
|
2586
|
+
kernel void kernel_moe_expand_scale_q5_0(
|
|
2587
|
+
global const half * src_d, // [expert][block][row]
|
|
2588
|
+
global half * dst_scale, // [expert][row][block][2]
|
|
2589
|
+
global half * dst_min, // [expert][row][block]
|
|
2590
|
+
int ne00,
|
|
2591
|
+
int ne01
|
|
2592
|
+
) {
|
|
2593
|
+
int row = get_global_id(0);
|
|
2594
|
+
int blk = get_global_id(1);
|
|
2595
|
+
int e = get_global_id(2);
|
|
2596
|
+
if (row >= ne01) { return; }
|
|
2597
|
+
|
|
2598
|
+
long nb = ne00 / 32;
|
|
2599
|
+
half d = src_d[(long)e*nb*ne01 + (long)blk*ne01 + row]; // [expert][block][row]
|
|
2600
|
+
long sb = (((long)e*ne01 + row)*nb + blk) * 2;
|
|
2601
|
+
long mb = ((long)e*ne01 + row)*nb + blk;
|
|
2602
|
+
dst_scale[sb + 0] = d;
|
|
2603
|
+
dst_scale[sb + 1] = d;
|
|
2604
|
+
dst_min[mb] = (half)((float)d * 16.0f);
|
|
2605
|
+
}
|
|
2606
|
+
|
|
2607
|
+
// ---------------------------------------------------------------------------
|
|
2608
|
+
// kernel_moe_expand_scale_q5_K
|
|
2609
|
+
//
|
|
2610
|
+
// q5_K value = d*sv*code + (-dm*mn), with the 6-bit packed per-sub-block scale sv
|
|
2611
|
+
// and min mn (8 sub-blocks of 32 per 256-superblock, decoded by get_scale_min_k4
|
|
2612
|
+
// from the 12-byte s[]). The generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a,
|
|
2613
|
+
// MOE_QT=5) keeps the unsigned 5-bit code and applies scale/min via the uniform
|
|
2614
|
+
// per-32-block buffers:
|
|
2615
|
+
// acc += sc0*a_d*raw1 + sc1*a_d*raw2 - mn_u*a_s,
|
|
2616
|
+
// sc0 = sc1 = d*sv (both per-16 segments of a 32-block share the sub-block scale),
|
|
2617
|
+
// mn_u = dm*mn (positive; the GEMM subtracts it -> the -dm*mn min term).
|
|
2618
|
+
// q5_K's q_img (low nibbles) + qh (hi-bit plane) are already in the layout the GEMM
|
|
2619
|
+
// reads (same trans4_ns convert that feeds gemm_moe_q5_k_f32_ns), so only the scale
|
|
2620
|
+
// is rebuilt here.
|
|
2621
|
+
//
|
|
2622
|
+
// One work-item per (row, superblock, expert); each emits 8 sub-blocks.
|
|
2623
|
+
// ---------------------------------------------------------------------------
|
|
2624
|
+
kernel void kernel_moe_expand_scale_q5_K(
|
|
2625
|
+
global const uchar * src_s, // [expert][row][superblock][12]
|
|
2626
|
+
global const half * src_d, // [expert][superblock][row]
|
|
2627
|
+
global const half * src_dm, // [expert][superblock][row]
|
|
2628
|
+
global half * dst_scale, // [expert][row][32block][2]
|
|
2629
|
+
global half * dst_min, // [expert][row][32block]
|
|
2630
|
+
int ne00,
|
|
2631
|
+
int ne01
|
|
2632
|
+
) {
|
|
2633
|
+
int row = get_global_id(0);
|
|
2634
|
+
int sb = get_global_id(1); // superblock index along K
|
|
2635
|
+
int e = get_global_id(2);
|
|
2636
|
+
if (row >= ne01) { return; }
|
|
2637
|
+
|
|
2638
|
+
long nsb = ne00 / 256; // superblocks per row
|
|
2639
|
+
long nblk32 = ne00 / 32; // 32-blocks per row
|
|
2640
|
+
|
|
2641
|
+
float d = (float)src_d [((long)e*nsb + sb)*ne01 + row];
|
|
2642
|
+
float dm = (float)src_dm[((long)e*nsb + sb)*ne01 + row];
|
|
2643
|
+
|
|
2644
|
+
__global const uchar * sc = src_s + ((long)e*ne01 + row)*nsb*12 + (long)sb*12;
|
|
2645
|
+
|
|
2646
|
+
for (int j = 0; j < 8; ++j) {
|
|
2647
|
+
uchar sv, mn;
|
|
2648
|
+
// get_scale_min_k4 (6-bit packed scale/min for sub-block j of 8)
|
|
2649
|
+
if (j < 4) {
|
|
2650
|
+
sv = sc[j] & 63;
|
|
2651
|
+
mn = sc[j+4] & 63;
|
|
2652
|
+
} else {
|
|
2653
|
+
sv = (sc[j+4] & 0x0F) | ((sc[j-4] & 0xC0) >> 2);
|
|
2654
|
+
mn = ((sc[j+4] >> 4) & 0x0F) | ((sc[j] & 0xC0) >> 2);
|
|
2655
|
+
}
|
|
2656
|
+
long sub = (long)sb*8 + j;
|
|
2657
|
+
long sbase = (((long)e*ne01 + row)*nblk32 + sub) * 2;
|
|
2658
|
+
half s_val = (half)(d * (float)sv);
|
|
2659
|
+
dst_scale[sbase + 0] = s_val;
|
|
2660
|
+
dst_scale[sbase + 1] = s_val;
|
|
2661
|
+
dst_min[((long)e*ne01 + row)*nblk32 + sub] = (half)(dm * (float)mn);
|
|
2662
|
+
}
|
|
2663
|
+
}
|