elelem-ggml 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/exe/elelem-ggml +8 -0
- data/ext/elelem/ggml/elelem.c +466 -0
- data/ext/elelem/ggml/extconf.rb +52 -0
- data/lib/elelem/ggml/chat_ml.rb +23 -0
- data/lib/elelem/ggml/gguf.rb +82 -0
- data/lib/elelem/ggml/provider.rb +63 -0
- data/lib/elelem/ggml/tokenizer.rb +95 -0
- data/lib/elelem/ggml/version.rb +7 -0
- data/lib/elelem/ggml.rb +37 -0
- data/vendor/ggml/AUTHORS +335 -0
- data/vendor/ggml/CMakeLists.txt +499 -0
- data/vendor/ggml/CONTRIBUTING.md +3 -0
- data/vendor/ggml/LICENSE +21 -0
- data/vendor/ggml/README.md +49 -0
- data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
- data/vendor/ggml/cmake/GitVars.cmake +22 -0
- data/vendor/ggml/cmake/common.cmake +50 -0
- data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
- data/vendor/ggml/ggml.pc.in +10 -0
- data/vendor/ggml/include/ggml-alloc.h +86 -0
- data/vendor/ggml/include/ggml-backend.h +437 -0
- data/vendor/ggml/include/ggml-blas.h +25 -0
- data/vendor/ggml/include/ggml-cann.h +123 -0
- data/vendor/ggml/include/ggml-cpp.h +39 -0
- data/vendor/ggml/include/ggml-cpu.h +152 -0
- data/vendor/ggml/include/ggml-cuda.h +47 -0
- data/vendor/ggml/include/ggml-et.h +28 -0
- data/vendor/ggml/include/ggml-hexagon.h +19 -0
- data/vendor/ggml/include/ggml-metal.h +61 -0
- data/vendor/ggml/include/ggml-opencl.h +26 -0
- data/vendor/ggml/include/ggml-openvino.h +37 -0
- data/vendor/ggml/include/ggml-opt.h +256 -0
- data/vendor/ggml/include/ggml-rpc.h +35 -0
- data/vendor/ggml/include/ggml-sycl.h +57 -0
- data/vendor/ggml/include/ggml-virtgpu.h +14 -0
- data/vendor/ggml/include/ggml-vulkan.h +29 -0
- data/vendor/ggml/include/ggml-webgpu.h +19 -0
- data/vendor/ggml/include/ggml-zdnn.h +17 -0
- data/vendor/ggml/include/ggml-zendnn.h +22 -0
- data/vendor/ggml/include/ggml.h +2963 -0
- data/vendor/ggml/include/gguf.h +211 -0
- data/vendor/ggml/requirements.txt +12 -0
- data/vendor/ggml/scripts/gen-authors.sh +9 -0
- data/vendor/ggml/scripts/make-release-checks.sh +96 -0
- data/vendor/ggml/scripts/make-release-desc.sh +69 -0
- data/vendor/ggml/scripts/make-release-summary.txt +33 -0
- data/vendor/ggml/scripts/release.sh +195 -0
- data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
- data/vendor/ggml/scripts/sync-llama.last +1 -0
- data/vendor/ggml/scripts/sync-llama.sh +21 -0
- data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
- data/vendor/ggml/scripts/sync-whisper.last +1 -0
- data/vendor/ggml/scripts/sync-whisper.sh +17 -0
- data/vendor/ggml/src/CMakeLists.txt +634 -0
- data/vendor/ggml/src/ggml-alloc.c +1249 -0
- data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
- data/vendor/ggml/src/ggml-backend-dl.h +45 -0
- data/vendor/ggml/src/ggml-backend-impl.h +291 -0
- data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
- data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
- data/vendor/ggml/src/ggml-backend.cpp +2517 -0
- data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
- data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
- data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
- data/vendor/ggml/src/ggml-cann/common.h +651 -0
- data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
- data/vendor/ggml/src/ggml-common.h +1911 -0
- data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
- data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
- data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
- data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
- data/vendor/ggml/src/ggml-cpu/common.h +95 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
- data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
- data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
- data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
- data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
- data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
- data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
- data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
- data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
- data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
- data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
- data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
- data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
- data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
- data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
- data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
- data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
- data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
- data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
- data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
- data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
- data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
- data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
- data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
- data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
- data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
- data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
- data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
- data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
- data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
- data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
- data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
- data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
- data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
- data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
- data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
- data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
- data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
- data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
- data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
- data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
- data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
- data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
- data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
- data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
- data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
- data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
- data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
- data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
- data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
- data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
- data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
- data/vendor/ggml/src/ggml-feats.h +166 -0
- data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
- data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
- data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
- data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
- data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
- data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
- data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
- data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
- data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
- data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
- data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
- data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
- data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
- data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
- data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
- data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
- data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
- data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
- data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
- data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
- data/vendor/ggml/src/ggml-impl.h +783 -0
- data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
- data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
- data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
- data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
- data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
- data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
- data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
- data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
- data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
- data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
- data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
- data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
- data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
- data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
- data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
- data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
- data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
- data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
- data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
- data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
- data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
- data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
- data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
- data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
- data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
- data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
- data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
- data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
- data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
- data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
- data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
- data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
- data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
- data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
- data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
- data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
- data/vendor/ggml/src/ggml-opt.cpp +1094 -0
- data/vendor/ggml/src/ggml-quants.c +5667 -0
- data/vendor/ggml/src/ggml-quants.h +115 -0
- data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
- data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
- data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
- data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
- data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
- data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
- data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
- data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
- data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
- data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
- data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
- data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
- data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
- data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
- data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
- data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
- data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
- data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
- data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
- data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
- data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
- data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
- data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
- data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
- data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
- data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
- data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
- data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
- data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
- data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
- data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
- data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
- data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
- data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
- data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
- data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
- data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
- data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
- data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
- data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
- data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
- data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
- data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
- data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
- data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
- data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
- data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
- data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
- data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
- data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/vendor/ggml/src/ggml-threading.cpp +12 -0
- data/vendor/ggml/src/ggml-threading.h +14 -0
- data/vendor/ggml/src/ggml-version.h.in +4 -0
- data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
- data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
- data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
- data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
- data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
- data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
- data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
- data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
- data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
- data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
- data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
- data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
- data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
- data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
- data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
- data/vendor/ggml/src/ggml.c +8088 -0
- data/vendor/ggml/src/ggml.cpp +26 -0
- data/vendor/ggml/src/gguf.cpp +1706 -0
- metadata +1459 -0
|
@@ -0,0 +1,1880 @@
|
|
|
1
|
+
#include "ggml-et.h"
|
|
2
|
+
|
|
3
|
+
#include "ggml-backend-impl.h"
|
|
4
|
+
#include "ggml-backend.h"
|
|
5
|
+
#include "ggml-et-common.h"
|
|
6
|
+
#include "ggml-et-kernels.h"
|
|
7
|
+
#include "ggml-et-memops.h"
|
|
8
|
+
#include "ggml-et-ops.h"
|
|
9
|
+
#include "ggml-impl.h"
|
|
10
|
+
#include "ggml.h"
|
|
11
|
+
|
|
12
|
+
#include <stdarg.h>
|
|
13
|
+
|
|
14
|
+
#include <cstdarg>
|
|
15
|
+
#include <cstdio>
|
|
16
|
+
#include <cstdlib>
|
|
17
|
+
#include <cstring>
|
|
18
|
+
#include <vector>
|
|
19
|
+
|
|
20
|
+
#if __has_include(<filesystem>)
|
|
21
|
+
# include <filesystem>
|
|
22
|
+
namespace fs = std::filesystem;
|
|
23
|
+
#elif __has_include(<experimental/filesystem>)
|
|
24
|
+
# include <experimental/filesystem>
|
|
25
|
+
namespace fs = std::experimental::filesystem;
|
|
26
|
+
#else
|
|
27
|
+
# error "cannot include the filesystem library"
|
|
28
|
+
#endif
|
|
29
|
+
|
|
30
|
+
/*
|
|
31
|
+
* ggml_et_dump_tensor_metadata
|
|
32
|
+
* @brief prints the metadata of a single tensorf
|
|
33
|
+
*/
|
|
34
|
+
static void ggml_et_dump_tensor_metadata(const ggml_tensor * ggtensor, size_t indent_level, const char * title) {
|
|
35
|
+
char * spaces = (char *) alloca(indent_level + 1);
|
|
36
|
+
memset(spaces, ' ', indent_level);
|
|
37
|
+
spaces[indent_level] = '\0';
|
|
38
|
+
fprintf(stderr,
|
|
39
|
+
"%s%s: %s\n"
|
|
40
|
+
"%s type: %s\n"
|
|
41
|
+
"%s ne: %lld %lld %lld %lld\n"
|
|
42
|
+
"%s nb: %zu %zu %zu %zu\n"
|
|
43
|
+
"%s op: %s\n"
|
|
44
|
+
"%s data: %p\n"
|
|
45
|
+
"%s src0: %p\n",
|
|
46
|
+
spaces, title, ggtensor->name, spaces, ggml_type_name(ggtensor->type), spaces, (long long) ggtensor->ne[0],
|
|
47
|
+
(long long) ggtensor->ne[1], (long long) ggtensor->ne[2], (long long) ggtensor->ne[3], spaces,
|
|
48
|
+
ggtensor->nb[0], ggtensor->nb[1], ggtensor->nb[2], ggtensor->nb[3], spaces, ggml_op_name(ggtensor->op),
|
|
49
|
+
spaces, ggtensor->data, spaces, (void *) ggtensor->src[0]);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/*
|
|
53
|
+
* ggml_et_dump_operator_metadata
|
|
54
|
+
* @brief prints the metadata of a single tensor (or operator) including it's input and views
|
|
55
|
+
*/
|
|
56
|
+
static void ggml_et_dump_operator_metadata(const ggml_tensor * ggtensor) {
|
|
57
|
+
GGML_ASSERT(ggtensor != NULL);
|
|
58
|
+
ggml_et_dump_tensor_metadata(ggtensor, 0, "GGML tensor");
|
|
59
|
+
for (int i = 0; i < GGML_MAX_SRC && ggtensor->src[i]; i++) {
|
|
60
|
+
char arr[16];
|
|
61
|
+
int n = snprintf(arr, sizeof(arr), "src[%i]->name", i);
|
|
62
|
+
GGML_ASSERT((unsigned) n < sizeof(arr) && "printed too much data to stack buffer");
|
|
63
|
+
ggml_et_dump_tensor_metadata(ggtensor->src[i], 2, arr);
|
|
64
|
+
}
|
|
65
|
+
if (ggtensor->view_src) {
|
|
66
|
+
ggml_et_dump_tensor_metadata(ggtensor, 2, "view_src");
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
static struct ggml_et_driver {
|
|
71
|
+
std::shared_ptr<dev::IDeviceLayer> device_layer;
|
|
72
|
+
std::shared_ptr<rt::IRuntime> runtime;
|
|
73
|
+
std::unique_ptr<std::ofstream> profile_stream;
|
|
74
|
+
std::unique_ptr<std::ofstream> kernel_id_stream;
|
|
75
|
+
std::vector<std::pair<std::string, rt::KernelId>> kernel_map;
|
|
76
|
+
bool profiling_enabled = false;
|
|
77
|
+
} _drv;
|
|
78
|
+
|
|
79
|
+
// Check at runtime environment variables for paths likely holding ET toolchain with sysemu elf files
|
|
80
|
+
static std::string ggml_et_get_default_et_path() {
|
|
81
|
+
// List of environment variables to check in order of preference
|
|
82
|
+
const char * const env_vars[] = { "ET_TOOLCHAIN", "TOOLCHAIN_ROOT" };
|
|
83
|
+
|
|
84
|
+
for (const char * var : env_vars) {
|
|
85
|
+
if (const char * et_path = std::getenv(var)) {
|
|
86
|
+
if (et_path && *et_path != '\0') {
|
|
87
|
+
return fs::path(et_path).string();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// Otherwise assume default
|
|
93
|
+
return fs::path("/opt/et").string();
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// config when using sysemu instead of PCIe hardware device
|
|
97
|
+
// adapted from `ainekko/et-platform/esperanto-tools-libs/tools/src/bench.cpp`
|
|
98
|
+
static inline auto ggml_et_get_default_sysemu_options() {
|
|
99
|
+
constexpr uint64_t kSysEmuMaxCycles = std::numeric_limits<uint64_t>::max();
|
|
100
|
+
constexpr uint64_t kSysEmuMinionShiresMask = 0x1FFFFFFFFu;
|
|
101
|
+
const std::string et_path = ggml_et_get_default_et_path() + "/";
|
|
102
|
+
|
|
103
|
+
emu::SysEmuOptions sysEmuOptions;
|
|
104
|
+
|
|
105
|
+
// Construct all paths
|
|
106
|
+
sysEmuOptions.bootromTrampolineToBL2ElfPath =
|
|
107
|
+
et_path + "lib/esperanto-fw/BootromTrampolineToBL2/BootromTrampolineToBL2.elf";
|
|
108
|
+
sysEmuOptions.spBL2ElfPath =
|
|
109
|
+
et_path + "lib/esperanto-fw/ServiceProcessorBL2/fast-boot/ServiceProcessorBL2_fast-boot.elf";
|
|
110
|
+
sysEmuOptions.machineMinionElfPath = et_path + "lib/esperanto-fw/MachineMinion/MachineMinion.elf";
|
|
111
|
+
sysEmuOptions.masterMinionElfPath = et_path + "lib/esperanto-fw/MasterMinion/MasterMinion.elf";
|
|
112
|
+
sysEmuOptions.workerMinionElfPath = et_path + "lib/esperanto-fw/WorkerMinion/WorkerMinion.elf";
|
|
113
|
+
sysEmuOptions.executablePath = et_path + "bin/sys_emu";
|
|
114
|
+
|
|
115
|
+
// Check that each path has a valid existing non-zero file otherwise emulator just silently hangs
|
|
116
|
+
const std::vector<std::string> required_files = {
|
|
117
|
+
sysEmuOptions.bootromTrampolineToBL2ElfPath, sysEmuOptions.spBL2ElfPath,
|
|
118
|
+
sysEmuOptions.machineMinionElfPath, sysEmuOptions.masterMinionElfPath,
|
|
119
|
+
sysEmuOptions.workerMinionElfPath, sysEmuOptions.executablePath,
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
for (const auto & file : required_files) {
|
|
123
|
+
if (!fs::exists(file) || fs::file_size(file) == 0) {
|
|
124
|
+
// Check that each path has a valid existing non-zero file otherwise emulator just silently hangs
|
|
125
|
+
GGML_LOG_ERROR("ET: Unable to find required sysemu file: %s", file.c_str());
|
|
126
|
+
GGML_LOG_ERROR("ET: Confirm et-platform is correctly installed at configured path.");
|
|
127
|
+
abort();
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
sysEmuOptions.runDir = (fs::current_path().string() + "/");
|
|
132
|
+
sysEmuOptions.maxCycles = kSysEmuMaxCycles;
|
|
133
|
+
sysEmuOptions.minionShiresMask = kSysEmuMinionShiresMask;
|
|
134
|
+
sysEmuOptions.puUart0Path = sysEmuOptions.runDir + "pu_uart0_tx.log";
|
|
135
|
+
sysEmuOptions.puUart1Path = sysEmuOptions.runDir + "pu_uart1_tx.log";
|
|
136
|
+
sysEmuOptions.spUart0Path = sysEmuOptions.runDir + "spio_uart0_tx.log";
|
|
137
|
+
sysEmuOptions.spUart1Path = sysEmuOptions.runDir + "spio_uart1_tx.log";
|
|
138
|
+
sysEmuOptions.startGdb = false;
|
|
139
|
+
sysEmuOptions.memcheck = false;
|
|
140
|
+
|
|
141
|
+
return sysEmuOptions;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Forward declaration
|
|
145
|
+
static void ggml_et_driver_cleanup();
|
|
146
|
+
|
|
147
|
+
static bool ggml_et_driver_init() {
|
|
148
|
+
if (_drv.runtime != nullptr) {
|
|
149
|
+
assert(_drv.device_layer != nullptr);
|
|
150
|
+
} else {
|
|
151
|
+
try {
|
|
152
|
+
#if defined GGML_ET_SYSEMU && GGML_ET_SYSEMU
|
|
153
|
+
// For emulator device using sysEmuOptions provided by function above enabled compiling with `-DGGML_ET_SYSEMU=ON`
|
|
154
|
+
_drv.device_layer = dev::IDeviceLayer::createSysEmuDeviceLayer(ggml_et_get_default_sysemu_options());
|
|
155
|
+
#else
|
|
156
|
+
// For physical PCIe device
|
|
157
|
+
_drv.device_layer = dev::IDeviceLayer::createPcieDeviceLayer();
|
|
158
|
+
#endif // GGML_ET_SYSEMU
|
|
159
|
+
|
|
160
|
+
_drv.runtime = rt::IRuntime::create(_drv.device_layer);
|
|
161
|
+
|
|
162
|
+
// Initialize profiler if requested via environment variable
|
|
163
|
+
const char * profile_path = getenv("GGML_ET_PROFILE");
|
|
164
|
+
if (profile_path) {
|
|
165
|
+
std::string output_path = std::string(profile_path) + "/et_runtime_trace.json";
|
|
166
|
+
std::string kernel_id_path = std::string(profile_path) + "/kernel_id.json";
|
|
167
|
+
|
|
168
|
+
_drv.profile_stream = std::make_unique<std::ofstream>(output_path);
|
|
169
|
+
_drv.kernel_id_stream = std::make_unique<std::ofstream>(kernel_id_path);
|
|
170
|
+
if (!_drv.profile_stream->is_open()) {
|
|
171
|
+
GGML_LOG_ERROR("ET: Failed to open profiling output file: %s", output_path.c_str());
|
|
172
|
+
abort();
|
|
173
|
+
}
|
|
174
|
+
if (!_drv.kernel_id_stream->is_open()) {
|
|
175
|
+
GGML_LOG_ERROR("ET: Failed to open profiling kernel map: %s", kernel_id_path.c_str());
|
|
176
|
+
abort();
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
auto * profiler = _drv.runtime->getProfiler();
|
|
180
|
+
profiler->start(*_drv.profile_stream, rt::IProfiler::OutputType::Json);
|
|
181
|
+
_drv.profiling_enabled = true;
|
|
182
|
+
GGML_LOG_INFO("ET: Runtime profiler started (JSON format)");
|
|
183
|
+
|
|
184
|
+
// Register cleanup at program exit
|
|
185
|
+
std::atexit(ggml_et_driver_cleanup);
|
|
186
|
+
}
|
|
187
|
+
} catch (const std::exception & e) {
|
|
188
|
+
GGML_LOG_ERROR("ggml_et: %s", e.what());
|
|
189
|
+
if (_drv.device_layer != nullptr) {
|
|
190
|
+
_drv.device_layer.reset();
|
|
191
|
+
}
|
|
192
|
+
if (_drv.runtime != nullptr) {
|
|
193
|
+
_drv.runtime.reset();
|
|
194
|
+
}
|
|
195
|
+
return false;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
return true;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
static std::shared_ptr<dev::IDeviceLayer> ggml_et_devicelayer() {
|
|
202
|
+
return _drv.device_layer;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
std::shared_ptr<rt::IRuntime> ggml_et_runtime() {
|
|
206
|
+
return _drv.runtime;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
static void ggml_et_driver_cleanup() {
|
|
210
|
+
if (_drv.profiling_enabled && _drv.runtime) {
|
|
211
|
+
GGML_LOG_INFO("ET: Stopping runtime profiler");
|
|
212
|
+
auto * profiler = _drv.runtime->getProfiler();
|
|
213
|
+
profiler->stop();
|
|
214
|
+
_drv.profiling_enabled = false;
|
|
215
|
+
|
|
216
|
+
if (_drv.profile_stream) {
|
|
217
|
+
_drv.profile_stream->close();
|
|
218
|
+
_drv.profile_stream.reset();
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// Save kernel map
|
|
222
|
+
if (_drv.kernel_id_stream && !_drv.kernel_map.empty()) {
|
|
223
|
+
auto & os = *_drv.kernel_id_stream;
|
|
224
|
+
// XXX: Manual JSON construction. Not pretty but removes dependency
|
|
225
|
+
os << "{\n";
|
|
226
|
+
for (size_t i = 0; i < _drv.kernel_map.size(); i++) {
|
|
227
|
+
os << " \"" << _drv.kernel_map[i].first << "\": " << (int) _drv.kernel_map[i].second;
|
|
228
|
+
if (i + 1 < _drv.kernel_map.size()) {
|
|
229
|
+
os << ",";
|
|
230
|
+
}
|
|
231
|
+
os << "\n";
|
|
232
|
+
}
|
|
233
|
+
os << "}\n";
|
|
234
|
+
_drv.kernel_id_stream->close();
|
|
235
|
+
_drv.kernel_id_stream.reset();
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
static ggml_backend_dev_t ggml_backend_et_reg_get_device(ggml_backend_reg_t reg, size_t devidx);
|
|
241
|
+
|
|
242
|
+
static void ggml_backend_et_buffer_free_buffer(ggml_backend_buffer_t buffer) {
|
|
243
|
+
ggml_backend_et_buffer_context * ctx = (ggml_backend_et_buffer_context *) buffer->context;
|
|
244
|
+
if (ctx->data != nullptr) {
|
|
245
|
+
std::shared_ptr<rt::IRuntime> runtime = ggml_et_runtime();
|
|
246
|
+
if (runtime) {
|
|
247
|
+
runtime->freeDevice(ctx->rtid, static_cast<std::byte *>(ctx->data));
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
delete ctx;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
static void * ggml_backend_et_buffer_get_base(ggml_backend_buffer_t buffer) {
|
|
254
|
+
ggml_backend_et_buffer_context * ctx = (ggml_backend_et_buffer_context *) buffer->context;
|
|
255
|
+
return ctx->data;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
static ggml_status ggml_backend_et_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
|
|
259
|
+
// View tensors share buffer with their view_src, no additional initialization needed
|
|
260
|
+
if (tensor->view_src != NULL) {
|
|
261
|
+
return GGML_STATUS_SUCCESS;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
const size_t original_size = ggml_nbytes(tensor);
|
|
265
|
+
const size_t padded_size = ggml_backend_buft_get_alloc_size(buffer->buft, tensor);
|
|
266
|
+
|
|
267
|
+
// Clear padding bytes to avoid NaN values
|
|
268
|
+
// XXX: Martin - do we need this?
|
|
269
|
+
if (padded_size > original_size) {
|
|
270
|
+
const size_t padding_size = padded_size - original_size;
|
|
271
|
+
|
|
272
|
+
// Get device context to access memops kernel
|
|
273
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buffer->buft->device->context;
|
|
274
|
+
if (!dev_ctx) {
|
|
275
|
+
GGML_LOG_ERROR("ET: Failed to get device context for padding clear");
|
|
276
|
+
return GGML_STATUS_FAILED;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// Use device-side memset kernel for efficient padding clear
|
|
280
|
+
std::byte * padding_ptr = static_cast<std::byte *>(tensor->data) + original_size;
|
|
281
|
+
if (!ggml_et_memset(dev_ctx, padding_ptr, 0, padding_size)) {
|
|
282
|
+
GGML_LOG_ERROR("ET: Failed to clear padding using memset kernel for tensor %s", tensor->name);
|
|
283
|
+
return GGML_STATUS_FAILED;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
return GGML_STATUS_SUCCESS;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
static void ggml_backend_et_buffer_set_tensor(ggml_backend_buffer_t buffer,
|
|
291
|
+
ggml_tensor * tensor,
|
|
292
|
+
const void * data,
|
|
293
|
+
size_t offset,
|
|
294
|
+
size_t size) {
|
|
295
|
+
std::shared_ptr<rt::IRuntime> runtime = ggml_et_runtime();
|
|
296
|
+
if (!runtime) {
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Create short-lived stream for this transfer
|
|
301
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buffer->buft->device->context;
|
|
302
|
+
rt::StreamId stream = dev_ctx->default_stream;
|
|
303
|
+
|
|
304
|
+
std::byte * dst_ptr = static_cast<std::byte *>(tensor->data) + offset;
|
|
305
|
+
const std::byte * src_ptr = static_cast<const std::byte *>(data);
|
|
306
|
+
|
|
307
|
+
rt::EventId event = runtime->memcpyHostToDevice(stream, src_ptr, dst_ptr, size, true /*barrier*/);
|
|
308
|
+
|
|
309
|
+
runtime->waitForEvent(event);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
static void ggml_backend_et_buffer_get_tensor(ggml_backend_buffer_t buffer,
|
|
313
|
+
const ggml_tensor * tensor,
|
|
314
|
+
void * data,
|
|
315
|
+
size_t offset,
|
|
316
|
+
size_t size) {
|
|
317
|
+
std::shared_ptr<rt::IRuntime> runtime = ggml_et_runtime();
|
|
318
|
+
if (!runtime) {
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buffer->buft->device->context;
|
|
323
|
+
rt::StreamId stream = dev_ctx->default_stream;
|
|
324
|
+
|
|
325
|
+
const std::byte * src_ptr = static_cast<const std::byte *>(tensor->data) + offset;
|
|
326
|
+
std::byte * dst_ptr = static_cast<std::byte *>(data);
|
|
327
|
+
|
|
328
|
+
rt::EventId event = runtime->memcpyDeviceToHost(stream, src_ptr, dst_ptr, size, true /*barrier*/);
|
|
329
|
+
|
|
330
|
+
runtime->waitForEvent(event);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
static bool ggml_backend_et_buffer_cpy_tensor(ggml_backend_buffer_t buffer,
|
|
334
|
+
const ggml_tensor * src,
|
|
335
|
+
ggml_tensor * dst) {
|
|
336
|
+
GGML_UNUSED(buffer);
|
|
337
|
+
GGML_UNUSED(src);
|
|
338
|
+
GGML_UNUSED(dst);
|
|
339
|
+
return false;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
static void ggml_backend_et_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
|
|
343
|
+
ggml_backend_et_buffer_context * ctx = (ggml_backend_et_buffer_context *) buffer->context;
|
|
344
|
+
|
|
345
|
+
if (ctx->size == 0 || ctx->data == nullptr) {
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// Get device context to access memops kernel
|
|
350
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buffer->buft->device->context;
|
|
351
|
+
if (!dev_ctx) {
|
|
352
|
+
GGML_LOG_ERROR("ET: Failed to get device context for buffer clear");
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Use device-side memset kernel for efficient clearing
|
|
357
|
+
if (!ggml_et_memset(dev_ctx, ctx->data, value, ctx->size)) {
|
|
358
|
+
GGML_LOG_ERROR("ET: buffer_clear failed using memset kernel");
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
GGML_LOG_DEBUG("ET: Buffer cleared successfully using memops kernel");
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
static const struct ggml_backend_buffer_i ggml_backend_et_buffer_i = {
|
|
366
|
+
/* .free_buffer = */ ggml_backend_et_buffer_free_buffer,
|
|
367
|
+
/* .get_base = */ ggml_backend_et_buffer_get_base,
|
|
368
|
+
/* .init_tensor = */ ggml_backend_et_buffer_init_tensor,
|
|
369
|
+
/* .memset_tensor = */ NULL,
|
|
370
|
+
/* .set_tensor = */ ggml_backend_et_buffer_set_tensor,
|
|
371
|
+
/* .get_tensor = */ ggml_backend_et_buffer_get_tensor,
|
|
372
|
+
/* .set_tensor_2d = */ NULL,
|
|
373
|
+
/* .get_tensor_2d = */ NULL,
|
|
374
|
+
/* .cpy_tensor = */ ggml_backend_et_buffer_cpy_tensor,
|
|
375
|
+
/* .clear = */ ggml_backend_et_buffer_clear,
|
|
376
|
+
/* .reset = */ NULL,
|
|
377
|
+
};
|
|
378
|
+
|
|
379
|
+
static const char * ggml_backend_et_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
|
|
380
|
+
GGML_UNUSED(buft);
|
|
381
|
+
return GGML_ET_NAME;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
static ggml_backend_buffer_t ggml_backend_et_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
|
|
385
|
+
ggml_backend_et_buffer_type_context * btctx = (ggml_backend_et_buffer_type_context *) buft->context;
|
|
386
|
+
|
|
387
|
+
ggml_backend_et_buffer_context * ctx = new ggml_backend_et_buffer_context;
|
|
388
|
+
ctx->devidx = btctx->devidx;
|
|
389
|
+
ctx->size = size;
|
|
390
|
+
|
|
391
|
+
std::shared_ptr<rt::IRuntime> runtime = ggml_et_runtime();
|
|
392
|
+
if (!runtime) {
|
|
393
|
+
delete ctx;
|
|
394
|
+
return nullptr;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
std::vector<rt::DeviceId> rtids = runtime->getDevices();
|
|
398
|
+
if (static_cast<size_t>(btctx->devidx) >= rtids.size()) {
|
|
399
|
+
delete ctx;
|
|
400
|
+
return nullptr;
|
|
401
|
+
}
|
|
402
|
+
ctx->rtid = rtids[btctx->devidx];
|
|
403
|
+
|
|
404
|
+
ctx->data = runtime->mallocDevice(ctx->rtid, size);
|
|
405
|
+
if (ctx->data == nullptr) {
|
|
406
|
+
delete ctx;
|
|
407
|
+
return nullptr;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
return ggml_backend_buffer_init(buft, ggml_backend_et_buffer_i, ctx, size);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
static size_t ggml_backend_et_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
|
|
414
|
+
std::shared_ptr<rt::IRuntime> runtime = ggml_et_runtime();
|
|
415
|
+
if (!runtime || !buft->device) {
|
|
416
|
+
return GGML_MEM_ALIGN;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buft->device->context;
|
|
420
|
+
rt::DeviceProperties prop = runtime->getDeviceProperties(dev_ctx->rtid);
|
|
421
|
+
return prop.cacheLineSize_;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
static size_t ggml_backend_et_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
|
|
425
|
+
if (buft->device) {
|
|
426
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) buft->device->context;
|
|
427
|
+
return dev_ctx->total_mem;
|
|
428
|
+
}
|
|
429
|
+
return SIZE_MAX;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
static size_t ggml_backend_et_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
|
|
433
|
+
GGML_UNUSED(buft);
|
|
434
|
+
return ggml_nbytes_pad(tensor);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
static bool ggml_backend_et_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
|
|
438
|
+
GGML_UNUSED(buft);
|
|
439
|
+
return false;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
static const struct ggml_backend_buffer_type_i ggml_backend_et_buffer_type_i = {
|
|
443
|
+
/* .get_name = */ ggml_backend_et_buffer_type_get_name,
|
|
444
|
+
/* .alloc_buffer = */ ggml_backend_et_buffer_type_alloc_buffer,
|
|
445
|
+
/* .get_alignment = */ ggml_backend_et_buffer_type_get_alignment,
|
|
446
|
+
/* .get_max_size = */ ggml_backend_et_buffer_type_get_max_size,
|
|
447
|
+
/* .get_alloc_size = */ ggml_backend_et_buffer_type_get_alloc_size,
|
|
448
|
+
/* .is_host = */ ggml_backend_et_buffer_type_is_host,
|
|
449
|
+
};
|
|
450
|
+
|
|
451
|
+
static const char * ggml_backend_et_get_name(ggml_backend_t backend) {
|
|
452
|
+
GGML_UNUSED(backend);
|
|
453
|
+
return GGML_ET_NAME;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
static void ggml_backend_et_free(ggml_backend_t backend) {
|
|
457
|
+
ggml_backend_et_context * et_ctx = (ggml_backend_et_context *) backend->context;
|
|
458
|
+
std::shared_ptr<rt::IRuntime> runtime = ggml_et_runtime();
|
|
459
|
+
|
|
460
|
+
// Clean up kernels on this device before freeing backend
|
|
461
|
+
ggml_backend_dev_t dev = ggml_backend_et_reg_get_device(ggml_backend_et_reg(), et_ctx->devidx);
|
|
462
|
+
if (dev && dev->context) {
|
|
463
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context;
|
|
464
|
+
|
|
465
|
+
if (_drv.profiling_enabled) {
|
|
466
|
+
auto kernels = ggml_et_get_loaded_kernels(dev_ctx);
|
|
467
|
+
_drv.kernel_map.insert(_drv.kernel_map.end(), kernels.begin(), kernels.end());
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
ggml_et_unload_all_kernels(dev_ctx);
|
|
471
|
+
|
|
472
|
+
if (runtime) {
|
|
473
|
+
if (dev_ctx->trace_buffer) {
|
|
474
|
+
runtime->freeDevice(dev_ctx->rtid, dev_ctx->trace_buffer);
|
|
475
|
+
dev_ctx->trace_buffer = nullptr;
|
|
476
|
+
}
|
|
477
|
+
// Drain any in-flight uberkernel launches before freeing the
|
|
478
|
+
// device buffers they read from.
|
|
479
|
+
runtime->waitForStream(dev_ctx->default_stream);
|
|
480
|
+
for (auto & slot : dev_ctx->uberkernel.slots) {
|
|
481
|
+
if (slot.device_insts) {
|
|
482
|
+
runtime->freeDevice(dev_ctx->rtid, slot.device_insts);
|
|
483
|
+
slot.device_insts = nullptr;
|
|
484
|
+
}
|
|
485
|
+
if (slot.device_params) {
|
|
486
|
+
runtime->freeDevice(dev_ctx->rtid, slot.device_params);
|
|
487
|
+
slot.device_params = nullptr;
|
|
488
|
+
}
|
|
489
|
+
slot.has_pending = false;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
delete et_ctx;
|
|
495
|
+
delete backend;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
static ggml_backend_buffer_type_t ggml_backend_et_get_default_buffer_type(ggml_backend_t backend) {
|
|
499
|
+
ggml_backend_et_context * et_ctx = (ggml_backend_et_context *) backend->context;
|
|
500
|
+
|
|
501
|
+
return ggml_backend_et_buffer_type(et_ctx->devidx);
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
static void ggml_backend_et_set_tensor_async(ggml_backend_t backend,
|
|
505
|
+
ggml_tensor * tensor,
|
|
506
|
+
const void * data,
|
|
507
|
+
size_t offset,
|
|
508
|
+
size_t size) {
|
|
509
|
+
std::shared_ptr<rt::IRuntime> runtime = ggml_et_runtime();
|
|
510
|
+
if (!runtime) {
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) backend->device->context;
|
|
515
|
+
rt::StreamId stream = dev_ctx->default_stream;
|
|
516
|
+
|
|
517
|
+
std::byte * dst_ptr = static_cast<std::byte *>(tensor->data) + offset;
|
|
518
|
+
const std::byte * src_ptr = static_cast<const std::byte *>(data);
|
|
519
|
+
|
|
520
|
+
runtime->memcpyHostToDevice(stream, src_ptr, dst_ptr, size, true /*barrier*/);
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
static void ggml_backend_et_get_tensor_async(ggml_backend_t backend,
|
|
524
|
+
const ggml_tensor * tensor,
|
|
525
|
+
void * data,
|
|
526
|
+
size_t offset,
|
|
527
|
+
size_t size) {
|
|
528
|
+
std::shared_ptr<rt::IRuntime> runtime = ggml_et_runtime();
|
|
529
|
+
if (!runtime) {
|
|
530
|
+
return;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) backend->device->context;
|
|
534
|
+
rt::StreamId stream = dev_ctx->default_stream;
|
|
535
|
+
|
|
536
|
+
const std::byte * src_ptr = static_cast<const std::byte *>(tensor->data) + offset;
|
|
537
|
+
std::byte * dst_ptr = static_cast<std::byte *>(data);
|
|
538
|
+
|
|
539
|
+
runtime->memcpyDeviceToHost(stream, src_ptr, dst_ptr, size, true /*barrier*/);
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
static bool ggml_backend_et_cpy_tensor_async(ggml_backend_t backend_src,
|
|
543
|
+
ggml_backend_t backend_dst,
|
|
544
|
+
const ggml_tensor * src,
|
|
545
|
+
ggml_tensor * dst) {
|
|
546
|
+
GGML_UNUSED(backend_src);
|
|
547
|
+
GGML_UNUSED(backend_dst);
|
|
548
|
+
GGML_UNUSED(src);
|
|
549
|
+
GGML_UNUSED(dst);
|
|
550
|
+
return false;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
static void ggml_backend_et_synchronize(ggml_backend_t backend) {
|
|
554
|
+
std::shared_ptr<rt::IRuntime> runtime = ggml_et_runtime();
|
|
555
|
+
if (!runtime) {
|
|
556
|
+
return;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) backend->device->context;
|
|
560
|
+
runtime->waitForStream(dev_ctx->default_stream);
|
|
561
|
+
|
|
562
|
+
auto errors = runtime->retrieveStreamErrors(dev_ctx->default_stream);
|
|
563
|
+
if (errors.empty()) {
|
|
564
|
+
return;
|
|
565
|
+
}
|
|
566
|
+
for (const auto & err : errors) {
|
|
567
|
+
GGML_LOG_ERROR("ET: stream error detected at synchronization point. Code: %d,Type: %d\n", (int) err.errorCode_,
|
|
568
|
+
(int) err.errorContext_.value()[0].type_);
|
|
569
|
+
}
|
|
570
|
+
abort();
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
static bool ggml_et_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializer_list<ggml_op> ops) {
|
|
574
|
+
if (!ggml_can_fuse(cgraph, node_idx, ops)) {
|
|
575
|
+
return false;
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
if (ops.size() == 2 && ops.begin()[0] == GGML_OP_MUL_MAT && ops.begin()[1] == GGML_OP_ADD) {
|
|
579
|
+
const ggml_tensor * mm = cgraph->nodes[node_idx];
|
|
580
|
+
const ggml_tensor * add = cgraph->nodes[node_idx + 1];
|
|
581
|
+
|
|
582
|
+
// Only Q8_0 weights x F32 activations -> F32 (the kernel that has
|
|
583
|
+
// the bias path). Other MM variants must wait for their own kernel
|
|
584
|
+
// bias support.
|
|
585
|
+
if (mm->type != GGML_TYPE_F32 || mm->src[0]->type != GGML_TYPE_Q8_0 || mm->src[1]->type != GGML_TYPE_F32) {
|
|
586
|
+
return false;
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
// ADD must be F32 and one of its operands must be the MM output.
|
|
590
|
+
if (add->type != GGML_TYPE_F32) {
|
|
591
|
+
return false;
|
|
592
|
+
}
|
|
593
|
+
if (add->src[0] != mm && add->src[1] != mm) {
|
|
594
|
+
return false;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
const ggml_tensor * bias = (add->src[0] == mm) ? add->src[1] : add->src[0];
|
|
598
|
+
|
|
599
|
+
if (bias->type != GGML_TYPE_F32) {
|
|
600
|
+
return false;
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
// No broadcasting: bias shape must equal MM output shape.
|
|
604
|
+
for (int i = 0; i < GGML_MAX_DIMS; ++i) {
|
|
605
|
+
if (bias->ne[i] != mm->ne[i]) {
|
|
606
|
+
return false;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
// Bias and dst must be contiguous and have identical strides - the
|
|
611
|
+
// kernel uses dst-style offset arithmetic against bias's nb[].
|
|
612
|
+
if (!ggml_is_contiguous(bias) || !ggml_is_contiguous(mm)) {
|
|
613
|
+
return false;
|
|
614
|
+
}
|
|
615
|
+
for (int i = 0; i < GGML_MAX_DIMS; ++i) {
|
|
616
|
+
if ((int64_t) bias->nb[i] != (int64_t) add->nb[i]) {
|
|
617
|
+
return false;
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) {
|
|
623
|
+
const ggml_tensor * rms_norm = cgraph->nodes[node_idx];
|
|
624
|
+
const ggml_tensor * mul = cgraph->nodes[node_idx + 1];
|
|
625
|
+
|
|
626
|
+
// ET only supports F32
|
|
627
|
+
if (rms_norm->src[0]->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) {
|
|
628
|
+
return false;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
// Identify the weights tensor (the MUL operand that isn't rms_norm output)
|
|
632
|
+
const ggml_tensor * weights = (mul->src[0] == rms_norm) ? mul->src[1] : mul->src[0];
|
|
633
|
+
|
|
634
|
+
if (weights->type != GGML_TYPE_F32) {
|
|
635
|
+
return false;
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// Both inputs must be contiguous (ET hardware requirement)
|
|
639
|
+
if (!ggml_is_contiguous(rms_norm->src[0]) || !ggml_is_contiguous_rows(weights)) {
|
|
640
|
+
return false;
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
// ET requires cache-aligned rows (ne[0] % 16 == 0)
|
|
644
|
+
if (rms_norm->src[0]->ne[0] % 16 != 0 || weights->ne[0] % 16 != 0) {
|
|
645
|
+
return false;
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
// Fused kernel doesn't handle dim-0 broadcasting
|
|
649
|
+
if (weights->ne[0] != rms_norm->src[0]->ne[0]) {
|
|
650
|
+
return false;
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
return true;
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
static ggml_status ggml_backend_et_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {
|
|
658
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) backend->device->context;
|
|
659
|
+
ggml_et_uberkernel_begin_graph(&dev_ctx->uberkernel);
|
|
660
|
+
|
|
661
|
+
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
662
|
+
ggml_tensor * node = cgraph->nodes[i];
|
|
663
|
+
|
|
664
|
+
if (node->op == GGML_OP_NONE || node->op == GGML_OP_VIEW || node->op == GGML_OP_RESHAPE ||
|
|
665
|
+
node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE) {
|
|
666
|
+
continue;
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
// --- Fusion checks (before regular dispatch) ---
|
|
670
|
+
if (ggml_et_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {
|
|
671
|
+
ggml_et_op_rms_norm_mul(dev_ctx, node, cgraph->nodes[i + 1]);
|
|
672
|
+
i++; // skip the MUL node
|
|
673
|
+
continue;
|
|
674
|
+
}
|
|
675
|
+
if (ggml_et_can_fuse(cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_ADD })) {
|
|
676
|
+
ggml_et_op_mul_mat(dev_ctx, node, cgraph->nodes[i + 1]);
|
|
677
|
+
i++; // skip the ADD node
|
|
678
|
+
continue;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
switch (node->op) {
|
|
682
|
+
case GGML_OP_SQR:
|
|
683
|
+
ggml_et_op_sqr(dev_ctx, node);
|
|
684
|
+
break;
|
|
685
|
+
|
|
686
|
+
case GGML_OP_UNARY:
|
|
687
|
+
ggml_et_op_unary(dev_ctx, node);
|
|
688
|
+
break;
|
|
689
|
+
|
|
690
|
+
case GGML_OP_SUM_ROWS:
|
|
691
|
+
ggml_et_op_sum_rows(dev_ctx, node);
|
|
692
|
+
break;
|
|
693
|
+
|
|
694
|
+
case GGML_OP_MEAN:
|
|
695
|
+
ggml_et_op_mean(dev_ctx, node);
|
|
696
|
+
break;
|
|
697
|
+
|
|
698
|
+
case GGML_OP_CLAMP:
|
|
699
|
+
ggml_et_op_clamp(dev_ctx, node);
|
|
700
|
+
break;
|
|
701
|
+
|
|
702
|
+
case GGML_OP_MUL:
|
|
703
|
+
ggml_et_op_mul(dev_ctx, node);
|
|
704
|
+
break;
|
|
705
|
+
|
|
706
|
+
case GGML_OP_ADD:
|
|
707
|
+
ggml_et_op_add(dev_ctx, node);
|
|
708
|
+
break;
|
|
709
|
+
|
|
710
|
+
case GGML_OP_SUB:
|
|
711
|
+
ggml_et_op_sub(dev_ctx, node);
|
|
712
|
+
break;
|
|
713
|
+
|
|
714
|
+
case GGML_OP_CUMSUM:
|
|
715
|
+
ggml_et_op_cumsum(dev_ctx, node);
|
|
716
|
+
break;
|
|
717
|
+
|
|
718
|
+
case GGML_OP_MUL_MAT:
|
|
719
|
+
ggml_et_op_mul_mat(dev_ctx, node);
|
|
720
|
+
break;
|
|
721
|
+
|
|
722
|
+
case GGML_OP_MUL_MAT_ID:
|
|
723
|
+
ggml_et_op_mul_mat_id(dev_ctx, node);
|
|
724
|
+
break;
|
|
725
|
+
|
|
726
|
+
case GGML_OP_ROPE:
|
|
727
|
+
ggml_et_op_rope(dev_ctx, node);
|
|
728
|
+
break;
|
|
729
|
+
|
|
730
|
+
case GGML_OP_RMS_NORM:
|
|
731
|
+
ggml_et_op_rms_norm(dev_ctx, node);
|
|
732
|
+
break;
|
|
733
|
+
|
|
734
|
+
case GGML_OP_NORM:
|
|
735
|
+
ggml_et_op_norm(dev_ctx, node);
|
|
736
|
+
break;
|
|
737
|
+
|
|
738
|
+
case GGML_OP_L2_NORM:
|
|
739
|
+
ggml_et_op_l2_norm(dev_ctx, node);
|
|
740
|
+
break;
|
|
741
|
+
|
|
742
|
+
case GGML_OP_GROUP_NORM:
|
|
743
|
+
ggml_et_op_group_norm(dev_ctx, node);
|
|
744
|
+
break;
|
|
745
|
+
|
|
746
|
+
case GGML_OP_SCALE:
|
|
747
|
+
ggml_et_op_scale(dev_ctx, node);
|
|
748
|
+
break;
|
|
749
|
+
|
|
750
|
+
case GGML_OP_GLU:
|
|
751
|
+
ggml_et_op_glu(dev_ctx, node);
|
|
752
|
+
break;
|
|
753
|
+
|
|
754
|
+
case GGML_OP_SOFT_MAX:
|
|
755
|
+
ggml_et_op_softmax(dev_ctx, node);
|
|
756
|
+
break;
|
|
757
|
+
|
|
758
|
+
case GGML_OP_IM2COL:
|
|
759
|
+
ggml_et_op_im2col(dev_ctx, node);
|
|
760
|
+
break;
|
|
761
|
+
|
|
762
|
+
case GGML_OP_CONV_2D:
|
|
763
|
+
ggml_et_op_conv_2d(dev_ctx, node);
|
|
764
|
+
break;
|
|
765
|
+
|
|
766
|
+
case GGML_OP_FLASH_ATTN_EXT:
|
|
767
|
+
ggml_et_op_flash_attn_ext(dev_ctx, node);
|
|
768
|
+
break;
|
|
769
|
+
|
|
770
|
+
case GGML_OP_GET_ROWS:
|
|
771
|
+
ggml_et_op_get_rows(dev_ctx, node);
|
|
772
|
+
break;
|
|
773
|
+
|
|
774
|
+
case GGML_OP_CONT:
|
|
775
|
+
ggml_et_op_cont(dev_ctx, node);
|
|
776
|
+
break;
|
|
777
|
+
|
|
778
|
+
case GGML_OP_CPY:
|
|
779
|
+
ggml_et_op_cpy(dev_ctx, node);
|
|
780
|
+
break;
|
|
781
|
+
|
|
782
|
+
case GGML_OP_CONCAT:
|
|
783
|
+
ggml_et_op_concat(dev_ctx, node);
|
|
784
|
+
break;
|
|
785
|
+
|
|
786
|
+
case GGML_OP_REPEAT:
|
|
787
|
+
ggml_et_op_repeat(dev_ctx, node);
|
|
788
|
+
break;
|
|
789
|
+
|
|
790
|
+
case GGML_OP_SSM_CONV:
|
|
791
|
+
ggml_et_op_ssm_conv(dev_ctx, node);
|
|
792
|
+
break;
|
|
793
|
+
|
|
794
|
+
case GGML_OP_SSM_SCAN:
|
|
795
|
+
ggml_et_op_ssm_scan(dev_ctx, node);
|
|
796
|
+
break;
|
|
797
|
+
|
|
798
|
+
case GGML_OP_PAD:
|
|
799
|
+
ggml_et_op_pad(dev_ctx, node);
|
|
800
|
+
break;
|
|
801
|
+
|
|
802
|
+
case GGML_OP_SET_ROWS:
|
|
803
|
+
ggml_et_op_set_rows(dev_ctx, node);
|
|
804
|
+
break;
|
|
805
|
+
|
|
806
|
+
case GGML_OP_FILL:
|
|
807
|
+
ggml_et_op_fill(dev_ctx, node);
|
|
808
|
+
break;
|
|
809
|
+
|
|
810
|
+
case GGML_OP_DIAG:
|
|
811
|
+
ggml_et_op_diag(dev_ctx, node);
|
|
812
|
+
break;
|
|
813
|
+
|
|
814
|
+
case GGML_OP_TRI:
|
|
815
|
+
ggml_et_op_tri(dev_ctx, node);
|
|
816
|
+
break;
|
|
817
|
+
|
|
818
|
+
case GGML_OP_SOLVE_TRI:
|
|
819
|
+
ggml_et_op_solve_tri(dev_ctx, node);
|
|
820
|
+
break;
|
|
821
|
+
|
|
822
|
+
case GGML_OP_SET:
|
|
823
|
+
ggml_et_op_set(dev_ctx, node);
|
|
824
|
+
break;
|
|
825
|
+
|
|
826
|
+
case GGML_OP_RWKV_WKV6:
|
|
827
|
+
ggml_et_op_rwkv_wkv6(dev_ctx, node);
|
|
828
|
+
break;
|
|
829
|
+
|
|
830
|
+
case GGML_OP_RWKV_WKV7:
|
|
831
|
+
ggml_et_op_rwkv_wkv7(dev_ctx, node);
|
|
832
|
+
break;
|
|
833
|
+
|
|
834
|
+
case GGML_OP_GATED_DELTA_NET:
|
|
835
|
+
ggml_et_op_gated_delta_net(dev_ctx, node);
|
|
836
|
+
break;
|
|
837
|
+
|
|
838
|
+
default:
|
|
839
|
+
ggml_et_uberkernel_abort_graph(&dev_ctx->uberkernel);
|
|
840
|
+
GGML_LOG_ERROR("ET: Unsupported operation in graph: %s", ggml_op_name(node->op));
|
|
841
|
+
return GGML_STATUS_FAILED;
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
if (ggml_et_uberkernel_failed(&dev_ctx->uberkernel)) {
|
|
845
|
+
ggml_et_uberkernel_abort_graph(&dev_ctx->uberkernel);
|
|
846
|
+
return GGML_STATUS_FAILED;
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
if (!ggml_et_uberkernel_end_graph(dev_ctx)) {
|
|
851
|
+
ggml_et_uberkernel_abort_graph(&dev_ctx->uberkernel);
|
|
852
|
+
return GGML_STATUS_FAILED;
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
return GGML_STATUS_SUCCESS;
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
// Check that elements within each row are contiguous (nb[0] == type_size).
|
|
859
|
+
// Higher-dim strides can be arbitrary - kernels navigate them via byte offsets.
|
|
860
|
+
static bool et_ggml_is_row_contiguous(const ggml_tensor * t) {
|
|
861
|
+
return t->nb[0] == ggml_type_size(t->type);
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
|
|
865
|
+
GGML_UNUSED(dev);
|
|
866
|
+
|
|
867
|
+
bool supported = false;
|
|
868
|
+
switch (op->op) {
|
|
869
|
+
case GGML_OP_CUMSUM:
|
|
870
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
871
|
+
op->src[0]->nb[0] == sizeof(float) && ggml_is_contiguous(op);
|
|
872
|
+
break;
|
|
873
|
+
case GGML_OP_SQR:
|
|
874
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
875
|
+
op->ne[0] % 16 == 0 && ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]);
|
|
876
|
+
break;
|
|
877
|
+
case GGML_OP_SUM_ROWS:
|
|
878
|
+
// dst has ne[0]=1, src0 row length must be cache-aligned
|
|
879
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
880
|
+
op->src[0]->ne[0] % 16 == 0 && ggml_is_contiguous(op->src[0]);
|
|
881
|
+
break;
|
|
882
|
+
case GGML_OP_MEAN:
|
|
883
|
+
// Kernel handles arbitrary ne00 (per-row alignment guard with
|
|
884
|
+
// scalar tail), so no row-length divisibility constraint here.
|
|
885
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
886
|
+
ggml_is_contiguous(op->src[0]);
|
|
887
|
+
break;
|
|
888
|
+
case GGML_OP_CLAMP:
|
|
889
|
+
// Element-wise; kernel distributes by cache lines and handles a
|
|
890
|
+
// scalar tail, so any contiguous F32 size is fine - including the
|
|
891
|
+
// 1x1x1x1 scalar case.
|
|
892
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
893
|
+
ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]);
|
|
894
|
+
break;
|
|
895
|
+
case GGML_OP_UNARY:
|
|
896
|
+
// Only require dim-0 contiguity (nb[0] == sizeof(float)). Higher
|
|
897
|
+
// dims may be arbitrarily strided views; the kernel walks per-row
|
|
898
|
+
// using all four nb[] values. See unary_f32.c entry_point.
|
|
899
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
900
|
+
ggml_nelements(op) % 16 == 0 && op->nb[0] == sizeof(float) && op->src[0]->nb[0] == sizeof(float)) {
|
|
901
|
+
switch (ggml_get_unary_op(op)) {
|
|
902
|
+
case GGML_UNARY_OP_ABS:
|
|
903
|
+
case GGML_UNARY_OP_SGN:
|
|
904
|
+
case GGML_UNARY_OP_NEG:
|
|
905
|
+
case GGML_UNARY_OP_STEP:
|
|
906
|
+
case GGML_UNARY_OP_TANH:
|
|
907
|
+
case GGML_UNARY_OP_ELU:
|
|
908
|
+
case GGML_UNARY_OP_RELU:
|
|
909
|
+
case GGML_UNARY_OP_SIGMOID:
|
|
910
|
+
case GGML_UNARY_OP_GELU:
|
|
911
|
+
case GGML_UNARY_OP_GELU_QUICK:
|
|
912
|
+
case GGML_UNARY_OP_SILU:
|
|
913
|
+
case GGML_UNARY_OP_HARDSWISH:
|
|
914
|
+
case GGML_UNARY_OP_HARDSIGMOID:
|
|
915
|
+
case GGML_UNARY_OP_EXP:
|
|
916
|
+
case GGML_UNARY_OP_EXPM1:
|
|
917
|
+
case GGML_UNARY_OP_SOFTPLUS:
|
|
918
|
+
case GGML_UNARY_OP_GELU_ERF:
|
|
919
|
+
case GGML_UNARY_OP_FLOOR:
|
|
920
|
+
case GGML_UNARY_OP_CEIL:
|
|
921
|
+
case GGML_UNARY_OP_ROUND:
|
|
922
|
+
case GGML_UNARY_OP_TRUNC:
|
|
923
|
+
supported = true;
|
|
924
|
+
break;
|
|
925
|
+
default:
|
|
926
|
+
break;
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
break;
|
|
930
|
+
case GGML_OP_MUL:
|
|
931
|
+
case GGML_OP_ADD:
|
|
932
|
+
case GGML_OP_SUB:
|
|
933
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
|
|
934
|
+
op->src[1]->type == GGML_TYPE_F32 && op->nb[0] == sizeof(float) &&
|
|
935
|
+
op->src[0]->nb[0] == sizeof(float) &&
|
|
936
|
+
(op->src[1]->nb[0] == sizeof(float) || op->src[1]->ne[0] == 1) &&
|
|
937
|
+
op->nb[1] == op->ne[0] * sizeof(float);
|
|
938
|
+
break;
|
|
939
|
+
case GGML_OP_MUL_MAT:
|
|
940
|
+
// Support Q8_0 x F32 -> F32, F16 x F32 -> F32, F16 x F16 -> F32, and F32 x F32 -> F32 matrix multiplication
|
|
941
|
+
// Stride requirements: first dimension must be contiguous for all tensors
|
|
942
|
+
if (op->type == GGML_TYPE_F32 &&
|
|
943
|
+
((op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32) ||
|
|
944
|
+
(op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16)) &&
|
|
945
|
+
op->ne[0] % 16 == 0 && // dst row length for tensor-store path
|
|
946
|
+
op->src[0]->ne[1] % 16 == 0 && // m
|
|
947
|
+
op->src[0]->ne[0] % 16 == 0 && // k
|
|
948
|
+
ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) {
|
|
949
|
+
// Special path for the FP32 TensorFMA kernel
|
|
950
|
+
// Limitation - generic kernels can tolerate non-cache-aligned dst rows
|
|
951
|
+
// because they publish each output element atomically. The matrix
|
|
952
|
+
// engine path still uses tiled tensor stores, so keep dst rows aligned.
|
|
953
|
+
// The m edge is difficult to do because of the 4 conseqtive load hardware limitation
|
|
954
|
+
// And the k edge is impossible because that is encoded as `stride & 0xFFFFFFFFFFC0ULL` which becomes 0 for stride 16 (4x FP32) :(
|
|
955
|
+
// FIXME: Right now this overwrites the mul_mat_f32 kernel - whatever. Fix later. Demo code
|
|
956
|
+
supported = true;
|
|
957
|
+
} else if (op->type == GGML_TYPE_F32 && op->src[0] &&
|
|
958
|
+
(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) && op->src[1] &&
|
|
959
|
+
(op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32)) {
|
|
960
|
+
// Check first dimension contiguity requirements
|
|
961
|
+
bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type));
|
|
962
|
+
bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type));
|
|
963
|
+
bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float));
|
|
964
|
+
|
|
965
|
+
// Check destination stride ordering (only for dimensions with ne > 1)
|
|
966
|
+
bool dst_properly_ordered = true;
|
|
967
|
+
for (int d = 0; d < 3; d++) {
|
|
968
|
+
if (op->ne[d] > 1 && op->ne[d + 1] > 1 && op->nb[d] > op->nb[d + 1]) {
|
|
969
|
+
dst_properly_ordered = false;
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
|
|
973
|
+
supported = src0_first_dim_contiguous && src1_first_dim_contiguous && dst_first_dim_contiguous &&
|
|
974
|
+
dst_properly_ordered;
|
|
975
|
+
} else if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_Q8_0 && op->src[1] &&
|
|
976
|
+
op->src[1]->type == GGML_TYPE_F32) {
|
|
977
|
+
// Keep the existing quantized path constraints separate from the
|
|
978
|
+
// relaxed non-quant generic fallback.
|
|
979
|
+
bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type));
|
|
980
|
+
bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type));
|
|
981
|
+
bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float));
|
|
982
|
+
|
|
983
|
+
bool dst_properly_ordered = true;
|
|
984
|
+
for (int d = 0; d < 3; d++) {
|
|
985
|
+
if (op->ne[d] > 1 && op->ne[d + 1] > 1 && op->nb[d] > op->nb[d + 1]) {
|
|
986
|
+
dst_properly_ordered = false;
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
supported = src0_first_dim_contiguous && src1_first_dim_contiguous && dst_first_dim_contiguous &&
|
|
991
|
+
dst_properly_ordered;
|
|
992
|
+
|
|
993
|
+
} else if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_Q4_0 && op->src[1] &&
|
|
994
|
+
op->src[1]->type == GGML_TYPE_F32) {
|
|
995
|
+
// Keep the existing quantized path constraints separate from the
|
|
996
|
+
// relaxed non-quant generic fallback.
|
|
997
|
+
bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type));
|
|
998
|
+
bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type));
|
|
999
|
+
bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float));
|
|
1000
|
+
|
|
1001
|
+
bool dst_properly_ordered = true;
|
|
1002
|
+
for (int d = 0; d < 3; d++) {
|
|
1003
|
+
if (op->ne[d] > 1 && op->ne[d + 1] > 1 && op->nb[d] > op->nb[d + 1]) {
|
|
1004
|
+
dst_properly_ordered = false;
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
supported = src0_first_dim_contiguous && src1_first_dim_contiguous && dst_first_dim_contiguous &&
|
|
1009
|
+
dst_properly_ordered;
|
|
1010
|
+
} else {
|
|
1011
|
+
supported = false;
|
|
1012
|
+
}
|
|
1013
|
+
break;
|
|
1014
|
+
case GGML_OP_MUL_MAT_ID:
|
|
1015
|
+
// Support MUL_MAT_ID for Mixture of Experts: (Q8_0/Q4_0/F16/F32) x F32 -> F32 with I32 expert indices
|
|
1016
|
+
// src0 (as): [K, M, n_expert] - expert weight matrices (can be quantized)
|
|
1017
|
+
// src1 (b): [K, n_expert_used, batch] - activations (F32)
|
|
1018
|
+
// src2 (ids): [n_expert_used, batch] - expert selection indices (I32)
|
|
1019
|
+
// dst: [M, n_expert_used, batch, 1] - output (F32)
|
|
1020
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] &&
|
|
1021
|
+
(op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q4_0 ||
|
|
1022
|
+
op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) &&
|
|
1023
|
+
op->src[1] && op->src[1]->type == GGML_TYPE_F32 && op->src[2] && op->src[2]->type == GGML_TYPE_I32) {
|
|
1024
|
+
// Check first dimension contiguity requirements (matching CPU backend)
|
|
1025
|
+
bool src0_first_dim_contiguous = (op->src[0]->nb[0] == ggml_type_size(op->src[0]->type));
|
|
1026
|
+
bool src1_first_dim_contiguous = (op->src[1]->nb[0] == ggml_type_size(op->src[1]->type));
|
|
1027
|
+
bool src2_first_dim_contiguous = (op->src[2]->nb[0] == ggml_type_size(op->src[2]->type));
|
|
1028
|
+
bool dst_first_dim_contiguous = (op->nb[0] == sizeof(float));
|
|
1029
|
+
|
|
1030
|
+
// Check destination stride ordering (only for dimensions with ne > 1)
|
|
1031
|
+
bool dst_properly_ordered = true;
|
|
1032
|
+
for (int d = 0; d < 3; d++) {
|
|
1033
|
+
if (op->ne[d] > 1 && op->ne[d + 1] > 1 && op->nb[d] > op->nb[d + 1]) {
|
|
1034
|
+
dst_properly_ordered = false;
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
// Validate tensor dimension constraints from GGML definition
|
|
1039
|
+
bool dims_valid = (op->src[0]->ne[3] == 1) && // as is 3d (one matrix per expert)
|
|
1040
|
+
(op->src[1]->ne[3] == 1) && // b is 3d
|
|
1041
|
+
(op->src[2]->ne[2] == 1 && op->src[2]->ne[3] == 1) && // ids is 2d
|
|
1042
|
+
(op->src[2]->ne[1] == op->src[1]->ne[2]) && // must have expert list per b row
|
|
1043
|
+
(op->src[0]->ne[0] == op->src[1]->ne[0]) && // K dimension must match
|
|
1044
|
+
(op->src[2]->ne[0] % op->src[1]->ne[1] == 0); // can broadcast
|
|
1045
|
+
|
|
1046
|
+
supported = src0_first_dim_contiguous && src1_first_dim_contiguous && src2_first_dim_contiguous &&
|
|
1047
|
+
dst_first_dim_contiguous && dst_properly_ordered && dims_valid;
|
|
1048
|
+
} else {
|
|
1049
|
+
supported = false;
|
|
1050
|
+
}
|
|
1051
|
+
break;
|
|
1052
|
+
case GGML_OP_ROPE:
|
|
1053
|
+
// Support F32 x I32 -> F32 RoPE for the modes implemented by rope_f32.
|
|
1054
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
|
|
1055
|
+
op->src[1]->type == GGML_TYPE_I32 && ggml_is_contiguous(op) && et_ggml_is_row_contiguous(op->src[0])) {
|
|
1056
|
+
const int mode = ggml_get_op_params_i32(op, 2);
|
|
1057
|
+
const int ndims = ggml_get_op_params_i32(op, 1);
|
|
1058
|
+
const bool is_normal = mode == GGML_ROPE_TYPE_NORMAL;
|
|
1059
|
+
const bool is_neox = mode == GGML_ROPE_TYPE_NEOX;
|
|
1060
|
+
const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE;
|
|
1061
|
+
const bool zero_view_offset = op->src[0]->view_src == nullptr || op->src[0]->view_offs == 0;
|
|
1062
|
+
const bool has_sections = ggml_get_op_params_i32(op, 11) > 0 || ggml_get_op_params_i32(op, 12) > 0 ||
|
|
1063
|
+
ggml_get_op_params_i32(op, 13) > 0;
|
|
1064
|
+
// FIXME: support ggml_rope_set_offset
|
|
1065
|
+
const bool zero_rot_offset = ggml_get_op_params_i32(op, 15) == 0;
|
|
1066
|
+
|
|
1067
|
+
supported =
|
|
1068
|
+
zero_view_offset && zero_rot_offset && ndims <= 512 &&
|
|
1069
|
+
(is_normal || (is_neox && ndims % 16 == 0) || (is_imrope && ndims % 16 == 0 && has_sections));
|
|
1070
|
+
} else {
|
|
1071
|
+
supported = false;
|
|
1072
|
+
}
|
|
1073
|
+
break;
|
|
1074
|
+
case GGML_OP_RMS_NORM:
|
|
1075
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1076
|
+
op->ne[0] % 16 == 0 && ggml_is_contiguous(op) && et_ggml_is_row_contiguous(op->src[0]);
|
|
1077
|
+
break;
|
|
1078
|
+
case GGML_OP_NORM:
|
|
1079
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1080
|
+
op->ne[0] % 16 == 0 && ggml_is_contiguous(op) && et_ggml_is_row_contiguous(op->src[0]);
|
|
1081
|
+
break;
|
|
1082
|
+
case GGML_OP_L2_NORM:
|
|
1083
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1084
|
+
op->ne[0] % 16 == 0 && ggml_is_contiguous(op) && et_ggml_is_row_contiguous(op->src[0]);
|
|
1085
|
+
break;
|
|
1086
|
+
case GGML_OP_GROUP_NORM:
|
|
1087
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1088
|
+
ggml_is_contiguous(op) && et_ggml_is_row_contiguous(op->src[0]) &&
|
|
1089
|
+
ggml_get_op_params_i32(op, 0) > 0;
|
|
1090
|
+
break;
|
|
1091
|
+
case GGML_OP_IM2COL:
|
|
1092
|
+
supported = op->src[0] && op->src[1] &&
|
|
1093
|
+
((op->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32) ||
|
|
1094
|
+
(op->type == GGML_TYPE_F16 &&
|
|
1095
|
+
(op->src[1]->type == GGML_TYPE_F16 || op->src[1]->type == GGML_TYPE_F32))) &&
|
|
1096
|
+
ggml_is_contiguous(op) && ggml_is_contiguous(op->src[1]) &&
|
|
1097
|
+
op->nb[0] == ggml_type_size(op->type) && op->src[1]->nb[0] == ggml_type_size(op->src[1]->type);
|
|
1098
|
+
break;
|
|
1099
|
+
case GGML_OP_CONV_2D:
|
|
1100
|
+
{
|
|
1101
|
+
// First-cut conv_2d_f32_me kernel constraints. Anything outside
|
|
1102
|
+
// this falls back to CPU (it's a strict subset on purpose).
|
|
1103
|
+
if (!op->src[0] || !op->src[1]) {
|
|
1104
|
+
supported = false;
|
|
1105
|
+
break;
|
|
1106
|
+
}
|
|
1107
|
+
if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 ||
|
|
1108
|
+
op->src[1]->type != GGML_TYPE_F32) {
|
|
1109
|
+
supported = false;
|
|
1110
|
+
break;
|
|
1111
|
+
}
|
|
1112
|
+
if (!ggml_is_contiguous(op) || !ggml_is_contiguous(op->src[0]) || !ggml_is_contiguous(op->src[1])) {
|
|
1113
|
+
supported = false;
|
|
1114
|
+
break;
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
const ggml_tensor * flt = op->src[0]; // [Kw, Kh, Cin, Cout]
|
|
1118
|
+
const ggml_tensor * in = op->src[1]; // [W, H, Cin, N]
|
|
1119
|
+
const int32_t s0 = ggml_get_op_params_i32(op, 0);
|
|
1120
|
+
const int32_t s1 = ggml_get_op_params_i32(op, 1);
|
|
1121
|
+
const int32_t p0 = ggml_get_op_params_i32(op, 2);
|
|
1122
|
+
const int32_t p1 = ggml_get_op_params_i32(op, 3);
|
|
1123
|
+
const int32_t d0 = ggml_get_op_params_i32(op, 4);
|
|
1124
|
+
const int32_t d1 = ggml_get_op_params_i32(op, 5);
|
|
1125
|
+
|
|
1126
|
+
const int64_t Kw = flt->ne[0];
|
|
1127
|
+
const int64_t Kh = flt->ne[1];
|
|
1128
|
+
const int64_t Cin = flt->ne[2];
|
|
1129
|
+
const int64_t Cout = flt->ne[3];
|
|
1130
|
+
const int64_t H = in->ne[1];
|
|
1131
|
+
(void) in->ne[0];
|
|
1132
|
+
|
|
1133
|
+
if (s0 < 1 || s1 < 1 || !(d0 == 1 && d1 == 1) || Cin % 16 != 0 || Cout % 16 != 0 || in->ne[3] != 1) {
|
|
1134
|
+
supported = false;
|
|
1135
|
+
break;
|
|
1136
|
+
}
|
|
1137
|
+
const int64_t OW = op->ne[0];
|
|
1138
|
+
const int64_t OH = op->ne[1];
|
|
1139
|
+
if (OW <= 0 || OH <= 0) {
|
|
1140
|
+
supported = false;
|
|
1141
|
+
break;
|
|
1142
|
+
}
|
|
1143
|
+
(void) p0;
|
|
1144
|
+
(void) p1;
|
|
1145
|
+
|
|
1146
|
+
// Mirror the kernel's sizing:
|
|
1147
|
+
// if K_TILES * per_KT_bytes <= budget: 1 buffer, n_chunks=1
|
|
1148
|
+
// else: 2 buffers (double-buffer), shrink chunk_KT until
|
|
1149
|
+
// 2*chunk_KT*per_KT_bytes <= budget.
|
|
1150
|
+
const int64_t Hp = H + 2 * p1;
|
|
1151
|
+
const int64_t OW_pad = (OW + 15) & ~15;
|
|
1152
|
+
const int64_t Wp_a = OW_pad;
|
|
1153
|
+
const bool need_stage = (OW % 16 != 0);
|
|
1154
|
+
const int64_t stage_bytes = need_stage ? (Cout * OH * OW_pad * 4) : 0;
|
|
1155
|
+
const int64_t L2SCP_BUDGET = 1500 * 1024;
|
|
1156
|
+
// Per-hart partial-TenC scratch (mirrors kernel MAX_TILES_PER_HART=2):
|
|
1157
|
+
// 32 minions x 2 tiles x 1024 bytes = 64 KB per shire.
|
|
1158
|
+
const int64_t scratch_bytes = 32 * 2 * 16 * 16 * 4;
|
|
1159
|
+
const int64_t budget = L2SCP_BUDGET - stage_bytes - scratch_bytes;
|
|
1160
|
+
const int64_t per_KT_bytes = Kh * Kw * Cout * 16 * 4 + Kw * 16 * Hp * Wp_a * 4;
|
|
1161
|
+
const int64_t K_TILES = Cin / 16;
|
|
1162
|
+
|
|
1163
|
+
int64_t chunk_KT_calc;
|
|
1164
|
+
int64_t n_chunks_calc;
|
|
1165
|
+
if (K_TILES * per_KT_bytes <= budget) {
|
|
1166
|
+
chunk_KT_calc = K_TILES;
|
|
1167
|
+
n_chunks_calc = 1;
|
|
1168
|
+
} else {
|
|
1169
|
+
chunk_KT_calc = K_TILES;
|
|
1170
|
+
while (chunk_KT_calc > 1 && 2 * chunk_KT_calc * per_KT_bytes > budget) {
|
|
1171
|
+
chunk_KT_calc--;
|
|
1172
|
+
}
|
|
1173
|
+
while (chunk_KT_calc > 1 && K_TILES % chunk_KT_calc != 0) {
|
|
1174
|
+
chunk_KT_calc--;
|
|
1175
|
+
}
|
|
1176
|
+
if (chunk_KT_calc < 1) {
|
|
1177
|
+
supported = false;
|
|
1178
|
+
break;
|
|
1179
|
+
}
|
|
1180
|
+
n_chunks_calc = K_TILES / chunk_KT_calc;
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
if (n_chunks_calc > 1) {
|
|
1184
|
+
const int64_t M_TILES = Cout / 16;
|
|
1185
|
+
const int64_t w_tiles = (OW + 15) / 16;
|
|
1186
|
+
const int64_t total_tiles = OH * w_tiles * M_TILES;
|
|
1187
|
+
// MAX_TILES_PER_HART = 2 (mirrors kernel constant).
|
|
1188
|
+
const int64_t max_workers = (need_stage ? 32 : 1024) * 2;
|
|
1189
|
+
if (total_tiles > max_workers) {
|
|
1190
|
+
supported = false;
|
|
1191
|
+
break;
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
supported = true;
|
|
1196
|
+
break;
|
|
1197
|
+
}
|
|
1198
|
+
case GGML_OP_SCALE:
|
|
1199
|
+
// F32 contiguous, total elements must be cache line aligned (16 floats)
|
|
1200
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1201
|
+
ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && (ggml_nelements(op) % 16 == 0);
|
|
1202
|
+
break;
|
|
1203
|
+
case GGML_OP_GLU:
|
|
1204
|
+
// Note: we only require row-wise contiguity (ggml_is_contiguous_1) so that
|
|
1205
|
+
// strided views over a packed up_proj tensor (the common split-GLU layout)
|
|
1206
|
+
// are accepted. The kernel walks rows via nb[1] strides, so the inner
|
|
1207
|
+
// dimension just needs to be densely packed.
|
|
1208
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1209
|
+
ggml_nelements(op) % 16 == 0 && ggml_is_contiguous_1(op) && ggml_is_contiguous_1(op->src[0])) {
|
|
1210
|
+
// Check GLU variant - support SWIGLU, SWIGLU_OAI, GEGLU, GEGLU_ERF, GEGLU_QUICK, REGLU
|
|
1211
|
+
ggml_glu_op glu_type = ggml_get_glu_op(op);
|
|
1212
|
+
const bool supported_variant = glu_type == GGML_GLU_OP_SWIGLU || glu_type == GGML_GLU_OP_SWIGLU_OAI ||
|
|
1213
|
+
glu_type == GGML_GLU_OP_SWIGLU_CLAMP || glu_type == GGML_GLU_OP_GEGLU ||
|
|
1214
|
+
glu_type == GGML_GLU_OP_GEGLU_ERF ||
|
|
1215
|
+
glu_type == GGML_GLU_OP_GEGLU_QUICK || glu_type == GGML_GLU_OP_REGLU;
|
|
1216
|
+
|
|
1217
|
+
if (op->src[1]) {
|
|
1218
|
+
supported = supported_variant && op->src[1]->type == GGML_TYPE_F32 &&
|
|
1219
|
+
ggml_is_contiguous_1(op->src[1]) && op->src[0]->ne[0] == op->ne[0] &&
|
|
1220
|
+
op->src[1]->ne[0] == op->ne[0];
|
|
1221
|
+
} else {
|
|
1222
|
+
supported = supported_variant && op->src[0]->ne[0] == 2 * op->ne[0];
|
|
1223
|
+
}
|
|
1224
|
+
} else {
|
|
1225
|
+
supported = false;
|
|
1226
|
+
}
|
|
1227
|
+
break;
|
|
1228
|
+
case GGML_OP_SOFT_MAX:
|
|
1229
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1230
|
+
ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && op->src[0]->ne[0] > 1) {
|
|
1231
|
+
// Check optional mask tensor (F32 only)
|
|
1232
|
+
if (op->src[1]) {
|
|
1233
|
+
supported = op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[1]);
|
|
1234
|
+
if (!supported) {
|
|
1235
|
+
break;
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
// Check optional sinks tensor (F32 only)
|
|
1239
|
+
if (op->src[2]) {
|
|
1240
|
+
supported = op->src[2]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[2]);
|
|
1241
|
+
} else {
|
|
1242
|
+
supported = true;
|
|
1243
|
+
}
|
|
1244
|
+
} else {
|
|
1245
|
+
supported = false;
|
|
1246
|
+
}
|
|
1247
|
+
break;
|
|
1248
|
+
case GGML_OP_SSM_SCAN:
|
|
1249
|
+
supported = op->type == GGML_TYPE_F32 && ggml_is_contiguous(op) && op->src[0] &&
|
|
1250
|
+
op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]) && op->src[1] &&
|
|
1251
|
+
op->src[1]->type == GGML_TYPE_F32 && op->src[2] && op->src[2]->type == GGML_TYPE_F32 &&
|
|
1252
|
+
ggml_is_contiguous(op->src[2]) && op->src[3] && op->src[3]->type == GGML_TYPE_F32 &&
|
|
1253
|
+
ggml_is_contiguous(op->src[3]) && op->src[4] && op->src[4]->type == GGML_TYPE_F32 &&
|
|
1254
|
+
op->src[5] && op->src[5]->type == GGML_TYPE_F32 && op->src[6] &&
|
|
1255
|
+
op->src[6]->type == GGML_TYPE_I32 && ggml_is_contiguous(op->src[6]) &&
|
|
1256
|
+
op->src[1]->nb[0] == sizeof(float) && op->src[4]->nb[0] == sizeof(float) &&
|
|
1257
|
+
op->src[5]->nb[0] == sizeof(float) &&
|
|
1258
|
+
op->src[1]->nb[1] == (size_t) op->src[1]->ne[0] * sizeof(float) &&
|
|
1259
|
+
op->src[4]->nb[1] == (size_t) op->src[4]->ne[0] * sizeof(float) &&
|
|
1260
|
+
op->src[5]->nb[1] == (size_t) op->src[5]->ne[0] * sizeof(float) &&
|
|
1261
|
+
op->src[0]->ne[0] == op->src[4]->ne[0] && op->src[0]->ne[1] == op->src[1]->ne[0] &&
|
|
1262
|
+
op->src[0]->ne[2] == op->src[1]->ne[1] && op->src[1]->ne[2] == op->src[2]->ne[1] &&
|
|
1263
|
+
op->src[1]->ne[3] == op->src[2]->ne[2] && op->src[4]->ne[2] == op->src[1]->ne[2] &&
|
|
1264
|
+
op->src[4]->ne[3] == op->src[1]->ne[3] && ggml_are_same_shape(op->src[4], op->src[5]) &&
|
|
1265
|
+
op->src[6]->ne[0] == op->src[1]->ne[3] && op->src[3]->ne[1] == op->src[1]->ne[1] &&
|
|
1266
|
+
(op->src[3]->ne[0] == 1 || op->src[3]->ne[0] == op->src[0]->ne[0]) &&
|
|
1267
|
+
(op->src[1]->ne[1] % op->src[4]->ne[1] == 0);
|
|
1268
|
+
break;
|
|
1269
|
+
case GGML_OP_FLASH_ATTN_EXT:
|
|
1270
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
|
|
1271
|
+
(op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) && op->src[2] &&
|
|
1272
|
+
(op->src[2]->type == GGML_TYPE_F32 || op->src[2]->type == GGML_TYPE_F16) && op->src[4] == nullptr &&
|
|
1273
|
+
ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0])) {
|
|
1274
|
+
float max_bias = 0.0f;
|
|
1275
|
+
float logit_softcap = 0.0f;
|
|
1276
|
+
memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(max_bias));
|
|
1277
|
+
memcpy(&logit_softcap, (const float *) op->op_params + 2, sizeof(logit_softcap));
|
|
1278
|
+
|
|
1279
|
+
const ggml_prec prec = ggml_flash_attn_ext_get_prec(op);
|
|
1280
|
+
|
|
1281
|
+
// Mask must be F16 or F32 if present
|
|
1282
|
+
bool mask_ok = (op->src[3] == nullptr) || (op->src[3]->type == GGML_TYPE_F32) ||
|
|
1283
|
+
(op->src[3]->type == GGML_TYPE_F16);
|
|
1284
|
+
|
|
1285
|
+
// GQA: n_head_q must be a multiple of n_head_kv
|
|
1286
|
+
const int64_t nhq = op->src[0]->ne[2];
|
|
1287
|
+
const int64_t nhk = op->src[1]->ne[2];
|
|
1288
|
+
|
|
1289
|
+
// K/V row stride must match element size
|
|
1290
|
+
const size_t k_elem = op->src[1]->type == GGML_TYPE_F16 ? 2 : 4;
|
|
1291
|
+
const size_t v_elem = op->src[2]->type == GGML_TYPE_F16 ? 2 : 4;
|
|
1292
|
+
|
|
1293
|
+
// Only support matrix engine path (F16 K/V, dk%32==0);
|
|
1294
|
+
// mask scalar F32 fallback to get baseline perf readings
|
|
1295
|
+
const bool me_eligible = op->src[1]->type == GGML_TYPE_F16 && op->src[2]->type == GGML_TYPE_F16 &&
|
|
1296
|
+
(op->src[0]->ne[0] % 32) == 0;
|
|
1297
|
+
|
|
1298
|
+
supported = me_eligible && mask_ok && (prec == GGML_PREC_F32 || prec == GGML_PREC_DEFAULT) &&
|
|
1299
|
+
max_bias == 0.0f && logit_softcap == 0.0f && op->src[0]->nb[0] == sizeof(float) &&
|
|
1300
|
+
op->src[1]->nb[0] == k_elem && op->src[2]->nb[0] == v_elem && op->nb[0] == sizeof(float) &&
|
|
1301
|
+
op->src[0]->ne[0] == op->src[1]->ne[0] && // dk matches
|
|
1302
|
+
op->src[2]->ne[0] == op->ne[0] && // dv matches
|
|
1303
|
+
op->src[2]->ne[0] <= 512 && // dv limit
|
|
1304
|
+
op->src[0]->ne[0] <= 512 && // dk limit
|
|
1305
|
+
nhq % nhk == 0 && // GQA ratio is integer
|
|
1306
|
+
op->src[0]->ne[1] == op->ne[2] && op->src[0]->ne[2] == op->ne[1] &&
|
|
1307
|
+
op->src[0]->ne[3] == op->ne[3] && op->src[1]->ne[1] == op->src[2]->ne[1] &&
|
|
1308
|
+
op->src[1]->ne[2] == op->src[2]->ne[2] && op->src[1]->ne[3] == op->src[2]->ne[3] &&
|
|
1309
|
+
op->src[0]->ne[3] == op->src[1]->ne[3];
|
|
1310
|
+
} else {
|
|
1311
|
+
supported = false;
|
|
1312
|
+
}
|
|
1313
|
+
break;
|
|
1314
|
+
case GGML_OP_GET_ROWS:
|
|
1315
|
+
// Support F32/F16/Q4_0/Q8_0/Q4_K data with I32 indices -> F32 output
|
|
1316
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] &&
|
|
1317
|
+
(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 ||
|
|
1318
|
+
op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 ||
|
|
1319
|
+
op->src[0]->type == GGML_TYPE_Q4_K) &&
|
|
1320
|
+
op->src[1] && op->src[1]->type == GGML_TYPE_I32 && ggml_is_contiguous(op) &&
|
|
1321
|
+
ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) {
|
|
1322
|
+
// Validate dimension constraints from ggml implementation
|
|
1323
|
+
supported = (op->src[0]->ne[2] == op->src[1]->ne[1]) && (op->src[1]->ne[3] == 1);
|
|
1324
|
+
} else {
|
|
1325
|
+
supported = false;
|
|
1326
|
+
}
|
|
1327
|
+
break;
|
|
1328
|
+
case GGML_OP_CONT:
|
|
1329
|
+
// Support F32->F32 and F16->F16 CONT operations (rearrange non-contiguous to contiguous)
|
|
1330
|
+
if ((op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && op->src[0] &&
|
|
1331
|
+
op->src[0]->type == op->type && ggml_is_contiguous(op)) {
|
|
1332
|
+
// Defensive check: ensure dst and src0 are not aliased (separate buffers)
|
|
1333
|
+
// While GGML design currently guarantees this, check for future robustness
|
|
1334
|
+
if (op->data && op->src[0]->data && op->data == op->src[0]->data) {
|
|
1335
|
+
GGML_LOG_WARN("ET: CONT operation detected aliased tensors (dst == src0), unsupported");
|
|
1336
|
+
supported = false;
|
|
1337
|
+
} else {
|
|
1338
|
+
supported = true;
|
|
1339
|
+
}
|
|
1340
|
+
} else {
|
|
1341
|
+
supported = false;
|
|
1342
|
+
}
|
|
1343
|
+
break;
|
|
1344
|
+
case GGML_OP_CPY:
|
|
1345
|
+
// CPY copies src[0] data into dst layout (same as CONT for same-type)
|
|
1346
|
+
// Special path: zero-element tensors (scalars) are accepted as no-ops
|
|
1347
|
+
if (op->src[0]) {
|
|
1348
|
+
const int64_t nelements = op->ne[0] * op->ne[1] * op->ne[2] * op->ne[3];
|
|
1349
|
+
if (nelements == 0) {
|
|
1350
|
+
// Zero-element / scalar no-op case - always supported
|
|
1351
|
+
supported = true;
|
|
1352
|
+
} else if ((op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && op->src[0]->type == op->type &&
|
|
1353
|
+
ggml_is_contiguous(op)) {
|
|
1354
|
+
// Same-type with contiguous dst - reuse CONT kernel
|
|
1355
|
+
if (op->data && op->src[0]->data && op->data == op->src[0]->data) {
|
|
1356
|
+
GGML_LOG_WARN("ET: CPY operation detected aliased tensors, unsupported");
|
|
1357
|
+
supported = false;
|
|
1358
|
+
} else {
|
|
1359
|
+
supported = true;
|
|
1360
|
+
}
|
|
1361
|
+
} else if (op->type == GGML_TYPE_F16 && op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op)) {
|
|
1362
|
+
// F32 -> F16 conversion copy
|
|
1363
|
+
supported = true;
|
|
1364
|
+
} else {
|
|
1365
|
+
supported = false;
|
|
1366
|
+
}
|
|
1367
|
+
} else {
|
|
1368
|
+
supported = false;
|
|
1369
|
+
}
|
|
1370
|
+
break;
|
|
1371
|
+
case GGML_OP_CONCAT:
|
|
1372
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
|
|
1373
|
+
op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op)) {
|
|
1374
|
+
const int32_t dim = ((const int32_t *) op->op_params)[0];
|
|
1375
|
+
if (dim == 0 && op->src[0]->ne[0] % 16 == 0 && op->src[1]->ne[0] % 16 == 0 &&
|
|
1376
|
+
ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) {
|
|
1377
|
+
// Fast dim==0 path: both source row segments are cacheline-aligned
|
|
1378
|
+
// and contiguous, so the kernel can use vector row copies.
|
|
1379
|
+
supported = true;
|
|
1380
|
+
} else if (dim == 0 && ((op->src[0]->nb[0] % sizeof(float) == 0) || op->src[0]->ne[0] == 1) &&
|
|
1381
|
+
((op->src[1]->nb[0] % sizeof(float) == 0) || op->src[1]->ne[0] == 1)) {
|
|
1382
|
+
// Slow dim==0 path: scalar, stride-aware copies for non-contiguous
|
|
1383
|
+
// or non-aligned source row segments. Destination remains contiguous.
|
|
1384
|
+
supported = true;
|
|
1385
|
+
} else if (op->ne[0] % 16 == 0 && op->src[0]->ne[0] % 16 == 0 && op->src[1]->ne[0] % 16 == 0 &&
|
|
1386
|
+
ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])) {
|
|
1387
|
+
// Dim >= 1 path: full aligned row copies from one source or the other.
|
|
1388
|
+
supported = true;
|
|
1389
|
+
}
|
|
1390
|
+
}
|
|
1391
|
+
break;
|
|
1392
|
+
case GGML_OP_SSM_CONV:
|
|
1393
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
|
|
1394
|
+
op->src[1]->type == GGML_TYPE_F32 && op->src[0]->nb[0] == sizeof(float) &&
|
|
1395
|
+
op->src[1]->nb[0] == sizeof(float) && op->src[0]->nb[1] == op->src[0]->ne[0] * sizeof(float) &&
|
|
1396
|
+
op->src[1]->nb[1] == op->src[1]->ne[0] * sizeof(float) && ggml_is_contiguous(op) &&
|
|
1397
|
+
op->src[1]->ne[1] == op->src[0]->ne[1] && op->ne[0] == op->src[0]->ne[1] &&
|
|
1398
|
+
op->ne[1] == op->src[0]->ne[0] - op->src[1]->ne[0] + 1 && op->ne[2] == op->src[0]->ne[2];
|
|
1399
|
+
break;
|
|
1400
|
+
case GGML_OP_PAD:
|
|
1401
|
+
// F32 zero-pad only, no dim0 padding, dst contiguous
|
|
1402
|
+
// ne[0] must be CL-aligned (% 16 == 0) or evenly divide a CL (16 % ne[0] == 0)
|
|
1403
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1404
|
+
ggml_is_contiguous(op) && (op->ne[0] % 16 == 0 || 16 % op->ne[0] == 0) &&
|
|
1405
|
+
op->src[0]->nb[0] == sizeof(float)) {
|
|
1406
|
+
const int32_t lp0 = ((const int32_t *) op->op_params)[0];
|
|
1407
|
+
const int32_t rp0 = ((const int32_t *) op->op_params)[1];
|
|
1408
|
+
const bool circular = (bool) ((const int32_t *) op->op_params)[8];
|
|
1409
|
+
if (lp0 == 0 && rp0 == 0 && !circular) {
|
|
1410
|
+
supported = true;
|
|
1411
|
+
} else {
|
|
1412
|
+
supported = false;
|
|
1413
|
+
}
|
|
1414
|
+
} else {
|
|
1415
|
+
supported = false;
|
|
1416
|
+
}
|
|
1417
|
+
break;
|
|
1418
|
+
case GGML_OP_REPEAT:
|
|
1419
|
+
// Two acceptable shapes:
|
|
1420
|
+
// 1. No-op REPEAT (src and dst have identical shape): dispatched
|
|
1421
|
+
// to cont_f32, which handles arbitrary contiguous sizes.
|
|
1422
|
+
// 2. Real REPEAT via repeat_f32 kernel: dst ne[0] cacheline-aligned,
|
|
1423
|
+
// src0 ne[0] cacheline-aligned or 1, dst.ne[i] % src0.ne[i] == 0.
|
|
1424
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1425
|
+
ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && ggml_are_same_shape(op->src[0], op)) {
|
|
1426
|
+
supported = true;
|
|
1427
|
+
} else if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1428
|
+
(op->src[0]->ne[0] == 1 || op->src[0]->ne[0] % 16 == 0) && op->ne[0] % 16 == 0 &&
|
|
1429
|
+
ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && op->ne[0] % op->src[0]->ne[0] == 0 &&
|
|
1430
|
+
op->ne[1] % op->src[0]->ne[1] == 0 && op->ne[2] % op->src[0]->ne[2] == 0 &&
|
|
1431
|
+
op->ne[3] % op->src[0]->ne[3] == 0) {
|
|
1432
|
+
supported = true;
|
|
1433
|
+
} else {
|
|
1434
|
+
supported = false;
|
|
1435
|
+
}
|
|
1436
|
+
break;
|
|
1437
|
+
case GGML_OP_FILL:
|
|
1438
|
+
// F32 contiguous, ne[0] cacheline-aligned for SIMD fill
|
|
1439
|
+
supported = op->type == GGML_TYPE_F32 && ggml_is_contiguous(op) && op->ne[0] % 16 == 0;
|
|
1440
|
+
break;
|
|
1441
|
+
case GGML_OP_DIAG:
|
|
1442
|
+
// F32 contiguous dst, src0 is 1D vector [N,1,...], dst is [N,N,...]
|
|
1443
|
+
// ne[0] must be cacheline-aligned for SIMD zeroing
|
|
1444
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1445
|
+
op->ne[0] % 16 == 0 && op->ne[0] == op->ne[1] && op->src[0]->ne[0] == op->ne[0] &&
|
|
1446
|
+
op->src[0]->ne[1] == 1 && ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]);
|
|
1447
|
+
break;
|
|
1448
|
+
case GGML_OP_TRI:
|
|
1449
|
+
// F32 contiguous, same shape in/out
|
|
1450
|
+
// Kernel handles arbitrary ne[0] with aligned fast path + scalar fallback
|
|
1451
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 &&
|
|
1452
|
+
ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]);
|
|
1453
|
+
break;
|
|
1454
|
+
case GGML_OP_SOLVE_TRI:
|
|
1455
|
+
// F32 contiguous, A square, shapes compatible
|
|
1456
|
+
// Only lower-triangular left-side non-unit variant
|
|
1457
|
+
// Require k % 16 == 0 for cache-line-safe column parallelism
|
|
1458
|
+
supported = op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
|
|
1459
|
+
op->src[1]->type == GGML_TYPE_F32 && op->src[0]->ne[0] == op->src[0]->ne[1] &&
|
|
1460
|
+
op->src[0]->ne[1] == op->src[1]->ne[1] && op->src[1]->ne[0] % 16 == 0 &&
|
|
1461
|
+
ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);
|
|
1462
|
+
break;
|
|
1463
|
+
case GGML_OP_SET:
|
|
1464
|
+
// Minimal useful support: inplace F32 SET of a contiguous src1 view into
|
|
1465
|
+
// a contiguous dst/base tensor using explicit destination view strides.
|
|
1466
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
|
|
1467
|
+
op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous(op) && ggml_is_contiguous(op->src[0]) &&
|
|
1468
|
+
ggml_is_contiguous(op->src[1]) && ggml_are_same_shape(op, op->src[0]) && op->src[1]->ne[0] % 16 == 0) {
|
|
1469
|
+
const bool inplace = (bool) ((const int32_t *) op->op_params)[4];
|
|
1470
|
+
const size_t nb1 = ((const int32_t *) op->op_params)[0];
|
|
1471
|
+
const size_t nb2 = ((const int32_t *) op->op_params)[1];
|
|
1472
|
+
const size_t nb3 = ((const int32_t *) op->op_params)[2];
|
|
1473
|
+
const size_t offset = ((const int32_t *) op->op_params)[3];
|
|
1474
|
+
const size_t nb0 = ggml_element_size(op);
|
|
1475
|
+
const size_t im0 = op->src[1]->ne[0] == 0 ? 0 : op->src[1]->ne[0] - 1;
|
|
1476
|
+
const size_t im1 = op->src[1]->ne[1] == 0 ? 0 : op->src[1]->ne[1] - 1;
|
|
1477
|
+
const size_t im2 = op->src[1]->ne[2] == 0 ? 0 : op->src[1]->ne[2] - 1;
|
|
1478
|
+
const size_t im3 = op->src[1]->ne[3] == 0 ? 0 : op->src[1]->ne[3] - 1;
|
|
1479
|
+
|
|
1480
|
+
const bool view_bounds_ok = offset + im0 * nb0 + im1 * nb1 + im2 * nb2 + im3 * nb3 <= ggml_nbytes(op);
|
|
1481
|
+
|
|
1482
|
+
const bool cacheline_aligned =
|
|
1483
|
+
(nb1 % 64 == 0) && (nb2 % 64 == 0) && (nb3 % 64 == 0) && (offset % 64 == 0);
|
|
1484
|
+
|
|
1485
|
+
supported = inplace && view_bounds_ok && cacheline_aligned;
|
|
1486
|
+
}
|
|
1487
|
+
break;
|
|
1488
|
+
case GGML_OP_RWKV_WKV6:
|
|
1489
|
+
// F32 contiguous, head_size must be multiple of 8 for vectorization
|
|
1490
|
+
// 6 sources: k, v, r, tf, td, state
|
|
1491
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
|
|
1492
|
+
op->src[1]->type == GGML_TYPE_F32 && op->src[2] && op->src[2]->type == GGML_TYPE_F32 && op->src[3] &&
|
|
1493
|
+
op->src[3]->type == GGML_TYPE_F32 && op->src[4] && op->src[4]->type == GGML_TYPE_F32 && op->src[5] &&
|
|
1494
|
+
op->src[5]->type == GGML_TYPE_F32 && op->src[0]->ne[0] % 8 == 0 && // head_size multiple of 8
|
|
1495
|
+
ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) && ggml_is_contiguous(op->src[2]) &&
|
|
1496
|
+
ggml_is_contiguous(op->src[3]) && ggml_is_contiguous(op->src[4]) && ggml_is_contiguous(op->src[5])) {
|
|
1497
|
+
supported = true;
|
|
1498
|
+
} else {
|
|
1499
|
+
supported = false;
|
|
1500
|
+
}
|
|
1501
|
+
break;
|
|
1502
|
+
case GGML_OP_RWKV_WKV7:
|
|
1503
|
+
// F32 contiguous, head_size must be multiple of 8 for vectorization
|
|
1504
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] &&
|
|
1505
|
+
op->src[1]->type == GGML_TYPE_F32 && op->src[2] && op->src[2]->type == GGML_TYPE_F32 && op->src[3] &&
|
|
1506
|
+
op->src[3]->type == GGML_TYPE_F32 && op->src[4] && op->src[4]->type == GGML_TYPE_F32 && op->src[5] &&
|
|
1507
|
+
op->src[5]->type == GGML_TYPE_F32 && op->src[6] && op->src[6]->type == GGML_TYPE_F32 &&
|
|
1508
|
+
op->src[2]->ne[0] % 8 == 0 && // head_size multiple of 8
|
|
1509
|
+
ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) && ggml_is_contiguous(op->src[2]) &&
|
|
1510
|
+
ggml_is_contiguous(op->src[3]) && ggml_is_contiguous(op->src[4]) && ggml_is_contiguous(op->src[5]) &&
|
|
1511
|
+
ggml_is_contiguous(op->src[6])) {
|
|
1512
|
+
supported = true;
|
|
1513
|
+
} else {
|
|
1514
|
+
supported = false;
|
|
1515
|
+
}
|
|
1516
|
+
break;
|
|
1517
|
+
case GGML_OP_GATED_DELTA_NET:
|
|
1518
|
+
// F32, S_v must be multiple of 8 for vectorization
|
|
1519
|
+
// q, k, v may be row-contiguous with strided higher dimensions.
|
|
1520
|
+
// g, beta, state stay contiguous.
|
|
1521
|
+
if (op->type == GGML_TYPE_F32 && op->src[0] && op->src[0]->type == GGML_TYPE_F32 && // q
|
|
1522
|
+
op->src[1] && op->src[1]->type == GGML_TYPE_F32 && // k
|
|
1523
|
+
op->src[2] && op->src[2]->type == GGML_TYPE_F32 && // v
|
|
1524
|
+
op->src[3] && op->src[3]->type == GGML_TYPE_F32 && // g
|
|
1525
|
+
op->src[4] && op->src[4]->type == GGML_TYPE_F32 && // beta
|
|
1526
|
+
op->src[5] && op->src[5]->type == GGML_TYPE_F32 && // state
|
|
1527
|
+
op->src[2]->ne[0] % 8 == 0 && // S_v multiple of 8
|
|
1528
|
+
(op->src[3]->ne[0] == 1 || op->src[3]->ne[0] == op->src[2]->ne[0]) && // g is scalar or per-element
|
|
1529
|
+
op->src[4]->ne[0] == 1 && // beta is scalar per position
|
|
1530
|
+
et_ggml_is_row_contiguous(op->src[0]) && et_ggml_is_row_contiguous(op->src[1]) &&
|
|
1531
|
+
et_ggml_is_row_contiguous(op->src[2]) && ggml_is_contiguous(op->src[3]) &&
|
|
1532
|
+
ggml_is_contiguous(op->src[4]) && ggml_is_contiguous(op->src[5])) {
|
|
1533
|
+
supported = true;
|
|
1534
|
+
} else {
|
|
1535
|
+
supported = false;
|
|
1536
|
+
}
|
|
1537
|
+
break;
|
|
1538
|
+
case GGML_OP_VIEW:
|
|
1539
|
+
case GGML_OP_PERMUTE:
|
|
1540
|
+
case GGML_OP_TRANSPOSE:
|
|
1541
|
+
case GGML_OP_RESHAPE:
|
|
1542
|
+
// Metadata-only no-ops, accept any type
|
|
1543
|
+
supported = true;
|
|
1544
|
+
break;
|
|
1545
|
+
case GGML_OP_SET_ROWS:
|
|
1546
|
+
// Support F32 data with I64 indices -> F16/F32 output (scatter operation)
|
|
1547
|
+
if (op->src[0] && op->src[0]->type == GGML_TYPE_F32 && op->src[1] && op->src[1]->type == GGML_TYPE_I64 &&
|
|
1548
|
+
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && ggml_is_contiguous_rows(op) &&
|
|
1549
|
+
ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous(op->src[1])) {
|
|
1550
|
+
// Validate dimension constraints from ggml implementation
|
|
1551
|
+
supported = (op->ne[0] == op->src[0]->ne[0]) && // same number of columns
|
|
1552
|
+
(op->ne[2] == op->src[0]->ne[2]) && // same batch size
|
|
1553
|
+
(op->ne[3] == op->src[0]->ne[3]) && // same outer dimension
|
|
1554
|
+
(op->src[0]->ne[1] == op->src[1]->ne[0]) && // src rows = index count
|
|
1555
|
+
(op->src[0]->ne[2] % op->src[1]->ne[1] == 0) && // batch constraint
|
|
1556
|
+
(op->src[0]->ne[3] % op->src[1]->ne[2] == 0) && // outer constraint
|
|
1557
|
+
(op->src[1]->ne[3] == 1); // indices tensor constraint
|
|
1558
|
+
} else {
|
|
1559
|
+
supported = false;
|
|
1560
|
+
}
|
|
1561
|
+
break;
|
|
1562
|
+
case GGML_OP_NONE:
|
|
1563
|
+
// Always support NONE operations - they represent leaf nodes (parameters, inputs, constants)
|
|
1564
|
+
// No computation needed, just memory management
|
|
1565
|
+
supported = true;
|
|
1566
|
+
break;
|
|
1567
|
+
default:
|
|
1568
|
+
supported = false;
|
|
1569
|
+
break;
|
|
1570
|
+
}
|
|
1571
|
+
// if(!supported) {
|
|
1572
|
+
// ggml_et_dump_operator_metadata(op);
|
|
1573
|
+
// }
|
|
1574
|
+
return supported;
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
static bool ggml_backend_et_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
|
|
1578
|
+
GGML_UNUSED(dev);
|
|
1579
|
+
return buft->iface.get_name == ggml_backend_et_buffer_type_get_name;
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
static bool ggml_backend_et_device_offload_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
|
|
1583
|
+
// GET_ROWS (embedding lookup) uses a large weight (tok_embd) that lives on CPU (dev_input).
|
|
1584
|
+
// The scheduler has no mechanism to cache cross-backend weight copies - it re-copies split
|
|
1585
|
+
// inputs every graph_compute call. For GET_ROWS this means copying the entire embedding table
|
|
1586
|
+
// (e.g. 266MB for Llama 3.1 1B) from host to device on every token, just to look up a few rows.
|
|
1587
|
+
// Keep GET_ROWS on CPU and let the scheduler copy only the small result to the device.
|
|
1588
|
+
// The other backends either only offload if the tensor lives on device or is large enough to
|
|
1589
|
+
// justify the copy cost.
|
|
1590
|
+
if (op->op == GGML_OP_GET_ROWS) {
|
|
1591
|
+
return false;
|
|
1592
|
+
}
|
|
1593
|
+
return true;
|
|
1594
|
+
|
|
1595
|
+
GGML_UNUSED(dev);
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
static const struct ggml_backend_i ggml_backend_et_i = {
|
|
1599
|
+
/* .get_name = */ ggml_backend_et_get_name,
|
|
1600
|
+
/* .free = */ ggml_backend_et_free,
|
|
1601
|
+
/* .set_tensor_async = */ ggml_backend_et_set_tensor_async,
|
|
1602
|
+
/* .get_tensor_async = */ ggml_backend_et_get_tensor_async,
|
|
1603
|
+
/* .set_tensor_2d_async = */ NULL,
|
|
1604
|
+
/* .get_tensor_2d_async = */ NULL,
|
|
1605
|
+
/* .cpy_tensor_async = */ NULL,
|
|
1606
|
+
/* .synchronize = */ ggml_backend_et_synchronize,
|
|
1607
|
+
/* .graph_plan_create = */ NULL,
|
|
1608
|
+
/* .graph_plan_free = */ NULL,
|
|
1609
|
+
/* .graph_plan_update = */ NULL,
|
|
1610
|
+
/* .graph_plan_compute = */ NULL,
|
|
1611
|
+
/* .graph_compute = */ ggml_backend_et_graph_compute,
|
|
1612
|
+
/* .event_record = */ NULL,
|
|
1613
|
+
/* .event_wait = */ NULL,
|
|
1614
|
+
/* .graph_optimize = */ NULL,
|
|
1615
|
+
};
|
|
1616
|
+
|
|
1617
|
+
static const char * ggml_backend_et_device_get_name(ggml_backend_dev_t dev) {
|
|
1618
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context;
|
|
1619
|
+
return dev_ctx->name.c_str();
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1622
|
+
static const char * ggml_backend_et_device_get_description(ggml_backend_dev_t dev) {
|
|
1623
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context;
|
|
1624
|
+
return dev_ctx->desc.c_str();
|
|
1625
|
+
}
|
|
1626
|
+
|
|
1627
|
+
static void ggml_backend_et_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
|
|
1628
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context;
|
|
1629
|
+
// Currently getFreeMemory is not available on a runtime without server.
|
|
1630
|
+
// For now, report total memory as free.
|
|
1631
|
+
*free = dev_ctx->total_mem;
|
|
1632
|
+
*total = dev_ctx->total_mem;
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1635
|
+
static enum ggml_backend_dev_type ggml_backend_et_device_get_type(ggml_backend_dev_t dev) {
|
|
1636
|
+
GGML_UNUSED(dev);
|
|
1637
|
+
return GGML_BACKEND_DEVICE_TYPE_GPU;
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
static void ggml_backend_et_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) {
|
|
1641
|
+
GGML_UNUSED(dev);
|
|
1642
|
+
props->name = ggml_backend_et_device_get_name(dev);
|
|
1643
|
+
props->description = ggml_backend_et_device_get_description(dev);
|
|
1644
|
+
props->type = ggml_backend_et_device_get_type(dev);
|
|
1645
|
+
ggml_backend_et_device_get_memory(dev, &props->memory_free, &props->memory_total);
|
|
1646
|
+
props->device_id = NULL; // No PCI device ID available
|
|
1647
|
+
props->caps = {
|
|
1648
|
+
/* .async = */ true,
|
|
1649
|
+
/* .host_buffer = */ false,
|
|
1650
|
+
/* .buffer_from_host_ptr = */ false,
|
|
1651
|
+
/* .events = */ false,
|
|
1652
|
+
/* .mmap_support = */ true,
|
|
1653
|
+
};
|
|
1654
|
+
}
|
|
1655
|
+
|
|
1656
|
+
static ggml_backend_t ggml_backend_et_device_init_backend(ggml_backend_dev_t dev, const char * params) {
|
|
1657
|
+
GGML_UNUSED(params);
|
|
1658
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context;
|
|
1659
|
+
return ggml_backend_et_init(dev_ctx->devidx);
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
static ggml_backend_buffer_type_t ggml_backend_et_device_get_buffer_type(ggml_backend_dev_t dev) {
|
|
1663
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context;
|
|
1664
|
+
return dev_ctx->buftype;
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
static ggml_backend_buffer_type_t ggml_backend_et_device_get_host_buffer_type(ggml_backend_dev_t dev) {
|
|
1668
|
+
GGML_UNUSED(dev);
|
|
1669
|
+
return ggml_backend_cpu_buffer_type();
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
static const struct ggml_backend_device_i ggml_backend_et_device_i = {
|
|
1673
|
+
/* .get_name = */ ggml_backend_et_device_get_name,
|
|
1674
|
+
/* .get_description = */ ggml_backend_et_device_get_description,
|
|
1675
|
+
/* .get_memory = */ ggml_backend_et_device_get_memory,
|
|
1676
|
+
/* .get_type = */ ggml_backend_et_device_get_type,
|
|
1677
|
+
/* .get_props = */ ggml_backend_et_device_get_props,
|
|
1678
|
+
/* .init_backend = */ ggml_backend_et_device_init_backend,
|
|
1679
|
+
/* .get_buffer_type = */ ggml_backend_et_device_get_buffer_type,
|
|
1680
|
+
/* .get_host_buffer_type = */ ggml_backend_et_device_get_host_buffer_type,
|
|
1681
|
+
/* .buffer_from_host_ptr = */ NULL,
|
|
1682
|
+
/* .supports_op = */ ggml_backend_et_device_supports_op,
|
|
1683
|
+
/* .supports_buft = */ ggml_backend_et_device_supports_buft,
|
|
1684
|
+
/* .offload_op = */ ggml_backend_et_device_offload_op,
|
|
1685
|
+
/* .event_new = */ NULL,
|
|
1686
|
+
/* .event_free = */ NULL,
|
|
1687
|
+
/* .event_synchronize = */ NULL,
|
|
1688
|
+
};
|
|
1689
|
+
|
|
1690
|
+
/*
|
|
1691
|
+
Backend Registry.
|
|
1692
|
+
*/
|
|
1693
|
+
|
|
1694
|
+
static const char * ggml_backend_et_reg_get_name(ggml_backend_reg_t reg) {
|
|
1695
|
+
GGML_UNUSED(reg);
|
|
1696
|
+
return GGML_ET_NAME;
|
|
1697
|
+
}
|
|
1698
|
+
|
|
1699
|
+
static size_t ggml_backend_et_reg_get_device_count(ggml_backend_reg_t reg) {
|
|
1700
|
+
ggml_backend_et_reg_ctx * ctx = (ggml_backend_et_reg_ctx *) reg->context;
|
|
1701
|
+
return ctx->devices.size();
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1704
|
+
static ggml_backend_dev_t ggml_backend_et_reg_get_device(ggml_backend_reg_t reg, size_t devidx) {
|
|
1705
|
+
ggml_backend_et_reg_ctx * ctx = (ggml_backend_et_reg_ctx *) reg->context;
|
|
1706
|
+
if (devidx >= ctx->devices.size()) {
|
|
1707
|
+
return nullptr;
|
|
1708
|
+
}
|
|
1709
|
+
return ctx->devices[devidx];
|
|
1710
|
+
}
|
|
1711
|
+
|
|
1712
|
+
static void * ggml_backend_et_get_proc_address(ggml_backend_reg_t reg, const char * name) {
|
|
1713
|
+
GGML_UNUSED(reg);
|
|
1714
|
+
GGML_UNUSED(name);
|
|
1715
|
+
return nullptr;
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
static const struct ggml_backend_reg_i ggml_backend_et_reg_i = {
|
|
1719
|
+
/* .get_name = */ ggml_backend_et_reg_get_name,
|
|
1720
|
+
/* .get_device_count = */ ggml_backend_et_reg_get_device_count,
|
|
1721
|
+
/* .get_device = */ ggml_backend_et_reg_get_device,
|
|
1722
|
+
/* .get_proc_address = */ ggml_backend_et_get_proc_address,
|
|
1723
|
+
};
|
|
1724
|
+
|
|
1725
|
+
ggml_backend_reg_t ggml_backend_et_reg(void) {
|
|
1726
|
+
static ggml_backend_reg_t _reg = []() -> ggml_backend_reg_t {
|
|
1727
|
+
ggml_backend_et_reg_ctx * ctx = new ggml_backend_et_reg_ctx;
|
|
1728
|
+
|
|
1729
|
+
if (!ggml_et_driver_init()) {
|
|
1730
|
+
return nullptr;
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
ggml_backend_reg_t r = new ggml_backend_reg{
|
|
1734
|
+
/* .api_version = */ GGML_BACKEND_API_VERSION,
|
|
1735
|
+
/* .iface = */ ggml_backend_et_reg_i,
|
|
1736
|
+
/* .context = */ nullptr, // Set later
|
|
1737
|
+
};
|
|
1738
|
+
|
|
1739
|
+
std::vector<rt::DeviceId> rtids = ggml_et_runtime()->getDevices();
|
|
1740
|
+
|
|
1741
|
+
for (int i = 0; i < ggml_et_devicelayer()->getDevicesCount(); i++) {
|
|
1742
|
+
ggml_backend_dev_t dev = new ggml_backend_device{
|
|
1743
|
+
/* .iface = */ ggml_backend_et_device_i,
|
|
1744
|
+
/* .reg = */ r,
|
|
1745
|
+
/* .context = */ nullptr // Set later
|
|
1746
|
+
};
|
|
1747
|
+
|
|
1748
|
+
rt::DeviceId rtid = rtids[i];
|
|
1749
|
+
rt::DeviceProperties prop = ggml_et_runtime()->getDeviceProperties(rtid);
|
|
1750
|
+
|
|
1751
|
+
// Create device context.
|
|
1752
|
+
ggml_backend_et_device_context * dev_ctx = new ggml_backend_et_device_context;
|
|
1753
|
+
dev_ctx->devidx = i;
|
|
1754
|
+
dev_ctx->rtid = rtid;
|
|
1755
|
+
dev_ctx->name = GGML_ET_NAME + std::to_string(i);
|
|
1756
|
+
dev_ctx->desc = "ET device " + std::to_string(i);
|
|
1757
|
+
dev_ctx->total_mem = static_cast<size_t>(prop.memorySize_);
|
|
1758
|
+
{
|
|
1759
|
+
const char * env = getenv("GGML_ET_UBERKERNEL");
|
|
1760
|
+
dev_ctx->uberkernel_enabled = env && env[0] != '\0' && strcmp(env, "0") != 0;
|
|
1761
|
+
}
|
|
1762
|
+
// Add buffer type for device to device context.
|
|
1763
|
+
ggml_backend_et_buffer_type_context * bufty_ctx = new ggml_backend_et_buffer_type_context;
|
|
1764
|
+
bufty_ctx->devidx = i;
|
|
1765
|
+
bufty_ctx->name = GGML_ET_NAME + std::to_string(i);
|
|
1766
|
+
dev_ctx->buftype = new ggml_backend_buffer_type{ /* .iface = */ ggml_backend_et_buffer_type_i,
|
|
1767
|
+
/* .device = */ dev,
|
|
1768
|
+
/* .context = */ bufty_ctx };
|
|
1769
|
+
|
|
1770
|
+
// Create default stream for ordered execution on this device
|
|
1771
|
+
dev_ctx->default_stream = ggml_et_runtime()->createStream(rtid);
|
|
1772
|
+
|
|
1773
|
+
dev_ctx->trace_buffer = ggml_et_runtime()->mallocDevice(rtid, ET_TRACE_BUFFER_SIZE);
|
|
1774
|
+
// Pre-size each slot's host buffers and device-side scratch so the
|
|
1775
|
+
// first few graph_compute calls don't pay a malloc/grow penalty.
|
|
1776
|
+
for (auto & slot : dev_ctx->uberkernel.slots) {
|
|
1777
|
+
slot.insts.reserve(256);
|
|
1778
|
+
slot.params_blob.reserve(1 << 20);
|
|
1779
|
+
slot.device_insts_capacity = 256 * sizeof(ggml_et_uberkernel_inst);
|
|
1780
|
+
slot.device_params_capacity = 1 << 20;
|
|
1781
|
+
slot.device_insts = ggml_et_runtime()->mallocDevice(rtid, slot.device_insts_capacity);
|
|
1782
|
+
slot.device_params = ggml_et_runtime()->mallocDevice(rtid, slot.device_params_capacity);
|
|
1783
|
+
if (slot.device_insts == nullptr) {
|
|
1784
|
+
slot.device_insts_capacity = 0;
|
|
1785
|
+
}
|
|
1786
|
+
if (slot.device_params == nullptr) {
|
|
1787
|
+
slot.device_params_capacity = 0;
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
dev->context = dev_ctx;
|
|
1792
|
+
|
|
1793
|
+
ctx->devices.push_back(dev);
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
r->context = ctx;
|
|
1797
|
+
return r;
|
|
1798
|
+
}();
|
|
1799
|
+
|
|
1800
|
+
return _reg;
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
ggml_guid_t ggml_backend_et_guid(void) {
|
|
1804
|
+
static ggml_guid guid = { 0x4b, 0xe0, 0x72, 0x88, 0xc0, 0xf6, 0x29, 0xb4,
|
|
1805
|
+
0x79, 0x9f, 0x70, 0x68, 0x71, 0x0f, 0x6d, 0xc8 };
|
|
1806
|
+
return &guid;
|
|
1807
|
+
}
|
|
1808
|
+
|
|
1809
|
+
ggml_backend_t ggml_backend_et_init(size_t devidx) {
|
|
1810
|
+
if (!ggml_et_driver_init()) {
|
|
1811
|
+
return nullptr;
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
if (devidx >= (size_t) ggml_backend_et_get_device_count()) {
|
|
1815
|
+
return nullptr;
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
ggml_backend_et_context * ctx = new ggml_backend_et_context;
|
|
1819
|
+
ctx->devidx = (int) devidx;
|
|
1820
|
+
|
|
1821
|
+
ggml_backend_t backend = new ggml_backend{
|
|
1822
|
+
/* .guid = */ ggml_backend_et_guid(),
|
|
1823
|
+
/* .iface = */ ggml_backend_et_i,
|
|
1824
|
+
/* .device = */ ggml_backend_et_reg_get_device(ggml_backend_et_reg(), devidx),
|
|
1825
|
+
/* .context = */ ctx,
|
|
1826
|
+
};
|
|
1827
|
+
|
|
1828
|
+
return backend;
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1831
|
+
bool ggml_backend_is_et(ggml_backend_t backend) {
|
|
1832
|
+
return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_et_guid());
|
|
1833
|
+
}
|
|
1834
|
+
|
|
1835
|
+
int ggml_backend_et_get_device_count(void) {
|
|
1836
|
+
return ggml_backend_et_reg_get_device_count(ggml_backend_et_reg());
|
|
1837
|
+
}
|
|
1838
|
+
|
|
1839
|
+
void ggml_backend_et_get_device_description(int devidx, char * description, size_t description_size) {
|
|
1840
|
+
if (devidx < 0 || devidx >= ggml_backend_et_get_device_count()) {
|
|
1841
|
+
snprintf(description, description_size, "ET Device %d (invalid)", devidx);
|
|
1842
|
+
return;
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
ggml_backend_dev_t dev = ggml_backend_et_reg_get_device(ggml_backend_et_reg(), devidx);
|
|
1846
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context;
|
|
1847
|
+
snprintf(description, description_size, "%s", dev_ctx->desc.c_str());
|
|
1848
|
+
}
|
|
1849
|
+
|
|
1850
|
+
void ggml_backend_et_get_device_memory(int devidx, size_t * free, size_t * total) {
|
|
1851
|
+
if (devidx < 0 || devidx >= ggml_backend_et_get_device_count()) {
|
|
1852
|
+
*free = 0;
|
|
1853
|
+
*total = 0;
|
|
1854
|
+
return;
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
ggml_backend_dev_t dev = ggml_backend_et_reg_get_device(ggml_backend_et_reg(), devidx);
|
|
1858
|
+
ggml_backend_et_device_get_memory(dev, free, total);
|
|
1859
|
+
}
|
|
1860
|
+
|
|
1861
|
+
ggml_backend_buffer_type_t ggml_backend_et_buffer_type(size_t dev_num) {
|
|
1862
|
+
if (dev_num >= (size_t) ggml_backend_et_get_device_count()) {
|
|
1863
|
+
return nullptr;
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1866
|
+
ggml_backend_dev_t dev = ggml_backend_et_reg_get_device(ggml_backend_et_reg(), dev_num);
|
|
1867
|
+
ggml_backend_et_device_context * dev_ctx = (ggml_backend_et_device_context *) dev->context;
|
|
1868
|
+
return dev_ctx->buftype;
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
ggml_backend_buffer_type_t ggml_backend_et_host_buffer_type(void) {
|
|
1872
|
+
static ggml_backend_buffer_type host_buffer_type = {
|
|
1873
|
+
/* .iface = */ ggml_backend_et_buffer_type_i,
|
|
1874
|
+
/* .device = */ nullptr,
|
|
1875
|
+
/* .context = */ nullptr,
|
|
1876
|
+
};
|
|
1877
|
+
return &host_buffer_type;
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1880
|
+
GGML_BACKEND_DL_IMPL(ggml_backend_et_reg)
|