elelem-ggml 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/exe/elelem-ggml +8 -0
- data/ext/elelem/ggml/elelem.c +466 -0
- data/ext/elelem/ggml/extconf.rb +52 -0
- data/lib/elelem/ggml/chat_ml.rb +23 -0
- data/lib/elelem/ggml/gguf.rb +82 -0
- data/lib/elelem/ggml/provider.rb +63 -0
- data/lib/elelem/ggml/tokenizer.rb +95 -0
- data/lib/elelem/ggml/version.rb +7 -0
- data/lib/elelem/ggml.rb +37 -0
- data/vendor/ggml/AUTHORS +335 -0
- data/vendor/ggml/CMakeLists.txt +499 -0
- data/vendor/ggml/CONTRIBUTING.md +3 -0
- data/vendor/ggml/LICENSE +21 -0
- data/vendor/ggml/README.md +49 -0
- data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
- data/vendor/ggml/cmake/GitVars.cmake +22 -0
- data/vendor/ggml/cmake/common.cmake +50 -0
- data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
- data/vendor/ggml/ggml.pc.in +10 -0
- data/vendor/ggml/include/ggml-alloc.h +86 -0
- data/vendor/ggml/include/ggml-backend.h +437 -0
- data/vendor/ggml/include/ggml-blas.h +25 -0
- data/vendor/ggml/include/ggml-cann.h +123 -0
- data/vendor/ggml/include/ggml-cpp.h +39 -0
- data/vendor/ggml/include/ggml-cpu.h +152 -0
- data/vendor/ggml/include/ggml-cuda.h +47 -0
- data/vendor/ggml/include/ggml-et.h +28 -0
- data/vendor/ggml/include/ggml-hexagon.h +19 -0
- data/vendor/ggml/include/ggml-metal.h +61 -0
- data/vendor/ggml/include/ggml-opencl.h +26 -0
- data/vendor/ggml/include/ggml-openvino.h +37 -0
- data/vendor/ggml/include/ggml-opt.h +256 -0
- data/vendor/ggml/include/ggml-rpc.h +35 -0
- data/vendor/ggml/include/ggml-sycl.h +57 -0
- data/vendor/ggml/include/ggml-virtgpu.h +14 -0
- data/vendor/ggml/include/ggml-vulkan.h +29 -0
- data/vendor/ggml/include/ggml-webgpu.h +19 -0
- data/vendor/ggml/include/ggml-zdnn.h +17 -0
- data/vendor/ggml/include/ggml-zendnn.h +22 -0
- data/vendor/ggml/include/ggml.h +2963 -0
- data/vendor/ggml/include/gguf.h +211 -0
- data/vendor/ggml/requirements.txt +12 -0
- data/vendor/ggml/scripts/gen-authors.sh +9 -0
- data/vendor/ggml/scripts/make-release-checks.sh +96 -0
- data/vendor/ggml/scripts/make-release-desc.sh +69 -0
- data/vendor/ggml/scripts/make-release-summary.txt +33 -0
- data/vendor/ggml/scripts/release.sh +195 -0
- data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
- data/vendor/ggml/scripts/sync-llama.last +1 -0
- data/vendor/ggml/scripts/sync-llama.sh +21 -0
- data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
- data/vendor/ggml/scripts/sync-whisper.last +1 -0
- data/vendor/ggml/scripts/sync-whisper.sh +17 -0
- data/vendor/ggml/src/CMakeLists.txt +634 -0
- data/vendor/ggml/src/ggml-alloc.c +1249 -0
- data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
- data/vendor/ggml/src/ggml-backend-dl.h +45 -0
- data/vendor/ggml/src/ggml-backend-impl.h +291 -0
- data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
- data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
- data/vendor/ggml/src/ggml-backend.cpp +2517 -0
- data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
- data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
- data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
- data/vendor/ggml/src/ggml-cann/common.h +651 -0
- data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
- data/vendor/ggml/src/ggml-common.h +1911 -0
- data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
- data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
- data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
- data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
- data/vendor/ggml/src/ggml-cpu/common.h +95 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
- data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
- data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
- data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
- data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
- data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
- data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
- data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
- data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
- data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
- data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
- data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
- data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
- data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
- data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
- data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
- data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
- data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
- data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
- data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
- data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
- data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
- data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
- data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
- data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
- data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
- data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
- data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
- data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
- data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
- data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
- data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
- data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
- data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
- data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
- data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
- data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
- data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
- data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
- data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
- data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
- data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
- data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
- data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
- data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
- data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
- data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
- data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
- data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
- data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
- data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
- data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
- data/vendor/ggml/src/ggml-feats.h +166 -0
- data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
- data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
- data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
- data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
- data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
- data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
- data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
- data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
- data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
- data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
- data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
- data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
- data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
- data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
- data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
- data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
- data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
- data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
- data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
- data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
- data/vendor/ggml/src/ggml-impl.h +783 -0
- data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
- data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
- data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
- data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
- data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
- data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
- data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
- data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
- data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
- data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
- data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
- data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
- data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
- data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
- data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
- data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
- data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
- data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
- data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
- data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
- data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
- data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
- data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
- data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
- data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
- data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
- data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
- data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
- data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
- data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
- data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
- data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
- data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
- data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
- data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
- data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
- data/vendor/ggml/src/ggml-opt.cpp +1094 -0
- data/vendor/ggml/src/ggml-quants.c +5667 -0
- data/vendor/ggml/src/ggml-quants.h +115 -0
- data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
- data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
- data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
- data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
- data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
- data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
- data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
- data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
- data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
- data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
- data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
- data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
- data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
- data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
- data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
- data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
- data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
- data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
- data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
- data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
- data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
- data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
- data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
- data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
- data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
- data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
- data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
- data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
- data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
- data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
- data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
- data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
- data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
- data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
- data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
- data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
- data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
- data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
- data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
- data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
- data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
- data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
- data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
- data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
- data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
- data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
- data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
- data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
- data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
- data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/vendor/ggml/src/ggml-threading.cpp +12 -0
- data/vendor/ggml/src/ggml-threading.h +14 -0
- data/vendor/ggml/src/ggml-version.h.in +4 -0
- data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
- data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
- data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
- data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
- data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
- data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
- data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
- data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
- data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
- data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
- data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
- data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
- data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
- data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
- data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
- data/vendor/ggml/src/ggml.c +8088 -0
- data/vendor/ggml/src/ggml.cpp +26 -0
- data/vendor/ggml/src/gguf.cpp +1706 -0
- metadata +1459 -0
|
@@ -0,0 +1,2584 @@
|
|
|
1
|
+
#include "ggml-et-ops.h"
|
|
2
|
+
|
|
3
|
+
#include "ggml-et-cpu-compare.h"
|
|
4
|
+
#include "ggml-et-kernels.h"
|
|
5
|
+
#include "ggml-impl.h"
|
|
6
|
+
|
|
7
|
+
#include <stdio.h>
|
|
8
|
+
|
|
9
|
+
#include <cstdint>
|
|
10
|
+
|
|
11
|
+
// CPU comparison configuration - can be enabled for debugging
|
|
12
|
+
static ggml_et_cpu_compare_config rope_cpu_compare_config = {
|
|
13
|
+
/* .enabled = */ false,
|
|
14
|
+
/* .use_cpu_result = */ false, // Replace ET result with CPU result
|
|
15
|
+
/* .log_differences = */ true,
|
|
16
|
+
/* .tolerance = */ 1e-5f,
|
|
17
|
+
/* .max_log_elements = */ 4096
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
static ggml_et_cpu_compare_config rms_norm_cpu_compare_config = {
|
|
21
|
+
/* .enabled = */ false,
|
|
22
|
+
/* .use_cpu_result = */ false,
|
|
23
|
+
/* .log_differences = */ true,
|
|
24
|
+
/* .tolerance = */ 1e-5f,
|
|
25
|
+
/* .max_log_elements = */ 4096
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
static ggml_et_cpu_compare_config norm_cpu_compare_config = {
|
|
29
|
+
/* .enabled = */ false,
|
|
30
|
+
/* .use_cpu_result = */ false,
|
|
31
|
+
/* .log_differences = */ true,
|
|
32
|
+
/* .tolerance = */ 1e-5f,
|
|
33
|
+
/* .max_log_elements = */ 4096
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
static ggml_et_cpu_compare_config l2_norm_cpu_compare_config = {
|
|
37
|
+
/* .enabled = */ false,
|
|
38
|
+
/* .use_cpu_result = */ false,
|
|
39
|
+
/* .log_differences = */ true,
|
|
40
|
+
/* .tolerance = */ 1e-5f,
|
|
41
|
+
/* .max_log_elements = */ 4096
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
static ggml_et_cpu_compare_config group_norm_cpu_compare_config = {
|
|
45
|
+
/* .enabled = */ false,
|
|
46
|
+
/* .use_cpu_result = */ false,
|
|
47
|
+
/* .log_differences = */ true,
|
|
48
|
+
/* .tolerance = */ 1e-5f,
|
|
49
|
+
/* .max_log_elements = */ 4096
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
static ggml_et_cpu_compare_config im2col_cpu_compare_config = {
|
|
53
|
+
/* .enabled = */ false,
|
|
54
|
+
/* .use_cpu_result = */ false,
|
|
55
|
+
/* .log_differences = */ true,
|
|
56
|
+
/* .tolerance = */ 1e-5f,
|
|
57
|
+
/* .max_log_elements = */ 4096
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
static ggml_et_cpu_compare_config unary_cpu_compare_config = {
|
|
61
|
+
/* .enabled = */ false,
|
|
62
|
+
/* .use_cpu_result = */ false,
|
|
63
|
+
/* .log_differences = */ true,
|
|
64
|
+
/* .tolerance = */ 1e-4f,
|
|
65
|
+
/* .max_log_elements = */ 4096
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
static ggml_et_cpu_compare_config sum_rows_cpu_compare_config = {
|
|
69
|
+
/* .enabled = */ false,
|
|
70
|
+
/* .use_cpu_result = */ false,
|
|
71
|
+
/* .log_differences = */ true,
|
|
72
|
+
/* .tolerance = */ 1e-5f,
|
|
73
|
+
/* .max_log_elements = */ 4096
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
static ggml_et_cpu_compare_config clamp_cpu_compare_config = {
|
|
77
|
+
/* .enabled = */ false,
|
|
78
|
+
/* .use_cpu_result = */ false,
|
|
79
|
+
/* .log_differences = */ true,
|
|
80
|
+
/* .tolerance = */ 1e-6f,
|
|
81
|
+
/* .max_log_elements = */ 4096
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
static ggml_et_cpu_compare_config mean_cpu_compare_config = {
|
|
85
|
+
/* .enabled = */ false,
|
|
86
|
+
/* .use_cpu_result = */ false,
|
|
87
|
+
/* .log_differences = */ true,
|
|
88
|
+
/* .tolerance = */ 1e-5f,
|
|
89
|
+
/* .max_log_elements = */ 4096
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
static ggml_et_cpu_compare_config sqr_cpu_compare_config = {
|
|
93
|
+
/* .enabled = */ false,
|
|
94
|
+
/* .use_cpu_result = */ false,
|
|
95
|
+
/* .log_differences = */ true,
|
|
96
|
+
/* .tolerance = */ 1e-6f,
|
|
97
|
+
/* .max_log_elements = */ 4096
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
static ggml_et_cpu_compare_config elmap_cpu_compare_config = {
|
|
101
|
+
/* .enabled = */ false,
|
|
102
|
+
/* .use_cpu_result = */ false,
|
|
103
|
+
/* .log_differences = */ true,
|
|
104
|
+
/* .tolerance = */ 1e-6f,
|
|
105
|
+
/* .max_log_elements = */ 4096
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
static ggml_et_cpu_compare_config glu_cpu_compare_config = {
|
|
109
|
+
/* .enabled = */ false,
|
|
110
|
+
/* .use_cpu_result = */ false,
|
|
111
|
+
/* .log_differences = */ true,
|
|
112
|
+
/* .tolerance = */ 1e-5f,
|
|
113
|
+
/* .max_log_elements = */ 4096
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
static ggml_et_cpu_compare_config mul_mat_cpu_compare_config = {
|
|
117
|
+
/* .enabled = */ false,
|
|
118
|
+
/* .use_cpu_result = */ false,
|
|
119
|
+
/* .log_differences = */ true,
|
|
120
|
+
/* .tolerance = */ 0.01,
|
|
121
|
+
/* .max_log_elements = */ 4096
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
static ggml_et_cpu_compare_config mul_mat_id_cpu_compare_config = {
|
|
125
|
+
/* .enabled = */ false,
|
|
126
|
+
/* .use_cpu_result = */ false,
|
|
127
|
+
/* .log_differences = */ true,
|
|
128
|
+
/* .tolerance = */ 0.01,
|
|
129
|
+
/* .max_log_elements = */ 4096
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
static ggml_et_cpu_compare_config softmax_cpu_compare_config = {
|
|
133
|
+
/* .enabled = */ false,
|
|
134
|
+
/* .use_cpu_result = */ false,
|
|
135
|
+
/* .log_differences = */ true,
|
|
136
|
+
/* .tolerance = */ 1e-5f,
|
|
137
|
+
/* .max_log_elements = */ 1024
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
static ggml_et_cpu_compare_config get_rows_cpu_compare_config = {
|
|
141
|
+
/* .enabled = */ false,
|
|
142
|
+
/* .use_cpu_result = */ false,
|
|
143
|
+
/* .log_differences = */ true,
|
|
144
|
+
/* .tolerance = */ 1e-6f,
|
|
145
|
+
/* .max_log_elements = */ 2048
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
static ggml_et_cpu_compare_config pad_cpu_compare_config = {
|
|
149
|
+
/* .enabled = */ false,
|
|
150
|
+
/* .use_cpu_result = */ false,
|
|
151
|
+
/* .log_differences = */ true,
|
|
152
|
+
/* .tolerance = */ 1e-6f,
|
|
153
|
+
/* .max_log_elements = */ 4096
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
static ggml_et_cpu_compare_config cont_cpu_compare_config = {
|
|
157
|
+
/* .enabled = */ false,
|
|
158
|
+
/* .use_cpu_result = */ false,
|
|
159
|
+
/* .log_differences = */ true,
|
|
160
|
+
/* .tolerance = */ 1e-6f,
|
|
161
|
+
/* .max_log_elements = */ 4096
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
static ggml_et_cpu_compare_config concat_cpu_compare_config = {
|
|
165
|
+
/* .enabled = */ false,
|
|
166
|
+
/* .use_cpu_result = */ false,
|
|
167
|
+
/* .log_differences = */ true,
|
|
168
|
+
/* .tolerance = */ 1e-6f,
|
|
169
|
+
/* .max_log_elements = */ 4096
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
static ggml_et_cpu_compare_config cumsum_cpu_compare_config = {
|
|
173
|
+
/* .enabled = */ false,
|
|
174
|
+
/* .use_cpu_result = */ false,
|
|
175
|
+
/* .log_differences = */ true,
|
|
176
|
+
/* .tolerance = */ 1e-6f,
|
|
177
|
+
/* .max_log_elements = */ 4096
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
static ggml_et_cpu_compare_config repeat_cpu_compare_config = {
|
|
181
|
+
/* .enabled = */ false,
|
|
182
|
+
/* .use_cpu_result = */ false,
|
|
183
|
+
/* .log_differences = */ true,
|
|
184
|
+
/* .tolerance = */ 1e-6f,
|
|
185
|
+
/* .max_log_elements = */ 4096
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
static ggml_et_cpu_compare_config ssm_conv_cpu_compare_config = {
|
|
189
|
+
/* .enabled = */ false,
|
|
190
|
+
/* .use_cpu_result = */ false,
|
|
191
|
+
/* .log_differences = */ true,
|
|
192
|
+
/* .tolerance = */ 1e-6f,
|
|
193
|
+
/* .max_log_elements = */ 4096
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
static ggml_et_cpu_compare_config rwkv_wkv6_cpu_compare_config = {
|
|
197
|
+
/* .enabled = */ false,
|
|
198
|
+
/* .use_cpu_result = */ false,
|
|
199
|
+
/* .log_differences = */ true,
|
|
200
|
+
/* .tolerance = */ 1e-4f,
|
|
201
|
+
/* .max_log_elements = */ 4096
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
static ggml_et_cpu_compare_config rwkv_wkv7_cpu_compare_config = {
|
|
205
|
+
/* .enabled = */ false,
|
|
206
|
+
/* .use_cpu_result = */ false,
|
|
207
|
+
/* .log_differences = */ true,
|
|
208
|
+
/* .tolerance = */ 1e-4f,
|
|
209
|
+
/* .max_log_elements = */ 4096
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
static ggml_et_cpu_compare_config set_rows_cpu_compare_config = {
|
|
213
|
+
/* .enabled = */ false,
|
|
214
|
+
/* .use_cpu_result = */ false,
|
|
215
|
+
/* .log_differences = */ true,
|
|
216
|
+
/* .tolerance = */ 1e-6f,
|
|
217
|
+
/* .max_log_elements = */ 2048
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
bool ggml_et_op_rms_norm_mul(ggml_backend_et_device_context * dev_ctx,
|
|
221
|
+
const ggml_tensor * rms_norm_node,
|
|
222
|
+
const ggml_tensor * mul_node) {
|
|
223
|
+
ET_PERF_START();
|
|
224
|
+
|
|
225
|
+
if (!dev_ctx || !rms_norm_node || !mul_node) {
|
|
226
|
+
GGML_LOG_ERROR("ET: Invalid parameters for fused RMS_NORM_MUL operation\n");
|
|
227
|
+
return false;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
if (!rms_norm_node->src[0]) {
|
|
231
|
+
GGML_LOG_ERROR("ET: Fused RMS_NORM_MUL missing required input\n");
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// Extract weights: the MUL operand that isn't the rms_norm output
|
|
236
|
+
const ggml_tensor * weights = (mul_node->src[0] == rms_norm_node) ? mul_node->src[1] : mul_node->src[0];
|
|
237
|
+
|
|
238
|
+
if (!weights) {
|
|
239
|
+
GGML_LOG_ERROR("ET: Fused RMS_NORM_MUL missing weights tensor\n");
|
|
240
|
+
return false;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
float eps;
|
|
244
|
+
memcpy(&eps, rms_norm_node->op_params, sizeof(float));
|
|
245
|
+
|
|
246
|
+
ggml_et_rms_norm_mul_params params;
|
|
247
|
+
params.src0 = *rms_norm_node->src[0]; // input to normalize
|
|
248
|
+
params.src1 = *weights; // normalization weights
|
|
249
|
+
params.dst = *mul_node; // final output
|
|
250
|
+
params.eps = eps;
|
|
251
|
+
|
|
252
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "rms_norm_mul_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
253
|
+
|
|
254
|
+
ET_PERF_END_EXT("RMS_NORM_MUL", "rms_norm_mul_f32", mul_node, "eps=%.6f", (double) eps);
|
|
255
|
+
return kernel_result;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
bool ggml_et_op_scale(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
259
|
+
ET_PERF_START();
|
|
260
|
+
|
|
261
|
+
if (!dev_ctx || !node) {
|
|
262
|
+
GGML_LOG_ERROR("ET: Invalid parameters for SCALE operation\n");
|
|
263
|
+
return false;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
if (!node->src[0]) {
|
|
267
|
+
GGML_LOG_ERROR("ET: SCALE operation missing required input\n");
|
|
268
|
+
return false;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
|
|
272
|
+
GGML_LOG_ERROR("ET: SCALE operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
273
|
+
ggml_type_name(node->src[0]->type));
|
|
274
|
+
return false;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
float scale, bias;
|
|
278
|
+
memcpy(&scale, (const float *) node->op_params + 0, sizeof(float));
|
|
279
|
+
memcpy(&bias, (const float *) node->op_params + 1, sizeof(float));
|
|
280
|
+
|
|
281
|
+
ggml_et_scale_params params;
|
|
282
|
+
params.src0 = *node->src[0];
|
|
283
|
+
params.dst = *node;
|
|
284
|
+
params.scale = scale;
|
|
285
|
+
params.bias = bias;
|
|
286
|
+
|
|
287
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "scale_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
288
|
+
|
|
289
|
+
ET_PERF_END_EXT("SCALE", "scale_f32", node, "scale=%.6f|bias=%.6f", (double) scale, (double) bias);
|
|
290
|
+
return kernel_result;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
bool ggml_et_op_sqr(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
294
|
+
ET_PERF_START();
|
|
295
|
+
|
|
296
|
+
if (!dev_ctx || !node) {
|
|
297
|
+
GGML_LOG_ERROR("ET: Invalid parameters for SQR operation\n");
|
|
298
|
+
return false;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
if (!node->src[0]) {
|
|
302
|
+
GGML_LOG_ERROR("ET: SQR operation missing required input\n");
|
|
303
|
+
return false;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
|
|
307
|
+
GGML_LOG_ERROR("ET: SQR operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
308
|
+
ggml_type_name(node->src[0]->type));
|
|
309
|
+
return false;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
ggml_et_sqr_params params;
|
|
313
|
+
params.src0 = *node->src[0]; // F32 input tensor
|
|
314
|
+
params.dst = *node; // F32 output tensor
|
|
315
|
+
|
|
316
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
317
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
318
|
+
bool cpu_comparison_active = false;
|
|
319
|
+
if (sqr_cpu_compare_config.enabled) {
|
|
320
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SQR)) {
|
|
321
|
+
cpu_comparison_active = true;
|
|
322
|
+
} else {
|
|
323
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SQR operation\n");
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "sqr_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
328
|
+
|
|
329
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
330
|
+
if (cpu_comparison_active) {
|
|
331
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &sqr_cpu_compare_config)) {
|
|
332
|
+
GGML_LOG_WARN("ET: CPU comparison failed for SQR operation\n");
|
|
333
|
+
}
|
|
334
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
ET_PERF_END("SQR", "sqr_f32", node);
|
|
338
|
+
return kernel_result;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
bool ggml_et_op_sum_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
342
|
+
ET_PERF_START();
|
|
343
|
+
|
|
344
|
+
if (!dev_ctx || !node) {
|
|
345
|
+
GGML_LOG_ERROR("ET: Invalid parameters for SUM_ROWS operation\n");
|
|
346
|
+
return false;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
if (!node->src[0]) {
|
|
350
|
+
GGML_LOG_ERROR("ET: SUM_ROWS operation missing required input\n");
|
|
351
|
+
return false;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
|
|
355
|
+
GGML_LOG_ERROR("ET: SUM_ROWS operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
356
|
+
ggml_type_name(node->src[0]->type));
|
|
357
|
+
return false;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
ggml_et_sum_rows_params params;
|
|
361
|
+
params.src0 = *node->src[0];
|
|
362
|
+
params.dst = *node;
|
|
363
|
+
|
|
364
|
+
// Phase 1: Initialize CPU comparison context
|
|
365
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
366
|
+
bool cpu_comparison_active = false;
|
|
367
|
+
if (sum_rows_cpu_compare_config.enabled) {
|
|
368
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SUM_ROWS)) {
|
|
369
|
+
cpu_comparison_active = true;
|
|
370
|
+
} else {
|
|
371
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SUM_ROWS operation\n");
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "sum_rows_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
376
|
+
|
|
377
|
+
// Phase 2: Execute CPU computation and compare
|
|
378
|
+
if (cpu_comparison_active) {
|
|
379
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &sum_rows_cpu_compare_config)) {
|
|
380
|
+
GGML_LOG_WARN("ET: CPU comparison failed for SUM_ROWS operation\n");
|
|
381
|
+
}
|
|
382
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
ET_PERF_END("SUM_ROWS", "sum_rows_f32", node);
|
|
386
|
+
return kernel_result;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
bool ggml_et_op_mean(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
390
|
+
ET_PERF_START();
|
|
391
|
+
|
|
392
|
+
if (!dev_ctx || !node) {
|
|
393
|
+
GGML_LOG_ERROR("ET: Invalid parameters for MEAN operation\n");
|
|
394
|
+
return false;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
if (!node->src[0]) {
|
|
398
|
+
GGML_LOG_ERROR("ET: MEAN operation missing required input\n");
|
|
399
|
+
return false;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
|
|
403
|
+
GGML_LOG_ERROR("ET: MEAN operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
404
|
+
ggml_type_name(node->src[0]->type));
|
|
405
|
+
return false;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
ggml_et_mean_params params;
|
|
409
|
+
params.src0 = *node->src[0];
|
|
410
|
+
params.dst = *node;
|
|
411
|
+
|
|
412
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
413
|
+
bool cpu_comparison_active = false;
|
|
414
|
+
if (mean_cpu_compare_config.enabled) {
|
|
415
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_MEAN)) {
|
|
416
|
+
cpu_comparison_active = true;
|
|
417
|
+
} else {
|
|
418
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for MEAN operation\n");
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "mean_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
423
|
+
|
|
424
|
+
if (cpu_comparison_active) {
|
|
425
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &mean_cpu_compare_config)) {
|
|
426
|
+
GGML_LOG_WARN("ET: CPU comparison failed for MEAN operation\n");
|
|
427
|
+
}
|
|
428
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
ET_PERF_END("MEAN", "mean_f32", node);
|
|
432
|
+
return kernel_result;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
bool ggml_et_op_clamp(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
436
|
+
ET_PERF_START();
|
|
437
|
+
|
|
438
|
+
if (!dev_ctx || !node) {
|
|
439
|
+
GGML_LOG_ERROR("ET: Invalid parameters for CLAMP operation\n");
|
|
440
|
+
return false;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
if (!node->src[0]) {
|
|
444
|
+
GGML_LOG_ERROR("ET: CLAMP operation missing required input\n");
|
|
445
|
+
return false;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
|
|
449
|
+
GGML_LOG_ERROR("ET: CLAMP operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
450
|
+
ggml_type_name(node->src[0]->type));
|
|
451
|
+
return false;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
ggml_et_clamp_params params;
|
|
455
|
+
params.src0 = *node->src[0];
|
|
456
|
+
params.dst = *node;
|
|
457
|
+
// op_params layout per ggml.c::ggml_clamp: { min, max } as floats
|
|
458
|
+
memcpy(¶ms.min_val, (const float *) node->op_params + 0, sizeof(float));
|
|
459
|
+
memcpy(¶ms.max_val, (const float *) node->op_params + 1, sizeof(float));
|
|
460
|
+
|
|
461
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
462
|
+
bool cpu_comparison_active = false;
|
|
463
|
+
if (clamp_cpu_compare_config.enabled) {
|
|
464
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CLAMP)) {
|
|
465
|
+
cpu_comparison_active = true;
|
|
466
|
+
} else {
|
|
467
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CLAMP operation\n");
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "clamp_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
472
|
+
|
|
473
|
+
if (cpu_comparison_active) {
|
|
474
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &clamp_cpu_compare_config)) {
|
|
475
|
+
GGML_LOG_WARN("ET: CPU comparison failed for CLAMP operation\n");
|
|
476
|
+
}
|
|
477
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
ET_PERF_END("CLAMP", "clamp_f32", node);
|
|
481
|
+
return kernel_result;
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
bool ggml_et_op_unary(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
485
|
+
ET_PERF_START();
|
|
486
|
+
|
|
487
|
+
if (!dev_ctx || !node) {
|
|
488
|
+
GGML_LOG_ERROR("ET: Invalid parameters for UNARY operation\n");
|
|
489
|
+
return false;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
if (!node->src[0]) {
|
|
493
|
+
GGML_LOG_ERROR("ET: UNARY operation missing required input\n");
|
|
494
|
+
return false;
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
|
|
498
|
+
GGML_LOG_ERROR("ET: UNARY operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
499
|
+
ggml_type_name(node->src[0]->type));
|
|
500
|
+
return false;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
const ggml_unary_op uop = ggml_get_unary_op(node);
|
|
504
|
+
const char * op_name = ggml_unary_op_name(uop);
|
|
505
|
+
|
|
506
|
+
ggml_et_unary_params params;
|
|
507
|
+
params.src0 = *node->src[0]; // F32 input tensor
|
|
508
|
+
params.dst = *node; // F32 output tensor
|
|
509
|
+
params.unary_op = (int32_t) uop;
|
|
510
|
+
|
|
511
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
512
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
513
|
+
bool cpu_comparison_active = false;
|
|
514
|
+
if (unary_cpu_compare_config.enabled) {
|
|
515
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_UNARY)) {
|
|
516
|
+
cpu_comparison_active = true;
|
|
517
|
+
} else {
|
|
518
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for UNARY/%s operation\n", op_name);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "unary_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
523
|
+
|
|
524
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
525
|
+
if (cpu_comparison_active) {
|
|
526
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &unary_cpu_compare_config)) {
|
|
527
|
+
GGML_LOG_WARN("ET: CPU comparison failed for UNARY/%s operation\n", op_name);
|
|
528
|
+
}
|
|
529
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
ET_PERF_END_EXT("UNARY", "unary_f32", node, "op=%s", op_name);
|
|
533
|
+
return kernel_result;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
bool ggml_et_op_mul(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
537
|
+
// Delegate to generic element map operation
|
|
538
|
+
return ggml_et_op_elmap(dev_ctx, node);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
bool ggml_et_op_add(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
542
|
+
// Delegate to generic element map operation
|
|
543
|
+
return ggml_et_op_elmap(dev_ctx, node);
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
bool ggml_et_op_sub(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
547
|
+
// Delegate to generic element map operation
|
|
548
|
+
return ggml_et_op_elmap(dev_ctx, node);
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
bool ggml_et_op_elmap(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
552
|
+
ET_PERF_START();
|
|
553
|
+
|
|
554
|
+
if (!dev_ctx || !node) {
|
|
555
|
+
GGML_LOG_ERROR("ET: Invalid parameters for element map operation\n");
|
|
556
|
+
return false;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
if (!node->src[0] || !node->src[1]) {
|
|
560
|
+
GGML_LOG_ERROR("ET: Element map operation missing required inputs\n");
|
|
561
|
+
return false;
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32) {
|
|
565
|
+
GGML_LOG_ERROR("ET: Element map operation with unsupported types: dst=%s src0=%s src1=%s\n",
|
|
566
|
+
ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
|
|
567
|
+
ggml_type_name(node->src[1]->type));
|
|
568
|
+
return false;
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
const char * op_name = ggml_op_name(node->op);
|
|
572
|
+
|
|
573
|
+
ggml_et_elmap_params params;
|
|
574
|
+
params.src0 = *node->src[0];
|
|
575
|
+
params.src1 = *node->src[1];
|
|
576
|
+
params.dst = *node; // F32 output tensor (op type stored in dst.op)
|
|
577
|
+
|
|
578
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
579
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
580
|
+
bool cpu_comparison_active = false;
|
|
581
|
+
if (elmap_cpu_compare_config.enabled) {
|
|
582
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, node->op)) {
|
|
583
|
+
cpu_comparison_active = true;
|
|
584
|
+
} else {
|
|
585
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for %s operation\n", op_name);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
// fprintf(stderr, "ET: el_map s0 [%ld, %ld, %ld, %ld] s1 [%ld, %ld, %ld, %ld]\n",
|
|
590
|
+
// node->src[0]->ne[0], node->src[0]->ne[1], node->src[0]->ne[2], node->src[0]->ne[3],
|
|
591
|
+
// node->src[1]->ne[0], node->src[1]->ne[1], node->src[1]->ne[2], node->src[1]->ne[3]);
|
|
592
|
+
|
|
593
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "el_map_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
594
|
+
|
|
595
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
596
|
+
if (cpu_comparison_active) {
|
|
597
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &elmap_cpu_compare_config)) {
|
|
598
|
+
GGML_LOG_WARN("ET: CPU comparison failed for %s operation\n", op_name);
|
|
599
|
+
}
|
|
600
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
ET_PERF_END(op_name, "el_map_f32", node);
|
|
604
|
+
return kernel_result;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
bool ggml_et_op_glu(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
608
|
+
ET_PERF_START();
|
|
609
|
+
|
|
610
|
+
// Validate inputs
|
|
611
|
+
if (!dev_ctx || !node) {
|
|
612
|
+
GGML_LOG_ERROR("ET: Invalid parameters for GLU operation\n");
|
|
613
|
+
return false;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
if (!node->src[0]) {
|
|
617
|
+
GGML_LOG_ERROR("ET: GLU operation missing required input\n");
|
|
618
|
+
return false;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
const bool is_split_mode = node->src[1] != nullptr;
|
|
622
|
+
|
|
623
|
+
// Only support F32 (as validated by supports_op)
|
|
624
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 ||
|
|
625
|
+
(is_split_mode && node->src[1]->type != GGML_TYPE_F32)) {
|
|
626
|
+
return false;
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
// Extract GLU operation parameters from op_params
|
|
630
|
+
int32_t glu_op_type = ggml_get_op_params_i32(node, 0); // GLU variant (REGLU, GEGLU, SWIGLU, etc.)
|
|
631
|
+
int32_t swapped = ggml_get_op_params_i32(node, 1); // Whether gate/value are swapped
|
|
632
|
+
|
|
633
|
+
// Supported variants
|
|
634
|
+
switch (glu_op_type) {
|
|
635
|
+
case GGML_GLU_OP_REGLU:
|
|
636
|
+
case GGML_GLU_OP_GEGLU:
|
|
637
|
+
case GGML_GLU_OP_SWIGLU:
|
|
638
|
+
case GGML_GLU_OP_SWIGLU_OAI:
|
|
639
|
+
case GGML_GLU_OP_SWIGLU_CLAMP:
|
|
640
|
+
case GGML_GLU_OP_GEGLU_ERF:
|
|
641
|
+
case GGML_GLU_OP_GEGLU_QUICK:
|
|
642
|
+
break;
|
|
643
|
+
default:
|
|
644
|
+
GGML_LOG_ERROR("ET: GLU operation with unsupported variant: %s\n",
|
|
645
|
+
ggml_glu_op_name((ggml_glu_op) glu_op_type));
|
|
646
|
+
return false;
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
// Get GLU operation name for logging
|
|
650
|
+
const char * glu_op_name = ggml_glu_op_name((ggml_glu_op) glu_op_type);
|
|
651
|
+
|
|
652
|
+
// Pack parameters. Single-tensor mode is encoded by zeroing src1.
|
|
653
|
+
ggml_et_glu_params params = {};
|
|
654
|
+
params.src0 = *node->src[0];
|
|
655
|
+
if (is_split_mode) {
|
|
656
|
+
params.src1 = *node->src[1];
|
|
657
|
+
}
|
|
658
|
+
params.dst = *node;
|
|
659
|
+
params.glu_op_type = glu_op_type;
|
|
660
|
+
params.swapped = swapped;
|
|
661
|
+
params.alpha = 0.0f;
|
|
662
|
+
params.limit = 0.0f;
|
|
663
|
+
if (glu_op_type == GGML_GLU_OP_SWIGLU_OAI) {
|
|
664
|
+
params.alpha = ggml_get_op_params_f32(node, 2);
|
|
665
|
+
}
|
|
666
|
+
if (glu_op_type == GGML_GLU_OP_SWIGLU_OAI || glu_op_type == GGML_GLU_OP_SWIGLU_CLAMP) {
|
|
667
|
+
params.limit = ggml_get_op_params_f32(node, 3);
|
|
668
|
+
}
|
|
669
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
670
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
671
|
+
bool cpu_comparison_active = false;
|
|
672
|
+
if (glu_cpu_compare_config.enabled) {
|
|
673
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GLU)) {
|
|
674
|
+
cpu_comparison_active = true;
|
|
675
|
+
} else {
|
|
676
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for %s operation\n", glu_op_name);
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
// Launch ET kernel
|
|
681
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "glu_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
682
|
+
|
|
683
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
684
|
+
if (cpu_comparison_active) {
|
|
685
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &glu_cpu_compare_config)) {
|
|
686
|
+
GGML_LOG_WARN("ET: CPU comparison failed for %s operation\n", glu_op_name);
|
|
687
|
+
}
|
|
688
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
ET_PERF_END("GLU", "glu_f32", node);
|
|
692
|
+
return kernel_result;
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
bool ggml_et_op_mul_mat(ggml_backend_et_device_context * dev_ctx,
|
|
696
|
+
const ggml_tensor * node,
|
|
697
|
+
const ggml_tensor * add_node) {
|
|
698
|
+
ET_PERF_START();
|
|
699
|
+
|
|
700
|
+
if (!dev_ctx || !node) {
|
|
701
|
+
GGML_LOG_ERROR("ET: Invalid parameters for MUL_MAT operation\n");
|
|
702
|
+
return false;
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
if (!node->src[0] || !node->src[1]) {
|
|
706
|
+
GGML_LOG_ERROR("ET: MUL_MAT operation missing required inputs\n");
|
|
707
|
+
return false;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
// Fused MM+ADD: when add_node is non-NULL the caller has already validated
|
|
711
|
+
// (Q8_0 weights, F32 acts, exact-shape ADD with stride parity to dst) via
|
|
712
|
+
// ggml_et_can_fuse({MUL_MAT, ADD}). The kernel writes dst = mm + bias and
|
|
713
|
+
// the ADD's output replaces MM's as the actual dst.
|
|
714
|
+
const ggml_tensor * fused_dst = add_node ? add_node : node;
|
|
715
|
+
const ggml_tensor * bias_tensor = nullptr;
|
|
716
|
+
if (add_node) {
|
|
717
|
+
bias_tensor = (add_node->src[0] == node) ? add_node->src[1] : add_node->src[0];
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
const char * kernel_name;
|
|
721
|
+
const char * src0_type_name;
|
|
722
|
+
|
|
723
|
+
if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q4_0 && node->src[1]->type == GGML_TYPE_F32 &&
|
|
724
|
+
node->src[1]->ne[1] >= 53 && // N >= 53
|
|
725
|
+
node->src[0]->ne[1] % 16 == 0 && // M % TILE_M
|
|
726
|
+
node->src[0]->ne[0] % 32 == 0) { // K % BLOCK_K (Q4_0 block)
|
|
727
|
+
|
|
728
|
+
// Matrix engine for N >= 53; partial N (via n_cur-1) and errata padding are handled in-kernel.
|
|
729
|
+
kernel_name = "mul_mat_Q4_0_matrix_engine";
|
|
730
|
+
src0_type_name = "Q4_0";
|
|
731
|
+
|
|
732
|
+
} else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q4_0 &&
|
|
733
|
+
node->src[1]->type == GGML_TYPE_F32) {
|
|
734
|
+
kernel_name = "mul_mat_Q4_0"; // N < 53, or M % 16 != 0 or K % 32 != 0
|
|
735
|
+
src0_type_name = "Q4_0";
|
|
736
|
+
|
|
737
|
+
} else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q8_0 &&
|
|
738
|
+
node->src[1]->type == GGML_TYPE_F32) {
|
|
739
|
+
kernel_name = "mul_mat_Q8_0";
|
|
740
|
+
src0_type_name = "Q8_0";
|
|
741
|
+
|
|
742
|
+
} else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 &&
|
|
743
|
+
node->src[1]->type == GGML_TYPE_F16 && node->ne[0] % 16 == 0 && node->src[0]->ne[0] % 16 == 0 &&
|
|
744
|
+
node->src[0]->ne[1] % 16 == 0 && node->src[1]->ne[0] != 1) {
|
|
745
|
+
kernel_name = "mul_mat_f16_matrix_engine";
|
|
746
|
+
src0_type_name = "F16";
|
|
747
|
+
|
|
748
|
+
} else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 &&
|
|
749
|
+
(node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32)) {
|
|
750
|
+
kernel_name = "mul_mat_f16";
|
|
751
|
+
src0_type_name = "F16";
|
|
752
|
+
|
|
753
|
+
} else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 &&
|
|
754
|
+
node->src[1]->type == GGML_TYPE_F32 && node->ne[0] % 16 == 0 && node->src[0]->ne[0] % 16 == 0 &&
|
|
755
|
+
node->src[0]->ne[1] % 16 == 0 && node->src[1]->ne[0] != 1) { // GEMV is faster with the generic path
|
|
756
|
+
|
|
757
|
+
kernel_name = "mul_mat_f32_matrix_engine";
|
|
758
|
+
src0_type_name = "F32";
|
|
759
|
+
} else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 &&
|
|
760
|
+
(node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32)) {
|
|
761
|
+
kernel_name = "mul_mat_f32";
|
|
762
|
+
src0_type_name = "F32";
|
|
763
|
+
} else {
|
|
764
|
+
GGML_LOG_ERROR("ET: MUL_MAT operation with unsupported types: dst=%s src0=%s src1=%s\n",
|
|
765
|
+
ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
|
|
766
|
+
ggml_type_name(node->src[1]->type));
|
|
767
|
+
return false;
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
ggml_et_binary_params params;
|
|
771
|
+
params.src0 = *node->src[0]; // weight matrix
|
|
772
|
+
params.src1 = *node->src[1]; // activation matrix
|
|
773
|
+
params.dst = *fused_dst; // output (= add_node when fused, else node)
|
|
774
|
+
|
|
775
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
776
|
+
bool cpu_comparison_active = false;
|
|
777
|
+
if (mul_mat_cpu_compare_config.enabled) {
|
|
778
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, fused_dst, GGML_OP_MUL_MAT)) {
|
|
779
|
+
cpu_comparison_active = true;
|
|
780
|
+
} else {
|
|
781
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for MUL_MAT operation\n");
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
bool kernel_result;
|
|
786
|
+
if (node->src[0]->type == GGML_TYPE_Q8_0) {
|
|
787
|
+
// Q8_0 kernel always takes the extended struct. bias.data is non-NULL
|
|
788
|
+
// only on the fused path; otherwise the kernel skips the add entirely.
|
|
789
|
+
ggml_et_mm_q8_params q8_params = {};
|
|
790
|
+
q8_params.src0 = params.src0;
|
|
791
|
+
q8_params.src1 = params.src1;
|
|
792
|
+
q8_params.dst = params.dst;
|
|
793
|
+
if (bias_tensor) {
|
|
794
|
+
q8_params.bias = *bias_tensor;
|
|
795
|
+
}
|
|
796
|
+
kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, &q8_params, sizeof(q8_params), 0xFFFFFFFF);
|
|
797
|
+
} else {
|
|
798
|
+
// Non-Q8 MM kernels don't yet support fused-add; the graph fuse check
|
|
799
|
+
// already rejects non-Q8 pairs, so add_node is always nullptr here.
|
|
800
|
+
kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
// printf("Tensor error:");
|
|
804
|
+
// if (params.src0.data != NULL)
|
|
805
|
+
// {
|
|
806
|
+
// printf("Ptr OK\n");
|
|
807
|
+
// printf("node->data ptr = %p\n", node->data);
|
|
808
|
+
// // if (once < 100){
|
|
809
|
+
// // // uint64_t * host_data = (uint64_t *) node->data;
|
|
810
|
+
// // // printf("Tensor error: %lu\n", host_data[0]);
|
|
811
|
+
|
|
812
|
+
// // // printf("Tensor error:");
|
|
813
|
+
// // once++;
|
|
814
|
+
// // }
|
|
815
|
+
// }
|
|
816
|
+
|
|
817
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
818
|
+
if (cpu_comparison_active) {
|
|
819
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, fused_dst, &mul_mat_cpu_compare_config)) {
|
|
820
|
+
GGML_LOG_WARN("ET: CPU comparison failed for MUL_MAT operation\n");
|
|
821
|
+
}
|
|
822
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
{
|
|
826
|
+
// Calculate actual FLOPs including batch/sequence dimensions
|
|
827
|
+
// dst shape: [M, N, ne2, ne3] where M=ne[1], N=ne[0]
|
|
828
|
+
int64_t m = node->ne[1];
|
|
829
|
+
int64_t n = node->ne[0];
|
|
830
|
+
int64_t k = node->src[0]->ne[0];
|
|
831
|
+
int64_t ne2 = node->ne[2];
|
|
832
|
+
int64_t ne3 = node->ne[3];
|
|
833
|
+
|
|
834
|
+
// Total FLOPs = (batch_size) * M * N * (2*K - 1)
|
|
835
|
+
// Each MxN matrix-matrix multiply does M*N*(2*K-1) FLOPs
|
|
836
|
+
// Broadcasting is handled by repeating computation, so count actual operations
|
|
837
|
+
int64_t batch_size = ne2 * ne3;
|
|
838
|
+
int64_t total_flops = batch_size * m * n * (2 * k - 1);
|
|
839
|
+
|
|
840
|
+
char kernel_variant[64];
|
|
841
|
+
snprintf(kernel_variant, sizeof(kernel_variant), "%s_%sx%s", kernel_name, src0_type_name,
|
|
842
|
+
ggml_type_name(node->src[1]->type));
|
|
843
|
+
ET_PERF_END_EXT("MUL_MAT", kernel_variant, node, "flops=%" PRId64, total_flops);
|
|
844
|
+
}
|
|
845
|
+
return kernel_result;
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
bool ggml_et_op_mul_mat_id(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
849
|
+
ET_PERF_START();
|
|
850
|
+
if (!dev_ctx || !node) {
|
|
851
|
+
GGML_LOG_ERROR("ET: Invalid parameters for MUL_MAT_ID operation\n");
|
|
852
|
+
return false;
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
if (!node->src[0] || !node->src[1] || !node->src[2]) {
|
|
856
|
+
GGML_LOG_ERROR("ET: MUL_MAT_ID operation missing required inputs\n");
|
|
857
|
+
return false;
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
const char * kernel_name;
|
|
861
|
+
const char * src0_type_name;
|
|
862
|
+
|
|
863
|
+
// Support Q8_0/Q4_0/F16/F32 x F32 -> F32 matrix multiplication with expert selection
|
|
864
|
+
if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q8_0 && node->src[1]->type == GGML_TYPE_F32 &&
|
|
865
|
+
node->src[2]->type == GGML_TYPE_I32) {
|
|
866
|
+
kernel_name = "mul_mat_id_Q8_0";
|
|
867
|
+
src0_type_name = "Q8_0";
|
|
868
|
+
|
|
869
|
+
} else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_Q4_0 &&
|
|
870
|
+
node->src[1]->type == GGML_TYPE_F32 && node->src[2]->type == GGML_TYPE_I32) {
|
|
871
|
+
kernel_name = "mul_mat_id_Q4_0";
|
|
872
|
+
src0_type_name = "Q4_0";
|
|
873
|
+
|
|
874
|
+
} else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F16 &&
|
|
875
|
+
node->src[1]->type == GGML_TYPE_F32 && node->src[2]->type == GGML_TYPE_I32) {
|
|
876
|
+
kernel_name = "mul_mat_id_f32";
|
|
877
|
+
src0_type_name = "F16";
|
|
878
|
+
|
|
879
|
+
} else if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 &&
|
|
880
|
+
node->src[1]->type == GGML_TYPE_F32 && node->src[2]->type == GGML_TYPE_I32) {
|
|
881
|
+
kernel_name = "mul_mat_id_f32";
|
|
882
|
+
src0_type_name = "F32";
|
|
883
|
+
|
|
884
|
+
} else {
|
|
885
|
+
GGML_LOG_ERROR("ET: MUL_MAT_ID operation with unsupported types: dst=%s src0=%s src1=%s src2=%s\n",
|
|
886
|
+
ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
|
|
887
|
+
ggml_type_name(node->src[1]->type), ggml_type_name(node->src[2]->type));
|
|
888
|
+
return false;
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
// Pack parameters - copy full tensor structures
|
|
892
|
+
ggml_et_mul_mat_id_params params;
|
|
893
|
+
params.src0 = *node->src[0]; // Expert weight matrices (Q8_0/F16/F32)
|
|
894
|
+
params.src1 = *node->src[1]; // Activation matrix (F32)
|
|
895
|
+
params.src2 = *node->src[2]; // Expert indices (I32)
|
|
896
|
+
params.dst = *node; // Output matrix (F32)
|
|
897
|
+
|
|
898
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
899
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
900
|
+
bool cpu_comparison_active = false;
|
|
901
|
+
if (mul_mat_id_cpu_compare_config.enabled) {
|
|
902
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_MUL_MAT_ID)) {
|
|
903
|
+
cpu_comparison_active = true;
|
|
904
|
+
} else {
|
|
905
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for MUL_MAT_ID operation\n");
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
// Launch ET kernel
|
|
910
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
911
|
+
|
|
912
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
913
|
+
if (cpu_comparison_active) {
|
|
914
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &mul_mat_id_cpu_compare_config)) {
|
|
915
|
+
GGML_LOG_WARN("ET: CPU comparison failed for MUL_MAT_ID operation\n");
|
|
916
|
+
}
|
|
917
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
// Calculate FLOPs (approximate - similar to MUL_MAT but with expert routing overhead)
|
|
921
|
+
// Each expert computation is similar to a MUL_MAT, but we only compute for selected experts
|
|
922
|
+
int64_t K = node->src[0]->ne[0];
|
|
923
|
+
int64_t M = node->src[0]->ne[1];
|
|
924
|
+
int64_t n_expert_used = node->src[2]->ne[0];
|
|
925
|
+
int64_t batch = node->src[2]->ne[1];
|
|
926
|
+
|
|
927
|
+
int64_t total_flops = batch * n_expert_used * M * (2 * K - 1);
|
|
928
|
+
|
|
929
|
+
char kernel_variant[64];
|
|
930
|
+
snprintf(kernel_variant, sizeof(kernel_variant), "%s_%sx%s", kernel_name, src0_type_name,
|
|
931
|
+
ggml_type_name(node->src[1]->type));
|
|
932
|
+
ET_PERF_END_EXT("MUL_MAT_ID", kernel_variant, node, "flops=%" PRId64 "|n_expert=%lld|n_expert_used=%lld",
|
|
933
|
+
total_flops, (long long) node->src[0]->ne[2], (long long) n_expert_used);
|
|
934
|
+
|
|
935
|
+
return kernel_result;
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
bool ggml_et_op_rope(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
939
|
+
ET_PERF_START();
|
|
940
|
+
|
|
941
|
+
if (!dev_ctx || !node) {
|
|
942
|
+
GGML_LOG_ERROR("ET: Invalid parameters for ROPE operation\n");
|
|
943
|
+
return false;
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
if (!node->src[0] || !node->src[1]) {
|
|
947
|
+
GGML_LOG_ERROR("ET: ROPE operation missing required inputs\n");
|
|
948
|
+
return false;
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
const char * kernel_name;
|
|
952
|
+
|
|
953
|
+
if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_I32) {
|
|
954
|
+
kernel_name = "rope_f32";
|
|
955
|
+
} else {
|
|
956
|
+
return false;
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
// Pack parameters - copy full tensor structures and op_params
|
|
960
|
+
ggml_et_rope_params params;
|
|
961
|
+
params.src0 = *node->src[0]; // F32 input tensor
|
|
962
|
+
params.src1 = *node->src[1]; // I32 position tensor
|
|
963
|
+
if (node->src[2]) {
|
|
964
|
+
params.src2 = *node->src[2]; // F32 frequency factors (optional)
|
|
965
|
+
} else {
|
|
966
|
+
memset(¶ms.src2, 0, sizeof(params.src2)); // Zero if not provided
|
|
967
|
+
}
|
|
968
|
+
params.dst = *node; // F32 output tensor
|
|
969
|
+
|
|
970
|
+
params.rope_params.n_past = ((const int32_t *) node->op_params)[0];
|
|
971
|
+
params.rope_params.n_dims = ((const int32_t *) node->op_params)[1];
|
|
972
|
+
params.rope_params.mode = ((const int32_t *) node->op_params)[2];
|
|
973
|
+
params.rope_params.n_ctx = ((const int32_t *) node->op_params)[3];
|
|
974
|
+
params.rope_params.n_ctx_orig = ((const int32_t *) node->op_params)[4];
|
|
975
|
+
memcpy(¶ms.rope_params.freq_base, (const int32_t *) node->op_params + 5, sizeof(float));
|
|
976
|
+
memcpy(¶ms.rope_params.freq_scale, (const int32_t *) node->op_params + 6, sizeof(float));
|
|
977
|
+
memcpy(¶ms.rope_params.ext_factor, (const int32_t *) node->op_params + 7, sizeof(float));
|
|
978
|
+
memcpy(¶ms.rope_params.attn_factor, (const int32_t *) node->op_params + 8, sizeof(float));
|
|
979
|
+
memcpy(¶ms.rope_params.beta_fast, (const int32_t *) node->op_params + 9, sizeof(float));
|
|
980
|
+
memcpy(¶ms.rope_params.beta_slow, (const int32_t *) node->op_params + 10, sizeof(float));
|
|
981
|
+
if (params.rope_params.mode & GGML_ROPE_TYPE_MROPE) {
|
|
982
|
+
memcpy(params.rope_params.sections, (const int32_t *) node->op_params + 11, sizeof(int32_t) * 4);
|
|
983
|
+
} else {
|
|
984
|
+
memset(params.rope_params.sections, 0, sizeof(params.rope_params.sections));
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
988
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
989
|
+
bool cpu_comparison_active = false;
|
|
990
|
+
if (rope_cpu_compare_config.enabled) {
|
|
991
|
+
GGML_LOG_DEBUG("ET: Initializing CPU comparison for ROPE operation\n");
|
|
992
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_ROPE)) {
|
|
993
|
+
cpu_comparison_active = true;
|
|
994
|
+
} else {
|
|
995
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for ROPE operation\n");
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1000
|
+
|
|
1001
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
1002
|
+
if (cpu_comparison_active) {
|
|
1003
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rope_cpu_compare_config)) {
|
|
1004
|
+
GGML_LOG_WARN("ET: CPU comparison failed for ROPE operation\n");
|
|
1005
|
+
}
|
|
1006
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
ET_PERF_END_EXT("ROPE", kernel_name, node, "mode=0x%x|n_dims=%d|freq_base=%.2f|freq_scale=%.2f",
|
|
1010
|
+
params.rope_params.mode, params.rope_params.n_dims, (double) params.rope_params.freq_base,
|
|
1011
|
+
(double) params.rope_params.freq_scale);
|
|
1012
|
+
return kernel_result;
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
bool ggml_et_op_rms_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1016
|
+
ET_PERF_START();
|
|
1017
|
+
|
|
1018
|
+
if (!dev_ctx || !node) {
|
|
1019
|
+
GGML_LOG_ERROR("ET: Invalid parameters for RMS_NORM operation\n");
|
|
1020
|
+
return false;
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
if (!node->src[0]) {
|
|
1024
|
+
GGML_LOG_ERROR("ET: RMS_NORM operation missing required input\n");
|
|
1025
|
+
return false;
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
const char * kernel_name;
|
|
1029
|
+
|
|
1030
|
+
if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) {
|
|
1031
|
+
kernel_name = "rms_norm_f32";
|
|
1032
|
+
|
|
1033
|
+
} else {
|
|
1034
|
+
GGML_LOG_ERROR("ET: RMS_NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
1035
|
+
ggml_type_name(node->src[0]->type));
|
|
1036
|
+
return false;
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
float eps;
|
|
1040
|
+
memcpy(&eps, node->op_params, sizeof(float));
|
|
1041
|
+
|
|
1042
|
+
ggml_et_rms_norm_params params;
|
|
1043
|
+
params.src0 = *node->src[0]; // F32 input tensor
|
|
1044
|
+
params.dst = *node; // F32 output tensor
|
|
1045
|
+
params.eps = eps; // Epsilon parameter for numerical stability
|
|
1046
|
+
|
|
1047
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
1048
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1049
|
+
bool cpu_comparison_active = false;
|
|
1050
|
+
if (rms_norm_cpu_compare_config.enabled) {
|
|
1051
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_RMS_NORM)) {
|
|
1052
|
+
cpu_comparison_active = true;
|
|
1053
|
+
} else {
|
|
1054
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for RMS_NORM operation\n");
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1059
|
+
|
|
1060
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
1061
|
+
if (cpu_comparison_active) {
|
|
1062
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rms_norm_cpu_compare_config)) {
|
|
1063
|
+
GGML_LOG_WARN("ET: CPU comparison failed for RMS_NORM operation\n");
|
|
1064
|
+
}
|
|
1065
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
ET_PERF_END_EXT("RMS_NORM", kernel_name, node, "eps=%.6f", (double) eps);
|
|
1069
|
+
return kernel_result;
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
bool ggml_et_op_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1073
|
+
ET_PERF_START();
|
|
1074
|
+
|
|
1075
|
+
if (!dev_ctx || !node) {
|
|
1076
|
+
GGML_LOG_ERROR("ET: Invalid parameters for NORM operation\n");
|
|
1077
|
+
return false;
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
if (!node->src[0]) {
|
|
1081
|
+
GGML_LOG_ERROR("ET: NORM operation missing required input\n");
|
|
1082
|
+
return false;
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
const char * kernel_name;
|
|
1086
|
+
|
|
1087
|
+
if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) {
|
|
1088
|
+
kernel_name = "norm_f32";
|
|
1089
|
+
|
|
1090
|
+
} else {
|
|
1091
|
+
GGML_LOG_ERROR("ET: NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
1092
|
+
ggml_type_name(node->src[0]->type));
|
|
1093
|
+
return false;
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
float eps;
|
|
1097
|
+
memcpy(&eps, node->op_params, sizeof(float));
|
|
1098
|
+
|
|
1099
|
+
ggml_et_norm_params params;
|
|
1100
|
+
params.src0 = *node->src[0]; // F32 input tensor
|
|
1101
|
+
params.dst = *node; // F32 output tensor
|
|
1102
|
+
params.eps = eps; // Epsilon parameter for numerical stability
|
|
1103
|
+
|
|
1104
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
1105
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1106
|
+
bool cpu_comparison_active = false;
|
|
1107
|
+
if (norm_cpu_compare_config.enabled) {
|
|
1108
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_NORM)) {
|
|
1109
|
+
cpu_comparison_active = true;
|
|
1110
|
+
} else {
|
|
1111
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for NORM operation\n");
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1116
|
+
|
|
1117
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
1118
|
+
if (cpu_comparison_active) {
|
|
1119
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &norm_cpu_compare_config)) {
|
|
1120
|
+
GGML_LOG_WARN("ET: CPU comparison failed for NORM operation\n");
|
|
1121
|
+
}
|
|
1122
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
ET_PERF_END_EXT("NORM", kernel_name, node, "eps=%.6f", (double) eps);
|
|
1126
|
+
return kernel_result;
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
bool ggml_et_op_l2_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1130
|
+
ET_PERF_START();
|
|
1131
|
+
|
|
1132
|
+
if (!dev_ctx || !node) {
|
|
1133
|
+
GGML_LOG_ERROR("ET: Invalid parameters for L2_NORM operation\n");
|
|
1134
|
+
return false;
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
if (!node->src[0]) {
|
|
1138
|
+
GGML_LOG_ERROR("ET: L2_NORM operation missing required input\n");
|
|
1139
|
+
return false;
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
const char * kernel_name;
|
|
1143
|
+
|
|
1144
|
+
if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) {
|
|
1145
|
+
kernel_name = "l2_norm_f32";
|
|
1146
|
+
|
|
1147
|
+
} else {
|
|
1148
|
+
GGML_LOG_ERROR("ET: L2_NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
1149
|
+
ggml_type_name(node->src[0]->type));
|
|
1150
|
+
return false;
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
float eps;
|
|
1154
|
+
memcpy(&eps, node->op_params, sizeof(float));
|
|
1155
|
+
|
|
1156
|
+
ggml_et_l2_norm_params params;
|
|
1157
|
+
params.src0 = *node->src[0]; // F32 input tensor
|
|
1158
|
+
params.dst = *node; // F32 output tensor
|
|
1159
|
+
params.eps = eps; // Epsilon parameter for numerical stability
|
|
1160
|
+
|
|
1161
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
1162
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1163
|
+
bool cpu_comparison_active = false;
|
|
1164
|
+
if (l2_norm_cpu_compare_config.enabled) {
|
|
1165
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_L2_NORM)) {
|
|
1166
|
+
cpu_comparison_active = true;
|
|
1167
|
+
} else {
|
|
1168
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for L2_NORM operation\n");
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1173
|
+
|
|
1174
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
1175
|
+
if (cpu_comparison_active) {
|
|
1176
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &l2_norm_cpu_compare_config)) {
|
|
1177
|
+
GGML_LOG_WARN("ET: CPU comparison failed for L2_NORM operation\n");
|
|
1178
|
+
}
|
|
1179
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
ET_PERF_END_EXT("L2_NORM", kernel_name, node, "eps=%.6f", (double) eps);
|
|
1183
|
+
return kernel_result;
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
bool ggml_et_op_group_norm(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1187
|
+
ET_PERF_START();
|
|
1188
|
+
|
|
1189
|
+
if (!dev_ctx || !node) {
|
|
1190
|
+
GGML_LOG_ERROR("ET: Invalid parameters for GROUP_NORM operation\n");
|
|
1191
|
+
return false;
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
if (!node->src[0]) {
|
|
1195
|
+
GGML_LOG_ERROR("ET: GROUP_NORM operation missing required input\n");
|
|
1196
|
+
return false;
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
|
|
1200
|
+
GGML_LOG_ERROR("ET: GROUP_NORM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
1201
|
+
ggml_type_name(node->src[0]->type));
|
|
1202
|
+
return false;
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
const int32_t n_groups = ggml_get_op_params_i32(node, 0);
|
|
1206
|
+
float eps;
|
|
1207
|
+
memcpy(&eps, (const float *) node->op_params + 1, sizeof(float));
|
|
1208
|
+
|
|
1209
|
+
ggml_et_group_norm_params params;
|
|
1210
|
+
params.src0 = *node->src[0];
|
|
1211
|
+
params.dst = *node;
|
|
1212
|
+
params.n_groups = n_groups;
|
|
1213
|
+
params.eps = eps;
|
|
1214
|
+
|
|
1215
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1216
|
+
bool cpu_comparison_active = false;
|
|
1217
|
+
if (group_norm_cpu_compare_config.enabled) {
|
|
1218
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GROUP_NORM)) {
|
|
1219
|
+
cpu_comparison_active = true;
|
|
1220
|
+
} else {
|
|
1221
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for GROUP_NORM operation\n");
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1225
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "group_norm_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1226
|
+
|
|
1227
|
+
if (cpu_comparison_active) {
|
|
1228
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &group_norm_cpu_compare_config)) {
|
|
1229
|
+
GGML_LOG_WARN("ET: CPU comparison failed for GROUP_NORM operation\n");
|
|
1230
|
+
}
|
|
1231
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
ET_PERF_END_EXT("GROUP_NORM", "group_norm_f32", node, "eps=%.6f|n_groups=%d", (double) eps, n_groups);
|
|
1235
|
+
return kernel_result;
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
bool ggml_et_op_im2col(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1239
|
+
ET_PERF_START();
|
|
1240
|
+
|
|
1241
|
+
if (!dev_ctx || !node) {
|
|
1242
|
+
GGML_LOG_ERROR("ET: Invalid parameters for IM2COL operation\n");
|
|
1243
|
+
return false;
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
if (!node->src[0] || !node->src[1]) {
|
|
1247
|
+
GGML_LOG_ERROR("ET: IM2COL operation missing required inputs\n");
|
|
1248
|
+
return false;
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
const bool supported_types =
|
|
1252
|
+
(node->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32) ||
|
|
1253
|
+
(node->type == GGML_TYPE_F16 && (node->src[1]->type == GGML_TYPE_F16 || node->src[1]->type == GGML_TYPE_F32));
|
|
1254
|
+
|
|
1255
|
+
if (!supported_types) {
|
|
1256
|
+
GGML_LOG_ERROR("ET: IM2COL operation with unsupported types: dst=%s src1=%s\n", ggml_type_name(node->type),
|
|
1257
|
+
ggml_type_name(node->src[1]->type));
|
|
1258
|
+
return false;
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
ggml_et_im2col_params params;
|
|
1262
|
+
params.src0 = *node->src[0];
|
|
1263
|
+
params.src1 = *node->src[1];
|
|
1264
|
+
params.dst = *node;
|
|
1265
|
+
|
|
1266
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1267
|
+
bool cpu_comparison_active = false;
|
|
1268
|
+
if (im2col_cpu_compare_config.enabled) {
|
|
1269
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_IM2COL)) {
|
|
1270
|
+
cpu_comparison_active = true;
|
|
1271
|
+
} else {
|
|
1272
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for IM2COL operation\n");
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "im2col", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1277
|
+
|
|
1278
|
+
if (cpu_comparison_active) {
|
|
1279
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &im2col_cpu_compare_config)) {
|
|
1280
|
+
GGML_LOG_WARN("ET: CPU comparison failed for IM2COL operation\n");
|
|
1281
|
+
}
|
|
1282
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1285
|
+
ET_PERF_END("IM2COL", "im2col", node);
|
|
1286
|
+
return kernel_result;
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
bool ggml_et_op_conv_2d(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1290
|
+
ET_PERF_START();
|
|
1291
|
+
|
|
1292
|
+
if (!dev_ctx || !node) {
|
|
1293
|
+
return false;
|
|
1294
|
+
}
|
|
1295
|
+
if (!node->src[0] || !node->src[1]) {
|
|
1296
|
+
return false;
|
|
1297
|
+
}
|
|
1298
|
+
if (!node->data || !node->src[0]->data || !node->src[1]->data) {
|
|
1299
|
+
return false;
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
// Kernel constraints (mirror supports_op; recheck here as a guard).
|
|
1303
|
+
const ggml_tensor * flt = node->src[0]; // [Kw, Kh, Cin, Cout]
|
|
1304
|
+
const ggml_tensor * in = node->src[1]; // [W, H, Cin, N]
|
|
1305
|
+
if (node->type != GGML_TYPE_F32 || flt->type != GGML_TYPE_F32 || in->type != GGML_TYPE_F32) {
|
|
1306
|
+
return false;
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
const int32_t s0 = ggml_get_op_params_i32(node, 0);
|
|
1310
|
+
const int32_t s1 = ggml_get_op_params_i32(node, 1);
|
|
1311
|
+
const int32_t p0 = ggml_get_op_params_i32(node, 2);
|
|
1312
|
+
const int32_t p1 = ggml_get_op_params_i32(node, 3);
|
|
1313
|
+
const int32_t d0 = ggml_get_op_params_i32(node, 4);
|
|
1314
|
+
const int32_t d1 = ggml_get_op_params_i32(node, 5);
|
|
1315
|
+
|
|
1316
|
+
if (s0 < 1 || s1 < 1) {
|
|
1317
|
+
return false;
|
|
1318
|
+
}
|
|
1319
|
+
if (d0 != 1 || d1 != 1) {
|
|
1320
|
+
return false;
|
|
1321
|
+
}
|
|
1322
|
+
if (flt->ne[2] % 16 != 0 || flt->ne[3] % 16 != 0) {
|
|
1323
|
+
return false;
|
|
1324
|
+
}
|
|
1325
|
+
if (in->ne[3] != 1) {
|
|
1326
|
+
return false;
|
|
1327
|
+
}
|
|
1328
|
+
if (node->ne[0] <= 0) {
|
|
1329
|
+
return false; // OW > 0 (any width OK; staging path handles non-16)
|
|
1330
|
+
}
|
|
1331
|
+
(void) p0;
|
|
1332
|
+
(void) p1;
|
|
1333
|
+
|
|
1334
|
+
ggml_et_binary_params params;
|
|
1335
|
+
params.src0 = *node->src[0];
|
|
1336
|
+
params.src1 = *node->src[1];
|
|
1337
|
+
params.dst = *node;
|
|
1338
|
+
|
|
1339
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "conv_2d_f32_me", ¶ms, sizeof(params), 0xFFFFFFFFu);
|
|
1340
|
+
|
|
1341
|
+
ET_PERF_END("CONV_2D", "conv_2d_f32_me", node);
|
|
1342
|
+
return kernel_result;
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
bool ggml_et_op_softmax(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1346
|
+
ET_PERF_START();
|
|
1347
|
+
|
|
1348
|
+
if (!dev_ctx || !node) {
|
|
1349
|
+
GGML_LOG_ERROR("ET: Invalid parameters for SOFTMAX operation\n");
|
|
1350
|
+
return false;
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
if (!node->src[0]) {
|
|
1354
|
+
GGML_LOG_ERROR("ET: SOFTMAX operation missing required input\n");
|
|
1355
|
+
return false;
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
const char * kernel_name;
|
|
1359
|
+
|
|
1360
|
+
if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) {
|
|
1361
|
+
kernel_name = "softmax_f32";
|
|
1362
|
+
|
|
1363
|
+
} else {
|
|
1364
|
+
GGML_LOG_ERROR("ET: SOFTMAX operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
1365
|
+
ggml_type_name(node->src[0]->type));
|
|
1366
|
+
return false;
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
// Validate contiguity requirements
|
|
1370
|
+
if (!ggml_is_contiguous(node)) {
|
|
1371
|
+
GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous destination tensor\n");
|
|
1372
|
+
return false;
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
if (!ggml_is_contiguous(node->src[0])) {
|
|
1376
|
+
GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous source tensor\n");
|
|
1377
|
+
return false;
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
// Check optional mask tensor
|
|
1381
|
+
if (node->src[1]) {
|
|
1382
|
+
if (node->src[1]->type != GGML_TYPE_F32) {
|
|
1383
|
+
GGML_LOG_ERROR("ET: SOFTMAX operation with unsupported mask type: %s (F32 required)\n",
|
|
1384
|
+
ggml_type_name(node->src[1]->type));
|
|
1385
|
+
return false;
|
|
1386
|
+
}
|
|
1387
|
+
if (!ggml_is_contiguous(node->src[1])) {
|
|
1388
|
+
GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous mask tensor\n");
|
|
1389
|
+
return false;
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
// Check optional sinks tensor
|
|
1394
|
+
if (node->src[2]) {
|
|
1395
|
+
if (node->src[2]->type != GGML_TYPE_F32) {
|
|
1396
|
+
GGML_LOG_ERROR("ET: SOFTMAX operation with unsupported sinks type: %s (F32 required)\n",
|
|
1397
|
+
ggml_type_name(node->src[2]->type));
|
|
1398
|
+
return false;
|
|
1399
|
+
}
|
|
1400
|
+
if (!ggml_is_contiguous(node->src[2])) {
|
|
1401
|
+
GGML_LOG_ERROR("ET: SOFTMAX operation requires contiguous sinks tensor\n");
|
|
1402
|
+
return false;
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
// Extract scale and max_bias from op_params
|
|
1407
|
+
float scale = 1.0f;
|
|
1408
|
+
float max_bias = 0.0f;
|
|
1409
|
+
if (node->op_params) {
|
|
1410
|
+
memcpy(&scale, (const float *) node->op_params + 0, sizeof(float));
|
|
1411
|
+
memcpy(&max_bias, (const float *) node->op_params + 1, sizeof(float));
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
ggml_et_softmax_params params;
|
|
1415
|
+
params.src0 = *node->src[0]; // F32 input tensor
|
|
1416
|
+
if (node->src[1]) {
|
|
1417
|
+
params.src1 = *node->src[1]; // F32 mask tensor
|
|
1418
|
+
} else {
|
|
1419
|
+
memset(¶ms.src1, 0, sizeof(params.src1)); // Zero if no mask
|
|
1420
|
+
}
|
|
1421
|
+
if (node->src[2]) {
|
|
1422
|
+
params.src2 = *node->src[2]; // F32 sinks tensor
|
|
1423
|
+
} else {
|
|
1424
|
+
memset(¶ms.src2, 0, sizeof(params.src2)); // Zero if no sinks
|
|
1425
|
+
}
|
|
1426
|
+
params.dst = *node; // F32 output tensor
|
|
1427
|
+
params.scale = scale; // Scale factor
|
|
1428
|
+
params.max_bias = max_bias; // ALiBi bias
|
|
1429
|
+
|
|
1430
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
1431
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1432
|
+
bool cpu_comparison_active = false;
|
|
1433
|
+
if (softmax_cpu_compare_config.enabled) {
|
|
1434
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SOFT_MAX)) {
|
|
1435
|
+
cpu_comparison_active = true;
|
|
1436
|
+
} else {
|
|
1437
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SOFTMAX operation\n");
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1442
|
+
|
|
1443
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
1444
|
+
if (cpu_comparison_active) {
|
|
1445
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &softmax_cpu_compare_config)) {
|
|
1446
|
+
GGML_LOG_WARN("ET: CPU comparison failed for SOFTMAX operation\n");
|
|
1447
|
+
}
|
|
1448
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
ET_PERF_END_EXT("SOFTMAX", kernel_name, node, "scale=%.6f|max_bias=%.6f|has_mask=%s", (double) scale,
|
|
1452
|
+
(double) max_bias, node->src[1] ? "yes" : "no");
|
|
1453
|
+
return kernel_result;
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
bool ggml_et_op_flash_attn_ext(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1457
|
+
ET_PERF_START();
|
|
1458
|
+
|
|
1459
|
+
if (!dev_ctx || !node) {
|
|
1460
|
+
GGML_LOG_ERROR("ET: Invalid parameters for FLASH_ATTN_EXT operation\n");
|
|
1461
|
+
return false;
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
if (!node->src[0] || !node->src[1] || !node->src[2]) {
|
|
1465
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT operation missing required inputs\n");
|
|
1466
|
+
return false;
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
|
|
1470
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT requires F32 Q and dst, got dst=%s q=%s\n", ggml_type_name(node->type),
|
|
1471
|
+
ggml_type_name(node->src[0]->type));
|
|
1472
|
+
return false;
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
// K and V can be F16 or F32
|
|
1476
|
+
if ((node->src[1]->type != GGML_TYPE_F32 && node->src[1]->type != GGML_TYPE_F16) ||
|
|
1477
|
+
(node->src[2]->type != GGML_TYPE_F32 && node->src[2]->type != GGML_TYPE_F16)) {
|
|
1478
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT K/V must be F16 or F32, got k=%s v=%s\n", ggml_type_name(node->src[1]->type),
|
|
1479
|
+
ggml_type_name(node->src[2]->type));
|
|
1480
|
+
return false;
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
if (node->src[4] != nullptr) {
|
|
1484
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT baseline kernel does not support sinks\n");
|
|
1485
|
+
return false;
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
// Mask is optional; if present must be F16 or F32
|
|
1489
|
+
if (node->src[3] != nullptr && node->src[3]->type != GGML_TYPE_F32 && node->src[3]->type != GGML_TYPE_F16) {
|
|
1490
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT mask must be F16 or F32, got %s\n", ggml_type_name(node->src[3]->type));
|
|
1491
|
+
return false;
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
// Q and dst must be row-contiguous F32
|
|
1495
|
+
if (!ggml_is_contiguous_rows(node) || !ggml_is_contiguous_rows(node->src[0])) {
|
|
1496
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT requires row-contiguous Q and dst\n");
|
|
1497
|
+
return false;
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
if (node->nb[0] != sizeof(float) || node->src[0]->nb[0] != sizeof(float)) {
|
|
1501
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT requires contiguous F32 rows for Q and dst\n");
|
|
1502
|
+
return false;
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
// K/V must have element-sized stride in dim 0
|
|
1506
|
+
const size_t k_elem = node->src[1]->type == GGML_TYPE_F16 ? 2 : 4;
|
|
1507
|
+
const size_t v_elem = node->src[2]->type == GGML_TYPE_F16 ? 2 : 4;
|
|
1508
|
+
if (node->src[1]->nb[0] != k_elem || node->src[2]->nb[0] != v_elem) {
|
|
1509
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT K/V must have element-sized stride in dim 0\n");
|
|
1510
|
+
return false;
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
float scale = 1.0f;
|
|
1514
|
+
float max_bias = 0.0f;
|
|
1515
|
+
float logit_softcap = 0.0f;
|
|
1516
|
+
memcpy(&scale, (const float *) node->op_params + 0, sizeof(scale));
|
|
1517
|
+
memcpy(&max_bias, (const float *) node->op_params + 1, sizeof(max_bias));
|
|
1518
|
+
memcpy(&logit_softcap, (const float *) node->op_params + 2, sizeof(logit_softcap));
|
|
1519
|
+
|
|
1520
|
+
if (max_bias != 0.0f || logit_softcap != 0.0f) {
|
|
1521
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT baseline kernel does not support max_bias or logit_softcap\n");
|
|
1522
|
+
return false;
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
const ggml_prec prec = ggml_flash_attn_ext_get_prec(node);
|
|
1526
|
+
if (prec != GGML_PREC_F32 && prec != GGML_PREC_DEFAULT) {
|
|
1527
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT baseline kernel only supports F32 precision\n");
|
|
1528
|
+
return false;
|
|
1529
|
+
}
|
|
1530
|
+
|
|
1531
|
+
// dk must match between Q and K; dv must match between V and dst
|
|
1532
|
+
if (node->src[0]->ne[0] != node->src[1]->ne[0]) {
|
|
1533
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dk mismatch: Q=%lld K=%lld\n", (long long) node->src[0]->ne[0],
|
|
1534
|
+
(long long) node->src[1]->ne[0]);
|
|
1535
|
+
return false;
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
if (node->src[2]->ne[0] != node->ne[0]) {
|
|
1539
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dv mismatch: V=%lld dst=%lld\n", (long long) node->src[2]->ne[0],
|
|
1540
|
+
(long long) node->ne[0]);
|
|
1541
|
+
return false;
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
if (node->src[2]->ne[0] > 512) {
|
|
1545
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dv=%lld exceeds maximum 512\n", (long long) node->src[2]->ne[0]);
|
|
1546
|
+
return false;
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
if (node->src[0]->ne[0] > 512) {
|
|
1550
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dk=%lld exceeds maximum 512\n", (long long) node->src[0]->ne[0]);
|
|
1551
|
+
return false;
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
// GQA: n_head_q must be a multiple of n_head_kv
|
|
1555
|
+
const int64_t nhq = node->src[0]->ne[2];
|
|
1556
|
+
const int64_t nhk = node->src[1]->ne[2];
|
|
1557
|
+
if (nhq % nhk != 0) {
|
|
1558
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT n_head_q (%lld) not divisible by n_head_kv (%lld)\n", (long long) nhq,
|
|
1559
|
+
(long long) nhk);
|
|
1560
|
+
return false;
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
// K and V must have matching sequence length, heads, and batch dims
|
|
1564
|
+
if (node->src[1]->ne[1] != node->src[2]->ne[1] || node->src[1]->ne[2] != node->src[2]->ne[2] ||
|
|
1565
|
+
node->src[1]->ne[3] != node->src[2]->ne[3]) {
|
|
1566
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT K/V shape mismatch\n");
|
|
1567
|
+
return false;
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
// dst layout checks: [dv, nhq, nq, no]
|
|
1571
|
+
if (node->src[0]->ne[1] != node->ne[2] || node->src[0]->ne[2] != node->ne[1] ||
|
|
1572
|
+
node->src[0]->ne[3] != node->ne[3]) {
|
|
1573
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT dst shape mismatch\n");
|
|
1574
|
+
return false;
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
// Batch dims: Q batch must match K batch
|
|
1578
|
+
if (node->src[0]->ne[3] != node->src[1]->ne[3]) {
|
|
1579
|
+
GGML_LOG_ERROR("ET: FLASH_ATTN_EXT batch dimension mismatch\n");
|
|
1580
|
+
return false;
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1583
|
+
ggml_et_flash_attn_ext_params params;
|
|
1584
|
+
memset(¶ms, 0, sizeof(params));
|
|
1585
|
+
params.src0 = *node->src[0];
|
|
1586
|
+
params.src1 = *node->src[1];
|
|
1587
|
+
params.src2 = *node->src[2];
|
|
1588
|
+
if (node->src[3] != nullptr) {
|
|
1589
|
+
params.mask = *node->src[3];
|
|
1590
|
+
params.has_mask = 1;
|
|
1591
|
+
}
|
|
1592
|
+
params.dst = *node;
|
|
1593
|
+
params.scale = scale;
|
|
1594
|
+
|
|
1595
|
+
// Use matrix engine kernel when K/V are F16 and dk is a multiple of 32
|
|
1596
|
+
const char * kernel_name;
|
|
1597
|
+
if (node->src[1]->type == GGML_TYPE_F16 && node->src[2]->type == GGML_TYPE_F16 && (node->src[0]->ne[0] % 32) == 0) {
|
|
1598
|
+
kernel_name = "flash_attn_ext_f16_me";
|
|
1599
|
+
} else {
|
|
1600
|
+
kernel_name = "flash_attn_ext_f32";
|
|
1601
|
+
}
|
|
1602
|
+
|
|
1603
|
+
const bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1604
|
+
|
|
1605
|
+
ET_PERF_END_EXT("FLASH_ATTN_EXT", kernel_name, node, "scale=%.6f", (double) scale);
|
|
1606
|
+
return kernel_result;
|
|
1607
|
+
}
|
|
1608
|
+
|
|
1609
|
+
bool ggml_et_op_get_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1610
|
+
ET_PERF_START();
|
|
1611
|
+
|
|
1612
|
+
if (!dev_ctx || !node) {
|
|
1613
|
+
GGML_LOG_ERROR("ET: Invalid parameters for GET_ROWS operation\n");
|
|
1614
|
+
return false;
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
if (!node->src[0] || !node->src[1]) {
|
|
1618
|
+
GGML_LOG_ERROR("ET: GET_ROWS operation missing required inputs\n");
|
|
1619
|
+
return false;
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1622
|
+
const char * kernel_name;
|
|
1623
|
+
|
|
1624
|
+
if (node->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_I32 &&
|
|
1625
|
+
(node->src[0]->type == GGML_TYPE_F32 || node->src[0]->type == GGML_TYPE_F16 ||
|
|
1626
|
+
node->src[0]->type == GGML_TYPE_Q4_0 || node->src[0]->type == GGML_TYPE_Q8_0 ||
|
|
1627
|
+
node->src[0]->type == GGML_TYPE_Q4_K)) {
|
|
1628
|
+
kernel_name = "get_rows_f32";
|
|
1629
|
+
|
|
1630
|
+
} else {
|
|
1631
|
+
GGML_LOG_ERROR("ET: GET_ROWS operation with unsupported types: dst=%s src0=%s src1=%s\n",
|
|
1632
|
+
ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
|
|
1633
|
+
ggml_type_name(node->src[1]->type));
|
|
1634
|
+
return false;
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
// Validate contiguity requirements
|
|
1638
|
+
if (!ggml_is_contiguous(node)) {
|
|
1639
|
+
GGML_LOG_ERROR("ET: GET_ROWS operation requires contiguous destination tensor\n");
|
|
1640
|
+
return false;
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1643
|
+
if (!ggml_is_contiguous(node->src[0])) {
|
|
1644
|
+
GGML_LOG_ERROR("ET: GET_ROWS operation requires contiguous data tensor\n");
|
|
1645
|
+
return false;
|
|
1646
|
+
}
|
|
1647
|
+
|
|
1648
|
+
if (!ggml_is_contiguous(node->src[1])) {
|
|
1649
|
+
GGML_LOG_ERROR("ET: GET_ROWS operation requires contiguous indices tensor\n");
|
|
1650
|
+
return false;
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
// Validate dimension constraints from ggml implementation
|
|
1654
|
+
if (node->src[0]->ne[2] != node->src[1]->ne[1] || node->src[1]->ne[3] != 1) {
|
|
1655
|
+
GGML_LOG_ERROR(
|
|
1656
|
+
"ET: GET_ROWS operation dimension constraint failed: src0.ne[2]=%lld != src1.ne[1]=%lld or src1.ne[3]=%lld "
|
|
1657
|
+
"!= 1\n",
|
|
1658
|
+
(long long) node->src[0]->ne[2], (long long) node->src[1]->ne[1], (long long) node->src[1]->ne[3]);
|
|
1659
|
+
return false;
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
ggml_et_get_rows_params params;
|
|
1663
|
+
params.src0 = *node->src[0]; // Data tensor (F32 or Q8_0)
|
|
1664
|
+
params.src1 = *node->src[1]; // Indices tensor (I32)
|
|
1665
|
+
params.dst = *node; // Output tensor (F32)
|
|
1666
|
+
|
|
1667
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
1668
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1669
|
+
bool cpu_comparison_active = false;
|
|
1670
|
+
if (get_rows_cpu_compare_config.enabled) {
|
|
1671
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GET_ROWS)) {
|
|
1672
|
+
cpu_comparison_active = true;
|
|
1673
|
+
} else {
|
|
1674
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for GET_ROWS operation\n");
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1678
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1679
|
+
|
|
1680
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
1681
|
+
if (cpu_comparison_active) {
|
|
1682
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &get_rows_cpu_compare_config)) {
|
|
1683
|
+
GGML_LOG_WARN("ET: CPU comparison failed for GET_ROWS operation\n");
|
|
1684
|
+
}
|
|
1685
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
ET_PERF_END("GET_ROWS", kernel_name, node);
|
|
1689
|
+
return kernel_result;
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
bool ggml_et_op_cont(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1693
|
+
ET_PERF_START();
|
|
1694
|
+
|
|
1695
|
+
// Validate source tensor exists
|
|
1696
|
+
if (!node->src[0]) {
|
|
1697
|
+
GGML_LOG_ERROR("ET: CONT operation missing source tensor\n");
|
|
1698
|
+
return false;
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
// Validate types match (input and output must be same type)
|
|
1702
|
+
if (node->type != node->src[0]->type) {
|
|
1703
|
+
GGML_LOG_ERROR("ET: CONT operation type mismatch: src=%s dst=%s\n", ggml_type_name(node->src[0]->type),
|
|
1704
|
+
ggml_type_name(node->type));
|
|
1705
|
+
return false;
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
// Validate supported types
|
|
1709
|
+
if (node->type != GGML_TYPE_F32 && node->type != GGML_TYPE_F16) {
|
|
1710
|
+
GGML_LOG_ERROR("ET: CONT operation unsupported type: %s (only F32 and F16 supported)\n",
|
|
1711
|
+
ggml_type_name(node->type));
|
|
1712
|
+
return false;
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
// Validate contiguity - output must be contiguous, input can be non-contiguous
|
|
1716
|
+
if (!ggml_is_contiguous(node)) {
|
|
1717
|
+
GGML_LOG_ERROR("ET: CONT operation requires contiguous output tensor\n");
|
|
1718
|
+
return false;
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
// Select kernel based on type
|
|
1722
|
+
const char * kernel_name;
|
|
1723
|
+
if (node->type == GGML_TYPE_F32) {
|
|
1724
|
+
kernel_name = "cont_f32";
|
|
1725
|
+
} else if (node->type == GGML_TYPE_F16) {
|
|
1726
|
+
kernel_name = "cont_f16";
|
|
1727
|
+
} else {
|
|
1728
|
+
GGML_LOG_ERROR("ET: CONT operation with unsupported type: %s\n", ggml_type_name(node->type));
|
|
1729
|
+
return false;
|
|
1730
|
+
}
|
|
1731
|
+
|
|
1732
|
+
ggml_et_cont_params params;
|
|
1733
|
+
params.src0 = *node->src[0]; // Input tensor (potentially non-contiguous)
|
|
1734
|
+
params.dst = *node; // Output tensor (contiguous)
|
|
1735
|
+
|
|
1736
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
1737
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1738
|
+
bool cpu_comparison_active = false;
|
|
1739
|
+
if (cont_cpu_compare_config.enabled) {
|
|
1740
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CONT)) {
|
|
1741
|
+
cpu_comparison_active = true;
|
|
1742
|
+
} else {
|
|
1743
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CONT operation\n");
|
|
1744
|
+
}
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1748
|
+
|
|
1749
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
1750
|
+
if (cpu_comparison_active) {
|
|
1751
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &cont_cpu_compare_config)) {
|
|
1752
|
+
GGML_LOG_WARN("ET: CPU comparison failed for CONT operation\n");
|
|
1753
|
+
}
|
|
1754
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1755
|
+
}
|
|
1756
|
+
|
|
1757
|
+
ET_PERF_END("CONT", kernel_name, node);
|
|
1758
|
+
return kernel_result;
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1761
|
+
bool ggml_et_op_cumsum(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1762
|
+
ET_PERF_START();
|
|
1763
|
+
|
|
1764
|
+
if (!dev_ctx || !node || !node->src[0]) {
|
|
1765
|
+
GGML_LOG_ERROR("ET: Invalid parameters for CUMSUM operation\n");
|
|
1766
|
+
return false;
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
|
|
1770
|
+
GGML_LOG_ERROR("ET: CUMSUM operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
1771
|
+
ggml_type_name(node->src[0]->type));
|
|
1772
|
+
return false;
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
const char * kernel_name = "cumsum_f32";
|
|
1776
|
+
|
|
1777
|
+
ggml_et_cumsum_params params;
|
|
1778
|
+
params.src0 = *node->src[0];
|
|
1779
|
+
params.dst = *node;
|
|
1780
|
+
|
|
1781
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1782
|
+
bool cpu_comparison_active = false;
|
|
1783
|
+
if (cumsum_cpu_compare_config.enabled) {
|
|
1784
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CUMSUM)) {
|
|
1785
|
+
cpu_comparison_active = true;
|
|
1786
|
+
} else {
|
|
1787
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CUMSUM operation\n");
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1792
|
+
|
|
1793
|
+
if (cpu_comparison_active) {
|
|
1794
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &cumsum_cpu_compare_config)) {
|
|
1795
|
+
GGML_LOG_WARN("ET: CPU comparison failed for CUMSUM operation\n");
|
|
1796
|
+
}
|
|
1797
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
ET_PERF_END("CUMSUM", kernel_name, node);
|
|
1801
|
+
return kernel_result;
|
|
1802
|
+
}
|
|
1803
|
+
|
|
1804
|
+
bool ggml_et_op_cpy(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1805
|
+
ET_PERF_START();
|
|
1806
|
+
|
|
1807
|
+
// CPY copies data from src[0] into the layout of dst (which matches src[1])
|
|
1808
|
+
// For same-type with contiguous dst, this is identical to CONT
|
|
1809
|
+
if (!node->src[0]) {
|
|
1810
|
+
GGML_LOG_ERROR("ET: CPY operation missing source tensor\n");
|
|
1811
|
+
return false;
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
// Scalar / zero-element special path: if any dimension is 0, nothing to copy
|
|
1815
|
+
const int64_t nelements = node->ne[0] * node->ne[1] * node->ne[2] * node->ne[3];
|
|
1816
|
+
if (nelements == 0) {
|
|
1817
|
+
GGML_LOG_DEBUG("ET: CPY no-op (zero elements): ne=[%" PRId64 ",%" PRId64 ",%" PRId64 ",%" PRId64 "]\n",
|
|
1818
|
+
node->ne[0], node->ne[1], node->ne[2], node->ne[3]);
|
|
1819
|
+
ET_PERF_END("CPY", "noop", node);
|
|
1820
|
+
return true;
|
|
1821
|
+
}
|
|
1822
|
+
|
|
1823
|
+
// Only F32 and F16 supported for dst
|
|
1824
|
+
if (node->type != GGML_TYPE_F32 && node->type != GGML_TYPE_F16) {
|
|
1825
|
+
GGML_LOG_ERROR("ET: CPY unsupported dst type: %s\n", ggml_type_name(node->type));
|
|
1826
|
+
return false;
|
|
1827
|
+
}
|
|
1828
|
+
|
|
1829
|
+
// Select kernel based on src/dst type combination
|
|
1830
|
+
const char * kernel_name;
|
|
1831
|
+
if (node->src[0]->type == GGML_TYPE_F32 && node->type == GGML_TYPE_F32) {
|
|
1832
|
+
kernel_name = "cont_f32";
|
|
1833
|
+
} else if (node->src[0]->type == GGML_TYPE_F16 && node->type == GGML_TYPE_F16) {
|
|
1834
|
+
kernel_name = "cont_f16";
|
|
1835
|
+
} else if (node->src[0]->type == GGML_TYPE_F32 && node->type == GGML_TYPE_F16) {
|
|
1836
|
+
kernel_name = "cpy_f32_f16";
|
|
1837
|
+
} else {
|
|
1838
|
+
GGML_LOG_ERROR("ET: CPY unsupported type combination: src=%s dst=%s\n", ggml_type_name(node->src[0]->type),
|
|
1839
|
+
ggml_type_name(node->type));
|
|
1840
|
+
return false;
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1843
|
+
ggml_et_cont_params params;
|
|
1844
|
+
params.src0 = *node->src[0];
|
|
1845
|
+
params.dst = *node;
|
|
1846
|
+
|
|
1847
|
+
// CPU comparison for debugging
|
|
1848
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1849
|
+
bool cpu_comparison_active = false;
|
|
1850
|
+
if (cont_cpu_compare_config.enabled) {
|
|
1851
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CPY)) {
|
|
1852
|
+
cpu_comparison_active = true;
|
|
1853
|
+
} else {
|
|
1854
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CPY operation\n");
|
|
1855
|
+
}
|
|
1856
|
+
}
|
|
1857
|
+
|
|
1858
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1859
|
+
|
|
1860
|
+
if (cpu_comparison_active) {
|
|
1861
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &cont_cpu_compare_config)) {
|
|
1862
|
+
GGML_LOG_WARN("ET: CPU comparison failed for CPY operation\n");
|
|
1863
|
+
}
|
|
1864
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1865
|
+
}
|
|
1866
|
+
|
|
1867
|
+
ET_PERF_END("CPY", kernel_name, node);
|
|
1868
|
+
return kernel_result;
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
bool ggml_et_op_concat(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1872
|
+
ET_PERF_START();
|
|
1873
|
+
|
|
1874
|
+
if (!dev_ctx || !node) {
|
|
1875
|
+
GGML_LOG_ERROR("ET: Invalid parameters for CONCAT operation\n");
|
|
1876
|
+
return false;
|
|
1877
|
+
}
|
|
1878
|
+
|
|
1879
|
+
if (!node->src[0] || !node->src[1]) {
|
|
1880
|
+
GGML_LOG_ERROR("ET: CONCAT operation missing required inputs\n");
|
|
1881
|
+
return false;
|
|
1882
|
+
}
|
|
1883
|
+
|
|
1884
|
+
const char * kernel_name;
|
|
1885
|
+
|
|
1886
|
+
if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32) {
|
|
1887
|
+
kernel_name = "concat_f32";
|
|
1888
|
+
|
|
1889
|
+
} else {
|
|
1890
|
+
GGML_LOG_ERROR("ET: CONCAT operation with unsupported types: dst=%s src0=%s src1=%s\n",
|
|
1891
|
+
ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
|
|
1892
|
+
ggml_type_name(node->src[1]->type));
|
|
1893
|
+
return false;
|
|
1894
|
+
}
|
|
1895
|
+
|
|
1896
|
+
int32_t dim;
|
|
1897
|
+
memcpy(&dim, node->op_params, sizeof(int32_t));
|
|
1898
|
+
|
|
1899
|
+
ggml_et_concat_params params;
|
|
1900
|
+
params.src0 = *node->src[0];
|
|
1901
|
+
params.src1 = *node->src[1];
|
|
1902
|
+
params.dst = *node;
|
|
1903
|
+
params.dim = dim;
|
|
1904
|
+
|
|
1905
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
1906
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1907
|
+
bool cpu_comparison_active = false;
|
|
1908
|
+
if (concat_cpu_compare_config.enabled) {
|
|
1909
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_CONCAT)) {
|
|
1910
|
+
cpu_comparison_active = true;
|
|
1911
|
+
} else {
|
|
1912
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for CONCAT operation\n");
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1915
|
+
|
|
1916
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1917
|
+
|
|
1918
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
1919
|
+
if (cpu_comparison_active) {
|
|
1920
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &concat_cpu_compare_config)) {
|
|
1921
|
+
GGML_LOG_WARN("ET: CPU comparison failed for CONCAT operation\n");
|
|
1922
|
+
}
|
|
1923
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1924
|
+
}
|
|
1925
|
+
|
|
1926
|
+
ET_PERF_END_EXT("CONCAT", kernel_name, node, "dim=%d", dim);
|
|
1927
|
+
return kernel_result;
|
|
1928
|
+
}
|
|
1929
|
+
|
|
1930
|
+
bool ggml_et_op_repeat(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1931
|
+
ET_PERF_START();
|
|
1932
|
+
|
|
1933
|
+
if (!dev_ctx || !node) {
|
|
1934
|
+
GGML_LOG_ERROR("ET: Invalid parameters for REPEAT operation\n");
|
|
1935
|
+
return false;
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1938
|
+
if (!node->src[0]) {
|
|
1939
|
+
GGML_LOG_ERROR("ET: REPEAT operation missing required input\n");
|
|
1940
|
+
return false;
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1943
|
+
const char * kernel_name;
|
|
1944
|
+
|
|
1945
|
+
if (node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32) {
|
|
1946
|
+
// No-op REPEAT (every repeat factor is 1): the output is just a copy
|
|
1947
|
+
// of the input. Route to cont_f32, whose contiguous fast path handles
|
|
1948
|
+
// arbitrary sizes (including those rejected by repeat_f32's gate,
|
|
1949
|
+
// e.g. ne[0]=1).
|
|
1950
|
+
if (ggml_are_same_shape(node->src[0], node)) {
|
|
1951
|
+
kernel_name = "cont_f32";
|
|
1952
|
+
} else {
|
|
1953
|
+
kernel_name = "repeat_f32";
|
|
1954
|
+
}
|
|
1955
|
+
|
|
1956
|
+
} else {
|
|
1957
|
+
GGML_LOG_ERROR("ET: REPEAT operation with unsupported types: dst=%s src0=%s\n", ggml_type_name(node->type),
|
|
1958
|
+
ggml_type_name(node->src[0]->type));
|
|
1959
|
+
return false;
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
// ggml_et_cont_params and ggml_et_repeat_params have identical layouts
|
|
1963
|
+
// (just src0 + dst), so the same payload works for either kernel.
|
|
1964
|
+
ggml_et_repeat_params params;
|
|
1965
|
+
params.src0 = *node->src[0];
|
|
1966
|
+
params.dst = *node;
|
|
1967
|
+
|
|
1968
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
1969
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
1970
|
+
bool cpu_comparison_active = false;
|
|
1971
|
+
if (repeat_cpu_compare_config.enabled) {
|
|
1972
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_REPEAT)) {
|
|
1973
|
+
cpu_comparison_active = true;
|
|
1974
|
+
} else {
|
|
1975
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for REPEAT operation\n");
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
1980
|
+
|
|
1981
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
1982
|
+
if (cpu_comparison_active) {
|
|
1983
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &repeat_cpu_compare_config)) {
|
|
1984
|
+
GGML_LOG_WARN("ET: CPU comparison failed for REPEAT operation\n");
|
|
1985
|
+
}
|
|
1986
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
1987
|
+
}
|
|
1988
|
+
|
|
1989
|
+
ET_PERF_END("REPEAT", kernel_name, node);
|
|
1990
|
+
return kernel_result;
|
|
1991
|
+
}
|
|
1992
|
+
|
|
1993
|
+
bool ggml_et_op_ssm_conv(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
1994
|
+
ET_PERF_START();
|
|
1995
|
+
|
|
1996
|
+
if (!dev_ctx || !node || !node->src[0] || !node->src[1]) {
|
|
1997
|
+
GGML_LOG_ERROR("ET: Invalid parameters for SSM_CONV operation\n");
|
|
1998
|
+
return false;
|
|
1999
|
+
}
|
|
2000
|
+
|
|
2001
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32) {
|
|
2002
|
+
GGML_LOG_ERROR("ET: SSM_CONV operation with unsupported types: dst=%s src0=%s src1=%s\n",
|
|
2003
|
+
ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
|
|
2004
|
+
ggml_type_name(node->src[1]->type));
|
|
2005
|
+
return false;
|
|
2006
|
+
}
|
|
2007
|
+
|
|
2008
|
+
const char * kernel_name = "ssm_conv_f32";
|
|
2009
|
+
|
|
2010
|
+
ggml_et_ssm_conv_params params;
|
|
2011
|
+
params.src0 = *node->src[0];
|
|
2012
|
+
params.src1 = *node->src[1];
|
|
2013
|
+
params.dst = *node;
|
|
2014
|
+
|
|
2015
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
2016
|
+
bool cpu_comparison_active = false;
|
|
2017
|
+
if (ssm_conv_cpu_compare_config.enabled) {
|
|
2018
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SSM_CONV)) {
|
|
2019
|
+
cpu_comparison_active = true;
|
|
2020
|
+
} else {
|
|
2021
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SSM_CONV operation\n");
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
|
|
2025
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2026
|
+
|
|
2027
|
+
if (cpu_comparison_active) {
|
|
2028
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &ssm_conv_cpu_compare_config)) {
|
|
2029
|
+
GGML_LOG_WARN("ET: CPU comparison failed for SSM_CONV operation\n");
|
|
2030
|
+
}
|
|
2031
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
ET_PERF_END("SSM_CONV", kernel_name, node);
|
|
2035
|
+
return kernel_result;
|
|
2036
|
+
}
|
|
2037
|
+
|
|
2038
|
+
bool ggml_et_op_ssm_scan(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
2039
|
+
ET_PERF_START();
|
|
2040
|
+
|
|
2041
|
+
if (!dev_ctx || !node) {
|
|
2042
|
+
GGML_LOG_ERROR("ET: Invalid parameters for SSM_SCAN operation\n");
|
|
2043
|
+
return false;
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
for (int i = 0; i < 7; ++i) {
|
|
2047
|
+
if (!node->src[i]) {
|
|
2048
|
+
GGML_LOG_ERROR("ET: SSM_SCAN missing required input %d\n", i);
|
|
2049
|
+
return false;
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
|
|
2053
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32 ||
|
|
2054
|
+
node->src[2]->type != GGML_TYPE_F32 || node->src[3]->type != GGML_TYPE_F32 ||
|
|
2055
|
+
node->src[4]->type != GGML_TYPE_F32 || node->src[5]->type != GGML_TYPE_F32 ||
|
|
2056
|
+
node->src[6]->type != GGML_TYPE_I32) {
|
|
2057
|
+
GGML_LOG_ERROR("ET: SSM_SCAN operation with unsupported types\n");
|
|
2058
|
+
return false;
|
|
2059
|
+
}
|
|
2060
|
+
|
|
2061
|
+
ggml_et_ssm_scan_params params;
|
|
2062
|
+
params.src0 = *node->src[0];
|
|
2063
|
+
params.src1 = *node->src[1];
|
|
2064
|
+
params.src2 = *node->src[2];
|
|
2065
|
+
params.src3 = *node->src[3];
|
|
2066
|
+
params.src4 = *node->src[4];
|
|
2067
|
+
params.src5 = *node->src[5];
|
|
2068
|
+
params.src6 = *node->src[6];
|
|
2069
|
+
params.dst = *node;
|
|
2070
|
+
params.K = ggml_get_op_params_i32(node, 0);
|
|
2071
|
+
|
|
2072
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "ssm_scan_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2073
|
+
|
|
2074
|
+
ET_PERF_END("SSM_SCAN", "ssm_scan_f32", node);
|
|
2075
|
+
return kernel_result;
|
|
2076
|
+
}
|
|
2077
|
+
|
|
2078
|
+
bool ggml_et_op_rwkv_wkv6(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
2079
|
+
ET_PERF_START();
|
|
2080
|
+
|
|
2081
|
+
if (!dev_ctx || !node) {
|
|
2082
|
+
GGML_LOG_ERROR("ET: Invalid parameters for RWKV_WKV6 operation\n");
|
|
2083
|
+
return false;
|
|
2084
|
+
}
|
|
2085
|
+
|
|
2086
|
+
// Validate all 6 source tensors exist
|
|
2087
|
+
for (int i = 0; i <= 5; i++) {
|
|
2088
|
+
if (!node->src[i]) {
|
|
2089
|
+
GGML_LOG_ERROR("ET: RWKV_WKV6 operation missing src[%d]\n", i);
|
|
2090
|
+
return false;
|
|
2091
|
+
}
|
|
2092
|
+
}
|
|
2093
|
+
|
|
2094
|
+
if (node->type != GGML_TYPE_F32) {
|
|
2095
|
+
GGML_LOG_ERROR("ET: RWKV_WKV6 only supports F32, got %s\n", ggml_type_name(node->type));
|
|
2096
|
+
return false;
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
const char * kernel_name = "rwkv_wkv6_f32";
|
|
2100
|
+
|
|
2101
|
+
const int64_t S = node->src[0]->ne[0]; // head_size
|
|
2102
|
+
const int64_t H = node->src[0]->ne[1]; // num heads
|
|
2103
|
+
const int64_t T = node->src[1]->ne[2]; // num tokens
|
|
2104
|
+
const int64_t n_seqs = node->src[5]->ne[1]; // num sequences
|
|
2105
|
+
const int64_t C = S * H;
|
|
2106
|
+
|
|
2107
|
+
ggml_et_rwkv_wkv6_params params;
|
|
2108
|
+
params.k = (float *) node->src[0]->data;
|
|
2109
|
+
params.v = (float *) node->src[1]->data;
|
|
2110
|
+
params.r = (float *) node->src[2]->data;
|
|
2111
|
+
params.tf = (float *) node->src[3]->data;
|
|
2112
|
+
params.td = (float *) node->src[4]->data;
|
|
2113
|
+
params.state_in = (float *) node->src[5]->data;
|
|
2114
|
+
params.dst = (float *) node->data;
|
|
2115
|
+
params.C = (int32_t) C;
|
|
2116
|
+
params.H = (int32_t) H;
|
|
2117
|
+
params.S = (int32_t) S;
|
|
2118
|
+
params.T = (int32_t) T;
|
|
2119
|
+
params.n_seqs = (int32_t) n_seqs;
|
|
2120
|
+
|
|
2121
|
+
// Phase 1: Initialize CPU comparison context
|
|
2122
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
2123
|
+
bool cpu_comparison_active = false;
|
|
2124
|
+
if (rwkv_wkv6_cpu_compare_config.enabled) {
|
|
2125
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_RWKV_WKV6)) {
|
|
2126
|
+
cpu_comparison_active = true;
|
|
2127
|
+
} else {
|
|
2128
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for RWKV_WKV6 operation\n");
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
|
|
2132
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2133
|
+
|
|
2134
|
+
// Phase 2: Execute CPU computation and compare
|
|
2135
|
+
if (cpu_comparison_active) {
|
|
2136
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rwkv_wkv6_cpu_compare_config)) {
|
|
2137
|
+
GGML_LOG_WARN("ET: CPU comparison failed for RWKV_WKV6 operation\n");
|
|
2138
|
+
}
|
|
2139
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
ET_PERF_END_EXT("RWKV_WKV6", kernel_name, node, "S=%d H=%d T=%d n_seqs=%d", (int) S, (int) H, (int) T,
|
|
2143
|
+
(int) n_seqs);
|
|
2144
|
+
return kernel_result;
|
|
2145
|
+
}
|
|
2146
|
+
|
|
2147
|
+
bool ggml_et_op_rwkv_wkv7(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
2148
|
+
ET_PERF_START();
|
|
2149
|
+
|
|
2150
|
+
if (!dev_ctx || !node) {
|
|
2151
|
+
GGML_LOG_ERROR("ET: Invalid parameters for RWKV_WKV7 operation\n");
|
|
2152
|
+
return false;
|
|
2153
|
+
}
|
|
2154
|
+
|
|
2155
|
+
// Validate all 7 source tensors exist
|
|
2156
|
+
for (int i = 0; i <= 6; i++) {
|
|
2157
|
+
if (!node->src[i]) {
|
|
2158
|
+
GGML_LOG_ERROR("ET: RWKV_WKV7 operation missing src[%d]\n", i);
|
|
2159
|
+
return false;
|
|
2160
|
+
}
|
|
2161
|
+
}
|
|
2162
|
+
|
|
2163
|
+
if (node->type != GGML_TYPE_F32) {
|
|
2164
|
+
GGML_LOG_ERROR("ET: RWKV_WKV7 only supports F32, got %s\n", ggml_type_name(node->type));
|
|
2165
|
+
return false;
|
|
2166
|
+
}
|
|
2167
|
+
|
|
2168
|
+
const char * kernel_name = "rwkv_wkv7_f32";
|
|
2169
|
+
|
|
2170
|
+
const int64_t S = node->src[2]->ne[0]; // head_size
|
|
2171
|
+
const int64_t H = node->src[2]->ne[1]; // num heads
|
|
2172
|
+
const int64_t T = node->src[1]->ne[2]; // num tokens
|
|
2173
|
+
const int64_t n_seqs = node->src[6]->ne[1]; // num sequences
|
|
2174
|
+
const int64_t C = S * H;
|
|
2175
|
+
|
|
2176
|
+
ggml_et_rwkv_wkv7_params params;
|
|
2177
|
+
params.r = (float *) node->src[0]->data;
|
|
2178
|
+
params.w = (float *) node->src[1]->data;
|
|
2179
|
+
params.k = (float *) node->src[2]->data;
|
|
2180
|
+
params.v = (float *) node->src[3]->data;
|
|
2181
|
+
params.a = (float *) node->src[4]->data;
|
|
2182
|
+
params.b = (float *) node->src[5]->data;
|
|
2183
|
+
params.state_in = (float *) node->src[6]->data;
|
|
2184
|
+
params.dst = (float *) node->data;
|
|
2185
|
+
params.C = (int32_t) C;
|
|
2186
|
+
params.H = (int32_t) H;
|
|
2187
|
+
params.S = (int32_t) S;
|
|
2188
|
+
params.T = (int32_t) T;
|
|
2189
|
+
params.n_seqs = (int32_t) n_seqs;
|
|
2190
|
+
|
|
2191
|
+
// Phase 1: Initialize CPU comparison context
|
|
2192
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
2193
|
+
bool cpu_comparison_active = false;
|
|
2194
|
+
if (rwkv_wkv7_cpu_compare_config.enabled) {
|
|
2195
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_RWKV_WKV7)) {
|
|
2196
|
+
cpu_comparison_active = true;
|
|
2197
|
+
} else {
|
|
2198
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for RWKV_WKV7 operation\n");
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2202
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2203
|
+
|
|
2204
|
+
// Phase 2: Execute CPU computation and compare
|
|
2205
|
+
if (cpu_comparison_active) {
|
|
2206
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &rwkv_wkv7_cpu_compare_config)) {
|
|
2207
|
+
GGML_LOG_WARN("ET: CPU comparison failed for RWKV_WKV7 operation\n");
|
|
2208
|
+
}
|
|
2209
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
2210
|
+
}
|
|
2211
|
+
|
|
2212
|
+
ET_PERF_END_EXT("RWKV_WKV7", kernel_name, node, "S=%d H=%d T=%d n_seqs=%d", (int) S, (int) H, (int) T,
|
|
2213
|
+
(int) n_seqs);
|
|
2214
|
+
return kernel_result;
|
|
2215
|
+
}
|
|
2216
|
+
|
|
2217
|
+
static ggml_et_cpu_compare_config gated_delta_net_cpu_compare_config = {
|
|
2218
|
+
/* .enabled = */ false,
|
|
2219
|
+
/* .use_cpu_result = */ false,
|
|
2220
|
+
/* .log_differences = */ true,
|
|
2221
|
+
/* .tolerance = */ 1e-4f,
|
|
2222
|
+
/* .max_log_elements = */ 4096
|
|
2223
|
+
};
|
|
2224
|
+
|
|
2225
|
+
bool ggml_et_op_gated_delta_net(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
2226
|
+
ET_PERF_START();
|
|
2227
|
+
|
|
2228
|
+
if (!dev_ctx || !node) {
|
|
2229
|
+
GGML_LOG_ERROR("ET: Invalid parameters for GATED_DELTA_NET operation\n");
|
|
2230
|
+
return false;
|
|
2231
|
+
}
|
|
2232
|
+
|
|
2233
|
+
// Validate all 6 source tensors exist
|
|
2234
|
+
for (int i = 0; i <= 5; i++) {
|
|
2235
|
+
if (!node->src[i]) {
|
|
2236
|
+
GGML_LOG_ERROR("ET: GATED_DELTA_NET operation missing src[%d]\n", i);
|
|
2237
|
+
return false;
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
|
|
2241
|
+
if (node->type != GGML_TYPE_F32) {
|
|
2242
|
+
GGML_LOG_ERROR("ET: GATED_DELTA_NET only supports F32, got %s\n", ggml_type_name(node->type));
|
|
2243
|
+
return false;
|
|
2244
|
+
}
|
|
2245
|
+
|
|
2246
|
+
const char * kernel_name = "gated_delta_net_f32";
|
|
2247
|
+
|
|
2248
|
+
const ggml_tensor * src_q = node->src[0];
|
|
2249
|
+
const ggml_tensor * src_k = node->src[1];
|
|
2250
|
+
const ggml_tensor * src_v = node->src[2];
|
|
2251
|
+
const ggml_tensor * src_g = node->src[3];
|
|
2252
|
+
const ggml_tensor * src_beta = node->src[4];
|
|
2253
|
+
const ggml_tensor * src_state = node->src[5];
|
|
2254
|
+
|
|
2255
|
+
const int64_t S_v = src_v->ne[0];
|
|
2256
|
+
const int64_t H = src_v->ne[1];
|
|
2257
|
+
const int64_t n_tokens = src_v->ne[2];
|
|
2258
|
+
const int64_t n_seqs = src_v->ne[3];
|
|
2259
|
+
const int64_t H_q = src_q->ne[1];
|
|
2260
|
+
const int64_t H_k = src_k->ne[1];
|
|
2261
|
+
const int64_t n_seqs_q = src_q->ne[3];
|
|
2262
|
+
const int64_t n_seqs_k = src_k->ne[3];
|
|
2263
|
+
|
|
2264
|
+
ggml_et_gated_delta_net_params params;
|
|
2265
|
+
params.q = *src_q;
|
|
2266
|
+
params.k = *src_k;
|
|
2267
|
+
params.v = *src_v;
|
|
2268
|
+
params.g = *src_g;
|
|
2269
|
+
params.beta = *src_beta;
|
|
2270
|
+
params.state_in = *src_state;
|
|
2271
|
+
params.dst = *node;
|
|
2272
|
+
params.S_v = (int32_t) S_v;
|
|
2273
|
+
params.H = (int32_t) H;
|
|
2274
|
+
params.H_q = (int32_t) H_q;
|
|
2275
|
+
params.H_k = (int32_t) H_k;
|
|
2276
|
+
params.n_tokens = (int32_t) n_tokens;
|
|
2277
|
+
params.n_seqs = (int32_t) n_seqs;
|
|
2278
|
+
params.n_seqs_q = (int32_t) n_seqs_q;
|
|
2279
|
+
params.n_seqs_k = (int32_t) n_seqs_k;
|
|
2280
|
+
params.kda = (src_g->ne[0] == S_v) ? 1 : 0;
|
|
2281
|
+
params.K = ggml_get_op_params_i32(node, 0);
|
|
2282
|
+
params.scale = 1.0f / sqrtf((float) S_v);
|
|
2283
|
+
|
|
2284
|
+
// CPU comparison for debugging
|
|
2285
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
2286
|
+
bool cpu_comparison_active = false;
|
|
2287
|
+
if (gated_delta_net_cpu_compare_config.enabled) {
|
|
2288
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_GATED_DELTA_NET)) {
|
|
2289
|
+
cpu_comparison_active = true;
|
|
2290
|
+
} else {
|
|
2291
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for GATED_DELTA_NET operation\n");
|
|
2292
|
+
}
|
|
2293
|
+
}
|
|
2294
|
+
|
|
2295
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2296
|
+
|
|
2297
|
+
if (cpu_comparison_active) {
|
|
2298
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &gated_delta_net_cpu_compare_config)) {
|
|
2299
|
+
GGML_LOG_WARN("ET: CPU comparison failed for GATED_DELTA_NET operation\n");
|
|
2300
|
+
}
|
|
2301
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
2302
|
+
}
|
|
2303
|
+
|
|
2304
|
+
ET_PERF_END_EXT("GATED_DELTA_NET", kernel_name, node, "S_v=%d H=%d n_tokens=%d n_seqs=%d kda=%d", (int) S_v,
|
|
2305
|
+
(int) H, (int) n_tokens, (int) n_seqs, params.kda);
|
|
2306
|
+
return kernel_result;
|
|
2307
|
+
}
|
|
2308
|
+
|
|
2309
|
+
bool ggml_et_op_set_rows(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
2310
|
+
ET_PERF_START();
|
|
2311
|
+
|
|
2312
|
+
if (!dev_ctx || !node) {
|
|
2313
|
+
GGML_LOG_ERROR("ET: Invalid parameters for SET_ROWS operation\n");
|
|
2314
|
+
return false;
|
|
2315
|
+
}
|
|
2316
|
+
|
|
2317
|
+
if (!node->src[0] || !node->src[1] || !node->src[2]) {
|
|
2318
|
+
GGML_LOG_ERROR(
|
|
2319
|
+
"ET: SET_ROWS operation missing required inputs (needs src[0]=base, src[1]=indices, src[2]=data)\n");
|
|
2320
|
+
return false;
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2323
|
+
const char * kernel_name;
|
|
2324
|
+
|
|
2325
|
+
// Support F32 data with I64 indices -> F32/F16 output (scatter operation)
|
|
2326
|
+
if (node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_I64 &&
|
|
2327
|
+
(node->type == GGML_TYPE_F32 || node->type == GGML_TYPE_F16)) {
|
|
2328
|
+
if (node->type == GGML_TYPE_F32 || node->type == GGML_TYPE_F16) {
|
|
2329
|
+
kernel_name = "set_rows_f32";
|
|
2330
|
+
} else {
|
|
2331
|
+
GGML_LOG_ERROR("ET: SET_ROWS unsupported output type: %s\n", ggml_type_name(node->type));
|
|
2332
|
+
return false;
|
|
2333
|
+
}
|
|
2334
|
+
|
|
2335
|
+
} else {
|
|
2336
|
+
GGML_LOG_ERROR("ET: SET_ROWS operation with unsupported types: dst=%s src0=%s src1=%s\n",
|
|
2337
|
+
ggml_type_name(node->type), ggml_type_name(node->src[0]->type),
|
|
2338
|
+
ggml_type_name(node->src[1]->type));
|
|
2339
|
+
return false;
|
|
2340
|
+
}
|
|
2341
|
+
|
|
2342
|
+
// Validate contiguity requirements
|
|
2343
|
+
if (!ggml_is_contiguous_rows(node)) {
|
|
2344
|
+
GGML_LOG_ERROR("ET: SET_ROWS operation requires contiguous-rows destination tensor\n");
|
|
2345
|
+
return false;
|
|
2346
|
+
}
|
|
2347
|
+
|
|
2348
|
+
if (!ggml_is_contiguous_rows(node->src[0])) {
|
|
2349
|
+
GGML_LOG_ERROR("ET: SET_ROWS operation requires contiguous-rows source tensor\n");
|
|
2350
|
+
return false;
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
if (!ggml_is_contiguous(node->src[1])) {
|
|
2354
|
+
GGML_LOG_ERROR("ET: SET_ROWS operation requires contiguous indices tensor\n");
|
|
2355
|
+
return false;
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2358
|
+
// Validate dimension constraints from ggml implementation
|
|
2359
|
+
if (!(node->ne[0] == node->src[0]->ne[0] && // same number of columns
|
|
2360
|
+
node->ne[2] == node->src[0]->ne[2] && // same batch size
|
|
2361
|
+
node->ne[3] == node->src[0]->ne[3] && // same outer dimension
|
|
2362
|
+
node->src[0]->ne[1] == node->src[1]->ne[0] && // src rows = index count
|
|
2363
|
+
node->src[0]->ne[2] % node->src[1]->ne[1] == 0 && // batch constraint
|
|
2364
|
+
node->src[0]->ne[3] % node->src[1]->ne[2] == 0 && // outer constraint
|
|
2365
|
+
node->src[1]->ne[3] == 1)) { // indices constraint
|
|
2366
|
+
GGML_LOG_ERROR("ET: SET_ROWS operation dimension constraint failed\n");
|
|
2367
|
+
return false;
|
|
2368
|
+
}
|
|
2369
|
+
|
|
2370
|
+
ggml_et_set_rows_params params;
|
|
2371
|
+
params.src0 = *node->src[0]; // F32 source data tensor
|
|
2372
|
+
params.src1 = *node->src[1]; // I64 indices tensor
|
|
2373
|
+
params.dst = *node; // F32/F16 destination tensor
|
|
2374
|
+
|
|
2375
|
+
// Phase 1: Initialize CPU comparison context and copy source buffers (before ET kernel)
|
|
2376
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
2377
|
+
bool cpu_comparison_active = false;
|
|
2378
|
+
if (set_rows_cpu_compare_config.enabled) {
|
|
2379
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_SET_ROWS)) {
|
|
2380
|
+
cpu_comparison_active = true;
|
|
2381
|
+
} else {
|
|
2382
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for SET_ROWS operation\n");
|
|
2383
|
+
}
|
|
2384
|
+
}
|
|
2385
|
+
|
|
2386
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, kernel_name, ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2387
|
+
|
|
2388
|
+
// Phase 2: Execute CPU computation and compare with ET result (after ET kernel)
|
|
2389
|
+
if (cpu_comparison_active) {
|
|
2390
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &set_rows_cpu_compare_config)) {
|
|
2391
|
+
GGML_LOG_WARN("ET: CPU comparison failed for SET_ROWS operation\n");
|
|
2392
|
+
}
|
|
2393
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
2394
|
+
}
|
|
2395
|
+
|
|
2396
|
+
ET_PERF_END("SET_ROWS", kernel_name, node);
|
|
2397
|
+
return kernel_result;
|
|
2398
|
+
}
|
|
2399
|
+
|
|
2400
|
+
bool ggml_et_op_fill(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
2401
|
+
ET_PERF_START();
|
|
2402
|
+
|
|
2403
|
+
ggml_et_fill_params params;
|
|
2404
|
+
params.dst = *node;
|
|
2405
|
+
memcpy(¶ms.c, node->op_params, sizeof(float));
|
|
2406
|
+
|
|
2407
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "fill_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2408
|
+
|
|
2409
|
+
ET_PERF_END("FILL", "fill_f32", node);
|
|
2410
|
+
return kernel_result;
|
|
2411
|
+
}
|
|
2412
|
+
|
|
2413
|
+
bool ggml_et_op_diag(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
2414
|
+
ET_PERF_START();
|
|
2415
|
+
|
|
2416
|
+
if (!node->src[0]) {
|
|
2417
|
+
GGML_LOG_ERROR("ET: DIAG operation missing source tensor\n");
|
|
2418
|
+
return false;
|
|
2419
|
+
}
|
|
2420
|
+
|
|
2421
|
+
ggml_et_diag_params params;
|
|
2422
|
+
params.src0 = *node->src[0];
|
|
2423
|
+
params.dst = *node;
|
|
2424
|
+
|
|
2425
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "diag_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2426
|
+
|
|
2427
|
+
ET_PERF_END("DIAG", "diag_f32", node);
|
|
2428
|
+
return kernel_result;
|
|
2429
|
+
}
|
|
2430
|
+
|
|
2431
|
+
bool ggml_et_op_tri(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
2432
|
+
ET_PERF_START();
|
|
2433
|
+
|
|
2434
|
+
if (!node->src[0]) {
|
|
2435
|
+
GGML_LOG_ERROR("ET: TRI operation missing source tensor\n");
|
|
2436
|
+
return false;
|
|
2437
|
+
}
|
|
2438
|
+
|
|
2439
|
+
ggml_et_tri_params params;
|
|
2440
|
+
params.src0 = *node->src[0];
|
|
2441
|
+
params.dst = *node;
|
|
2442
|
+
memcpy(¶ms.tri_type, node->op_params, sizeof(int32_t));
|
|
2443
|
+
|
|
2444
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "tri_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2445
|
+
|
|
2446
|
+
ET_PERF_END("TRI", "tri_f32", node);
|
|
2447
|
+
return kernel_result;
|
|
2448
|
+
}
|
|
2449
|
+
|
|
2450
|
+
bool ggml_et_op_solve_tri(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
2451
|
+
ET_PERF_START();
|
|
2452
|
+
|
|
2453
|
+
if (!node->src[0] || !node->src[1]) {
|
|
2454
|
+
GGML_LOG_ERROR("ET: SOLVE_TRI operation missing source tensor(s)\n");
|
|
2455
|
+
return false;
|
|
2456
|
+
}
|
|
2457
|
+
|
|
2458
|
+
ggml_et_solve_tri_params params;
|
|
2459
|
+
params.src0 = *node->src[0]; // A (lower-triangular)
|
|
2460
|
+
params.src1 = *node->src[1]; // B (RHS)
|
|
2461
|
+
params.dst = *node; // X (solution)
|
|
2462
|
+
|
|
2463
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "solve_tri_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2464
|
+
|
|
2465
|
+
ET_PERF_END("SOLVE_TRI", "solve_tri_f32", node);
|
|
2466
|
+
return kernel_result;
|
|
2467
|
+
}
|
|
2468
|
+
|
|
2469
|
+
bool ggml_et_op_set(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
2470
|
+
ET_PERF_START();
|
|
2471
|
+
|
|
2472
|
+
if (!node->src[0] || !node->src[1]) {
|
|
2473
|
+
GGML_LOG_ERROR("ET: SET operation missing source tensor(s)\n");
|
|
2474
|
+
return false;
|
|
2475
|
+
}
|
|
2476
|
+
|
|
2477
|
+
const bool inplace = (bool) ((const int32_t *) node->op_params)[4];
|
|
2478
|
+
const size_t offset = ((const int32_t *) node->op_params)[3];
|
|
2479
|
+
const size_t nb1 = ((const int32_t *) node->op_params)[0];
|
|
2480
|
+
const size_t nb2 = ((const int32_t *) node->op_params)[1];
|
|
2481
|
+
const size_t nb3 = ((const int32_t *) node->op_params)[2];
|
|
2482
|
+
|
|
2483
|
+
if (!inplace) {
|
|
2484
|
+
GGML_LOG_ERROR("ET: SET only supports inplace (inplace=%d)\n", inplace);
|
|
2485
|
+
return false;
|
|
2486
|
+
}
|
|
2487
|
+
|
|
2488
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32 || node->src[1]->type != GGML_TYPE_F32) {
|
|
2489
|
+
GGML_LOG_ERROR("ET: SET only supports F32 (dst=%s src0=%s src1=%s)\n", ggml_type_name(node->type),
|
|
2490
|
+
ggml_type_name(node->src[0]->type), ggml_type_name(node->src[1]->type));
|
|
2491
|
+
return false;
|
|
2492
|
+
}
|
|
2493
|
+
|
|
2494
|
+
if (!ggml_are_same_shape(node, node->src[0])) {
|
|
2495
|
+
GGML_LOG_ERROR("ET: SET requires same-shape src0 and dst\n");
|
|
2496
|
+
return false;
|
|
2497
|
+
}
|
|
2498
|
+
|
|
2499
|
+
if (!ggml_is_contiguous(node) || !ggml_is_contiguous(node->src[0]) || !ggml_is_contiguous(node->src[1])) {
|
|
2500
|
+
GGML_LOG_ERROR("ET: SET requires contiguous dst, src0, and src1\n");
|
|
2501
|
+
return false;
|
|
2502
|
+
}
|
|
2503
|
+
|
|
2504
|
+
ggml_et_set_params params;
|
|
2505
|
+
params.src1 = *node->src[1];
|
|
2506
|
+
params.dst = *node;
|
|
2507
|
+
params.nb1 = (int32_t) nb1;
|
|
2508
|
+
params.nb2 = (int32_t) nb2;
|
|
2509
|
+
params.nb3 = (int32_t) nb3;
|
|
2510
|
+
params.offset = (int32_t) offset;
|
|
2511
|
+
|
|
2512
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "set_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2513
|
+
|
|
2514
|
+
ET_PERF_END("SET", "set_f32", node);
|
|
2515
|
+
return kernel_result;
|
|
2516
|
+
}
|
|
2517
|
+
|
|
2518
|
+
bool ggml_et_op_pad(ggml_backend_et_device_context * dev_ctx, const ggml_tensor * node) {
|
|
2519
|
+
ET_PERF_START();
|
|
2520
|
+
|
|
2521
|
+
if (!node->src[0]) {
|
|
2522
|
+
GGML_LOG_ERROR("ET: PAD operation missing source tensor\n");
|
|
2523
|
+
return false;
|
|
2524
|
+
}
|
|
2525
|
+
|
|
2526
|
+
if (node->type != GGML_TYPE_F32 || node->src[0]->type != GGML_TYPE_F32) {
|
|
2527
|
+
GGML_LOG_ERROR("ET: PAD only supports F32 (src=%s dst=%s)\n", ggml_type_name(node->src[0]->type),
|
|
2528
|
+
ggml_type_name(node->type));
|
|
2529
|
+
return false;
|
|
2530
|
+
}
|
|
2531
|
+
|
|
2532
|
+
if (!ggml_is_contiguous(node)) {
|
|
2533
|
+
GGML_LOG_ERROR("ET: PAD requires contiguous output tensor\n");
|
|
2534
|
+
return false;
|
|
2535
|
+
}
|
|
2536
|
+
|
|
2537
|
+
if (node->src[0]->nb[0] != sizeof(float)) {
|
|
2538
|
+
GGML_LOG_ERROR("ET: PAD requires element-contiguous src dim0 (nb[0]=%zu)\n", (size_t) node->src[0]->nb[0]);
|
|
2539
|
+
return false;
|
|
2540
|
+
}
|
|
2541
|
+
|
|
2542
|
+
// Extract padding parameters from op_params
|
|
2543
|
+
const int32_t * op_params = (const int32_t *) node->op_params;
|
|
2544
|
+
|
|
2545
|
+
ggml_et_pad_params params;
|
|
2546
|
+
params.src0 = *node->src[0];
|
|
2547
|
+
params.dst = *node;
|
|
2548
|
+
params.lp[0] = op_params[0];
|
|
2549
|
+
params.rp[0] = op_params[1];
|
|
2550
|
+
params.lp[1] = op_params[2];
|
|
2551
|
+
params.rp[1] = op_params[3];
|
|
2552
|
+
params.lp[2] = op_params[4];
|
|
2553
|
+
params.rp[2] = op_params[5];
|
|
2554
|
+
params.lp[3] = op_params[6];
|
|
2555
|
+
params.rp[3] = op_params[7];
|
|
2556
|
+
|
|
2557
|
+
// v1: no dim0 padding
|
|
2558
|
+
if (params.lp[0] != 0 || params.rp[0] != 0) {
|
|
2559
|
+
GGML_LOG_ERROR("ET: PAD dim0 padding not supported (lp0=%d rp0=%d)\n", params.lp[0], params.rp[0]);
|
|
2560
|
+
return false;
|
|
2561
|
+
}
|
|
2562
|
+
|
|
2563
|
+
ggml_et_cpu_compare_ctx cpu_cmp_ctx;
|
|
2564
|
+
bool cpu_comparison_active = false;
|
|
2565
|
+
if (pad_cpu_compare_config.enabled) {
|
|
2566
|
+
if (ggml_et_cpu_compare_init_pre(&cpu_cmp_ctx, node, GGML_OP_PAD)) {
|
|
2567
|
+
cpu_comparison_active = true;
|
|
2568
|
+
} else {
|
|
2569
|
+
GGML_LOG_WARN("ET: Failed to initialize CPU comparison for PAD operation\n");
|
|
2570
|
+
}
|
|
2571
|
+
}
|
|
2572
|
+
|
|
2573
|
+
bool kernel_result = ggml_et_launch_kernel(dev_ctx, "pad_f32", ¶ms, sizeof(params), 0xFFFFFFFF);
|
|
2574
|
+
|
|
2575
|
+
if (cpu_comparison_active) {
|
|
2576
|
+
if (!ggml_et_cpu_compare_compute_and_check(&cpu_cmp_ctx, node, &pad_cpu_compare_config)) {
|
|
2577
|
+
GGML_LOG_WARN("ET: CPU comparison failed for PAD operation\n");
|
|
2578
|
+
}
|
|
2579
|
+
ggml_et_cpu_compare_free(&cpu_cmp_ctx);
|
|
2580
|
+
}
|
|
2581
|
+
|
|
2582
|
+
ET_PERF_END("PAD", "pad_f32", node);
|
|
2583
|
+
return kernel_result;
|
|
2584
|
+
}
|