elelem-ggml 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/exe/elelem-ggml +8 -0
- data/ext/elelem/ggml/elelem.c +466 -0
- data/ext/elelem/ggml/extconf.rb +52 -0
- data/lib/elelem/ggml/chat_ml.rb +23 -0
- data/lib/elelem/ggml/gguf.rb +82 -0
- data/lib/elelem/ggml/provider.rb +63 -0
- data/lib/elelem/ggml/tokenizer.rb +95 -0
- data/lib/elelem/ggml/version.rb +7 -0
- data/lib/elelem/ggml.rb +37 -0
- data/vendor/ggml/AUTHORS +335 -0
- data/vendor/ggml/CMakeLists.txt +499 -0
- data/vendor/ggml/CONTRIBUTING.md +3 -0
- data/vendor/ggml/LICENSE +21 -0
- data/vendor/ggml/README.md +49 -0
- data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
- data/vendor/ggml/cmake/GitVars.cmake +22 -0
- data/vendor/ggml/cmake/common.cmake +50 -0
- data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
- data/vendor/ggml/ggml.pc.in +10 -0
- data/vendor/ggml/include/ggml-alloc.h +86 -0
- data/vendor/ggml/include/ggml-backend.h +437 -0
- data/vendor/ggml/include/ggml-blas.h +25 -0
- data/vendor/ggml/include/ggml-cann.h +123 -0
- data/vendor/ggml/include/ggml-cpp.h +39 -0
- data/vendor/ggml/include/ggml-cpu.h +152 -0
- data/vendor/ggml/include/ggml-cuda.h +47 -0
- data/vendor/ggml/include/ggml-et.h +28 -0
- data/vendor/ggml/include/ggml-hexagon.h +19 -0
- data/vendor/ggml/include/ggml-metal.h +61 -0
- data/vendor/ggml/include/ggml-opencl.h +26 -0
- data/vendor/ggml/include/ggml-openvino.h +37 -0
- data/vendor/ggml/include/ggml-opt.h +256 -0
- data/vendor/ggml/include/ggml-rpc.h +35 -0
- data/vendor/ggml/include/ggml-sycl.h +57 -0
- data/vendor/ggml/include/ggml-virtgpu.h +14 -0
- data/vendor/ggml/include/ggml-vulkan.h +29 -0
- data/vendor/ggml/include/ggml-webgpu.h +19 -0
- data/vendor/ggml/include/ggml-zdnn.h +17 -0
- data/vendor/ggml/include/ggml-zendnn.h +22 -0
- data/vendor/ggml/include/ggml.h +2963 -0
- data/vendor/ggml/include/gguf.h +211 -0
- data/vendor/ggml/requirements.txt +12 -0
- data/vendor/ggml/scripts/gen-authors.sh +9 -0
- data/vendor/ggml/scripts/make-release-checks.sh +96 -0
- data/vendor/ggml/scripts/make-release-desc.sh +69 -0
- data/vendor/ggml/scripts/make-release-summary.txt +33 -0
- data/vendor/ggml/scripts/release.sh +195 -0
- data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
- data/vendor/ggml/scripts/sync-llama.last +1 -0
- data/vendor/ggml/scripts/sync-llama.sh +21 -0
- data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
- data/vendor/ggml/scripts/sync-whisper.last +1 -0
- data/vendor/ggml/scripts/sync-whisper.sh +17 -0
- data/vendor/ggml/src/CMakeLists.txt +634 -0
- data/vendor/ggml/src/ggml-alloc.c +1249 -0
- data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
- data/vendor/ggml/src/ggml-backend-dl.h +45 -0
- data/vendor/ggml/src/ggml-backend-impl.h +291 -0
- data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
- data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
- data/vendor/ggml/src/ggml-backend.cpp +2517 -0
- data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
- data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
- data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
- data/vendor/ggml/src/ggml-cann/common.h +651 -0
- data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
- data/vendor/ggml/src/ggml-common.h +1911 -0
- data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
- data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
- data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
- data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
- data/vendor/ggml/src/ggml-cpu/common.h +95 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
- data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
- data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
- data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
- data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
- data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
- data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
- data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
- data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
- data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
- data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
- data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
- data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
- data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
- data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
- data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
- data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
- data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
- data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
- data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
- data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
- data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
- data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
- data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
- data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
- data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
- data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
- data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
- data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
- data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
- data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
- data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
- data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
- data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
- data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
- data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
- data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
- data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
- data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
- data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
- data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
- data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
- data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
- data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
- data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
- data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
- data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
- data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
- data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
- data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
- data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
- data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
- data/vendor/ggml/src/ggml-feats.h +166 -0
- data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
- data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
- data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
- data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
- data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
- data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
- data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
- data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
- data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
- data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
- data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
- data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
- data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
- data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
- data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
- data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
- data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
- data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
- data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
- data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
- data/vendor/ggml/src/ggml-impl.h +783 -0
- data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
- data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
- data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
- data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
- data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
- data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
- data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
- data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
- data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
- data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
- data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
- data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
- data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
- data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
- data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
- data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
- data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
- data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
- data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
- data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
- data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
- data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
- data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
- data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
- data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
- data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
- data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
- data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
- data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
- data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
- data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
- data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
- data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
- data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
- data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
- data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
- data/vendor/ggml/src/ggml-opt.cpp +1094 -0
- data/vendor/ggml/src/ggml-quants.c +5667 -0
- data/vendor/ggml/src/ggml-quants.h +115 -0
- data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
- data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
- data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
- data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
- data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
- data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
- data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
- data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
- data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
- data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
- data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
- data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
- data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
- data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
- data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
- data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
- data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
- data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
- data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
- data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
- data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
- data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
- data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
- data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
- data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
- data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
- data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
- data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
- data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
- data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
- data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
- data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
- data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
- data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
- data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
- data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
- data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
- data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
- data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
- data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
- data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
- data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
- data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
- data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
- data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
- data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
- data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
- data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
- data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
- data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/vendor/ggml/src/ggml-threading.cpp +12 -0
- data/vendor/ggml/src/ggml-threading.h +14 -0
- data/vendor/ggml/src/ggml-version.h.in +4 -0
- data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
- data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
- data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
- data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
- data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
- data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
- data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
- data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
- data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
- data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
- data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
- data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
- data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
- data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
- data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
- data/vendor/ggml/src/ggml.c +8088 -0
- data/vendor/ggml/src/ggml.cpp +26 -0
- data/vendor/ggml/src/gguf.cpp +1706 -0
- metadata +1459 -0
|
@@ -0,0 +1,2473 @@
|
|
|
1
|
+
#import "ggml-metal-device.h"
|
|
2
|
+
|
|
3
|
+
#import "ggml-impl.h"
|
|
4
|
+
#import "ggml-backend-impl.h"
|
|
5
|
+
#import "ggml-metal-impl.h"
|
|
6
|
+
#import "ggml-metal-common.h"
|
|
7
|
+
|
|
8
|
+
#include <Foundation/Foundation.h>
|
|
9
|
+
|
|
10
|
+
#include <Metal/Metal.h>
|
|
11
|
+
|
|
12
|
+
#include <stdatomic.h>
|
|
13
|
+
|
|
14
|
+
#ifndef TARGET_OS_VISION
|
|
15
|
+
#define TARGET_OS_VISION 0
|
|
16
|
+
#endif
|
|
17
|
+
|
|
18
|
+
// create residency sets only on macOS >= 15.0
|
|
19
|
+
#if !TARGET_CPU_X86_64 && TARGET_OS_OSX && __MAC_OS_X_VERSION_MAX_ALLOWED >= 150000 || \
|
|
20
|
+
TARGET_OS_IOS && __IPHONE_OS_VERSION_MAX_ALLOWED >= 180000 || \
|
|
21
|
+
TARGET_OS_TV && __TV_OS_VERSION_MAX_ALLOWED >= 180000 || \
|
|
22
|
+
TARGET_OS_VISION && __VISION_OS_VERSION_MAX_ALLOWED >= 200000
|
|
23
|
+
#define GGML_METAL_HAS_RESIDENCY_SETS 1
|
|
24
|
+
#endif
|
|
25
|
+
|
|
26
|
+
// overload of MTLGPUFamilyMetalX (not available in some environments)
|
|
27
|
+
static const NSInteger MTLGPUFamilyMetal3_GGML = 5001;
|
|
28
|
+
static const NSInteger MTLGPUFamilyMetal4_GGML = 5002;
|
|
29
|
+
|
|
30
|
+
// MTLLanguageVersion4_0 is not present in older SDKs
|
|
31
|
+
static const NSUInteger MTLLanguageVersion4_0_GGML = 4 << 16;
|
|
32
|
+
|
|
33
|
+
#if !GGML_METAL_EMBED_LIBRARY
|
|
34
|
+
// Here to assist with NSBundle Path Hack
|
|
35
|
+
@interface GGMLMetalClass : NSObject
|
|
36
|
+
@end
|
|
37
|
+
@implementation GGMLMetalClass
|
|
38
|
+
@end
|
|
39
|
+
#endif
|
|
40
|
+
|
|
41
|
+
//
|
|
42
|
+
// MTLFunctionConstantValues wrapper
|
|
43
|
+
//
|
|
44
|
+
|
|
45
|
+
struct ggml_metal_cv {
|
|
46
|
+
MTLFunctionConstantValues * obj;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
ggml_metal_cv_t ggml_metal_cv_init(void) {
|
|
50
|
+
ggml_metal_cv_t res = calloc(1, sizeof(struct ggml_metal_cv));
|
|
51
|
+
|
|
52
|
+
res->obj = [[MTLFunctionConstantValues alloc] init];
|
|
53
|
+
|
|
54
|
+
return res;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
void ggml_metal_cv_free(ggml_metal_cv_t cv) {
|
|
58
|
+
[cv->obj release];
|
|
59
|
+
free(cv);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
void ggml_metal_cv_set_int16(ggml_metal_cv_t cv, int16_t value, int32_t idx) {
|
|
63
|
+
[cv->obj setConstantValue:&value type:MTLDataTypeShort atIndex:idx];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
void ggml_metal_cv_set_int32(ggml_metal_cv_t cv, int32_t value, int32_t idx) {
|
|
67
|
+
[cv->obj setConstantValue:&value type:MTLDataTypeInt atIndex:idx];
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
void ggml_metal_cv_set_bool(ggml_metal_cv_t cv, bool value, int32_t idx) {
|
|
71
|
+
[cv->obj setConstantValue:&value type:MTLDataTypeBool atIndex:idx];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
//
|
|
75
|
+
// MTLComputePipelineState wrapper
|
|
76
|
+
//
|
|
77
|
+
|
|
78
|
+
struct ggml_metal_pipeline {
|
|
79
|
+
id<MTLComputePipelineState> obj;
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
ggml_metal_pipeline_t ggml_metal_pipeline_init(void) {
|
|
83
|
+
ggml_metal_pipeline_t res = calloc(1, sizeof(struct ggml_metal_pipeline));
|
|
84
|
+
|
|
85
|
+
*res = (struct ggml_metal_pipeline) {
|
|
86
|
+
/*.obj =*/ nil,
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
return res;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
void ggml_metal_pipeline_free(ggml_metal_pipeline_t pipeline) {
|
|
93
|
+
[pipeline->obj release];
|
|
94
|
+
|
|
95
|
+
free(pipeline);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
int ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_with_params pipeline) {
|
|
99
|
+
return pipeline.pipeline->obj.maxTotalThreadsPerThreadgroup;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
//
|
|
103
|
+
// MTLLibrary collection (one library per op-source, compiled separately)
|
|
104
|
+
//
|
|
105
|
+
|
|
106
|
+
// Single source of truth for the per-kind metal libraries. The order here
|
|
107
|
+
// defines the enum values and every per-kind table below, so adding a library
|
|
108
|
+
// is a one-line change here (plus adding its source to CMakeLists.txt).
|
|
109
|
+
// X(suffix, name): name is both the kernels/<name>.metal basename and the
|
|
110
|
+
// ggml_metallib_<name>_{start,end} embed-symbol stem.
|
|
111
|
+
#define GGML_METAL_LIBS \
|
|
112
|
+
X(FA, fa) \
|
|
113
|
+
X(MUL_MV, mul_mv) \
|
|
114
|
+
X(MUL_MM, mul_mm) \
|
|
115
|
+
X(QUANTIZE, quantize) \
|
|
116
|
+
X(SOFTMAX, softmax) \
|
|
117
|
+
X(NORM, norm) \
|
|
118
|
+
X(UNARY, unary) \
|
|
119
|
+
X(BINBCAST, binbcast) \
|
|
120
|
+
X(REDUCE, reduce) \
|
|
121
|
+
X(TRI, tri) \
|
|
122
|
+
X(SSM, ssm) \
|
|
123
|
+
X(WKV, wkv) \
|
|
124
|
+
X(GATED_DELTA_NET, gated_delta_net)\
|
|
125
|
+
X(SOLVE_TRI, solve_tri) \
|
|
126
|
+
X(ROPE, rope) \
|
|
127
|
+
X(CONV, conv) \
|
|
128
|
+
X(UPSCALE, upscale) \
|
|
129
|
+
X(ARGSORT, argsort) \
|
|
130
|
+
X(POOL, pool) \
|
|
131
|
+
X(MISC, misc)
|
|
132
|
+
|
|
133
|
+
enum ggml_metal_lib_kind {
|
|
134
|
+
#define X(e, s) GGML_METAL_LIB_##e,
|
|
135
|
+
GGML_METAL_LIBS
|
|
136
|
+
#undef X
|
|
137
|
+
GGML_METAL_LIB_COUNT,
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
static const char * const k_lib_names[GGML_METAL_LIB_COUNT] = {
|
|
141
|
+
#define X(e, s) [GGML_METAL_LIB_##e] = #s,
|
|
142
|
+
GGML_METAL_LIBS
|
|
143
|
+
#undef X
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
struct ggml_metal_library {
|
|
147
|
+
// Per-kind compiled libraries. When single_library is true, the whole library
|
|
148
|
+
// (e.g. a pre-compiled default.metallib or a from-source build) lives at
|
|
149
|
+
// objs[0] and the remaining slots are nil.
|
|
150
|
+
id<MTLLibrary> objs[GGML_METAL_LIB_COUNT];
|
|
151
|
+
bool single_library; // true: combined library at objs[0]; false: per-kind libs in objs[*]
|
|
152
|
+
|
|
153
|
+
// Routing table: kernel function name -> objs[] index, populated from each
|
|
154
|
+
// compiled library's -[MTLLibrary functionNames]. The actual compiled
|
|
155
|
+
// libraries are the single source of truth for which library owns a kernel,
|
|
156
|
+
// so adding kernels later requires no manual routing maintenance.
|
|
157
|
+
// nil in single_library mode (everything resolves to objs[0]).
|
|
158
|
+
NSMutableDictionary<NSString *, NSNumber *> * fn_to_lib;
|
|
159
|
+
|
|
160
|
+
// kernels from a second metallib, resolved ahead of the combined library
|
|
161
|
+
NSSet<NSString *> * override_fns;
|
|
162
|
+
|
|
163
|
+
ggml_metal_device_t dev;
|
|
164
|
+
ggml_metal_pipelines_t pipelines; // cache of compiled pipelines
|
|
165
|
+
|
|
166
|
+
NSLock * lock;
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
// Build the fn_to_lib routing table by querying each compiled library's public
|
|
170
|
+
// function names. Call once after all per-kind libraries have been compiled.
|
|
171
|
+
static void ggml_metal_library_build_index(ggml_metal_library_t lib) {
|
|
172
|
+
@autoreleasepool {
|
|
173
|
+
NSMutableDictionary<NSString *, NSNumber *> * index = [[NSMutableDictionary alloc] init];
|
|
174
|
+
for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
|
|
175
|
+
for (NSString * fname in [lib->objs[kind] functionNames]) {
|
|
176
|
+
index[fname] = @(kind);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
lib->fn_to_lib = index;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// note: defined below, after struct ggml_metal_device
|
|
184
|
+
static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev);
|
|
185
|
+
|
|
186
|
+
// the tensor API headers are exposed to the shader compiler only at Metal language version 4.0
|
|
187
|
+
static void ggml_metal_compile_options_set_lang(MTLCompileOptions * options, bool has_tensor) {
|
|
188
|
+
if (!has_tensor) {
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
options.languageVersion = (MTLLanguageVersion) MTLLanguageVersion4_0_GGML;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// Parse a `#include "name"` line. Returns the quoted name in *include_name on
|
|
196
|
+
// success. Whitespace-tolerant; ignores `#include <...>` (system headers).
|
|
197
|
+
static bool ggml_metal_library_parse_quoted_include(NSString * line, NSString ** include_name) {
|
|
198
|
+
NSScanner * scanner = [NSScanner scannerWithString:line];
|
|
199
|
+
scanner.charactersToBeSkipped = [NSCharacterSet whitespaceCharacterSet];
|
|
200
|
+
|
|
201
|
+
if (![scanner scanString:@"#" intoString:NULL] ||
|
|
202
|
+
![scanner scanString:@"include" intoString:NULL] ||
|
|
203
|
+
![scanner scanString:@"\"" intoString:NULL]) {
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
NSString * name = nil;
|
|
208
|
+
if (![scanner scanUpToString:@"\"" intoString:&name]) {
|
|
209
|
+
return false;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
if (include_name) {
|
|
213
|
+
*include_name = name;
|
|
214
|
+
}
|
|
215
|
+
return true;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Recursively inline `#include "name"` directives. System includes (<...>),
|
|
219
|
+
// `#if/#else/#endif`, and other preprocessor lines are passed through to the
|
|
220
|
+
// Metal compiler unchanged. `#pragma once` is dropped since `seen` already
|
|
221
|
+
// guards against double-inclusion.
|
|
222
|
+
static bool ggml_metal_library_flatten_file(NSMutableString * dst, NSString * path,
|
|
223
|
+
NSArray<NSString *> * search_paths,
|
|
224
|
+
NSMutableSet<NSString *> * seen, NSError ** error) {
|
|
225
|
+
NSString * key = [path stringByStandardizingPath];
|
|
226
|
+
if ([seen containsObject:key]) {
|
|
227
|
+
return true;
|
|
228
|
+
}
|
|
229
|
+
[seen addObject:key];
|
|
230
|
+
|
|
231
|
+
NSString * src = [NSString stringWithContentsOfFile:path encoding:NSUTF8StringEncoding error:error];
|
|
232
|
+
if (!src) {
|
|
233
|
+
return false;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
NSFileManager * fm = [NSFileManager defaultManager];
|
|
237
|
+
for (NSString * line in [src componentsSeparatedByString:@"\n"]) {
|
|
238
|
+
NSString * trimmed = [line stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
|
|
239
|
+
if ([trimmed isEqualToString:@"#pragma once"]) {
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
NSString * include_name = nil;
|
|
244
|
+
if (ggml_metal_library_parse_quoted_include(line, &include_name)) {
|
|
245
|
+
NSString * resolved = nil;
|
|
246
|
+
for (NSString * dir in search_paths) {
|
|
247
|
+
NSString * candidate = [dir stringByAppendingPathComponent:include_name];
|
|
248
|
+
if ([fm isReadableFileAtPath:candidate]) {
|
|
249
|
+
resolved = candidate;
|
|
250
|
+
break;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
if (!resolved) {
|
|
254
|
+
if (error) {
|
|
255
|
+
NSString * msg = [NSString stringWithFormat:@"could not resolve include \"%@\" from '%@'", include_name, path];
|
|
256
|
+
*error = [NSError errorWithDomain:@"ggml-metal-source-flatten" code:1
|
|
257
|
+
userInfo:@{NSLocalizedDescriptionKey: msg}];
|
|
258
|
+
}
|
|
259
|
+
return false;
|
|
260
|
+
}
|
|
261
|
+
if (!ggml_metal_library_flatten_file(dst, resolved, search_paths, seen, error)) {
|
|
262
|
+
return false;
|
|
263
|
+
}
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
[dst appendString:line];
|
|
268
|
+
[dst appendString:@"\n"];
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
return true;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
static NSString * ggml_metal_library_flatten_source(NSString * path_source, NSError ** error) {
|
|
275
|
+
// Search paths cover both runtime layout (build/bin/kernels + build/bin)
|
|
276
|
+
// and source-tree layout (ggml/src/ggml-metal/kernels + ggml/src/ggml-metal + ggml/src).
|
|
277
|
+
NSString * path_kernels = [path_source stringByDeletingLastPathComponent];
|
|
278
|
+
NSString * path_base = [path_kernels stringByDeletingLastPathComponent];
|
|
279
|
+
NSArray<NSString *> * search_paths = @[
|
|
280
|
+
path_kernels,
|
|
281
|
+
path_base,
|
|
282
|
+
[path_base stringByDeletingLastPathComponent],
|
|
283
|
+
];
|
|
284
|
+
|
|
285
|
+
NSMutableString * src = [[NSMutableString alloc] init];
|
|
286
|
+
NSMutableSet<NSString *> * seen = [NSMutableSet set];
|
|
287
|
+
|
|
288
|
+
if (!ggml_metal_library_flatten_file(src, path_source, search_paths, seen, error)) {
|
|
289
|
+
[src release];
|
|
290
|
+
return nil;
|
|
291
|
+
}
|
|
292
|
+
return src;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// Compile all per-kind libraries in parallel. `source_for_kind` returns the MSL
|
|
296
|
+
// source for a kind (the helper takes ownership and releases it), or nil with
|
|
297
|
+
// *err set on failure. On success the objs[] slots are populated and the routing
|
|
298
|
+
// index is built; on any failure every error is logged and false is returned
|
|
299
|
+
// (the caller is responsible for freeing `res`).
|
|
300
|
+
static bool ggml_metal_library_compile_all(
|
|
301
|
+
ggml_metal_library_t res,
|
|
302
|
+
id<MTLDevice> device,
|
|
303
|
+
NSDictionary * prep,
|
|
304
|
+
NSString * (^source_for_kind)(int kind, NSError ** err),
|
|
305
|
+
const char * origin) {
|
|
306
|
+
const int64_t t_start = ggml_time_us();
|
|
307
|
+
|
|
308
|
+
int64_t * t_per_lib = calloc(GGML_METAL_LIB_COUNT, sizeof(int64_t));
|
|
309
|
+
NSError ** err_per_lib = calloc(GGML_METAL_LIB_COUNT, sizeof(NSError *));
|
|
310
|
+
__block atomic_bool any_failure = false;
|
|
311
|
+
|
|
312
|
+
dispatch_group_t group = dispatch_group_create();
|
|
313
|
+
dispatch_queue_t queue = dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0);
|
|
314
|
+
|
|
315
|
+
for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
|
|
316
|
+
dispatch_group_async(group, queue, ^{
|
|
317
|
+
|
|
318
|
+
const int64_t t0 = ggml_time_us();
|
|
319
|
+
|
|
320
|
+
NSError * error = nil;
|
|
321
|
+
|
|
322
|
+
NSString * src = source_for_kind(kind, &error);
|
|
323
|
+
if (!src) {
|
|
324
|
+
err_per_lib[kind] = [error retain];
|
|
325
|
+
atomic_store(&any_failure, true);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
id<MTLLibrary> lib = nil;
|
|
330
|
+
|
|
331
|
+
@autoreleasepool {
|
|
332
|
+
MTLCompileOptions * options = [MTLCompileOptions new];
|
|
333
|
+
options.preprocessorMacros = prep;
|
|
334
|
+
ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(res->dev)->has_tensor);
|
|
335
|
+
|
|
336
|
+
lib = [device newLibraryWithSource:src options:options error:&error];
|
|
337
|
+
|
|
338
|
+
[options release];
|
|
339
|
+
|
|
340
|
+
// retain the error before the autorelease pool drains it
|
|
341
|
+
if (!lib) {
|
|
342
|
+
err_per_lib[kind] = [error retain];
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
[src release];
|
|
347
|
+
|
|
348
|
+
t_per_lib[kind] = ggml_time_us() - t0;
|
|
349
|
+
|
|
350
|
+
if (!lib) {
|
|
351
|
+
atomic_store(&any_failure, true);
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
res->objs[kind] = lib;
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
|
|
359
|
+
dispatch_release(group);
|
|
360
|
+
|
|
361
|
+
const bool ok = !atomic_load(&any_failure);
|
|
362
|
+
|
|
363
|
+
if (ok) {
|
|
364
|
+
const int64_t t_total = ggml_time_us() - t_start;
|
|
365
|
+
int64_t t_max = 0;
|
|
366
|
+
for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
|
|
367
|
+
GGML_LOG_DEBUG("%s: compiled '%s' library in %.3f sec\n",
|
|
368
|
+
__func__, k_lib_names[kind], t_per_lib[kind] / 1e6);
|
|
369
|
+
if (t_per_lib[kind] > t_max) t_max = t_per_lib[kind];
|
|
370
|
+
}
|
|
371
|
+
GGML_LOG_INFO("%s: loaded %d libraries from %s in %.3f sec (max single = %.3f sec)\n",
|
|
372
|
+
__func__, GGML_METAL_LIB_COUNT, origin, t_total / 1e6, t_max / 1e6);
|
|
373
|
+
|
|
374
|
+
ggml_metal_library_build_index(res);
|
|
375
|
+
} else {
|
|
376
|
+
for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
|
|
377
|
+
if (err_per_lib[kind]) {
|
|
378
|
+
GGML_LOG_ERROR("%s: failed to build '%s' library: %s\n", __func__,
|
|
379
|
+
k_lib_names[kind], [[err_per_lib[kind] description] UTF8String]);
|
|
380
|
+
[err_per_lib[kind] release];
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
free(err_per_lib);
|
|
386
|
+
free(t_per_lib);
|
|
387
|
+
|
|
388
|
+
return ok;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// look for <name>.metallib as a bundle resource, then next to the running binary
|
|
392
|
+
static NSString * ggml_metal_find_metallib(NSBundle * bundle, NSString * name) {
|
|
393
|
+
NSError * error = nil;
|
|
394
|
+
|
|
395
|
+
NSString * path_lib = [bundle pathForResource:name ofType:@"metallib"];
|
|
396
|
+
if (path_lib == nil) {
|
|
397
|
+
// Try to find the resource in the directory where the current binary located.
|
|
398
|
+
NSString * bin_cur = [[NSProcessInfo processInfo] arguments][0];
|
|
399
|
+
NSString * bin_dir = [bin_cur stringByDeletingLastPathComponent];
|
|
400
|
+
|
|
401
|
+
NSString * path_lib_default = [NSString pathWithComponents:@[bin_dir, [name stringByAppendingPathExtension:@"metallib"]]];
|
|
402
|
+
if ([[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
|
|
403
|
+
GGML_LOG_INFO("%s: found '%s'\n", __func__, [path_lib_default UTF8String]);
|
|
404
|
+
|
|
405
|
+
NSDictionary * atts = [[NSFileManager defaultManager] attributesOfItemAtPath:path_lib_default error:&error];
|
|
406
|
+
if (atts && atts[NSFileType] == NSFileTypeSymbolicLink) {
|
|
407
|
+
// Optionally, if this is a symlink, try to resolve it.
|
|
408
|
+
path_lib_default = [[NSFileManager defaultManager] destinationOfSymbolicLinkAtPath:path_lib_default error:&error];
|
|
409
|
+
if (path_lib_default && [path_lib_default length] > 0 && ![[path_lib_default substringToIndex:1] isEqualToString:@"/"]) {
|
|
410
|
+
// It is a relative path, adding the binary directory as directory prefix.
|
|
411
|
+
path_lib_default = [NSString pathWithComponents:@[bin_dir, path_lib_default]];
|
|
412
|
+
}
|
|
413
|
+
if (!path_lib_default || ![[NSFileManager defaultManager] isReadableFileAtPath:path_lib_default]) {
|
|
414
|
+
// Link to the resource could not be resolved.
|
|
415
|
+
path_lib_default = nil;
|
|
416
|
+
} else {
|
|
417
|
+
GGML_LOG_INFO("%s: symlink resolved '%s'\n", __func__, [path_lib_default UTF8String]);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
} else {
|
|
421
|
+
// The resource couldn't be found in the binary's directory.
|
|
422
|
+
path_lib_default = nil;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
path_lib = path_lib_default;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
return path_lib;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
|
|
432
|
+
id<MTLDevice> device = ggml_metal_device_get_obj(dev);
|
|
433
|
+
|
|
434
|
+
ggml_metal_library_t res = calloc(1, sizeof(struct ggml_metal_library));
|
|
435
|
+
res->dev = dev;
|
|
436
|
+
res->pipelines = ggml_metal_pipelines_init();
|
|
437
|
+
res->lock = [NSLock new];
|
|
438
|
+
|
|
439
|
+
// shared MTLCompileOptions preprocessor macros (matches the build-time defines)
|
|
440
|
+
NSMutableDictionary * prep = [NSMutableDictionary dictionary];
|
|
441
|
+
if (ggml_metal_device_get_props(dev)->has_bfloat) {
|
|
442
|
+
[prep setObject:@"1" forKey:@"GGML_METAL_HAS_BF16"];
|
|
443
|
+
}
|
|
444
|
+
if (ggml_metal_device_get_props(dev)->has_tensor) {
|
|
445
|
+
[prep setObject:@"1" forKey:@"GGML_METAL_HAS_TENSOR"];
|
|
446
|
+
}
|
|
447
|
+
#if GGML_METAL_EMBED_LIBRARY
|
|
448
|
+
[prep setObject:@"1" forKey:@"GGML_METAL_EMBED_LIBRARY"];
|
|
449
|
+
#endif
|
|
450
|
+
|
|
451
|
+
#if GGML_METAL_EMBED_LIBRARY
|
|
452
|
+
GGML_LOG_INFO("%s: using embedded metal library\n", __func__);
|
|
453
|
+
|
|
454
|
+
// start/end symbols emitted by CMake (see CMakeLists.txt), one pair per kind
|
|
455
|
+
#define X(e, s) extern const char ggml_metallib_##s##_start[]; extern const char ggml_metallib_##s##_end[];
|
|
456
|
+
GGML_METAL_LIBS
|
|
457
|
+
#undef X
|
|
458
|
+
|
|
459
|
+
static const char * const lib_start[GGML_METAL_LIB_COUNT] = {
|
|
460
|
+
#define X(e, s) [GGML_METAL_LIB_##e] = ggml_metallib_##s##_start,
|
|
461
|
+
GGML_METAL_LIBS
|
|
462
|
+
#undef X
|
|
463
|
+
};
|
|
464
|
+
static const char * const lib_end[GGML_METAL_LIB_COUNT] = {
|
|
465
|
+
#define X(e, s) [GGML_METAL_LIB_##e] = ggml_metallib_##s##_end,
|
|
466
|
+
GGML_METAL_LIBS
|
|
467
|
+
#undef X
|
|
468
|
+
};
|
|
469
|
+
|
|
470
|
+
const bool ok = ggml_metal_library_compile_all(res, device, prep,
|
|
471
|
+
^NSString * (int kind, NSError ** err) {
|
|
472
|
+
(void) err;
|
|
473
|
+
return [[NSString alloc] initWithBytes:lib_start[kind]
|
|
474
|
+
length:(lib_end[kind] - lib_start[kind])
|
|
475
|
+
encoding:NSUTF8StringEncoding];
|
|
476
|
+
}, "embedded data");
|
|
477
|
+
|
|
478
|
+
if (!ok) {
|
|
479
|
+
ggml_metal_library_free(res);
|
|
480
|
+
return NULL;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
return res;
|
|
484
|
+
#else
|
|
485
|
+
#ifdef SWIFT_PACKAGE
|
|
486
|
+
NSBundle * bundle = SWIFTPM_MODULE_BUNDLE;
|
|
487
|
+
#else
|
|
488
|
+
NSBundle * bundle = [NSBundle bundleForClass:[GGMLMetalClass class]];
|
|
489
|
+
#endif
|
|
490
|
+
|
|
491
|
+
const int64_t t_start = ggml_time_us();
|
|
492
|
+
|
|
493
|
+
NSError * error = nil;
|
|
494
|
+
NSString * path_lib = ggml_metal_find_metallib(bundle, @"default");
|
|
495
|
+
|
|
496
|
+
if (path_lib != nil) {
|
|
497
|
+
// pre-compiled library found: a single combined default.metallib
|
|
498
|
+
NSURL * libURL = [NSURL fileURLWithPath:path_lib];
|
|
499
|
+
GGML_LOG_INFO("%s: loading '%s'\n", __func__, [path_lib UTF8String]);
|
|
500
|
+
|
|
501
|
+
res->objs[0] = [device newLibraryWithURL:libURL error:&error];
|
|
502
|
+
res->single_library = true;
|
|
503
|
+
if (!res->objs[0]) {
|
|
504
|
+
GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
|
|
505
|
+
ggml_metal_library_free(res);
|
|
506
|
+
return NULL;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
// the tensor API kernels are built into a separate metallib
|
|
510
|
+
if (ggml_metal_device_get_props(dev)->has_tensor) {
|
|
511
|
+
NSString * path_mm = ggml_metal_find_metallib(bundle, @"ggml-tensor");
|
|
512
|
+
|
|
513
|
+
id<MTLLibrary> lib_mm = nil;
|
|
514
|
+
if (path_mm != nil) {
|
|
515
|
+
lib_mm = [device newLibraryWithURL:[NSURL fileURLWithPath:path_mm] error:&error];
|
|
516
|
+
if (!lib_mm && error) {
|
|
517
|
+
GGML_LOG_ERROR("%s: %s\n", __func__, [[error description] UTF8String]);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
if (lib_mm) {
|
|
522
|
+
GGML_LOG_INFO("%s: loaded '%s'\n", __func__, [path_mm UTF8String]);
|
|
523
|
+
|
|
524
|
+
res->objs[GGML_METAL_LIB_MUL_MM] = [lib_mm retain];
|
|
525
|
+
res->override_fns = [[NSSet setWithArray:[lib_mm functionNames]] retain];
|
|
526
|
+
} else {
|
|
527
|
+
GGML_LOG_INFO("%s: ggml-tensor.metallib not found - disabling the tensor API\n", __func__);
|
|
528
|
+
|
|
529
|
+
ggml_metal_device_disable_tensor(dev);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
GGML_LOG_INFO("%s: loaded in %.3f sec\n", __func__, (ggml_time_us() - t_start) / 1e6);
|
|
534
|
+
return res;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
// no pre-compiled metallib: fall back to compiling each kernel source separately
|
|
538
|
+
GGML_LOG_INFO("%s: default.metallib not found, loading kernel sources\n", __func__);
|
|
539
|
+
|
|
540
|
+
NSString * path_resource = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"];
|
|
541
|
+
if (path_resource) {
|
|
542
|
+
GGML_LOG_INFO("%s: GGML_METAL_PATH_RESOURCES = %s\n", __func__, [path_resource UTF8String]);
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// resolve each kind's source path up front (file lookup/logging stays on the calling thread)
|
|
546
|
+
NSString ** path_per_kind = calloc(GGML_METAL_LIB_COUNT, sizeof(NSString *));
|
|
547
|
+
for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
|
|
548
|
+
NSString * rel = [NSString stringWithFormat:@"kernels/%s.metal", k_lib_names[kind]];
|
|
549
|
+
|
|
550
|
+
NSString * path_source = nil;
|
|
551
|
+
if (path_resource) {
|
|
552
|
+
path_source = [path_resource stringByAppendingPathComponent:rel];
|
|
553
|
+
} else {
|
|
554
|
+
NSString * stem = [NSString stringWithFormat:@"kernels/%s", k_lib_names[kind]];
|
|
555
|
+
path_source = [bundle pathForResource:stem ofType:@"metal"];
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
if (path_source == nil || ![[NSFileManager defaultManager] isReadableFileAtPath:path_source]) {
|
|
559
|
+
GGML_LOG_WARN("%s: could not locate %s in bundle, falling back to cwd\n", __func__, [rel UTF8String]);
|
|
560
|
+
path_source = rel;
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
GGML_LOG_DEBUG("%s: loading '%s'\n", __func__, [path_source UTF8String]);
|
|
564
|
+
|
|
565
|
+
path_per_kind[kind] = [path_source retain];
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
const bool ok = ggml_metal_library_compile_all(res, device, prep,
|
|
569
|
+
^NSString * (int kind, NSError ** err) {
|
|
570
|
+
return ggml_metal_library_flatten_source(path_per_kind[kind], err);
|
|
571
|
+
}, "source");
|
|
572
|
+
|
|
573
|
+
for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
|
|
574
|
+
[path_per_kind[kind] release];
|
|
575
|
+
}
|
|
576
|
+
free(path_per_kind);
|
|
577
|
+
|
|
578
|
+
if (!ok) {
|
|
579
|
+
ggml_metal_library_free(res);
|
|
580
|
+
return NULL;
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
return res;
|
|
584
|
+
#endif
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
ggml_metal_library_t ggml_metal_library_init_from_source(ggml_metal_device_t dev, const char * source, bool verbose) {
|
|
588
|
+
if (source == NULL) {
|
|
589
|
+
GGML_LOG_ERROR("%s: source is NULL\n", __func__);
|
|
590
|
+
return NULL;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
id<MTLDevice> device = ggml_metal_device_get_obj(dev);
|
|
594
|
+
id<MTLLibrary> library = nil;
|
|
595
|
+
NSError * error = nil;
|
|
596
|
+
|
|
597
|
+
const int64_t t_start = ggml_time_us();
|
|
598
|
+
|
|
599
|
+
NSString * src = [[NSString alloc] initWithBytes:source
|
|
600
|
+
length:strlen(source)
|
|
601
|
+
encoding:NSUTF8StringEncoding];
|
|
602
|
+
if (!src) {
|
|
603
|
+
GGML_LOG_ERROR("%s: failed to create NSString from source\n", __func__);
|
|
604
|
+
return NULL;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
@autoreleasepool {
|
|
608
|
+
NSMutableDictionary * prep = [NSMutableDictionary dictionary];
|
|
609
|
+
|
|
610
|
+
MTLCompileOptions * options = [MTLCompileOptions new];
|
|
611
|
+
options.preprocessorMacros = prep;
|
|
612
|
+
ggml_metal_compile_options_set_lang(options, ggml_metal_device_get_props(dev)->has_tensor);
|
|
613
|
+
|
|
614
|
+
library = [device newLibraryWithSource:src options:options error:&error];
|
|
615
|
+
if (error) {
|
|
616
|
+
if (verbose) {
|
|
617
|
+
GGML_LOG_ERROR("%s: error compiling source: %s\n", __func__, [[error description] UTF8String]);
|
|
618
|
+
} else {
|
|
619
|
+
GGML_LOG_ERROR("%s: error compiling source\n", __func__);
|
|
620
|
+
}
|
|
621
|
+
library = nil;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
[options release];
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
[src release];
|
|
628
|
+
|
|
629
|
+
if (!library) {
|
|
630
|
+
if (verbose) {
|
|
631
|
+
GGML_LOG_ERROR("%s: failed to create Metal library from source\n", __func__);
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
return NULL;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
if (verbose) {
|
|
638
|
+
GGML_LOG_INFO("%s: compiled in %.3f sec\n", __func__, (ggml_time_us() - t_start) / 1e6);
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
ggml_metal_library_t res = calloc(1, sizeof(struct ggml_metal_library));
|
|
642
|
+
if (!res) {
|
|
643
|
+
GGML_LOG_ERROR("%s: calloc failed\n", __func__);
|
|
644
|
+
return NULL;
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
res->objs[0] = library;
|
|
648
|
+
res->single_library = true;
|
|
649
|
+
res->dev = dev;
|
|
650
|
+
res->pipelines = ggml_metal_pipelines_init();
|
|
651
|
+
res->lock = [NSLock new];
|
|
652
|
+
|
|
653
|
+
return res;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
void ggml_metal_library_free(ggml_metal_library_t lib) {
|
|
657
|
+
if (!lib) {
|
|
658
|
+
return;
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
for (int kind = 0; kind < GGML_METAL_LIB_COUNT; ++kind) {
|
|
662
|
+
if (lib->objs[kind]) {
|
|
663
|
+
[lib->objs[kind] release];
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
if (lib->fn_to_lib) {
|
|
668
|
+
[lib->fn_to_lib release];
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
if (lib->override_fns) {
|
|
672
|
+
[lib->override_fns release];
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
ggml_metal_pipelines_free(lib->pipelines);
|
|
676
|
+
|
|
677
|
+
[lib->lock release];
|
|
678
|
+
|
|
679
|
+
free(lib);
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
ggml_metal_device_t ggml_metal_library_get_device(ggml_metal_library_t lib) {
|
|
683
|
+
return lib->dev;
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline(ggml_metal_library_t lib, const char * name) {
|
|
687
|
+
[lib->lock lock];
|
|
688
|
+
|
|
689
|
+
struct ggml_metal_pipeline_with_params res = {
|
|
690
|
+
/*.pipeline =*/ nil,
|
|
691
|
+
/*.nsg =*/ 0,
|
|
692
|
+
/*.nr0 =*/ 0,
|
|
693
|
+
/*.nr1 =*/ 0,
|
|
694
|
+
/*.smem =*/ 0,
|
|
695
|
+
/*.c4 =*/ false,
|
|
696
|
+
/*.cnt =*/ false,
|
|
697
|
+
};
|
|
698
|
+
|
|
699
|
+
res.pipeline = ggml_metal_pipelines_get(lib->pipelines, name);
|
|
700
|
+
|
|
701
|
+
[lib->lock unlock];
|
|
702
|
+
|
|
703
|
+
return res;
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
struct ggml_metal_pipeline_with_params ggml_metal_library_compile_pipeline(ggml_metal_library_t lib, const char * base, const char * name, ggml_metal_cv_t cv) {
|
|
707
|
+
struct ggml_metal_pipeline_with_params res = {
|
|
708
|
+
/*.pipeline =*/ nil,
|
|
709
|
+
/*.nsg =*/ 0,
|
|
710
|
+
/*.nr0 =*/ 0,
|
|
711
|
+
/*.nr1 =*/ 0,
|
|
712
|
+
/*.smem =*/ 0,
|
|
713
|
+
/*.c4 =*/ false,
|
|
714
|
+
/*.cnt =*/ false,
|
|
715
|
+
};
|
|
716
|
+
|
|
717
|
+
[lib->lock lock];
|
|
718
|
+
|
|
719
|
+
res.pipeline = ggml_metal_pipelines_get(lib->pipelines, name);
|
|
720
|
+
if (res.pipeline) {
|
|
721
|
+
[lib->lock unlock];
|
|
722
|
+
|
|
723
|
+
return res;
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
@autoreleasepool {
|
|
727
|
+
NSError * error = nil;
|
|
728
|
+
|
|
729
|
+
NSString * base_func = [NSString stringWithUTF8String:base];
|
|
730
|
+
|
|
731
|
+
GGML_LOG_DEBUG("%s: compiling pipeline: base = '%s', name = '%s'\n", __func__, base, name);
|
|
732
|
+
|
|
733
|
+
// route to the library that actually defines this kernel; fn_to_lib is
|
|
734
|
+
// built from -[MTLLibrary functionNames] so it's always in sync
|
|
735
|
+
int lib_idx = 0;
|
|
736
|
+
if (lib->override_fns && [lib->override_fns containsObject:base_func]) {
|
|
737
|
+
lib_idx = GGML_METAL_LIB_MUL_MM;
|
|
738
|
+
} else if (!lib->single_library) {
|
|
739
|
+
NSNumber * idx = lib->fn_to_lib[base_func];
|
|
740
|
+
if (!idx) {
|
|
741
|
+
[lib->lock unlock];
|
|
742
|
+
|
|
743
|
+
GGML_LOG_ERROR("%s: kernel not found in any metal library: base = '%s', name = '%s'\n", __func__, base, name);
|
|
744
|
+
|
|
745
|
+
return res;
|
|
746
|
+
}
|
|
747
|
+
lib_idx = [idx intValue];
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
id<MTLLibrary> mtl_lib = lib->objs[lib_idx];
|
|
751
|
+
|
|
752
|
+
id<MTLFunction> mtl_function;
|
|
753
|
+
if (!cv) {
|
|
754
|
+
mtl_function = [mtl_lib newFunctionWithName:base_func];
|
|
755
|
+
} else {
|
|
756
|
+
mtl_function = [mtl_lib newFunctionWithName:base_func constantValues:cv->obj error:&error];
|
|
757
|
+
}
|
|
758
|
+
if (!mtl_function) {
|
|
759
|
+
[lib->lock unlock];
|
|
760
|
+
|
|
761
|
+
GGML_LOG_ERROR("%s: failed to compile pipeline: base = '%s', name = '%s'\n", __func__, base, name);
|
|
762
|
+
if (error) {
|
|
763
|
+
GGML_LOG_ERROR("%s: %s\n", __func__, [[error description] UTF8String]);
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
return res;
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
id<MTLDevice> device = ggml_metal_device_get_obj(lib->dev);
|
|
770
|
+
id<MTLComputePipelineState> obj = [device newComputePipelineStateWithFunction:mtl_function error:&error];
|
|
771
|
+
|
|
772
|
+
[mtl_function release];
|
|
773
|
+
|
|
774
|
+
if (!obj) {
|
|
775
|
+
[lib->lock unlock];
|
|
776
|
+
|
|
777
|
+
GGML_LOG_ERROR("%s: failed to create pipeline state: base = '%s', name = '%s'\n", __func__, base, name);
|
|
778
|
+
if (error) {
|
|
779
|
+
GGML_LOG_ERROR("%s: %s\n", __func__, [[error description] UTF8String]);
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
return res;
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
GGML_LOG_DEBUG("%s: loaded %-40s %16p | th_max = %4d | th_width = %4d\n", __func__, name,
|
|
786
|
+
(void *) obj,
|
|
787
|
+
(int) obj.maxTotalThreadsPerThreadgroup,
|
|
788
|
+
(int) obj.threadExecutionWidth);
|
|
789
|
+
|
|
790
|
+
if (obj.maxTotalThreadsPerThreadgroup == 0 || obj.threadExecutionWidth == 0) {
|
|
791
|
+
[obj release];
|
|
792
|
+
|
|
793
|
+
[lib->lock unlock];
|
|
794
|
+
|
|
795
|
+
GGML_LOG_ERROR("%s: incompatible pipeline %s\n", __func__, name);
|
|
796
|
+
|
|
797
|
+
return res;
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
res.pipeline = ggml_metal_pipeline_init();
|
|
801
|
+
res.pipeline->obj = obj;
|
|
802
|
+
|
|
803
|
+
ggml_metal_pipelines_add(lib->pipelines, name, res.pipeline);
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
[lib->lock unlock];
|
|
807
|
+
|
|
808
|
+
return res;
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
//
|
|
812
|
+
// MTLComputeCommandEncoder wrapper
|
|
813
|
+
//
|
|
814
|
+
|
|
815
|
+
struct ggml_metal_encoder {
|
|
816
|
+
id<MTLComputeCommandEncoder> obj;
|
|
817
|
+
};
|
|
818
|
+
|
|
819
|
+
ggml_metal_encoder_t ggml_metal_encoder_init(ggml_metal_cmd_buf_t cmd_buf_raw, bool concurrent) {
|
|
820
|
+
ggml_metal_encoder_t res = calloc(1, sizeof(struct ggml_metal_encoder));
|
|
821
|
+
|
|
822
|
+
id<MTLCommandBuffer> cmd_buf = (id<MTLCommandBuffer>) cmd_buf_raw;
|
|
823
|
+
|
|
824
|
+
if (concurrent) {
|
|
825
|
+
res->obj = [cmd_buf computeCommandEncoderWithDispatchType: MTLDispatchTypeConcurrent];
|
|
826
|
+
} else {
|
|
827
|
+
res->obj = [cmd_buf computeCommandEncoder];
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
[res->obj retain];
|
|
831
|
+
|
|
832
|
+
return res;
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
void ggml_metal_encoder_free(ggml_metal_encoder_t encoder) {
|
|
836
|
+
[encoder->obj release];
|
|
837
|
+
free(encoder);
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
void ggml_metal_encoder_debug_group_push(ggml_metal_encoder_t encoder, const char * name) {
|
|
841
|
+
@autoreleasepool {
|
|
842
|
+
[encoder->obj pushDebugGroup:[NSString stringWithCString:name encoding:NSUTF8StringEncoding]];
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
void ggml_metal_encoder_debug_group_pop (ggml_metal_encoder_t encoder) {
|
|
847
|
+
[encoder->obj popDebugGroup];
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
void ggml_metal_encoder_set_pipeline(ggml_metal_encoder_t encoder, struct ggml_metal_pipeline_with_params pipeline) {
|
|
851
|
+
if (!pipeline.pipeline) {
|
|
852
|
+
GGML_ABORT("%s: nil Metal pipeline (missing kernel; see compile_pipeline log above)\n", __func__);
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
[encoder->obj setComputePipelineState:pipeline.pipeline->obj];
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
void ggml_metal_encoder_set_bytes(ggml_metal_encoder_t encoder, void * data, size_t size, int idx) {
|
|
859
|
+
[encoder->obj setBytes:data length:size atIndex:idx];
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
void ggml_metal_encoder_set_buffer(ggml_metal_encoder_t encoder, struct ggml_metal_buffer_id buffer, int idx) {
|
|
863
|
+
[encoder->obj setBuffer:buffer.metal offset:buffer.offs atIndex:idx];
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
void ggml_metal_encoder_set_threadgroup_memory_size(ggml_metal_encoder_t encoder, size_t size, int idx) {
|
|
867
|
+
// ref: https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setthreadgroupmemorylength(_:index:)
|
|
868
|
+
GGML_ASSERT(size % 16 == 0);
|
|
869
|
+
|
|
870
|
+
[encoder->obj setThreadgroupMemoryLength:size atIndex:idx];
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
void ggml_metal_encoder_dispatch_threadgroups(ggml_metal_encoder_t encoder, int tg0, int tg1, int tg2, int tptg0, int tptg1, int tptg2) {
|
|
874
|
+
[encoder->obj dispatchThreadgroups:MTLSizeMake(tg0, tg1, tg2) threadsPerThreadgroup:MTLSizeMake(tptg0, tptg1, tptg2)];
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
void ggml_metal_encoder_memory_barrier(ggml_metal_encoder_t encoder) {
|
|
878
|
+
[encoder->obj memoryBarrierWithScope:MTLBarrierScopeBuffers];
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
void ggml_metal_encoder_end_encoding(ggml_metal_encoder_t encoder) {
|
|
882
|
+
[encoder->obj endEncoding];
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
struct ggml_metal_device {
|
|
886
|
+
id<MTLDevice> mtl_device;
|
|
887
|
+
|
|
888
|
+
// a single global queue shared by all Metal backends
|
|
889
|
+
// technically not needed for devices with unified memory, but enables discrete GPUs support
|
|
890
|
+
// ref: https://github.com/ggml-org/llama.cpp/pull/15906
|
|
891
|
+
id<MTLCommandQueue> mtl_queue;
|
|
892
|
+
|
|
893
|
+
ggml_metal_rsets_t rsets;
|
|
894
|
+
|
|
895
|
+
ggml_metal_library_t library;
|
|
896
|
+
|
|
897
|
+
struct ggml_metal_device_props props;
|
|
898
|
+
|
|
899
|
+
// virtual address for GPU memory allocations
|
|
900
|
+
atomic_uintptr_t addr_virt;
|
|
901
|
+
};
|
|
902
|
+
|
|
903
|
+
//
|
|
904
|
+
// MTLResidenceSet wrapper
|
|
905
|
+
//
|
|
906
|
+
|
|
907
|
+
struct ggml_metal_rsets {
|
|
908
|
+
NSLock * lock;
|
|
909
|
+
|
|
910
|
+
NSMutableArray * data;
|
|
911
|
+
|
|
912
|
+
// number of seconds since the last graph computation
|
|
913
|
+
// keep the residency sets wired for that amount of time to avoid being collected by the OS
|
|
914
|
+
int keep_alive_s;
|
|
915
|
+
int loops_per_s;
|
|
916
|
+
int time_per_loop_ms;
|
|
917
|
+
|
|
918
|
+
// background heartbeat thread to keep the residency sets alive
|
|
919
|
+
atomic_bool d_stop;
|
|
920
|
+
atomic_int d_loop;
|
|
921
|
+
|
|
922
|
+
dispatch_group_t d_group;
|
|
923
|
+
};
|
|
924
|
+
|
|
925
|
+
#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
|
|
926
|
+
static void ggml_metal_dummy_work(ggml_metal_device_t dev) {
|
|
927
|
+
if (dev->mtl_queue == nil) {
|
|
928
|
+
return;
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
@autoreleasepool {
|
|
932
|
+
// perform a minimal dummy operation on the GPU
|
|
933
|
+
id<MTLBuffer> buf = [dev->mtl_device newBufferWithLength:1 options:MTLResourceStorageModePrivate];
|
|
934
|
+
id<MTLCommandBuffer> cmd_buf = [dev->mtl_queue commandBuffer];
|
|
935
|
+
|
|
936
|
+
{
|
|
937
|
+
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
|
938
|
+
|
|
939
|
+
[encoder fillBuffer:buf range:NSMakeRange(0, 1) value:0];
|
|
940
|
+
|
|
941
|
+
[encoder endEncoding];
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
[cmd_buf commit];
|
|
945
|
+
[buf release];
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
#endif
|
|
949
|
+
|
|
950
|
+
ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev) {
|
|
951
|
+
ggml_metal_rsets_t res = calloc(1, sizeof(struct ggml_metal_rsets));
|
|
952
|
+
|
|
953
|
+
res->lock = [[NSLock alloc] init];
|
|
954
|
+
res->data = [[NSMutableArray alloc] init];
|
|
955
|
+
|
|
956
|
+
// by default keep the memory wired for 3 minutes
|
|
957
|
+
res->keep_alive_s = 3*60;
|
|
958
|
+
|
|
959
|
+
const char * GGML_METAL_RESIDENCY_KEEP_ALIVE_S = getenv("GGML_METAL_RESIDENCY_KEEP_ALIVE_S");
|
|
960
|
+
if (GGML_METAL_RESIDENCY_KEEP_ALIVE_S) {
|
|
961
|
+
res->keep_alive_s = atoi(GGML_METAL_RESIDENCY_KEEP_ALIVE_S);
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
if (res->keep_alive_s <= 0) {
|
|
965
|
+
res->keep_alive_s = 3*60;
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
res->time_per_loop_ms = 5;
|
|
969
|
+
res->loops_per_s = 1000/res->time_per_loop_ms;
|
|
970
|
+
|
|
971
|
+
GGML_LOG_INFO("%s: creating a residency set collection (keep_alive = %d s)\n", __func__, res->keep_alive_s);
|
|
972
|
+
|
|
973
|
+
atomic_store_explicit(&res->d_stop, false, memory_order_relaxed);
|
|
974
|
+
atomic_store_explicit(&res->d_loop, res->loops_per_s*res->keep_alive_s, memory_order_relaxed);
|
|
975
|
+
|
|
976
|
+
res->d_group = dispatch_group_create();
|
|
977
|
+
|
|
978
|
+
// start a background thread that periodically requests residency for all the currently active sets in the collection
|
|
979
|
+
// the requests stop after a certain amount of time (keep_alive_s) of inactivity
|
|
980
|
+
dispatch_queue_t d_queue = dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0);
|
|
981
|
+
dispatch_group_async(res->d_group, d_queue, ^{
|
|
982
|
+
#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
|
|
983
|
+
if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {
|
|
984
|
+
while (!atomic_load_explicit(&res->d_stop, memory_order_relaxed)) {
|
|
985
|
+
if (atomic_load_explicit(&res->d_loop, memory_order_relaxed) > 0) {
|
|
986
|
+
[res->lock lock];
|
|
987
|
+
|
|
988
|
+
for (int i = 0; i < (int) res->data.count; ++i) {
|
|
989
|
+
[res->data[i] requestResidency];
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
atomic_fetch_sub_explicit(&res->d_loop, 1, memory_order_relaxed);
|
|
993
|
+
|
|
994
|
+
[res->lock unlock];
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
usleep(res->time_per_loop_ms * 1000);
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
#endif
|
|
1001
|
+
});
|
|
1002
|
+
|
|
1003
|
+
#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
|
|
1004
|
+
if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {
|
|
1005
|
+
// workaround for residency set memory not being released if no GPU operation occurs
|
|
1006
|
+
// https://developer.apple.com/forums/thread/839089
|
|
1007
|
+
// https://github.com/ggml-org/llama.cpp/issues/25937
|
|
1008
|
+
ggml_metal_dummy_work(dev);
|
|
1009
|
+
}
|
|
1010
|
+
#endif
|
|
1011
|
+
|
|
1012
|
+
return res;
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
void ggml_metal_rsets_free(ggml_metal_rsets_t rsets) {
|
|
1016
|
+
if (rsets == NULL) {
|
|
1017
|
+
return;
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
// note: if you hit this assert, most likely you haven't deallocated all Metal resources before exiting
|
|
1021
|
+
GGML_ASSERT([rsets->data count] == 0);
|
|
1022
|
+
|
|
1023
|
+
atomic_store_explicit(&rsets->d_stop, true, memory_order_relaxed);
|
|
1024
|
+
|
|
1025
|
+
dispatch_group_wait(rsets->d_group, DISPATCH_TIME_FOREVER);
|
|
1026
|
+
dispatch_release(rsets->d_group);
|
|
1027
|
+
|
|
1028
|
+
[rsets->data release];
|
|
1029
|
+
[rsets->lock release];
|
|
1030
|
+
|
|
1031
|
+
free(rsets);
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
static const struct {
|
|
1035
|
+
const char * name;
|
|
1036
|
+
const char * token;
|
|
1037
|
+
enum ggml_metal_device_id id;
|
|
1038
|
+
} k_metal_devices[] = {
|
|
1039
|
+
#define DEV(name, id) { name, #id, id }
|
|
1040
|
+
DEV("M1", GGML_METAL_DEVICE_M1),
|
|
1041
|
+
DEV("M1 Pro", GGML_METAL_DEVICE_M1_PRO),
|
|
1042
|
+
DEV("M1 Max", GGML_METAL_DEVICE_M1_MAX),
|
|
1043
|
+
DEV("M1 Ultra", GGML_METAL_DEVICE_M1_ULTRA),
|
|
1044
|
+
DEV("M2", GGML_METAL_DEVICE_M2),
|
|
1045
|
+
DEV("M2 Pro", GGML_METAL_DEVICE_M2_PRO),
|
|
1046
|
+
DEV("M2 Max", GGML_METAL_DEVICE_M2_MAX),
|
|
1047
|
+
DEV("M2 Ultra", GGML_METAL_DEVICE_M2_ULTRA),
|
|
1048
|
+
DEV("M3", GGML_METAL_DEVICE_M3),
|
|
1049
|
+
DEV("M3 Pro", GGML_METAL_DEVICE_M3_PRO),
|
|
1050
|
+
DEV("M3 Max", GGML_METAL_DEVICE_M3_MAX),
|
|
1051
|
+
DEV("M3 Ultra", GGML_METAL_DEVICE_M3_ULTRA),
|
|
1052
|
+
DEV("M4", GGML_METAL_DEVICE_M4),
|
|
1053
|
+
DEV("M4 Pro", GGML_METAL_DEVICE_M4_PRO),
|
|
1054
|
+
DEV("M4 Max", GGML_METAL_DEVICE_M4_MAX),
|
|
1055
|
+
DEV("M5", GGML_METAL_DEVICE_M5),
|
|
1056
|
+
DEV("M5 Pro", GGML_METAL_DEVICE_M5_PRO),
|
|
1057
|
+
DEV("M5 Max", GGML_METAL_DEVICE_M5_MAX),
|
|
1058
|
+
DEV("M5 Ultra", GGML_METAL_DEVICE_M5_ULTRA),
|
|
1059
|
+
DEV("A18 Pro", GGML_METAL_DEVICE_A18_PRO),
|
|
1060
|
+
#undef DEV
|
|
1061
|
+
};
|
|
1062
|
+
|
|
1063
|
+
static enum ggml_metal_device_id ggml_metal_device_id_parse(const char * name) {
|
|
1064
|
+
if (!name) {
|
|
1065
|
+
return GGML_METAL_DEVICE_GENERIC;
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
static const char prefix[] = "Apple ";
|
|
1069
|
+
if (strncmp(name, prefix, sizeof(prefix) - 1) != 0) {
|
|
1070
|
+
return GGML_METAL_DEVICE_GENERIC;
|
|
1071
|
+
}
|
|
1072
|
+
const char * suffix = name + sizeof(prefix) - 1;
|
|
1073
|
+
|
|
1074
|
+
for (size_t i = 0; i < sizeof(k_metal_devices)/sizeof(k_metal_devices[0]); ++i) {
|
|
1075
|
+
if (strcmp(suffix, k_metal_devices[i].name) == 0) {
|
|
1076
|
+
return k_metal_devices[i].id;
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
return GGML_METAL_DEVICE_GENERIC;
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
const char * ggml_metal_device_id_token(enum ggml_metal_device_id id) {
|
|
1083
|
+
for (size_t i = 0; i < sizeof(k_metal_devices)/sizeof(k_metal_devices[0]); ++i) {
|
|
1084
|
+
if (k_metal_devices[i].id == id) {
|
|
1085
|
+
return k_metal_devices[i].token;
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
return "GGML_METAL_DEVICE_GENERIC";
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
ggml_metal_device_t ggml_metal_device_init(int device, int n_devices) {
|
|
1092
|
+
ggml_metal_device_t dev = calloc(1, sizeof(struct ggml_metal_device));
|
|
1093
|
+
|
|
1094
|
+
assert(dev != NULL);
|
|
1095
|
+
|
|
1096
|
+
@autoreleasepool {
|
|
1097
|
+
if (dev->mtl_device == nil) {
|
|
1098
|
+
dev->mtl_device = MTLCreateSystemDefaultDevice();
|
|
1099
|
+
|
|
1100
|
+
if (dev->mtl_device) {
|
|
1101
|
+
dev->mtl_queue = [dev->mtl_device newCommandQueue];
|
|
1102
|
+
if (dev->mtl_queue == nil) {
|
|
1103
|
+
GGML_LOG_ERROR("%s: error: failed to create command queue\n", __func__);
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
dev->addr_virt = 0x000000400ULL;
|
|
1107
|
+
|
|
1108
|
+
dev->props.device = device;
|
|
1109
|
+
|
|
1110
|
+
// the Metal backend uses the system default device as the single physical device;
|
|
1111
|
+
// additional (virtual) devices are emulated on top of it via GGML_METAL_DEVICES
|
|
1112
|
+
dev->props.device_phys = 0;
|
|
1113
|
+
dev->props.device_virt = device;
|
|
1114
|
+
|
|
1115
|
+
dev->props.has_simdgroup_reduction = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
|
|
1116
|
+
dev->props.has_simdgroup_reduction |= [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
|
|
1117
|
+
|
|
1118
|
+
dev->props.has_simdgroup_mm = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
|
|
1119
|
+
dev->props.has_unified_memory = dev->mtl_device.hasUnifiedMemory;
|
|
1120
|
+
|
|
1121
|
+
dev->props.has_bfloat = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal3_GGML];
|
|
1122
|
+
dev->props.has_bfloat |= [dev->mtl_device supportsFamily:MTLGPUFamilyApple6];
|
|
1123
|
+
if (getenv("GGML_METAL_BF16_DISABLE") != NULL) {
|
|
1124
|
+
dev->props.has_bfloat = false;
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
dev->props.has_tensor = [dev->mtl_device supportsFamily:MTLGPUFamilyMetal4_GGML];
|
|
1128
|
+
if (getenv("GGML_METAL_TENSOR_DISABLE") != NULL) {
|
|
1129
|
+
dev->props.has_tensor = false;
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
// note: disable the tensor API by default for old chips because with the current implementation it is not useful
|
|
1133
|
+
// - M2 Ultra: ~5% slower
|
|
1134
|
+
// - M4, M4 Max: no significant difference
|
|
1135
|
+
//
|
|
1136
|
+
// TODO: try to update the tensor API kernels to at least match the simdgroup performance
|
|
1137
|
+
if (getenv("GGML_METAL_TENSOR_ENABLE") == NULL &&
|
|
1138
|
+
![[dev->mtl_device name] containsString:@"M5"] &&
|
|
1139
|
+
![[dev->mtl_device name] containsString:@"M6"] &&
|
|
1140
|
+
![[dev->mtl_device name] containsString:@"A19"] &&
|
|
1141
|
+
![[dev->mtl_device name] containsString:@"A20"]) {
|
|
1142
|
+
GGML_LOG_INFO("%s: tensor API disabled for pre-M5 and pre-A19 devices\n", __func__);
|
|
1143
|
+
dev->props.has_tensor = false;
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
// double-check that the tensor API compiles
|
|
1147
|
+
if (dev->props.has_tensor) {
|
|
1148
|
+
const char * src_tensor_f16 = "\n"
|
|
1149
|
+
"#include <metal_stdlib> \n"
|
|
1150
|
+
"#include <metal_tensor> \n"
|
|
1151
|
+
"#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
|
|
1152
|
+
" \n"
|
|
1153
|
+
"using namespace metal; \n"
|
|
1154
|
+
"using namespace mpp::tensor_ops; \n"
|
|
1155
|
+
" \n"
|
|
1156
|
+
"kernel void dummy_kernel( \n"
|
|
1157
|
+
" tensor<device half, dextents<int32_t, 2>> A [[buffer(0)]], \n"
|
|
1158
|
+
" tensor<device half, dextents<int32_t, 2>> B [[buffer(1)]], \n"
|
|
1159
|
+
" device float * C [[buffer(2)]], \n"
|
|
1160
|
+
" uint2 tgid [[threadgroup_position_in_grid]]) \n"
|
|
1161
|
+
"{ \n"
|
|
1162
|
+
" auto tA = A.slice(0, (int)tgid.y); \n"
|
|
1163
|
+
" auto tB = B.slice((int)tgid.x, 0); \n"
|
|
1164
|
+
" \n"
|
|
1165
|
+
" matmul2d< \n"
|
|
1166
|
+
" matmul2d_descriptor(16, 16, dynamic_extent), \n"
|
|
1167
|
+
" execution_simdgroups<4>> mm; \n"
|
|
1168
|
+
" \n"
|
|
1169
|
+
" auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
|
|
1170
|
+
" \n"
|
|
1171
|
+
" auto sA = tA.slice(0, 0); \n"
|
|
1172
|
+
" auto sB = tB.slice(0, 0); \n"
|
|
1173
|
+
" mm.run(sB, sA, cT); \n"
|
|
1174
|
+
" \n"
|
|
1175
|
+
" auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
|
|
1176
|
+
" \n"
|
|
1177
|
+
" cT.store(tC); \n"
|
|
1178
|
+
"}";
|
|
1179
|
+
|
|
1180
|
+
GGML_LOG_INFO("%s: testing tensor API for f16 support\n", __func__);
|
|
1181
|
+
ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_f16, false);
|
|
1182
|
+
if (lib == NULL) {
|
|
1183
|
+
GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);
|
|
1184
|
+
dev->props.has_tensor = false;
|
|
1185
|
+
} else {
|
|
1186
|
+
struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
|
|
1187
|
+
if (!ppl.pipeline) {
|
|
1188
|
+
GGML_LOG_WARN("%s: - the tensor API is not supported in this environment - disabling\n", __func__);
|
|
1189
|
+
dev->props.has_tensor = false;
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
ggml_metal_library_free(lib);
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
// try to compile a dummy kernel to determine if the tensor API is supported for bfloat
|
|
1197
|
+
if (dev->props.has_tensor && dev->props.has_bfloat) {
|
|
1198
|
+
const char * src_tensor_bf16 = "\n"
|
|
1199
|
+
"#include <metal_stdlib> \n"
|
|
1200
|
+
"#include <metal_tensor> \n"
|
|
1201
|
+
"#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h> \n"
|
|
1202
|
+
" \n"
|
|
1203
|
+
"using namespace metal; \n"
|
|
1204
|
+
"using namespace mpp::tensor_ops; \n"
|
|
1205
|
+
" \n"
|
|
1206
|
+
"kernel void dummy_kernel( \n"
|
|
1207
|
+
" tensor<device bfloat, dextents<int32_t, 2>> A [[buffer(0)]], \n"
|
|
1208
|
+
" tensor<device bfloat, dextents<int32_t, 2>> B [[buffer(1)]], \n"
|
|
1209
|
+
" device float * C [[buffer(2)]], \n"
|
|
1210
|
+
" uint2 tgid [[threadgroup_position_in_grid]]) \n"
|
|
1211
|
+
"{ \n"
|
|
1212
|
+
" auto tA = A.slice(0, (int)tgid.y); \n"
|
|
1213
|
+
" auto tB = B.slice((int)tgid.x, 0); \n"
|
|
1214
|
+
" \n"
|
|
1215
|
+
" matmul2d< \n"
|
|
1216
|
+
" matmul2d_descriptor(16, 16, dynamic_extent), \n"
|
|
1217
|
+
" execution_simdgroups<4>> mm; \n"
|
|
1218
|
+
" \n"
|
|
1219
|
+
" auto cT = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB), float>(); \n"
|
|
1220
|
+
" \n"
|
|
1221
|
+
" auto sA = tA.slice(0, 0); \n"
|
|
1222
|
+
" auto sB = tB.slice(0, 0); \n"
|
|
1223
|
+
" mm.run(sB, sA, cT); \n"
|
|
1224
|
+
" \n"
|
|
1225
|
+
" auto tC = tensor<device float, dextents<int32_t, 2>, tensor_inline>(C, dextents<int32_t, 2>(16, 16)); \n"
|
|
1226
|
+
" \n"
|
|
1227
|
+
" cT.store(tC); \n"
|
|
1228
|
+
"}";
|
|
1229
|
+
|
|
1230
|
+
GGML_LOG_INFO("%s: testing tensor API for bfloat support\n", __func__);
|
|
1231
|
+
ggml_metal_library_t lib = ggml_metal_library_init_from_source(dev, src_tensor_bf16, false);
|
|
1232
|
+
if (lib == NULL) {
|
|
1233
|
+
GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);
|
|
1234
|
+
dev->props.has_bfloat = false;
|
|
1235
|
+
} else {
|
|
1236
|
+
struct ggml_metal_pipeline_with_params ppl = ggml_metal_library_compile_pipeline(lib, "dummy_kernel", "dummy_kernel", nil);
|
|
1237
|
+
if (!ppl.pipeline) {
|
|
1238
|
+
GGML_LOG_WARN("%s: - the tensor API does not support bfloat - disabling bfloat support\n", __func__);
|
|
1239
|
+
dev->props.has_bfloat = false;
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
ggml_metal_library_free(lib);
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
dev->props.use_residency_sets = true;
|
|
1247
|
+
#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
|
|
1248
|
+
dev->props.use_residency_sets = getenv("GGML_METAL_NO_RESIDENCY") == nil;
|
|
1249
|
+
#endif
|
|
1250
|
+
|
|
1251
|
+
dev->props.use_shared_buffers = dev->props.has_unified_memory;
|
|
1252
|
+
#if TARGET_OS_OSX
|
|
1253
|
+
// In case of eGPU, shared memory may be preferable.
|
|
1254
|
+
dev->props.use_shared_buffers |= [dev->mtl_device location] == MTLDeviceLocationExternal;
|
|
1255
|
+
#endif
|
|
1256
|
+
if (getenv("GGML_METAL_SHARED_BUFFERS_DISABLE") != NULL) {
|
|
1257
|
+
dev->props.use_shared_buffers = false;
|
|
1258
|
+
}
|
|
1259
|
+
if (getenv("GGML_METAL_SHARED_BUFFERS_ENABLE") != NULL) {
|
|
1260
|
+
dev->props.use_shared_buffers = true;
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
dev->props.supports_gpu_family_apple7 = [dev->mtl_device supportsFamily:MTLGPUFamilyApple7];
|
|
1264
|
+
|
|
1265
|
+
dev->props.device_id = ggml_metal_device_id_parse([[dev->mtl_device name] UTF8String]);
|
|
1266
|
+
|
|
1267
|
+
dev->props.op_offload_min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32;
|
|
1268
|
+
|
|
1269
|
+
dev->props.max_buffer_size = dev->mtl_device.maxBufferLength;
|
|
1270
|
+
dev->props.max_theadgroup_memory_size = dev->mtl_device.maxThreadgroupMemoryLength;
|
|
1271
|
+
if (@available(macOS 10.12, iOS 16.0, *)) {
|
|
1272
|
+
dev->props.max_working_set_size = dev->mtl_device.recommendedMaxWorkingSetSize;
|
|
1273
|
+
} else {
|
|
1274
|
+
dev->props.max_working_set_size = dev->mtl_device.maxBufferLength;
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
snprintf(dev->props.name, sizeof(dev->props.name), "%s%d", "MTL", device);
|
|
1278
|
+
const char * gpu_name = [[dev->mtl_device name] UTF8String];
|
|
1279
|
+
if (n_devices > 1) {
|
|
1280
|
+
snprintf(dev->props.desc, sizeof(dev->props.desc), "%s (dev p%d/v%d)",
|
|
1281
|
+
gpu_name, dev->props.device_phys, dev->props.device_virt);
|
|
1282
|
+
} else {
|
|
1283
|
+
snprintf(dev->props.desc, sizeof(dev->props.desc), "%s", gpu_name);
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
dev->library = ggml_metal_library_init(dev);
|
|
1287
|
+
if (!dev->library) {
|
|
1288
|
+
GGML_LOG_ERROR("%s: error: failed to create library\n", __func__);
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
if (dev->props.use_residency_sets) {
|
|
1292
|
+
dev->rsets = ggml_metal_rsets_init(dev);
|
|
1293
|
+
} else {
|
|
1294
|
+
dev->rsets = nil;
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
// print MTL GPU family:
|
|
1298
|
+
GGML_LOG_INFO("%s: GPU name: %s (%s)\n", __func__, dev->props.name, dev->props.desc);
|
|
1299
|
+
|
|
1300
|
+
// determine max supported GPU family
|
|
1301
|
+
// https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf
|
|
1302
|
+
// https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
|
|
1303
|
+
{
|
|
1304
|
+
for (int i = MTLGPUFamilyApple1 + 20; i >= MTLGPUFamilyApple1; --i) {
|
|
1305
|
+
if ([dev->mtl_device supportsFamily:i]) {
|
|
1306
|
+
dev->props.gpu_family = i - (int) MTLGPUFamilyApple1 + 1;
|
|
1307
|
+
GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyApple%d (%d)\n", __func__, dev->props.gpu_family, i);
|
|
1308
|
+
break;
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
for (int i = MTLGPUFamilyCommon1 + 5; i >= MTLGPUFamilyCommon1; --i) {
|
|
1313
|
+
if ([dev->mtl_device supportsFamily:i]) {
|
|
1314
|
+
GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyCommon%d (%d)\n", __func__, i - (int) MTLGPUFamilyCommon1 + 1, i);
|
|
1315
|
+
break;
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
for (int i = MTLGPUFamilyMetal3_GGML + 5; i >= MTLGPUFamilyMetal3_GGML; --i) {
|
|
1320
|
+
if ([dev->mtl_device supportsFamily:i]) {
|
|
1321
|
+
GGML_LOG_INFO("%s: GPU family: MTLGPUFamilyMetal%d (%d)\n", __func__, i - (int) MTLGPUFamilyMetal3_GGML + 3, i);
|
|
1322
|
+
break;
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
GGML_LOG_INFO("%s: simdgroup reduction = %s\n", __func__, dev->props.has_simdgroup_reduction ? "true" : "false");
|
|
1328
|
+
GGML_LOG_INFO("%s: simdgroup matrix mul. = %s\n", __func__, dev->props.has_simdgroup_mm ? "true" : "false");
|
|
1329
|
+
GGML_LOG_INFO("%s: has unified memory = %s\n", __func__, dev->props.has_unified_memory ? "true" : "false");
|
|
1330
|
+
GGML_LOG_INFO("%s: has bfloat = %s\n", __func__, dev->props.has_bfloat ? "true" : "false");
|
|
1331
|
+
GGML_LOG_INFO("%s: has tensor = %s\n", __func__, dev->props.has_tensor ? "true" : "false");
|
|
1332
|
+
GGML_LOG_INFO("%s: use residency sets = %s\n", __func__, dev->props.use_residency_sets ? "true" : "false");
|
|
1333
|
+
GGML_LOG_INFO("%s: use shared buffers = %s\n", __func__, dev->props.use_shared_buffers ? "true" : "false");
|
|
1334
|
+
|
|
1335
|
+
#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
|
|
1336
|
+
if (@available(macOS 10.12, iOS 16.0, *)) {
|
|
1337
|
+
GGML_LOG_INFO("%s: recommendedMaxWorkingSetSize = %8.2f MB\n", __func__, dev->props.max_working_set_size / 1e6);
|
|
1338
|
+
}
|
|
1339
|
+
#endif
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
return dev;
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
void ggml_metal_device_free(ggml_metal_device_t dev) {
|
|
1348
|
+
assert(dev != NULL);
|
|
1349
|
+
|
|
1350
|
+
@autoreleasepool {
|
|
1351
|
+
ggml_metal_rsets_free(dev->rsets);
|
|
1352
|
+
|
|
1353
|
+
ggml_metal_library_free(dev->library);
|
|
1354
|
+
dev->library = NULL;
|
|
1355
|
+
|
|
1356
|
+
if (dev->mtl_queue) {
|
|
1357
|
+
[dev->mtl_queue release];
|
|
1358
|
+
dev->mtl_queue = nil;
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
if (dev->mtl_device) {
|
|
1362
|
+
[dev->mtl_device release];
|
|
1363
|
+
dev->mtl_device = nil;
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
free(dev);
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
void * ggml_metal_device_get_obj(ggml_metal_device_t dev) {
|
|
1371
|
+
return dev->mtl_device;
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
void * ggml_metal_device_get_queue(ggml_metal_device_t dev) {
|
|
1375
|
+
return dev->mtl_queue;
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
ggml_metal_library_t ggml_metal_device_get_library(ggml_metal_device_t dev) {
|
|
1379
|
+
return dev->library;
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
void ggml_metal_device_rsets_add(ggml_metal_device_t dev, ggml_metal_rset_t rset) {
|
|
1383
|
+
if (rset == nil) {
|
|
1384
|
+
return;
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
GGML_ASSERT(dev->rsets);
|
|
1388
|
+
|
|
1389
|
+
[dev->rsets->lock lock];
|
|
1390
|
+
|
|
1391
|
+
[dev->rsets->data addObject:rset];
|
|
1392
|
+
|
|
1393
|
+
[dev->rsets->lock unlock];
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
void ggml_metal_device_rsets_rm(ggml_metal_device_t dev, ggml_metal_rset_t rset) {
|
|
1397
|
+
if (rset == nil) {
|
|
1398
|
+
return;
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
GGML_ASSERT(dev->rsets);
|
|
1402
|
+
|
|
1403
|
+
[dev->rsets->lock lock];
|
|
1404
|
+
|
|
1405
|
+
[dev->rsets->data removeObject:rset];
|
|
1406
|
+
|
|
1407
|
+
[dev->rsets->lock unlock];
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
void ggml_metal_device_rsets_keep_alive(ggml_metal_device_t dev) {
|
|
1411
|
+
if (dev->rsets == NULL) {
|
|
1412
|
+
return;
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
atomic_store_explicit(&dev->rsets->d_loop, dev->rsets->loops_per_s*dev->rsets->keep_alive_s, memory_order_relaxed);
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
struct ggml_metal_event {
|
|
1419
|
+
void * obj; // id<MTLSharedEvent>
|
|
1420
|
+
|
|
1421
|
+
atomic_int value;
|
|
1422
|
+
};
|
|
1423
|
+
|
|
1424
|
+
void ggml_metal_event_encode_signal(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf_raw) {
|
|
1425
|
+
id<MTLSharedEvent> event = (id<MTLSharedEvent>)ev->obj;
|
|
1426
|
+
|
|
1427
|
+
id<MTLCommandBuffer> cmd_buf = (id<MTLCommandBuffer>) cmd_buf_raw;
|
|
1428
|
+
|
|
1429
|
+
[cmd_buf encodeSignalEvent:event value:atomic_fetch_add_explicit(&ev->value, 1, memory_order_relaxed) + 1];
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
void ggml_metal_event_encode_wait(ggml_metal_event_t ev, ggml_metal_cmd_buf_t cmd_buf_raw) {
|
|
1433
|
+
id<MTLSharedEvent> event = (id<MTLSharedEvent>)ev->obj;
|
|
1434
|
+
|
|
1435
|
+
id<MTLCommandBuffer> cmd_buf = (id<MTLCommandBuffer>) cmd_buf_raw;
|
|
1436
|
+
|
|
1437
|
+
[cmd_buf encodeWaitForEvent:event value:atomic_load_explicit(&ev->value, memory_order_relaxed)];
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
ggml_metal_event_t ggml_metal_device_event_init(ggml_metal_device_t dev) {
|
|
1441
|
+
id<MTLSharedEvent> event = [dev->mtl_device newSharedEvent];
|
|
1442
|
+
|
|
1443
|
+
ggml_metal_event_t ev = calloc(1, sizeof(struct ggml_metal_event));
|
|
1444
|
+
|
|
1445
|
+
ev->obj = (__bridge void *)event;
|
|
1446
|
+
ev->value = 0;
|
|
1447
|
+
|
|
1448
|
+
return ev;
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
void ggml_metal_device_event_free(ggml_metal_device_t dev, ggml_metal_event_t ev) {
|
|
1452
|
+
@autoreleasepool {
|
|
1453
|
+
id<MTLSharedEvent> event = ev->obj;
|
|
1454
|
+
[event release];
|
|
1455
|
+
|
|
1456
|
+
free(ev);
|
|
1457
|
+
|
|
1458
|
+
GGML_UNUSED(dev);
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
void ggml_metal_device_event_synchronize(ggml_metal_device_t dev, ggml_metal_event_t ev) {
|
|
1463
|
+
id<MTLSharedEvent> event = ev->obj;
|
|
1464
|
+
const bool res = [event waitUntilSignaledValue:atomic_load_explicit(&ev->value, memory_order_relaxed) timeoutMS:60000];
|
|
1465
|
+
if (!res) {
|
|
1466
|
+
GGML_ABORT("%s: failed to wait for event\n", __func__);
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
GGML_UNUSED(dev);
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
void ggml_metal_device_get_memory(ggml_metal_device_t dev, size_t * free, size_t * total) {
|
|
1473
|
+
if (@available(macOS 10.12, iOS 16.0, *)) {
|
|
1474
|
+
*total = dev->mtl_device.recommendedMaxWorkingSetSize;
|
|
1475
|
+
size_t cur = dev->mtl_device.currentAllocatedSize;
|
|
1476
|
+
// it's possible to allocate more than `recommendedMaxWorkingSetSize`
|
|
1477
|
+
*free = *total > cur ? *total - cur : 0;
|
|
1478
|
+
} else {
|
|
1479
|
+
*free = 0;
|
|
1480
|
+
*total = 0;
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
static bool ggml_metal_supports_mul_mat_op(
|
|
1485
|
+
bool has_simdgroup_reduction,
|
|
1486
|
+
const struct ggml_tensor * op,
|
|
1487
|
+
bool src0_f16_has_mv,
|
|
1488
|
+
bool mm_path) {
|
|
1489
|
+
if (!has_simdgroup_reduction || op->src[0]->type == GGML_TYPE_NVFP4) {
|
|
1490
|
+
return false;
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
if (op->src[1]->type != GGML_TYPE_F16) {
|
|
1494
|
+
return true;
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
if (op->src[0]->type == GGML_TYPE_BF16) {
|
|
1498
|
+
return false;
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
if (src0_f16_has_mv && op->src[0]->type == GGML_TYPE_F16) {
|
|
1502
|
+
return true;
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
return mm_path;
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tensor * op) {
|
|
1509
|
+
const bool has_simdgroup_mm = dev->props.has_simdgroup_mm;
|
|
1510
|
+
const bool has_simdgroup_reduction = dev->props.has_simdgroup_reduction;
|
|
1511
|
+
const bool has_bfloat = dev->props.has_bfloat;
|
|
1512
|
+
|
|
1513
|
+
if (!has_bfloat) {
|
|
1514
|
+
if (op->type == GGML_TYPE_BF16) {
|
|
1515
|
+
return false;
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
for (size_t i = 0, n = 3; i < n; ++i) {
|
|
1519
|
+
if (op->src[i] != NULL && op->src[i]->type == GGML_TYPE_BF16) {
|
|
1520
|
+
return false;
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
switch (op->op) {
|
|
1526
|
+
case GGML_OP_SCALE:
|
|
1527
|
+
case GGML_OP_FILL:
|
|
1528
|
+
case GGML_OP_CLAMP:
|
|
1529
|
+
case GGML_OP_SQR:
|
|
1530
|
+
case GGML_OP_SQRT:
|
|
1531
|
+
case GGML_OP_SIN:
|
|
1532
|
+
case GGML_OP_COS:
|
|
1533
|
+
case GGML_OP_LOG:
|
|
1534
|
+
return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
|
|
1535
|
+
case GGML_OP_UNARY:
|
|
1536
|
+
switch (ggml_get_unary_op(op)) {
|
|
1537
|
+
case GGML_UNARY_OP_TANH:
|
|
1538
|
+
case GGML_UNARY_OP_RELU:
|
|
1539
|
+
case GGML_UNARY_OP_SIGMOID:
|
|
1540
|
+
case GGML_UNARY_OP_GELU:
|
|
1541
|
+
case GGML_UNARY_OP_GELU_ERF:
|
|
1542
|
+
case GGML_UNARY_OP_GELU_QUICK:
|
|
1543
|
+
case GGML_UNARY_OP_SILU:
|
|
1544
|
+
case GGML_UNARY_OP_ELU:
|
|
1545
|
+
case GGML_UNARY_OP_NEG:
|
|
1546
|
+
case GGML_UNARY_OP_ABS:
|
|
1547
|
+
case GGML_UNARY_OP_SGN:
|
|
1548
|
+
case GGML_UNARY_OP_STEP:
|
|
1549
|
+
case GGML_UNARY_OP_HARDSWISH:
|
|
1550
|
+
case GGML_UNARY_OP_HARDSIGMOID:
|
|
1551
|
+
case GGML_UNARY_OP_EXP:
|
|
1552
|
+
case GGML_UNARY_OP_SOFTPLUS:
|
|
1553
|
+
case GGML_UNARY_OP_EXPM1:
|
|
1554
|
+
case GGML_UNARY_OP_FLOOR:
|
|
1555
|
+
case GGML_UNARY_OP_CEIL:
|
|
1556
|
+
case GGML_UNARY_OP_ROUND:
|
|
1557
|
+
case GGML_UNARY_OP_TRUNC:
|
|
1558
|
+
case GGML_UNARY_OP_XIELU:
|
|
1559
|
+
return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
|
|
1560
|
+
default:
|
|
1561
|
+
return false;
|
|
1562
|
+
}
|
|
1563
|
+
case GGML_OP_SILU_BACK:
|
|
1564
|
+
return (op->src[0]->type == GGML_TYPE_F32) &&
|
|
1565
|
+
(op->src[1]->type == GGML_TYPE_F32) &&
|
|
1566
|
+
(op->type == GGML_TYPE_F32) &&
|
|
1567
|
+
ggml_is_contiguous(op->src[0]) &&
|
|
1568
|
+
ggml_is_contiguous(op->src[1]) &&
|
|
1569
|
+
ggml_is_contiguous(op) &&
|
|
1570
|
+
ggml_are_same_shape(op->src[0], op->src[1]);
|
|
1571
|
+
case GGML_OP_GLU:
|
|
1572
|
+
switch (ggml_get_glu_op(op)) {
|
|
1573
|
+
case GGML_GLU_OP_REGLU:
|
|
1574
|
+
case GGML_GLU_OP_GEGLU:
|
|
1575
|
+
case GGML_GLU_OP_SWIGLU:
|
|
1576
|
+
case GGML_GLU_OP_SWIGLU_OAI:
|
|
1577
|
+
case GGML_GLU_OP_GEGLU_ERF:
|
|
1578
|
+
case GGML_GLU_OP_GEGLU_QUICK:
|
|
1579
|
+
case GGML_GLU_OP_SWIGLU_CLAMP:
|
|
1580
|
+
return ggml_is_contiguous_1(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
|
|
1581
|
+
default:
|
|
1582
|
+
return false;
|
|
1583
|
+
}
|
|
1584
|
+
case GGML_OP_NONE:
|
|
1585
|
+
case GGML_OP_RESHAPE:
|
|
1586
|
+
case GGML_OP_VIEW:
|
|
1587
|
+
case GGML_OP_TRANSPOSE:
|
|
1588
|
+
case GGML_OP_PERMUTE:
|
|
1589
|
+
return true;
|
|
1590
|
+
case GGML_OP_CONCAT:
|
|
1591
|
+
{
|
|
1592
|
+
const enum ggml_type src0_type = op->src[0]->type;
|
|
1593
|
+
const enum ggml_type src1_type = op->src[1]->type;
|
|
1594
|
+
if (src0_type != src1_type || src0_type != op->type) {
|
|
1595
|
+
return false;
|
|
1596
|
+
}
|
|
1597
|
+
switch (src0_type) {
|
|
1598
|
+
case GGML_TYPE_F32:
|
|
1599
|
+
case GGML_TYPE_F16:
|
|
1600
|
+
case GGML_TYPE_I8:
|
|
1601
|
+
case GGML_TYPE_I16:
|
|
1602
|
+
case GGML_TYPE_I32:
|
|
1603
|
+
case GGML_TYPE_I64:
|
|
1604
|
+
return true;
|
|
1605
|
+
case GGML_TYPE_BF16:
|
|
1606
|
+
return has_bfloat;
|
|
1607
|
+
case GGML_TYPE_Q4_0:
|
|
1608
|
+
case GGML_TYPE_Q4_1:
|
|
1609
|
+
case GGML_TYPE_Q5_0:
|
|
1610
|
+
case GGML_TYPE_Q5_1:
|
|
1611
|
+
case GGML_TYPE_Q8_0:
|
|
1612
|
+
return true;
|
|
1613
|
+
default:
|
|
1614
|
+
return false;
|
|
1615
|
+
}
|
|
1616
|
+
}
|
|
1617
|
+
case GGML_OP_ADD:
|
|
1618
|
+
case GGML_OP_SUB:
|
|
1619
|
+
case GGML_OP_MUL:
|
|
1620
|
+
case GGML_OP_DIV:
|
|
1621
|
+
case GGML_OP_ADD_ID:
|
|
1622
|
+
return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && (op->src[0]->type == op->src[1]->type);
|
|
1623
|
+
case GGML_OP_ACC:
|
|
1624
|
+
return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]) && op->src[0]->type == GGML_TYPE_F32;
|
|
1625
|
+
case GGML_OP_REPEAT:
|
|
1626
|
+
case GGML_OP_CONV_TRANSPOSE_1D:
|
|
1627
|
+
return true;
|
|
1628
|
+
case GGML_OP_CONV_TRANSPOSE_2D:
|
|
1629
|
+
return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) &&
|
|
1630
|
+
(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) &&
|
|
1631
|
+
op->src[1]->type == GGML_TYPE_F32 &&
|
|
1632
|
+
op->type == GGML_TYPE_F32;
|
|
1633
|
+
case GGML_OP_COL2IM_1D:
|
|
1634
|
+
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) &&
|
|
1635
|
+
op->type == op->src[0]->type &&
|
|
1636
|
+
ggml_is_contiguous(op->src[0]) &&
|
|
1637
|
+
ggml_is_contiguous(op);
|
|
1638
|
+
case GGML_OP_CONV_3D:
|
|
1639
|
+
return ggml_is_contiguous(op->src[0]) &&
|
|
1640
|
+
ggml_is_contiguous(op->src[1]) &&
|
|
1641
|
+
(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32) &&
|
|
1642
|
+
op->src[1]->type == GGML_TYPE_F32;
|
|
1643
|
+
case GGML_OP_SUM:
|
|
1644
|
+
return has_simdgroup_reduction && ggml_is_contiguous(op->src[0]);
|
|
1645
|
+
case GGML_OP_TRI:
|
|
1646
|
+
return ggml_is_contiguous_rows(op->src[0]);
|
|
1647
|
+
case GGML_OP_SUM_ROWS:
|
|
1648
|
+
case GGML_OP_CUMSUM:
|
|
1649
|
+
case GGML_OP_MEAN:
|
|
1650
|
+
case GGML_OP_SOFT_MAX:
|
|
1651
|
+
case GGML_OP_GROUP_NORM:
|
|
1652
|
+
case GGML_OP_L2_NORM:
|
|
1653
|
+
return has_simdgroup_reduction && ggml_is_contiguous_rows(op->src[0]);
|
|
1654
|
+
case GGML_OP_COUNT_EQUAL:
|
|
1655
|
+
return has_simdgroup_reduction &&
|
|
1656
|
+
op->src[0]->type == GGML_TYPE_I32 &&
|
|
1657
|
+
op->src[1]->type == GGML_TYPE_I32 &&
|
|
1658
|
+
op->type == GGML_TYPE_I64;
|
|
1659
|
+
case GGML_OP_ARGMAX:
|
|
1660
|
+
return has_simdgroup_reduction;
|
|
1661
|
+
case GGML_OP_NORM:
|
|
1662
|
+
case GGML_OP_RMS_NORM:
|
|
1663
|
+
return has_simdgroup_reduction && (ggml_is_contiguous_rows(op->src[0]));
|
|
1664
|
+
case GGML_OP_ROPE:
|
|
1665
|
+
case GGML_OP_ROPE_BACK:
|
|
1666
|
+
return true;
|
|
1667
|
+
case GGML_OP_IM2COL:
|
|
1668
|
+
return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32);
|
|
1669
|
+
case GGML_OP_CONV_2D:
|
|
1670
|
+
return ggml_is_contiguous(op->src[0]) &&
|
|
1671
|
+
op->src[1]->type == GGML_TYPE_F32 &&
|
|
1672
|
+
op->type == GGML_TYPE_F32 &&
|
|
1673
|
+
(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32);
|
|
1674
|
+
case GGML_OP_CONV_2D_DW:
|
|
1675
|
+
return op->src[1]->type == GGML_TYPE_F32 &&
|
|
1676
|
+
op->type == GGML_TYPE_F32 &&
|
|
1677
|
+
(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32);
|
|
1678
|
+
case GGML_OP_UPSCALE:
|
|
1679
|
+
return op->src[0]->type == GGML_TYPE_F32;
|
|
1680
|
+
case GGML_OP_POOL_1D:
|
|
1681
|
+
return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;
|
|
1682
|
+
case GGML_OP_POOL_2D:
|
|
1683
|
+
return op->src[0]->type == GGML_TYPE_F32;
|
|
1684
|
+
case GGML_OP_PAD:
|
|
1685
|
+
// TODO: add circular padding support for metal, see https://github.com/ggml-org/llama.cpp/pull/16985
|
|
1686
|
+
if (ggml_get_op_params_i32(op, 8) != 0) {
|
|
1687
|
+
return false;
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
return (ggml_get_op_params_i32(op, 0) == 0) && (ggml_get_op_params_i32(op, 2) == 0) &&
|
|
1691
|
+
(ggml_get_op_params_i32(op, 4) == 0) && (ggml_get_op_params_i32(op, 6) == 0);
|
|
1692
|
+
case GGML_OP_PAD_REFLECT_1D:
|
|
1693
|
+
case GGML_OP_TIMESTEP_EMBEDDING:
|
|
1694
|
+
return op->src[0]->type == GGML_TYPE_F32;
|
|
1695
|
+
case GGML_OP_LEAKY_RELU:
|
|
1696
|
+
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
|
|
1697
|
+
case GGML_OP_ARGSORT:
|
|
1698
|
+
case GGML_OP_TOP_K:
|
|
1699
|
+
case GGML_OP_ARANGE:
|
|
1700
|
+
return true;
|
|
1701
|
+
case GGML_OP_ROLL:
|
|
1702
|
+
return ggml_is_contiguous(op->src[0]);
|
|
1703
|
+
case GGML_OP_FLASH_ATTN_EXT:
|
|
1704
|
+
// for new head sizes, add checks here
|
|
1705
|
+
if (op->src[0]->ne[0] != 32 &&
|
|
1706
|
+
op->src[0]->ne[0] != 40 &&
|
|
1707
|
+
op->src[0]->ne[0] != 48 &&
|
|
1708
|
+
op->src[0]->ne[0] != 64 &&
|
|
1709
|
+
op->src[0]->ne[0] != 72 &&
|
|
1710
|
+
op->src[0]->ne[0] != 80 &&
|
|
1711
|
+
op->src[0]->ne[0] != 96 &&
|
|
1712
|
+
op->src[0]->ne[0] != 112 &&
|
|
1713
|
+
op->src[0]->ne[0] != 128 &&
|
|
1714
|
+
op->src[0]->ne[0] != 192 &&
|
|
1715
|
+
op->src[0]->ne[0] != 256 &&
|
|
1716
|
+
op->src[0]->ne[0] != 320 &&
|
|
1717
|
+
op->src[0]->ne[0] != 512 &&
|
|
1718
|
+
op->src[0]->ne[0] != 576) {
|
|
1719
|
+
return false;
|
|
1720
|
+
}
|
|
1721
|
+
if (op->src[1]->type != op->src[2]->type) {
|
|
1722
|
+
return false;
|
|
1723
|
+
}
|
|
1724
|
+
switch (op->src[1]->type) {
|
|
1725
|
+
case GGML_TYPE_F32:
|
|
1726
|
+
case GGML_TYPE_F16:
|
|
1727
|
+
case GGML_TYPE_Q8_0:
|
|
1728
|
+
case GGML_TYPE_Q4_0:
|
|
1729
|
+
case GGML_TYPE_Q4_1:
|
|
1730
|
+
case GGML_TYPE_Q5_0:
|
|
1731
|
+
case GGML_TYPE_Q5_1:
|
|
1732
|
+
break;
|
|
1733
|
+
case GGML_TYPE_BF16:
|
|
1734
|
+
if (!has_bfloat) {
|
|
1735
|
+
return false;
|
|
1736
|
+
}
|
|
1737
|
+
break;
|
|
1738
|
+
default:
|
|
1739
|
+
return false;
|
|
1740
|
+
}
|
|
1741
|
+
return has_simdgroup_mm; // TODO: over-restricted for vec-kernels
|
|
1742
|
+
case GGML_OP_LIGHTNING_INDEXER:
|
|
1743
|
+
if (op->src[0]->ne[0] != OP_LIGHTNING_INDEXER_DK ||
|
|
1744
|
+
op->src[0]->ne[1] != OP_LIGHTNING_INDEXER_NH) {
|
|
1745
|
+
return false;
|
|
1746
|
+
}
|
|
1747
|
+
if (!has_simdgroup_mm ||
|
|
1748
|
+
op->src[0]->type != GGML_TYPE_F32 ||
|
|
1749
|
+
op->src[2]->type != GGML_TYPE_F32 ||
|
|
1750
|
+
op->src[3]->type != GGML_TYPE_F16 ||
|
|
1751
|
+
op->type != GGML_TYPE_F32 ||
|
|
1752
|
+
!ggml_is_contiguous_rows(op->src[0]) ||
|
|
1753
|
+
!ggml_is_contiguous_rows(op->src[1]) ||
|
|
1754
|
+
!ggml_is_contiguous_rows(op->src[2]) ||
|
|
1755
|
+
!ggml_is_contiguous_rows(op->src[3])) {
|
|
1756
|
+
return false;
|
|
1757
|
+
}
|
|
1758
|
+
switch (op->src[1]->type) {
|
|
1759
|
+
case GGML_TYPE_F32:
|
|
1760
|
+
case GGML_TYPE_F16:
|
|
1761
|
+
case GGML_TYPE_Q4_0:
|
|
1762
|
+
case GGML_TYPE_Q4_1:
|
|
1763
|
+
case GGML_TYPE_Q5_0:
|
|
1764
|
+
case GGML_TYPE_Q5_1:
|
|
1765
|
+
case GGML_TYPE_Q8_0:
|
|
1766
|
+
return true;
|
|
1767
|
+
case GGML_TYPE_BF16:
|
|
1768
|
+
return has_bfloat;
|
|
1769
|
+
default:
|
|
1770
|
+
return false;
|
|
1771
|
+
}
|
|
1772
|
+
case GGML_OP_DSV4_HC_COMB:
|
|
1773
|
+
return has_simdgroup_reduction &&
|
|
1774
|
+
op->src[0]->type == GGML_TYPE_F32 &&
|
|
1775
|
+
op->src[1]->type == GGML_TYPE_F32 &&
|
|
1776
|
+
op->src[2]->type == GGML_TYPE_F32 &&
|
|
1777
|
+
op->type == GGML_TYPE_F32 &&
|
|
1778
|
+
op->src[0]->ne[0] == 24 &&
|
|
1779
|
+
op->src[1]->ne[0] >= 3 &&
|
|
1780
|
+
op->src[2]->ne[0] == 24 &&
|
|
1781
|
+
ggml_is_contiguous_rows(op->src[0]) &&
|
|
1782
|
+
ggml_is_contiguous_rows(op->src[1]) &&
|
|
1783
|
+
ggml_is_contiguous_rows(op->src[2]);
|
|
1784
|
+
case GGML_OP_DSV4_HC_PRE:
|
|
1785
|
+
return has_simdgroup_reduction &&
|
|
1786
|
+
op->src[0]->type == GGML_TYPE_F32 &&
|
|
1787
|
+
op->src[1]->type == GGML_TYPE_F32 &&
|
|
1788
|
+
op->type == GGML_TYPE_F32 &&
|
|
1789
|
+
op->src[0]->ne[1] == 4 &&
|
|
1790
|
+
op->src[1]->ne[0] == 4 &&
|
|
1791
|
+
ggml_is_contiguous_rows(op->src[0]) &&
|
|
1792
|
+
ggml_is_contiguous_rows(op->src[1]);
|
|
1793
|
+
case GGML_OP_DSV4_HC_POST:
|
|
1794
|
+
return has_simdgroup_reduction &&
|
|
1795
|
+
op->src[0]->type == GGML_TYPE_F32 &&
|
|
1796
|
+
op->src[1]->type == GGML_TYPE_F32 &&
|
|
1797
|
+
op->src[2]->type == GGML_TYPE_F32 &&
|
|
1798
|
+
op->src[3]->type == GGML_TYPE_F32 &&
|
|
1799
|
+
op->type == GGML_TYPE_F32 &&
|
|
1800
|
+
op->src[1]->ne[1] == 4 &&
|
|
1801
|
+
op->src[2]->ne[0] == 4 &&
|
|
1802
|
+
op->src[3]->ne[0] == 4 &&
|
|
1803
|
+
op->src[3]->ne[1] == 4 &&
|
|
1804
|
+
ggml_is_contiguous_rows(op->src[0]) &&
|
|
1805
|
+
ggml_is_contiguous_rows(op->src[1]) &&
|
|
1806
|
+
ggml_is_contiguous_rows(op->src[2]) &&
|
|
1807
|
+
ggml_is_contiguous_rows(op->src[3]);
|
|
1808
|
+
case GGML_OP_SSM_SCAN:
|
|
1809
|
+
return has_simdgroup_reduction;
|
|
1810
|
+
case GGML_OP_SSM_CONV:
|
|
1811
|
+
return has_simdgroup_reduction;
|
|
1812
|
+
case GGML_OP_RWKV_WKV6:
|
|
1813
|
+
case GGML_OP_RWKV_WKV7:
|
|
1814
|
+
return true;
|
|
1815
|
+
case GGML_OP_GATED_DELTA_NET:
|
|
1816
|
+
return has_simdgroup_reduction && op->src[2]->ne[0] % 32 == 0;
|
|
1817
|
+
case GGML_OP_SOLVE_TRI:
|
|
1818
|
+
return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32;
|
|
1819
|
+
case GGML_OP_MUL_MAT:
|
|
1820
|
+
return ggml_metal_supports_mul_mat_op(
|
|
1821
|
+
has_simdgroup_reduction, op, true,
|
|
1822
|
+
ggml_metal_op_mul_mat_use_mm(op, has_simdgroup_mm));
|
|
1823
|
+
case GGML_OP_MUL_MAT_ID:
|
|
1824
|
+
return ggml_metal_supports_mul_mat_op(
|
|
1825
|
+
has_simdgroup_reduction, op, false,
|
|
1826
|
+
ggml_metal_op_mul_mat_id_use_mm(op, has_simdgroup_mm));
|
|
1827
|
+
case GGML_OP_SET:
|
|
1828
|
+
case GGML_OP_CPY:
|
|
1829
|
+
case GGML_OP_DUP:
|
|
1830
|
+
case GGML_OP_CONT:
|
|
1831
|
+
{
|
|
1832
|
+
switch (op->src[0]->type) {
|
|
1833
|
+
case GGML_TYPE_F32:
|
|
1834
|
+
switch (op->type) {
|
|
1835
|
+
case GGML_TYPE_F32:
|
|
1836
|
+
case GGML_TYPE_F16:
|
|
1837
|
+
case GGML_TYPE_BF16:
|
|
1838
|
+
case GGML_TYPE_Q8_0:
|
|
1839
|
+
case GGML_TYPE_Q1_0:
|
|
1840
|
+
case GGML_TYPE_Q2_0:
|
|
1841
|
+
case GGML_TYPE_Q4_0:
|
|
1842
|
+
case GGML_TYPE_Q4_1:
|
|
1843
|
+
case GGML_TYPE_Q5_0:
|
|
1844
|
+
case GGML_TYPE_Q5_1:
|
|
1845
|
+
case GGML_TYPE_IQ4_NL:
|
|
1846
|
+
case GGML_TYPE_TQ2_0:
|
|
1847
|
+
case GGML_TYPE_I32:
|
|
1848
|
+
return true;
|
|
1849
|
+
default:
|
|
1850
|
+
return false;
|
|
1851
|
+
}
|
|
1852
|
+
case GGML_TYPE_F16:
|
|
1853
|
+
switch (op->type) {
|
|
1854
|
+
case GGML_TYPE_F32:
|
|
1855
|
+
case GGML_TYPE_F16:
|
|
1856
|
+
return true;
|
|
1857
|
+
default:
|
|
1858
|
+
return false;
|
|
1859
|
+
}
|
|
1860
|
+
case GGML_TYPE_BF16:
|
|
1861
|
+
switch (op->type) {
|
|
1862
|
+
case GGML_TYPE_F32:
|
|
1863
|
+
case GGML_TYPE_BF16:
|
|
1864
|
+
return true;
|
|
1865
|
+
default:
|
|
1866
|
+
return false;
|
|
1867
|
+
}
|
|
1868
|
+
case GGML_TYPE_Q1_0:
|
|
1869
|
+
case GGML_TYPE_Q2_0:
|
|
1870
|
+
case GGML_TYPE_Q4_0:
|
|
1871
|
+
case GGML_TYPE_Q4_1:
|
|
1872
|
+
case GGML_TYPE_Q5_0:
|
|
1873
|
+
case GGML_TYPE_Q5_1:
|
|
1874
|
+
case GGML_TYPE_Q8_0:
|
|
1875
|
+
case GGML_TYPE_TQ2_0:
|
|
1876
|
+
switch (op->type) {
|
|
1877
|
+
case GGML_TYPE_F32:
|
|
1878
|
+
case GGML_TYPE_F16:
|
|
1879
|
+
return true;
|
|
1880
|
+
default:
|
|
1881
|
+
return false;
|
|
1882
|
+
}
|
|
1883
|
+
case GGML_TYPE_I32:
|
|
1884
|
+
return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_I32;
|
|
1885
|
+
default:
|
|
1886
|
+
return false;
|
|
1887
|
+
};
|
|
1888
|
+
}
|
|
1889
|
+
case GGML_OP_GET_ROWS:
|
|
1890
|
+
return op->src[0]->type != GGML_TYPE_NVFP4;
|
|
1891
|
+
case GGML_OP_SET_ROWS:
|
|
1892
|
+
{
|
|
1893
|
+
if (op->src[0]->type == GGML_TYPE_F16) {
|
|
1894
|
+
return op->type == GGML_TYPE_F16;
|
|
1895
|
+
}
|
|
1896
|
+
|
|
1897
|
+
if (op->src[0]->type != GGML_TYPE_F32) {
|
|
1898
|
+
return false;
|
|
1899
|
+
}
|
|
1900
|
+
|
|
1901
|
+
switch (op->type) {
|
|
1902
|
+
case GGML_TYPE_F32:
|
|
1903
|
+
case GGML_TYPE_F16:
|
|
1904
|
+
case GGML_TYPE_BF16:
|
|
1905
|
+
case GGML_TYPE_Q8_0:
|
|
1906
|
+
case GGML_TYPE_Q4_0:
|
|
1907
|
+
case GGML_TYPE_Q4_1:
|
|
1908
|
+
case GGML_TYPE_Q5_0:
|
|
1909
|
+
case GGML_TYPE_Q5_1:
|
|
1910
|
+
case GGML_TYPE_IQ4_NL:
|
|
1911
|
+
case GGML_TYPE_TQ2_0:
|
|
1912
|
+
return true;
|
|
1913
|
+
default:
|
|
1914
|
+
return false;
|
|
1915
|
+
};
|
|
1916
|
+
}
|
|
1917
|
+
case GGML_OP_DIAG:
|
|
1918
|
+
return true;
|
|
1919
|
+
case GGML_OP_OPT_STEP_ADAMW:
|
|
1920
|
+
case GGML_OP_OPT_STEP_SGD:
|
|
1921
|
+
return has_simdgroup_reduction;
|
|
1922
|
+
default:
|
|
1923
|
+
return false;
|
|
1924
|
+
}
|
|
1925
|
+
}
|
|
1926
|
+
|
|
1927
|
+
const struct ggml_metal_device_props * ggml_metal_device_get_props(ggml_metal_device_t dev) {
|
|
1928
|
+
return &dev->props;
|
|
1929
|
+
}
|
|
1930
|
+
|
|
1931
|
+
static void ggml_metal_device_disable_tensor(ggml_metal_device_t dev) {
|
|
1932
|
+
dev->props.has_tensor = false;
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
//
|
|
1936
|
+
// device buffers
|
|
1937
|
+
//
|
|
1938
|
+
|
|
1939
|
+
// max memory buffers that can be mapped to the device
|
|
1940
|
+
#define GGML_METAL_MAX_BUFFERS 64
|
|
1941
|
+
|
|
1942
|
+
struct ggml_metal_buffer_wrapper {
|
|
1943
|
+
void * data;
|
|
1944
|
+
size_t size;
|
|
1945
|
+
|
|
1946
|
+
id<MTLBuffer> metal;
|
|
1947
|
+
};
|
|
1948
|
+
|
|
1949
|
+
struct ggml_metal_buffer {
|
|
1950
|
+
void * all_data;
|
|
1951
|
+
size_t all_size;
|
|
1952
|
+
|
|
1953
|
+
// if false, the Metal buffer data is allocated in private GPU memory and is not shared with the host
|
|
1954
|
+
bool is_shared;
|
|
1955
|
+
bool owned;
|
|
1956
|
+
|
|
1957
|
+
// multiple buffers are used only to avoid the maximum buffer size limitation when using mmap
|
|
1958
|
+
int n_buffers;
|
|
1959
|
+
struct ggml_metal_buffer_wrapper buffers[GGML_METAL_MAX_BUFFERS];
|
|
1960
|
+
|
|
1961
|
+
bool use_residency_sets;
|
|
1962
|
+
|
|
1963
|
+
// optional MTLResidencySet
|
|
1964
|
+
// note: cannot use explicitly "id<MTLResidencySet>" here because it is not available on certain OSes
|
|
1965
|
+
id rset;
|
|
1966
|
+
|
|
1967
|
+
// pointers to global device
|
|
1968
|
+
ggml_metal_device_t dev;
|
|
1969
|
+
};
|
|
1970
|
+
|
|
1971
|
+
static void ggml_metal_log_allocated_size(id<MTLDevice> device, size_t size_aligned) {
|
|
1972
|
+
#ifndef GGML_METAL_NDEBUG
|
|
1973
|
+
#if TARGET_OS_OSX || (TARGET_OS_IOS && __clang_major__ >= 15)
|
|
1974
|
+
if (@available(macOS 10.12, iOS 16.0, *)) {
|
|
1975
|
+
GGML_LOG_DEBUG("%s: allocated buffer, size = %8.2f MiB, (%8.2f / %8.2f)\n",
|
|
1976
|
+
__func__,
|
|
1977
|
+
size_aligned / 1024.0 / 1024.0,
|
|
1978
|
+
device.currentAllocatedSize / 1024.0 / 1024.0,
|
|
1979
|
+
device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);
|
|
1980
|
+
|
|
1981
|
+
if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {
|
|
1982
|
+
GGML_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);
|
|
1983
|
+
}
|
|
1984
|
+
} else {
|
|
1985
|
+
GGML_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, (%8.2f)\n",
|
|
1986
|
+
__func__,
|
|
1987
|
+
size_aligned / 1024.0 / 1024.0,
|
|
1988
|
+
device.currentAllocatedSize / 1024.0 / 1024.0);
|
|
1989
|
+
}
|
|
1990
|
+
#endif
|
|
1991
|
+
#endif
|
|
1992
|
+
GGML_UNUSED(device);
|
|
1993
|
+
GGML_UNUSED(size_aligned);
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
// rset init
|
|
1997
|
+
static bool ggml_metal_buffer_rset_init(ggml_metal_buffer_t buf) {
|
|
1998
|
+
buf->rset = nil;
|
|
1999
|
+
|
|
2000
|
+
if (!buf->use_residency_sets) {
|
|
2001
|
+
return true;
|
|
2002
|
+
}
|
|
2003
|
+
|
|
2004
|
+
#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
|
|
2005
|
+
if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {
|
|
2006
|
+
MTLResidencySetDescriptor * desc = [[MTLResidencySetDescriptor alloc] init];
|
|
2007
|
+
desc.label = @"ggml_metal";
|
|
2008
|
+
desc.initialCapacity = buf->n_buffers;
|
|
2009
|
+
|
|
2010
|
+
NSError * error;
|
|
2011
|
+
buf->rset = [buf->dev->mtl_device newResidencySetWithDescriptor:desc error:&error];
|
|
2012
|
+
if (error) {
|
|
2013
|
+
GGML_LOG_ERROR("%s: error: %s\n", __func__, [[error description] UTF8String]);
|
|
2014
|
+
[desc release];
|
|
2015
|
+
return false;
|
|
2016
|
+
}
|
|
2017
|
+
|
|
2018
|
+
[desc release];
|
|
2019
|
+
|
|
2020
|
+
for (int i = 0; i < buf->n_buffers; i++) {
|
|
2021
|
+
[buf->rset addAllocation:buf->buffers[i].metal];
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
[buf->rset commit];
|
|
2025
|
+
[buf->rset requestResidency];
|
|
2026
|
+
|
|
2027
|
+
return true;
|
|
2028
|
+
}
|
|
2029
|
+
#endif
|
|
2030
|
+
|
|
2031
|
+
return true;
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
// rset free
|
|
2035
|
+
static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) {
|
|
2036
|
+
#if defined(GGML_METAL_HAS_RESIDENCY_SETS)
|
|
2037
|
+
if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) {
|
|
2038
|
+
if (buf->rset) {
|
|
2039
|
+
[buf->rset endResidency];
|
|
2040
|
+
[buf->rset removeAllAllocations];
|
|
2041
|
+
[buf->rset commit];
|
|
2042
|
+
[buf->rset release];
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
2045
|
+
#else
|
|
2046
|
+
GGML_UNUSED(buf);
|
|
2047
|
+
#endif
|
|
2048
|
+
}
|
|
2049
|
+
|
|
2050
|
+
static void * ggml_metal_host_malloc(size_t n) {
|
|
2051
|
+
void * data = NULL;
|
|
2052
|
+
|
|
2053
|
+
#if TARGET_OS_OSX
|
|
2054
|
+
kern_return_t err = vm_allocate((vm_map_t) mach_task_self(), (void *) &data, n, VM_FLAGS_ANYWHERE);
|
|
2055
|
+
if (err != KERN_SUCCESS) {
|
|
2056
|
+
GGML_LOG_ERROR("%s: error: vm_allocate failed\n", __func__);
|
|
2057
|
+
return NULL;
|
|
2058
|
+
}
|
|
2059
|
+
#else
|
|
2060
|
+
const int result = posix_memalign((void **) &data, sysconf(_SC_PAGESIZE), n);
|
|
2061
|
+
if (result != 0) {
|
|
2062
|
+
GGML_LOG_ERROR("%s: error: posix_memalign failed\n", __func__);
|
|
2063
|
+
return NULL;
|
|
2064
|
+
}
|
|
2065
|
+
#endif
|
|
2066
|
+
|
|
2067
|
+
return data;
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
ggml_metal_buffer_t ggml_metal_buffer_init(ggml_metal_device_t dev, size_t size, bool shared) {
|
|
2071
|
+
ggml_metal_buffer_t res = calloc(1, sizeof(struct ggml_metal_buffer));
|
|
2072
|
+
|
|
2073
|
+
res->dev = dev;
|
|
2074
|
+
|
|
2075
|
+
const size_t size_page = sysconf(_SC_PAGESIZE);
|
|
2076
|
+
|
|
2077
|
+
size_t size_aligned = size;
|
|
2078
|
+
if ((size_aligned % size_page) != 0) {
|
|
2079
|
+
size_aligned += (size_page - (size_aligned % size_page));
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);
|
|
2083
|
+
|
|
2084
|
+
shared = shared && props_dev->use_shared_buffers;
|
|
2085
|
+
|
|
2086
|
+
// allocate shared buffer if the device supports it and it is required by the buffer type
|
|
2087
|
+
if (shared) {
|
|
2088
|
+
res->all_data = ggml_metal_host_malloc(size_aligned);
|
|
2089
|
+
res->is_shared = true;
|
|
2090
|
+
} else {
|
|
2091
|
+
// use virtual address
|
|
2092
|
+
res->all_data = (void *) atomic_fetch_add_explicit(&dev->addr_virt, size_aligned, memory_order_relaxed);
|
|
2093
|
+
res->is_shared = false;
|
|
2094
|
+
}
|
|
2095
|
+
res->all_size = size_aligned;
|
|
2096
|
+
|
|
2097
|
+
res->owned = true;
|
|
2098
|
+
|
|
2099
|
+
res->n_buffers = 1;
|
|
2100
|
+
|
|
2101
|
+
if (res->all_data != NULL) {
|
|
2102
|
+
res->buffers[0].size = size;
|
|
2103
|
+
res->buffers[0].metal = nil;
|
|
2104
|
+
|
|
2105
|
+
if (size_aligned > 0) {
|
|
2106
|
+
if (props_dev->use_shared_buffers && shared) {
|
|
2107
|
+
res->buffers[0].metal = [res->dev->mtl_device newBufferWithBytesNoCopy:res->all_data
|
|
2108
|
+
length:size_aligned
|
|
2109
|
+
options:MTLResourceStorageModeShared
|
|
2110
|
+
deallocator:nil];
|
|
2111
|
+
} else {
|
|
2112
|
+
res->buffers[0].metal = [res->dev->mtl_device newBufferWithLength:size_aligned options:MTLResourceStorageModePrivate];
|
|
2113
|
+
}
|
|
2114
|
+
}
|
|
2115
|
+
|
|
2116
|
+
res->buffers[0].data = res->all_data;
|
|
2117
|
+
}
|
|
2118
|
+
|
|
2119
|
+
if (size_aligned > 0 && (res->all_data == NULL || res->buffers[0].metal == nil)) {
|
|
2120
|
+
GGML_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);
|
|
2121
|
+
free(res);
|
|
2122
|
+
return NULL;
|
|
2123
|
+
}
|
|
2124
|
+
|
|
2125
|
+
res->use_residency_sets = props_dev->use_residency_sets;
|
|
2126
|
+
|
|
2127
|
+
if (!ggml_metal_buffer_rset_init(res)) {
|
|
2128
|
+
GGML_LOG_ERROR("%s: error: failed to initialize residency set\n", __func__);
|
|
2129
|
+
free(res);
|
|
2130
|
+
return NULL;
|
|
2131
|
+
}
|
|
2132
|
+
|
|
2133
|
+
ggml_metal_device_rsets_add(dev, res->rset);
|
|
2134
|
+
|
|
2135
|
+
//ggml_metal_log_allocated_size(device, size_aligned);
|
|
2136
|
+
|
|
2137
|
+
return res;
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
ggml_metal_buffer_t ggml_metal_buffer_map(ggml_metal_device_t dev, void * ptr, size_t size, size_t max_tensor_size) {
|
|
2141
|
+
ggml_metal_buffer_t res = calloc(1, sizeof(struct ggml_metal_buffer));
|
|
2142
|
+
|
|
2143
|
+
res->dev = dev;
|
|
2144
|
+
|
|
2145
|
+
res->all_data = ptr;
|
|
2146
|
+
res->all_size = size;
|
|
2147
|
+
|
|
2148
|
+
res->is_shared = true;
|
|
2149
|
+
res->owned = false;
|
|
2150
|
+
|
|
2151
|
+
res->n_buffers = 0;
|
|
2152
|
+
|
|
2153
|
+
const size_t size_page = sysconf(_SC_PAGESIZE);
|
|
2154
|
+
|
|
2155
|
+
// page-align the data ptr
|
|
2156
|
+
{
|
|
2157
|
+
const uintptr_t offs = (uintptr_t) ptr % size_page;
|
|
2158
|
+
ptr = (void *) ((char *) ptr - offs);
|
|
2159
|
+
size += offs;
|
|
2160
|
+
}
|
|
2161
|
+
|
|
2162
|
+
size_t size_aligned = size;
|
|
2163
|
+
if ((size_aligned % size_page) != 0) {
|
|
2164
|
+
size_aligned += (size_page - (size_aligned % size_page));
|
|
2165
|
+
}
|
|
2166
|
+
|
|
2167
|
+
const struct ggml_metal_device_props * props_dev = ggml_metal_device_get_props(dev);
|
|
2168
|
+
|
|
2169
|
+
// the buffer fits into the max buffer size allowed by the device
|
|
2170
|
+
if (size_aligned <= props_dev->max_buffer_size) {
|
|
2171
|
+
res->buffers[res->n_buffers].data = ptr;
|
|
2172
|
+
res->buffers[res->n_buffers].size = size;
|
|
2173
|
+
res->buffers[res->n_buffers].metal = nil;
|
|
2174
|
+
|
|
2175
|
+
if (size_aligned > 0) {
|
|
2176
|
+
res->buffers[res->n_buffers].metal = [res->dev->mtl_device newBufferWithBytesNoCopy:ptr length:size_aligned options:MTLResourceStorageModeShared deallocator:nil];
|
|
2177
|
+
|
|
2178
|
+
if (res->buffers[res->n_buffers].metal == nil) {
|
|
2179
|
+
GGML_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);
|
|
2180
|
+
free(res);
|
|
2181
|
+
return NULL;
|
|
2182
|
+
}
|
|
2183
|
+
}
|
|
2184
|
+
|
|
2185
|
+
ggml_metal_log_allocated_size(res->dev->mtl_device, size_aligned);
|
|
2186
|
+
|
|
2187
|
+
++res->n_buffers;
|
|
2188
|
+
} else {
|
|
2189
|
+
// this overlap between the views will guarantee that the tensor with the maximum size will fully fit into
|
|
2190
|
+
// one of the views
|
|
2191
|
+
const size_t size_ovlp = ((max_tensor_size + size_page - 1) / size_page + 1) * size_page; // round-up 2 pages just in case
|
|
2192
|
+
const size_t size_step = props_dev->max_buffer_size - size_ovlp;
|
|
2193
|
+
const size_t size_view = props_dev->max_buffer_size;
|
|
2194
|
+
|
|
2195
|
+
for (size_t i = 0; i < size; i += size_step) {
|
|
2196
|
+
const size_t size_step_aligned = (i + size_view <= size) ? size_view : (size_aligned - i);
|
|
2197
|
+
|
|
2198
|
+
res->buffers[res->n_buffers].data = (void *) ((uint8_t *) ptr + i);
|
|
2199
|
+
res->buffers[res->n_buffers].size = size_step_aligned;
|
|
2200
|
+
res->buffers[res->n_buffers].metal = nil;
|
|
2201
|
+
|
|
2202
|
+
if (size_step_aligned > 0) {
|
|
2203
|
+
res->buffers[res->n_buffers].metal = [res->dev->mtl_device newBufferWithBytesNoCopy:(void *) ((uint8_t *) ptr + i) length:size_step_aligned options:MTLResourceStorageModeShared deallocator:nil];
|
|
2204
|
+
|
|
2205
|
+
if (res->buffers[res->n_buffers].metal == nil) {
|
|
2206
|
+
GGML_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_step_aligned / 1024.0 / 1024.0);
|
|
2207
|
+
free(res);
|
|
2208
|
+
return NULL;
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
|
|
2212
|
+
ggml_metal_log_allocated_size(res->dev->mtl_device, size_step_aligned);
|
|
2213
|
+
|
|
2214
|
+
if (i + size_step < size) {
|
|
2215
|
+
GGML_LOG_INFO("\n");
|
|
2216
|
+
}
|
|
2217
|
+
|
|
2218
|
+
++res->n_buffers;
|
|
2219
|
+
}
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
res->use_residency_sets = props_dev->use_residency_sets;
|
|
2223
|
+
|
|
2224
|
+
if (!ggml_metal_buffer_rset_init(res)) {
|
|
2225
|
+
GGML_LOG_ERROR("%s: error: failed to initialize residency set\n", __func__);
|
|
2226
|
+
free(res);
|
|
2227
|
+
return NULL;
|
|
2228
|
+
}
|
|
2229
|
+
|
|
2230
|
+
ggml_metal_device_rsets_add(dev, res->rset);
|
|
2231
|
+
|
|
2232
|
+
return res;
|
|
2233
|
+
}
|
|
2234
|
+
|
|
2235
|
+
void ggml_metal_buffer_free(ggml_metal_buffer_t buf) {
|
|
2236
|
+
@autoreleasepool {
|
|
2237
|
+
ggml_metal_device_rsets_rm(buf->dev, buf->rset);
|
|
2238
|
+
|
|
2239
|
+
for (int i = 0; i < buf->n_buffers; i++) {
|
|
2240
|
+
[buf->buffers[i].metal release];
|
|
2241
|
+
}
|
|
2242
|
+
|
|
2243
|
+
ggml_metal_buffer_rset_free(buf);
|
|
2244
|
+
}
|
|
2245
|
+
|
|
2246
|
+
if (buf->is_shared && buf->owned) {
|
|
2247
|
+
#if TARGET_OS_OSX
|
|
2248
|
+
vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)buf->all_data, buf->all_size);
|
|
2249
|
+
#else
|
|
2250
|
+
free(buf->all_data);
|
|
2251
|
+
#endif
|
|
2252
|
+
}
|
|
2253
|
+
|
|
2254
|
+
free(buf);
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
void * ggml_metal_buffer_get_base(ggml_metal_buffer_t buf) {
|
|
2258
|
+
return buf->all_data;
|
|
2259
|
+
}
|
|
2260
|
+
|
|
2261
|
+
bool ggml_metal_buffer_is_shared(ggml_metal_buffer_t buf) {
|
|
2262
|
+
return buf->is_shared;
|
|
2263
|
+
}
|
|
2264
|
+
|
|
2265
|
+
void ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
|
|
2266
|
+
if (buf->is_shared) {
|
|
2267
|
+
memset((char *) tensor->data + offset, value, size);
|
|
2268
|
+
return;
|
|
2269
|
+
}
|
|
2270
|
+
|
|
2271
|
+
@autoreleasepool {
|
|
2272
|
+
// dst
|
|
2273
|
+
struct ggml_metal_buffer_id bid_dst = ggml_metal_buffer_get_id(buf, tensor);
|
|
2274
|
+
bid_dst.offs += offset;
|
|
2275
|
+
|
|
2276
|
+
id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];
|
|
2277
|
+
|
|
2278
|
+
{
|
|
2279
|
+
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
|
2280
|
+
|
|
2281
|
+
[encoder fillBuffer:bid_dst.metal
|
|
2282
|
+
range:NSMakeRange(bid_dst.offs, bid_dst.offs + size)
|
|
2283
|
+
value:value];
|
|
2284
|
+
|
|
2285
|
+
[encoder endEncoding];
|
|
2286
|
+
}
|
|
2287
|
+
|
|
2288
|
+
[cmd_buf commit];
|
|
2289
|
+
[cmd_buf waitUntilCompleted];
|
|
2290
|
+
}
|
|
2291
|
+
}
|
|
2292
|
+
|
|
2293
|
+
void ggml_metal_buffer_set_tensor(ggml_metal_buffer_t buf, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
|
|
2294
|
+
if (buf->is_shared) {
|
|
2295
|
+
memcpy((char *) tensor->data + offset, data, size);
|
|
2296
|
+
return;
|
|
2297
|
+
}
|
|
2298
|
+
|
|
2299
|
+
@autoreleasepool {
|
|
2300
|
+
// src
|
|
2301
|
+
void * data_ptr = (void *)(uintptr_t) data; // "const cast" the src data
|
|
2302
|
+
id<MTLBuffer> buf_src = [buf->dev->mtl_device newBufferWithBytesNoCopy:data_ptr
|
|
2303
|
+
length:size
|
|
2304
|
+
options:MTLResourceStorageModeShared
|
|
2305
|
+
deallocator:nil];
|
|
2306
|
+
|
|
2307
|
+
GGML_ASSERT(buf_src);
|
|
2308
|
+
|
|
2309
|
+
// dst
|
|
2310
|
+
struct ggml_metal_buffer_id bid_dst = ggml_metal_buffer_get_id(buf, tensor);
|
|
2311
|
+
bid_dst.offs += offset;
|
|
2312
|
+
|
|
2313
|
+
// note: for experimentation purposes, here we use a semaphore to wait for the copy to complete
|
|
2314
|
+
// this is alternative to waitUntilCompleted, which should be faster, but don't seem to make much difference
|
|
2315
|
+
dispatch_semaphore_t completion_semaphore = dispatch_semaphore_create(0);
|
|
2316
|
+
|
|
2317
|
+
id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];
|
|
2318
|
+
|
|
2319
|
+
{
|
|
2320
|
+
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
|
2321
|
+
|
|
2322
|
+
[encoder copyFromBuffer:buf_src
|
|
2323
|
+
sourceOffset:0
|
|
2324
|
+
toBuffer:bid_dst.metal
|
|
2325
|
+
destinationOffset:bid_dst.offs
|
|
2326
|
+
size:size];
|
|
2327
|
+
|
|
2328
|
+
[encoder endEncoding];
|
|
2329
|
+
}
|
|
2330
|
+
|
|
2331
|
+
[cmd_buf addCompletedHandler:^(id<MTLCommandBuffer> cb) {
|
|
2332
|
+
// TODO: can check for errors here
|
|
2333
|
+
GGML_UNUSED(cb);
|
|
2334
|
+
|
|
2335
|
+
dispatch_semaphore_signal(completion_semaphore);
|
|
2336
|
+
}];
|
|
2337
|
+
|
|
2338
|
+
[cmd_buf commit];
|
|
2339
|
+
|
|
2340
|
+
dispatch_semaphore_wait(completion_semaphore, DISPATCH_TIME_FOREVER);
|
|
2341
|
+
dispatch_release(completion_semaphore);
|
|
2342
|
+
|
|
2343
|
+
//[cmd_buf waitUntilCompleted];
|
|
2344
|
+
}
|
|
2345
|
+
}
|
|
2346
|
+
|
|
2347
|
+
void ggml_metal_buffer_get_tensor(ggml_metal_buffer_t buf, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
|
|
2348
|
+
if (buf->is_shared) {
|
|
2349
|
+
memcpy(data, (const char *) tensor->data + offset, size);
|
|
2350
|
+
return;
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
@autoreleasepool {
|
|
2354
|
+
// src
|
|
2355
|
+
struct ggml_metal_buffer_id bid_src = ggml_metal_buffer_get_id(buf, tensor);
|
|
2356
|
+
bid_src.offs += offset;
|
|
2357
|
+
|
|
2358
|
+
// dst
|
|
2359
|
+
id<MTLBuffer> buf_dst = [buf->dev->mtl_device newBufferWithBytesNoCopy:data
|
|
2360
|
+
length:size
|
|
2361
|
+
options:MTLResourceStorageModeShared
|
|
2362
|
+
deallocator:nil];
|
|
2363
|
+
|
|
2364
|
+
GGML_ASSERT(buf_dst);
|
|
2365
|
+
|
|
2366
|
+
id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];
|
|
2367
|
+
|
|
2368
|
+
{
|
|
2369
|
+
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
|
2370
|
+
|
|
2371
|
+
[encoder copyFromBuffer:bid_src.metal
|
|
2372
|
+
sourceOffset:bid_src.offs
|
|
2373
|
+
toBuffer:buf_dst
|
|
2374
|
+
destinationOffset:0
|
|
2375
|
+
size:size];
|
|
2376
|
+
|
|
2377
|
+
[encoder endEncoding];
|
|
2378
|
+
}
|
|
2379
|
+
|
|
2380
|
+
[cmd_buf commit];
|
|
2381
|
+
[cmd_buf waitUntilCompleted];
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
|
|
2385
|
+
bool ggml_metal_buffer_cpy_tensor(ggml_metal_buffer_t buf_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {
|
|
2386
|
+
ggml_metal_buffer_t buf_src = (ggml_metal_buffer_t)src->buffer->context;
|
|
2387
|
+
|
|
2388
|
+
const size_t size = ggml_nbytes(src);
|
|
2389
|
+
|
|
2390
|
+
// if both buffers are shared, we can use memcpy directly
|
|
2391
|
+
if (buf_dst->is_shared && buf_src->is_shared) {
|
|
2392
|
+
memcpy(dst->data, src->data, size);
|
|
2393
|
+
return true;
|
|
2394
|
+
}
|
|
2395
|
+
|
|
2396
|
+
// for private buffers, we need to use Metal blit commands
|
|
2397
|
+
@autoreleasepool {
|
|
2398
|
+
struct ggml_metal_buffer_id bid_src = ggml_metal_buffer_get_id(buf_src, src);
|
|
2399
|
+
struct ggml_metal_buffer_id bid_dst = ggml_metal_buffer_get_id(buf_dst, dst);
|
|
2400
|
+
|
|
2401
|
+
if (bid_src.metal == nil || bid_dst.metal == nil) {
|
|
2402
|
+
return false;
|
|
2403
|
+
}
|
|
2404
|
+
|
|
2405
|
+
id<MTLCommandBuffer> cmd_buf = [buf_dst->dev->mtl_queue commandBufferWithUnretainedReferences];
|
|
2406
|
+
|
|
2407
|
+
{
|
|
2408
|
+
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
|
2409
|
+
|
|
2410
|
+
[encoder copyFromBuffer:bid_src.metal
|
|
2411
|
+
sourceOffset:bid_src.offs
|
|
2412
|
+
toBuffer:bid_dst.metal
|
|
2413
|
+
destinationOffset:bid_dst.offs
|
|
2414
|
+
size:size];
|
|
2415
|
+
|
|
2416
|
+
[encoder endEncoding];
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
[cmd_buf commit];
|
|
2420
|
+
[cmd_buf waitUntilCompleted];
|
|
2421
|
+
}
|
|
2422
|
+
|
|
2423
|
+
return true;
|
|
2424
|
+
}
|
|
2425
|
+
|
|
2426
|
+
void ggml_metal_buffer_clear(ggml_metal_buffer_t buf, uint8_t value) {
|
|
2427
|
+
if (buf->is_shared) {
|
|
2428
|
+
memset(buf->all_data, value, buf->all_size);
|
|
2429
|
+
return;
|
|
2430
|
+
}
|
|
2431
|
+
|
|
2432
|
+
@autoreleasepool {
|
|
2433
|
+
id<MTLCommandBuffer> cmd_buf = [buf->dev->mtl_queue commandBufferWithUnretainedReferences];
|
|
2434
|
+
|
|
2435
|
+
{
|
|
2436
|
+
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
|
2437
|
+
|
|
2438
|
+
[encoder fillBuffer:buf->buffers[0].metal
|
|
2439
|
+
range:NSMakeRange(0, buf->buffers[0].size)
|
|
2440
|
+
value:value];
|
|
2441
|
+
|
|
2442
|
+
[encoder endEncoding];
|
|
2443
|
+
}
|
|
2444
|
+
|
|
2445
|
+
[cmd_buf commit];
|
|
2446
|
+
[cmd_buf waitUntilCompleted];
|
|
2447
|
+
}
|
|
2448
|
+
}
|
|
2449
|
+
|
|
2450
|
+
struct ggml_metal_buffer_id ggml_metal_buffer_get_id(ggml_metal_buffer_t buf, const struct ggml_tensor * t) {
|
|
2451
|
+
struct ggml_metal_buffer_id res = { nil, 0 };
|
|
2452
|
+
|
|
2453
|
+
const int64_t tsize = ggml_nbytes(t);
|
|
2454
|
+
|
|
2455
|
+
// find the view that contains the tensor fully
|
|
2456
|
+
for (int i = 0; i < buf->n_buffers; ++i) {
|
|
2457
|
+
const int64_t ioffs = (int64_t) t->data - (int64_t) buf->buffers[i].data;
|
|
2458
|
+
|
|
2459
|
+
//GGML_LOG_INFO("ioffs = %10ld, tsize = %10ld, sum = %10ld, buf->buffers[%d].size = %10ld\n", ioffs, tsize, ioffs + tsize, i, buf->buffers[i].size);
|
|
2460
|
+
if (ioffs >= 0 && ioffs + tsize <= (int64_t) buf->buffers[i].size) {
|
|
2461
|
+
res.metal = buf->buffers[i].metal;
|
|
2462
|
+
res.offs = (size_t) ioffs;
|
|
2463
|
+
|
|
2464
|
+
//GGML_LOG_INFO("%s: tensor '%16s', offs = %8ld\n", __func__, t->name, *offs);
|
|
2465
|
+
|
|
2466
|
+
return res;
|
|
2467
|
+
}
|
|
2468
|
+
}
|
|
2469
|
+
|
|
2470
|
+
GGML_LOG_ERROR("%s: error: tensor '%s' buffer is nil\n", __func__, t->name);
|
|
2471
|
+
|
|
2472
|
+
return res;
|
|
2473
|
+
}
|