elelem-ggml 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/exe/elelem-ggml +8 -0
- data/ext/elelem/ggml/elelem.c +466 -0
- data/ext/elelem/ggml/extconf.rb +52 -0
- data/lib/elelem/ggml/chat_ml.rb +23 -0
- data/lib/elelem/ggml/gguf.rb +82 -0
- data/lib/elelem/ggml/provider.rb +63 -0
- data/lib/elelem/ggml/tokenizer.rb +95 -0
- data/lib/elelem/ggml/version.rb +7 -0
- data/lib/elelem/ggml.rb +37 -0
- data/vendor/ggml/AUTHORS +335 -0
- data/vendor/ggml/CMakeLists.txt +499 -0
- data/vendor/ggml/CONTRIBUTING.md +3 -0
- data/vendor/ggml/LICENSE +21 -0
- data/vendor/ggml/README.md +49 -0
- data/vendor/ggml/cmake/FindNCCL.cmake +36 -0
- data/vendor/ggml/cmake/GitVars.cmake +22 -0
- data/vendor/ggml/cmake/common.cmake +50 -0
- data/vendor/ggml/cmake/ggml-config.cmake.in +216 -0
- data/vendor/ggml/ggml.pc.in +10 -0
- data/vendor/ggml/include/ggml-alloc.h +86 -0
- data/vendor/ggml/include/ggml-backend.h +437 -0
- data/vendor/ggml/include/ggml-blas.h +25 -0
- data/vendor/ggml/include/ggml-cann.h +123 -0
- data/vendor/ggml/include/ggml-cpp.h +39 -0
- data/vendor/ggml/include/ggml-cpu.h +152 -0
- data/vendor/ggml/include/ggml-cuda.h +47 -0
- data/vendor/ggml/include/ggml-et.h +28 -0
- data/vendor/ggml/include/ggml-hexagon.h +19 -0
- data/vendor/ggml/include/ggml-metal.h +61 -0
- data/vendor/ggml/include/ggml-opencl.h +26 -0
- data/vendor/ggml/include/ggml-openvino.h +37 -0
- data/vendor/ggml/include/ggml-opt.h +256 -0
- data/vendor/ggml/include/ggml-rpc.h +35 -0
- data/vendor/ggml/include/ggml-sycl.h +57 -0
- data/vendor/ggml/include/ggml-virtgpu.h +14 -0
- data/vendor/ggml/include/ggml-vulkan.h +29 -0
- data/vendor/ggml/include/ggml-webgpu.h +19 -0
- data/vendor/ggml/include/ggml-zdnn.h +17 -0
- data/vendor/ggml/include/ggml-zendnn.h +22 -0
- data/vendor/ggml/include/ggml.h +2963 -0
- data/vendor/ggml/include/gguf.h +211 -0
- data/vendor/ggml/requirements.txt +12 -0
- data/vendor/ggml/scripts/gen-authors.sh +9 -0
- data/vendor/ggml/scripts/make-release-checks.sh +96 -0
- data/vendor/ggml/scripts/make-release-desc.sh +69 -0
- data/vendor/ggml/scripts/make-release-summary.txt +33 -0
- data/vendor/ggml/scripts/release.sh +195 -0
- data/vendor/ggml/scripts/sync-llama-am.sh +167 -0
- data/vendor/ggml/scripts/sync-llama.last +1 -0
- data/vendor/ggml/scripts/sync-llama.sh +21 -0
- data/vendor/ggml/scripts/sync-whisper-am.sh +138 -0
- data/vendor/ggml/scripts/sync-whisper.last +1 -0
- data/vendor/ggml/scripts/sync-whisper.sh +17 -0
- data/vendor/ggml/src/CMakeLists.txt +634 -0
- data/vendor/ggml/src/ggml-alloc.c +1249 -0
- data/vendor/ggml/src/ggml-backend-dl.cpp +48 -0
- data/vendor/ggml/src/ggml-backend-dl.h +45 -0
- data/vendor/ggml/src/ggml-backend-impl.h +291 -0
- data/vendor/ggml/src/ggml-backend-meta.cpp +2511 -0
- data/vendor/ggml/src/ggml-backend-reg.cpp +605 -0
- data/vendor/ggml/src/ggml-backend.cpp +2517 -0
- data/vendor/ggml/src/ggml-blas/CMakeLists.txt +101 -0
- data/vendor/ggml/src/ggml-blas/ggml-blas.cpp +530 -0
- data/vendor/ggml/src/ggml-cann/CMakeLists.txt +89 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.cpp +195 -0
- data/vendor/ggml/src/ggml-cann/acl_tensor.h +349 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.cpp +4479 -0
- data/vendor/ggml/src/ggml-cann/aclnn_ops.h +1191 -0
- data/vendor/ggml/src/ggml-cann/common.h +651 -0
- data/vendor/ggml/src/ggml-cann/ggml-cann.cpp +3070 -0
- data/vendor/ggml/src/ggml-common.h +1911 -0
- data/vendor/ggml/src/ggml-cpu/CMakeLists.txt +685 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
- data/vendor/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/vendor/ggml/src/ggml-cpu/amx/common.h +115 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
- data/vendor/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +41 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/quants.c +4319 -0
- data/vendor/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
- data/vendor/ggml/src/ggml-cpu/arch/loongarch/quants.c +2309 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
- data/vendor/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/quants.c +6596 -0
- data/vendor/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
- data/vendor/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
- data/vendor/ggml/src/ggml-cpu/arch/wasm/quants.c +1292 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/quants.c +4108 -0
- data/vendor/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
- data/vendor/ggml/src/ggml-cpu/arch-fallback.h +353 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
- data/vendor/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/vendor/ggml/src/ggml-cpu/cmake/FindSMTIME.cmake +32 -0
- data/vendor/ggml/src/ggml-cpu/common.h +95 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.c +3936 -0
- data/vendor/ggml/src/ggml-cpu/ggml-cpu.cpp +712 -0
- data/vendor/ggml/src/ggml-cpu/hbm.cpp +55 -0
- data/vendor/ggml/src/ggml-cpu/hbm.h +8 -0
- data/vendor/ggml/src/ggml-cpu/iqp.cpp +1253 -0
- data/vendor/ggml/src/ggml-cpu/iqp.h +39 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/CMakeLists.txt +14 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.cpp +1101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kernels.h +101 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1919 -0
- data/vendor/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4164 -0
- data/vendor/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
- data/vendor/ggml/src/ggml-cpu/ops.cpp +12164 -0
- data/vendor/ggml/src/ggml-cpu/ops.h +124 -0
- data/vendor/ggml/src/ggml-cpu/quants.c +1339 -0
- data/vendor/ggml/src/ggml-cpu/quants.h +106 -0
- data/vendor/ggml/src/ggml-cpu/repack.cpp +4836 -0
- data/vendor/ggml/src/ggml-cpu/repack.h +245 -0
- data/vendor/ggml/src/ggml-cpu/simd-gemm.h +226 -0
- data/vendor/ggml/src/ggml-cpu/simd-mappings.h +1319 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.cpp +1742 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime.h +21 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +1027 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp +5768 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.cpp +320 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_env.h +55 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/ime_kernels.h +189 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.cpp +1795 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/repack.h +14 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp +3178 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/rvv_kernels.h +95 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_barrier.h +34 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp +760 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_mem_pool.h +32 -0
- data/vendor/ggml/src/ggml-cpu/spacemit/spine_tcm.h +409 -0
- data/vendor/ggml/src/ggml-cpu/traits.cpp +36 -0
- data/vendor/ggml/src/ggml-cpu/traits.h +38 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.cpp +337 -0
- data/vendor/ggml/src/ggml-cpu/unary-ops.h +35 -0
- data/vendor/ggml/src/ggml-cpu/vec.cpp +613 -0
- data/vendor/ggml/src/ggml-cpu/vec.h +1570 -0
- data/vendor/ggml/src/ggml-cuda/CMakeLists.txt +267 -0
- data/vendor/ggml/src/ggml-cuda/acc.cu +61 -0
- data/vendor/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cu +58 -0
- data/vendor/ggml/src/ggml-cuda/add-id.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cu +971 -0
- data/vendor/ggml/src/ggml-cuda/allreduce.cuh +29 -0
- data/vendor/ggml/src/ggml-cuda/arange.cu +34 -0
- data/vendor/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cu +292 -0
- data/vendor/ggml/src/ggml-cuda/argsort.cuh +20 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cu +574 -0
- data/vendor/ggml/src/ggml-cuda/binbcast.cuh +12 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/vendor/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
- data/vendor/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/common.cuh +1678 -0
- data/vendor/ggml/src/ggml-cuda/concat.cu +242 -0
- data/vendor/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cu +88 -0
- data/vendor/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
- data/vendor/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cu +167 -0
- data/vendor/ggml/src/ggml-cuda/conv2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/convert.cu +705 -0
- data/vendor/ggml/src/ggml-cuda/convert.cuh +66 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/vendor/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/vendor/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cu +617 -0
- data/vendor/ggml/src/ggml-cuda/cpy.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
- data/vendor/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cu +307 -0
- data/vendor/ggml/src/ggml-cuda/cumsum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dequantize.cuh +452 -0
- data/vendor/ggml/src/ggml-cuda/diag.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/diag.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/vendor/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cu +294 -0
- data/vendor/ggml/src/ggml-cuda/dsv4-hc.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/fattn-common.cuh +1289 -0
- data/vendor/ggml/src/ggml-cuda/fattn-mma-f16.cuh +2136 -0
- data/vendor/ggml/src/ggml-cuda/fattn-swizzle.cuh +126 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cu +60 -0
- data/vendor/ggml/src/ggml-cuda/fattn-tile.cuh +1355 -0
- data/vendor/ggml/src/ggml-cuda/fattn-vec.cuh +611 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cu +722 -0
- data/vendor/ggml/src/ggml-cuda/fattn.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/fill.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/fill.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cu +101 -0
- data/vendor/ggml/src/ggml-cuda/fwht.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cu +327 -0
- data/vendor/ggml/src/ggml-cuda/gated_delta_net.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cu +490 -0
- data/vendor/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/vendor/ggml/src/ggml-cuda/ggml-cuda.cu +5781 -0
- data/vendor/ggml/src/ggml-cuda/gla.cu +93 -0
- data/vendor/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cu +267 -0
- data/vendor/ggml/src/ggml-cuda/im2col.cuh +6 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cu +588 -0
- data/vendor/ggml/src/ggml-cuda/lightning-indexer.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/mean.cu +77 -0
- data/vendor/ggml/src/ggml-cuda/mean.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/mma.cuh +1456 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cu +191 -0
- data/vendor/ggml/src/ggml-cuda/mmf.cuh +908 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cu +180 -0
- data/vendor/ggml/src/ggml-cuda/mmid.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-ampere.cuh +383 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-blackwell.cuh +37 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-cdna.cuh +185 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +273 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +274 -0
- data/vendor/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +290 -0
- data/vendor/ggml/src/ggml-cuda/mmq-load-tiles.cuh +1768 -0
- data/vendor/ggml/src/ggml-cuda/mmq-vec-dot.cuh +1240 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cu +386 -0
- data/vendor/ggml/src/ggml-cuda/mmq.cuh +1598 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cu +875 -0
- data/vendor/ggml/src/ggml-cuda/mmvf.cuh +14 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cu +1504 -0
- data/vendor/ggml/src/ggml-cuda/mmvq.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cu +65 -0
- data/vendor/ggml/src/ggml-cuda/moe-weighted-reduction.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/norm.cu +698 -0
- data/vendor/ggml/src/ggml-cuda/norm.cuh +18 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
- data/vendor/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cu +125 -0
- data/vendor/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/pad.cu +106 -0
- data/vendor/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
- data/vendor/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cu +85 -0
- data/vendor/ggml/src/ggml-cuda/pool1d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/vendor/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cu +697 -0
- data/vendor/ggml/src/ggml-cuda/quantize.cuh +70 -0
- data/vendor/ggml/src/ggml-cuda/reduce_rows.cuh +43 -0
- data/vendor/ggml/src/ggml-cuda/roll.cu +67 -0
- data/vendor/ggml/src/ggml-cuda/roll.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/rope.cu +941 -0
- data/vendor/ggml/src/ggml-cuda/rope.cuh +11 -0
- data/vendor/ggml/src/ggml-cuda/scale.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cu +398 -0
- data/vendor/ggml/src/ggml-cuda/set-rows.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/set.cu +39 -0
- data/vendor/ggml/src/ggml-cuda/set.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/snake.cu +72 -0
- data/vendor/ggml/src/ggml-cuda/snake.cuh +8 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cu +37 -0
- data/vendor/ggml/src/ggml-cuda/softcap.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cu +480 -0
- data/vendor/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cu +273 -0
- data/vendor/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cu +206 -0
- data/vendor/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cu +859 -0
- data/vendor/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/sum.cu +41 -0
- data/vendor/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/sumrows.cuh +4 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +6 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +11 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +12 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +111 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cu +275 -0
- data/vendor/ggml/src/ggml-cuda/top-k.cuh +3 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cu +442 -0
- data/vendor/ggml/src/ggml-cuda/topk-moe.cuh +31 -0
- data/vendor/ggml/src/ggml-cuda/tri.cu +136 -0
- data/vendor/ggml/src/ggml-cuda/tri.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/vendor/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/unary.cu +721 -0
- data/vendor/ggml/src/ggml-cuda/unary.cuh +123 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cu +293 -0
- data/vendor/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/vendor/ggml/src/ggml-cuda/vecdotq.cuh +1371 -0
- data/vendor/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
- data/vendor/ggml/src/ggml-cuda/vendors/hip.h +301 -0
- data/vendor/ggml/src/ggml-cuda/vendors/musa.h +151 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cu +253 -0
- data/vendor/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/vendor/ggml/src/ggml-et/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-et/cmake/embed_one_kernel.cmake +15 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.cpp.in +6 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-kernels-embed.hpp.in +12 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.cpp.in +18 -0
- data/vendor/ggml/src/ggml-et/cmake/ggml-et-uberkernel-kernel-map.h.in +13 -0
- data/vendor/ggml/src/ggml-et/et-kernels/CMakeLists.txt +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/scripts/check_unimplemented_instructions.sh +36 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/RunBackend.sh +23 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/block_ops.h +997 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/clamp_f32.c +120 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/concat_f32.c +175 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f16.c +107 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cont_f32.c +248 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/conv_2d_f32_me.c +807 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cpy_f32_f16.c +110 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/crt.S +15 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/cumsum_f32.c +96 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/diag_f32.c +90 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/el_map_f32.c +377 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/fill_f32.c +87 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f16_me.c +1000 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/flash_attn_ext_f32.c +217 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/gated_delta_net_f32.c +346 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/get_rows_f32.c +612 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ggml_tensor.h +44 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/glu_f32.c +606 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/group_norm_f32.c +171 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/im2col.c +130 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/l2_norm_f32.c +237 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/linker.ld +85 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/math_fp.h +299 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mean_f32.c +220 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/memops.c +181 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0.c +358 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q4_0_matrix_engine.c +368 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_Q8_0.c +413 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16.c +142 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f16_matrix_engine.c +329 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32.c +137 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_f32_matrix_engine.c +155 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q4_0.c +169 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_Q8_0.c +160 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/mul_mat_id_f32.c +288 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/norm_f32.c +328 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/pad_f32.c +165 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/platform.h +545 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/quants.h +72 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/repeat_f32.c +118 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_f32.c +270 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rms_norm_mul_f32.c +290 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rope_f32.c +656 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv6_f32.c +184 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/rwkv_wkv7_f32.c +272 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/scale_f32.c +94 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_f32.c +101 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/set_rows_f32.c +394 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/softmax_f32.c +698 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/solve_tri_f32.c +109 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sqr_f32.c +88 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_conv_f32.c +129 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/ssm_scan_f32.c +282 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/sum_rows_f32.c +103 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tensor.h +897 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/tri_f32.c +244 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/uberkernel.c +497 -0
- data/vendor/ggml/src/ggml-et/et-kernels/src/unary_f32.c +705 -0
- data/vendor/ggml/src/ggml-et/ggml-et-common.h +86 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.cpp +502 -0
- data/vendor/ggml/src/ggml-et/ggml-et-cpu-compare.h +54 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.cpp +508 -0
- data/vendor/ggml/src/ggml-et/ggml-et-kernels.h +48 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.cpp +36 -0
- data/vendor/ggml/src/ggml-et/ggml-et-memops.h +18 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.cpp +2584 -0
- data/vendor/ggml/src/ggml-et/ggml-et-ops.h +393 -0
- data/vendor/ggml/src/ggml-et/ggml-et-uberkernel-common.h +17 -0
- data/vendor/ggml/src/ggml-et/ggml-et.cpp +1880 -0
- data/vendor/ggml/src/ggml-feats.h +166 -0
- data/vendor/ggml/src/ggml-hexagon/CMakeLists.txt +113 -0
- data/vendor/ggml/src/ggml-hexagon/ggml-hexagon.cpp +6523 -0
- data/vendor/ggml/src/ggml-hexagon/htp/CMakeLists.txt +58 -0
- data/vendor/ggml/src/ggml-hexagon/htp/act-ops.c +585 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.c +398 -0
- data/vendor/ggml/src/ggml-hexagon/htp/allreduce-ops.h +40 -0
- data/vendor/ggml/src/ggml-hexagon/htp/argsort-ops.c +514 -0
- data/vendor/ggml/src/ggml-hexagon/htp/binary-ops.c +873 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +155 -0
- data/vendor/ggml/src/ggml-hexagon/htp/concat-ops.c +277 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cpy-ops.c +354 -0
- data/vendor/ggml/src/ggml-hexagon/htp/cumsum-ops.c +265 -0
- data/vendor/ggml/src/ggml-hexagon/htp/diag-ops.c +216 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.c +104 -0
- data/vendor/ggml/src/ggml-hexagon/htp/dma-queue.h +408 -0
- data/vendor/ggml/src/ggml-hexagon/htp/fill-ops.c +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.c +2479 -0
- data/vendor/ggml/src/ggml-hexagon/htp/flash-attn-ops.h +344 -0
- data/vendor/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c +1157 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.c +260 -0
- data/vendor/ggml/src/ggml-hexagon/htp/get-rows-ops.h +77 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-bitmap.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-common.h +80 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dma.h +2 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-dump.h +86 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-fastdiv.h +37 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-profile.h +64 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hex-utils.h +65 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h +555 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +1378 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.c +170 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-queue.h +160 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hmx-utils.h +222 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ctx.h +146 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-ops.h +242 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.c +300 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-tensor.h +29 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp-vtcm.h +19 -0
- data/vendor/ggml/src/ggml-hexagon/htp/htp_iface.idl +23 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-arith.h +636 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-base.h +292 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-copy.h +262 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-div.h +291 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-dump.h +129 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-exp.h +255 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h +303 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h +47 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-floor.h +100 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-inverse.h +210 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-log.h +118 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-flat.h +1511 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +1200 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-norm.h +454 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-pow.h +42 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-quant.h +165 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-reduce.h +336 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-repl.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-scale.h +199 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +181 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +90 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-sqrt.h +189 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-types.h +36 -0
- data/vendor/ggml/src/ggml-hexagon/htp/hvx-utils.h +24 -0
- data/vendor/ggml/src/ggml-hexagon/htp/im2col-ops.c +306 -0
- data/vendor/ggml/src/ggml-hexagon/htp/main.c +1245 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.c +4078 -0
- data/vendor/ggml/src/ggml-hexagon/htp/matmul-ops.h +778 -0
- data/vendor/ggml/src/ggml-hexagon/htp/pad-ops.c +547 -0
- data/vendor/ggml/src/ggml-hexagon/htp/repeat-ops.c +148 -0
- data/vendor/ggml/src/ggml-hexagon/htp/rope-ops.c +733 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.c +230 -0
- data/vendor/ggml/src/ggml-hexagon/htp/set-rows-ops.h +74 -0
- data/vendor/ggml/src/ggml-hexagon/htp/softmax-ops.c +407 -0
- data/vendor/ggml/src/ggml-hexagon/htp/solve-tri-ops.c +267 -0
- data/vendor/ggml/src/ggml-hexagon/htp/ssm-conv.c +433 -0
- data/vendor/ggml/src/ggml-hexagon/htp/sum-rows-ops.c +128 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.c +1264 -0
- data/vendor/ggml/src/ggml-hexagon/htp/unary-ops.h +176 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.c +244 -0
- data/vendor/ggml/src/ggml-hexagon/htp/work-queue.h +38 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.cpp +425 -0
- data/vendor/ggml/src/ggml-hexagon/htp-drv.h +123 -0
- data/vendor/ggml/src/ggml-hexagon/htp-opnode.h +372 -0
- data/vendor/ggml/src/ggml-hexagon/libdl.h +79 -0
- data/vendor/ggml/src/ggml-hexagon/libggml-htp.inf +36 -0
- data/vendor/ggml/src/ggml-hip/CMakeLists.txt +153 -0
- data/vendor/ggml/src/ggml-impl.h +783 -0
- data/vendor/ggml/src/ggml-metal/CMakeLists.txt +240 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.cpp +474 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-common.h +56 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-context.m +759 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.cpp +2330 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.h +355 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-device.m +2473 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-impl.h +1345 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.cpp +5610 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-ops.h +106 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.cpp +3732 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal-tuning.h +77 -0
- data/vendor/ggml/src/ggml-metal/ggml-metal.cpp +1006 -0
- data/vendor/ggml/src/ggml-metal/kernels/argsort.metal +337 -0
- data/vendor/ggml/src/ggml-metal/kernels/binbcast.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/common.h +126 -0
- data/vendor/ggml/src/ggml-metal/kernels/conv.metal +724 -0
- data/vendor/ggml/src/ggml-metal/kernels/dequantize.h +735 -0
- data/vendor/ggml/src/ggml-metal/kernels/fa.metal +2445 -0
- data/vendor/ggml/src/ggml-metal/kernels/gated_delta_net.metal +250 -0
- data/vendor/ggml/src/ggml-metal/kernels/misc.metal +595 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mm.metal +853 -0
- data/vendor/ggml/src/ggml-metal/kernels/mul_mv.metal +3225 -0
- data/vendor/ggml/src/ggml-metal/kernels/norm.metal +308 -0
- data/vendor/ggml/src/ggml-metal/kernels/pool.metal +148 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.h +262 -0
- data/vendor/ggml/src/ggml-metal/kernels/quantize.metal +480 -0
- data/vendor/ggml/src/ggml-metal/kernels/reduce.metal +228 -0
- data/vendor/ggml/src/ggml-metal/kernels/rope.metal +333 -0
- data/vendor/ggml/src/ggml-metal/kernels/softmax.metal +223 -0
- data/vendor/ggml/src/ggml-metal/kernels/solve_tri.metal +75 -0
- data/vendor/ggml/src/ggml-metal/kernels/ssm.metal +473 -0
- data/vendor/ggml/src/ggml-metal/kernels/tri.metal +69 -0
- data/vendor/ggml/src/ggml-metal/kernels/unary.metal +400 -0
- data/vendor/ggml/src/ggml-metal/kernels/upscale.metal +179 -0
- data/vendor/ggml/src/ggml-metal/kernels/wkv.metal +179 -0
- data/vendor/ggml/src/ggml-musa/CMakeLists.txt +123 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/vendor/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/vendor/ggml/src/ggml-opencl/CMakeLists.txt +246 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.cpp +453 -0
- data/vendor/ggml/src/ggml-opencl/cl-program-cache.h +75 -0
- data/vendor/ggml/src/ggml-opencl/fa_tune.h +92 -0
- data/vendor/ggml/src/ggml-opencl/ggml-opencl.cpp +26982 -0
- data/vendor/ggml/src/ggml-opencl/kernels/abs.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/argsort.cl +86 -0
- data/vendor/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/vendor/ggml/src/ggml-opencl/kernels/concat.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d.cl +185 -0
- data/vendor/ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cpy.cl +288 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cumsum.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/cvt.cl +2663 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/vendor/ggml/src/ggml-opencl/kernels/div.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/vendor/ggml/src/ggml-opencl/kernels/exp.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/expm1.cl +113 -0
- data/vendor/ggml/src/ggml-opencl/kernels/fill.cl +17 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +421 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +422 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +2656 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q4_0.cl +2011 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_f32_q8_0.cl +1840 -0
- data/vendor/ggml/src/ggml-opencl/kernels/flash_attn_pre_f16.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gated_delta_net.cl +249 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gelu.cl +89 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl +162 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl +376 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl +324 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl +169 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl +326 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl +348 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl +209 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl +328 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl +330 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl +356 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl +335 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl +200 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl +221 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl +226 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl +150 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl +94 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl +139 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +127 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl +132 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl +490 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +281 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl +131 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl +235 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl +134 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl +245 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_tiled.cl +136 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl +129 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +212 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl +233 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +257 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +266 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl +123 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl +160 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl +383 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl +384 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl +823 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_o4.cl +349 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32_tiled.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl +296 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl +299 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl +453 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl +409 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_o4.cl +372 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32_tiled.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl +280 -0
- data/vendor/ggml/src/ggml-opencl/kernels/get_rows.cl +193 -0
- data/vendor/ggml/src/ggml-opencl/kernels/glu.cl +443 -0
- data/vendor/ggml/src/ggml-opencl/kernels/group_norm.cl +121 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/vendor/ggml/src/ggml-opencl/kernels/l2_norm.cl +71 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mean.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_add_id_glu.cl +76 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_combine.cl +79 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_b.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl +64 -0
- data/vendor/ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl +155 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul.cl +152 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl +130 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl +273 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +146 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +196 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl +171 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl +156 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl +163 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl +165 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl +173 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl +175 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl +158 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl +154 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +101 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +1237 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_mrow.cl +306 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl +176 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl +283 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl +222 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +182 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +205 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +244 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +272 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
- data/vendor/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
- data/vendor/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
- data/vendor/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
- data/vendor/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl +42 -0
- data/vendor/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/vendor/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rms_norm.cl +369 -0
- data/vendor/ggml/src/ggml-opencl/kernels/rope.cl +759 -0
- data/vendor/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
- data/vendor/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
- data/vendor/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
- data/vendor/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
- data/vendor/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
- data/vendor/ggml/src/ggml-opencl/kernels/ssm_scan.cl +216 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
- data/vendor/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
- data/vendor/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
- data/vendor/ggml/src/ggml-opencl/kernels/tsembd.cl +48 -0
- data/vendor/ggml/src/ggml-opencl/kernels/upscale.cl +120 -0
- data/vendor/ggml/src/ggml-opencl/libdl.h +79 -0
- data/vendor/ggml/src/ggml-openvino/CMakeLists.txt +22 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.cpp +2057 -0
- data/vendor/ggml/src/ggml-openvino/ggml-decoder.h +417 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +493 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino-extra.h +216 -0
- data/vendor/ggml/src/ggml-openvino/ggml-openvino.cpp +1566 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.cpp +1273 -0
- data/vendor/ggml/src/ggml-openvino/ggml-quants.h +186 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.cpp +272 -0
- data/vendor/ggml/src/ggml-openvino/model-cache.h +56 -0
- data/vendor/ggml/src/ggml-openvino/openvino/decoder.h +133 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/frontend.h +23 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.cpp +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/input_model.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/node_context.h +171 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add.cpp +45 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/add_id.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/argsort.cpp +47 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/clamp.cpp +33 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/concat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cont.cpp +40 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cpy.cpp +295 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/diag.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/div.cpp +146 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/fill.cpp +34 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +241 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +327 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +65 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +43 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +142 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +75 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +64 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +110 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/im2col.cpp +120 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +283 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +100 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/norm.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pad.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/permute.cpp +147 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +53 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/repeat.cpp +48 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/reshape.cpp +113 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +81 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/roll.cpp +36 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/rope.cpp +303 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/scale.cpp +86 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set.cpp +76 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +95 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/softmax.cpp +166 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +108 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sqr.cpp +35 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +61 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/transpose.cpp +54 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/tri.cpp +82 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +27 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp +38 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op/view.cpp +245 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.cpp +88 -0
- data/vendor/ggml/src/ggml-openvino/openvino/op_table.h +67 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +212 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +60 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +44 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +58 -0
- data/vendor/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +17 -0
- data/vendor/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +41 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.cpp +432 -0
- data/vendor/ggml/src/ggml-openvino/openvino/translate_session.h +29 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.cpp +886 -0
- data/vendor/ggml/src/ggml-openvino/openvino/utils.h +88 -0
- data/vendor/ggml/src/ggml-openvino/utils.cpp +1496 -0
- data/vendor/ggml/src/ggml-openvino/utils.h +195 -0
- data/vendor/ggml/src/ggml-opt.cpp +1094 -0
- data/vendor/ggml/src/ggml-quants.c +5667 -0
- data/vendor/ggml/src/ggml-quants.h +115 -0
- data/vendor/ggml/src/ggml-rpc/CMakeLists.txt +49 -0
- data/vendor/ggml/src/ggml-rpc/ggml-rpc.cpp +2351 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.cpp +481 -0
- data/vendor/ggml/src/ggml-rpc/transport-apple.h +27 -0
- data/vendor/ggml/src/ggml-rpc/transport.cpp +739 -0
- data/vendor/ggml/src/ggml-rpc/transport.h +38 -0
- data/vendor/ggml/src/ggml-sycl/CMakeLists.txt +218 -0
- data/vendor/ggml/src/ggml-sycl/add-id.cpp +81 -0
- data/vendor/ggml/src/ggml-sycl/add-id.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/backend.hpp +52 -0
- data/vendor/ggml/src/ggml-sycl/base.hpp +36 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.cpp +650 -0
- data/vendor/ggml/src/ggml-sycl/binbcast.hpp +69 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.cpp +102 -0
- data/vendor/ggml/src/ggml-sycl/col2im-1d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/common.cpp +153 -0
- data/vendor/ggml/src/ggml-sycl/common.hpp +1029 -0
- data/vendor/ggml/src/ggml-sycl/concat.cpp +509 -0
- data/vendor/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv.cpp +101 -0
- data/vendor/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-dw.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.cpp +125 -0
- data/vendor/ggml/src/ggml-sycl/conv2d-transpose.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.cpp +150 -0
- data/vendor/ggml/src/ggml-sycl/conv2d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.cpp +224 -0
- data/vendor/ggml/src/ggml-sycl/conv3d.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/convert.cpp +877 -0
- data/vendor/ggml/src/ggml-sycl/convert.hpp +64 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.cpp +79 -0
- data/vendor/ggml/src/ggml-sycl/count-equal.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/cpy.cpp +1433 -0
- data/vendor/ggml/src/ggml-sycl/cpy.hpp +537 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.cpp +255 -0
- data/vendor/ggml/src/ggml-sycl/cross_entropy_loss.hpp +7 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.cpp +148 -0
- data/vendor/ggml/src/ggml-sycl/cumsum.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dequantize.hpp +1681 -0
- data/vendor/ggml/src/ggml-sycl/diag.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/diag.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.cpp +2227 -0
- data/vendor/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/vendor/ggml/src/ggml-sycl/dpct/helper.hpp +3782 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.cpp +280 -0
- data/vendor/ggml/src/ggml-sycl/dsv4-hc.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.cpp +1432 -0
- data/vendor/ggml/src/ggml-sycl/element_wise.hpp +135 -0
- data/vendor/ggml/src/ggml-sycl/esimd.hpp +589 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.cpp +56 -0
- data/vendor/ggml/src/ggml-sycl/fattn-buffers.hpp +63 -0
- data/vendor/ggml/src/ggml-sycl/fattn-common.hpp +1185 -0
- data/vendor/ggml/src/ggml-sycl/fattn-mkl.cpp +689 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.cpp +450 -0
- data/vendor/ggml/src/ggml-sycl/fattn-onednn.hpp +18 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.cpp +59 -0
- data/vendor/ggml/src/ggml-sycl/fattn-tile.hpp +1246 -0
- data/vendor/ggml/src/ggml-sycl/fattn-vec.hpp +684 -0
- data/vendor/ggml/src/ggml-sycl/fattn.cpp +452 -0
- data/vendor/ggml/src/ggml-sycl/fattn.hpp +42 -0
- data/vendor/ggml/src/ggml-sycl/fill.cpp +55 -0
- data/vendor/ggml/src/ggml-sycl/fill.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/fusion.cpp +212 -0
- data/vendor/ggml/src/ggml-sycl/fusion.hpp +17 -0
- data/vendor/ggml/src/ggml-sycl/fwht.cpp +119 -0
- data/vendor/ggml/src/ggml-sycl/fwht.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.cpp +366 -0
- data/vendor/ggml/src/ggml-sycl/gated_delta_net.hpp +19 -0
- data/vendor/ggml/src/ggml-sycl/gemm.hpp +93 -0
- data/vendor/ggml/src/ggml-sycl/getrows.cpp +368 -0
- data/vendor/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/ggml-sycl.cpp +7028 -0
- data/vendor/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/vendor/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/im2col.cpp +400 -0
- data/vendor/ggml/src/ggml-sycl/im2col.hpp +23 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.cpp +197 -0
- data/vendor/ggml/src/ggml-sycl/lightning-indexer.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/mem.cpp +162 -0
- data/vendor/ggml/src/ggml-sycl/mem.hpp +16 -0
- data/vendor/ggml/src/ggml-sycl/mmq.cpp +3030 -0
- data/vendor/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.cpp +3033 -0
- data/vendor/ggml/src/ggml-sycl/mmvq.hpp +76 -0
- data/vendor/ggml/src/ggml-sycl/norm.cpp +963 -0
- data/vendor/ggml/src/ggml-sycl/norm.hpp +32 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.cpp +131 -0
- data/vendor/ggml/src/ggml-sycl/opt-step.hpp +6 -0
- data/vendor/ggml/src/ggml-sycl/outprod.cpp +83 -0
- data/vendor/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pad.cpp +97 -0
- data/vendor/ggml/src/ggml-sycl/pad.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.cpp +100 -0
- data/vendor/ggml/src/ggml-sycl/pad_reflect_1d.hpp +10 -0
- data/vendor/ggml/src/ggml-sycl/pool.cpp +185 -0
- data/vendor/ggml/src/ggml-sycl/pool.hpp +22 -0
- data/vendor/ggml/src/ggml-sycl/presets.hpp +79 -0
- data/vendor/ggml/src/ggml-sycl/quantize.hpp +133 -0
- data/vendor/ggml/src/ggml-sycl/quants.hpp +204 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.cpp +76 -0
- data/vendor/ggml/src/ggml-sycl/repeat_back.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/roll.cpp +122 -0
- data/vendor/ggml/src/ggml-sycl/roll.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/rope.cpp +651 -0
- data/vendor/ggml/src/ggml-sycl/rope.hpp +26 -0
- data/vendor/ggml/src/ggml-sycl/set.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/set.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.cpp +577 -0
- data/vendor/ggml/src/ggml-sycl/set_rows.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/softmax.cpp +425 -0
- data/vendor/ggml/src/ggml-sycl/softmax.hpp +24 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.cpp +172 -0
- data/vendor/ggml/src/ggml-sycl/solve_tri.hpp +8 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
- data/vendor/ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
- data/vendor/ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
- data/vendor/ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp +8 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.cpp +620 -0
- data/vendor/ggml/src/ggml-sycl/topk-moe.hpp +12 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.cpp +73 -0
- data/vendor/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/vendor/ggml/src/ggml-sycl/type.hpp +112 -0
- data/vendor/ggml/src/ggml-sycl/upscale.cpp +410 -0
- data/vendor/ggml/src/ggml-sycl/upscale.hpp +9 -0
- data/vendor/ggml/src/ggml-sycl/vecdotq.hpp +1726 -0
- data/vendor/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/vendor/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/vendor/ggml/src/ggml-threading.cpp +12 -0
- data/vendor/ggml/src/ggml-threading.h +14 -0
- data/vendor/ggml/src/ggml-version.h.in +4 -0
- data/vendor/ggml/src/ggml-virtgpu/CMakeLists.txt +70 -0
- data/vendor/ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp +87 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/CMakeLists.txt +21 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp +115 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-convert.h +13 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp +102 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp +105 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp +149 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp +51 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h +73 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-dispatched.h +27 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h +32 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/backend.cpp +144 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h +95 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h +94 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_backend.h +50 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs.h +378 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h +232 -0
- data/vendor/ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h +58 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp +81 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp +123 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-device.cpp +160 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend-reg.cpp +213 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-backend.cpp +72 -0
- data/vendor/ggml/src/ggml-virtgpu/ggml-remoting.h +71 -0
- data/vendor/ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml +166 -0
- data/vendor/ggml/src/ggml-virtgpu/include/apir_hw.h +9 -0
- data/vendor/ggml/src/ggml-virtgpu/regenerate_remoting.py +333 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-apir.h +15 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp +58 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp +110 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp +173 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp +194 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward-impl.h +36 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h +54 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.cpp +99 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-shm.h +23 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.cpp +179 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu-utils.h +86 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.cpp +545 -0
- data/vendor/ggml/src/ggml-virtgpu/virtgpu.h +115 -0
- data/vendor/ggml/src/ggml-vulkan/CMakeLists.txt +254 -0
- data/vendor/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/vendor/ggml/src/ggml-vulkan/ggml-vulkan.cpp +20349 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +69 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add1.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/arange.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +86 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp +114 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +473 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv3d_mm.comp +424 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp +98 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +350 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp +67 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose_02.comp +61 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp +135 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss.comp +78 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cross_entropy_loss_back.comp +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp +83 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +741 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +1466 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +40 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_0.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dequant_tq2_0.comp +31 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp +28 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fa_types.glsl +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e2m1.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/float_e4m3.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp +7 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +760 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +223 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +647 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +487 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +146 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp +162 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl +203 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +121 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp +115 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +189 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp +13 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl +11 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl +75 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_back.comp +25 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +51 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/gla.comp +82 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +138 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/lightning_indexer.comp +151 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +93 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_tq2_0.comp +102 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +564 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +475 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +690 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +670 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +89 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +317 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +488 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +99 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/out_prod.comp +59 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp +43 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl +218 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl +35 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +17 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/snake.comp +49 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +195 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +54 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp +62 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp +79 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp +65 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl +53 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp +81 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp +60 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp +134 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +47 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl +19 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp +9 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_clamp.comp +12 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp +118 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp +221 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp +246 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/topk_radix_select.comp +144 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp +42 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +1938 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp +168 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +178 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl +37 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +1367 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/vendor/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/vendor/ggml/src/ggml-webgpu/CMakeLists.txt +83 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +3457 -0
- data/vendor/ggml/src/ggml-webgpu/ggml-webgpu.cpp +4790 -0
- data/vendor/ggml/src/ggml-webgpu/pre_wgsl.hpp +808 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl +64 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl +72 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl +102 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl +142 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +967 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl +92 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl +127 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/conv2d_dw.wgsl +100 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl +66 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +89 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +408 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_decls.tmpl +134 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +136 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +240 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl +101 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl +84 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +503 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl +149 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl +794 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl +77 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +40 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +1052 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl +195 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl +52 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl +154 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl +159 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl +167 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl +1560 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl +297 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl +86 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +21 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl +178 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl +70 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl +151 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl +228 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl +152 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl +109 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +108 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl +224 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl +210 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl +120 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl +65 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +247 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl +55 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl +213 -0
- data/vendor/ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl +240 -0
- data/vendor/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
- data/vendor/ggml/src/ggml-zdnn/common.hpp +59 -0
- data/vendor/ggml/src/ggml-zdnn/ggml-zdnn.cpp +638 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.cpp +80 -0
- data/vendor/ggml/src/ggml-zdnn/mmf.hpp +12 -0
- data/vendor/ggml/src/ggml-zdnn/utils.cpp +79 -0
- data/vendor/ggml/src/ggml-zdnn/utils.hpp +19 -0
- data/vendor/ggml/src/ggml-zendnn/CMakeLists.txt +91 -0
- data/vendor/ggml/src/ggml-zendnn/ggml-zendnn.cpp +838 -0
- data/vendor/ggml/src/ggml.c +8088 -0
- data/vendor/ggml/src/ggml.cpp +26 -0
- data/vendor/ggml/src/gguf.cpp +1706 -0
- metadata +1459 -0
|
@@ -0,0 +1,2351 @@
|
|
|
1
|
+
#include "ggml-rpc.h"
|
|
2
|
+
#include "ggml-impl.h"
|
|
3
|
+
#include "ggml-backend-impl.h"
|
|
4
|
+
#include "ggml-cpp.h"
|
|
5
|
+
#include "transport.h"
|
|
6
|
+
|
|
7
|
+
#include <array>
|
|
8
|
+
#include <cinttypes>
|
|
9
|
+
#include <optional>
|
|
10
|
+
#include <string>
|
|
11
|
+
#include <vector>
|
|
12
|
+
#include <queue>
|
|
13
|
+
#include <condition_variable>
|
|
14
|
+
#include <future>
|
|
15
|
+
#include <memory>
|
|
16
|
+
#include <mutex>
|
|
17
|
+
#include <unordered_map>
|
|
18
|
+
#include <unordered_set>
|
|
19
|
+
#include <cstring>
|
|
20
|
+
#include <fstream>
|
|
21
|
+
#include <filesystem>
|
|
22
|
+
#include <algorithm>
|
|
23
|
+
#include <atomic>
|
|
24
|
+
#include <thread>
|
|
25
|
+
|
|
26
|
+
static const char * RPC_DEBUG = std::getenv("GGML_RPC_DEBUG");
|
|
27
|
+
|
|
28
|
+
#define LOG_DBG(...) \
|
|
29
|
+
do { if (RPC_DEBUG) GGML_LOG_DEBUG(__VA_ARGS__); } while (0)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
namespace fs = std::filesystem;
|
|
33
|
+
|
|
34
|
+
// macro for nicer error messages on server crash
|
|
35
|
+
#define RPC_STATUS_ASSERT(x) if (!(x)) GGML_ABORT("Remote RPC server crashed or returned malformed response")
|
|
36
|
+
|
|
37
|
+
// all RPC structures must be packed
|
|
38
|
+
#pragma pack(push, 1)
|
|
39
|
+
// ggml_tensor is serialized into rpc_tensor
|
|
40
|
+
struct rpc_tensor {
|
|
41
|
+
uint64_t id;
|
|
42
|
+
uint32_t type;
|
|
43
|
+
uint64_t buffer;
|
|
44
|
+
uint32_t ne[GGML_MAX_DIMS];
|
|
45
|
+
uint32_t nb[GGML_MAX_DIMS];
|
|
46
|
+
uint32_t op;
|
|
47
|
+
int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];
|
|
48
|
+
int32_t flags;
|
|
49
|
+
uint64_t src[GGML_MAX_SRC];
|
|
50
|
+
uint64_t view_src;
|
|
51
|
+
uint64_t view_offs;
|
|
52
|
+
uint64_t data;
|
|
53
|
+
char name[GGML_MAX_NAME];
|
|
54
|
+
|
|
55
|
+
int32_t use_count;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
static_assert(sizeof(rpc_tensor) % 8 == 0, "rpc_tensor size must be multiple of 8");
|
|
59
|
+
|
|
60
|
+
// RPC commands
|
|
61
|
+
enum rpc_cmd {
|
|
62
|
+
RPC_CMD_ALLOC_BUFFER = 0,
|
|
63
|
+
RPC_CMD_GET_ALIGNMENT,
|
|
64
|
+
RPC_CMD_GET_MAX_SIZE,
|
|
65
|
+
RPC_CMD_BUFFER_GET_BASE,
|
|
66
|
+
RPC_CMD_FREE_BUFFER,
|
|
67
|
+
RPC_CMD_BUFFER_CLEAR,
|
|
68
|
+
RPC_CMD_SET_TENSOR,
|
|
69
|
+
RPC_CMD_SET_TENSOR_HASH,
|
|
70
|
+
RPC_CMD_GET_TENSOR,
|
|
71
|
+
RPC_CMD_COPY_TENSOR,
|
|
72
|
+
RPC_CMD_GRAPH_COMPUTE,
|
|
73
|
+
RPC_CMD_GET_DEVICE_MEMORY,
|
|
74
|
+
RPC_CMD_INIT_TENSOR,
|
|
75
|
+
RPC_CMD_GET_ALLOC_SIZE,
|
|
76
|
+
RPC_CMD_HELLO,
|
|
77
|
+
RPC_CMD_DEVICE_COUNT,
|
|
78
|
+
RPC_CMD_GRAPH_RECOMPUTE,
|
|
79
|
+
RPC_CMD_MEMSET_TENSOR,
|
|
80
|
+
RPC_CMD_NONE,
|
|
81
|
+
RPC_CMD_COUNT,
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
static_assert(RPC_CMD_HELLO == 14, "RPC_CMD_HELLO must be always 14");
|
|
85
|
+
|
|
86
|
+
// Try RPC_CMD_SET_TENSOR_HASH first when data size is larger than this threshold
|
|
87
|
+
const size_t HASH_THRESHOLD = 10 * 1024 * 1024;
|
|
88
|
+
|
|
89
|
+
struct rpc_msg_hello_req {
|
|
90
|
+
uint8_t conn_caps[RPC_CONN_CAPS_SIZE];
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
struct rpc_msg_hello_rsp {
|
|
94
|
+
uint8_t major;
|
|
95
|
+
uint8_t minor;
|
|
96
|
+
uint8_t patch;
|
|
97
|
+
uint8_t padding;
|
|
98
|
+
uint8_t conn_caps[RPC_CONN_CAPS_SIZE];
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
struct rpc_msg_device_count_rsp {
|
|
102
|
+
uint32_t device_count;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
struct rpc_msg_get_alloc_size_req {
|
|
106
|
+
uint32_t device;
|
|
107
|
+
rpc_tensor tensor;
|
|
108
|
+
rpc_tensor srcs[GGML_MAX_SRC];
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
struct rpc_msg_get_alloc_size_rsp {
|
|
112
|
+
uint64_t alloc_size;
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
struct rpc_msg_init_tensor_req {
|
|
116
|
+
rpc_tensor tensor;
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
struct rpc_msg_alloc_buffer_req {
|
|
120
|
+
uint32_t device;
|
|
121
|
+
uint64_t size;
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
struct rpc_msg_alloc_buffer_rsp {
|
|
125
|
+
uint64_t remote_ptr;
|
|
126
|
+
uint64_t remote_size;
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
struct rpc_msg_get_alignment_req {
|
|
130
|
+
uint32_t device;
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
struct rpc_msg_get_alignment_rsp {
|
|
134
|
+
uint64_t alignment;
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
struct rpc_msg_get_max_size_req {
|
|
138
|
+
uint32_t device;
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
struct rpc_msg_get_max_size_rsp {
|
|
142
|
+
uint64_t max_size;
|
|
143
|
+
};
|
|
144
|
+
|
|
145
|
+
struct rpc_msg_buffer_get_base_req {
|
|
146
|
+
uint64_t remote_ptr;
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
struct rpc_msg_buffer_get_base_rsp {
|
|
150
|
+
uint64_t base_ptr;
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
struct rpc_msg_free_buffer_req {
|
|
154
|
+
uint64_t remote_ptr;
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
struct rpc_msg_buffer_clear_req {
|
|
158
|
+
uint64_t remote_ptr;
|
|
159
|
+
uint8_t value;
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
struct rpc_msg_memset_tensor_req {
|
|
163
|
+
rpc_tensor tensor;
|
|
164
|
+
uint64_t offset;
|
|
165
|
+
uint64_t size;
|
|
166
|
+
uint8_t value;
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
struct rpc_msg_set_tensor_hash_req {
|
|
170
|
+
rpc_tensor tensor;
|
|
171
|
+
uint64_t offset;
|
|
172
|
+
uint64_t hash;
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
struct rpc_msg_set_tensor_hash_rsp {
|
|
176
|
+
uint8_t result;
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
struct rpc_msg_get_tensor_req {
|
|
180
|
+
rpc_tensor tensor;
|
|
181
|
+
uint64_t offset;
|
|
182
|
+
uint64_t size;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
struct rpc_msg_copy_tensor_req {
|
|
186
|
+
rpc_tensor src;
|
|
187
|
+
rpc_tensor dst;
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
struct rpc_msg_copy_tensor_rsp {
|
|
191
|
+
uint8_t result;
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
struct rpc_msg_get_device_memory_req {
|
|
195
|
+
uint32_t device;
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
struct rpc_msg_get_device_memory_rsp {
|
|
199
|
+
uint64_t free_mem;
|
|
200
|
+
uint64_t total_mem;
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
struct rpc_msg_graph_recompute_req {
|
|
204
|
+
uint32_t device;
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
#pragma pack(pop)
|
|
208
|
+
|
|
209
|
+
// RPC data structures
|
|
210
|
+
|
|
211
|
+
static ggml_guid_t ggml_backend_rpc_guid() {
|
|
212
|
+
static ggml_guid guid = {0x99, 0x68, 0x5b, 0x6c, 0xd2, 0x83, 0x3d, 0x24, 0x25, 0x36, 0x72, 0xe1, 0x5b, 0x0e, 0x14, 0x03};
|
|
213
|
+
return &guid;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
struct ggml_backend_rpc_device_context {
|
|
217
|
+
std::string endpoint;
|
|
218
|
+
uint32_t device;
|
|
219
|
+
std::string name;
|
|
220
|
+
std::string description;
|
|
221
|
+
uint64_t last_graph_uid;
|
|
222
|
+
};
|
|
223
|
+
|
|
224
|
+
struct ggml_backend_rpc_buffer_type_context {
|
|
225
|
+
std::string endpoint;
|
|
226
|
+
uint32_t device;
|
|
227
|
+
std::string name;
|
|
228
|
+
size_t alignment;
|
|
229
|
+
size_t max_size;
|
|
230
|
+
};
|
|
231
|
+
|
|
232
|
+
class rpc_dispatcher;
|
|
233
|
+
struct ggml_backend_rpc_context {
|
|
234
|
+
std::shared_ptr<rpc_dispatcher> dispatcher;
|
|
235
|
+
uint32_t device;
|
|
236
|
+
std::string name;
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
struct ggml_backend_rpc_buffer_context {
|
|
240
|
+
std::shared_ptr<rpc_dispatcher> dispatcher;
|
|
241
|
+
void * base_ptr;
|
|
242
|
+
uint64_t remote_ptr;
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
// RPC helper functions
|
|
246
|
+
|
|
247
|
+
// Computes FNV-1a hash of the data
|
|
248
|
+
static uint64_t fnv_hash(const uint8_t * data, size_t len, uint64_t hash = 0xcbf29ce484222325ULL) {
|
|
249
|
+
const uint64_t fnv_prime = 0x100000001b3ULL;
|
|
250
|
+
|
|
251
|
+
for (size_t i = 0; i < len; ++i) {
|
|
252
|
+
hash ^= data[i];
|
|
253
|
+
hash *= fnv_prime;
|
|
254
|
+
}
|
|
255
|
+
return hash;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
static bool send_msg(socket_ptr sock, const void * msg, size_t msg_size) {
|
|
259
|
+
if (!sock->send_data(&msg_size, sizeof(msg_size))) {
|
|
260
|
+
return false;
|
|
261
|
+
}
|
|
262
|
+
if (!sock->send_data(msg, msg_size)) {
|
|
263
|
+
return false;
|
|
264
|
+
}
|
|
265
|
+
return sock->flush();
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
static bool recv_msg(socket_ptr sock, void * msg, size_t msg_size) {
|
|
269
|
+
uint64_t size;
|
|
270
|
+
if (!sock->recv_data(&size, sizeof(size))) {
|
|
271
|
+
return false;
|
|
272
|
+
}
|
|
273
|
+
if (size != msg_size) {
|
|
274
|
+
return false;
|
|
275
|
+
}
|
|
276
|
+
return sock->recv_data(msg, msg_size);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
static bool recv_msg(socket_ptr sock, std::vector<uint8_t> & input) {
|
|
280
|
+
uint64_t size;
|
|
281
|
+
if (!sock->recv_data(&size, sizeof(size))) {
|
|
282
|
+
return false;
|
|
283
|
+
}
|
|
284
|
+
try {
|
|
285
|
+
input.resize(size);
|
|
286
|
+
} catch (const std::bad_alloc & e) {
|
|
287
|
+
GGML_LOG_ERROR("Failed to allocate input buffer of size %" PRIu64 "\n", size);
|
|
288
|
+
return false;
|
|
289
|
+
}
|
|
290
|
+
return sock->recv_data(input.data(), size);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
static bool parse_endpoint(const std::string & endpoint, std::string & host, int & port) {
|
|
294
|
+
size_t pos = endpoint.find(':');
|
|
295
|
+
if (pos == std::string::npos) {
|
|
296
|
+
return false;
|
|
297
|
+
}
|
|
298
|
+
host = endpoint.substr(0, pos);
|
|
299
|
+
try {
|
|
300
|
+
port = std::stoi(endpoint.substr(pos + 1));
|
|
301
|
+
} catch (...) {
|
|
302
|
+
return false;
|
|
303
|
+
}
|
|
304
|
+
return true;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
// RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) |
|
|
308
|
+
// No response
|
|
309
|
+
static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size) {
|
|
310
|
+
uint8_t cmd_byte = cmd;
|
|
311
|
+
if (!sock->send_data(&cmd_byte, sizeof(cmd_byte))) {
|
|
312
|
+
return false;
|
|
313
|
+
}
|
|
314
|
+
if (!sock->send_data(&input_size, sizeof(input_size))) {
|
|
315
|
+
return false;
|
|
316
|
+
}
|
|
317
|
+
if (!sock->send_data(input, input_size)) {
|
|
318
|
+
return false;
|
|
319
|
+
}
|
|
320
|
+
return sock->flush();
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) |
|
|
324
|
+
// RPC response: | response_size (8 bytes) | response_data (response_size bytes) |
|
|
325
|
+
static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size, void * output, size_t output_size) {
|
|
326
|
+
if (!send_rpc_cmd(sock, cmd, input, input_size)) {
|
|
327
|
+
return false;
|
|
328
|
+
}
|
|
329
|
+
uint64_t out_size;
|
|
330
|
+
if (!sock->recv_data(&out_size, sizeof(out_size))) {
|
|
331
|
+
return false;
|
|
332
|
+
}
|
|
333
|
+
if (out_size != output_size) {
|
|
334
|
+
return false;
|
|
335
|
+
}
|
|
336
|
+
if (!sock->recv_data(output, output_size)) {
|
|
337
|
+
return false;
|
|
338
|
+
}
|
|
339
|
+
return true;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// RPC client-side implementation
|
|
343
|
+
|
|
344
|
+
// Performs HELLO handshake with transport auto-negotiation.
|
|
345
|
+
// Advertises local capabilities via conn_caps; if the server responds with
|
|
346
|
+
// matching capabilities, the socket is upgraded transparently.
|
|
347
|
+
static bool negotiate_hello(const std::shared_ptr<socket_t> & sock) {
|
|
348
|
+
rpc_msg_hello_req request = {};
|
|
349
|
+
rpc_msg_hello_rsp response = {};
|
|
350
|
+
|
|
351
|
+
sock->get_caps(request.conn_caps);
|
|
352
|
+
|
|
353
|
+
bool status = send_rpc_cmd(sock, RPC_CMD_HELLO, &request, sizeof(request), &response, sizeof(response));
|
|
354
|
+
RPC_STATUS_ASSERT(status);
|
|
355
|
+
|
|
356
|
+
if (response.major != RPC_PROTO_MAJOR_VERSION || response.minor > RPC_PROTO_MINOR_VERSION) {
|
|
357
|
+
GGML_LOG_ERROR("RPC server version mismatch: %d.%d.%d\n",
|
|
358
|
+
response.major, response.minor, response.patch);
|
|
359
|
+
return false;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
sock->update_caps(response.conn_caps);
|
|
363
|
+
return true;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
template <typename T>
|
|
367
|
+
class message_queue {
|
|
368
|
+
public:
|
|
369
|
+
message_queue() {}
|
|
370
|
+
|
|
371
|
+
bool push(const T &value) {
|
|
372
|
+
std::unique_lock<std::mutex> lock(mutex);
|
|
373
|
+
if (interrupted) {
|
|
374
|
+
return false;
|
|
375
|
+
}
|
|
376
|
+
queue.push(value);
|
|
377
|
+
cvar.notify_all();
|
|
378
|
+
return true;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
bool pop(T* out) {
|
|
382
|
+
std::unique_lock<std::mutex> lock(mutex);
|
|
383
|
+
cvar.wait(lock, [this] { return !queue.empty() || interrupted; });
|
|
384
|
+
if (interrupted) {
|
|
385
|
+
return false;
|
|
386
|
+
}
|
|
387
|
+
*out = queue.front();
|
|
388
|
+
queue.pop();
|
|
389
|
+
return true;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
void interrupt() {
|
|
393
|
+
std::unique_lock<std::mutex> lock(mutex);
|
|
394
|
+
interrupted = true;
|
|
395
|
+
lock.unlock();
|
|
396
|
+
cvar.notify_all();
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
private:
|
|
400
|
+
bool interrupted = false;
|
|
401
|
+
std::queue<T> queue;
|
|
402
|
+
std::mutex mutex;
|
|
403
|
+
std::condition_variable cvar;
|
|
404
|
+
};
|
|
405
|
+
|
|
406
|
+
class rpc_dispatcher {
|
|
407
|
+
public:
|
|
408
|
+
rpc_dispatcher() {
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
void send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size);
|
|
412
|
+
void send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size);
|
|
413
|
+
void send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size);
|
|
414
|
+
void send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size);
|
|
415
|
+
|
|
416
|
+
ggml_backend_event_t event_new(ggml_backend_dev_t dev);
|
|
417
|
+
void event_free(ggml_backend_event_t event);
|
|
418
|
+
void event_synchronize(ggml_backend_event_t event);
|
|
419
|
+
void event_record(ggml_backend_event_t event);
|
|
420
|
+
void synchronize();
|
|
421
|
+
|
|
422
|
+
void start(const std::string & endpoint);
|
|
423
|
+
void work();
|
|
424
|
+
|
|
425
|
+
~rpc_dispatcher();
|
|
426
|
+
|
|
427
|
+
private:
|
|
428
|
+
struct rpc_msg {
|
|
429
|
+
rpc_cmd cmd;
|
|
430
|
+
std::shared_ptr<const void> input;
|
|
431
|
+
size_t input_size;
|
|
432
|
+
void * output;
|
|
433
|
+
size_t output_size;
|
|
434
|
+
std::promise<void> completion;
|
|
435
|
+
};
|
|
436
|
+
using rpc_msg_ptr = std::shared_ptr<rpc_msg>;
|
|
437
|
+
using rpc_msg_queue = message_queue<rpc_msg_ptr>;
|
|
438
|
+
struct rpc_event {
|
|
439
|
+
rpc_msg_ptr msg;
|
|
440
|
+
std::shared_future<void> sf;
|
|
441
|
+
};
|
|
442
|
+
rpc_msg_queue queue;
|
|
443
|
+
socket_ptr sock;
|
|
444
|
+
std::atomic_bool running;
|
|
445
|
+
std::thread thread;
|
|
446
|
+
};
|
|
447
|
+
|
|
448
|
+
static void rpc_dispatcher_trampoline(rpc_dispatcher * dispatcher)
|
|
449
|
+
{
|
|
450
|
+
dispatcher->work();
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size) {
|
|
454
|
+
auto msg = std::make_shared<rpc_msg>();
|
|
455
|
+
msg->cmd = cmd;
|
|
456
|
+
msg->input = input;
|
|
457
|
+
msg->input_size = input_size;
|
|
458
|
+
msg->output = nullptr;
|
|
459
|
+
msg->output_size = 0;
|
|
460
|
+
GGML_ASSERT(queue.push(msg));
|
|
461
|
+
auto future = msg->completion.get_future();
|
|
462
|
+
future.wait();
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
void rpc_dispatcher::send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size) {
|
|
466
|
+
auto msg = std::make_shared<rpc_msg>();
|
|
467
|
+
msg->cmd = cmd;
|
|
468
|
+
msg->input = input;
|
|
469
|
+
msg->input_size = input_size;
|
|
470
|
+
msg->output = nullptr;
|
|
471
|
+
msg->output_size = 0;
|
|
472
|
+
GGML_ASSERT(queue.push(msg));
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size) {
|
|
476
|
+
auto msg = std::make_shared<rpc_msg>();
|
|
477
|
+
msg->cmd = cmd;
|
|
478
|
+
msg->input = input;
|
|
479
|
+
msg->input_size = input_size;
|
|
480
|
+
msg->output = output;
|
|
481
|
+
msg->output_size = output_size;
|
|
482
|
+
GGML_ASSERT(queue.push(msg));
|
|
483
|
+
auto future = msg->completion.get_future();
|
|
484
|
+
future.wait();
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
void rpc_dispatcher::send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size) {
|
|
488
|
+
auto msg = std::make_shared<rpc_msg>();
|
|
489
|
+
msg->cmd = cmd;
|
|
490
|
+
msg->input = input;
|
|
491
|
+
msg->input_size = input_size;
|
|
492
|
+
msg->output = output;
|
|
493
|
+
msg->output_size = output_size;
|
|
494
|
+
GGML_ASSERT(queue.push(msg));
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
ggml_backend_event_t rpc_dispatcher::event_new(ggml_backend_dev_t dev) {
|
|
498
|
+
rpc_event * ev = new rpc_event;
|
|
499
|
+
ev->msg = std::make_shared<rpc_msg>();
|
|
500
|
+
ev->msg->cmd = RPC_CMD_NONE;
|
|
501
|
+
ev->sf = ev->msg->completion.get_future().share();
|
|
502
|
+
GGML_ASSERT(queue.push(ev->msg));
|
|
503
|
+
return new ggml_backend_event {
|
|
504
|
+
/* .device = */ dev,
|
|
505
|
+
/* .context = */ ev,
|
|
506
|
+
};
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
void rpc_dispatcher::event_free(ggml_backend_event_t event) {
|
|
510
|
+
rpc_event * ev = (rpc_event *)event->context;
|
|
511
|
+
delete ev;
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
void rpc_dispatcher::event_synchronize(ggml_backend_event_t event) {
|
|
515
|
+
rpc_event * ev = (rpc_event *)event->context;
|
|
516
|
+
ev->sf.wait();
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
void rpc_dispatcher::event_record(ggml_backend_event_t event) {
|
|
520
|
+
rpc_event * ev = (rpc_event *)event->context;
|
|
521
|
+
ev->msg = std::make_shared<rpc_msg>();
|
|
522
|
+
ev->msg->cmd = RPC_CMD_NONE;
|
|
523
|
+
ev->sf = ev->msg->completion.get_future().share();
|
|
524
|
+
GGML_ASSERT(queue.push(ev->msg));
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
void rpc_dispatcher::synchronize() {
|
|
528
|
+
// to ensure all messages are processed, submit dummy message and wait for it to complete
|
|
529
|
+
auto msg = std::make_shared<rpc_msg>();
|
|
530
|
+
msg->cmd = RPC_CMD_NONE;
|
|
531
|
+
GGML_ASSERT(queue.push(msg));
|
|
532
|
+
msg->completion.get_future().wait();
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
void rpc_dispatcher::start(const std::string & endpoint) {
|
|
536
|
+
std::string host;
|
|
537
|
+
int port;
|
|
538
|
+
if (!parse_endpoint(endpoint, host, port)) {
|
|
539
|
+
GGML_ABORT("Failed to parse endpoint: %s\n", endpoint.c_str());
|
|
540
|
+
}
|
|
541
|
+
if (!rpc_transport_init()) {
|
|
542
|
+
GGML_ABORT("RPC transport initialization failed\n");
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
sock = socket_t::connect(host.c_str(), port);
|
|
546
|
+
if (sock == nullptr) {
|
|
547
|
+
GGML_ABORT("Failed to connect to %s\n", endpoint.c_str());
|
|
548
|
+
}
|
|
549
|
+
if (!negotiate_hello(sock)) {
|
|
550
|
+
GGML_ABORT("RPC handshake failed for %s\n", endpoint.c_str());
|
|
551
|
+
}
|
|
552
|
+
LOG_DBG("[%s] connected to %s\n", __func__, endpoint.c_str());
|
|
553
|
+
running = true;
|
|
554
|
+
thread = std::thread(rpc_dispatcher_trampoline, this);
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
void rpc_dispatcher::work() {
|
|
558
|
+
while (running) {
|
|
559
|
+
rpc_msg_ptr msg_ptr;
|
|
560
|
+
if (!queue.pop(&msg_ptr)) {
|
|
561
|
+
break;
|
|
562
|
+
}
|
|
563
|
+
if (msg_ptr->cmd != RPC_CMD_NONE) {
|
|
564
|
+
if (msg_ptr->output) {
|
|
565
|
+
bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size, msg_ptr->output, msg_ptr->output_size);
|
|
566
|
+
RPC_STATUS_ASSERT(status);
|
|
567
|
+
} else {
|
|
568
|
+
bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size);
|
|
569
|
+
RPC_STATUS_ASSERT(status);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
msg_ptr->completion.set_value();
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
rpc_dispatcher::~rpc_dispatcher() {
|
|
577
|
+
running = false;
|
|
578
|
+
queue.interrupt();
|
|
579
|
+
sock = nullptr;
|
|
580
|
+
if (thread.joinable()) {
|
|
581
|
+
thread.join();
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
static std::shared_ptr<rpc_dispatcher> get_dispatcher(const std::string & endpoint) {
|
|
586
|
+
static std::mutex mutex;
|
|
587
|
+
std::lock_guard<std::mutex> lock(mutex);
|
|
588
|
+
static std::unordered_map<std::string, std::weak_ptr<rpc_dispatcher>> dispatchers;
|
|
589
|
+
|
|
590
|
+
auto it = dispatchers.find(endpoint);
|
|
591
|
+
if (it != dispatchers.end()) {
|
|
592
|
+
if (auto dispatcher = it->second.lock()) {
|
|
593
|
+
return dispatcher;
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
auto dispatcher = std::make_shared<rpc_dispatcher>();
|
|
598
|
+
dispatcher->start(endpoint);
|
|
599
|
+
dispatchers[endpoint] = dispatcher;
|
|
600
|
+
return dispatcher;
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
static void ggml_backend_rpc_buffer_free_buffer(ggml_backend_buffer_t buffer) {
|
|
604
|
+
ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
|
|
605
|
+
auto request = std::make_shared<rpc_msg_free_buffer_req>();
|
|
606
|
+
request->remote_ptr = ctx->remote_ptr;
|
|
607
|
+
ctx->dispatcher->send(RPC_CMD_FREE_BUFFER, request, sizeof(*request));
|
|
608
|
+
delete ctx;
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
static void * ggml_backend_rpc_buffer_get_base(ggml_backend_buffer_t buffer) {
|
|
612
|
+
ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
|
|
613
|
+
if (ctx->base_ptr != nullptr) {
|
|
614
|
+
return ctx->base_ptr;
|
|
615
|
+
}
|
|
616
|
+
auto request = std::make_shared<rpc_msg_buffer_get_base_req>();
|
|
617
|
+
request->remote_ptr = ctx->remote_ptr;
|
|
618
|
+
rpc_msg_buffer_get_base_rsp response;
|
|
619
|
+
ctx->dispatcher->send(RPC_CMD_BUFFER_GET_BASE, request, sizeof(*request), &response, sizeof(response));
|
|
620
|
+
ctx->base_ptr = reinterpret_cast<void *>(response.base_ptr);
|
|
621
|
+
return ctx->base_ptr;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
static bool ggml_backend_buffer_is_rpc(ggml_backend_buffer_t buffer) {
|
|
625
|
+
return buffer->iface.free_buffer == ggml_backend_rpc_buffer_free_buffer;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
static rpc_tensor serialize_tensor(const ggml_tensor * tensor, const std::shared_ptr<rpc_dispatcher> & dispatcher = nullptr) {
|
|
629
|
+
rpc_tensor result;
|
|
630
|
+
if (!tensor) {
|
|
631
|
+
memset(&result, 0, sizeof(result));
|
|
632
|
+
return result;
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
result.id = reinterpret_cast<uint64_t>(tensor);
|
|
636
|
+
result.type = tensor->type;
|
|
637
|
+
if (tensor->buffer && ggml_backend_buffer_is_rpc(tensor->buffer)) {
|
|
638
|
+
ggml_backend_buffer_t buffer = tensor->buffer;
|
|
639
|
+
ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
|
|
640
|
+
// ref: https://github.com/ggml-org/llama.cpp/pull/26500
|
|
641
|
+
if (ctx != nullptr && (dispatcher == nullptr || ctx->dispatcher == dispatcher)) {
|
|
642
|
+
result.buffer = ctx->remote_ptr;
|
|
643
|
+
result.data = reinterpret_cast<uint64_t>(tensor->data);
|
|
644
|
+
} else {
|
|
645
|
+
result.buffer = 0;
|
|
646
|
+
result.data = 0;
|
|
647
|
+
}
|
|
648
|
+
} else {
|
|
649
|
+
result.buffer = 0;
|
|
650
|
+
result.data = 0;
|
|
651
|
+
}
|
|
652
|
+
for (uint32_t i = 0; i < GGML_MAX_DIMS; i++) {
|
|
653
|
+
result.ne[i] = tensor->ne[i];
|
|
654
|
+
result.nb[i] = tensor->nb[i];
|
|
655
|
+
}
|
|
656
|
+
result.op = tensor->op;
|
|
657
|
+
for (uint32_t i = 0; i < GGML_MAX_OP_PARAMS / sizeof(int32_t); i++) {
|
|
658
|
+
result.op_params[i] = tensor->op_params[i];
|
|
659
|
+
}
|
|
660
|
+
result.flags = tensor->flags;
|
|
661
|
+
for (uint32_t i = 0; i < GGML_MAX_SRC; i++) {
|
|
662
|
+
result.src[i] = reinterpret_cast<uint64_t>(tensor->src[i]);
|
|
663
|
+
}
|
|
664
|
+
result.view_src = reinterpret_cast<uint64_t>(tensor->view_src);
|
|
665
|
+
result.view_offs = tensor->view_offs;
|
|
666
|
+
|
|
667
|
+
// Avoid sending uninitialized data over the wire
|
|
668
|
+
memset(result.name, 0, sizeof(result.name));
|
|
669
|
+
result.use_count = 0;
|
|
670
|
+
|
|
671
|
+
snprintf(result.name, GGML_MAX_NAME, "%s", tensor->name);
|
|
672
|
+
return result;
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
static enum ggml_status ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
|
|
676
|
+
ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
|
|
677
|
+
|
|
678
|
+
// CUDA backend on the server pads everything to 512 due to CUDA limitations.
|
|
679
|
+
// Due to bandwidth constraints, we only call the server init tensor functions if necessary.
|
|
680
|
+
// In particular, only quantized tensors need padding
|
|
681
|
+
if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0) && (tensor->view_src == nullptr)) {
|
|
682
|
+
auto request = std::make_shared<rpc_msg_init_tensor_req>();
|
|
683
|
+
request->tensor = serialize_tensor(tensor);
|
|
684
|
+
ctx->dispatcher->send(RPC_CMD_INIT_TENSOR, request, sizeof(*request));
|
|
685
|
+
}
|
|
686
|
+
return GGML_STATUS_SUCCESS;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
static void ggml_backend_rpc_buffer_memset_tensor(
|
|
690
|
+
ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
|
|
691
|
+
ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
|
|
692
|
+
auto request = std::make_shared<rpc_msg_memset_tensor_req>();
|
|
693
|
+
request->tensor = serialize_tensor(tensor);
|
|
694
|
+
request->offset = offset;
|
|
695
|
+
request->size = size;
|
|
696
|
+
request->value = value;
|
|
697
|
+
ctx->dispatcher->send(RPC_CMD_MEMSET_TENSOR, request, sizeof(*request));
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
|
|
701
|
+
ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
|
|
702
|
+
rpc_tensor rpc_tensor = serialize_tensor(tensor);
|
|
703
|
+
if (size > HASH_THRESHOLD) {
|
|
704
|
+
auto request = std::make_shared<rpc_msg_set_tensor_hash_req>();
|
|
705
|
+
request->tensor = rpc_tensor;
|
|
706
|
+
request->offset = offset;
|
|
707
|
+
request->hash = fnv_hash((const uint8_t*)data, size);
|
|
708
|
+
rpc_msg_set_tensor_hash_rsp response;
|
|
709
|
+
ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), &response, sizeof(response));
|
|
710
|
+
if (response.result) {
|
|
711
|
+
// the server has the same data, no need to send it
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
// input serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes)
|
|
716
|
+
size_t input_size = sizeof(rpc_tensor) + sizeof(uint64_t) + size;
|
|
717
|
+
uint8_t * input = new uint8_t[input_size]();
|
|
718
|
+
memcpy(input, &rpc_tensor, sizeof(rpc_tensor));
|
|
719
|
+
memcpy(input + sizeof(rpc_tensor), &offset, sizeof(offset));
|
|
720
|
+
memcpy(input + sizeof(rpc_tensor) + sizeof(offset), data, size);
|
|
721
|
+
std::shared_ptr<uint8_t> input_ptr(input, std::default_delete<uint8_t[]>());
|
|
722
|
+
ctx->dispatcher->send(RPC_CMD_SET_TENSOR, input_ptr, input_size);
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
static void ggml_backend_rpc_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
|
|
726
|
+
ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
|
|
727
|
+
auto request = std::make_shared<rpc_msg_get_tensor_req>();
|
|
728
|
+
request->tensor = serialize_tensor(tensor);
|
|
729
|
+
request->offset = offset;
|
|
730
|
+
request->size = size;
|
|
731
|
+
ctx->dispatcher->send(RPC_CMD_GET_TENSOR, request, sizeof(*request), data, size);
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
static bool ggml_backend_rpc_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
|
|
735
|
+
if (ggml_backend_buffer_is_rpc(src->buffer)) {
|
|
736
|
+
// check if src and dst are on the same server
|
|
737
|
+
ggml_backend_buffer_t src_buffer = src->buffer;
|
|
738
|
+
ggml_backend_rpc_buffer_context * src_ctx = (ggml_backend_rpc_buffer_context *)src_buffer->context;
|
|
739
|
+
ggml_backend_buffer_t dst_buffer = dst->buffer;
|
|
740
|
+
ggml_backend_rpc_buffer_context * dst_ctx = (ggml_backend_rpc_buffer_context *)dst_buffer->context;
|
|
741
|
+
if (src_ctx->dispatcher != dst_ctx->dispatcher) {
|
|
742
|
+
return false;
|
|
743
|
+
}
|
|
744
|
+
ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
|
|
745
|
+
auto request = std::make_shared<rpc_msg_copy_tensor_req>();
|
|
746
|
+
request->src = serialize_tensor(src);
|
|
747
|
+
request->dst = serialize_tensor(dst);
|
|
748
|
+
rpc_msg_copy_tensor_rsp response;
|
|
749
|
+
ctx->dispatcher->send(RPC_CMD_COPY_TENSOR, request, sizeof(*request), &response, sizeof(response));
|
|
750
|
+
return response.result;
|
|
751
|
+
}
|
|
752
|
+
return false;
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
static void ggml_backend_rpc_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
|
|
756
|
+
ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
|
|
757
|
+
auto request = std::make_shared<rpc_msg_buffer_clear_req>();
|
|
758
|
+
request->remote_ptr = ctx->remote_ptr;
|
|
759
|
+
request->value = value;
|
|
760
|
+
ctx->dispatcher->send(RPC_CMD_BUFFER_CLEAR, request, sizeof(*request));
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
static ggml_backend_buffer_i ggml_backend_rpc_buffer_interface = {
|
|
764
|
+
/* .free_buffer = */ ggml_backend_rpc_buffer_free_buffer,
|
|
765
|
+
/* .get_base = */ ggml_backend_rpc_buffer_get_base,
|
|
766
|
+
/* .init_tensor = */ ggml_backend_rpc_buffer_init_tensor,
|
|
767
|
+
/* .memset_tensor = */ ggml_backend_rpc_buffer_memset_tensor,
|
|
768
|
+
/* .set_tensor = */ ggml_backend_rpc_buffer_set_tensor,
|
|
769
|
+
/* .get_tensor = */ ggml_backend_rpc_buffer_get_tensor,
|
|
770
|
+
/* .set_tensor_2d = */ NULL,
|
|
771
|
+
/* .get_tensor_2d = */ NULL,
|
|
772
|
+
/* .cpy_tensor = */ ggml_backend_rpc_buffer_cpy_tensor,
|
|
773
|
+
/* .clear = */ ggml_backend_rpc_buffer_clear,
|
|
774
|
+
/* .reset = */ NULL,
|
|
775
|
+
};
|
|
776
|
+
|
|
777
|
+
static const char * ggml_backend_rpc_buffer_type_name(ggml_backend_buffer_type_t buft) {
|
|
778
|
+
ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;
|
|
779
|
+
return buft_ctx->name.c_str();
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
static ggml_backend_buffer_t ggml_backend_rpc_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
|
|
783
|
+
ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;
|
|
784
|
+
auto request = std::make_shared<rpc_msg_alloc_buffer_req>();
|
|
785
|
+
request->device = buft_ctx->device;
|
|
786
|
+
request->size = size;
|
|
787
|
+
rpc_msg_alloc_buffer_rsp response;
|
|
788
|
+
|
|
789
|
+
auto dispatcher = get_dispatcher(buft_ctx->endpoint);
|
|
790
|
+
dispatcher->send(RPC_CMD_ALLOC_BUFFER, request, sizeof(*request), &response, sizeof(response));
|
|
791
|
+
if (response.remote_ptr != 0) {
|
|
792
|
+
ggml_backend_buffer_t buffer = ggml_backend_buffer_init(buft,
|
|
793
|
+
ggml_backend_rpc_buffer_interface,
|
|
794
|
+
new ggml_backend_rpc_buffer_context{dispatcher, nullptr, response.remote_ptr},
|
|
795
|
+
response.remote_size);
|
|
796
|
+
return buffer;
|
|
797
|
+
} else {
|
|
798
|
+
return nullptr;
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
static size_t get_alignment(const std::shared_ptr<rpc_dispatcher> & dispatcher, uint32_t device) {
|
|
803
|
+
auto request = std::make_shared<rpc_msg_get_alignment_req>();
|
|
804
|
+
request->device = device;
|
|
805
|
+
rpc_msg_get_alignment_rsp response;
|
|
806
|
+
dispatcher->send(RPC_CMD_GET_ALIGNMENT, request, sizeof(*request), &response, sizeof(response));
|
|
807
|
+
return response.alignment;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
static size_t ggml_backend_rpc_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
|
|
811
|
+
ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;
|
|
812
|
+
return buft_ctx->alignment;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
static size_t get_max_size(const std::shared_ptr<rpc_dispatcher> & dispatcher, uint32_t device) {
|
|
816
|
+
auto request = std::make_shared<rpc_msg_get_max_size_req>();
|
|
817
|
+
request->device = device;
|
|
818
|
+
rpc_msg_get_max_size_rsp response;
|
|
819
|
+
dispatcher->send(RPC_CMD_GET_MAX_SIZE, request, sizeof(*request), &response, sizeof(response));
|
|
820
|
+
return response.max_size;
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
static size_t ggml_backend_rpc_get_max_size(ggml_backend_buffer_type_t buft) {
|
|
824
|
+
ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;
|
|
825
|
+
return buft_ctx->max_size;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
|
|
829
|
+
// should we query the remote server for the actual size
|
|
830
|
+
bool rpc_get = false;
|
|
831
|
+
|
|
832
|
+
// See comments in init_tensor.
|
|
833
|
+
rpc_get |= ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0) && (tensor->view_src == nullptr);
|
|
834
|
+
|
|
835
|
+
// [TAG_ALLOC_SIZE_EXPAND]
|
|
836
|
+
// ops that may require additional memory for fleeting data on certain backends
|
|
837
|
+
// ref: https://github.com/ggml-org/llama.cpp/pull/15966
|
|
838
|
+
rpc_get |= ggml_op_alloc_size_may_expand(tensor->op);
|
|
839
|
+
|
|
840
|
+
if (rpc_get) {
|
|
841
|
+
ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;
|
|
842
|
+
|
|
843
|
+
// Cache key for calls to read the alloc_size.
|
|
844
|
+
// We deliberately exclude src tensor dimensions from the key because:
|
|
845
|
+
// 1. For CPU backends, alloc_size = ggml_nbytes(output) regardless of src shapes
|
|
846
|
+
// 2. For GPU backends, the reservation graph uses max dimensions, so the
|
|
847
|
+
// cached value from reservation is always >= any subsequent request
|
|
848
|
+
// 3. Including src dims causes cache misses per-ubatch (e.g. growing KV cache)
|
|
849
|
+
// which blocks the main thread behind in-flight GRAPH_COMPUTE commands
|
|
850
|
+
struct alloc_size_cache_key {
|
|
851
|
+
uint32_t device;
|
|
852
|
+
uint32_t type;
|
|
853
|
+
uint32_t op;
|
|
854
|
+
int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];
|
|
855
|
+
uint32_t ne[GGML_MAX_DIMS];
|
|
856
|
+
};
|
|
857
|
+
|
|
858
|
+
alloc_size_cache_key key = {};
|
|
859
|
+
key.device = buft_ctx->device;
|
|
860
|
+
key.type = tensor->type;
|
|
861
|
+
key.op = tensor->op;
|
|
862
|
+
memcpy(key.op_params, tensor->op_params, sizeof(key.op_params));
|
|
863
|
+
for (int i = 0; i < GGML_MAX_DIMS; i++) {
|
|
864
|
+
key.ne[i] = (uint32_t)tensor->ne[i];
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
uint64_t cache_hash = fnv_hash((const uint8_t *)&key, sizeof(key));
|
|
868
|
+
cache_hash = fnv_hash((const uint8_t *)buft_ctx->endpoint.data(), buft_ctx->endpoint.size(), cache_hash);
|
|
869
|
+
|
|
870
|
+
// alloc sizes are immutable for a given tensor configuration
|
|
871
|
+
static std::mutex cache_mutex;
|
|
872
|
+
static std::unordered_map<uint64_t, size_t> cache;
|
|
873
|
+
|
|
874
|
+
{
|
|
875
|
+
std::lock_guard<std::mutex> lock(cache_mutex);
|
|
876
|
+
auto it = cache.find(cache_hash);
|
|
877
|
+
if (it != cache.end()) {
|
|
878
|
+
return it->second;
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
auto request = std::make_shared<rpc_msg_get_alloc_size_req>();
|
|
883
|
+
request->device = buft_ctx->device;
|
|
884
|
+
request->tensor = serialize_tensor(tensor);
|
|
885
|
+
|
|
886
|
+
// .get_alloc_size could be a function of the tensor's srcs, so we must serialize them as well
|
|
887
|
+
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
|
888
|
+
request->srcs[i] = serialize_tensor(tensor->src[i]);
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
rpc_msg_get_alloc_size_rsp response;
|
|
892
|
+
auto dispatcher = get_dispatcher(buft_ctx->endpoint);
|
|
893
|
+
dispatcher->send(RPC_CMD_GET_ALLOC_SIZE, request, sizeof(*request), &response, sizeof(response));
|
|
894
|
+
|
|
895
|
+
{
|
|
896
|
+
std::lock_guard<std::mutex> lock(cache_mutex);
|
|
897
|
+
cache[cache_hash] = response.alloc_size;
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
return response.alloc_size;
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
return ggml_nbytes(tensor);
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
static ggml_backend_buffer_type_i ggml_backend_rpc_buffer_type_interface = {
|
|
907
|
+
/* .get_name = */ ggml_backend_rpc_buffer_type_name,
|
|
908
|
+
/* .alloc_buffer = */ ggml_backend_rpc_buffer_type_alloc_buffer,
|
|
909
|
+
/* .get_alignment = */ ggml_backend_rpc_buffer_type_get_alignment,
|
|
910
|
+
/* .get_max_size = */ ggml_backend_rpc_get_max_size,
|
|
911
|
+
/* .get_alloc_size = */ ggml_backend_rpc_buffer_type_get_alloc_size,
|
|
912
|
+
/* .is_host = */ NULL,
|
|
913
|
+
};
|
|
914
|
+
|
|
915
|
+
static const char * ggml_backend_rpc_name(ggml_backend_t backend) {
|
|
916
|
+
ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context;
|
|
917
|
+
|
|
918
|
+
return rpc_ctx->name.c_str();
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
static void ggml_backend_rpc_free(ggml_backend_t backend) {
|
|
922
|
+
ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context;
|
|
923
|
+
delete rpc_ctx;
|
|
924
|
+
delete backend;
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
|
|
928
|
+
ggml_backend_rpc_context * ctx = (ggml_backend_rpc_context *)backend->context;
|
|
929
|
+
rpc_tensor rpc_tensor = serialize_tensor(tensor);
|
|
930
|
+
if (size > HASH_THRESHOLD) {
|
|
931
|
+
auto request = std::make_shared<rpc_msg_set_tensor_hash_req>();
|
|
932
|
+
request->tensor = rpc_tensor;
|
|
933
|
+
request->offset = offset;
|
|
934
|
+
request->hash = fnv_hash((const uint8_t*)data, size);
|
|
935
|
+
rpc_msg_set_tensor_hash_rsp response;
|
|
936
|
+
// TODO: make this async
|
|
937
|
+
ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), &response, sizeof(response));
|
|
938
|
+
if (response.result) {
|
|
939
|
+
// the server has the same data, no need to send it
|
|
940
|
+
return;
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
// input serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes)
|
|
944
|
+
size_t input_size = sizeof(rpc_tensor) + sizeof(uint64_t) + size;
|
|
945
|
+
uint8_t * input = new uint8_t[input_size]();
|
|
946
|
+
memcpy(input, &rpc_tensor, sizeof(rpc_tensor));
|
|
947
|
+
memcpy(input + sizeof(rpc_tensor), &offset, sizeof(offset));
|
|
948
|
+
memcpy(input + sizeof(rpc_tensor) + sizeof(offset), data, size);
|
|
949
|
+
std::shared_ptr<uint8_t> input_ptr(input, std::default_delete<uint8_t[]>());
|
|
950
|
+
ctx->dispatcher->send_async(RPC_CMD_SET_TENSOR, input_ptr, input_size);
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
static void ggml_backend_rpc_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
|
|
954
|
+
ggml_backend_rpc_context * ctx = (ggml_backend_rpc_context *)backend->context;
|
|
955
|
+
auto request = std::make_shared<rpc_msg_get_tensor_req>();
|
|
956
|
+
request->tensor = serialize_tensor(tensor);
|
|
957
|
+
request->offset = offset;
|
|
958
|
+
request->size = size;
|
|
959
|
+
ctx->dispatcher->send_async(RPC_CMD_GET_TENSOR, request, sizeof(*request), data, size);
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
static void ggml_backend_rpc_synchronize(ggml_backend_t backend) {
|
|
963
|
+
ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context;
|
|
964
|
+
rpc_ctx->dispatcher->synchronize();
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
static void add_tensor(ggml_tensor * tensor, const ggml_cgraph * cgraph, const std::shared_ptr<rpc_dispatcher> & dispatcher, std::vector<rpc_tensor> & tensors, std::unordered_set<ggml_tensor*> & visited) {
|
|
968
|
+
if (tensor == nullptr) {
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
if (visited.find(tensor) != visited.end()) {
|
|
972
|
+
return;
|
|
973
|
+
}
|
|
974
|
+
visited.insert(tensor);
|
|
975
|
+
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
|
976
|
+
add_tensor(tensor->src[i], cgraph, dispatcher, tensors, visited);
|
|
977
|
+
}
|
|
978
|
+
add_tensor(tensor->view_src, cgraph, dispatcher, tensors, visited);
|
|
979
|
+
rpc_tensor result = serialize_tensor(tensor, dispatcher);
|
|
980
|
+
const size_t hash_pos = ggml_hash_find(&cgraph->visited_hash_set, tensor);
|
|
981
|
+
if (hash_pos != GGML_HASHSET_FULL && ggml_bitset_get(cgraph->visited_hash_set.used, hash_pos)) {
|
|
982
|
+
result.use_count = cgraph->use_counts[hash_pos];
|
|
983
|
+
}
|
|
984
|
+
tensors.push_back(result);
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
static uint8_t * serialize_graph(uint32_t device, const ggml_cgraph * cgraph, const std::shared_ptr<rpc_dispatcher> & dispatcher, size_t * output_size) {
|
|
988
|
+
uint32_t n_nodes = cgraph->n_nodes;
|
|
989
|
+
std::vector<rpc_tensor> tensors;
|
|
990
|
+
std::unordered_set<ggml_tensor*> visited;
|
|
991
|
+
for (uint32_t i = 0; i < n_nodes; i++) {
|
|
992
|
+
add_tensor(cgraph->nodes[i], cgraph, dispatcher, tensors, visited);
|
|
993
|
+
}
|
|
994
|
+
// serialization format:
|
|
995
|
+
// | device (4 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) |
|
|
996
|
+
uint32_t n_tensors = tensors.size();
|
|
997
|
+
*output_size = 2*sizeof(uint32_t) + n_nodes * sizeof(uint64_t) + sizeof(uint32_t) + n_tensors * sizeof(rpc_tensor);
|
|
998
|
+
uint8_t * output = new uint8_t[*output_size]();
|
|
999
|
+
uint8_t * dest = output;
|
|
1000
|
+
memcpy(dest, &device, sizeof(device));
|
|
1001
|
+
dest += sizeof(device);
|
|
1002
|
+
memcpy(dest, &n_nodes, sizeof(n_nodes));
|
|
1003
|
+
dest += sizeof(n_nodes);
|
|
1004
|
+
for (uint32_t i = 0; i < n_nodes; i++) {
|
|
1005
|
+
memcpy(dest + i * sizeof(uint64_t), &cgraph->nodes[i], sizeof(uint64_t));
|
|
1006
|
+
}
|
|
1007
|
+
dest += n_nodes * sizeof(uint64_t);
|
|
1008
|
+
memcpy(dest, &n_tensors, sizeof(n_tensors));
|
|
1009
|
+
dest += sizeof(n_tensors);
|
|
1010
|
+
rpc_tensor * out_tensors = (rpc_tensor *)dest;
|
|
1011
|
+
memcpy(out_tensors, tensors.data(), n_tensors * sizeof(rpc_tensor));
|
|
1012
|
+
return output;
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
static enum ggml_status ggml_backend_rpc_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {
|
|
1016
|
+
ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context;
|
|
1017
|
+
ggml_backend_dev_t rpc_dev = ggml_backend_get_device(backend);
|
|
1018
|
+
ggml_backend_rpc_device_context * rpc_dev_ctx = (ggml_backend_rpc_device_context *)rpc_dev->context;
|
|
1019
|
+
|
|
1020
|
+
GGML_ASSERT(cgraph->n_nodes > 0);
|
|
1021
|
+
bool reuse = cgraph->uid != 0 && rpc_dev_ctx->last_graph_uid == cgraph->uid;
|
|
1022
|
+
if (reuse) {
|
|
1023
|
+
auto request = std::make_shared<rpc_msg_graph_recompute_req>();
|
|
1024
|
+
request->device = rpc_ctx->device;
|
|
1025
|
+
rpc_ctx->dispatcher->send_async(RPC_CMD_GRAPH_RECOMPUTE, request, sizeof(*request));
|
|
1026
|
+
} else {
|
|
1027
|
+
rpc_dev_ctx->last_graph_uid = cgraph->uid;
|
|
1028
|
+
size_t input_size = 0;
|
|
1029
|
+
uint8_t * input = serialize_graph(rpc_ctx->device, cgraph, rpc_ctx->dispatcher, &input_size);
|
|
1030
|
+
std::shared_ptr<uint8_t> input_ptr(input, std::default_delete<uint8_t[]>());
|
|
1031
|
+
rpc_ctx->dispatcher->send_async(RPC_CMD_GRAPH_COMPUTE, input_ptr, input_size);
|
|
1032
|
+
}
|
|
1033
|
+
return GGML_STATUS_SUCCESS;
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
static void ggml_backend_rpc_event_record(ggml_backend_t backend, ggml_backend_event_t event) {
|
|
1037
|
+
ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context;
|
|
1038
|
+
rpc_ctx->dispatcher->event_record(event);
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
static void ggml_backend_rpc_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {
|
|
1042
|
+
// this is noop for RPC as we have a single stream
|
|
1043
|
+
GGML_UNUSED(backend);
|
|
1044
|
+
GGML_UNUSED(event);
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
static ggml_backend_i ggml_backend_rpc_interface = {
|
|
1048
|
+
/* .get_name = */ ggml_backend_rpc_name,
|
|
1049
|
+
/* .free = */ ggml_backend_rpc_free,
|
|
1050
|
+
/* .set_tensor_async = */ ggml_backend_rpc_set_tensor_async,
|
|
1051
|
+
/* .get_tensor_async = */ ggml_backend_rpc_get_tensor_async,
|
|
1052
|
+
/* .set_tensor_2d_async = */ NULL,
|
|
1053
|
+
/* .get_tensor_2d_async = */ NULL,
|
|
1054
|
+
/* .cpy_tensor_async = */ NULL,
|
|
1055
|
+
/* .synchronize = */ ggml_backend_rpc_synchronize,
|
|
1056
|
+
/* .graph_plan_create = */ NULL,
|
|
1057
|
+
/* .graph_plan_free = */ NULL,
|
|
1058
|
+
/* .graph_plan_update = */ NULL,
|
|
1059
|
+
/* .graph_plan_compute = */ NULL,
|
|
1060
|
+
/* .graph_compute = */ ggml_backend_rpc_graph_compute,
|
|
1061
|
+
/* .event_record = */ ggml_backend_rpc_event_record,
|
|
1062
|
+
/* .event_wait = */ ggml_backend_rpc_event_wait,
|
|
1063
|
+
/* .graph_optimize = */ NULL,
|
|
1064
|
+
};
|
|
1065
|
+
|
|
1066
|
+
ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, uint32_t device) {
|
|
1067
|
+
static std::mutex mutex;
|
|
1068
|
+
std::lock_guard<std::mutex> lock(mutex);
|
|
1069
|
+
std::string buft_name = "RPC" + std::to_string(device) + "[" + std::string(endpoint) + "]";
|
|
1070
|
+
// NOTE: buffer types are allocated and never freed; this is by design
|
|
1071
|
+
static std::unordered_map<std::string, ggml_backend_buffer_type_t> buft_map;
|
|
1072
|
+
auto it = buft_map.find(buft_name);
|
|
1073
|
+
if (it != buft_map.end()) {
|
|
1074
|
+
return it->second;
|
|
1075
|
+
}
|
|
1076
|
+
auto dispatcher = get_dispatcher(endpoint);
|
|
1077
|
+
size_t alignment = get_alignment(dispatcher, device);
|
|
1078
|
+
size_t max_size = get_max_size(dispatcher, device);
|
|
1079
|
+
ggml_backend_rpc_buffer_type_context * buft_ctx = new ggml_backend_rpc_buffer_type_context {
|
|
1080
|
+
/* .endpoint = */ endpoint,
|
|
1081
|
+
/* .device = */ device,
|
|
1082
|
+
/* .name = */ buft_name,
|
|
1083
|
+
/* .alignment = */ alignment,
|
|
1084
|
+
/* .max_size = */ max_size
|
|
1085
|
+
};
|
|
1086
|
+
auto reg = ggml_backend_rpc_add_server(endpoint);
|
|
1087
|
+
ggml_backend_buffer_type_t buft = new ggml_backend_buffer_type {
|
|
1088
|
+
/* .iface = */ ggml_backend_rpc_buffer_type_interface,
|
|
1089
|
+
/* .device = */ ggml_backend_reg_dev_get(reg, device),
|
|
1090
|
+
/* .context = */ buft_ctx
|
|
1091
|
+
};
|
|
1092
|
+
buft_map[buft_name] = buft;
|
|
1093
|
+
return buft;
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
ggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device) {
|
|
1097
|
+
std::string dev_name = "RPC" + std::to_string(device) + "[" + std::string(endpoint) + "]";
|
|
1098
|
+
auto dispatcher = get_dispatcher(endpoint);
|
|
1099
|
+
ggml_backend_rpc_context * ctx = new ggml_backend_rpc_context {
|
|
1100
|
+
/* .dispatcher = */ dispatcher,
|
|
1101
|
+
/* .device = */ device,
|
|
1102
|
+
/* .name = */ dev_name,
|
|
1103
|
+
};
|
|
1104
|
+
auto reg = ggml_backend_rpc_add_server(endpoint);
|
|
1105
|
+
ggml_backend_t backend = new ggml_backend {
|
|
1106
|
+
/* .guid = */ ggml_backend_rpc_guid(),
|
|
1107
|
+
/* .iface = */ ggml_backend_rpc_interface,
|
|
1108
|
+
/* .device = */ ggml_backend_reg_dev_get(reg, device),
|
|
1109
|
+
/* .context = */ ctx
|
|
1110
|
+
};
|
|
1111
|
+
return backend;
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
bool ggml_backend_is_rpc(ggml_backend_t backend) {
|
|
1115
|
+
return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_rpc_guid());
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device, size_t * free, size_t * total) {
|
|
1119
|
+
auto dispatcher = get_dispatcher(endpoint);
|
|
1120
|
+
auto request = std::make_shared<rpc_msg_get_device_memory_req>();
|
|
1121
|
+
request->device = device;
|
|
1122
|
+
rpc_msg_get_device_memory_rsp response;
|
|
1123
|
+
dispatcher->send(RPC_CMD_GET_DEVICE_MEMORY, request, sizeof(*request), &response, sizeof(response));
|
|
1124
|
+
*free = response.free_mem;
|
|
1125
|
+
*total = response.total_mem;
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
// RPC server-side implementation
|
|
1129
|
+
|
|
1130
|
+
class rpc_server {
|
|
1131
|
+
public:
|
|
1132
|
+
rpc_server(std::vector<ggml_backend_t> all_backends, const char * cache_dir)
|
|
1133
|
+
: backends(std::move(all_backends)), cache_dir(cache_dir) {
|
|
1134
|
+
stored_graphs.resize(backends.size());
|
|
1135
|
+
}
|
|
1136
|
+
~rpc_server();
|
|
1137
|
+
|
|
1138
|
+
void hello(rpc_msg_hello_rsp & response);
|
|
1139
|
+
bool alloc_buffer(const rpc_msg_alloc_buffer_req & request, rpc_msg_alloc_buffer_rsp & response);
|
|
1140
|
+
bool get_alignment(const rpc_msg_get_alignment_req & request, rpc_msg_get_alignment_rsp & response);
|
|
1141
|
+
bool get_max_size(const rpc_msg_get_max_size_req & request, rpc_msg_get_max_size_rsp & response);
|
|
1142
|
+
bool buffer_get_base(const rpc_msg_buffer_get_base_req & request, rpc_msg_buffer_get_base_rsp & response);
|
|
1143
|
+
bool free_buffer(const rpc_msg_free_buffer_req & request);
|
|
1144
|
+
bool buffer_clear(const rpc_msg_buffer_clear_req & request);
|
|
1145
|
+
bool memset_tensor(const rpc_msg_memset_tensor_req & request);
|
|
1146
|
+
bool set_tensor(const std::vector<uint8_t> & input);
|
|
1147
|
+
bool set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response);
|
|
1148
|
+
bool get_tensor(const rpc_msg_get_tensor_req & request, std::vector<uint8_t> & response);
|
|
1149
|
+
bool copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_copy_tensor_rsp & response);
|
|
1150
|
+
bool graph_compute(const std::vector<uint8_t> & input);
|
|
1151
|
+
bool graph_recompute(const rpc_msg_graph_recompute_req & request);
|
|
1152
|
+
bool init_tensor(const rpc_msg_init_tensor_req & request);
|
|
1153
|
+
bool get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_msg_get_alloc_size_rsp & response);
|
|
1154
|
+
bool get_device_memory(const rpc_msg_get_device_memory_req & request, rpc_msg_get_device_memory_rsp & response);
|
|
1155
|
+
|
|
1156
|
+
struct stored_graph {
|
|
1157
|
+
std::vector<uint8_t> buffer;
|
|
1158
|
+
ggml_cgraph * graph;
|
|
1159
|
+
};
|
|
1160
|
+
|
|
1161
|
+
private:
|
|
1162
|
+
bool get_cached_file(uint64_t hash, std::vector<uint8_t> & data);
|
|
1163
|
+
ggml_tensor * deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor);
|
|
1164
|
+
ggml_tensor * create_node(uint64_t id,
|
|
1165
|
+
struct ggml_context * ctx,
|
|
1166
|
+
const std::unordered_map<uint64_t, const rpc_tensor*> & tensor_ptrs,
|
|
1167
|
+
std::unordered_map<uint64_t, struct ggml_tensor*> & tensor_map);
|
|
1168
|
+
|
|
1169
|
+
|
|
1170
|
+
std::vector<ggml_backend_t> backends;
|
|
1171
|
+
const char * cache_dir;
|
|
1172
|
+
std::unordered_set<ggml_backend_buffer_t> buffers;
|
|
1173
|
+
// store the last computed graph for each backend
|
|
1174
|
+
std::vector<stored_graph> stored_graphs;
|
|
1175
|
+
};
|
|
1176
|
+
|
|
1177
|
+
void rpc_server::hello(rpc_msg_hello_rsp & response) {
|
|
1178
|
+
response.major = RPC_PROTO_MAJOR_VERSION;
|
|
1179
|
+
response.minor = RPC_PROTO_MINOR_VERSION;
|
|
1180
|
+
response.patch = RPC_PROTO_PATCH_VERSION;
|
|
1181
|
+
LOG_DBG("[%s] version: %d.%d.%d\n", __func__, response.major, response.minor, response.patch);
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_msg_get_alloc_size_rsp & response) {
|
|
1185
|
+
uint32_t dev_id = request.device;
|
|
1186
|
+
if (dev_id >= backends.size()) {
|
|
1187
|
+
return false;
|
|
1188
|
+
}
|
|
1189
|
+
ggml_backend_buffer_type_t buft;
|
|
1190
|
+
struct ggml_init_params params {
|
|
1191
|
+
/*.mem_size =*/ ggml_tensor_overhead()*(1 + GGML_MAX_SRC),
|
|
1192
|
+
/*.mem_buffer =*/ NULL,
|
|
1193
|
+
/*.no_alloc =*/ true,
|
|
1194
|
+
};
|
|
1195
|
+
|
|
1196
|
+
ggml_context_ptr ctx_ptr { ggml_init(params) };
|
|
1197
|
+
GGML_ASSERT(ctx_ptr != nullptr);
|
|
1198
|
+
ggml_context * ctx = ctx_ptr.get();
|
|
1199
|
+
|
|
1200
|
+
ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);
|
|
1201
|
+
if (tensor == nullptr) {
|
|
1202
|
+
GGML_LOG_ERROR("Null tensor pointer passed to server get_alloc_size function.\n");
|
|
1203
|
+
return false;
|
|
1204
|
+
}
|
|
1205
|
+
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
|
1206
|
+
if (request.srcs[i].id != 0) {
|
|
1207
|
+
tensor->src[i] = deserialize_tensor(ctx, &request.srcs[i]);
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
LOG_DBG("[%s] device: %d, buffer: %p, data: %p\n", __func__, dev_id, (void*)tensor->buffer, tensor->data);
|
|
1212
|
+
if (tensor->buffer == nullptr) {
|
|
1213
|
+
//No buffer allocated.
|
|
1214
|
+
buft = ggml_backend_get_default_buffer_type(backends[dev_id]);
|
|
1215
|
+
} else {
|
|
1216
|
+
buft = tensor->buffer->buft;
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
response.alloc_size = ggml_backend_buft_get_alloc_size(buft, tensor);
|
|
1220
|
+
|
|
1221
|
+
return true;
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
bool rpc_server::alloc_buffer(const rpc_msg_alloc_buffer_req & request, rpc_msg_alloc_buffer_rsp & response) {
|
|
1225
|
+
uint32_t dev_id = request.device;
|
|
1226
|
+
if (dev_id >= backends.size()) {
|
|
1227
|
+
return false;
|
|
1228
|
+
}
|
|
1229
|
+
ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]);
|
|
1230
|
+
ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, request.size);
|
|
1231
|
+
response.remote_ptr = 0;
|
|
1232
|
+
response.remote_size = 0;
|
|
1233
|
+
if (buffer != nullptr) {
|
|
1234
|
+
response.remote_ptr = reinterpret_cast<uint64_t>(buffer);
|
|
1235
|
+
response.remote_size = buffer->size;
|
|
1236
|
+
LOG_DBG("[%s] device: %d, size: %" PRIu64 " -> remote_ptr: %" PRIx64 ", remote_size: %" PRIu64 "\n",
|
|
1237
|
+
__func__, dev_id, request.size, response.remote_ptr, response.remote_size);
|
|
1238
|
+
buffers.insert(buffer);
|
|
1239
|
+
} else {
|
|
1240
|
+
LOG_DBG("[%s] device: %d, size: %" PRIu64 " -> failed\n", __func__, dev_id, request.size);
|
|
1241
|
+
}
|
|
1242
|
+
return true;
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
bool rpc_server::get_alignment(const rpc_msg_get_alignment_req & request, rpc_msg_get_alignment_rsp & response) {
|
|
1246
|
+
uint32_t dev_id = request.device;
|
|
1247
|
+
if (dev_id >= backends.size()) {
|
|
1248
|
+
return false;
|
|
1249
|
+
}
|
|
1250
|
+
ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]);
|
|
1251
|
+
size_t alignment = ggml_backend_buft_get_alignment(buft);
|
|
1252
|
+
LOG_DBG("[%s] device: %d, alignment: %lu\n", __func__, dev_id, alignment);
|
|
1253
|
+
response.alignment = alignment;
|
|
1254
|
+
return true;
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
bool rpc_server::get_max_size(const rpc_msg_get_max_size_req & request, rpc_msg_get_max_size_rsp & response) {
|
|
1258
|
+
uint32_t dev_id = request.device;
|
|
1259
|
+
if (dev_id >= backends.size()) {
|
|
1260
|
+
return false;
|
|
1261
|
+
}
|
|
1262
|
+
ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]);
|
|
1263
|
+
size_t max_size = ggml_backend_buft_get_max_size(buft);
|
|
1264
|
+
LOG_DBG("[%s] device: %d, max_size: %lu\n", __func__, dev_id, max_size);
|
|
1265
|
+
response.max_size = max_size;
|
|
1266
|
+
return true;
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
bool rpc_server::buffer_get_base(const rpc_msg_buffer_get_base_req & request, rpc_msg_buffer_get_base_rsp & response) {
|
|
1270
|
+
LOG_DBG("[%s] remote_ptr: %" PRIx64 "\n", __func__, request.remote_ptr);
|
|
1271
|
+
ggml_backend_buffer_t buffer = reinterpret_cast<ggml_backend_buffer_t>(request.remote_ptr);
|
|
1272
|
+
if (buffers.find(buffer) == buffers.end()) {
|
|
1273
|
+
GGML_LOG_ERROR("[%s] buffer not found\n", __func__);
|
|
1274
|
+
return false;
|
|
1275
|
+
}
|
|
1276
|
+
void * base = ggml_backend_buffer_get_base(buffer);
|
|
1277
|
+
response.base_ptr = reinterpret_cast<uint64_t>(base);
|
|
1278
|
+
return true;
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
bool rpc_server::free_buffer(const rpc_msg_free_buffer_req & request) {
|
|
1282
|
+
LOG_DBG("[%s] remote_ptr: %" PRIx64 "\n", __func__, request.remote_ptr);
|
|
1283
|
+
ggml_backend_buffer_t buffer = reinterpret_cast<ggml_backend_buffer_t>(request.remote_ptr);
|
|
1284
|
+
if (buffers.find(buffer) == buffers.end()) {
|
|
1285
|
+
GGML_LOG_ERROR("[%s] buffer not found\n", __func__);
|
|
1286
|
+
return false;
|
|
1287
|
+
}
|
|
1288
|
+
ggml_backend_buffer_free(buffer);
|
|
1289
|
+
buffers.erase(buffer);
|
|
1290
|
+
return true;
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1293
|
+
bool rpc_server::buffer_clear(const rpc_msg_buffer_clear_req & request) {
|
|
1294
|
+
LOG_DBG("[%s] remote_ptr: %" PRIx64 ", value: %u\n", __func__, request.remote_ptr, request.value);
|
|
1295
|
+
ggml_backend_buffer_t buffer = reinterpret_cast<ggml_backend_buffer_t>(request.remote_ptr);
|
|
1296
|
+
if (buffers.find(buffer) == buffers.end()) {
|
|
1297
|
+
GGML_LOG_ERROR("[%s] buffer not found\n", __func__);
|
|
1298
|
+
return false;
|
|
1299
|
+
}
|
|
1300
|
+
ggml_backend_buffer_clear(buffer, request.value);
|
|
1301
|
+
return true;
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
bool rpc_server::memset_tensor(const rpc_msg_memset_tensor_req & request) {
|
|
1305
|
+
struct ggml_init_params params {
|
|
1306
|
+
/*.mem_size =*/ ggml_tensor_overhead(),
|
|
1307
|
+
/*.mem_buffer =*/ NULL,
|
|
1308
|
+
/*.no_alloc =*/ true,
|
|
1309
|
+
};
|
|
1310
|
+
ggml_context_ptr ctx_ptr { ggml_init(params) };
|
|
1311
|
+
GGML_ASSERT(ctx_ptr != nullptr);
|
|
1312
|
+
ggml_context * ctx = ctx_ptr.get();
|
|
1313
|
+
ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);
|
|
1314
|
+
if (tensor == nullptr || tensor->buffer == nullptr) {
|
|
1315
|
+
GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__);
|
|
1316
|
+
return false;
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
const uint64_t tensor_size = ggml_nbytes(tensor);
|
|
1320
|
+
if (request.offset > tensor_size || request.size > tensor_size - request.offset) {
|
|
1321
|
+
GGML_LOG_ERROR("[%s] tensor region (offset=%" PRIu64 ", size=%" PRIu64 ") out of tensor bounds [0, %" PRIu64 ")\n",
|
|
1322
|
+
__func__, request.offset, request.size, tensor_size);
|
|
1323
|
+
return false;
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
const uint64_t buffer_start = (uint64_t) ggml_backend_buffer_get_base(tensor->buffer);
|
|
1327
|
+
const uint64_t buffer_size = ggml_backend_buffer_get_size(tensor->buffer);
|
|
1328
|
+
if (request.tensor.data < buffer_start) {
|
|
1329
|
+
GGML_LOG_ERROR("[%s] tensor data before buffer start\n", __func__);
|
|
1330
|
+
return false;
|
|
1331
|
+
}
|
|
1332
|
+
const uint64_t data_offset = request.tensor.data - buffer_start;
|
|
1333
|
+
if (data_offset > buffer_size ||
|
|
1334
|
+
request.offset > buffer_size - data_offset ||
|
|
1335
|
+
request.size > buffer_size - data_offset - request.offset) {
|
|
1336
|
+
GGML_LOG_ERROR("[%s] tensor region out of buffer bounds\n", __func__);
|
|
1337
|
+
return false;
|
|
1338
|
+
}
|
|
1339
|
+
if (tensor->buffer->iface.memset_tensor == nullptr) {
|
|
1340
|
+
GGML_LOG_ERROR("[%s] memset not implemented by backend buffer\n", __func__);
|
|
1341
|
+
return false;
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 ", value: %u\n",
|
|
1345
|
+
__func__, (void *) tensor->buffer, tensor->data, request.offset, request.size, request.value);
|
|
1346
|
+
ggml_backend_tensor_memset(tensor, request.value, request.offset, request.size);
|
|
1347
|
+
return true;
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor) {
|
|
1351
|
+
// Validate tensor type before using it
|
|
1352
|
+
if (tensor->type >= GGML_TYPE_COUNT) {
|
|
1353
|
+
GGML_LOG_ERROR("[%s] invalid tensor type received: %u\n", __func__, tensor->type);
|
|
1354
|
+
return nullptr;
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
// Fix: Prevent division by zero if blck_size is 0 (e.g., deprecated types)
|
|
1358
|
+
if (ggml_blck_size((enum ggml_type)tensor->type) == 0) {
|
|
1359
|
+
GGML_LOG_ERROR("[%s] invalid tensor type received (blck_size is 0): %u\n", __func__, tensor->type);
|
|
1360
|
+
return nullptr;
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
ggml_tensor * result = ggml_new_tensor_4d(ctx, (ggml_type) tensor->type,
|
|
1364
|
+
tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]);
|
|
1365
|
+
|
|
1366
|
+
// ggml_new_tensor_4d might fail if dimensions are invalid, although less likely to crash than invalid type
|
|
1367
|
+
if (result == nullptr) {
|
|
1368
|
+
GGML_LOG_ERROR("[%s] ggml_new_tensor_4d failed for type %u\n", __func__, tensor->type);
|
|
1369
|
+
return nullptr;
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
for (uint32_t i = 0; i < GGML_MAX_DIMS; i++) {
|
|
1373
|
+
result->nb[i] = tensor->nb[i];
|
|
1374
|
+
}
|
|
1375
|
+
result->buffer = reinterpret_cast<ggml_backend_buffer_t>(tensor->buffer);
|
|
1376
|
+
if (result->buffer && buffers.find(result->buffer) == buffers.end()) {
|
|
1377
|
+
result->buffer = nullptr;
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
if (result->buffer) {
|
|
1381
|
+
// require that the tensor data does not go beyond the buffer end
|
|
1382
|
+
uint64_t tensor_size = (uint64_t) ggml_nbytes(result);
|
|
1383
|
+
uint64_t buffer_start = (uint64_t) ggml_backend_buffer_get_base(result->buffer);
|
|
1384
|
+
uint64_t buffer_size = (uint64_t) ggml_backend_buffer_get_size(result->buffer);
|
|
1385
|
+
GGML_ASSERT(tensor->data + tensor_size >= tensor->data); // check for overflow
|
|
1386
|
+
GGML_ASSERT(tensor->data >= buffer_start && tensor->data + tensor_size <= buffer_start + buffer_size);
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
result->op = (ggml_op) tensor->op;
|
|
1390
|
+
for (uint32_t i = 0; i < GGML_MAX_OP_PARAMS / sizeof(int32_t); i++) {
|
|
1391
|
+
result->op_params[i] = tensor->op_params[i];
|
|
1392
|
+
}
|
|
1393
|
+
result->flags = tensor->flags;
|
|
1394
|
+
result->data = reinterpret_cast<void *>(tensor->data);
|
|
1395
|
+
ggml_set_name(result, tensor->name);
|
|
1396
|
+
return result;
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
|
|
1400
|
+
bool rpc_server::set_tensor(const std::vector<uint8_t> & input) {
|
|
1401
|
+
// serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) |
|
|
1402
|
+
if (input.size() < sizeof(rpc_tensor) + sizeof(uint64_t)) {
|
|
1403
|
+
return false;
|
|
1404
|
+
}
|
|
1405
|
+
const rpc_tensor * in_tensor = (const rpc_tensor *)input.data();
|
|
1406
|
+
uint64_t offset;
|
|
1407
|
+
memcpy(&offset, input.data() + sizeof(rpc_tensor), sizeof(offset));
|
|
1408
|
+
const size_t size = input.size() - sizeof(rpc_tensor) - sizeof(offset);
|
|
1409
|
+
|
|
1410
|
+
struct ggml_init_params params {
|
|
1411
|
+
/*.mem_size =*/ ggml_tensor_overhead(),
|
|
1412
|
+
/*.mem_buffer =*/ NULL,
|
|
1413
|
+
/*.no_alloc =*/ true,
|
|
1414
|
+
};
|
|
1415
|
+
ggml_context_ptr ctx_ptr { ggml_init(params) };
|
|
1416
|
+
GGML_ASSERT(ctx_ptr != nullptr);
|
|
1417
|
+
ggml_context * ctx = ctx_ptr.get();
|
|
1418
|
+
ggml_tensor * tensor = deserialize_tensor(ctx, in_tensor);
|
|
1419
|
+
if (tensor == nullptr || tensor->buffer == nullptr) {
|
|
1420
|
+
GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__);
|
|
1421
|
+
return false;
|
|
1422
|
+
}
|
|
1423
|
+
LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu\n", __func__, (void*)tensor->buffer, tensor->data, offset, size);
|
|
1424
|
+
|
|
1425
|
+
// sanitize tensor->data
|
|
1426
|
+
{
|
|
1427
|
+
const size_t p0 = (size_t) ggml_backend_buffer_get_base(tensor->buffer);
|
|
1428
|
+
const size_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer);
|
|
1429
|
+
|
|
1430
|
+
if (in_tensor->data + offset < p0 || in_tensor->data + offset >= p1 || size > (p1 - in_tensor->data - offset)) {
|
|
1431
|
+
GGML_LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu) out of buffer bounds [0x%zx, 0x%zx)\n",
|
|
1432
|
+
__func__, in_tensor->data, offset, size, p0, p1);
|
|
1433
|
+
return false;
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
const void * data = input.data() + sizeof(rpc_tensor) + sizeof(offset);
|
|
1438
|
+
if (cache_dir && size > HASH_THRESHOLD) {
|
|
1439
|
+
uint64_t hash = fnv_hash((const uint8_t*)data, size);
|
|
1440
|
+
char hash_str[17];
|
|
1441
|
+
snprintf(hash_str, sizeof(hash_str), "%016" PRIx64, hash);
|
|
1442
|
+
// save to cache_dir/hash_str
|
|
1443
|
+
fs::path cache_file = fs::path(cache_dir) / hash_str;
|
|
1444
|
+
std::ofstream ofs(cache_file, std::ios::binary);
|
|
1445
|
+
ofs.write((const char *)data, size);
|
|
1446
|
+
GGML_LOG_INFO("[%s] saved to '%s'\n", __func__, cache_file.string().c_str());
|
|
1447
|
+
}
|
|
1448
|
+
ggml_backend_tensor_set(tensor, data, offset, size);
|
|
1449
|
+
return true;
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
bool rpc_server::get_cached_file(uint64_t hash, std::vector<uint8_t> & data) {
|
|
1453
|
+
if (!cache_dir) {
|
|
1454
|
+
return false;
|
|
1455
|
+
}
|
|
1456
|
+
char hash_str[17];
|
|
1457
|
+
snprintf(hash_str, sizeof(hash_str), "%016" PRIx64, hash);
|
|
1458
|
+
fs::path cache_file = fs::path(cache_dir) / hash_str;
|
|
1459
|
+
std::error_code ec;
|
|
1460
|
+
if (!fs::exists(cache_file, ec)) {
|
|
1461
|
+
return false;
|
|
1462
|
+
}
|
|
1463
|
+
std::ifstream ifs(cache_file, std::ios::binary);
|
|
1464
|
+
ifs.seekg(0, std::ios::end);
|
|
1465
|
+
size_t size = ifs.tellg();
|
|
1466
|
+
ifs.seekg(0, std::ios::beg);
|
|
1467
|
+
data.resize(size);
|
|
1468
|
+
ifs.read((char *)data.data(), size);
|
|
1469
|
+
return true;
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response)
|
|
1473
|
+
{
|
|
1474
|
+
std::vector<uint8_t> cached_file;
|
|
1475
|
+
if (!get_cached_file(request.hash, cached_file)) {
|
|
1476
|
+
response.result = 0;
|
|
1477
|
+
return true;
|
|
1478
|
+
}
|
|
1479
|
+
size_t size = cached_file.size();
|
|
1480
|
+
struct ggml_init_params params {
|
|
1481
|
+
/*.mem_size =*/ ggml_tensor_overhead(),
|
|
1482
|
+
/*.mem_buffer =*/ NULL,
|
|
1483
|
+
/*.no_alloc =*/ true,
|
|
1484
|
+
};
|
|
1485
|
+
ggml_context_ptr ctx_ptr { ggml_init(params) };
|
|
1486
|
+
GGML_ASSERT(ctx_ptr != nullptr);
|
|
1487
|
+
ggml_context * ctx = ctx_ptr.get();
|
|
1488
|
+
ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);
|
|
1489
|
+
if (tensor == nullptr || tensor->buffer == nullptr) {
|
|
1490
|
+
GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__);
|
|
1491
|
+
return false;
|
|
1492
|
+
}
|
|
1493
|
+
LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu, hash: %" PRIx64 "\n",
|
|
1494
|
+
__func__, (void*)tensor->buffer, tensor->data, request.offset, size, request.hash);
|
|
1495
|
+
|
|
1496
|
+
// sanitize tensor->data
|
|
1497
|
+
{
|
|
1498
|
+
const size_t p0 = (size_t) ggml_backend_buffer_get_base(tensor->buffer);
|
|
1499
|
+
const size_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer);
|
|
1500
|
+
|
|
1501
|
+
if (request.tensor.data + request.offset < p0
|
|
1502
|
+
|| request.tensor.data + request.offset >= p1
|
|
1503
|
+
|| size > (p1 - request.tensor.data - request.offset)) {
|
|
1504
|
+
GGML_LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu, hash=0x%" PRIx64 ") out of buffer bounds [0x%zx, 0x%zx)\n",
|
|
1505
|
+
__func__, request.tensor.data, request.offset, size, request.hash, p0, p1);
|
|
1506
|
+
return false;
|
|
1507
|
+
}
|
|
1508
|
+
}
|
|
1509
|
+
ggml_backend_tensor_set(tensor, cached_file.data(), request.offset, size);
|
|
1510
|
+
response.result = 1;
|
|
1511
|
+
return true;
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) {
|
|
1515
|
+
struct ggml_init_params params {
|
|
1516
|
+
/*.mem_size =*/ ggml_tensor_overhead(),
|
|
1517
|
+
/*.mem_buffer =*/ NULL,
|
|
1518
|
+
/*.no_alloc =*/ true,
|
|
1519
|
+
};
|
|
1520
|
+
ggml_context_ptr ctx_ptr { ggml_init(params) };
|
|
1521
|
+
GGML_ASSERT(ctx_ptr != nullptr);
|
|
1522
|
+
ggml_context * ctx = ctx_ptr.get();
|
|
1523
|
+
ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);
|
|
1524
|
+
if (tensor == nullptr) {
|
|
1525
|
+
GGML_LOG_ERROR("Null tensor pointer passed to server init_tensor function.\n");
|
|
1526
|
+
return false;
|
|
1527
|
+
}
|
|
1528
|
+
LOG_DBG("[%s] buffer: %p, data: %p\n", __func__, (void*)tensor->buffer, tensor->data);
|
|
1529
|
+
// Call the backend's buffer_init_tensor function
|
|
1530
|
+
ggml_backend_buffer_t buffer = tensor->buffer;
|
|
1531
|
+
if (buffer && buffer->iface.init_tensor) {
|
|
1532
|
+
buffer->iface.init_tensor(buffer, tensor);
|
|
1533
|
+
} else {
|
|
1534
|
+
if (!buffer) {
|
|
1535
|
+
GGML_LOG_ERROR("Tensor with null buffer passed to init_tensor function\n");
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
if (tensor->extra != nullptr) {
|
|
1540
|
+
// This pointer can either be passed around client/server, or probably better stored server-side and kept track of.
|
|
1541
|
+
// Currently unimplemented.
|
|
1542
|
+
GGML_LOG_ERROR("tensor->extra populated by the backend, this is currently unsupported.\n");
|
|
1543
|
+
return false;
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
return true;
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector<uint8_t> & response) {
|
|
1550
|
+
struct ggml_init_params params {
|
|
1551
|
+
/*.mem_size =*/ ggml_tensor_overhead(),
|
|
1552
|
+
/*.mem_buffer =*/ NULL,
|
|
1553
|
+
/*.no_alloc =*/ true,
|
|
1554
|
+
};
|
|
1555
|
+
ggml_context_ptr ctx_ptr { ggml_init(params) };
|
|
1556
|
+
GGML_ASSERT(ctx_ptr != nullptr);
|
|
1557
|
+
ggml_context * ctx = ctx_ptr.get();
|
|
1558
|
+
ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);
|
|
1559
|
+
if (tensor == nullptr || tensor->buffer == nullptr) {
|
|
1560
|
+
GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__);
|
|
1561
|
+
return false;
|
|
1562
|
+
}
|
|
1563
|
+
LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 "\n", __func__, (void*)tensor->buffer, tensor->data, request.offset, request.size);
|
|
1564
|
+
|
|
1565
|
+
// sanitize tensor->data
|
|
1566
|
+
{
|
|
1567
|
+
const size_t p0 = (size_t) ggml_backend_buffer_get_base(tensor->buffer);
|
|
1568
|
+
const size_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer);
|
|
1569
|
+
|
|
1570
|
+
if (request.tensor.data + request.offset < p0 ||
|
|
1571
|
+
request.tensor.data + request.offset >= p1 ||
|
|
1572
|
+
request.size > (p1 - request.tensor.data - request.offset)) {
|
|
1573
|
+
GGML_LOG_ERROR("[%s] requested tensor region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%" PRIu64 ") out of buffer bounds [0x%zx, 0x%zx)\n",
|
|
1574
|
+
__func__, request.tensor.data, request.offset, request.size, p0, p1);
|
|
1575
|
+
return false;
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
response.resize(request.size, 0);
|
|
1580
|
+
ggml_backend_tensor_get(tensor, response.data(), request.offset, request.size);
|
|
1581
|
+
return true;
|
|
1582
|
+
}
|
|
1583
|
+
|
|
1584
|
+
bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_copy_tensor_rsp & response) {
|
|
1585
|
+
struct ggml_init_params params {
|
|
1586
|
+
/*.mem_size =*/ 2*ggml_tensor_overhead(),
|
|
1587
|
+
/*.mem_buffer =*/ NULL,
|
|
1588
|
+
/*.no_alloc =*/ true,
|
|
1589
|
+
};
|
|
1590
|
+
ggml_context_ptr ctx_ptr { ggml_init(params) };
|
|
1591
|
+
GGML_ASSERT(ctx_ptr != nullptr);
|
|
1592
|
+
ggml_context * ctx = ctx_ptr.get();
|
|
1593
|
+
|
|
1594
|
+
ggml_tensor * src = deserialize_tensor(ctx, &request.src);
|
|
1595
|
+
ggml_tensor * dst = deserialize_tensor(ctx, &request.dst);
|
|
1596
|
+
if (src == nullptr || dst == nullptr || src->buffer == nullptr || dst->buffer == nullptr) {
|
|
1597
|
+
GGML_LOG_ERROR("[%s] error deserializing tensors\n", __func__);
|
|
1598
|
+
return false;
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
uint64_t src_size = (uint64_t) ggml_nbytes(src);
|
|
1602
|
+
uint64_t dst_data = (uint64_t) dst->data;
|
|
1603
|
+
uint64_t dst_base = (uint64_t) ggml_backend_buffer_get_base(dst->buffer);
|
|
1604
|
+
uint64_t dst_buf_sz = (uint64_t) ggml_backend_buffer_get_size(dst->buffer);
|
|
1605
|
+
|
|
1606
|
+
if (dst_data + src_size > dst_base + dst_buf_sz) {
|
|
1607
|
+
GGML_LOG_ERROR("[%s] out-of-bounds write in rpc_server::copy_tensor:\n"
|
|
1608
|
+
" write range : [0x%" PRIx64 ", 0x%" PRIx64 "]\n"
|
|
1609
|
+
" buffer base: [0x%" PRIx64 ", 0x%" PRIx64 "]\n",
|
|
1610
|
+
__func__,
|
|
1611
|
+
dst_data,
|
|
1612
|
+
dst_data + src_size,
|
|
1613
|
+
dst_base,
|
|
1614
|
+
dst_base + dst_buf_sz);
|
|
1615
|
+
return false;
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
LOG_DBG("[%s] src->buffer: %p, dst->buffer: %p\n",
|
|
1619
|
+
__func__, (void*) src->buffer, (void*) dst->buffer);
|
|
1620
|
+
|
|
1621
|
+
response.result = ggml_backend_buffer_copy_tensor(src, dst);
|
|
1622
|
+
return true;
|
|
1623
|
+
}
|
|
1624
|
+
|
|
1625
|
+
ggml_tensor * rpc_server::create_node(uint64_t id,
|
|
1626
|
+
struct ggml_context * ctx,
|
|
1627
|
+
const std::unordered_map<uint64_t, const rpc_tensor*> & tensor_ptrs,
|
|
1628
|
+
std::unordered_map<uint64_t, struct ggml_tensor*> & tensor_map) {
|
|
1629
|
+
if (tensor_map.find(id) != tensor_map.end()) {
|
|
1630
|
+
return tensor_map[id];
|
|
1631
|
+
}
|
|
1632
|
+
// Safely find the tensor pointer
|
|
1633
|
+
auto it_ptr = tensor_ptrs.find(id);
|
|
1634
|
+
if (it_ptr == tensor_ptrs.end()) {
|
|
1635
|
+
return nullptr;
|
|
1636
|
+
}
|
|
1637
|
+
const rpc_tensor * tensor = it_ptr->second;
|
|
1638
|
+
|
|
1639
|
+
struct ggml_tensor * result = deserialize_tensor(ctx, tensor);
|
|
1640
|
+
if (result == nullptr) {
|
|
1641
|
+
return nullptr;
|
|
1642
|
+
}
|
|
1643
|
+
if (result->buffer == nullptr && result->data != nullptr) {
|
|
1644
|
+
GGML_LOG_ERROR("[%s] invalid data ptr", __func__);
|
|
1645
|
+
return nullptr;
|
|
1646
|
+
}
|
|
1647
|
+
tensor_map[id] = result;
|
|
1648
|
+
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
|
1649
|
+
// Check if the source ID is 0 before calling create_node recursively
|
|
1650
|
+
if (tensor->src[i] == 0) {
|
|
1651
|
+
result->src[i] = nullptr;
|
|
1652
|
+
} else {
|
|
1653
|
+
result->src[i] = create_node(tensor->src[i], ctx, tensor_ptrs, tensor_map);
|
|
1654
|
+
// If the recursive call failed for a non-zero ID, propagate the error
|
|
1655
|
+
if (result->src[i] == nullptr) {
|
|
1656
|
+
GGML_LOG_ERROR("[%s] failed to create source node %d (src_id=%" PRIu64 ") for node id %" PRIu64 "\n",
|
|
1657
|
+
__func__, i, tensor->src[i], id);
|
|
1658
|
+
// Must return nullptr to signal failure up the call stack
|
|
1659
|
+
return nullptr;
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
// Handle view_src similarly
|
|
1665
|
+
if (tensor->view_src == 0) {
|
|
1666
|
+
result->view_src = nullptr;
|
|
1667
|
+
} else {
|
|
1668
|
+
result->view_src = create_node(tensor->view_src, ctx, tensor_ptrs, tensor_map);
|
|
1669
|
+
// If the recursive call failed for a non-zero ID, propagate the error
|
|
1670
|
+
if (result->view_src == nullptr) {
|
|
1671
|
+
GGML_LOG_ERROR("[%s] failed to create view_src node (view_src_id=%" PRIu64 ") for node id %" PRIu64 "\n",
|
|
1672
|
+
__func__, tensor->view_src, id);
|
|
1673
|
+
// Must return nullptr to signal failure up the call stack
|
|
1674
|
+
return nullptr;
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1677
|
+
result->view_offs = tensor->view_offs;
|
|
1678
|
+
return result;
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
bool rpc_server::graph_compute(const std::vector<uint8_t> & input) {
|
|
1682
|
+
// serialization format:
|
|
1683
|
+
// | device (4 bytes) | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) |
|
|
1684
|
+
if (input.size() < 2*sizeof(uint32_t)) {
|
|
1685
|
+
return false;
|
|
1686
|
+
}
|
|
1687
|
+
const uint8_t * src = input.data();
|
|
1688
|
+
uint32_t device;
|
|
1689
|
+
memcpy(&device, src, sizeof(device));
|
|
1690
|
+
src += sizeof(device);
|
|
1691
|
+
if (device >= backends.size()) {
|
|
1692
|
+
return false;
|
|
1693
|
+
}
|
|
1694
|
+
uint32_t n_nodes;
|
|
1695
|
+
memcpy(&n_nodes, src, sizeof(n_nodes));
|
|
1696
|
+
src += sizeof(n_nodes);
|
|
1697
|
+
if (input.size() < 2*sizeof(uint32_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t)) {
|
|
1698
|
+
return false;
|
|
1699
|
+
}
|
|
1700
|
+
const uint64_t * nodes = (const uint64_t *)src;
|
|
1701
|
+
src += n_nodes*sizeof(uint64_t);
|
|
1702
|
+
uint32_t n_tensors;
|
|
1703
|
+
memcpy(&n_tensors, src, sizeof(n_tensors));
|
|
1704
|
+
src += sizeof(n_tensors);
|
|
1705
|
+
if (input.size() < 2*sizeof(uint32_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t) + n_tensors*sizeof(rpc_tensor)) {
|
|
1706
|
+
return false;
|
|
1707
|
+
}
|
|
1708
|
+
const rpc_tensor * tensors = (const rpc_tensor *)src;
|
|
1709
|
+
LOG_DBG("[%s] device: %u, n_nodes: %u, n_tensors: %u\n", __func__, device, n_nodes, n_tensors);
|
|
1710
|
+
|
|
1711
|
+
size_t buf_size = ggml_tensor_overhead()*(n_nodes + n_tensors) + ggml_graph_overhead_custom(n_nodes, false);
|
|
1712
|
+
if (stored_graphs[device].buffer.size() < buf_size) {
|
|
1713
|
+
stored_graphs[device].buffer.resize(buf_size);
|
|
1714
|
+
}
|
|
1715
|
+
struct ggml_init_params params = {
|
|
1716
|
+
/*.mem_size =*/ buf_size,
|
|
1717
|
+
/*.mem_buffer =*/ stored_graphs[device].buffer.data(),
|
|
1718
|
+
/*.no_alloc =*/ true,
|
|
1719
|
+
};
|
|
1720
|
+
ggml_context_ptr ctx_ptr { ggml_init(params) };
|
|
1721
|
+
GGML_ASSERT(ctx_ptr != nullptr);
|
|
1722
|
+
ggml_context * ctx = ctx_ptr.get();
|
|
1723
|
+
struct ggml_cgraph * graph = ggml_new_graph_custom(ctx, n_nodes, false);
|
|
1724
|
+
graph->n_nodes = n_nodes;
|
|
1725
|
+
std::unordered_map<uint64_t, const rpc_tensor*> tensor_ptrs;
|
|
1726
|
+
tensor_ptrs.reserve(n_tensors);
|
|
1727
|
+
for (uint32_t i = 0; i < n_tensors; i++) {
|
|
1728
|
+
tensor_ptrs.emplace(tensors[i].id, &tensors[i]);
|
|
1729
|
+
}
|
|
1730
|
+
std::unordered_map<uint64_t, ggml_tensor*> tensor_map;
|
|
1731
|
+
tensor_map.reserve(n_nodes);
|
|
1732
|
+
for (uint32_t i = 0; i < n_nodes; i++) {
|
|
1733
|
+
int64_t id;
|
|
1734
|
+
memcpy(&id, &nodes[i], sizeof(id));
|
|
1735
|
+
graph->nodes[i] = create_node(id, ctx, tensor_ptrs, tensor_map);
|
|
1736
|
+
|
|
1737
|
+
// Check if create_node failed for a *non-zero* ID.
|
|
1738
|
+
// If id was 0, create_node returning nullptr is expected.
|
|
1739
|
+
// If id was non-zero and create_node returned nullptr, it indicates a deserialization error.
|
|
1740
|
+
if (graph->nodes[i] == nullptr && id != 0) {
|
|
1741
|
+
GGML_LOG_ERROR("[%s] failed to create graph node %d (id=%" PRId64 ")\n", __func__, i, id);
|
|
1742
|
+
return false;
|
|
1743
|
+
}
|
|
1744
|
+
if (graph->nodes[i] != nullptr) {
|
|
1745
|
+
const size_t hash_pos = ggml_hash_insert(&graph->visited_hash_set, graph->nodes[i]);
|
|
1746
|
+
graph->use_counts[hash_pos] = tensor_ptrs.at(id)->use_count;
|
|
1747
|
+
}
|
|
1748
|
+
}
|
|
1749
|
+
ggml_status status = ggml_backend_graph_compute(backends[device], graph);
|
|
1750
|
+
GGML_ASSERT(status == GGML_STATUS_SUCCESS && "Unsuccessful graph computations are not supported with RPC");
|
|
1751
|
+
stored_graphs[device].graph = graph;
|
|
1752
|
+
return true;
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
bool rpc_server::graph_recompute(const rpc_msg_graph_recompute_req & request) {
|
|
1756
|
+
uint32_t device = request.device;
|
|
1757
|
+
if (device >= backends.size()) {
|
|
1758
|
+
return false;
|
|
1759
|
+
}
|
|
1760
|
+
if (stored_graphs[device].graph == nullptr) {
|
|
1761
|
+
return false;
|
|
1762
|
+
}
|
|
1763
|
+
ggml_cgraph * graph = stored_graphs[device].graph;
|
|
1764
|
+
LOG_DBG("[%s] device: %u\n", __func__, device);
|
|
1765
|
+
ggml_status status = ggml_backend_graph_compute(backends[device], graph);
|
|
1766
|
+
GGML_ASSERT(status == GGML_STATUS_SUCCESS && "Unsuccessful graph computations are not supported with RPC");
|
|
1767
|
+
return true;
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
bool rpc_server::get_device_memory(const rpc_msg_get_device_memory_req & request, rpc_msg_get_device_memory_rsp & response) {
|
|
1771
|
+
uint32_t dev_id = request.device;
|
|
1772
|
+
if (dev_id >= backends.size()) {
|
|
1773
|
+
return false;
|
|
1774
|
+
}
|
|
1775
|
+
size_t free, total;
|
|
1776
|
+
ggml_backend_dev_t dev = ggml_backend_get_device(backends[dev_id]);
|
|
1777
|
+
ggml_backend_dev_memory(dev, &free, &total);
|
|
1778
|
+
response.free_mem = free;
|
|
1779
|
+
response.total_mem = total;
|
|
1780
|
+
LOG_DBG("[%s] device: %u, free_mem: %" PRIu64 ", total_mem: %" PRIu64 "\n", __func__, dev_id, response.free_mem, response.total_mem);
|
|
1781
|
+
return true;
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
rpc_server::~rpc_server() {
|
|
1785
|
+
for (auto buffer : buffers) {
|
|
1786
|
+
ggml_backend_buffer_free(buffer);
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1790
|
+
static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const char * cache_dir,
|
|
1791
|
+
socket_ptr sock) {
|
|
1792
|
+
rpc_server server(backends, cache_dir);
|
|
1793
|
+
uint8_t cmd;
|
|
1794
|
+
if (!sock->recv_data(&cmd, 1)) {
|
|
1795
|
+
return;
|
|
1796
|
+
}
|
|
1797
|
+
if (cmd != RPC_CMD_HELLO) {
|
|
1798
|
+
GGML_LOG_ERROR("Expected HELLO command, update client\n");
|
|
1799
|
+
return;
|
|
1800
|
+
}
|
|
1801
|
+
|
|
1802
|
+
// Read input_size and validate protocol version
|
|
1803
|
+
uint64_t hello_input_size;
|
|
1804
|
+
if (!sock->recv_data(&hello_input_size, sizeof(hello_input_size))) {
|
|
1805
|
+
return;
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
if (hello_input_size != sizeof(rpc_msg_hello_req)) {
|
|
1809
|
+
GGML_LOG_ERROR("HELLO request size mismatch (%zu vs %zu) — client needs upgrade to protocol v%d.x\n",
|
|
1810
|
+
(size_t)hello_input_size, sizeof(rpc_msg_hello_req), RPC_PROTO_MAJOR_VERSION);
|
|
1811
|
+
return;
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
rpc_msg_hello_req req = {};
|
|
1815
|
+
if (!sock->recv_data(&req, sizeof(req))) {
|
|
1816
|
+
return;
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
rpc_msg_hello_rsp rsp = {};
|
|
1820
|
+
server.hello(rsp);
|
|
1821
|
+
// Advertise server transport capabilities based on client's caps
|
|
1822
|
+
sock->get_caps(rsp.conn_caps);
|
|
1823
|
+
if (!send_msg(sock, &rsp, sizeof(rsp))) {
|
|
1824
|
+
return;
|
|
1825
|
+
}
|
|
1826
|
+
|
|
1827
|
+
// Activate transport upgrade using client's caps
|
|
1828
|
+
sock->update_caps(req.conn_caps);
|
|
1829
|
+
while (true) {
|
|
1830
|
+
if (!sock->recv_data(&cmd, 1)) {
|
|
1831
|
+
break;
|
|
1832
|
+
}
|
|
1833
|
+
if (cmd >= RPC_CMD_COUNT) {
|
|
1834
|
+
// fail fast if the command is invalid
|
|
1835
|
+
GGML_LOG_ERROR("Unknown command: %d\n", cmd);
|
|
1836
|
+
break;
|
|
1837
|
+
}
|
|
1838
|
+
switch (cmd) {
|
|
1839
|
+
case RPC_CMD_HELLO: {
|
|
1840
|
+
// HELLO command is handled above
|
|
1841
|
+
return;
|
|
1842
|
+
}
|
|
1843
|
+
case RPC_CMD_DEVICE_COUNT: {
|
|
1844
|
+
if (!recv_msg(sock, nullptr, 0)) {
|
|
1845
|
+
return;
|
|
1846
|
+
}
|
|
1847
|
+
rpc_msg_device_count_rsp response;
|
|
1848
|
+
response.device_count = backends.size();
|
|
1849
|
+
if (!send_msg(sock, &response, sizeof(response))) {
|
|
1850
|
+
return;
|
|
1851
|
+
}
|
|
1852
|
+
break;
|
|
1853
|
+
}
|
|
1854
|
+
case RPC_CMD_ALLOC_BUFFER: {
|
|
1855
|
+
rpc_msg_alloc_buffer_req request;
|
|
1856
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
1857
|
+
return;
|
|
1858
|
+
}
|
|
1859
|
+
rpc_msg_alloc_buffer_rsp response;
|
|
1860
|
+
if (!server.alloc_buffer(request, response)) {
|
|
1861
|
+
return;
|
|
1862
|
+
}
|
|
1863
|
+
if (!send_msg(sock, &response, sizeof(response))) {
|
|
1864
|
+
return;
|
|
1865
|
+
}
|
|
1866
|
+
break;
|
|
1867
|
+
}
|
|
1868
|
+
case RPC_CMD_GET_ALLOC_SIZE: {
|
|
1869
|
+
rpc_msg_get_alloc_size_req request;
|
|
1870
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
1871
|
+
return;
|
|
1872
|
+
}
|
|
1873
|
+
rpc_msg_get_alloc_size_rsp response;
|
|
1874
|
+
if (!server.get_alloc_size(request, response)) {
|
|
1875
|
+
return;
|
|
1876
|
+
}
|
|
1877
|
+
if (!send_msg(sock, &response, sizeof(response))) {
|
|
1878
|
+
return;
|
|
1879
|
+
}
|
|
1880
|
+
break;
|
|
1881
|
+
}
|
|
1882
|
+
case RPC_CMD_GET_ALIGNMENT: {
|
|
1883
|
+
rpc_msg_get_alignment_req request;
|
|
1884
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
1885
|
+
return;
|
|
1886
|
+
}
|
|
1887
|
+
rpc_msg_get_alignment_rsp response;
|
|
1888
|
+
if (!server.get_alignment(request, response)) {
|
|
1889
|
+
return;
|
|
1890
|
+
}
|
|
1891
|
+
if (!send_msg(sock, &response, sizeof(response))) {
|
|
1892
|
+
return;
|
|
1893
|
+
}
|
|
1894
|
+
break;
|
|
1895
|
+
}
|
|
1896
|
+
case RPC_CMD_GET_MAX_SIZE: {
|
|
1897
|
+
rpc_msg_get_max_size_req request;
|
|
1898
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
1899
|
+
return;
|
|
1900
|
+
}
|
|
1901
|
+
rpc_msg_get_max_size_rsp response;
|
|
1902
|
+
if (!server.get_max_size(request, response)) {
|
|
1903
|
+
return;
|
|
1904
|
+
}
|
|
1905
|
+
if (!send_msg(sock, &response, sizeof(response))) {
|
|
1906
|
+
return;
|
|
1907
|
+
}
|
|
1908
|
+
break;
|
|
1909
|
+
}
|
|
1910
|
+
case RPC_CMD_BUFFER_GET_BASE: {
|
|
1911
|
+
rpc_msg_buffer_get_base_req request;
|
|
1912
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
1913
|
+
return;
|
|
1914
|
+
}
|
|
1915
|
+
rpc_msg_buffer_get_base_rsp response;
|
|
1916
|
+
if (!server.buffer_get_base(request, response)) {
|
|
1917
|
+
return;
|
|
1918
|
+
}
|
|
1919
|
+
if (!send_msg(sock, &response, sizeof(response))) {
|
|
1920
|
+
return;
|
|
1921
|
+
}
|
|
1922
|
+
break;
|
|
1923
|
+
}
|
|
1924
|
+
case RPC_CMD_FREE_BUFFER: {
|
|
1925
|
+
rpc_msg_free_buffer_req request;
|
|
1926
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
1927
|
+
return;
|
|
1928
|
+
}
|
|
1929
|
+
if (!server.free_buffer(request)) {
|
|
1930
|
+
return;
|
|
1931
|
+
}
|
|
1932
|
+
break;
|
|
1933
|
+
}
|
|
1934
|
+
case RPC_CMD_BUFFER_CLEAR: {
|
|
1935
|
+
rpc_msg_buffer_clear_req request;
|
|
1936
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
1937
|
+
return;
|
|
1938
|
+
}
|
|
1939
|
+
if (!server.buffer_clear(request)) {
|
|
1940
|
+
return;
|
|
1941
|
+
}
|
|
1942
|
+
break;
|
|
1943
|
+
}
|
|
1944
|
+
case RPC_CMD_MEMSET_TENSOR: {
|
|
1945
|
+
rpc_msg_memset_tensor_req request;
|
|
1946
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
1947
|
+
return;
|
|
1948
|
+
}
|
|
1949
|
+
if (!server.memset_tensor(request)) {
|
|
1950
|
+
return;
|
|
1951
|
+
}
|
|
1952
|
+
break;
|
|
1953
|
+
}
|
|
1954
|
+
case RPC_CMD_SET_TENSOR: {
|
|
1955
|
+
std::vector<uint8_t> input;
|
|
1956
|
+
if (!recv_msg(sock, input)) {
|
|
1957
|
+
return;
|
|
1958
|
+
}
|
|
1959
|
+
if (!server.set_tensor(input)) {
|
|
1960
|
+
return;
|
|
1961
|
+
}
|
|
1962
|
+
break;
|
|
1963
|
+
}
|
|
1964
|
+
case RPC_CMD_SET_TENSOR_HASH: {
|
|
1965
|
+
rpc_msg_set_tensor_hash_req request;
|
|
1966
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
1967
|
+
return;
|
|
1968
|
+
}
|
|
1969
|
+
rpc_msg_set_tensor_hash_rsp response;
|
|
1970
|
+
if (!server.set_tensor_hash(request, response)) {
|
|
1971
|
+
return;
|
|
1972
|
+
}
|
|
1973
|
+
if (!send_msg(sock, &response, sizeof(response))) {
|
|
1974
|
+
return;
|
|
1975
|
+
}
|
|
1976
|
+
break;
|
|
1977
|
+
}
|
|
1978
|
+
case RPC_CMD_INIT_TENSOR: {
|
|
1979
|
+
rpc_msg_init_tensor_req request;
|
|
1980
|
+
if (!recv_msg(sock, &request,sizeof(request))) {
|
|
1981
|
+
return;
|
|
1982
|
+
}
|
|
1983
|
+
if (!server.init_tensor(request)) {
|
|
1984
|
+
return;
|
|
1985
|
+
}
|
|
1986
|
+
break;
|
|
1987
|
+
}
|
|
1988
|
+
case RPC_CMD_GET_TENSOR: {
|
|
1989
|
+
rpc_msg_get_tensor_req request;
|
|
1990
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
1991
|
+
return;
|
|
1992
|
+
}
|
|
1993
|
+
std::vector<uint8_t> response;
|
|
1994
|
+
if (!server.get_tensor(request, response)) {
|
|
1995
|
+
return;
|
|
1996
|
+
}
|
|
1997
|
+
if (!send_msg(sock, response.data(), response.size())) {
|
|
1998
|
+
return;
|
|
1999
|
+
}
|
|
2000
|
+
break;
|
|
2001
|
+
}
|
|
2002
|
+
case RPC_CMD_COPY_TENSOR: {
|
|
2003
|
+
rpc_msg_copy_tensor_req request;
|
|
2004
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
2005
|
+
return;
|
|
2006
|
+
}
|
|
2007
|
+
rpc_msg_copy_tensor_rsp response;
|
|
2008
|
+
if (!server.copy_tensor(request, response)) {
|
|
2009
|
+
return;
|
|
2010
|
+
}
|
|
2011
|
+
if (!send_msg(sock, &response, sizeof(response))) {
|
|
2012
|
+
return;
|
|
2013
|
+
}
|
|
2014
|
+
break;
|
|
2015
|
+
}
|
|
2016
|
+
case RPC_CMD_GRAPH_COMPUTE: {
|
|
2017
|
+
std::vector<uint8_t> input;
|
|
2018
|
+
if (!recv_msg(sock, input)) {
|
|
2019
|
+
return;
|
|
2020
|
+
}
|
|
2021
|
+
if (!server.graph_compute(input)) {
|
|
2022
|
+
return;
|
|
2023
|
+
}
|
|
2024
|
+
break;
|
|
2025
|
+
}
|
|
2026
|
+
case RPC_CMD_GRAPH_RECOMPUTE: {
|
|
2027
|
+
rpc_msg_graph_recompute_req request;
|
|
2028
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
2029
|
+
return;
|
|
2030
|
+
}
|
|
2031
|
+
if (!server.graph_recompute(request)) {
|
|
2032
|
+
return;
|
|
2033
|
+
}
|
|
2034
|
+
break;
|
|
2035
|
+
}
|
|
2036
|
+
case RPC_CMD_GET_DEVICE_MEMORY: {
|
|
2037
|
+
rpc_msg_get_device_memory_req request;
|
|
2038
|
+
if (!recv_msg(sock, &request, sizeof(request))) {
|
|
2039
|
+
return;
|
|
2040
|
+
}
|
|
2041
|
+
rpc_msg_get_device_memory_rsp response;
|
|
2042
|
+
if (!server.get_device_memory(request, response)) {
|
|
2043
|
+
return;
|
|
2044
|
+
}
|
|
2045
|
+
if (!send_msg(sock, &response, sizeof(response))) {
|
|
2046
|
+
return;
|
|
2047
|
+
}
|
|
2048
|
+
break;
|
|
2049
|
+
}
|
|
2050
|
+
default: {
|
|
2051
|
+
GGML_LOG_ERROR("Unknown command: %d\n", cmd);
|
|
2052
|
+
return;
|
|
2053
|
+
}
|
|
2054
|
+
}
|
|
2055
|
+
}
|
|
2056
|
+
}
|
|
2057
|
+
|
|
2058
|
+
void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir,
|
|
2059
|
+
size_t n_threads, size_t n_devices, ggml_backend_dev_t * devices) {
|
|
2060
|
+
if (n_devices == 0 || devices == nullptr) {
|
|
2061
|
+
fprintf(stderr, "Invalid arguments to ggml_backend_rpc_start_server\n");
|
|
2062
|
+
return;
|
|
2063
|
+
}
|
|
2064
|
+
std::vector<ggml_backend_t> backends;
|
|
2065
|
+
printf("Starting RPC server v%d.%d.%d\n",
|
|
2066
|
+
RPC_PROTO_MAJOR_VERSION,
|
|
2067
|
+
RPC_PROTO_MINOR_VERSION,
|
|
2068
|
+
RPC_PROTO_PATCH_VERSION);
|
|
2069
|
+
printf(" endpoint : %s\n", endpoint);
|
|
2070
|
+
printf(" local cache : %s\n", cache_dir ? cache_dir : "n/a");
|
|
2071
|
+
printf("Devices:\n");
|
|
2072
|
+
for (size_t i = 0; i < n_devices; i++) {
|
|
2073
|
+
auto dev = devices[i];
|
|
2074
|
+
size_t free, total;
|
|
2075
|
+
ggml_backend_dev_memory(dev, &free, &total);
|
|
2076
|
+
printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),
|
|
2077
|
+
total / 1024 / 1024, free / 1024 / 1024);
|
|
2078
|
+
auto backend = ggml_backend_dev_init(dev, nullptr);
|
|
2079
|
+
if (!backend) {
|
|
2080
|
+
fprintf(stderr, "Failed to create backend for device %s\n", dev->iface.get_name(dev));
|
|
2081
|
+
return;
|
|
2082
|
+
}
|
|
2083
|
+
backends.push_back(backend);
|
|
2084
|
+
ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
|
|
2085
|
+
if (reg) {
|
|
2086
|
+
auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
|
|
2087
|
+
if (ggml_backend_set_n_threads_fn) {
|
|
2088
|
+
ggml_backend_set_n_threads_fn(backend, n_threads);
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
std::string host;
|
|
2094
|
+
int port;
|
|
2095
|
+
if (!parse_endpoint(endpoint, host, port)) {
|
|
2096
|
+
return;
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
#ifdef GGML_RPC_RDMA
|
|
2100
|
+
printf(" transport : TCP (RDMA auto-negotiate enabled)\n");
|
|
2101
|
+
#else
|
|
2102
|
+
printf(" transport : TCP\n");
|
|
2103
|
+
#endif // GGML_RPC_RDMA
|
|
2104
|
+
if (!rpc_transport_init()) {
|
|
2105
|
+
fprintf(stderr, "Failed to initialize RPC transport\n");
|
|
2106
|
+
return;
|
|
2107
|
+
}
|
|
2108
|
+
auto server_socket = socket_t::create_server(host.c_str(), port);
|
|
2109
|
+
if (server_socket == nullptr) {
|
|
2110
|
+
fprintf(stderr, "Failed to create server socket\n");
|
|
2111
|
+
return;
|
|
2112
|
+
}
|
|
2113
|
+
while (true) {
|
|
2114
|
+
auto client_socket = server_socket->accept();
|
|
2115
|
+
if (client_socket == nullptr) {
|
|
2116
|
+
fprintf(stderr, "Failed to accept client connection\n");
|
|
2117
|
+
return;
|
|
2118
|
+
}
|
|
2119
|
+
printf("Accepted client connection\n");
|
|
2120
|
+
fflush(stdout);
|
|
2121
|
+
rpc_serve_client(backends, cache_dir, client_socket);
|
|
2122
|
+
printf("Client connection closed\n");
|
|
2123
|
+
fflush(stdout);
|
|
2124
|
+
}
|
|
2125
|
+
rpc_transport_shutdown();
|
|
2126
|
+
for (auto backend : backends) {
|
|
2127
|
+
ggml_backend_free(backend);
|
|
2128
|
+
}
|
|
2129
|
+
}
|
|
2130
|
+
|
|
2131
|
+
static const char * ggml_backend_rpc_device_get_name(ggml_backend_dev_t dev) {
|
|
2132
|
+
ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;
|
|
2133
|
+
|
|
2134
|
+
return ctx->name.c_str();
|
|
2135
|
+
}
|
|
2136
|
+
|
|
2137
|
+
static const char * ggml_backend_rpc_device_get_description(ggml_backend_dev_t dev) {
|
|
2138
|
+
ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;
|
|
2139
|
+
|
|
2140
|
+
return ctx->description.c_str();
|
|
2141
|
+
}
|
|
2142
|
+
|
|
2143
|
+
static void ggml_backend_rpc_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
|
|
2144
|
+
ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;
|
|
2145
|
+
|
|
2146
|
+
ggml_backend_rpc_get_device_memory(ctx->endpoint.c_str(), ctx->device, free, total);
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
static enum ggml_backend_dev_type ggml_backend_rpc_device_get_type(ggml_backend_dev_t dev) {
|
|
2150
|
+
// TODO: obtain value from the server
|
|
2151
|
+
return GGML_BACKEND_DEVICE_TYPE_GPU;
|
|
2152
|
+
|
|
2153
|
+
GGML_UNUSED(dev);
|
|
2154
|
+
}
|
|
2155
|
+
|
|
2156
|
+
static void ggml_backend_rpc_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) {
|
|
2157
|
+
props->name = ggml_backend_rpc_device_get_name(dev);
|
|
2158
|
+
props->description = ggml_backend_rpc_device_get_description(dev);
|
|
2159
|
+
props->type = ggml_backend_rpc_device_get_type(dev);
|
|
2160
|
+
ggml_backend_rpc_device_get_memory(dev, &props->memory_free, &props->memory_total);
|
|
2161
|
+
props->caps = {
|
|
2162
|
+
/* .async = */ true,
|
|
2163
|
+
/* .host_buffer = */ false,
|
|
2164
|
+
/* .buffer_from_host_ptr = */ false,
|
|
2165
|
+
/* .events = */ true,
|
|
2166
|
+
/* .mmap_support = */ true,
|
|
2167
|
+
};
|
|
2168
|
+
}
|
|
2169
|
+
|
|
2170
|
+
static ggml_backend_t ggml_backend_rpc_device_init(ggml_backend_dev_t dev, const char * params) {
|
|
2171
|
+
ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;
|
|
2172
|
+
|
|
2173
|
+
return ggml_backend_rpc_init(ctx->endpoint.c_str(), ctx->device);
|
|
2174
|
+
|
|
2175
|
+
GGML_UNUSED(params);
|
|
2176
|
+
}
|
|
2177
|
+
|
|
2178
|
+
static ggml_backend_buffer_type_t ggml_backend_rpc_device_get_buffer_type(ggml_backend_dev_t dev) {
|
|
2179
|
+
ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;
|
|
2180
|
+
|
|
2181
|
+
return ggml_backend_rpc_buffer_type(ctx->endpoint.c_str(), ctx->device);
|
|
2182
|
+
|
|
2183
|
+
GGML_UNUSED(dev);
|
|
2184
|
+
}
|
|
2185
|
+
|
|
2186
|
+
static bool ggml_backend_rpc_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {
|
|
2187
|
+
GGML_UNUSED(dev);
|
|
2188
|
+
GGML_UNUSED(op);
|
|
2189
|
+
//TODO: call the remote backend and cache the results
|
|
2190
|
+
return true;
|
|
2191
|
+
}
|
|
2192
|
+
|
|
2193
|
+
static bool ggml_backend_rpc_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
|
|
2194
|
+
if (!buft || buft->iface.get_name != ggml_backend_rpc_buffer_type_name) {
|
|
2195
|
+
return false;
|
|
2196
|
+
}
|
|
2197
|
+
ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;
|
|
2198
|
+
ggml_backend_rpc_device_context * dev_ctx = (ggml_backend_rpc_device_context *)dev->context;
|
|
2199
|
+
return buft_ctx->endpoint == dev_ctx->endpoint && buft_ctx->device == dev_ctx->device;
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2202
|
+
static ggml_backend_event_t ggml_backend_rpc_device_event_new(ggml_backend_dev_t dev) {
|
|
2203
|
+
ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;
|
|
2204
|
+
auto dispatcher = get_dispatcher(ctx->endpoint);
|
|
2205
|
+
return dispatcher->event_new(dev);
|
|
2206
|
+
}
|
|
2207
|
+
|
|
2208
|
+
static void ggml_backend_rpc_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) {
|
|
2209
|
+
ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;
|
|
2210
|
+
auto dispatcher = get_dispatcher(ctx->endpoint);
|
|
2211
|
+
dispatcher->event_free(event);
|
|
2212
|
+
}
|
|
2213
|
+
|
|
2214
|
+
static void ggml_backend_rpc_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) {
|
|
2215
|
+
ggml_backend_rpc_device_context * ctx = (ggml_backend_rpc_device_context *)dev->context;
|
|
2216
|
+
auto dispatcher = get_dispatcher(ctx->endpoint);
|
|
2217
|
+
dispatcher->event_synchronize(event);
|
|
2218
|
+
}
|
|
2219
|
+
|
|
2220
|
+
static const struct ggml_backend_device_i ggml_backend_rpc_device_i = {
|
|
2221
|
+
/* .get_name = */ ggml_backend_rpc_device_get_name,
|
|
2222
|
+
/* .get_description = */ ggml_backend_rpc_device_get_description,
|
|
2223
|
+
/* .get_memory = */ ggml_backend_rpc_device_get_memory,
|
|
2224
|
+
/* .get_type = */ ggml_backend_rpc_device_get_type,
|
|
2225
|
+
/* .get_props = */ ggml_backend_rpc_device_get_props,
|
|
2226
|
+
/* .init_backend = */ ggml_backend_rpc_device_init,
|
|
2227
|
+
/* .get_buffer_type = */ ggml_backend_rpc_device_get_buffer_type,
|
|
2228
|
+
/* .get_host_buffer_type = */ NULL,
|
|
2229
|
+
/* .buffer_from_host_ptr = */ NULL,
|
|
2230
|
+
/* .supports_op = */ ggml_backend_rpc_device_supports_op,
|
|
2231
|
+
/* .supports_buft = */ ggml_backend_rpc_device_supports_buft,
|
|
2232
|
+
/* .offload_op = */ NULL,
|
|
2233
|
+
/* .event_new = */ ggml_backend_rpc_device_event_new,
|
|
2234
|
+
/* .event_free = */ ggml_backend_rpc_device_event_free,
|
|
2235
|
+
/* .event_synchronize = */ ggml_backend_rpc_device_event_synchronize,
|
|
2236
|
+
};
|
|
2237
|
+
|
|
2238
|
+
// backend reg interface
|
|
2239
|
+
|
|
2240
|
+
struct ggml_backend_rpc_reg_context {
|
|
2241
|
+
std::string name;
|
|
2242
|
+
std::vector<ggml_backend_dev_t> devices;
|
|
2243
|
+
};
|
|
2244
|
+
|
|
2245
|
+
static const char * ggml_backend_rpc_reg_get_name(ggml_backend_reg_t reg) {
|
|
2246
|
+
ggml_backend_rpc_reg_context * ctx = (ggml_backend_rpc_reg_context *)reg->context;
|
|
2247
|
+
return ctx ? ctx->name.c_str() : "RPC";
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2250
|
+
static size_t ggml_backend_rpc_reg_get_device_count(ggml_backend_reg_t reg) {
|
|
2251
|
+
ggml_backend_rpc_reg_context * ctx = (ggml_backend_rpc_reg_context *)reg->context;
|
|
2252
|
+
return ctx ? ctx->devices.size() : 0;
|
|
2253
|
+
}
|
|
2254
|
+
|
|
2255
|
+
static ggml_backend_dev_t ggml_backend_rpc_reg_get_device(ggml_backend_reg_t reg, size_t index) {
|
|
2256
|
+
ggml_backend_rpc_reg_context * ctx = (ggml_backend_rpc_reg_context *)reg->context;
|
|
2257
|
+
if (ctx == nullptr) {
|
|
2258
|
+
GGML_ABORT("The RPC backend does not have enumerated devices - use ggml_backend_rpc_add_server instead");
|
|
2259
|
+
} else {
|
|
2260
|
+
GGML_ASSERT(index < ctx->devices.size());
|
|
2261
|
+
return ctx->devices[index];
|
|
2262
|
+
}
|
|
2263
|
+
}
|
|
2264
|
+
|
|
2265
|
+
static void * ggml_backend_rpc_get_proc_address(ggml_backend_reg_t reg, const char * name) {
|
|
2266
|
+
if (std::strcmp(name, "ggml_backend_rpc_add_server") == 0) {
|
|
2267
|
+
return (void *)ggml_backend_rpc_add_server;
|
|
2268
|
+
}
|
|
2269
|
+
if (std::strcmp(name, "ggml_backend_rpc_start_server") == 0) {
|
|
2270
|
+
return (void *)ggml_backend_rpc_start_server;
|
|
2271
|
+
}
|
|
2272
|
+
return NULL;
|
|
2273
|
+
|
|
2274
|
+
GGML_UNUSED(reg);
|
|
2275
|
+
}
|
|
2276
|
+
|
|
2277
|
+
static const struct ggml_backend_reg_i ggml_backend_rpc_reg_i = {
|
|
2278
|
+
/* .get_name = */ ggml_backend_rpc_reg_get_name,
|
|
2279
|
+
/* .get_device_count = */ ggml_backend_rpc_reg_get_device_count,
|
|
2280
|
+
/* .get_device = */ ggml_backend_rpc_reg_get_device,
|
|
2281
|
+
/* .get_proc_address = */ ggml_backend_rpc_get_proc_address,
|
|
2282
|
+
};
|
|
2283
|
+
|
|
2284
|
+
ggml_backend_reg_t ggml_backend_rpc_reg(void) {
|
|
2285
|
+
static struct ggml_backend_reg ggml_backend_rpc_reg = {
|
|
2286
|
+
/* .api_version = */ GGML_BACKEND_API_VERSION,
|
|
2287
|
+
/* .iface = */ ggml_backend_rpc_reg_i,
|
|
2288
|
+
/* .context = */ NULL,
|
|
2289
|
+
};
|
|
2290
|
+
|
|
2291
|
+
return &ggml_backend_rpc_reg;
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
static uint32_t ggml_backend_rpc_get_device_count(const char * endpoint) {
|
|
2295
|
+
auto dispatcher = get_dispatcher(endpoint);
|
|
2296
|
+
rpc_msg_device_count_rsp response;
|
|
2297
|
+
dispatcher->send(RPC_CMD_DEVICE_COUNT, nullptr, 0, &response, sizeof(response));
|
|
2298
|
+
return response.device_count;
|
|
2299
|
+
}
|
|
2300
|
+
|
|
2301
|
+
static const ggml_backend_reg_i ggml_backend_rpc_reg_interface = {
|
|
2302
|
+
/* .get_name = */ ggml_backend_rpc_reg_get_name,
|
|
2303
|
+
/* .get_device_count = */ ggml_backend_rpc_reg_get_device_count,
|
|
2304
|
+
/* .get_device = */ ggml_backend_rpc_reg_get_device,
|
|
2305
|
+
/* .get_proc_address = */ ggml_backend_rpc_get_proc_address,
|
|
2306
|
+
};
|
|
2307
|
+
|
|
2308
|
+
ggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint) {
|
|
2309
|
+
static std::unordered_map<std::string, ggml_backend_reg_t> reg_map;
|
|
2310
|
+
static std::mutex mutex;
|
|
2311
|
+
static uint32_t dev_id = 0;
|
|
2312
|
+
std::lock_guard<std::mutex> lock(mutex);
|
|
2313
|
+
if (reg_map.find(endpoint) != reg_map.end()) {
|
|
2314
|
+
return reg_map[endpoint];
|
|
2315
|
+
}
|
|
2316
|
+
uint32_t dev_count = ggml_backend_rpc_get_device_count(endpoint);
|
|
2317
|
+
if (dev_count == 0) {
|
|
2318
|
+
return nullptr;
|
|
2319
|
+
}
|
|
2320
|
+
ggml_backend_rpc_reg_context * ctx = new ggml_backend_rpc_reg_context;
|
|
2321
|
+
ctx->name = "RPC[" + std::string(endpoint) + "]";
|
|
2322
|
+
for (uint32_t ind = 0; ind < dev_count; ind++) {
|
|
2323
|
+
std::string dev_name = "RPC" + std::to_string(dev_id);
|
|
2324
|
+
std::string dev_desc = std::string(endpoint);
|
|
2325
|
+
ggml_backend_rpc_device_context * dev_ctx = new ggml_backend_rpc_device_context {
|
|
2326
|
+
/* .endpoint = */ endpoint,
|
|
2327
|
+
/* .device = */ ind,
|
|
2328
|
+
/* .name = */ dev_name,
|
|
2329
|
+
/* .description = */ dev_desc,
|
|
2330
|
+
/* .last_graph_uid = */ 0,
|
|
2331
|
+
};
|
|
2332
|
+
|
|
2333
|
+
ggml_backend_dev_t dev = new ggml_backend_device {
|
|
2334
|
+
/* .iface = */ ggml_backend_rpc_device_i,
|
|
2335
|
+
/* .reg = */ ggml_backend_rpc_reg(),
|
|
2336
|
+
/* .context = */ dev_ctx,
|
|
2337
|
+
};
|
|
2338
|
+
ctx->devices.push_back(dev);
|
|
2339
|
+
dev_id++;
|
|
2340
|
+
}
|
|
2341
|
+
ggml_backend_reg_t reg = new ggml_backend_reg {
|
|
2342
|
+
/* .api_version = */ GGML_BACKEND_API_VERSION,
|
|
2343
|
+
/* .iface = */ ggml_backend_rpc_reg_interface,
|
|
2344
|
+
/* .context = */ ctx
|
|
2345
|
+
};
|
|
2346
|
+
reg_map[endpoint] = reg;
|
|
2347
|
+
return reg;
|
|
2348
|
+
}
|
|
2349
|
+
|
|
2350
|
+
|
|
2351
|
+
GGML_BACKEND_DL_IMPL(ggml_backend_rpc_reg)
|