whispercpp 1.3.1 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +4 -3
- data/README.md +92 -31
- data/Rakefile +26 -7
- data/ext/.gitignore +5 -7
- data/ext/dependencies.rb +61 -0
- data/ext/extconf.rb +21 -198
- data/ext/options.rb +221 -0
- data/ext/ruby_whisper.c +159 -0
- data/ext/ruby_whisper.h +17 -2
- data/ext/ruby_whisper_context.c +641 -0
- data/ext/ruby_whisper_error.c +52 -0
- data/ext/ruby_whisper_model.c +232 -0
- data/ext/ruby_whisper_params.c +1301 -0
- data/ext/ruby_whisper_segment.c +143 -0
- data/ext/ruby_whisper_transcribe.cpp +87 -0
- data/ext/ruby_whisper_vad_params.c +288 -0
- data/ext/sources/.dockerignore +3 -0
- data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
- data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
- data/ext/sources/CMakeLists.txt +251 -0
- data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
- data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
- data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
- data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
- data/ext/sources/bindings/javascript/package.json +26 -0
- data/ext/sources/bindings/javascript/whisper.js +19 -0
- data/ext/sources/build-xcframework.sh +547 -0
- data/ext/sources/ci/run.sh +336 -0
- data/ext/sources/close-issue.yml +28 -0
- data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
- data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
- data/ext/sources/cmake/build-info.cmake +60 -0
- data/ext/sources/cmake/git-vars.cmake +22 -0
- data/ext/sources/cmake/whisper-config.cmake.in +65 -0
- data/ext/sources/cmake/whisper.pc.in +10 -0
- data/ext/sources/examples/CMakeLists.txt +124 -0
- data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
- data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
- data/ext/sources/examples/addon.node/addon.cpp +438 -0
- data/ext/sources/examples/addon.node/index.js +54 -0
- data/ext/sources/examples/addon.node/package.json +16 -0
- data/ext/sources/examples/bench/CMakeLists.txt +8 -0
- data/ext/sources/examples/bench/bench.cpp +175 -0
- data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
- data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
- data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
- data/ext/sources/examples/cli/CMakeLists.txt +8 -0
- data/ext/sources/examples/cli/cli.cpp +1294 -0
- data/ext/sources/examples/coi-serviceworker.js +146 -0
- data/ext/sources/examples/command/CMakeLists.txt +10 -0
- data/ext/sources/examples/command/command.cpp +776 -0
- data/ext/sources/examples/command/commands.txt +9 -0
- data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
- data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
- data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
- data/ext/sources/examples/common-ggml.cpp +238 -0
- data/ext/sources/examples/common-ggml.h +18 -0
- data/ext/sources/examples/common-sdl.cpp +227 -0
- data/ext/sources/examples/common-sdl.h +49 -0
- data/ext/sources/examples/common-whisper.cpp +168 -0
- data/ext/sources/examples/common-whisper.h +24 -0
- data/ext/sources/examples/common.cpp +675 -0
- data/ext/sources/examples/common.h +322 -0
- data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
- data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
- data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
- data/ext/sources/examples/generate-karaoke.sh +57 -0
- data/ext/sources/examples/grammar-parser.cpp +423 -0
- data/ext/sources/examples/grammar-parser.h +29 -0
- data/ext/sources/examples/helpers.js +191 -0
- data/ext/sources/examples/json.hpp +24596 -0
- data/ext/sources/examples/livestream.sh +112 -0
- data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
- data/ext/sources/examples/lsp/lsp.cpp +467 -0
- data/ext/sources/examples/lsp/whisper.vim +362 -0
- data/ext/sources/examples/miniaudio.h +93468 -0
- data/ext/sources/examples/python/test_whisper_processor.py +7 -0
- data/ext/sources/examples/python/whisper_processor.py +54 -0
- data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
- data/ext/sources/examples/quantize/quantize.cpp +223 -0
- data/ext/sources/examples/server/CMakeLists.txt +12 -0
- data/ext/sources/examples/server/bench.js +29 -0
- data/ext/sources/examples/server/httplib.h +10497 -0
- data/ext/sources/examples/server/server.cpp +1091 -0
- data/ext/sources/examples/server.py +115 -0
- data/ext/sources/examples/stb_vorbis.c +5584 -0
- data/ext/sources/examples/stream/CMakeLists.txt +10 -0
- data/ext/sources/examples/stream/stream.cpp +429 -0
- data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
- data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
- data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
- data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
- data/ext/sources/examples/sycl/build.sh +22 -0
- data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
- data/ext/sources/examples/sycl/run-whisper.sh +17 -0
- data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
- data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
- data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
- data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
- data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
- data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
- data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
- data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
- data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
- data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
- data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
- data/ext/sources/examples/talk-llama/llama-context.h +276 -0
- data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
- data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
- data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
- data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
- data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
- data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
- data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
- data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
- data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
- data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
- data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
- data/ext/sources/examples/talk-llama/llama-io.h +35 -0
- data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
- data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
- data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
- data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
- data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
- data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
- data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
- data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
- data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
- data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
- data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
- data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
- data/ext/sources/examples/talk-llama/llama-model.h +425 -0
- data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
- data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
- data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
- data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
- data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
- data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
- data/ext/sources/examples/talk-llama/llama.cpp +354 -0
- data/ext/sources/examples/talk-llama/llama.h +1377 -0
- data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
- data/ext/sources/examples/talk-llama/speak +40 -0
- data/ext/sources/examples/talk-llama/speak.bat +1 -0
- data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
- data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
- data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
- data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
- data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
- data/ext/sources/examples/talk-llama/unicode.h +66 -0
- data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
- data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
- data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
- data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
- data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
- data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
- data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
- data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
- data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
- data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
- data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
- data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
- data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
- data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
- data/ext/sources/ggml/CMakeLists.txt +390 -0
- data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
- data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
- data/ext/sources/ggml/cmake/common.cmake +26 -0
- data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
- data/ext/{ggml → sources/ggml}/include/ggml-alloc.h +1 -1
- data/ext/{ggml → sources/ggml}/include/ggml-backend.h +9 -7
- data/ext/{ggml → sources/ggml}/include/ggml-cpp.h +2 -1
- data/ext/{ggml → sources/ggml}/include/ggml-cpu.h +9 -1
- data/ext/{ggml → sources/ggml}/include/ggml-metal.h +1 -1
- data/ext/{ggml → sources/ggml}/include/ggml-opt.h +49 -28
- data/ext/{ggml → sources/ggml}/include/ggml-rpc.h +6 -1
- data/ext/{ggml → sources/ggml}/include/ggml-vulkan.h +0 -2
- data/ext/{ggml → sources/ggml}/include/ggml.h +182 -265
- data/ext/sources/ggml/include/gguf.h +202 -0
- data/ext/sources/ggml/src/CMakeLists.txt +346 -0
- data/ext/{ggml → sources/ggml}/src/ggml-alloc.c +34 -29
- data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
- data/ext/{ggml → sources/ggml}/src/ggml-backend-impl.h +1 -2
- data/ext/{ggml → sources/ggml}/src/ggml-backend-reg.cpp +87 -53
- data/ext/{ggml → sources/ggml}/src/ggml-backend.cpp +26 -14
- data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
- data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
- data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.cpp +10 -4
- data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.h +5 -5
- data/ext/{ggml → sources/ggml}/src/ggml-cann/aclnn_ops.cpp +1272 -1506
- data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/common.h +135 -1
- data/ext/{ggml → sources/ggml}/src/ggml-cann/ggml-cann.cpp +564 -146
- data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/dup.cpp +3 -5
- data/ext/{ggml → sources/ggml}/src/ggml-common.h +12 -8
- data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.cpp +2 -1
- data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
- data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/cpu-feats-x86.cpp +5 -1
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-impl.h +163 -41
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.c +4029 -1117
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu.cpp +67 -18
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
- data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
- data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
- data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
- data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
- data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
- data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
- data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
- data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
- data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
- data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
- data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
- data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
- data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
- data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
- data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
- data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
- data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
- data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
- data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
- data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
- data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
- data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
- data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
- data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
- data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
- data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
- data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
- data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
- data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
- data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
- data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
- data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
- data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
- data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
- data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
- data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
- data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
- data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
- data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
- data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
- data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
- data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
- data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
- data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
- data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
- data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
- data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
- data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
- data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/cuda.h +1 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/hip.h +57 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/musa.h +7 -1
- data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
- data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
- data/ext/{ggml → sources/ggml}/src/ggml-impl.h +64 -19
- data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
- data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
- data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
- data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.m +2178 -1064
- data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.metal +1575 -1218
- data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
- data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
- data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
- data/ext/{ggml → sources/ggml}/src/ggml-opt.cpp +373 -190
- data/ext/{ggml → sources/ggml}/src/ggml-quants.c +114 -120
- data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
- data/ext/{ggml → sources/ggml}/src/ggml-rpc/ggml-rpc.cpp +480 -73
- data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
- data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
- data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
- data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/common.cpp +20 -32
- data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/concat.cpp +32 -33
- data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/conv.cpp +4 -2
- data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/convert.cpp +104 -28
- data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
- data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
- data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
- data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/dmmv.cpp +156 -17
- data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
- data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
- data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
- data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
- data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
- data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/ggml-sycl.cpp +1004 -1240
- data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
- data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmq.cpp +0 -1
- data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmvq.cpp +261 -166
- data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/norm.cpp +204 -81
- data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/outprod.cpp +8 -17
- data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
- data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
- data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
- data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/softmax.cpp +35 -25
- data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
- data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/tsembd.cpp +3 -3
- data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
- data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
- data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/ext/{ggml → sources/ggml}/src/ggml-vulkan/ggml-vulkan.cpp +3130 -1087
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
- data/ext/{ggml → sources/ggml}/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +193 -35
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/ext/{ggml → sources/ggml}/src/ggml.c +676 -1820
- data/ext/sources/ggml/src/gguf.cpp +1330 -0
- data/ext/{include → sources/include}/whisper.h +68 -2
- data/ext/sources/src/CMakeLists.txt +143 -0
- data/ext/{src → sources/src}/coreml/whisper-decoder-impl.h +27 -15
- data/ext/{src → sources/src}/coreml/whisper-decoder-impl.m +35 -10
- data/ext/{src → sources/src}/coreml/whisper-encoder-impl.h +21 -9
- data/ext/{src → sources/src}/coreml/whisper-encoder-impl.m +28 -3
- data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
- data/ext/sources/src/whisper-arch.h +197 -0
- data/ext/{src → sources/src}/whisper.cpp +1905 -374
- data/ext/sources/tests/CMakeLists.txt +105 -0
- data/ext/sources/tests/earnings21/eval.mk +58 -0
- data/ext/sources/tests/earnings21/eval.py +68 -0
- data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
- data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
- data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
- data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
- data/ext/sources/tests/earnings21/requirements.txt +6 -0
- data/ext/sources/tests/en-0-ref.txt +1 -0
- data/ext/sources/tests/en-1-ref.txt +1 -0
- data/ext/sources/tests/en-2-ref.txt +1 -0
- data/ext/sources/tests/es-0-ref.txt +1 -0
- data/ext/sources/tests/librispeech/eval.mk +39 -0
- data/ext/sources/tests/librispeech/eval.py +47 -0
- data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
- data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
- data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
- data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
- data/ext/sources/tests/librispeech/requirements.txt +6 -0
- data/ext/sources/tests/run-tests.sh +130 -0
- data/ext/sources/tests/test-c.c +3 -0
- data/ext/sources/tests/test-vad-full.cpp +54 -0
- data/ext/sources/tests/test-vad.cpp +83 -0
- data/ext/sources/tests/test-whisper.js +58 -0
- data/extsources.rb +33 -5
- data/lib/whisper/model/uri.rb +149 -128
- data/sig/whisper.rbs +480 -0
- data/tests/helper.rb +28 -0
- data/tests/test_callback.rb +45 -3
- data/tests/test_error.rb +2 -2
- data/tests/test_model.rb +38 -0
- data/tests/test_package.rb +18 -3
- data/tests/test_params.rb +145 -8
- data/tests/test_segment.rb +10 -19
- data/tests/test_vad.rb +19 -0
- data/tests/test_vad_params.rb +103 -0
- data/tests/test_whisper.rb +37 -37
- data/whispercpp.gemspec +5 -4
- metadata +766 -111
- data/ext/cpu.mk +0 -9
- data/ext/examples/dr_wav.h +0 -8815
- data/ext/ggml/src/ggml-cann/aclnn_ops.h +0 -592
- data/ext/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +0 -4262
- data/ext/ggml/src/ggml-cpu/ggml-cpu.c +0 -14123
- data/ext/ggml/src/ggml-cpu/llamafile/sgemm.cpp +0 -1884
- data/ext/ggml/src/ggml-cpu/llamafile/sgemm.h +0 -14
- data/ext/ggml/src/ggml-metal/ggml-metal-impl.h +0 -288
- data/ext/ggml/src/ggml-sycl/element_wise.cpp +0 -1030
- data/ext/ggml/src/ggml-sycl/im2col.cpp +0 -126
- data/ext/ggml/src/ggml-sycl/rope.cpp +0 -276
- data/ext/ggml/src/ggml-sycl/wkv6.cpp +0 -141
- data/ext/metal-embed.mk +0 -17
- data/ext/metal.mk +0 -6
- data/ext/ruby_whisper.cpp +0 -1909
- data/ext/scripts/get-flags.mk +0 -38
- data/lib/whisper.rb +0 -2
- /data/ext/{ggml → sources/ggml}/include/ggml-blas.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-cann.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-cuda.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-kompute.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-opencl.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-sycl.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/common.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/ggml-amx.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-blas/ggml-blas.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/ascendc_kernels.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f16.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f32.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q4_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/common.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-aarch64.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-kompute/ggml-kompute.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-quants.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-threading.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-threading.h +0 -0
- /data/ext/{src → sources/src}/coreml/whisper-encoder.h +0 -0
- /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.cpp +0 -0
- /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.h +0 -0
@@ -0,0 +1,1125 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2023-2024 The ggml authors
|
3
|
+
*
|
4
|
+
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
5
|
+
* of this software and associated documentation files (the "Software"), to
|
6
|
+
* deal in the Software without restriction, including without limitation the
|
7
|
+
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
8
|
+
* sell copies of the Software, and to permit persons to whom the Software is
|
9
|
+
* furnished to do so, subject to the following conditions:
|
10
|
+
*
|
11
|
+
* The above copyright notice and this permission notice shall be included in
|
12
|
+
* all copies or substantial portions of the Software.
|
13
|
+
*
|
14
|
+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
15
|
+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
16
|
+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
17
|
+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
18
|
+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
19
|
+
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
20
|
+
* IN THE SOFTWARE.
|
21
|
+
*/
|
22
|
+
|
23
|
+
#ifndef CANN_ACLNN_OPS
|
24
|
+
#define CANN_ACLNN_OPS
|
25
|
+
|
26
|
+
#include <functional>
|
27
|
+
#include <aclnnop/aclnn_abs.h>
|
28
|
+
#include <aclnnop/aclnn_neg.h>
|
29
|
+
#include <aclnnop/aclnn_exp.h>
|
30
|
+
#include <aclnnop/aclnn_arange.h>
|
31
|
+
#include <aclnnop/aclnn_argsort.h>
|
32
|
+
#include <aclnnop/aclnn_cat.h>
|
33
|
+
#include <aclnnop/aclnn_clamp.h>
|
34
|
+
#include <aclnnop/aclnn_gelu.h>
|
35
|
+
#include <aclnnop/aclnn_gelu_v2.h>
|
36
|
+
#include <aclnnop/aclnn_sigmoid.h>
|
37
|
+
#include <aclnnop/aclnn_hardsigmoid.h>
|
38
|
+
#include <aclnnop/aclnn_hardswish.h>
|
39
|
+
#include <aclnnop/aclnn_leaky_relu.h>
|
40
|
+
#include <aclnnop/aclnn_relu.h>
|
41
|
+
#include <aclnnop/aclnn_silu.h>
|
42
|
+
#include <aclnnop/aclnn_tanh.h>
|
43
|
+
#include <aclnnop/aclnn_sqrt.h>
|
44
|
+
#include <aclnnop/aclnn_sin.h>
|
45
|
+
#include <aclnnop/aclnn_cos.h>
|
46
|
+
#include <aclnnop/aclnn_log.h>
|
47
|
+
#include <aclnnop/aclnn_sign.h>
|
48
|
+
#include "acl_tensor.h"
|
49
|
+
#include "common.h"
|
50
|
+
|
51
|
+
/**
|
52
|
+
* @brief Repeats a ggml tensor along each dimension to match the dimensions
|
53
|
+
* of another tensor.
|
54
|
+
*
|
55
|
+
* @details This function repeats the elements of a source ggml tensor along
|
56
|
+
* each dimension to create a destination tensor with the specified
|
57
|
+
* dimensions. The operation is performed using the ACL backend and
|
58
|
+
* executed asynchronously on the device.
|
59
|
+
*
|
60
|
+
* @param ctx The CANN context used for operations.
|
61
|
+
* @param dst The ggml tensor representing the destination, which op is
|
62
|
+
* GGML_OP_REPEAT and specifies the desired dimensions.
|
63
|
+
*/
|
64
|
+
void ggml_cann_repeat(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
65
|
+
|
66
|
+
/**
|
67
|
+
* @brief Applies the Leaky ReLU activation function to a tensor using the CANN
|
68
|
+
* backend.
|
69
|
+
*
|
70
|
+
* @details This function computes the Leaky ReLU activation for each element of
|
71
|
+
* the input tensor. The Leaky ReLU function allows a small gradient
|
72
|
+
* when the unit is not active (i.e., when the input is negative). The
|
73
|
+
* Leaky ReLU function is defined as:
|
74
|
+
* \f[
|
75
|
+
* \text{dst} = \max(0, src) + \text{negativeSlope} \cdot \min(0,
|
76
|
+
* src)
|
77
|
+
* \f]
|
78
|
+
* `negativeSlope` is in dst->params.
|
79
|
+
*
|
80
|
+
* @param ctx The CANN context used for operations.
|
81
|
+
* @param dst The destination tensor where the result of the Leaky ReLU
|
82
|
+
* activation is stored, which op is `GGML_OP_LEAKY_RELU`
|
83
|
+
*/
|
84
|
+
void ggml_cann_leaky_relu(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
85
|
+
|
86
|
+
/**
|
87
|
+
* @brief Concatenates multiple tensors along a specified dimension using the
|
88
|
+
* CANN backend.
|
89
|
+
*
|
90
|
+
* @param ctx The CANN context used for operations.
|
91
|
+
* @param tensorList A pointer to the list of tensors to be concatenated.
|
92
|
+
* @param dst The destination tensor where the result of the
|
93
|
+
* concatenation is stored. dst->op is `GGML_OP_CONCAT`.
|
94
|
+
* @param concat_dim The dimension along which the tensors are concatenated.
|
95
|
+
*
|
96
|
+
* @attention tensorList length should be 2 and the dimension using for concat
|
97
|
+
* default to 1.
|
98
|
+
*/
|
99
|
+
void ggml_cann_concat(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
100
|
+
|
101
|
+
/**
|
102
|
+
* @brief Generates a sequence of evenly spaced values within a specified
|
103
|
+
* interval for a ggml tensor using the CANN backend.
|
104
|
+
*
|
105
|
+
* @details This function creates a sequence of numbers over a specified i
|
106
|
+
* nterval, starting from `start`, ending before `stop`, and
|
107
|
+
* incrementing by `step`. The sequence is stored in the destination
|
108
|
+
* tensor `dst`.
|
109
|
+
*
|
110
|
+
* @param ctx The CANN context used for operations.
|
111
|
+
* @param dst The destination tensor where the generated sequence will be stored.
|
112
|
+
* `start`, 'stop' and 'step' are in dst->op_params and dst->op is
|
113
|
+
* `GGML_OP_ARANGE`.
|
114
|
+
*/
|
115
|
+
void ggml_cann_arange(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
116
|
+
|
117
|
+
/**
|
118
|
+
* @brief Applies a clamp operation to the elements of a ggml tensor using the
|
119
|
+
* CANN backend.
|
120
|
+
*
|
121
|
+
* @details This function clamps the elements of the input tensor `src` to a
|
122
|
+
* specified range defined by `min` and `max` values. The result is
|
123
|
+
* stored in the destination tensor `dst`. The operation is defined as:
|
124
|
+
* \f[
|
125
|
+
* y = \max(\min(x, max\_value), min\_value)
|
126
|
+
* \f]
|
127
|
+
* where `x` is an element of the input tensor, and `y` is the
|
128
|
+
* corresponding element in the output tensor.
|
129
|
+
* @param ctx The CANN context used for operations.
|
130
|
+
* @param dst The destination tensor where the clamped values will be stored.
|
131
|
+
* dst->op is `GGML_OP_CLAMP`, `min` and `max` value is in dst->params.
|
132
|
+
*/
|
133
|
+
void ggml_cann_clamp(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
134
|
+
|
135
|
+
/**
|
136
|
+
* @brief Scales the elements of a ggml tensor by a constant factor using the
|
137
|
+
* CANN backend.
|
138
|
+
*
|
139
|
+
* @details This function multiplies each element of the input tensor `src` by
|
140
|
+
* a scaling factor `scale`, storing the result in the destination
|
141
|
+
* tensor `dst`. The operation is defined as:
|
142
|
+
* \f[
|
143
|
+
* dst = src \times scale
|
144
|
+
* \f]
|
145
|
+
*
|
146
|
+
* @param ctx The CANN context used for operations.
|
147
|
+
* @param dst The destination tensor where the scaled values will be stored.
|
148
|
+
* dst->op is `GGML_OP_SCALE` and `scale` value is in dst->params.
|
149
|
+
*/
|
150
|
+
void ggml_cann_scale(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
151
|
+
|
152
|
+
/**
|
153
|
+
* @brief Sorts the elements of a ggml tensor and returns the indices that
|
154
|
+
* would sort the tensor using the CANN backend.
|
155
|
+
*
|
156
|
+
* @details This function performs an argsort operation on the input tensor
|
157
|
+
* `src`. It sorts the elements of `src` in either ascending or
|
158
|
+
* descending order, depending on the `GGML_SORT_ORDER_DESC`,
|
159
|
+
* and returns the indices that would sort the original tensor.
|
160
|
+
*
|
161
|
+
* @param ctx The CANN context used for operations.
|
162
|
+
* @param dst The destination tensor where the sorted indices will be stored.
|
163
|
+
* dst->op is `GGML_OP_ARGSORT`.
|
164
|
+
*/
|
165
|
+
void ggml_cann_argsort(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
166
|
+
|
167
|
+
/**
|
168
|
+
* @brief Computes the Layer Normalization for a ggml tensor using the CANN
|
169
|
+
* backend.
|
170
|
+
*
|
171
|
+
* @details This function applies the Layer Normalization operation on the
|
172
|
+
* input tensor `src` and stores the result in the destination tensor
|
173
|
+
* `dst`. Layer Normalization normalizes the features at each sample in
|
174
|
+
* a mini-batch independently. It is commonly used in neural networks
|
175
|
+
* to normalize the activations of a layer by adjusting and scaling
|
176
|
+
* the outputs.
|
177
|
+
* The operation is defined as:
|
178
|
+
* \f[
|
179
|
+
* \text { out }=\frac{x-\mathrm{E}[x]}{\sqrt{\text{Var}[x]+eps}}
|
180
|
+
* \f]
|
181
|
+
* `Var` defaults dst->ne[0]. `eps` is in dst->params.
|
182
|
+
*
|
183
|
+
* @param ctx The CANN context used for operations.
|
184
|
+
* @param dst The destination tensor where the normalized values will be stored.
|
185
|
+
* @attention `Var` defaults to dst->ne[0].
|
186
|
+
*/
|
187
|
+
void ggml_cann_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
188
|
+
|
189
|
+
/**
|
190
|
+
* @brief Computes the Group Normalization for a ggml tensor using the CANN
|
191
|
+
* backend.
|
192
|
+
*
|
193
|
+
* @brief This function applies the Group Normalization operation on the input
|
194
|
+
* tensor `src` and stores the result in the destination tensor `dst`.
|
195
|
+
* Group Normalization divides the channels into groups and normalizes
|
196
|
+
* the features within each group across spatial locations.
|
197
|
+
* It is commonly used in convolutional neural networks to improve
|
198
|
+
* training stability and performance.
|
199
|
+
* The operation is defined as:
|
200
|
+
* \f[
|
201
|
+
* \text { out }=\frac{x-\mathrm{E}[x]}{\sqrt{\text{Var}[x]+eps}}
|
202
|
+
* \f]
|
203
|
+
*
|
204
|
+
* @param ctx The CANN context used for operations.
|
205
|
+
* @param dst The destination tensor where the normalized values will be stored.
|
206
|
+
* `n_groups` is in dst->params, which split C channel to `n_groups`.
|
207
|
+
* dst->op is `GGML_OP_GROUP_NORM`.
|
208
|
+
*
|
209
|
+
* @attention eps defaults to 1e-6f.
|
210
|
+
*/
|
211
|
+
void ggml_cann_group_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
212
|
+
|
213
|
+
/**
|
214
|
+
* @brief Computes the accumulation of tensors using the CANN backend.
|
215
|
+
*
|
216
|
+
* @details This function performs an accumulation operation on two tensors.
|
217
|
+
* Depending on the `inplace` flag, it either updates the destination
|
218
|
+
* tensor `dst` in place by adding `alpha * src1` to it, or it creates
|
219
|
+
* a new tensor as the result of `src0 + alpha * src1` and stores it in
|
220
|
+
* `dst`.
|
221
|
+
* The operation is defined as:
|
222
|
+
* \f[
|
223
|
+
* dst = src0 + alpha \times src1
|
224
|
+
* \f]
|
225
|
+
* if `inplace` is `true`, `src0` is equal to 'dst'.
|
226
|
+
* @param ctx The CANN context used for operations.
|
227
|
+
* @param dst The destination tensor where the accumulated values will be stored.
|
228
|
+
* `inplace` is in dst->params, and dst->op is `GGML_OP_ACC`.
|
229
|
+
*/
|
230
|
+
void ggml_cann_acc(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
231
|
+
|
232
|
+
/**
|
233
|
+
* @brief Computes the sum of elements along the last dimension of a ggml tensor
|
234
|
+
* using the CANN backend.
|
235
|
+
*
|
236
|
+
* @details This function performs a reduction sum operation along the last
|
237
|
+
* dimension of the input tensor `src`. The result of the sum is stored
|
238
|
+
* in the destination tensor `dst`.
|
239
|
+
*
|
240
|
+
* @param ctx The CANN context used for operations.
|
241
|
+
* @param dst The destination tensor where the reduced values will be stored。
|
242
|
+
* dst->op is `GGML_OP_SUM_ROWS`.
|
243
|
+
*
|
244
|
+
* @attention `reduce_dims` defaults to 3, which means the last dimension.
|
245
|
+
*/
|
246
|
+
void ggml_cann_sum_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
247
|
+
|
248
|
+
/**
|
249
|
+
* @brief Computes the sum of elements in a ggml tensor.
|
250
|
+
*
|
251
|
+
* @details This function performs a reduction sum operation along the last
|
252
|
+
* dimension of the input tensor `src`. The result of the sum is stored
|
253
|
+
* in the destination tensor `dst`.
|
254
|
+
*
|
255
|
+
* @param ctx The CANN context used for operations.
|
256
|
+
* @param dst The destination tensor where the reduced values will be stored。
|
257
|
+
*
|
258
|
+
*/
|
259
|
+
|
260
|
+
void ggml_cann_sum(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
261
|
+
|
262
|
+
/**
|
263
|
+
* @brief Upsamples a ggml tensor using nearest neighbor interpolation using
|
264
|
+
* the CANN backend.
|
265
|
+
*
|
266
|
+
* @details This function performs upsampling of the input tensor `src` using
|
267
|
+
* nearest neighbor interpolation. The upsampling is applied to the
|
268
|
+
* height and width dimensions (last two dimensions) of the tensor. The
|
269
|
+
* result is stored in the destination tensor `dst`, which must have
|
270
|
+
* the appropriate dimensions for the upsampled output.
|
271
|
+
*
|
272
|
+
* @param ctx The CANN context used for operations.
|
273
|
+
* @param dst The destination tensor where the upsampled values will be stored.
|
274
|
+
* dst->op is `GGML_OP_UPSCALE`.
|
275
|
+
*/
|
276
|
+
void ggml_cann_upsample_nearest2d(ggml_backend_cann_context& ctx,
|
277
|
+
ggml_tensor* dst);
|
278
|
+
|
279
|
+
/**
|
280
|
+
* @brief Pads a ggml tensor to match the dimensions of the destination tensor
|
281
|
+
* using the CANN backend.
|
282
|
+
*
|
283
|
+
* @details This function pads the input tensor `src` so that it matches the
|
284
|
+
* dimensions of the destination tensor `dst`. The amount of padding
|
285
|
+
* is calculated based on the difference in sizes between `src` and
|
286
|
+
* `dst` along each dimension. The padded tensor is stored in `dst`.
|
287
|
+
*
|
288
|
+
* @param ctx The CANN context used for operations.
|
289
|
+
* @param dst The destination tensor, which specifies the target dimensions for
|
290
|
+
* padding. dst->op is `GGML_OP_PAD`.
|
291
|
+
*/
|
292
|
+
void ggml_cann_pad(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
293
|
+
|
294
|
+
/**
|
295
|
+
* @brief Executes a 2D pooling operation on a ggml tensor using the CANN
|
296
|
+
* backend.
|
297
|
+
*
|
298
|
+
* @details This function dispatches the execution of a 2D pooling operation on
|
299
|
+
* the input tensor `dst`. The type of pooling (average or max) is
|
300
|
+
* determined by the `op` parameter, which is read from the operation
|
301
|
+
* parameters of `dst`. The function supports average pooling
|
302
|
+
* (`GGML_OP_POOL_AVG`) and max pooling (`GGML_OP_POOL_MAX`). If an
|
303
|
+
* invalid operation is encountered, the function asserts a failure.
|
304
|
+
*
|
305
|
+
* @param ctx The CANN context used for operations.
|
306
|
+
* @param dst The destination tensor on which the pooling operation is to be
|
307
|
+
* performed. dst->op is `GGML_OP_POOL_2D`.
|
308
|
+
*/
|
309
|
+
void ggml_cann_pool2d(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
310
|
+
|
311
|
+
/**
|
312
|
+
* @brief Duplicates a ggml tensor using the CANN backend.
|
313
|
+
*
|
314
|
+
* @details This function duplicates the contents of the source tensor `src` to
|
315
|
+
* the destination tensor `dst`. The function supports various tensor
|
316
|
+
* types and configurations, including handling of extra data, type
|
317
|
+
* conversions, and special cases for contiguous and non-contiguous
|
318
|
+
* tensors.
|
319
|
+
*
|
320
|
+
* @param ctx The CANN context used for operations.
|
321
|
+
* @param dst The destination tensor where the duplicated data will be stored.
|
322
|
+
* dst->op is `GGML_OP_DUP`
|
323
|
+
*
|
324
|
+
* @attention Only support Fp16/FP32. Not support when src and dst have
|
325
|
+
* different shape and dst is no-contiguous.
|
326
|
+
* @note: This func need to simplify.
|
327
|
+
*/
|
328
|
+
void ggml_cann_dup(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
329
|
+
|
330
|
+
/**
|
331
|
+
* @brief Computes the Root Mean Square (RMS) normalization of a ggml tensor
|
332
|
+
* using the CANN backend.
|
333
|
+
*
|
334
|
+
* @details This function applies RMS normalization to the input tensor `src`
|
335
|
+
* and stores the result in the destination tensor `dst`. RMS
|
336
|
+
* normalization involves computing the root mean square of the input
|
337
|
+
* tensor along a specified dimension and then dividing each element of
|
338
|
+
* the tensor by this value, adjusted by a small epsilon value to
|
339
|
+
* prevent division by zero.
|
340
|
+
* The operation is defined as:
|
341
|
+
* \f[
|
342
|
+
* \text{RmsNorm}\left(x_i\right)=\frac{x_i}{\text{Rms}(\mathbf{x})} g_i,
|
343
|
+
* \quad \text { where } \text{Rms}(\mathbf{x})=\sqrt{\frac{1}{n} \sum_{i=1}^n x_i^2+e p s}
|
344
|
+
* \f]
|
345
|
+
* `eps` is in dst->op_params.
|
346
|
+
* @param ctx The CANN context used for operations.
|
347
|
+
* @param dst The destination tensor where the normalized values will be stored.
|
348
|
+
* dst->op is `GGML_OP_RMS_NORM`.
|
349
|
+
*/
|
350
|
+
void ggml_cann_rms_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
351
|
+
|
352
|
+
/**
|
353
|
+
* @brief Applies a diagonal mask to the tensor with a specified value.
|
354
|
+
*
|
355
|
+
* @details This function creates a mask tensor filled with ones, then applies
|
356
|
+
* an upper triangular and lower triangular operation to it based on
|
357
|
+
* the number of past elements specified. Afterward, it adds the masked
|
358
|
+
* tensor to the destination tensor in-place.
|
359
|
+
*
|
360
|
+
* @param ctx The backend CANN context used for operations.
|
361
|
+
* @param dst The destination tensor where the result will be stored. dst->op is
|
362
|
+
* `GGML_OP_DIAG_MASK`
|
363
|
+
* @param value The value to use for masking.
|
364
|
+
*/
|
365
|
+
void ggml_cann_diag_mask(ggml_backend_cann_context& ctx, ggml_tensor* dst, float value);
|
366
|
+
|
367
|
+
/**
|
368
|
+
* @brief Performs an image-to-column transformation on the input tensor.
|
369
|
+
*
|
370
|
+
* @details This function takes an input tensor and applies an image-to-column
|
371
|
+
* operation, converting spatial dimensions into column-like
|
372
|
+
* structures suitable for convolutional operations. It supports both
|
373
|
+
* half-precision (F16) and single-precision (F32) floating-point data
|
374
|
+
* types.
|
375
|
+
*
|
376
|
+
* @param ctx The backend CANN context for executing operations.
|
377
|
+
* @param dst The destination tensor that stores the result of the operation.
|
378
|
+
* dst->op is `GGML_OP_IM2COL`.
|
379
|
+
*/
|
380
|
+
void ggml_cann_im2col(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
381
|
+
|
382
|
+
/**
|
383
|
+
* @brief Computes time step embeddings using sine and cosine functions.
|
384
|
+
*
|
385
|
+
* @details This function calculates time step embeddings by applying sine and
|
386
|
+
* cosine transformations to a given input tensor, which is typically
|
387
|
+
* used in temporal models like diffusion models or transformers to
|
388
|
+
* encode time information effectively.
|
389
|
+
*
|
390
|
+
* @param ctx The backend CANN context for executing operations.
|
391
|
+
* @param dst The destination tensor where the result of the embedding operation
|
392
|
+
* will be stored. dst->op is `GGML_OP_TIMESTEP_EMBEDDING`.
|
393
|
+
*/
|
394
|
+
void ggml_cann_timestep_embedding(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
395
|
+
|
396
|
+
// @see ggml_cann_dup.
|
397
|
+
void ggml_cann_cpy(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
398
|
+
|
399
|
+
/**
|
400
|
+
* @brief Computes the softmax activation with optional masking.
|
401
|
+
*
|
402
|
+
* @details This function computes the softmax activation over the input tensor,
|
403
|
+
* optionally applying a mask and scaling factor. It supports both FP16
|
404
|
+
* and FP32 data types and can handle masking by broadcasting the mask
|
405
|
+
* across rows if necessary.
|
406
|
+
* The function performs the following steps:
|
407
|
+
* 1. Multiplies the input tensor by a scale factor.
|
408
|
+
* 2. Optionally casts the mask tensor to FP32 if it is in FP16 format.
|
409
|
+
* 3. Broadcasts the mask tensor if its dimensions do not match the
|
410
|
+
* input tensor's dimensions.
|
411
|
+
* 4. Adds the mask to the scaled input tensor.
|
412
|
+
* 5. Applies the softmax activation function along the specified
|
413
|
+
* dimension.
|
414
|
+
*
|
415
|
+
* @param ctx The backend CANN context for executing operations.
|
416
|
+
* @param dst The destination tensor where the result will be stored. dst->op is
|
417
|
+
* `GGML_OP_SOFTMAX`.
|
418
|
+
*/
|
419
|
+
void ggml_cann_softmax(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
420
|
+
|
421
|
+
/**
|
422
|
+
* @brief Extracts specific rows from a tensor based on indices.
|
423
|
+
*
|
424
|
+
* @details This function retrieves rows from a source tensor src0 according to
|
425
|
+
* the indices provided in another tensor src1 and stores the result in
|
426
|
+
* a destination tensor (\p dst). It supports different data types
|
427
|
+
* including F32, F16, Q4_0, and Q8_0.
|
428
|
+
*
|
429
|
+
* @param ctx The backend CANN context for executing operations.
|
430
|
+
* @param dst The destination tensor where the extracted rows will be stored.
|
431
|
+
* dst->op is `GGML_OP_GET_ROWS`.
|
432
|
+
*/
|
433
|
+
void ggml_cann_get_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
434
|
+
|
435
|
+
/**
|
436
|
+
* @brief Executes matrix multiplication for the given tensor.
|
437
|
+
*
|
438
|
+
* @details This function performs matrix multiplication on the source tensors
|
439
|
+
* associated with the destination tensor. It supports matrix
|
440
|
+
* multiplication F32, F16, and Q8_0.
|
441
|
+
*
|
442
|
+
* @param ctx The backend CANN context for executing operations.
|
443
|
+
* @param dst The destination tensor for storing the result of the matrix
|
444
|
+
* multiplication. dst->op is `GGML_OP_MUL_MAT`.
|
445
|
+
*/
|
446
|
+
void ggml_cann_mul_mat(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
447
|
+
|
448
|
+
/**
|
449
|
+
* @brief Applies Rotary Positional Embedding (RoPE) to the input tensor.
|
450
|
+
*
|
451
|
+
* @details This function implements the RoPE mechanism, which is a method to
|
452
|
+
* encode positional information into sequence data, particularly
|
453
|
+
* useful in transformer models. It supports both F32 and F16 data
|
454
|
+
* types.
|
455
|
+
*
|
456
|
+
* @param ctx The backend CANN context for executing operations.
|
457
|
+
* @param dst The destination tensor where the RoPE-transformed data will be
|
458
|
+
* stored. dst->op is `GGML_OP_ROPE`.
|
459
|
+
*
|
460
|
+
* @note The function currently does not support cases where the n_dims is less
|
461
|
+
* than the input tensor's first dimension.
|
462
|
+
* @note The function currently does not support cases where the freq_factors is
|
463
|
+
* not NULL.
|
464
|
+
* @note The function currently does not support cases where the ext_factor is
|
465
|
+
* not equal 0.
|
466
|
+
* @note The function currently does not support cases where the freq_scale is
|
467
|
+
* not equal 1.
|
468
|
+
*/
|
469
|
+
void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
470
|
+
|
471
|
+
/**
|
472
|
+
* @brief Computes the index of the maximum value along the specified dimension
|
473
|
+
* of a ggml tensor using the CANN backend.
|
474
|
+
*
|
475
|
+
* @details This function performs an argmax operation on the input tensor.
|
476
|
+
* It finds the index of the maximum value along the specified axis
|
477
|
+
* and stores these indices in the destination tensor `dst`. The
|
478
|
+
* operation is executed using the CANN backend for optimized performance.
|
479
|
+
*
|
480
|
+
* @param ctx The CANN context used for operations.
|
481
|
+
* @param dst The destination tensor where the indices of the maximum values will
|
482
|
+
* be stored. dst->op is `GGML_OP_ARGMAX`.
|
483
|
+
*/
|
484
|
+
void ggml_cann_argmax(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
485
|
+
|
486
|
+
/**
|
487
|
+
* @brief Adds two tensors element-wise and stores the result in a destination
|
488
|
+
* tensor.
|
489
|
+
*
|
490
|
+
* This function performs the operation:
|
491
|
+
* \f[
|
492
|
+
* dst = acl\_src0 + alpha \times acl\_src1
|
493
|
+
* \f]
|
494
|
+
* where alpha is a scalar value and defaults to 1.0f.
|
495
|
+
*
|
496
|
+
* @param ctx The context for the CANN backend operations.
|
497
|
+
* @param acl_src0 The first source tensor.
|
498
|
+
* @param acl_src1 The second source tensor.
|
499
|
+
* @param acl_dst The destination tensor where the result will be stored.
|
500
|
+
*/
|
501
|
+
void aclnn_add(ggml_backend_cann_context& ctx, aclTensor* acl_src0,
|
502
|
+
aclTensor* acl_src1, aclTensor* acl_dst = nullptr);
|
503
|
+
|
504
|
+
/**
|
505
|
+
* @brief Sub two tensors element-wise and stores the result in a destination
|
506
|
+
* tensor.
|
507
|
+
*
|
508
|
+
* This function performs the operation:
|
509
|
+
* \f[
|
510
|
+
* dst = acl\_src0 - alpha \times acl\_src1
|
511
|
+
* \f]
|
512
|
+
* where alpha is a scalar value and defaults to 1.0f.
|
513
|
+
*
|
514
|
+
* @param ctx The context for the CANN backend operations.
|
515
|
+
* @param acl_src0 The first source tensor.
|
516
|
+
* @param acl_src1 The second source tensor.
|
517
|
+
* @param acl_dst The destination tensor where the result will be stored.
|
518
|
+
*/
|
519
|
+
void aclnn_sub(ggml_backend_cann_context& ctx, aclTensor* acl_src0,
|
520
|
+
aclTensor* acl_src1, aclTensor* acl_dst = nullptr);
|
521
|
+
|
522
|
+
/**
|
523
|
+
* @brief Performs element-wise multiplication of two tensors and stores the
|
524
|
+
* result in a destination tensor.
|
525
|
+
*
|
526
|
+
* This function performs element-wise multiplication of the tensors `acl_src`
|
527
|
+
* and `acl_other` and stores the result in the destination tensor `acl_dst`.
|
528
|
+
* The operation is defined as:
|
529
|
+
* \f[
|
530
|
+
* \text {acl_dst }_i=\text {acl_src }_i \times \text {acl_other }_i
|
531
|
+
* \f]
|
532
|
+
*
|
533
|
+
* @param ctx The context for the CANN backend operations.
|
534
|
+
* @param acl_src The first tensor for element-wise multiplication.
|
535
|
+
* @param acl_other The second tensor for element-wise multiplication.
|
536
|
+
* @param acl_dst The destination tensor where the result will be stored.
|
537
|
+
*/
|
538
|
+
void aclnn_mul(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
539
|
+
aclTensor* acl_other, aclTensor* acl_dst = nullptr);
|
540
|
+
|
541
|
+
/**
|
542
|
+
* @brief Matrix division, optionally in-place.
|
543
|
+
*
|
544
|
+
* This function division each element of the source tensor `acl_src` by the
|
545
|
+
* tensor `acl_other` and stores the result in the destination tensor `acl_dst`.
|
546
|
+
* If `inplace` is true, `acl_dst` will not be used and the operation is
|
547
|
+
* performed in-place on `acl_src`. The operation is defined as: \f[
|
548
|
+
* \text{dst}_i = \frac{\text{acl_src}_i}{\text{acl_other}_i}
|
549
|
+
* \f]
|
550
|
+
*
|
551
|
+
* @param ctx The context for the CANN backend operations.
|
552
|
+
* @param acl_src Numerator tensor..
|
553
|
+
* @param acl_other Denominator tensor.
|
554
|
+
* @param acl_dst The destination tensor where the result will be stored if
|
555
|
+
* `inplace` is false.
|
556
|
+
* @param inplace Flag indicating whether to perform the operation in-place on
|
557
|
+
* `acl_src`.
|
558
|
+
*/
|
559
|
+
void aclnn_div(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
560
|
+
aclTensor* acl_other, aclTensor* acl_dst = nullptr);
|
561
|
+
|
562
|
+
/**
|
563
|
+
* @brief Applies element-wise cosine function to the elements of a tensor.
|
564
|
+
*
|
565
|
+
* This function computes the cosine of each element in the source tensor
|
566
|
+
* `acl_src` and stores the result in the destination tensor `acl_dst`. The
|
567
|
+
* operation is defined as: \f[ \text {acl_dst }_i=\cos \left(\text {acl_src
|
568
|
+
* }_i\right) \f]
|
569
|
+
*
|
570
|
+
* @param ctx The context for the CANN backend operations.
|
571
|
+
* @param acl_src The source tensor on which the cosine function will be
|
572
|
+
* applied.
|
573
|
+
* @param acl_dst The destination tensor where the cosine results will be
|
574
|
+
* stored.
|
575
|
+
*/
|
576
|
+
void aclnn_cos(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
577
|
+
aclTensor* acl_dst);
|
578
|
+
|
579
|
+
/**
|
580
|
+
* @brief Applies element-wise sine function to the elements of a tensor.
|
581
|
+
*
|
582
|
+
* This function computes the sine of each element in the source tensor
|
583
|
+
`acl_src`
|
584
|
+
* and stores the result in the destination tensor `acl_dst`.
|
585
|
+
* The operation is defined as:
|
586
|
+
* \f[
|
587
|
+
* \text {acl_dst }_i=\sin \left(\text {acl_src }_i\right)
|
588
|
+
* \f]
|
589
|
+
|
590
|
+
* @param ctx The context for the CANN backend operations.
|
591
|
+
* @param acl_src The source tensor on which the sine function will be applied.
|
592
|
+
* @param acl_dst The destination tensor where the sine results will be stored.
|
593
|
+
*/
|
594
|
+
void aclnn_sin(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
595
|
+
aclTensor* acl_dst);
|
596
|
+
|
597
|
+
/**
|
598
|
+
* @brief Prepares broadcast-compatible ACL tensors for two input tensors and one
|
599
|
+
* output tensor.
|
600
|
+
*
|
601
|
+
* This function checks whether broadcasting is needed between `src0` and `src1`.
|
602
|
+
* If broadcasting is required, it calculates the proper shapes and creates
|
603
|
+
* ACL tensors with broadcast parameters. Otherwise, it directly creates ACL tensors
|
604
|
+
* based on the original tensor shapes.
|
605
|
+
*
|
606
|
+
* @param src0 The first input tensor (reference shape).
|
607
|
+
* @param src1 The second input tensor (possibly broadcasted).
|
608
|
+
* @param dst The destination/output tensor.
|
609
|
+
* @param acl_src0 Output pointer to the created ACL tensor corresponding to src0.
|
610
|
+
* @param acl_src1 Output pointer to the created ACL tensor corresponding to src1.
|
611
|
+
* @param acl_dst Output pointer to the created ACL tensor corresponding to dst.
|
612
|
+
*/
|
613
|
+
void bcast_shape(ggml_tensor * src0, ggml_tensor * src1, ggml_tensor * dst,
|
614
|
+
aclTensor ** acl_src0, aclTensor ** acl_src1, aclTensor ** acl_dst);
|
615
|
+
|
616
|
+
/**
|
617
|
+
* @brief Computes the 1D transposed convolution (deconvolution) of a ggml
|
618
|
+
* tensor using the CANN backend.
|
619
|
+
*
|
620
|
+
* @details This function performs a 1D transposed convolution (also known as
|
621
|
+
* deconvolution) operation on the input tensor. The computed result is stored
|
622
|
+
* in the destination tensor `dst`. The operation is optimized using the CANN
|
623
|
+
* backend for improved performance.
|
624
|
+
*
|
625
|
+
* @param ctx The CANN context used for operations.
|
626
|
+
* @param dst The destination tensor where the transposed convolution result
|
627
|
+
* will be stored. dst->op is `GGML_OP_CONV_TRANSPOSE_1D`.
|
628
|
+
*/
|
629
|
+
void ggml_cann_conv_transpose_1d(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
630
|
+
|
631
|
+
/**
|
632
|
+
* @brief Applies the ELU (Exponential Linear Unit) activation to a ggml tensor
|
633
|
+
* using the CANN backend.
|
634
|
+
*
|
635
|
+
* @details This function performs an element-wise ELU activation on the input
|
636
|
+
* tensor.
|
637
|
+
* The result is written to the destination tensor `dst` in-place.
|
638
|
+
* The ELU function is defined as:
|
639
|
+
*
|
640
|
+
* \text{ELU}(x) =
|
641
|
+
* \begin{cases}
|
642
|
+
* x, & \text{if } x > 0 \\
|
643
|
+
* \alpha \left( \exp(x) - 1 \right), & \text{if } x \leq 0
|
644
|
+
* \end{cases}
|
645
|
+
*
|
646
|
+
* where α (alpha) is a hyperparameter, typically set to 1.0.
|
647
|
+
* This operation is optimized using the CANN backend for high-performance
|
648
|
+
* inference or training.
|
649
|
+
*
|
650
|
+
* @param ctx The CANN context used for operations.
|
651
|
+
* @param dst The destination tensor where the ELU-activated result will be stored.
|
652
|
+
* dst->op is expected to be `GGML_OP_ELU`.
|
653
|
+
*/
|
654
|
+
void ggml_cann_elu(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
655
|
+
|
656
|
+
/**
|
657
|
+
* @brief Computes the mean of a ggml tensor element-wise using the CANN backend.
|
658
|
+
*
|
659
|
+
* @details This function calculates the element-wise mean of the input tensor.
|
660
|
+
* The result is written to the destination tensor `dst`.
|
661
|
+
* The mean is computed by averaging the values across the entire tensor.
|
662
|
+
*
|
663
|
+
* This operation is optimized using the CANN backend for high-performance inference or training.
|
664
|
+
*
|
665
|
+
* @param ctx The CANN context used for operations.
|
666
|
+
* @param dst The destination tensor where the mean result will be stored.
|
667
|
+
* dst->op is expected to be `GGML_OP_MEAN`.
|
668
|
+
*/
|
669
|
+
void ggml_cann_mean(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
670
|
+
|
671
|
+
/**
|
672
|
+
* @brief Applies 1D reflect padding to a ggml tensor using the CANN backend.
|
673
|
+
*
|
674
|
+
* @details This function performs 1D reflect padding on the input tensor.
|
675
|
+
* The amount of padding on each side is specified by parameters stored in `dst->op_params`.
|
676
|
+
* The operation reflects the values at the borders of the tensor to generate the padded output.
|
677
|
+
*
|
678
|
+
* This operation is optimized using the CANN backend for high-performance inference or training.
|
679
|
+
*
|
680
|
+
* @param ctx The CANN context used for operations.
|
681
|
+
* @param dst The destination tensor where the padded result will be stored.
|
682
|
+
* dst->op is expected to be `GGML_OP_PAD_REFLECT_1D`.
|
683
|
+
*/
|
684
|
+
void ggml_cann_pad_reflect_1d(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
685
|
+
|
686
|
+
/**
|
687
|
+
* @brief Counts the number of equal elements in two ggml tensors using the CANN backend.
|
688
|
+
*
|
689
|
+
* @details This function performs an element-wise comparison between two input tensors,
|
690
|
+
* and counts the number of positions where the elements are equal. The result is
|
691
|
+
* stored in the destination tensor `dst` as a scalar.
|
692
|
+
*
|
693
|
+
* The operation is optimized using the CANN backend, making it suitable for
|
694
|
+
* high-performance inference or training scenarios.
|
695
|
+
*
|
696
|
+
* @param ctx The CANN context used for operations.
|
697
|
+
* @param dst The destination tensor where the result will be stored.
|
698
|
+
* dst->op is expected to be `GGML_OP_COUNT_EQUAL`.
|
699
|
+
*/
|
700
|
+
void ggml_cann_count_equal(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
701
|
+
|
702
|
+
/**
|
703
|
+
* @brief Applies the Step activation function to a ggml tensor using the CANN backend.
|
704
|
+
*
|
705
|
+
* @details This function applies a step function element-wise to the input tensor, where
|
706
|
+
* each element is transformed to 1.0 if it is greater than 0, and 0.0 otherwise.
|
707
|
+
* The result is stored in the destination tensor `dst`.
|
708
|
+
*
|
709
|
+
* This operation is accelerated using the CANN backend to improve runtime performance.
|
710
|
+
*
|
711
|
+
* @param ctx The CANN context used for operations.
|
712
|
+
* @param dst The destination tensor where the result will be stored.
|
713
|
+
* dst->op is expected to be `GGML_OP_STEP`.
|
714
|
+
*/
|
715
|
+
void ggml_cann_step(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
716
|
+
|
717
|
+
/**
|
718
|
+
* @brief Performs the Flash Attention extended operator using the CANN backend.
|
719
|
+
*
|
720
|
+
* @details This function implements the memory-efficient Flash Attention algorithm
|
721
|
+
* for computing scaled dot-product attention with hardware acceleration.
|
722
|
+
* The result is stored in the destination tensor `dst`.
|
723
|
+
*
|
724
|
+
* This operation is accelerated using the CANN backend to improve runtime performance.
|
725
|
+
*
|
726
|
+
* @param ctx The CANN context used for operations.
|
727
|
+
* @param dst The destination tensor where the result will be stored.
|
728
|
+
* dst->op is expected to be `GGML_OP_FLASH_ATTN_EXT`.
|
729
|
+
*/
|
730
|
+
void ggml_cann_flash_attn_ext(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
731
|
+
|
732
|
+
/*
|
733
|
+
* @brief A generic wrapper for ACL resources with custom deleter support.
|
734
|
+
*/
|
735
|
+
using any_acl_resource = std::unique_ptr<void, std::function<void(void*)>>;
|
736
|
+
|
737
|
+
/**
|
738
|
+
* @brief Trait structure used to define how to destroy a given ACL resource type.
|
739
|
+
*
|
740
|
+
* @tparam T ACL resource type.
|
741
|
+
*/
|
742
|
+
template<typename T>
|
743
|
+
struct acl_resource_traits;
|
744
|
+
|
745
|
+
/**
|
746
|
+
* @brief Specialization for aclTensor, defines how to destroy an aclTensor resource.
|
747
|
+
*/
|
748
|
+
template<>
|
749
|
+
struct acl_resource_traits<aclTensor> {
|
750
|
+
static void destroy(void* p) {
|
751
|
+
ACL_CHECK(aclDestroyTensor(static_cast<aclTensor*>(p)));
|
752
|
+
}
|
753
|
+
};
|
754
|
+
|
755
|
+
/**
|
756
|
+
* @brief Specialization for aclIntArray, defines how to destroy an aclIntArray resource.
|
757
|
+
*/
|
758
|
+
template<>
|
759
|
+
struct acl_resource_traits<aclIntArray> {
|
760
|
+
static void destroy(void* p) {
|
761
|
+
ACL_CHECK(aclDestroyIntArray(static_cast<aclIntArray*>(p)));
|
762
|
+
}
|
763
|
+
};
|
764
|
+
|
765
|
+
/**
|
766
|
+
* @brief Specialization for aclScalar, defines how to destroy an aclScalar resource.
|
767
|
+
*/
|
768
|
+
template<>
|
769
|
+
struct acl_resource_traits<aclScalar> {
|
770
|
+
static void destroy(void* p) {
|
771
|
+
ACL_CHECK(aclDestroyScalar(static_cast<aclScalar*>(p)));
|
772
|
+
}
|
773
|
+
};
|
774
|
+
|
775
|
+
/**
|
776
|
+
* @brief Specialization for aclTensorList, defines how to destroy an aclTensorList resource.
|
777
|
+
*/
|
778
|
+
template<>
|
779
|
+
struct acl_resource_traits<aclTensorList> {
|
780
|
+
static void destroy(void* p) {
|
781
|
+
ACL_CHECK(aclDestroyTensorList(static_cast<aclTensorList*>(p)));
|
782
|
+
}
|
783
|
+
};
|
784
|
+
|
785
|
+
/**
|
786
|
+
* @brief Creates a generic ACL resource wrapper with proper destruction logic.
|
787
|
+
*
|
788
|
+
* @tparam T ACL resource type.
|
789
|
+
* @param ptr Raw pointer to ACL resource.
|
790
|
+
* @return any_acl_resource Smart pointer that handles destruction.
|
791
|
+
*/
|
792
|
+
template<typename T>
|
793
|
+
any_acl_resource make_acl_resource(T* ptr) {
|
794
|
+
return any_acl_resource(
|
795
|
+
static_cast<void*>(ptr),
|
796
|
+
[](void* p) {
|
797
|
+
acl_resource_traits<T>::destroy(p);
|
798
|
+
}
|
799
|
+
);
|
800
|
+
}
|
801
|
+
|
802
|
+
/**
|
803
|
+
* @brief Registers multiple ACL resources into a vector for lifetime management.
|
804
|
+
*
|
805
|
+
* @tparam Args Variadic list of ACL resource types.
|
806
|
+
* @param vec Target vector to hold ACL resources.
|
807
|
+
* @param args Raw pointers to ACL resources.
|
808
|
+
*/
|
809
|
+
template<typename... Args>
|
810
|
+
void register_acl_resources(std::vector<any_acl_resource>& vec, Args*... args) {
|
811
|
+
(vec.emplace_back(make_acl_resource(args)), ...);
|
812
|
+
}
|
813
|
+
|
814
|
+
/**
|
815
|
+
* @brief Task class that wraps the execution of an aclnn function call.
|
816
|
+
*/
|
817
|
+
class aclnn_task : public cann_task {
|
818
|
+
public:
|
819
|
+
aclnn_task(aclnn_func_t aclnn_func, void * workspace_addr,
|
820
|
+
uint64_t workspace_size, aclOpExecutor * executor,
|
821
|
+
aclrtStream stream) :
|
822
|
+
aclnn_func_(aclnn_func),
|
823
|
+
workspace_addr_(workspace_addr),
|
824
|
+
workspace_size_(workspace_size),
|
825
|
+
executor_(executor),
|
826
|
+
stream_(stream) {}
|
827
|
+
virtual void run_task() override {
|
828
|
+
ACL_CHECK(aclnn_func_(workspace_addr_, workspace_size_, executor_, stream_));
|
829
|
+
}
|
830
|
+
private:
|
831
|
+
aclnn_func_t aclnn_func_;
|
832
|
+
void * workspace_addr_;
|
833
|
+
uint64_t workspace_size_;
|
834
|
+
aclOpExecutor * executor_;
|
835
|
+
aclrtStream stream_;
|
836
|
+
};
|
837
|
+
|
838
|
+
/**
|
839
|
+
* @brief Task class that releases ACL resources after usage.
|
840
|
+
*/
|
841
|
+
class release_resource_task : public cann_task {
|
842
|
+
public:
|
843
|
+
release_resource_task(std::vector<any_acl_resource>&& resources){
|
844
|
+
resource_ = std::move(resources);
|
845
|
+
}
|
846
|
+
|
847
|
+
virtual void run_task() override {
|
848
|
+
resource_.clear();
|
849
|
+
}
|
850
|
+
private:
|
851
|
+
std::vector<any_acl_resource> resource_;
|
852
|
+
};
|
853
|
+
|
854
|
+
/**
|
855
|
+
* @brief Task class for performing asynchronous memory copy operations.
|
856
|
+
*/
|
857
|
+
class async_memcpy_task : public cann_task {
|
858
|
+
public:
|
859
|
+
async_memcpy_task(void* dst, const void* src, size_t size,
|
860
|
+
aclrtMemcpyKind kind, aclrtStream stream)
|
861
|
+
: dst_(dst), src_(src), size_(size), kind_(kind), stream_(stream) {}
|
862
|
+
|
863
|
+
virtual void run_task() override {
|
864
|
+
ACL_CHECK(aclrtMemcpyAsync(dst_, size_, src_, size_, kind_, stream_));
|
865
|
+
}
|
866
|
+
private:
|
867
|
+
void* dst_;
|
868
|
+
const void* src_;
|
869
|
+
size_t size_;
|
870
|
+
aclrtMemcpyKind kind_;
|
871
|
+
aclrtStream stream_;
|
872
|
+
};
|
873
|
+
|
874
|
+
/**
|
875
|
+
* @brief Task class for performing asynchronous memory set operations.
|
876
|
+
*/
|
877
|
+
class async_memset_task : public cann_task {
|
878
|
+
public:
|
879
|
+
async_memset_task(void* buffer, size_t size, int32_t value, aclrtStream stream)
|
880
|
+
: buffer_(buffer), size_(size), value_(value), stream_(stream) {}
|
881
|
+
|
882
|
+
virtual void run_task() override {
|
883
|
+
ACL_CHECK(aclrtMemsetAsync(buffer_, size_, value_, size_, stream_));
|
884
|
+
}
|
885
|
+
private:
|
886
|
+
void* buffer_;
|
887
|
+
size_t size_;
|
888
|
+
int32_t value_;
|
889
|
+
aclrtStream stream_;
|
890
|
+
};
|
891
|
+
|
892
|
+
/**
|
893
|
+
* @brief Launches an asynchronous task using the memory allocator.
|
894
|
+
*
|
895
|
+
* This macro submit an asynchronous task on the specified stream.
|
896
|
+
* The task uses memory allocated by the allocator. It is guaranteed
|
897
|
+
* that the memory will not be accessed by other tasks until this task
|
898
|
+
* completes, due to the sequential execution order within the same stream.
|
899
|
+
*
|
900
|
+
* @param OP_NAME aclnn operator name.
|
901
|
+
* @param args Additional arguments required by the task.
|
902
|
+
*
|
903
|
+
* @note
|
904
|
+
* Memory from the allocator will be "freed" immediately and can be
|
905
|
+
* reallocated to other pointers. However, it won't be accessed by any
|
906
|
+
* other task before this asynchronous task ends, because all tasks in the
|
907
|
+
* same stream are executed in queue order.
|
908
|
+
*/
|
909
|
+
|
910
|
+
#define GGML_CANN_CALL_ACLNN_OP(CTX, OP_NAME, ...) \
|
911
|
+
do { \
|
912
|
+
uint64_t workspaceSize = 0; \
|
913
|
+
aclOpExecutor * executor; \
|
914
|
+
void * workspaceAddr = nullptr; \
|
915
|
+
ACL_CHECK(aclnn##OP_NAME##GetWorkspaceSize(__VA_ARGS__, &workspaceSize, &executor));\
|
916
|
+
/* workspace should alloced in main thread to keep malloc order when using vmm. */ \
|
917
|
+
if (workspaceSize > 0) { \
|
918
|
+
ggml_cann_pool_alloc workspace_allocator(CTX.pool(), workspaceSize); \
|
919
|
+
workspaceAddr = workspace_allocator.get(); \
|
920
|
+
} \
|
921
|
+
if (CTX.async_mode) { \
|
922
|
+
auto task = \
|
923
|
+
std::make_unique<aclnn_task>(aclnn##OP_NAME, workspaceAddr, workspaceSize, \
|
924
|
+
executor, CTX.stream()); \
|
925
|
+
CTX.task_queue.submit_task(std::move(task)); \
|
926
|
+
} else { \
|
927
|
+
ACL_CHECK(aclnn##OP_NAME(workspaceAddr, workspaceSize, executor, CTX.stream()));\
|
928
|
+
} \
|
929
|
+
} while (0)
|
930
|
+
|
931
|
+
/**
|
932
|
+
* @brief Registers and releases multiple ACL resources, optionally deferring the release
|
933
|
+
* using a task.
|
934
|
+
*
|
935
|
+
* @tparam Args Types of the ACL resources.
|
936
|
+
* @param ctx Backend context which manages task submission and async mode.
|
937
|
+
* @param args Pointers to ACL resources to be released.
|
938
|
+
*/
|
939
|
+
template <typename... Args>
|
940
|
+
void ggml_cann_release_resources(ggml_backend_cann_context & ctx, Args &&... args) {
|
941
|
+
std::vector<any_acl_resource> resources;
|
942
|
+
register_acl_resources(resources, std::forward<Args>(args)...);
|
943
|
+
if(ctx.async_mode) {
|
944
|
+
auto task = std::make_unique<release_resource_task>(std::move(resources));
|
945
|
+
ctx.task_queue.submit_task(std::move(task));
|
946
|
+
}
|
947
|
+
}
|
948
|
+
|
949
|
+
/**
|
950
|
+
* @brief Performs an asynchronous memory copy operation, optionally deferred via task submission.
|
951
|
+
*
|
952
|
+
* @param ctx Backend context containing stream and async configuration.
|
953
|
+
* @param dst Destination memory address.
|
954
|
+
* @param src Source memory address.
|
955
|
+
* @param len Size of memory to copy (in bytes).
|
956
|
+
* @param kind Type of memory copy (host-to-device, device-to-host, etc).
|
957
|
+
*/
|
958
|
+
inline void ggml_cann_async_memcpy(ggml_backend_cann_context & ctx, void * dst,
|
959
|
+
const void * src, size_t len, aclrtMemcpyKind kind) {
|
960
|
+
if (ctx.async_mode) {
|
961
|
+
auto task = std::make_unique<async_memcpy_task>(dst, const_cast<void *>(src), len, kind, ctx.stream());
|
962
|
+
ctx.task_queue.submit_task(std::move(task));
|
963
|
+
} else {
|
964
|
+
ACL_CHECK(aclrtMemcpyAsync(dst, len, src, len, kind, ctx.stream()));
|
965
|
+
}
|
966
|
+
}
|
967
|
+
|
968
|
+
inline void ggml_cann_async_memcpy(ggml_backend_cann_context * ctx, void * dst,
|
969
|
+
const void * src, size_t len, aclrtMemcpyKind kind) {
|
970
|
+
if (ctx->async_mode) {
|
971
|
+
auto task = std::make_unique<async_memcpy_task>(dst, const_cast<void *>(src), len, kind, ctx->stream());
|
972
|
+
ctx->task_queue.submit_task(std::move(task));
|
973
|
+
} else {
|
974
|
+
ACL_CHECK(aclrtMemcpyAsync(dst, len, src, len, kind, ctx->stream()));
|
975
|
+
}
|
976
|
+
}
|
977
|
+
|
978
|
+
/**
|
979
|
+
* @brief Performs an asynchronous memory set operation, optionally deferred via task submission.
|
980
|
+
*
|
981
|
+
* @param ctx Backend context containing stream and async configuration.
|
982
|
+
* @param buffer Memory buffer to be set.
|
983
|
+
* @param size Size of the memory buffer (in bytes).
|
984
|
+
* @param value Value to set in the buffer.
|
985
|
+
*/
|
986
|
+
inline void ggml_cann_async_memset(ggml_backend_cann_context & ctx, void * buffer,
|
987
|
+
size_t size, int value) {
|
988
|
+
if (ctx.async_mode) {
|
989
|
+
auto task = std::make_unique<async_memset_task>(buffer, size, value, ctx.stream());
|
990
|
+
ctx.task_queue.submit_task(std::move(task));
|
991
|
+
} else {
|
992
|
+
ACL_CHECK(aclrtMemsetAsync(buffer, size, value, size, ctx.stream()));
|
993
|
+
}
|
994
|
+
}
|
995
|
+
|
996
|
+
/**
|
997
|
+
* @brief Performs sparse expert-based matrix multiplication using the CANN backend.
|
998
|
+
*
|
999
|
+
* @details This function implements a MoE-style batched matrix multiplication, where each input token
|
1000
|
+
* is routed to one or more experts, and each expert corresponds to a specific [D, M] weight matrix
|
1001
|
+
* in the source tensor `src0`. The routing indices are provided via the `ids` tensor.
|
1002
|
+
*
|
1003
|
+
* For each token (from `src1`), the function selects the corresponding expert(s) as specified by `ids`,
|
1004
|
+
* performs the matrix multiplication with the selected expert's weight submatrix (from `src0`),
|
1005
|
+
* and stores the results in `dst`. This operation is optimized and executed on the CANN backend.
|
1006
|
+
*
|
1007
|
+
* Dimensions:
|
1008
|
+
* - src0: [D, M, A, 1], where A is the number of experts
|
1009
|
+
* - src1: [D, B, N, 1], where N is batch size and B is the slot count per sample
|
1010
|
+
* - ids : [K, N], where K is the number of experts each token is routed to
|
1011
|
+
* - dst : [M, K, N, 1], output tensor storing the result of expert × token multiplication
|
1012
|
+
*
|
1013
|
+
* The function handles two main modes:
|
1014
|
+
* - If `ne12 == 1`, a simpler per-token loop is used.
|
1015
|
+
* - TODO: If `ne12 > 1`, grouped multiplication and memory copying is used for efficiency.
|
1016
|
+
*
|
1017
|
+
* @param ctx The CANN context used for operations.
|
1018
|
+
* @param dst The destination tensor where the expert-weighted token outputs are stored.
|
1019
|
+
* Expected to be of shape [M, K, N, 1].
|
1020
|
+
*/
|
1021
|
+
void ggml_cann_mul_mat_id(ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
1022
|
+
|
1023
|
+
/**
|
1024
|
+
* @brief Applies a element-wise operation to two input tensors using the CANN
|
1025
|
+
* backend.
|
1026
|
+
*
|
1027
|
+
* This templated function takes a binary operator and applies it to two source
|
1028
|
+
* tensors
|
1029
|
+
* associated with the destination tensor. The function handles broadcasting as
|
1030
|
+
* needed.
|
1031
|
+
*
|
1032
|
+
* @tparam binary_op A callable object (e.g., lambda or function pointer) representing
|
1033
|
+
* the binary operation to be performed. It must take three arguments:
|
1034
|
+
* (ggml_backend_cann_context&, aclTensor*, aclTensor*, aclTensor*).
|
1035
|
+
*
|
1036
|
+
* @param ctx The CANN backend context used to manage execution and resources.
|
1037
|
+
* @param dst The destination tensor.
|
1038
|
+
*/
|
1039
|
+
template <auto binary_op>
|
1040
|
+
void ggml_cann_binary_op(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
1041
|
+
ggml_tensor* src0 = dst->src[0];
|
1042
|
+
ggml_tensor* src1 = dst->src[1];
|
1043
|
+
|
1044
|
+
aclTensor* acl_src0;
|
1045
|
+
aclTensor* acl_src1;
|
1046
|
+
aclTensor* acl_dst;
|
1047
|
+
|
1048
|
+
// Need bcast
|
1049
|
+
bcast_shape(src0, src1, dst, &acl_src0, &acl_src1, &acl_dst);
|
1050
|
+
binary_op(ctx, acl_src0, acl_src1, acl_dst);
|
1051
|
+
|
1052
|
+
ggml_cann_release_resources(ctx, acl_src0, acl_src1, acl_dst);
|
1053
|
+
}
|
1054
|
+
|
1055
|
+
|
1056
|
+
/**
|
1057
|
+
* @brief Applies a unary operation to an input tensor using the CANN backend.
|
1058
|
+
*
|
1059
|
+
* This templated function applies a unary operator to the source tensor of `dst`
|
1060
|
+
* and stores the result in the destination tensor.
|
1061
|
+
*
|
1062
|
+
* @tparam unary_op A callable with the signature:
|
1063
|
+
* void(ggml_backend_cann_context&, aclTensor*, aclTensor*)
|
1064
|
+
* where the first aclTensor is the source and the second is the destination.
|
1065
|
+
* @param ctx The CANN backend context for managing resources and execution.
|
1066
|
+
* @param dst The destination tensor. Its src[0] is treated as the input tensor.
|
1067
|
+
*/
|
1068
|
+
template <void unary_op(ggml_backend_cann_context&, aclTensor*, aclTensor*)>
|
1069
|
+
void ggml_cann_unary_op(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
1070
|
+
ggml_tensor* src = dst->src[0];
|
1071
|
+
|
1072
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
1073
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
1074
|
+
|
1075
|
+
unary_op(ctx, acl_src, acl_dst);
|
1076
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst);
|
1077
|
+
}
|
1078
|
+
|
1079
|
+
/**
|
1080
|
+
* @brief Applies a unary operation to a ggml tensor using the CANN backend.
|
1081
|
+
*
|
1082
|
+
* @details This function performs a unary operation on the input tensor using
|
1083
|
+
* a user-provided lambda or callable object `unary_op`, which accepts the CANN
|
1084
|
+
* context and two ACL tensors (source and destination). Internally, this function
|
1085
|
+
* creates ACL representations of the ggml tensors and invokes the unary operation.
|
1086
|
+
* The result is stored in the destination tensor `dst`. This utility abstracts the
|
1087
|
+
* common boilerplate of tensor conversion and cleanup when implementing unary ops.
|
1088
|
+
*
|
1089
|
+
* @param unary_op A callable that performs the unary operation using CANN APIs.
|
1090
|
+
* @param ctx The CANN context used for operations.
|
1091
|
+
* @param dst The destination tensor where the result will be stored.
|
1092
|
+
* The source tensor is retrieved from `dst->src[0]`.
|
1093
|
+
*/
|
1094
|
+
void ggml_cann_unary_op(
|
1095
|
+
std::function<void(ggml_backend_cann_context&, aclTensor*, aclTensor*)> unary_op,
|
1096
|
+
ggml_backend_cann_context& ctx, ggml_tensor* dst);
|
1097
|
+
|
1098
|
+
/**
|
1099
|
+
* @brief Helper macro to invoke a unary ACL operation using ggml_cann_unary_op.
|
1100
|
+
*
|
1101
|
+
* This macro defines an inline lambda wrapping a specific ACL operation name,
|
1102
|
+
* and passes it to the templated ggml_cann_unary_op function. It simplifies
|
1103
|
+
* calling unary ops by hiding the lambda boilerplate.
|
1104
|
+
*
|
1105
|
+
* Internally, the lambda will call:
|
1106
|
+
* @code
|
1107
|
+
* GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst);
|
1108
|
+
* @endcode
|
1109
|
+
*
|
1110
|
+
* @param OP_NAME The name of the ACL unary operator to invoke via GGML_CANN_CALL_ACLNN_OP.
|
1111
|
+
*
|
1112
|
+
* @see ggml_cann_unary_op
|
1113
|
+
* @see GGML_CANN_CALL_ACLNN_OP
|
1114
|
+
*/
|
1115
|
+
#define GGML_CANN_CALL_UNARY_OP(OP_NAME) \
|
1116
|
+
do { \
|
1117
|
+
auto lambda = [](ggml_backend_cann_context& ctx, \
|
1118
|
+
aclTensor* acl_src, \
|
1119
|
+
aclTensor* acl_dst) { \
|
1120
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst); \
|
1121
|
+
}; \
|
1122
|
+
ggml_cann_unary_op(lambda, ctx, dst); \
|
1123
|
+
} \
|
1124
|
+
while (0)
|
1125
|
+
#endif // CANN_ACLNN_OPS
|