whispercpp 1.3.0 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +6 -0
- data/LICENSE +1 -1
- data/README.md +216 -424
- data/Rakefile +79 -11
- data/ext/.gitignore +11 -0
- data/ext/dependencies.rb +61 -0
- data/ext/extconf.rb +18 -26
- data/ext/options.rb +221 -0
- data/ext/ruby_whisper.c +159 -0
- data/ext/ruby_whisper.h +27 -2
- data/ext/ruby_whisper_context.c +641 -0
- data/ext/ruby_whisper_error.c +52 -0
- data/ext/ruby_whisper_model.c +232 -0
- data/ext/ruby_whisper_params.c +1301 -0
- data/ext/ruby_whisper_segment.c +143 -0
- data/ext/ruby_whisper_transcribe.cpp +87 -0
- data/ext/ruby_whisper_vad_params.c +288 -0
- data/ext/sources/.dockerignore +3 -0
- data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
- data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
- data/ext/sources/CMakeLists.txt +251 -0
- data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
- data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
- data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
- data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
- data/ext/sources/bindings/javascript/package.json +26 -0
- data/ext/sources/bindings/javascript/whisper.js +19 -0
- data/ext/sources/build-xcframework.sh +547 -0
- data/ext/sources/ci/run.sh +336 -0
- data/ext/sources/close-issue.yml +28 -0
- data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
- data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
- data/ext/sources/cmake/build-info.cmake +60 -0
- data/ext/sources/cmake/git-vars.cmake +22 -0
- data/ext/sources/cmake/whisper-config.cmake.in +65 -0
- data/ext/sources/cmake/whisper.pc.in +10 -0
- data/ext/sources/examples/CMakeLists.txt +124 -0
- data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
- data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
- data/ext/sources/examples/addon.node/addon.cpp +438 -0
- data/ext/sources/examples/addon.node/index.js +54 -0
- data/ext/sources/examples/addon.node/package.json +16 -0
- data/ext/sources/examples/bench/CMakeLists.txt +8 -0
- data/ext/sources/examples/bench/bench.cpp +175 -0
- data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
- data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
- data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
- data/ext/sources/examples/cli/CMakeLists.txt +8 -0
- data/ext/sources/examples/cli/cli.cpp +1294 -0
- data/ext/sources/examples/coi-serviceworker.js +146 -0
- data/ext/sources/examples/command/CMakeLists.txt +10 -0
- data/ext/sources/examples/command/command.cpp +776 -0
- data/ext/sources/examples/command/commands.txt +9 -0
- data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
- data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
- data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
- data/ext/sources/examples/common-ggml.cpp +238 -0
- data/ext/sources/examples/common-ggml.h +18 -0
- data/ext/sources/examples/common-sdl.cpp +227 -0
- data/ext/sources/examples/common-sdl.h +49 -0
- data/ext/sources/examples/common-whisper.cpp +168 -0
- data/ext/sources/examples/common-whisper.h +24 -0
- data/ext/sources/examples/common.cpp +675 -0
- data/ext/sources/examples/common.h +322 -0
- data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
- data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
- data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
- data/ext/sources/examples/generate-karaoke.sh +57 -0
- data/ext/sources/examples/grammar-parser.cpp +423 -0
- data/ext/sources/examples/grammar-parser.h +29 -0
- data/ext/sources/examples/helpers.js +191 -0
- data/ext/sources/examples/json.hpp +24596 -0
- data/ext/sources/examples/livestream.sh +112 -0
- data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
- data/ext/sources/examples/lsp/lsp.cpp +467 -0
- data/ext/sources/examples/lsp/whisper.vim +362 -0
- data/ext/sources/examples/miniaudio.h +93468 -0
- data/ext/sources/examples/python/test_whisper_processor.py +7 -0
- data/ext/sources/examples/python/whisper_processor.py +54 -0
- data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
- data/ext/sources/examples/quantize/quantize.cpp +223 -0
- data/ext/sources/examples/server/CMakeLists.txt +12 -0
- data/ext/sources/examples/server/bench.js +29 -0
- data/ext/sources/examples/server/httplib.h +10497 -0
- data/ext/sources/examples/server/server.cpp +1091 -0
- data/ext/sources/examples/server.py +115 -0
- data/ext/sources/examples/stb_vorbis.c +5584 -0
- data/ext/sources/examples/stream/CMakeLists.txt +10 -0
- data/ext/sources/examples/stream/stream.cpp +429 -0
- data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
- data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
- data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
- data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
- data/ext/sources/examples/sycl/build.sh +22 -0
- data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
- data/ext/sources/examples/sycl/run-whisper.sh +17 -0
- data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
- data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
- data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
- data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
- data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
- data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
- data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
- data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
- data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
- data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
- data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
- data/ext/sources/examples/talk-llama/llama-context.h +276 -0
- data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
- data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
- data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
- data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
- data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
- data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
- data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
- data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
- data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
- data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
- data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
- data/ext/sources/examples/talk-llama/llama-io.h +35 -0
- data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
- data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
- data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
- data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
- data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
- data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
- data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
- data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
- data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
- data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
- data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
- data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
- data/ext/sources/examples/talk-llama/llama-model.h +425 -0
- data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
- data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
- data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
- data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
- data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
- data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
- data/ext/sources/examples/talk-llama/llama.cpp +354 -0
- data/ext/sources/examples/talk-llama/llama.h +1377 -0
- data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
- data/ext/sources/examples/talk-llama/speak +40 -0
- data/ext/sources/examples/talk-llama/speak.bat +1 -0
- data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
- data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
- data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
- data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
- data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
- data/ext/sources/examples/talk-llama/unicode.h +66 -0
- data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
- data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
- data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
- data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
- data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
- data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
- data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
- data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
- data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
- data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
- data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
- data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
- data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
- data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
- data/ext/sources/ggml/CMakeLists.txt +390 -0
- data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
- data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
- data/ext/sources/ggml/cmake/common.cmake +26 -0
- data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
- data/ext/sources/ggml/include/ggml-alloc.h +76 -0
- data/ext/sources/ggml/include/ggml-backend.h +354 -0
- data/ext/sources/ggml/include/ggml-blas.h +25 -0
- data/ext/sources/ggml/include/ggml-cann.h +123 -0
- data/ext/sources/ggml/include/ggml-cpp.h +39 -0
- data/ext/sources/ggml/include/ggml-cpu.h +143 -0
- data/ext/sources/ggml/include/ggml-cuda.h +47 -0
- data/ext/sources/ggml/include/ggml-kompute.h +50 -0
- data/ext/sources/ggml/include/ggml-metal.h +66 -0
- data/ext/sources/ggml/include/ggml-opencl.h +26 -0
- data/ext/sources/ggml/include/ggml-opt.h +237 -0
- data/ext/sources/ggml/include/ggml-rpc.h +33 -0
- data/ext/sources/ggml/include/ggml-sycl.h +49 -0
- data/ext/sources/ggml/include/ggml-vulkan.h +29 -0
- data/ext/{ggml.h → sources/ggml/include/ggml.h} +621 -821
- data/ext/sources/ggml/include/gguf.h +202 -0
- data/ext/sources/ggml/src/CMakeLists.txt +346 -0
- data/ext/sources/ggml/src/ggml-alloc.c +1042 -0
- data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
- data/ext/sources/ggml/src/ggml-amx/common.h +94 -0
- data/ext/sources/ggml/src/ggml-amx/ggml-amx.cpp +446 -0
- data/ext/sources/ggml/src/ggml-amx/mmq.cpp +2510 -0
- data/ext/sources/ggml/src/ggml-amx/mmq.h +17 -0
- data/ext/sources/ggml/src/ggml-backend-impl.h +255 -0
- data/ext/sources/ggml/src/ggml-backend-reg.cpp +586 -0
- data/ext/sources/ggml/src/ggml-backend.cpp +2011 -0
- data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
- data/ext/sources/ggml/src/ggml-blas/ggml-blas.cpp +517 -0
- data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
- data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
- data/ext/sources/ggml/src/ggml-cann/acl_tensor.cpp +181 -0
- data/ext/sources/ggml/src/ggml-cann/acl_tensor.h +258 -0
- data/ext/sources/ggml/src/ggml-cann/aclnn_ops.cpp +3193 -0
- data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
- data/ext/sources/ggml/src/ggml-cann/common.h +420 -0
- data/ext/sources/ggml/src/ggml-cann/ggml-cann.cpp +2606 -0
- data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
- data/ext/sources/ggml/src/ggml-cann/kernels/ascendc_kernels.h +19 -0
- data/ext/sources/ggml/src/ggml-cann/kernels/dup.cpp +234 -0
- data/ext/sources/ggml/src/ggml-cann/kernels/get_row_f16.cpp +197 -0
- data/ext/sources/ggml/src/ggml-cann/kernels/get_row_f32.cpp +190 -0
- data/ext/sources/ggml/src/ggml-cann/kernels/get_row_q4_0.cpp +204 -0
- data/ext/sources/ggml/src/ggml-cann/kernels/get_row_q8_0.cpp +191 -0
- data/ext/sources/ggml/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +218 -0
- data/ext/sources/ggml/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +216 -0
- data/ext/sources/ggml/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +295 -0
- data/ext/sources/ggml/src/ggml-common.h +1857 -0
- data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
- data/ext/sources/ggml/src/ggml-cpu/amx/amx.cpp +221 -0
- data/ext/sources/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/ext/sources/ggml/src/ggml-cpu/amx/common.h +91 -0
- data/ext/sources/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
- data/ext/sources/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
- data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
- data/ext/sources/ggml/src/ggml-cpu/cpu-feats-x86.cpp +327 -0
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.h +8 -0
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-hbm.cpp +55 -0
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-hbm.h +8 -0
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-impl.h +508 -0
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-quants.c +13747 -0
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-quants.h +63 -0
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-traits.cpp +36 -0
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-traits.h +38 -0
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.cpp +671 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
- data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
- data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
- data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
- data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
- data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
- data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
- data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
- data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
- data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
- data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
- data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
- data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
- data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
- data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
- data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
- data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
- data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
- data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
- data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
- data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
- data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
- data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
- data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
- data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
- data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
- data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
- data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
- data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
- data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
- data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
- data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
- data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
- data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
- data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
- data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
- data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
- data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
- data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
- data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
- data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
- data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
- data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
- data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
- data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
- data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
- data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
- data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
- data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
- data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
- data/ext/sources/ggml/src/ggml-cuda/vendors/cuda.h +15 -0
- data/ext/sources/ggml/src/ggml-cuda/vendors/hip.h +243 -0
- data/ext/sources/ggml/src/ggml-cuda/vendors/musa.h +140 -0
- data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
- data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
- data/ext/sources/ggml/src/ggml-impl.h +601 -0
- data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
- data/ext/sources/ggml/src/ggml-kompute/ggml-kompute.cpp +2251 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
- data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
- data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
- data/ext/sources/ggml/src/ggml-metal/ggml-metal.m +5998 -0
- data/ext/sources/ggml/src/ggml-metal/ggml-metal.metal +7089 -0
- data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
- data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
- data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
- data/ext/sources/ggml/src/ggml-opt.cpp +1037 -0
- data/ext/sources/ggml/src/ggml-quants.c +5232 -0
- data/ext/sources/ggml/src/ggml-quants.h +100 -0
- data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
- data/ext/sources/ggml/src/ggml-rpc/ggml-rpc.cpp +1813 -0
- data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
- data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
- data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
- data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
- data/ext/sources/ggml/src/ggml-sycl/common.cpp +83 -0
- data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
- data/ext/sources/ggml/src/ggml-sycl/concat.cpp +195 -0
- data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/conv.cpp +101 -0
- data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/convert.cpp +623 -0
- data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
- data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
- data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
- data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
- data/ext/sources/ggml/src/ggml-sycl/dmmv.cpp +1162 -0
- data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
- data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
- data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
- data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
- data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
- data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/ggml-sycl.cpp +4493 -0
- data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
- data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
- data/ext/sources/ggml/src/ggml-sycl/mmq.cpp +3030 -0
- data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/ext/sources/ggml/src/ggml-sycl/mmvq.cpp +1110 -0
- data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
- data/ext/sources/ggml/src/ggml-sycl/norm.cpp +501 -0
- data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
- data/ext/sources/ggml/src/ggml-sycl/outprod.cpp +47 -0
- data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
- data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
- data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
- data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/softmax.cpp +261 -0
- data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
- data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
- data/ext/sources/ggml/src/ggml-sycl/tsembd.cpp +72 -0
- data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
- data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/ext/sources/ggml/src/ggml-threading.cpp +12 -0
- data/ext/sources/ggml/src/ggml-threading.h +14 -0
- data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
- data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/ext/sources/ggml/src/ggml-vulkan/ggml-vulkan.cpp +10700 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +751 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/ext/sources/ggml/src/ggml.c +6550 -0
- data/ext/sources/ggml/src/gguf.cpp +1330 -0
- data/ext/{whisper.h → sources/include/whisper.h} +91 -24
- data/ext/sources/src/CMakeLists.txt +143 -0
- data/ext/sources/src/coreml/whisper-decoder-impl.h +158 -0
- data/ext/sources/src/coreml/whisper-decoder-impl.m +226 -0
- data/ext/sources/src/coreml/whisper-encoder-impl.h +154 -0
- data/ext/sources/src/coreml/whisper-encoder-impl.m +222 -0
- data/ext/sources/src/coreml/whisper-encoder.h +26 -0
- data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
- data/ext/sources/src/openvino/whisper-openvino-encoder.cpp +108 -0
- data/ext/sources/src/openvino/whisper-openvino-encoder.h +31 -0
- data/ext/sources/src/whisper-arch.h +197 -0
- data/ext/{whisper.cpp → sources/src/whisper.cpp} +2535 -835
- data/ext/sources/tests/CMakeLists.txt +105 -0
- data/ext/sources/tests/earnings21/eval.mk +58 -0
- data/ext/sources/tests/earnings21/eval.py +68 -0
- data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
- data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
- data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
- data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
- data/ext/sources/tests/earnings21/requirements.txt +6 -0
- data/ext/sources/tests/en-0-ref.txt +1 -0
- data/ext/sources/tests/en-1-ref.txt +1 -0
- data/ext/sources/tests/en-2-ref.txt +1 -0
- data/ext/sources/tests/es-0-ref.txt +1 -0
- data/ext/sources/tests/librispeech/eval.mk +39 -0
- data/ext/sources/tests/librispeech/eval.py +47 -0
- data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
- data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
- data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
- data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
- data/ext/sources/tests/librispeech/requirements.txt +6 -0
- data/ext/sources/tests/run-tests.sh +130 -0
- data/ext/sources/tests/test-c.c +3 -0
- data/ext/sources/tests/test-vad-full.cpp +54 -0
- data/ext/sources/tests/test-vad.cpp +83 -0
- data/ext/sources/tests/test-whisper.js +58 -0
- data/extsources.rb +34 -0
- data/lib/whisper/model/uri.rb +178 -0
- data/sig/whisper.rbs +480 -0
- data/tests/helper.rb +35 -0
- data/tests/jfk_reader/.gitignore +5 -0
- data/tests/jfk_reader/extconf.rb +3 -0
- data/tests/jfk_reader/jfk_reader.c +68 -0
- data/tests/test_callback.rb +202 -0
- data/tests/test_error.rb +20 -0
- data/tests/test_model.rb +109 -0
- data/tests/test_package.rb +46 -0
- data/tests/test_params.rb +297 -0
- data/tests/test_segment.rb +74 -0
- data/tests/test_vad.rb +19 -0
- data/tests/test_vad_params.rb +103 -0
- data/tests/test_whisper.rb +212 -124
- data/whispercpp.gemspec +37 -0
- metadata +794 -13
- data/ext/dr_wav.h +0 -6434
- data/ext/ggml.c +0 -21755
- data/ext/ruby_whisper.cpp +0 -426
@@ -0,0 +1,3193 @@
|
|
1
|
+
/*
|
2
|
+
* Copyright (c) 2023-2024 The ggml authors
|
3
|
+
*
|
4
|
+
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
5
|
+
* of this software and associated documentation files (the "Software"), to
|
6
|
+
* deal in the Software without restriction, including without limitation the
|
7
|
+
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
8
|
+
* sell copies of the Software, and to permit persons to whom the Software is
|
9
|
+
* furnished to do so, subject to the following conditions:
|
10
|
+
*
|
11
|
+
* The above copyright notice and this permission notice shall be included in
|
12
|
+
* all copies or substantial portions of the Software.
|
13
|
+
*
|
14
|
+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
15
|
+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
16
|
+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
17
|
+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
18
|
+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
19
|
+
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
20
|
+
* IN THE SOFTWARE.
|
21
|
+
*/
|
22
|
+
|
23
|
+
#include "aclnn_ops.h"
|
24
|
+
|
25
|
+
#include <aclnnop/aclnn_addcdiv.h>
|
26
|
+
#include <aclnnop/aclnn_avgpool2d.h>
|
27
|
+
#include <aclnnop/aclnn_batch_matmul.h>
|
28
|
+
#include <aclnnop/aclnn_cast.h>
|
29
|
+
#include <aclnnop/aclnn_constant_pad_nd.h>
|
30
|
+
#include <aclnnop/aclnn_copy.h>
|
31
|
+
#include <aclnnop/aclnn_div.h>
|
32
|
+
#include <aclnnop/aclnn_embedding.h>
|
33
|
+
#include <aclnnop/aclnn_exp.h>
|
34
|
+
#include <aclnnop/aclnn_fill_scalar.h>
|
35
|
+
#include <aclnnop/aclnn_group_norm.h>
|
36
|
+
#include <aclnnop/aclnn_index_fill_tensor.h>
|
37
|
+
#include <aclnnop/aclnn_layer_norm.h>
|
38
|
+
#include <aclnnop/aclnn_matmul.h>
|
39
|
+
#include <aclnnop/aclnn_max_pool.h>
|
40
|
+
#include <aclnnop/aclnn_mm.h>
|
41
|
+
#include <aclnnop/aclnn_permute.h>
|
42
|
+
#include <aclnnop/aclnn_pow_tensor_tensor.h>
|
43
|
+
#include <aclnnop/aclnn_reduce_sum.h>
|
44
|
+
#include <aclnnop/aclnn_repeat.h>
|
45
|
+
#include <aclnnop/aclnn_repeat_interleave.h>
|
46
|
+
#include <aclnnop/aclnn_roll.h>
|
47
|
+
#include <aclnnop/aclnn_softmax.h>
|
48
|
+
#include <aclnnop/aclnn_tril.h>
|
49
|
+
#include <aclnnop/aclnn_triu.h>
|
50
|
+
#include <aclnnop/aclnn_upsample_nearest_2d.h>
|
51
|
+
#include <aclnnop/aclnn_weight_quant_batch_matmul_v2.h>
|
52
|
+
#include <aclnnop/aclnn_argmax.h>
|
53
|
+
#include <aclnnop/aclnn_sum.h>
|
54
|
+
#include <aclnnop/aclnn_rms_norm.h>
|
55
|
+
#include <aclnnop/aclnn_im2col.h>
|
56
|
+
#include <aclnnop/aclnn_add.h>
|
57
|
+
#include <aclnnop/aclnn_sub.h>
|
58
|
+
#include <aclnnop/aclnn_mul.h>
|
59
|
+
#include <aclnnop/aclnn_div.h>
|
60
|
+
#include <aclnnop/aclnn_convolution.h>
|
61
|
+
#include <aclnnop/aclnn_elu.h>
|
62
|
+
#include <aclnnop/aclnn_log.h>
|
63
|
+
#include <aclnnop/aclnn_mean.h>
|
64
|
+
#include <aclnnop/aclnn_reflection_pad1d.h>
|
65
|
+
#include <aclnnop/aclnn_eq_tensor.h>
|
66
|
+
#include <aclnnop/aclnn_gt_scalar.h>
|
67
|
+
#include <aclnnop/aclnn_pow.h>
|
68
|
+
#include <aclnnop/aclnn_grouped_matmul_v2.h>
|
69
|
+
#include <aclnnop/aclnn_fused_infer_attention_score_v2.h>
|
70
|
+
#include <float.h>
|
71
|
+
|
72
|
+
#include <cmath>
|
73
|
+
#include <cstring>
|
74
|
+
#include <exception>
|
75
|
+
#include <vector>
|
76
|
+
|
77
|
+
#include "ggml-impl.h"
|
78
|
+
#include "ggml.h"
|
79
|
+
|
80
|
+
#define GGML_COMMON_DECL_C
|
81
|
+
|
82
|
+
#include "../ggml-common.h"
|
83
|
+
|
84
|
+
|
85
|
+
void bcast_shape(ggml_tensor * src0, ggml_tensor * src1, ggml_tensor * dst, aclTensor ** acl_src0,
|
86
|
+
aclTensor ** acl_src1, aclTensor ** acl_dst) {
|
87
|
+
GGML_ASSERT(ggml_are_same_shape(src0, dst) && ggml_can_repeat(src1, src0));
|
88
|
+
// Need bcast
|
89
|
+
if (!ggml_are_same_shape(src0, src1) && ggml_cann_need_bcast(src0, src1)) {
|
90
|
+
BCAST_SHAPE(src0, src1)
|
91
|
+
*acl_src0 = ggml_cann_create_tensor(src0, BCAST_PARAM(src0));
|
92
|
+
*acl_src1 = ggml_cann_create_tensor(src1, BCAST_PARAM(src1));
|
93
|
+
*acl_dst = ggml_cann_create_tensor(dst, BCAST_PARAM(src0));
|
94
|
+
} else {
|
95
|
+
*acl_src0 = ggml_cann_create_tensor(src0);
|
96
|
+
*acl_src1 = ggml_cann_create_tensor(src1);
|
97
|
+
*acl_dst = ggml_cann_create_tensor(dst);
|
98
|
+
}
|
99
|
+
}
|
100
|
+
|
101
|
+
void ggml_cann_unary_op(
|
102
|
+
std::function<void(ggml_backend_cann_context&, aclTensor*, aclTensor*)> unary_op,
|
103
|
+
ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
104
|
+
ggml_tensor* src = dst->src[0];
|
105
|
+
|
106
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
107
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
108
|
+
|
109
|
+
unary_op(ctx, acl_src, acl_dst);
|
110
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst);
|
111
|
+
}
|
112
|
+
|
113
|
+
/**
|
114
|
+
* @brief Repeats elements of a tensor along each dimension according to the
|
115
|
+
* specified repeat array.
|
116
|
+
*
|
117
|
+
* @param ctx The context for the CANN backend operations.
|
118
|
+
* @param acl_src The source tensor to be repeated.
|
119
|
+
* @param acl_dst The destination tensor after repeating.
|
120
|
+
* @param repeat_array The array specifying the number of repetitions along each
|
121
|
+
* dimension.
|
122
|
+
*/
|
123
|
+
static void aclnn_repeat(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
124
|
+
aclTensor* acl_dst, int64_t* repeat_array) {
|
125
|
+
// repeat tensor along each dim with repeat_array
|
126
|
+
aclIntArray* repeats = aclCreateIntArray(repeat_array, GGML_MAX_DIMS);
|
127
|
+
|
128
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Repeat, acl_src, repeats, acl_dst);
|
129
|
+
ggml_cann_release_resources(ctx, repeats);
|
130
|
+
}
|
131
|
+
|
132
|
+
/**
|
133
|
+
* @brief Casts the data type of a source tensor to a destination tensor.
|
134
|
+
*
|
135
|
+
* This function casts the data type of the source tensor `acl_src` to the
|
136
|
+
* specified data type `cast_data_type` and stores the result in the destination
|
137
|
+
* tensor `acl_dst`.
|
138
|
+
*
|
139
|
+
* @param ctx The context for the CANN backend operations.
|
140
|
+
* @param acl_src The source tensor whose data type will be casted.
|
141
|
+
* @param acl_dst The destination tensor where the casted result will be stored.
|
142
|
+
* @param cast_data_type The target data type to which the source tensor will be
|
143
|
+
* casted.
|
144
|
+
*/
|
145
|
+
static void aclnn_cast(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
146
|
+
aclTensor* acl_dst, aclDataType cast_data_type) {
|
147
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Cast, acl_src, cast_data_type, acl_dst);
|
148
|
+
}
|
149
|
+
|
150
|
+
void ggml_cann_repeat(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
151
|
+
ggml_tensor* src = dst->src[0];
|
152
|
+
GGML_ASSERT(ggml_can_repeat(src, dst));
|
153
|
+
|
154
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
155
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
156
|
+
|
157
|
+
int64_t repeatsArray[] = {dst->ne[3] / src->ne[3], dst->ne[2] / src->ne[2],
|
158
|
+
dst->ne[1] / src->ne[1], dst->ne[0] / src->ne[0]};
|
159
|
+
|
160
|
+
aclnn_repeat(ctx, acl_src, acl_dst, repeatsArray);
|
161
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst);
|
162
|
+
}
|
163
|
+
|
164
|
+
void aclnn_add(ggml_backend_cann_context& ctx, aclTensor* acl_src0,
|
165
|
+
aclTensor* acl_src1, aclTensor* acl_dst) {
|
166
|
+
float alphaValue = 1.0f;
|
167
|
+
aclScalar* alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
|
168
|
+
if (acl_dst != nullptr)
|
169
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Add, acl_src0, acl_src1, alpha, acl_dst);
|
170
|
+
else
|
171
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_src0, acl_src1, alpha);
|
172
|
+
ggml_cann_release_resources(ctx, alpha);
|
173
|
+
}
|
174
|
+
|
175
|
+
void aclnn_sub(ggml_backend_cann_context& ctx, aclTensor* acl_src0,
|
176
|
+
aclTensor* acl_src1, aclTensor* acl_dst) {
|
177
|
+
float alphaValue = 1.0f;
|
178
|
+
aclScalar* alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
|
179
|
+
if (acl_dst != nullptr)
|
180
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Sub, acl_src0, acl_src1, alpha, acl_dst);
|
181
|
+
else
|
182
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceSub, acl_src0, acl_src1, alpha);
|
183
|
+
ggml_cann_release_resources(ctx, alpha);
|
184
|
+
}
|
185
|
+
|
186
|
+
void aclnn_mul(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
187
|
+
aclTensor* acl_other, aclTensor* acl_dst) {
|
188
|
+
if (acl_dst != nullptr)
|
189
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Mul, acl_src, acl_other, acl_dst);
|
190
|
+
else
|
191
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, acl_src, acl_other);
|
192
|
+
}
|
193
|
+
|
194
|
+
void aclnn_div(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
195
|
+
aclTensor* acl_other, aclTensor* acl_dst) {
|
196
|
+
if (acl_dst != nullptr)
|
197
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Div, acl_src, acl_other, acl_dst);
|
198
|
+
else
|
199
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceDiv, acl_src, acl_other);
|
200
|
+
}
|
201
|
+
|
202
|
+
/**
|
203
|
+
* @brief Multiplies elements of a tensor by a scalar value, optionally
|
204
|
+
* in-place.
|
205
|
+
*
|
206
|
+
* This function multiplies each element of the source tensor `acl_src` by the
|
207
|
+
* scalar `scale` and stores the result in the destination tensor `acl_dst`. If
|
208
|
+
* `inplace` is true, `acl_dst` will not be used and the operation is performed
|
209
|
+
* in-place on `acl_src`.
|
210
|
+
* The operation is defined as:
|
211
|
+
* \f[
|
212
|
+
* \text {acl_dst }_i=\text {acl_src }_i \times \text {scale}
|
213
|
+
* \f]
|
214
|
+
*
|
215
|
+
* @param ctx The context for the CANN backend operations.
|
216
|
+
* @param acl_src The source tensor whose elements will be multiplied.
|
217
|
+
* @param scale The scalar value by which each element of `acl_src` will be
|
218
|
+
* multiplied.
|
219
|
+
* @param acl_dst The destination tensor where the result will be stored if
|
220
|
+
* `inplace` is false.
|
221
|
+
* @param inplace Flag indicating whether to perform the operation in-place on
|
222
|
+
* `acl_src`.
|
223
|
+
*/
|
224
|
+
static void aclnn_muls(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
225
|
+
float scale, aclTensor* acl_dst, bool inplace) {
|
226
|
+
aclScalar* acl_scale = aclCreateScalar(&scale, aclDataType::ACL_FLOAT);
|
227
|
+
if (inplace) {
|
228
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMuls, acl_src, acl_scale);
|
229
|
+
} else {
|
230
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Muls, acl_src, acl_scale, acl_dst);
|
231
|
+
}
|
232
|
+
ggml_cann_release_resources(ctx, acl_scale);
|
233
|
+
}
|
234
|
+
|
235
|
+
void ggml_cann_leaky_relu(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
236
|
+
ggml_tensor* src = dst->src[0];
|
237
|
+
|
238
|
+
GGML_ASSERT(src->type == GGML_TYPE_F32);
|
239
|
+
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
240
|
+
|
241
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
242
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
243
|
+
|
244
|
+
float negative_slope;
|
245
|
+
memcpy(&negative_slope, dst->op_params, sizeof(float));
|
246
|
+
aclScalar* acl_negative_slope =
|
247
|
+
aclCreateScalar(&negative_slope, aclDataType::ACL_FLOAT);
|
248
|
+
|
249
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, LeakyRelu, acl_src, acl_negative_slope, acl_dst);
|
250
|
+
ggml_cann_release_resources(ctx, acl_negative_slope, acl_src, acl_dst);
|
251
|
+
}
|
252
|
+
|
253
|
+
/**
|
254
|
+
* @brief Concatenates a list of tensors along a specified dimension and stores
|
255
|
+
* the result in a destination tensor.
|
256
|
+
*
|
257
|
+
* @param ctx The context for the CANN backend operations.
|
258
|
+
* @param tensorList The list of tensors to be concatenated.
|
259
|
+
* @param acl_dst The destination tensor where the concatenated result will be
|
260
|
+
* stored.
|
261
|
+
* @param concat_dim The dimension along which the tensors will be concatenated.
|
262
|
+
*/
|
263
|
+
static void aclnn_concat(ggml_backend_cann_context& ctx,
|
264
|
+
aclTensorList* tensorList, aclTensor* acl_dst,
|
265
|
+
int64_t concat_dim) {
|
266
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Cat, tensorList, concat_dim, acl_dst);
|
267
|
+
}
|
268
|
+
|
269
|
+
void ggml_cann_concat(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
270
|
+
ggml_tensor* src0 = dst->src[0];
|
271
|
+
ggml_tensor* src1 = dst->src[1];
|
272
|
+
aclTensor* acl_src0 = ggml_cann_create_tensor(src0);
|
273
|
+
aclTensor* acl_src1 = ggml_cann_create_tensor(src1);
|
274
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
275
|
+
|
276
|
+
const int32_t dim = ggml_get_op_params_i32(dst, 0);
|
277
|
+
|
278
|
+
GGML_ASSERT(dim >= 0 && dim < 4);
|
279
|
+
int32_t acl_dim = 3 - dim;
|
280
|
+
|
281
|
+
aclTensor* tensors[] = {acl_src0, acl_src1};
|
282
|
+
aclTensorList* tensor_list = aclCreateTensorList(tensors, 2);
|
283
|
+
aclnn_concat(ctx, tensor_list, acl_dst, acl_dim);
|
284
|
+
|
285
|
+
ggml_cann_release_resources(ctx, tensor_list, acl_dst);
|
286
|
+
}
|
287
|
+
|
288
|
+
/**
|
289
|
+
* @brief Creates a tensor with values starting from `start`, incremented by
|
290
|
+
* `step`, and ending before `stop`.
|
291
|
+
*
|
292
|
+
* This function performs the operation:
|
293
|
+
* \f[
|
294
|
+
* \text {out }_{i+1}=\text {out }_i+\text {step}
|
295
|
+
* \f]
|
296
|
+
* the range is [start, stop).
|
297
|
+
*
|
298
|
+
* @param ctx The context for the CANN backend operations.
|
299
|
+
* @param acl_dst The destination tensor where the values will be stored.
|
300
|
+
* @param start The starting value of the range.
|
301
|
+
* @param stop The ending value of the range (exclusive).
|
302
|
+
* @param step The step size between consecutive values.
|
303
|
+
* @param n_elements The number of elements in the destination tensor.
|
304
|
+
*/
|
305
|
+
static void aclnn_arange(ggml_backend_cann_context& ctx, aclTensor* acl_dst,
|
306
|
+
float start, float stop, float step,
|
307
|
+
int64_t n_elements) {
|
308
|
+
int64_t steps = (int64_t)std::ceil((stop - start) / step);
|
309
|
+
GGML_ASSERT(n_elements == steps);
|
310
|
+
|
311
|
+
aclScalar* acl_start = aclCreateScalar(&start, aclDataType::ACL_FLOAT);
|
312
|
+
aclScalar* acl_end = aclCreateScalar(&stop, aclDataType::ACL_FLOAT);
|
313
|
+
aclScalar* acl_step = aclCreateScalar(&step, aclDataType::ACL_FLOAT);
|
314
|
+
|
315
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Arange, acl_start, acl_end, acl_step, acl_dst);
|
316
|
+
ggml_cann_release_resources(ctx, acl_start, acl_end, acl_step);
|
317
|
+
}
|
318
|
+
|
319
|
+
void ggml_cann_arange(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
320
|
+
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
321
|
+
|
322
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
323
|
+
|
324
|
+
int64_t n_elements = ggml_nelements(dst);
|
325
|
+
float start;
|
326
|
+
float stop;
|
327
|
+
float step;
|
328
|
+
memcpy(&start, (float*)dst->op_params + 0, sizeof(float));
|
329
|
+
memcpy(&stop, (float*)dst->op_params + 1, sizeof(float));
|
330
|
+
memcpy(&step, (float*)dst->op_params + 2, sizeof(float));
|
331
|
+
|
332
|
+
aclnn_arange(ctx, acl_dst, start, stop, step, n_elements);
|
333
|
+
ggml_cann_release_resources(ctx, acl_dst);
|
334
|
+
}
|
335
|
+
|
336
|
+
void ggml_cann_clamp(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
337
|
+
ggml_tensor* src = dst->src[0];
|
338
|
+
|
339
|
+
float min;
|
340
|
+
float max;
|
341
|
+
memcpy(&min, dst->op_params, sizeof(float));
|
342
|
+
memcpy(&max, (float*)dst->op_params + 1, sizeof(float));
|
343
|
+
|
344
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
345
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
346
|
+
|
347
|
+
aclScalar* acl_min = aclCreateScalar(&min, aclDataType::ACL_FLOAT);
|
348
|
+
aclScalar* acl_max = aclCreateScalar(&max, aclDataType::ACL_FLOAT);
|
349
|
+
|
350
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Clamp, acl_src, acl_min, acl_max, acl_dst);
|
351
|
+
ggml_cann_release_resources(ctx, acl_min, acl_max, acl_src, acl_dst);
|
352
|
+
}
|
353
|
+
|
354
|
+
void ggml_cann_scale(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
355
|
+
ggml_tensor* src = dst->src[0];
|
356
|
+
|
357
|
+
// scale factor
|
358
|
+
float v;
|
359
|
+
memcpy(&v, dst->op_params, sizeof(float));
|
360
|
+
|
361
|
+
aclScalar* scale = aclCreateScalar(&v, aclDataType::ACL_FLOAT);
|
362
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
363
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
364
|
+
|
365
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Muls, acl_src, scale, acl_dst);
|
366
|
+
ggml_cann_release_resources(ctx, scale, acl_src, acl_dst);
|
367
|
+
}
|
368
|
+
|
369
|
+
void ggml_cann_argsort(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
370
|
+
ggml_tensor* src = dst->src[0];
|
371
|
+
enum ggml_sort_order order = (enum ggml_sort_order)dst->op_params[0];
|
372
|
+
|
373
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
374
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
375
|
+
ggml_cann_pool_alloc temp_buffer_allocator(
|
376
|
+
ctx.pool(), ggml_nelements(dst) * sizeof(int64_t));
|
377
|
+
void* buffer = temp_buffer_allocator.get();
|
378
|
+
aclTensor* tmp_tensor =
|
379
|
+
ggml_cann_create_tensor(buffer, ACL_INT64, ggml_type_size(dst->type),
|
380
|
+
dst->ne, dst->nb, GGML_MAX_DIMS);
|
381
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Argsort, acl_src, -1, (order == GGML_SORT_ORDER_DESC ? true : false),
|
382
|
+
tmp_tensor);
|
383
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Cast, tmp_tensor, ggml_cann_type_mapping(dst->type), acl_dst);
|
384
|
+
ggml_cann_release_resources(ctx, acl_src, tmp_tensor, acl_dst);
|
385
|
+
}
|
386
|
+
|
387
|
+
void ggml_cann_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
388
|
+
ggml_tensor* src = dst->src[0];
|
389
|
+
|
390
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
391
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
392
|
+
|
393
|
+
float eps;
|
394
|
+
memcpy(&eps, dst->op_params, sizeof(float));
|
395
|
+
|
396
|
+
std::vector<int64_t> normData = {dst->ne[0]};
|
397
|
+
aclIntArray* norm = aclCreateIntArray(normData.data(), normData.size());
|
398
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, LayerNorm, acl_src, norm, nullptr, nullptr,
|
399
|
+
eps, acl_dst, nullptr, nullptr);
|
400
|
+
ggml_cann_release_resources(ctx, norm, acl_src, acl_dst);
|
401
|
+
}
|
402
|
+
|
403
|
+
void ggml_cann_group_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
404
|
+
ggml_tensor* src = dst->src[0];
|
405
|
+
|
406
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
407
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
408
|
+
|
409
|
+
int n_groups = dst->op_params[0];
|
410
|
+
|
411
|
+
float eps;
|
412
|
+
memcpy(&eps, dst->op_params + 1, sizeof(float));
|
413
|
+
|
414
|
+
int64_t N = src->ne[3];
|
415
|
+
int64_t C = src->ne[2];
|
416
|
+
int64_t HxW = src->ne[1] * src->ne[0];
|
417
|
+
|
418
|
+
size_t type_size = ggml_type_size(src->type);
|
419
|
+
int64_t ne[] = {n_groups, N};
|
420
|
+
size_t nb[] = {type_size, type_size * n_groups};
|
421
|
+
size_t n_bytes = N * n_groups;
|
422
|
+
|
423
|
+
ggml_cann_pool_alloc temp_buffer_allocator(ctx.pool(), n_bytes * 2);
|
424
|
+
void* buffer = temp_buffer_allocator.get();
|
425
|
+
aclTensor* acl_mean_out = ggml_cann_create_tensor(
|
426
|
+
buffer, ACL_FLOAT, type_size, ne, nb, ACL_FORMAT_ND);
|
427
|
+
aclTensor* acl_rstd_out = ggml_cann_create_tensor(
|
428
|
+
(char*)buffer + n_bytes, ACL_FLOAT, type_size, ne, nb, ACL_FORMAT_ND);
|
429
|
+
|
430
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, GroupNorm, acl_src, nullptr, nullptr, N, C, HxW, n_groups, eps,
|
431
|
+
acl_dst, acl_mean_out, acl_rstd_out);
|
432
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst, acl_mean_out, acl_rstd_out);
|
433
|
+
}
|
434
|
+
|
435
|
+
void ggml_cann_acc(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
436
|
+
ggml_tensor* src0 = dst->src[0];
|
437
|
+
ggml_tensor* src1 = dst->src[1];
|
438
|
+
|
439
|
+
size_t nb1 = ((int32_t*)dst->op_params)[0];
|
440
|
+
size_t nb2 = ((int32_t*)dst->op_params)[1];
|
441
|
+
size_t nb3 = ((int32_t*)dst->op_params)[2];
|
442
|
+
size_t offset = ((int32_t*)dst->op_params)[3];
|
443
|
+
bool inplace = (bool)((int32_t*)dst->op_params)[4];
|
444
|
+
|
445
|
+
size_t param_nb[] = {ggml_element_size(src0), nb1, nb2, nb3};
|
446
|
+
|
447
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(
|
448
|
+
dst, src1->ne, param_nb, GGML_MAX_DIMS, ACL_FORMAT_ND, offset);
|
449
|
+
aclTensor* acl_src1 = ggml_cann_create_tensor(src1);
|
450
|
+
|
451
|
+
aclScalar* alpha = nullptr;
|
452
|
+
float alphaValue = 1.0f;
|
453
|
+
alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
|
454
|
+
|
455
|
+
if (!inplace) {
|
456
|
+
size_t cpy_size = ggml_nbytes(dst);
|
457
|
+
ggml_cann_async_memcpy(ctx, dst->data, src0->data, cpy_size,
|
458
|
+
ACL_MEMCPY_DEVICE_TO_DEVICE);
|
459
|
+
aclTensor* acl_src0 = ggml_cann_create_tensor(
|
460
|
+
src0, src1->ne, src0->nb, GGML_MAX_DIMS, ACL_FORMAT_ND, offset);
|
461
|
+
|
462
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Add, acl_src0, acl_src1, alpha, acl_dst);
|
463
|
+
ggml_cann_release_resources(ctx, acl_src0);
|
464
|
+
} else {
|
465
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst, acl_src1, alpha);
|
466
|
+
}
|
467
|
+
ggml_cann_release_resources(ctx, acl_src1, acl_dst);
|
468
|
+
}
|
469
|
+
|
470
|
+
/**
|
471
|
+
* @brief Performs sum reduction on a given tensor along specified dimensions.
|
472
|
+
*
|
473
|
+
* This function reduces the input tensor by summing along the specified dimensions.
|
474
|
+
*
|
475
|
+
* @param ctx The context for the CANN backend operations.
|
476
|
+
* @param dst The destination tensor where the reduced result will be stored.
|
477
|
+
* @param dim An array of dimension indices.
|
478
|
+
* @param dim_size The number of dimensions.
|
479
|
+
*/
|
480
|
+
static void aclnn_reduce_sum(ggml_backend_cann_context& ctx, ggml_tensor* dst,
|
481
|
+
int64_t* dim, size_t dim_size) {
|
482
|
+
GGML_ASSERT(dst->ne[0] == 1);
|
483
|
+
ggml_tensor* src = dst->src[0];
|
484
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
485
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
486
|
+
aclIntArray* reduce_dims = aclCreateIntArray(dim, dim_size);
|
487
|
+
|
488
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, ReduceSum, acl_src, reduce_dims, true,
|
489
|
+
ggml_cann_type_mapping(dst->type), acl_dst);
|
490
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst, reduce_dims);
|
491
|
+
}
|
492
|
+
|
493
|
+
void ggml_cann_sum_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
494
|
+
int64_t reduce_dims[] = {3};
|
495
|
+
aclnn_reduce_sum(ctx, dst, reduce_dims, 1);
|
496
|
+
}
|
497
|
+
|
498
|
+
void ggml_cann_sum(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
499
|
+
int64_t reduce_dims[] = {0, 1, 2, 3};
|
500
|
+
aclnn_reduce_sum(ctx, dst, reduce_dims, 4);
|
501
|
+
}
|
502
|
+
|
503
|
+
void ggml_cann_upsample_nearest2d(ggml_backend_cann_context& ctx,
|
504
|
+
ggml_tensor* dst) {
|
505
|
+
ggml_tensor* src = dst->src[0];
|
506
|
+
aclTensor* acl_src =
|
507
|
+
ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
|
508
|
+
aclTensor* acl_dst =
|
509
|
+
ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
|
510
|
+
|
511
|
+
std::vector<int64_t> output_size{dst->ne[1], dst->ne[0]};
|
512
|
+
auto output_size_array = aclCreateIntArray(output_size.data(), 2);
|
513
|
+
|
514
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, UpsampleNearest2d, acl_src, output_size_array, acl_dst);
|
515
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst, output_size_array);
|
516
|
+
}
|
517
|
+
|
518
|
+
/**
|
519
|
+
* @brief Pads a tensor with a specified value along each dimension.
|
520
|
+
*
|
521
|
+
* This function performs padding of the source tensor `acl_src` and stores the
|
522
|
+
* result in the destination tensor `acl_dst`. The padding values for each
|
523
|
+
* dimension are specified in the `paddings` array.
|
524
|
+
*
|
525
|
+
* @param ctx The context for the CANN backend operations.
|
526
|
+
* @param acl_src The source tensor to be padded.
|
527
|
+
* @param acl_dst The destination tensor where the padded result will be stored.
|
528
|
+
* @param paddings An array specifying the padding values for each dimension.
|
529
|
+
* The size of the array should be twice the number of dimensions of the tensor.
|
530
|
+
* @param value The value to be used for padding. The default value is 0.0.
|
531
|
+
*/
|
532
|
+
static void aclnn_pad(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
533
|
+
aclTensor* acl_dst, int64_t* paddings,
|
534
|
+
float value = 0.0f) {
|
535
|
+
aclIntArray* acl_pad = aclCreateIntArray(paddings, GGML_MAX_DIMS * 2);
|
536
|
+
aclScalar* acl_value = aclCreateScalar(&value, aclDataType::ACL_FLOAT);
|
537
|
+
|
538
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, ConstantPadNd, acl_src, acl_pad, acl_value, acl_dst);
|
539
|
+
ggml_cann_release_resources(ctx, acl_pad, acl_value);
|
540
|
+
}
|
541
|
+
|
542
|
+
void ggml_cann_pad(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
543
|
+
ggml_tensor* src = dst->src[0];
|
544
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
545
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
546
|
+
|
547
|
+
// padding: value in the array means how much distance will be padding.
|
548
|
+
// the position of elements in the array means which dirction to padding,
|
549
|
+
// each position means: [dim0.front, dim0.behind, dim1.front, dim1.behind,
|
550
|
+
// dim2.front, dim2.behind, dim3.front, dim3.behind]
|
551
|
+
int64_t paddings[] = {
|
552
|
+
0, dst->ne[0] - src->ne[0], 0, dst->ne[1] - src->ne[1],
|
553
|
+
0, dst->ne[2] - src->ne[2], 0, dst->ne[3] - src->ne[3]};
|
554
|
+
aclnn_pad(ctx, acl_src, acl_dst, paddings);
|
555
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst);
|
556
|
+
}
|
557
|
+
|
558
|
+
/**
|
559
|
+
* @brief Performs 2D average pooling on the input tensor and stores the result
|
560
|
+
* in the destination tensor.
|
561
|
+
*
|
562
|
+
* This function performs average pooling on the source tensor and stores the
|
563
|
+
* result in the destination tensor. The pooling parameters (kernel size,
|
564
|
+
* strides, padding) are specified in the `op_params` of the destination tensor.
|
565
|
+
*
|
566
|
+
* @param ctx The context for the CANN backend operations.
|
567
|
+
* @param dst The destination tensor where the result will be stored. The source
|
568
|
+
* tensor is referenced by `dst->src[0]`.
|
569
|
+
*/
|
570
|
+
static void ggml_cann_avg_pool2d(ggml_backend_cann_context& ctx,
|
571
|
+
ggml_tensor* dst) {
|
572
|
+
ggml_tensor* src = dst->src[0];
|
573
|
+
GGML_ASSERT(src->type == GGML_TYPE_F32);
|
574
|
+
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
575
|
+
|
576
|
+
aclTensor* acl_src =
|
577
|
+
ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
|
578
|
+
aclTensor* acl_dst =
|
579
|
+
ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
|
580
|
+
|
581
|
+
const int32_t* opts = (const int32_t*)dst->op_params;
|
582
|
+
const int k0 = opts[1];
|
583
|
+
const int k1 = opts[2];
|
584
|
+
const int s0 = opts[3];
|
585
|
+
const int s1 = opts[4];
|
586
|
+
const int p0 = opts[5];
|
587
|
+
const int p1 = opts[6];
|
588
|
+
|
589
|
+
std::vector<int64_t> kernel_dims = {k1, k0};
|
590
|
+
std::vector<int64_t> stride_dims = {s1, s0};
|
591
|
+
std::vector<int64_t> padding_avg_dims = {p1, p0}; // (padH, padW)
|
592
|
+
|
593
|
+
auto* kernel_size = aclCreateIntArray(kernel_dims.data(), 2);
|
594
|
+
auto* strides = aclCreateIntArray(stride_dims.data(), 2);
|
595
|
+
auto* paddings_avg = aclCreateIntArray(padding_avg_dims.data(), 2);
|
596
|
+
|
597
|
+
bool ceil_mode = false;
|
598
|
+
bool count_include_pad = true;
|
599
|
+
int64_t divisor_override = 0;
|
600
|
+
int8_t cube_math_type = 0;
|
601
|
+
#ifdef ASCEND_310P
|
602
|
+
cube_math_type = 1;
|
603
|
+
#endif
|
604
|
+
|
605
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, AvgPool2d, acl_src, kernel_size, strides, paddings_avg,
|
606
|
+
ceil_mode, count_include_pad, divisor_override,
|
607
|
+
cube_math_type, acl_dst);
|
608
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst, kernel_size, strides,
|
609
|
+
paddings_avg);
|
610
|
+
}
|
611
|
+
|
612
|
+
/**
|
613
|
+
* @brief Performs 2D max pooling on the input tensor and stores the result in
|
614
|
+
* the destination tensor.
|
615
|
+
*
|
616
|
+
* This function performs max pooling on the source tensor and stores the result
|
617
|
+
* in the destination tensor. The pooling parameters (kernel size, strides,
|
618
|
+
* padding) are specified in the `op_params` of the destination tensor.
|
619
|
+
*
|
620
|
+
* @param ctx The context for the CANN backend operations.
|
621
|
+
* @param dst The destination tensor where the result will be stored. The source
|
622
|
+
* tensor is referenced by `dst->src[0]`.
|
623
|
+
*/
|
624
|
+
static void ggml_cann_max_pool2d(ggml_backend_cann_context& ctx,
|
625
|
+
ggml_tensor* dst) {
|
626
|
+
ggml_tensor* src = dst->src[0];
|
627
|
+
GGML_ASSERT(src->type == GGML_TYPE_F32);
|
628
|
+
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
629
|
+
|
630
|
+
aclTensor* acl_src =
|
631
|
+
ggml_cann_create_tensor(src, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
|
632
|
+
aclTensor* acl_dst =
|
633
|
+
ggml_cann_create_tensor(dst, nullptr, nullptr, 0, ACL_FORMAT_NCHW);
|
634
|
+
|
635
|
+
const int32_t* opts = (const int32_t*)dst->op_params;
|
636
|
+
const int k0 = opts[1];
|
637
|
+
const int k1 = opts[2];
|
638
|
+
const int s0 = opts[3];
|
639
|
+
const int s1 = opts[4];
|
640
|
+
const int p0 = opts[5];
|
641
|
+
const int p1 = opts[6];
|
642
|
+
|
643
|
+
int64_t temp_ne[] = {src->ne[0] + p0 * 2, src->ne[1] + p1 * 2, src->ne[2],
|
644
|
+
src->ne[3]};
|
645
|
+
size_t temp_nb[GGML_MAX_DIMS];
|
646
|
+
|
647
|
+
temp_nb[0] = ggml_element_size(src);
|
648
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
649
|
+
temp_nb[i] = temp_nb[i - 1] * temp_ne[i - 1];
|
650
|
+
}
|
651
|
+
|
652
|
+
ggml_cann_pool_alloc temp_buffer_allocator(
|
653
|
+
ctx.pool(), ggml_nbytes(src) + p0 * 2 + p1 * 2 * src->nb[1]);
|
654
|
+
void* buffer = temp_buffer_allocator.get();
|
655
|
+
aclTensor* tmp_tensor = ggml_cann_create_tensor(
|
656
|
+
buffer, ACL_FLOAT, ggml_element_size(src), temp_ne, temp_nb,
|
657
|
+
GGML_MAX_DIMS, ACL_FORMAT_NCHW);
|
658
|
+
|
659
|
+
// pad: see padding in ggml_cann_pad()
|
660
|
+
int64_t paddings[] = {p0, p0, p1, p1, 0, 0, 0, 0};
|
661
|
+
float value = -FLT_MAX;
|
662
|
+
aclnn_pad(ctx, acl_src, tmp_tensor, paddings, value);
|
663
|
+
|
664
|
+
// max_pool
|
665
|
+
std::vector<int64_t> kernel_dims = {k1, k0};
|
666
|
+
std::vector<int64_t> stride_dims = {s1, s0};
|
667
|
+
// padding_max_dims: [dim0_start, dim0_end, dim1_start, dim1_end]
|
668
|
+
std::vector<int64_t> padding_max_dims = {0, 0, 0, 0};
|
669
|
+
std::vector<int64_t> dilation_size = {1, 1};
|
670
|
+
auto* kernel_size = aclCreateIntArray(kernel_dims.data(), 2);
|
671
|
+
auto* strides = aclCreateIntArray(stride_dims.data(), 2);
|
672
|
+
auto* paddings_max = aclCreateIntArray(padding_max_dims.data(), 4);
|
673
|
+
auto* dilations = aclCreateIntArray(dilation_size.data(), 2);
|
674
|
+
|
675
|
+
bool ceil_mode = false;
|
676
|
+
int64_t auto_pads = 0;
|
677
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, MaxPool, tmp_tensor, kernel_size, strides, auto_pads,
|
678
|
+
paddings_max, dilations, ceil_mode, acl_dst);
|
679
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst, tmp_tensor, kernel_size,
|
680
|
+
strides, paddings_max, dilations);
|
681
|
+
}
|
682
|
+
|
683
|
+
void ggml_cann_pool2d(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
684
|
+
const int32_t* opts = (const int32_t*)dst->op_params;
|
685
|
+
enum ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);
|
686
|
+
switch (op) {
|
687
|
+
case GGML_OP_POOL_AVG:
|
688
|
+
ggml_cann_avg_pool2d(ctx, dst);
|
689
|
+
break;
|
690
|
+
case GGML_OP_POOL_MAX:
|
691
|
+
ggml_cann_max_pool2d(ctx, dst);
|
692
|
+
break;
|
693
|
+
case GGML_OP_POOL_COUNT:
|
694
|
+
GGML_ABORT("fatal error");
|
695
|
+
break;
|
696
|
+
}
|
697
|
+
}
|
698
|
+
|
699
|
+
/**
|
700
|
+
* @brief Copies data from the source tensor to the destination tensor.
|
701
|
+
*
|
702
|
+
* This function copies data from the source tensor `acl_src` to the destination
|
703
|
+
* tensor `acl_dst`.
|
704
|
+
*
|
705
|
+
* @param ctx The context for the CANN backend operations.
|
706
|
+
* @param acl_src The source tensor from which data will be copied.
|
707
|
+
* @param acl_dst The destination tensor where the data will be copied to.
|
708
|
+
*/
|
709
|
+
static void cann_copy(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
710
|
+
aclTensor* acl_dst) {
|
711
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceCopy, acl_dst, acl_src);
|
712
|
+
}
|
713
|
+
|
714
|
+
void ggml_cann_dup(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
715
|
+
ggml_tensor* src0 = dst->src[0];
|
716
|
+
|
717
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src0);
|
718
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
719
|
+
if (ggml_are_same_shape(src0, dst)) {
|
720
|
+
if (dst->type == src0->type) {
|
721
|
+
cann_copy(ctx, acl_src, acl_dst);
|
722
|
+
} else {
|
723
|
+
aclnn_cast(ctx, acl_src, acl_dst, ggml_cann_type_mapping(dst->type));
|
724
|
+
}
|
725
|
+
} else {
|
726
|
+
if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {
|
727
|
+
if (dst->type == src0->type) {
|
728
|
+
size_t cpy_size = ggml_nbytes(dst);
|
729
|
+
ggml_cann_async_memcpy(ctx, dst->data, src0->data, cpy_size,
|
730
|
+
ACL_MEMCPY_DEVICE_TO_DEVICE);
|
731
|
+
return;
|
732
|
+
} else {
|
733
|
+
ggml_cann_pool_alloc src_buffer_allocator(
|
734
|
+
ctx.pool(),
|
735
|
+
ggml_nelements(dst) * ggml_type_size(dst->type));
|
736
|
+
void* src_trans_buffer = src_buffer_allocator.get();
|
737
|
+
size_t src_trans_nb[GGML_MAX_DIMS];
|
738
|
+
src_trans_nb[0] = ggml_type_size(dst->type);
|
739
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
740
|
+
src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];
|
741
|
+
}
|
742
|
+
aclTensor* src_trans_tensor = ggml_cann_create_tensor(
|
743
|
+
src_trans_buffer, ggml_cann_type_mapping(dst->type),
|
744
|
+
ggml_type_size(dst->type), src0->ne, src_trans_nb,
|
745
|
+
GGML_MAX_DIMS);
|
746
|
+
|
747
|
+
aclnn_cast(ctx, acl_src, src_trans_tensor, ggml_cann_type_mapping(dst->type));
|
748
|
+
size_t cpy_size = ggml_nbytes(dst);
|
749
|
+
ggml_cann_async_memcpy(ctx, dst->data, src_trans_buffer, cpy_size,
|
750
|
+
ACL_MEMCPY_DEVICE_TO_DEVICE);
|
751
|
+
ggml_cann_release_resources(ctx, src_trans_tensor);
|
752
|
+
return;
|
753
|
+
}
|
754
|
+
} else if (ggml_is_contiguous(dst)) {
|
755
|
+
ggml_cann_pool_alloc src_buffer_allocator(
|
756
|
+
ctx.pool(), ggml_nelements(dst) * ggml_type_size(dst->type));
|
757
|
+
void* src_trans_buffer = src_buffer_allocator.get();
|
758
|
+
size_t src_trans_nb[GGML_MAX_DIMS];
|
759
|
+
src_trans_nb[0] = ggml_type_size(dst->type);
|
760
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
761
|
+
src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];
|
762
|
+
}
|
763
|
+
aclTensor* src_trans_tensor = ggml_cann_create_tensor(
|
764
|
+
src_trans_buffer, ggml_cann_type_mapping(dst->type),
|
765
|
+
ggml_type_size(dst->type), src0->ne, src_trans_nb,
|
766
|
+
GGML_MAX_DIMS);
|
767
|
+
|
768
|
+
aclnn_cast(ctx, acl_src, src_trans_tensor, ggml_cann_type_mapping(dst->type));
|
769
|
+
|
770
|
+
size_t cpy_size = ggml_nbytes(dst);
|
771
|
+
ggml_cann_async_memcpy(ctx, dst->data, src_trans_buffer, cpy_size,
|
772
|
+
ACL_MEMCPY_DEVICE_TO_DEVICE);
|
773
|
+
ggml_cann_release_resources(ctx, src_trans_tensor);
|
774
|
+
return;
|
775
|
+
} else {
|
776
|
+
GGML_ABORT("Unsupport dst is not tontiguous.");
|
777
|
+
}
|
778
|
+
}
|
779
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst);
|
780
|
+
}
|
781
|
+
|
782
|
+
/**
|
783
|
+
* @brief Creates an ACL tensor initialized with zeros using a provided buffer.
|
784
|
+
*
|
785
|
+
* This function initializes a tensor with zeros using the specified buffer and
|
786
|
+
* tensor parameters.
|
787
|
+
*
|
788
|
+
* @param ctx The context for the CANN backend operations.
|
789
|
+
* @param buffer The buffer to be used for the tensor data.
|
790
|
+
* @param n_bytes The size of the buffer in bytes.
|
791
|
+
* @param ne An array specifying the extents (sizes) of each dimension of the
|
792
|
+
* tensor.
|
793
|
+
* @param dims The number of dimensions of the tensor.
|
794
|
+
* @param type The data type of the tensor.
|
795
|
+
* @param type_size The size of each element in the tensor data type.
|
796
|
+
* @return An ACL tensor initialized with zeros.
|
797
|
+
*/
|
798
|
+
static aclTensor* aclnn_zero(ggml_backend_cann_context& ctx, void* buffer,
|
799
|
+
size_t n_bytes, int64_t* ne, int64_t dims,
|
800
|
+
aclDataType type, size_t type_size) {
|
801
|
+
size_t nb[GGML_MAX_DIMS];
|
802
|
+
nb[0] = type_size;
|
803
|
+
for (int i = 1; i < dims; i++) {
|
804
|
+
nb[i] = nb[i - 1] * ne[i - 1];
|
805
|
+
}
|
806
|
+
|
807
|
+
ggml_cann_async_memset(ctx, buffer, n_bytes, 0);
|
808
|
+
aclTensor* zero =
|
809
|
+
ggml_cann_create_tensor(buffer, type, type_size, ne, nb, dims);
|
810
|
+
return zero;
|
811
|
+
}
|
812
|
+
|
813
|
+
/**
|
814
|
+
* @brief Creates an ACL tensor initialized with value using a provided buffer.
|
815
|
+
*
|
816
|
+
* This function initializes a tensor with value using the specified buffer and
|
817
|
+
* tensor parameters.
|
818
|
+
*
|
819
|
+
* @param ctx The context for the CANN backend operations.
|
820
|
+
* @param buffer The buffer to be used for the tensor data.
|
821
|
+
* @param n_bytes The size of the buffer in bytes.
|
822
|
+
* @param ne An array specifying the extents (sizes) of each dimension of the
|
823
|
+
* tensor.
|
824
|
+
* @param dims The number of dimensions of the tensor.
|
825
|
+
* @param type The data type of the tensor.
|
826
|
+
* @param type_size The size of each element in the tensor data type.
|
827
|
+
* @param value The value to be used for initializing the tensor (default
|
828
|
+
* is 1.0).
|
829
|
+
* @return An ACL tensor initialized with value.
|
830
|
+
*/
|
831
|
+
static aclTensor* aclnn_values(ggml_backend_cann_context& ctx, void* buffer,
|
832
|
+
size_t n_bytes, int64_t* ne, int64_t dims,
|
833
|
+
aclDataType type, size_t type_size,
|
834
|
+
float value = 1.0f) {
|
835
|
+
aclTensor* acl_tensor =
|
836
|
+
aclnn_zero(ctx, buffer, n_bytes, ne, dims, type, type_size);
|
837
|
+
float alpha_host = 1.0f;
|
838
|
+
aclScalar* alpha = aclCreateScalar(&alpha_host, aclDataType::ACL_FLOAT);
|
839
|
+
aclScalar* other = aclCreateScalar(&value, aclDataType::ACL_FLOAT);
|
840
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdds, acl_tensor, other, alpha);
|
841
|
+
return acl_tensor;
|
842
|
+
}
|
843
|
+
|
844
|
+
void ggml_cann_rms_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
845
|
+
ggml_tensor* src = dst->src[0];
|
846
|
+
|
847
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
848
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
849
|
+
|
850
|
+
float eps;
|
851
|
+
memcpy(&eps, dst->op_params, sizeof(float));
|
852
|
+
size_t one_tensor_n_bytes = src->ne[0] * ggml_element_size(src);
|
853
|
+
ggml_cann_pool_alloc one_tensor_allocator(ctx.pool(), one_tensor_n_bytes);
|
854
|
+
|
855
|
+
aclTensor* acl_gamma = aclnn_values(
|
856
|
+
ctx, one_tensor_allocator.get(), one_tensor_n_bytes, src->ne, 1,
|
857
|
+
ggml_cann_type_mapping(src->type), ggml_element_size(src));
|
858
|
+
|
859
|
+
size_t zero_tensor_n_bytes =
|
860
|
+
src->ne[1] * src->ne[2] * src->ne[3] * ggml_element_size(src);
|
861
|
+
ggml_cann_pool_alloc zero_tensor_allocator(ctx.pool(), zero_tensor_n_bytes);
|
862
|
+
aclTensor* acl_rstd =
|
863
|
+
aclnn_zero(ctx, zero_tensor_allocator.get(), zero_tensor_n_bytes,
|
864
|
+
src->ne, GGML_MAX_DIMS, ggml_cann_type_mapping(src->type),
|
865
|
+
ggml_element_size(src));
|
866
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, RmsNorm, acl_src, acl_gamma, eps, acl_dst, acl_rstd);
|
867
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst, acl_gamma, acl_rstd);
|
868
|
+
}
|
869
|
+
|
870
|
+
// TODO: performace is low.
|
871
|
+
void ggml_cann_diag_mask(ggml_backend_cann_context& ctx, ggml_tensor* dst,
|
872
|
+
float value) {
|
873
|
+
ggml_tensor* src = dst->src[0];
|
874
|
+
|
875
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
876
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
877
|
+
|
878
|
+
const int n_past = ((int32_t*)dst->op_params)[0];
|
879
|
+
|
880
|
+
size_t one_tensor_n_bytes = src->ne[0] * src->ne[1] * src->ne[2] *
|
881
|
+
src->ne[3] * ggml_element_size(src);
|
882
|
+
ggml_cann_pool_alloc one_tensor_allocator(ctx.pool(), one_tensor_n_bytes);
|
883
|
+
|
884
|
+
aclTensor* mask_tensor =
|
885
|
+
aclnn_values(ctx, one_tensor_allocator.get(), one_tensor_n_bytes,
|
886
|
+
src->ne, GGML_MAX_DIMS, ggml_cann_type_mapping(src->type),
|
887
|
+
ggml_element_size(src), value);
|
888
|
+
|
889
|
+
aclScalar* alpha = nullptr;
|
890
|
+
float alphaValue = 1.0f;
|
891
|
+
alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
|
892
|
+
|
893
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceTriu, mask_tensor, n_past + 1);
|
894
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Tril, acl_src, n_past + 1, acl_dst);
|
895
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceAdd, acl_dst, mask_tensor, alpha);
|
896
|
+
ggml_cann_release_resources(ctx, alpha, acl_src, acl_dst, mask_tensor);
|
897
|
+
}
|
898
|
+
|
899
|
+
/**
|
900
|
+
* @brief Permutes the dimensions of a tensor according to a specified order.
|
901
|
+
*
|
902
|
+
* This function permutes the dimensions of the source tensor `acl_src`
|
903
|
+
* according to the order specified in the `new_dim` array and stores the result
|
904
|
+
* in the destination tensor `acl_dst`.
|
905
|
+
*
|
906
|
+
* @param ctx The context for the CANN backend operations.
|
907
|
+
* @param acl_src The source tensor whose dimensions will be permuted.
|
908
|
+
* @param acl_dst The destination tensor where the permuted result will be
|
909
|
+
* stored.
|
910
|
+
* @param new_dim An array specifying the new order of dimensions for the
|
911
|
+
* tensor.
|
912
|
+
* @param dims The number of dimensions in the tensor.
|
913
|
+
*/
|
914
|
+
static void aclnn_permute(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
915
|
+
aclTensor* acl_dst, int64_t* new_dim, uint64_t dims) {
|
916
|
+
aclIntArray* acl_dims = aclCreateIntArray(new_dim, dims);
|
917
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Permute, acl_src, acl_dims, acl_dst);
|
918
|
+
ggml_cann_release_resources(ctx, acl_dims);
|
919
|
+
}
|
920
|
+
|
921
|
+
static void ggml_cann_im2col_2d_post_process(ggml_backend_cann_context& ctx,
|
922
|
+
ggml_tensor* dst,
|
923
|
+
ggml_tensor* src1,
|
924
|
+
aclTensor* tmp_cast_tensor,
|
925
|
+
aclTensor* tmp_im2col_tensor) {
|
926
|
+
// Permute: [N, IC * KH * KW, OW * OH] -> [N, OW * OH, IC * KH * KW]
|
927
|
+
int64_t dst_ne[] = {dst->ne[0], dst->ne[1] * dst->ne[2], dst->ne[3]};
|
928
|
+
size_t dst_nb[] = {dst->nb[0], dst->nb[1], dst->nb[3]};
|
929
|
+
aclTensor* acl_dst =
|
930
|
+
ggml_cann_create_tensor(dst, dst_ne, dst_nb, GGML_MAX_DIMS - 1);
|
931
|
+
|
932
|
+
int64_t permute_dim[] = {0, 2, 1};
|
933
|
+
if (src1->type != dst->type) {
|
934
|
+
aclnn_permute(ctx, tmp_cast_tensor, acl_dst, permute_dim, 3);
|
935
|
+
} else {
|
936
|
+
aclnn_permute(ctx, tmp_im2col_tensor, acl_dst, permute_dim, 3);
|
937
|
+
}
|
938
|
+
|
939
|
+
ggml_cann_release_resources(ctx, acl_dst);
|
940
|
+
}
|
941
|
+
|
942
|
+
static void ggml_cann_im2col_1d_post_process(
|
943
|
+
ggml_backend_cann_context& ctx, ggml_tensor* dst, ggml_tensor* src1,
|
944
|
+
aclTensor* tmp_cast_tensor, aclTensor* tmp_im2col_tensor,
|
945
|
+
const std::vector<int64_t>& im2col_op_params) {
|
946
|
+
// get params
|
947
|
+
const int64_t KH = im2col_op_params[0];
|
948
|
+
const int64_t KW = im2col_op_params[1];
|
949
|
+
const int64_t IW = im2col_op_params[2];
|
950
|
+
const int64_t IC = im2col_op_params[3];
|
951
|
+
const int64_t N = im2col_op_params[4];
|
952
|
+
const int64_t OH = im2col_op_params[5];
|
953
|
+
const int64_t OW = im2col_op_params[6];
|
954
|
+
const int64_t s0 = im2col_op_params[7];
|
955
|
+
const int64_t p0 = im2col_op_params[8];
|
956
|
+
const int64_t d0 = im2col_op_params[9];
|
957
|
+
const int64_t n_bytes_factor = im2col_op_params[10];
|
958
|
+
|
959
|
+
// Permute: [N, IC * KH * KW, OW * OH] ->
|
960
|
+
// [N, OW * OH * n_bytes_factor, IC * KH * KW]
|
961
|
+
ggml_cann_pool_alloc tmp_permute_allocator(ctx.pool());
|
962
|
+
tmp_permute_allocator.alloc(ggml_nbytes(dst) * n_bytes_factor);
|
963
|
+
void* tmp_permute_buffer = tmp_permute_allocator.get();
|
964
|
+
|
965
|
+
int64_t tmp_permute_ne[] = {IC * KH * KW, OW * OH * n_bytes_factor, N};
|
966
|
+
size_t tmp_permute_nb[GGML_MAX_DIMS - 1];
|
967
|
+
tmp_permute_nb[0] = ggml_type_size(dst->type);
|
968
|
+
for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {
|
969
|
+
tmp_permute_nb[i] = tmp_permute_nb[i - 1] * tmp_permute_ne[i - 1];
|
970
|
+
}
|
971
|
+
|
972
|
+
aclTensor* tmp_permute_tensor = ggml_cann_create_tensor(
|
973
|
+
tmp_permute_buffer, ggml_cann_type_mapping(dst->type),
|
974
|
+
ggml_type_size(dst->type), tmp_permute_ne, tmp_permute_nb,
|
975
|
+
GGML_MAX_DIMS - 1, ACL_FORMAT_ND);
|
976
|
+
|
977
|
+
int64_t permute_dim[] = {0, 2, 1};
|
978
|
+
if (src1->type != dst->type) {
|
979
|
+
aclnn_permute(ctx, tmp_cast_tensor, tmp_permute_tensor, permute_dim, 3);
|
980
|
+
} else {
|
981
|
+
aclnn_permute(ctx, tmp_im2col_tensor, tmp_permute_tensor, permute_dim,
|
982
|
+
3);
|
983
|
+
}
|
984
|
+
|
985
|
+
// number of times the kernel moves in W dimension
|
986
|
+
const int n_step_w = (IW + 2 * p0 - d0 * (KW - 1) - 1) / s0 + 1;
|
987
|
+
size_t offset;
|
988
|
+
void *cur_dst_buffer = dst->data, *cur_permute_buffer = tmp_permute_buffer;
|
989
|
+
|
990
|
+
// memory copy with offset to restore 1D im2col from 2d
|
991
|
+
if (IC > 1) {
|
992
|
+
offset = IC * KH * KW * n_step_w * ggml_type_size(dst->type);
|
993
|
+
size_t size_cpy = KH * KW * ggml_type_size(dst->type);
|
994
|
+
|
995
|
+
for (int c = 0; c < IC; c++) {
|
996
|
+
cur_permute_buffer = (char*)tmp_permute_buffer + offset +
|
997
|
+
KH * KW * c * ggml_type_size(dst->type);
|
998
|
+
cur_dst_buffer = (char*)dst->data +
|
999
|
+
c * KH * KW * n_step_w * ggml_type_size(dst->type);
|
1000
|
+
|
1001
|
+
for (int i = 0; i < n_step_w; i++) {
|
1002
|
+
ggml_cann_async_memcpy(ctx, cur_dst_buffer, cur_permute_buffer, size_cpy,
|
1003
|
+
ACL_MEMCPY_DEVICE_TO_DEVICE);
|
1004
|
+
cur_dst_buffer =
|
1005
|
+
(char*)cur_dst_buffer + KH * KW * ggml_type_size(dst->type);
|
1006
|
+
cur_permute_buffer = (char*)cur_permute_buffer +
|
1007
|
+
KH * KW * IC * ggml_type_size(dst->type);
|
1008
|
+
}
|
1009
|
+
}
|
1010
|
+
} else {
|
1011
|
+
offset = KH * KW * n_step_w *
|
1012
|
+
ggml_type_size(dst->type); // equal to ggml_nbytes(dst)
|
1013
|
+
ggml_cann_async_memcpy(ctx, dst->data, (char*)tmp_permute_buffer + offset, offset,
|
1014
|
+
ACL_MEMCPY_DEVICE_TO_DEVICE);
|
1015
|
+
}
|
1016
|
+
|
1017
|
+
ggml_cann_release_resources(ctx, tmp_permute_tensor);
|
1018
|
+
}
|
1019
|
+
|
1020
|
+
void ggml_cann_im2col(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
1021
|
+
ggml_tensor* src0 = dst->src[0]; // kernel
|
1022
|
+
ggml_tensor* src1 = dst->src[1]; // input
|
1023
|
+
|
1024
|
+
GGML_TENSOR_BINARY_OP_LOCALS;
|
1025
|
+
|
1026
|
+
// aclnnIm2col only works on 2D. set s1, p1, d1 to 1 to perform 2D
|
1027
|
+
// im2col and do post-processing to restore it to 1D.
|
1028
|
+
const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;
|
1029
|
+
const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
|
1030
|
+
const int32_t s1 = is_2D ? ((const int32_t*)(dst->op_params))[1] : 1;
|
1031
|
+
const int32_t p0 = ((const int32_t*)(dst->op_params))[2];
|
1032
|
+
const int32_t p1 = is_2D ? ((const int32_t*)(dst->op_params))[3] : 1;
|
1033
|
+
const int32_t d0 = ((const int32_t*)(dst->op_params))[4];
|
1034
|
+
const int32_t d1 = is_2D ? ((const int32_t*)(dst->op_params))[5] : 1;
|
1035
|
+
|
1036
|
+
const int64_t N = ne13;
|
1037
|
+
const int64_t IC = ne12;
|
1038
|
+
const int64_t KH = ne01;
|
1039
|
+
const int64_t KW = ne00;
|
1040
|
+
const int64_t IW = ne10;
|
1041
|
+
|
1042
|
+
const int64_t OH = is_2D ? ne2 : 1;
|
1043
|
+
const int64_t OW = ne1;
|
1044
|
+
|
1045
|
+
// memory allocated increased to 3x when is_2D == false
|
1046
|
+
const int64_t n_bytes_factor = is_2D ? 1 : 3;
|
1047
|
+
|
1048
|
+
// im2col: [N,C,H,W] -> [N, IC * KH * KW, OW * OH * n_bytes_factor]
|
1049
|
+
aclTensor* acl_src1 = ggml_cann_create_tensor(src1);
|
1050
|
+
int64_t tmp_im2col_ne[] = {OW * OH * n_bytes_factor, IC * KH * KW, N};
|
1051
|
+
size_t tmp_im2col_nb[GGML_MAX_DIMS - 1];
|
1052
|
+
|
1053
|
+
tmp_im2col_nb[0] = ggml_type_size(src1->type);
|
1054
|
+
for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {
|
1055
|
+
tmp_im2col_nb[i] = tmp_im2col_nb[i - 1] * tmp_im2col_ne[i - 1];
|
1056
|
+
}
|
1057
|
+
|
1058
|
+
// Calculate im2col.
|
1059
|
+
// If dst is f16, tmp_buffer is f32, we need alloc src.typesize *
|
1060
|
+
// dst.elemcount.
|
1061
|
+
ggml_cann_pool_alloc im2col_allocator(
|
1062
|
+
ctx.pool(),
|
1063
|
+
ggml_nelements(dst) * ggml_element_size(src1) * n_bytes_factor);
|
1064
|
+
void* tmp_im2col_buffer = im2col_allocator.get();
|
1065
|
+
|
1066
|
+
aclTensor* tmp_im2col_tensor = ggml_cann_create_tensor(
|
1067
|
+
tmp_im2col_buffer, ggml_cann_type_mapping(src1->type),
|
1068
|
+
ggml_type_size(src1->type), tmp_im2col_ne, tmp_im2col_nb,
|
1069
|
+
GGML_MAX_DIMS - 1, ACL_FORMAT_ND);
|
1070
|
+
|
1071
|
+
std::vector<int64_t> kernel_dims = {KH, KW};
|
1072
|
+
std::vector<int64_t> dilation_size = {d1, d0};
|
1073
|
+
std::vector<int64_t> padding_dims = {p1, p0};
|
1074
|
+
std::vector<int64_t> stride_dims = {s1, s0};
|
1075
|
+
auto* kernel_size = aclCreateIntArray(kernel_dims.data(), 2);
|
1076
|
+
auto* dilations = aclCreateIntArray(dilation_size.data(), 2);
|
1077
|
+
auto* paddings = aclCreateIntArray(padding_dims.data(), 2);
|
1078
|
+
auto* strides = aclCreateIntArray(stride_dims.data(), 2);
|
1079
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Im2col, acl_src1, kernel_size, dilations,
|
1080
|
+
paddings, strides, tmp_im2col_tensor);
|
1081
|
+
|
1082
|
+
// Cast if dst is f16.
|
1083
|
+
aclTensor* tmp_cast_tensor = nullptr;
|
1084
|
+
ggml_cann_pool_alloc tmp_cast_allocator(ctx.pool());
|
1085
|
+
void* tmp_cast_buffer = nullptr;
|
1086
|
+
if (src1->type != dst->type) {
|
1087
|
+
tmp_cast_allocator.alloc(ggml_nbytes(dst) * n_bytes_factor);
|
1088
|
+
tmp_cast_buffer = tmp_cast_allocator.get();
|
1089
|
+
size_t temp_cast_nb[GGML_MAX_DIMS - 1];
|
1090
|
+
temp_cast_nb[0] = ggml_type_size(dst->type);
|
1091
|
+
for (int i = 1; i < GGML_MAX_DIMS - 1; i++) {
|
1092
|
+
temp_cast_nb[i] = temp_cast_nb[i - 1] * tmp_im2col_ne[i - 1];
|
1093
|
+
}
|
1094
|
+
|
1095
|
+
tmp_cast_tensor = ggml_cann_create_tensor(
|
1096
|
+
tmp_cast_buffer, ggml_cann_type_mapping(dst->type),
|
1097
|
+
ggml_type_size(dst->type), tmp_im2col_ne, temp_cast_nb,
|
1098
|
+
GGML_MAX_DIMS - 1, ACL_FORMAT_ND);
|
1099
|
+
aclnn_cast(ctx, tmp_im2col_tensor, tmp_cast_tensor, ggml_cann_type_mapping(dst->type));
|
1100
|
+
}
|
1101
|
+
|
1102
|
+
// post-processing
|
1103
|
+
if (is_2D) {
|
1104
|
+
ggml_cann_im2col_2d_post_process(ctx, dst, src1, tmp_cast_tensor,
|
1105
|
+
tmp_im2col_tensor);
|
1106
|
+
} else {
|
1107
|
+
std::vector<int64_t> im2col_op_params = {
|
1108
|
+
KH, KW, IW, IC, N, OH, OW, s0, p0, d0, n_bytes_factor};
|
1109
|
+
ggml_cann_im2col_1d_post_process(ctx, dst, src1, tmp_cast_tensor,
|
1110
|
+
tmp_im2col_tensor, im2col_op_params);
|
1111
|
+
}
|
1112
|
+
|
1113
|
+
ggml_cann_release_resources(ctx, acl_src1, tmp_im2col_tensor, tmp_cast_tensor,
|
1114
|
+
kernel_size, dilations, paddings, strides);
|
1115
|
+
}
|
1116
|
+
|
1117
|
+
/**
|
1118
|
+
* @brief Applies element-wise exponential function to the elements of a tensor.
|
1119
|
+
*
|
1120
|
+
* This function computes the exponential of each element in the source tensor
|
1121
|
+
* `acl_src` and stores the result back into the same tensor.
|
1122
|
+
* The operation is defined as:
|
1123
|
+
* \f[
|
1124
|
+
* \text {acl_src }_i=e^{acl\_src_i}
|
1125
|
+
* \f]
|
1126
|
+
*
|
1127
|
+
* @param ctx The context for the CANN backend operations.
|
1128
|
+
* @param acl_src The tensor on which the exponential function will be applied.
|
1129
|
+
*/
|
1130
|
+
static void aclnn_exp(ggml_backend_cann_context& ctx, aclTensor* acl_src) {
|
1131
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceExp, acl_src);
|
1132
|
+
}
|
1133
|
+
|
1134
|
+
void aclnn_cos(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
1135
|
+
aclTensor* acl_dst) {
|
1136
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Cos, acl_src, acl_dst);
|
1137
|
+
}
|
1138
|
+
|
1139
|
+
void aclnn_sin(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
1140
|
+
aclTensor* acl_dst) {
|
1141
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Sin, acl_src, acl_dst);
|
1142
|
+
}
|
1143
|
+
|
1144
|
+
void ggml_cann_timestep_embedding(ggml_backend_cann_context& ctx,
|
1145
|
+
ggml_tensor* dst) {
|
1146
|
+
const ggml_tensor* src = dst->src[0];
|
1147
|
+
|
1148
|
+
GGML_ASSERT(src->type == GGML_TYPE_F32);
|
1149
|
+
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
1150
|
+
|
1151
|
+
const int dim = dst->op_params[0];
|
1152
|
+
const int max_period = dst->op_params[1];
|
1153
|
+
int half = dim / 2;
|
1154
|
+
|
1155
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src);
|
1156
|
+
|
1157
|
+
// arange: [0, ..., half)
|
1158
|
+
float start = 0;
|
1159
|
+
float stop = half;
|
1160
|
+
float step = 1;
|
1161
|
+
int64_t n_elements_arange = half;
|
1162
|
+
int64_t tmp_arange_ne[] = {half};
|
1163
|
+
size_t tmp_arange_nb[] = {sizeof(dst->type)};
|
1164
|
+
|
1165
|
+
ggml_cann_pool_alloc arange_allocator(ctx.pool(), half * sizeof(dst->type));
|
1166
|
+
void* tmp_arange_buffer = arange_allocator.get();
|
1167
|
+
aclTensor* tmp_arange_tensor = ggml_cann_create_tensor(
|
1168
|
+
tmp_arange_buffer, ggml_cann_type_mapping(dst->type),
|
1169
|
+
ggml_type_size(dst->type), tmp_arange_ne, tmp_arange_nb,
|
1170
|
+
GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
1171
|
+
|
1172
|
+
aclnn_arange(ctx, tmp_arange_tensor, start, stop, step, n_elements_arange);
|
1173
|
+
|
1174
|
+
// freq
|
1175
|
+
float freq_param = -logf(max_period) / half;
|
1176
|
+
bool inplace = true;
|
1177
|
+
aclnn_muls(ctx, tmp_arange_tensor, freq_param, nullptr, inplace);
|
1178
|
+
aclnn_exp(ctx, tmp_arange_tensor);
|
1179
|
+
|
1180
|
+
// permute: src [0,1,2,3]->[0,1,3,2]
|
1181
|
+
int64_t tmp_permute_ne[] = {src->ne[1], src->ne[0], src->ne[2], src->ne[3]};
|
1182
|
+
size_t tmp_permute_nb[GGML_MAX_DIMS];
|
1183
|
+
tmp_permute_nb[0] = ggml_type_size(src->type);
|
1184
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
1185
|
+
tmp_permute_nb[i] = tmp_permute_nb[i - 1] * tmp_permute_ne[i - 1];
|
1186
|
+
}
|
1187
|
+
|
1188
|
+
ggml_cann_pool_alloc permute_allocator(ctx.pool(), ggml_nbytes(src));
|
1189
|
+
void* tmp_permute_buffer = permute_allocator.get();
|
1190
|
+
aclTensor* tmp_permute_tensor = ggml_cann_create_tensor(
|
1191
|
+
tmp_permute_buffer, ggml_cann_type_mapping(src->type),
|
1192
|
+
ggml_type_size(src->type), tmp_permute_ne, tmp_permute_nb,
|
1193
|
+
GGML_MAX_DIMS, ACL_FORMAT_ND);
|
1194
|
+
int64_t permute_dim[] = {0, 1, 3, 2};
|
1195
|
+
int64_t num_dims = 4;
|
1196
|
+
aclnn_permute(ctx, acl_src, tmp_permute_tensor, permute_dim, num_dims);
|
1197
|
+
|
1198
|
+
// timestep * freq
|
1199
|
+
int64_t tmp_mul_ne[] = {src->ne[1] * half, src->ne[0], src->ne[2],
|
1200
|
+
src->ne[3]};
|
1201
|
+
size_t tmp_mul_nb[GGML_MAX_DIMS];
|
1202
|
+
tmp_mul_nb[0] = ggml_type_size(src->type);
|
1203
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
1204
|
+
tmp_mul_nb[i] = tmp_mul_nb[i - 1] * tmp_mul_ne[i - 1];
|
1205
|
+
}
|
1206
|
+
|
1207
|
+
int mul_nelements =
|
1208
|
+
src->ne[1] * half * src->ne[0] * src->ne[2] * src->ne[3];
|
1209
|
+
|
1210
|
+
ggml_cann_pool_alloc mul_allocator(
|
1211
|
+
ctx.pool(), mul_nelements * ggml_type_size(src->type));
|
1212
|
+
void* tmp_mul_buffer = mul_allocator.get();
|
1213
|
+
aclTensor* tmp_mul_tensor = ggml_cann_create_tensor(
|
1214
|
+
tmp_mul_buffer, ggml_cann_type_mapping(src->type),
|
1215
|
+
ggml_type_size(src->type), tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS,
|
1216
|
+
ACL_FORMAT_ND);
|
1217
|
+
aclnn_mul(ctx, tmp_permute_tensor, tmp_arange_tensor, tmp_mul_tensor);
|
1218
|
+
|
1219
|
+
// cos
|
1220
|
+
ggml_cann_pool_alloc cos_allocator(
|
1221
|
+
ctx.pool(), mul_nelements * ggml_type_size(src->type));
|
1222
|
+
void* tmp_cos_buffer = cos_allocator.get();
|
1223
|
+
aclTensor* tmp_cos_tensor = ggml_cann_create_tensor(
|
1224
|
+
tmp_cos_buffer, ggml_cann_type_mapping(dst->type),
|
1225
|
+
ggml_type_size(dst->type), tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS,
|
1226
|
+
ACL_FORMAT_ND);
|
1227
|
+
|
1228
|
+
aclnn_cos(ctx, tmp_mul_tensor, tmp_cos_tensor);
|
1229
|
+
|
1230
|
+
// sin
|
1231
|
+
ggml_cann_pool_alloc sin_allocator(
|
1232
|
+
ctx.pool(), mul_nelements * ggml_type_size(src->type));
|
1233
|
+
void* tmp_sin_buffer = sin_allocator.get();
|
1234
|
+
aclTensor* tmp_sin_tensor = ggml_cann_create_tensor(
|
1235
|
+
tmp_sin_buffer, ggml_cann_type_mapping(dst->type),
|
1236
|
+
ggml_type_size(dst->type), tmp_mul_ne, tmp_mul_nb, GGML_MAX_DIMS,
|
1237
|
+
ACL_FORMAT_ND);
|
1238
|
+
|
1239
|
+
aclnn_sin(ctx, tmp_mul_tensor, tmp_sin_tensor);
|
1240
|
+
|
1241
|
+
// concat
|
1242
|
+
int64_t concat_dim = 3;
|
1243
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
1244
|
+
aclTensor* tensors[] = {tmp_cos_tensor, tmp_sin_tensor};
|
1245
|
+
aclTensorList* tensor_list = aclCreateTensorList(tensors, 2);
|
1246
|
+
aclnn_concat(ctx, tensor_list, acl_dst, concat_dim);
|
1247
|
+
|
1248
|
+
// release
|
1249
|
+
// segmentation fault when delete both tensorList and his elements.
|
1250
|
+
ggml_cann_release_resources(ctx, tensor_list, acl_src, tmp_arange_tensor,
|
1251
|
+
tmp_permute_tensor, tmp_mul_tensor, acl_dst);
|
1252
|
+
}
|
1253
|
+
|
1254
|
+
/**
|
1255
|
+
* @brief Fills a tensor with a scalar value.
|
1256
|
+
*
|
1257
|
+
* This function fills the destination tensor `acl_dst` with the scalar value
|
1258
|
+
* `scalar`.
|
1259
|
+
*
|
1260
|
+
* @param ctx The context for the CANN backend operations.
|
1261
|
+
* @param scalar The scalar value used to fill the tensor.
|
1262
|
+
* @param acl_dst The destination tensor to be filled with the scalar value.
|
1263
|
+
*/
|
1264
|
+
static void aclnn_fill_scalar(ggml_backend_cann_context& ctx, float scalar,
|
1265
|
+
aclTensor* acl_dst) {
|
1266
|
+
auto acl_scalar = aclCreateScalar(&scalar, aclDataType::ACL_FLOAT);
|
1267
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceFillScalar, acl_dst, acl_scalar);
|
1268
|
+
ggml_cann_release_resources(ctx, acl_scalar);
|
1269
|
+
}
|
1270
|
+
|
1271
|
+
/**
|
1272
|
+
* @brief Raises each element of a tensor to the power of the corresponding
|
1273
|
+
* element in another tensor.
|
1274
|
+
*
|
1275
|
+
* This function computes the element-wise power of the destination tensor
|
1276
|
+
* `acl_dst` raised to the power of the exponent tensor `acl_exp`.
|
1277
|
+
* The operation is defined as:
|
1278
|
+
* \f[
|
1279
|
+
* \text {acl_dst }_i=acl\_dst_i^{\text {acl_exp }_i}
|
1280
|
+
* \f]
|
1281
|
+
*
|
1282
|
+
* @param ctx The context for the CANN backend operations.
|
1283
|
+
* @param acl_dst The destination tensor, which also serves as the base tensor.
|
1284
|
+
* @param acl_exp The exponent tensor, each element of which is used to raise
|
1285
|
+
* the corresponding element in the destination tensor.
|
1286
|
+
*/
|
1287
|
+
static void aclnn_pow_tensor_tensor(ggml_backend_cann_context& ctx,
|
1288
|
+
aclTensor* acl_dst, aclTensor* acl_exp) {
|
1289
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplacePowTensorTensor, acl_dst, acl_exp);
|
1290
|
+
}
|
1291
|
+
|
1292
|
+
/**
|
1293
|
+
* @brief Applies the Alibi (Attention with Linear Biases) mechanism to the
|
1294
|
+
* @details This function implements the Alibi mechanism, which introduces
|
1295
|
+
* learnable biases into the attention scores to simulate relative
|
1296
|
+
* position encoding without the need for explicit positional
|
1297
|
+
* embeddings.
|
1298
|
+
*
|
1299
|
+
* @param ctx The backend CANN context for executing operations.
|
1300
|
+
* @param acl_src The source tensor representing the query or key.
|
1301
|
+
* @param acl_position The position tensor containing relative positions.
|
1302
|
+
* @param acl_dst The destination tensor where the result will be stored.
|
1303
|
+
* @param n_head The number of attention heads.
|
1304
|
+
* @param src_ne The dimensions of the source tensor.
|
1305
|
+
* @param src_nb0 The byte size of the first dimension of the source
|
1306
|
+
tensor.
|
1307
|
+
* @param max_bias The maximum bias value used in the Alibi mechanism.
|
1308
|
+
* @param dst The destination tensor object for additional metadata.
|
1309
|
+
*
|
1310
|
+
* The function performs the following steps:
|
1311
|
+
* 1. Calculates the logarithm floor of the number of heads to determine the
|
1312
|
+
base for bias calculation.
|
1313
|
+
* 2. Initializes arrays with arithmetic sequences and fills them with bias
|
1314
|
+
values.
|
1315
|
+
* 3. Computes the bias tensor based on the calculated biases and arithmetic
|
1316
|
+
sequences.
|
1317
|
+
* 4. Reshapes the bias tensor to match the dimensions of the input tensors.
|
1318
|
+
* 5. Multiplies the position tensor by the bias tensor.
|
1319
|
+
* 6. Adds the result of the multiplication to the source tensor to produce the
|
1320
|
+
final output.
|
1321
|
+
*/
|
1322
|
+
static void aclnn_alibi(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
1323
|
+
aclTensor* acl_position, aclTensor* acl_dst,
|
1324
|
+
const int n_head, int64_t* src_ne, const size_t src_nb0,
|
1325
|
+
float max_bias, ggml_tensor* dst) {
|
1326
|
+
const int64_t ne2_ne3 = src_ne[2] * src_ne[3];
|
1327
|
+
GGML_ASSERT(src_nb0 == sizeof(float));
|
1328
|
+
GGML_ASSERT(n_head == src_ne[2]);
|
1329
|
+
|
1330
|
+
const int n_heads_log2_floor = 1u << (uint32_t)floor(log2(n_head));
|
1331
|
+
|
1332
|
+
float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);
|
1333
|
+
float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);
|
1334
|
+
|
1335
|
+
// init arange
|
1336
|
+
ggml_cann_pool_alloc arange_allocator(ctx.pool(),
|
1337
|
+
ne2_ne3 * ggml_type_size(dst->type));
|
1338
|
+
void* tmp_arange_buffer = arange_allocator.get();
|
1339
|
+
|
1340
|
+
// arange1: [1, ..., n_heads_log2_floor+1)
|
1341
|
+
float start = 1;
|
1342
|
+
float stop = n_heads_log2_floor + 1;
|
1343
|
+
float step = 1;
|
1344
|
+
int64_t n_elements_arange = n_heads_log2_floor;
|
1345
|
+
|
1346
|
+
int64_t tmp_arange1_ne[] = {n_heads_log2_floor};
|
1347
|
+
size_t tmp_arange1_nb[] = {sizeof(dst->type)};
|
1348
|
+
aclTensor* tmp_arange1_tensor = ggml_cann_create_tensor(
|
1349
|
+
tmp_arange_buffer, ggml_cann_type_mapping(dst->type),
|
1350
|
+
ggml_type_size(dst->type), tmp_arange1_ne, tmp_arange1_nb,
|
1351
|
+
GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
1352
|
+
|
1353
|
+
aclnn_arange(ctx, tmp_arange1_tensor, start, stop, step, n_elements_arange);
|
1354
|
+
|
1355
|
+
aclTensor* tmp_arange2_tensor = nullptr;
|
1356
|
+
if (n_heads_log2_floor < ne2_ne3) {
|
1357
|
+
// arange2: [1, ..., 2 * (k - n_heads_log2_floor) + 1)
|
1358
|
+
start = 1;
|
1359
|
+
stop = 2 * (ne2_ne3 - n_heads_log2_floor) + 1;
|
1360
|
+
step = 2;
|
1361
|
+
n_elements_arange = ne2_ne3 - n_heads_log2_floor;
|
1362
|
+
int64_t tmp_arange2_ne[] = {ne2_ne3 - n_heads_log2_floor};
|
1363
|
+
size_t tmp_arange2_nb[] = {sizeof(dst->type)};
|
1364
|
+
|
1365
|
+
aclTensor* tmp_arange2_tensor = ggml_cann_create_tensor(
|
1366
|
+
(char*)tmp_arange_buffer +
|
1367
|
+
n_heads_log2_floor * ggml_type_size(dst->type),
|
1368
|
+
ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),
|
1369
|
+
tmp_arange2_ne, tmp_arange2_nb, GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
1370
|
+
aclnn_arange(ctx, tmp_arange2_tensor, start, stop, step,
|
1371
|
+
n_elements_arange);
|
1372
|
+
}
|
1373
|
+
|
1374
|
+
// init mk_base
|
1375
|
+
ggml_cann_pool_alloc mk_base_allocator(ctx.pool(),
|
1376
|
+
ne2_ne3 * ggml_type_size(dst->type));
|
1377
|
+
void* tmp_mk_base_buffer = mk_base_allocator.get();
|
1378
|
+
int64_t tmp_mk_base1_ne[] = {n_heads_log2_floor};
|
1379
|
+
size_t tmp_mk_base1_nb[] = {sizeof(dst->type)};
|
1380
|
+
aclTensor* tmp_mk_base1_tensor = ggml_cann_create_tensor(
|
1381
|
+
tmp_mk_base_buffer, ggml_cann_type_mapping(dst->type),
|
1382
|
+
ggml_type_size(dst->type), tmp_mk_base1_ne, tmp_mk_base1_nb,
|
1383
|
+
GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
1384
|
+
|
1385
|
+
aclnn_fill_scalar(ctx, m0, tmp_mk_base1_tensor);
|
1386
|
+
|
1387
|
+
aclTensor* tmp_mk_base2_tensor = nullptr;
|
1388
|
+
if (n_heads_log2_floor < ne2_ne3) {
|
1389
|
+
int64_t tmp_mk_base2_ne[] = {ne2_ne3 - n_heads_log2_floor};
|
1390
|
+
size_t tmp_mk_base2_nb[] = {sizeof(dst->type)};
|
1391
|
+
aclTensor* tmp_mk_base2_tensor = ggml_cann_create_tensor(
|
1392
|
+
(char*)tmp_mk_base_buffer +
|
1393
|
+
n_heads_log2_floor * ggml_type_size(dst->type),
|
1394
|
+
ggml_cann_type_mapping(dst->type), ggml_type_size(dst->type),
|
1395
|
+
tmp_mk_base2_ne, tmp_mk_base2_nb, GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
1396
|
+
aclnn_fill_scalar(ctx, m1, tmp_mk_base2_tensor);
|
1397
|
+
}
|
1398
|
+
|
1399
|
+
// init mk
|
1400
|
+
int64_t tmp_mk_base_ne[] = {ne2_ne3};
|
1401
|
+
size_t tmp_mk_base_nb[] = {sizeof(dst->type)};
|
1402
|
+
aclTensor* tmp_mk_base_tensor = ggml_cann_create_tensor(
|
1403
|
+
tmp_mk_base_buffer, ggml_cann_type_mapping(dst->type),
|
1404
|
+
ggml_type_size(dst->type), tmp_mk_base_ne, tmp_mk_base_nb,
|
1405
|
+
GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
1406
|
+
aclTensor* tmp_arange_tensor = ggml_cann_create_tensor(
|
1407
|
+
tmp_arange_buffer, ggml_cann_type_mapping(dst->type),
|
1408
|
+
ggml_type_size(dst->type), tmp_mk_base_ne, tmp_mk_base_nb,
|
1409
|
+
GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
1410
|
+
aclnn_pow_tensor_tensor(ctx, tmp_mk_base_tensor, tmp_arange_tensor);
|
1411
|
+
|
1412
|
+
// reshape mk
|
1413
|
+
int64_t tmp_mk_ne[] = {1, 1, src_ne[2], src_ne[3]};
|
1414
|
+
size_t tmp_mk_nb[GGML_MAX_DIMS];
|
1415
|
+
tmp_mk_nb[0] = ggml_type_size(dst->type);
|
1416
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
1417
|
+
tmp_mk_nb[i] = tmp_mk_nb[i - 1] * tmp_mk_ne[i - 1];
|
1418
|
+
}
|
1419
|
+
aclTensor* tmp_mk_tensor = ggml_cann_create_tensor(
|
1420
|
+
tmp_mk_base_buffer, ggml_cann_type_mapping(dst->type),
|
1421
|
+
ggml_type_size(dst->type), tmp_mk_ne, tmp_mk_nb, GGML_MAX_DIMS,
|
1422
|
+
ACL_FORMAT_ND);
|
1423
|
+
|
1424
|
+
// acl_position * mk
|
1425
|
+
int64_t tmp_output_ne[] = {src_ne[0], src_ne[1], src_ne[2], src_ne[3]};
|
1426
|
+
size_t tmp_output_nb[GGML_MAX_DIMS];
|
1427
|
+
tmp_output_nb[0] = ggml_type_size(dst->type);
|
1428
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
1429
|
+
tmp_output_nb[i] = tmp_output_nb[i - 1] * tmp_output_ne[i - 1];
|
1430
|
+
}
|
1431
|
+
ggml_cann_pool_alloc output_allocator(ctx.pool(), ggml_nbytes(dst));
|
1432
|
+
void* tmp_output_buffer = output_allocator.get();
|
1433
|
+
aclTensor* tmp_output_tensor = ggml_cann_create_tensor(
|
1434
|
+
tmp_output_buffer, ggml_cann_type_mapping(dst->type),
|
1435
|
+
ggml_type_size(dst->type), tmp_output_ne, tmp_output_nb, GGML_MAX_DIMS,
|
1436
|
+
ACL_FORMAT_ND);
|
1437
|
+
aclnn_mul(ctx, acl_position, tmp_mk_tensor, tmp_output_tensor);
|
1438
|
+
|
1439
|
+
// add
|
1440
|
+
aclnn_add(ctx, tmp_output_tensor, acl_src, acl_dst);
|
1441
|
+
ggml_cann_release_resources(ctx, tmp_arange1_tensor, tmp_arange2_tensor,
|
1442
|
+
tmp_mk_base1_tensor, tmp_mk_base2_tensor, tmp_mk_base_tensor,
|
1443
|
+
tmp_arange_tensor, tmp_mk_tensor, tmp_output_tensor);
|
1444
|
+
}
|
1445
|
+
|
1446
|
+
void ggml_cann_cpy(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
1447
|
+
ggml_cann_dup(ctx, dst);
|
1448
|
+
}
|
1449
|
+
|
1450
|
+
/**
|
1451
|
+
* @brief Applies the softmax function to a tensor along a specified dimension.
|
1452
|
+
*
|
1453
|
+
* This function computes the softmax of the source tensor `acl_src` along the
|
1454
|
+
* specified dimension `dim` and stores the result in the destination tensor
|
1455
|
+
* `acl_dst`.
|
1456
|
+
*
|
1457
|
+
* @param ctx The context for the CANN backend operations.
|
1458
|
+
* @param acl_src The source tensor on which the softmax function will be
|
1459
|
+
* applied.
|
1460
|
+
* @param dim The dimension along which the softmax function will be computed.
|
1461
|
+
* @param acl_dst The destination tensor where the softmax results will be
|
1462
|
+
* stored.
|
1463
|
+
*/
|
1464
|
+
static void aclnn_softmax(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
1465
|
+
int64_t dim, aclTensor* acl_dst) {
|
1466
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Softmax, acl_src, dim, acl_dst);
|
1467
|
+
}
|
1468
|
+
|
1469
|
+
void ggml_cann_softmax(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
1470
|
+
ggml_tensor* src0 = dst->src[0];
|
1471
|
+
ggml_tensor* src1 = dst->src[1]; // mask
|
1472
|
+
|
1473
|
+
aclTensor* acl_src0 = ggml_cann_create_tensor(src0);
|
1474
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
1475
|
+
|
1476
|
+
float scale = 1.0f;
|
1477
|
+
float max_bias = 0.0f;
|
1478
|
+
|
1479
|
+
memcpy(&scale, (float*)dst->op_params + 0, sizeof(float));
|
1480
|
+
memcpy(&max_bias, (float*)dst->op_params + 1, sizeof(float));
|
1481
|
+
|
1482
|
+
// input mul scale
|
1483
|
+
aclScalar* acl_scale = aclCreateScalar(&scale, aclDataType::ACL_FLOAT);
|
1484
|
+
|
1485
|
+
size_t n_bytes = ggml_nbytes(src0);
|
1486
|
+
ggml_cann_pool_alloc mul_scale_allocator(ctx.pool(), n_bytes);
|
1487
|
+
void* input_mul_scale_buffer = mul_scale_allocator.get();
|
1488
|
+
aclTensor* acl_input_mul_scale_tensor = ggml_cann_create_tensor(
|
1489
|
+
input_mul_scale_buffer, ACL_FLOAT, ggml_type_size(src0->type), src0->ne,
|
1490
|
+
src0->nb, GGML_MAX_DIMS);
|
1491
|
+
|
1492
|
+
bool inplace = false;
|
1493
|
+
aclnn_muls(ctx, acl_src0, scale, acl_input_mul_scale_tensor, inplace);
|
1494
|
+
|
1495
|
+
// mask
|
1496
|
+
aclTensor* acl_src1_fp32_tensor = nullptr;
|
1497
|
+
aclTensor* tmp_mask_tensor = nullptr;
|
1498
|
+
ggml_cann_pool_alloc src1_fp32_allocator(ctx.pool());
|
1499
|
+
if (src1) {
|
1500
|
+
const bool use_f16 = src1->type == GGML_TYPE_F16;
|
1501
|
+
if (use_f16) {
|
1502
|
+
// cast to fp32
|
1503
|
+
size_t n_bytes = ggml_nelements(src1) * sizeof(float_t);
|
1504
|
+
size_t src1_fp32_nb[GGML_MAX_DIMS];
|
1505
|
+
src1_fp32_nb[0] = sizeof(float_t);
|
1506
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
1507
|
+
src1_fp32_nb[i] = src1_fp32_nb[i - 1] * src1->ne[i - 1];
|
1508
|
+
}
|
1509
|
+
src1_fp32_allocator.alloc(n_bytes);
|
1510
|
+
void* src1_fp32_buffer = src1_fp32_allocator.get();
|
1511
|
+
acl_src1_fp32_tensor = ggml_cann_create_tensor(
|
1512
|
+
src1_fp32_buffer, ACL_FLOAT, sizeof(float), src1->ne,
|
1513
|
+
src1_fp32_nb, GGML_MAX_DIMS);
|
1514
|
+
aclTensor* acl_src1 = ggml_cann_create_tensor(src1);
|
1515
|
+
aclnn_cast(ctx, acl_src1, acl_src1_fp32_tensor, ACL_FLOAT);
|
1516
|
+
ggml_cann_release_resources(ctx, acl_src1);
|
1517
|
+
} else {
|
1518
|
+
acl_src1_fp32_tensor = ggml_cann_create_tensor(src1);
|
1519
|
+
}
|
1520
|
+
|
1521
|
+
// broadcast the mask across rows, only use ne11 of ne01 in mask
|
1522
|
+
if (src1->ne[1] != src0->ne[1]) {
|
1523
|
+
// mask shape: [1,1,ne11,ne10]
|
1524
|
+
int64_t tmp_mask_ne[] = {src0->ne[0], src0->ne[1], 1, 1};
|
1525
|
+
size_t tmp_mask_nb[GGML_MAX_DIMS];
|
1526
|
+
tmp_mask_nb[0] = sizeof(float_t);
|
1527
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
1528
|
+
tmp_mask_nb[i] = tmp_mask_nb[i - 1] * tmp_mask_ne[i - 1];
|
1529
|
+
}
|
1530
|
+
tmp_mask_tensor = ggml_cann_create_tensor(
|
1531
|
+
src1->data, ACL_FLOAT, sizeof(float), tmp_mask_ne, tmp_mask_nb,
|
1532
|
+
GGML_MAX_DIMS, ACL_FORMAT_ND);
|
1533
|
+
}
|
1534
|
+
|
1535
|
+
// alibi
|
1536
|
+
const int n_head = src0->ne[2];
|
1537
|
+
const size_t src_nb0 = src0->nb[0];
|
1538
|
+
|
1539
|
+
n_bytes = ggml_nbytes(dst);
|
1540
|
+
ggml_cann_pool_alloc output_allocator(ctx.pool(), n_bytes);
|
1541
|
+
void* output_buffer = output_allocator.get();
|
1542
|
+
aclTensor* alibi_output_tensor = ggml_cann_create_tensor(
|
1543
|
+
output_buffer, ACL_FLOAT, ggml_type_size(dst->type), dst->ne,
|
1544
|
+
dst->nb, GGML_MAX_DIMS);
|
1545
|
+
if (max_bias <= 0.0f) {
|
1546
|
+
// slope = 1.0
|
1547
|
+
if (tmp_mask_tensor) {
|
1548
|
+
aclnn_add(ctx, tmp_mask_tensor, acl_input_mul_scale_tensor,
|
1549
|
+
alibi_output_tensor);
|
1550
|
+
} else {
|
1551
|
+
aclnn_add(ctx, acl_src1_fp32_tensor, acl_input_mul_scale_tensor,
|
1552
|
+
alibi_output_tensor);
|
1553
|
+
}
|
1554
|
+
} else {
|
1555
|
+
// slope != 1.0
|
1556
|
+
if (tmp_mask_tensor) {
|
1557
|
+
aclnn_alibi(ctx, acl_input_mul_scale_tensor, tmp_mask_tensor,
|
1558
|
+
alibi_output_tensor, n_head, src0->ne, src_nb0,
|
1559
|
+
max_bias, dst);
|
1560
|
+
} else {
|
1561
|
+
aclnn_alibi(ctx, acl_input_mul_scale_tensor,
|
1562
|
+
acl_src1_fp32_tensor, alibi_output_tensor, n_head,
|
1563
|
+
src0->ne, src_nb0, max_bias, dst);
|
1564
|
+
}
|
1565
|
+
}
|
1566
|
+
|
1567
|
+
// softmax
|
1568
|
+
aclnn_softmax(ctx, alibi_output_tensor, 3, acl_dst);
|
1569
|
+
ggml_cann_release_resources(ctx, alibi_output_tensor);
|
1570
|
+
} else {
|
1571
|
+
aclnn_softmax(ctx, acl_input_mul_scale_tensor, 3, acl_dst);
|
1572
|
+
}
|
1573
|
+
|
1574
|
+
ggml_cann_release_resources(ctx, acl_src0, acl_src1_fp32_tensor, acl_dst,
|
1575
|
+
acl_scale, acl_input_mul_scale_tensor, tmp_mask_tensor);
|
1576
|
+
}
|
1577
|
+
|
1578
|
+
/**
|
1579
|
+
* @brief Performs embedding operation on a 4D tensor using the CANN backend.
|
1580
|
+
*
|
1581
|
+
* This function extracts slices from the source tensor (`src_buffer`),
|
1582
|
+
* index tensor (`index`), and destination tensor (`dst`), and performs an
|
1583
|
+
* embedding operation on them. The embedding operation is applied by iterating
|
1584
|
+
* over the last two dimensions of the source tensor, creating the necessary
|
1585
|
+
* tensors for the source, index, and output, and executing the embedding operation.
|
1586
|
+
*
|
1587
|
+
* @param ctx The context for CANN backend operations.
|
1588
|
+
* @param src_buffer The source buffer holding the data for the source tensor.
|
1589
|
+
* @param src_ne The dimensions of the source tensor.
|
1590
|
+
* @param src_nb The strides (byte offsets) of the source tensor.
|
1591
|
+
* @param index The index tensor used in the embedding operation.
|
1592
|
+
* @param dst The destination tensor where the result will be stored.
|
1593
|
+
*/
|
1594
|
+
static void aclnn_embedding_4d(ggml_backend_cann_context& ctx, void* src_buffer,
|
1595
|
+
int64_t* src_ne, size_t* src_nb, ggml_tensor* index,
|
1596
|
+
ggml_tensor* dst) {
|
1597
|
+
for (int64_t i = 0; i < src_ne[3]; i++) {
|
1598
|
+
for (int64_t j = 0; j < src_ne[2]; j++) {
|
1599
|
+
// src
|
1600
|
+
int64_t acl_src_ne[2] = {src_ne[0], src_ne[1]};
|
1601
|
+
size_t acl_src_nb[2] = {src_nb[0], src_nb[1]};
|
1602
|
+
aclTensor* acl_src_tensor = ggml_cann_create_tensor(
|
1603
|
+
(char*)src_buffer + i * src_nb[3] + j * src_nb[2],
|
1604
|
+
ggml_cann_type_mapping(dst->type), ggml_element_size(dst),
|
1605
|
+
acl_src_ne, acl_src_nb, 2);
|
1606
|
+
|
1607
|
+
// index
|
1608
|
+
int64_t acl_index_ne[1] = {index->ne[0]};
|
1609
|
+
size_t acl_index_nb[1] = {index->nb[0]};
|
1610
|
+
aclTensor* acl_index = ggml_cann_create_tensor(
|
1611
|
+
(char*)index->data + i * index->nb[2] + j * index->nb[1],
|
1612
|
+
ggml_cann_type_mapping(index->type), ggml_element_size(index),
|
1613
|
+
acl_index_ne, acl_index_nb, 1);
|
1614
|
+
|
1615
|
+
// out
|
1616
|
+
int64_t acl_out_ne[2] = {dst->ne[0], dst->ne[1]};
|
1617
|
+
size_t acl_out_nb[2] = {dst->nb[0], dst->nb[1]};
|
1618
|
+
aclTensor* acl_out = ggml_cann_create_tensor(
|
1619
|
+
(char*)dst->data + i * dst->nb[3] + j * dst->nb[2],
|
1620
|
+
ggml_cann_type_mapping(dst->type), ggml_element_size(dst),
|
1621
|
+
acl_out_ne, acl_out_nb, 2);
|
1622
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Embedding, acl_src_tensor, acl_index, acl_out);
|
1623
|
+
ggml_cann_release_resources(ctx, acl_src_tensor, acl_index, acl_out);
|
1624
|
+
}
|
1625
|
+
}
|
1626
|
+
}
|
1627
|
+
|
1628
|
+
void ggml_cann_get_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
1629
|
+
ggml_tensor* src0 = dst->src[0]; // src
|
1630
|
+
ggml_tensor* src1 = dst->src[1]; // index
|
1631
|
+
|
1632
|
+
switch (src0->type) {
|
1633
|
+
case GGML_TYPE_F32: {
|
1634
|
+
aclnn_embedding_4d(ctx, src0->data, src0->ne, src0->nb, src1,
|
1635
|
+
dst);
|
1636
|
+
break;
|
1637
|
+
}
|
1638
|
+
case GGML_TYPE_F16: {
|
1639
|
+
aclTensor* acl_src0 = ggml_cann_create_tensor(src0);
|
1640
|
+
ggml_cann_pool_alloc src_buffer_allocator(
|
1641
|
+
ctx.pool(), ggml_nelements(src0) * sizeof(float_t));
|
1642
|
+
void* src_trans_buffer = src_buffer_allocator.get();
|
1643
|
+
size_t src_trans_nb[GGML_MAX_DIMS];
|
1644
|
+
src_trans_nb[0] = sizeof(float_t);
|
1645
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
1646
|
+
src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];
|
1647
|
+
}
|
1648
|
+
aclTensor* src_trans_tensor = ggml_cann_create_tensor(
|
1649
|
+
src_trans_buffer, ACL_FLOAT, ggml_type_size(dst->type),
|
1650
|
+
src0->ne, src_trans_nb, GGML_MAX_DIMS);
|
1651
|
+
aclnn_cast(ctx, acl_src0, src_trans_tensor, ggml_cann_type_mapping(dst->type));
|
1652
|
+
aclnn_embedding_4d(ctx, src_trans_buffer, src0->ne,
|
1653
|
+
src_trans_nb, src1, dst);
|
1654
|
+
ggml_cann_release_resources(ctx, acl_src0, src_trans_tensor);
|
1655
|
+
break;
|
1656
|
+
}
|
1657
|
+
case GGML_TYPE_Q8_0: {
|
1658
|
+
// add 1 dim for bcast mul.
|
1659
|
+
size_t weight_nb[GGML_MAX_DIMS + 1], scale_nb[GGML_MAX_DIMS + 1],
|
1660
|
+
dequant_nb[GGML_MAX_DIMS + 1];
|
1661
|
+
int64_t weight_ne[GGML_MAX_DIMS + 1], scale_ne[GGML_MAX_DIMS + 1],
|
1662
|
+
*dequant_ne;
|
1663
|
+
int64_t scale_offset = 0;
|
1664
|
+
|
1665
|
+
// [3,4,5,64] -> [3,4,5,2,32]
|
1666
|
+
weight_ne[0] = QK8_0;
|
1667
|
+
weight_ne[1] = src0->ne[0] / QK8_0;
|
1668
|
+
weight_nb[0] = sizeof(int8_t);
|
1669
|
+
weight_nb[1] = weight_nb[0] * weight_ne[0];
|
1670
|
+
for (int i = 2; i < GGML_MAX_DIMS + 1; i++) {
|
1671
|
+
weight_ne[i] = src0->ne[i - 1];
|
1672
|
+
weight_nb[i] = weight_nb[i - 1] * weight_ne[i - 1];
|
1673
|
+
}
|
1674
|
+
|
1675
|
+
// [3,4,5,64] -> [3,4,5,2,1]
|
1676
|
+
scale_ne[0] = 1;
|
1677
|
+
scale_ne[1] = src0->ne[0] / QK8_0;
|
1678
|
+
scale_nb[0] = sizeof(uint16_t);
|
1679
|
+
scale_nb[1] = scale_nb[0] * scale_ne[0];
|
1680
|
+
for (int i = 2; i < GGML_MAX_DIMS + 1; i++) {
|
1681
|
+
scale_ne[i] = src0->ne[i - 1];
|
1682
|
+
scale_nb[i] = scale_nb[i - 1] * scale_ne[i - 1];
|
1683
|
+
}
|
1684
|
+
|
1685
|
+
// [3,4,5,64] -> [3,4,5,2,32]
|
1686
|
+
dequant_ne = weight_ne;
|
1687
|
+
dequant_nb[0] = sizeof(float_t);
|
1688
|
+
for (int i = 1; i < GGML_MAX_DIMS + 1; i++) {
|
1689
|
+
dequant_nb[i] = dequant_nb[i - 1] * dequant_ne[i - 1];
|
1690
|
+
}
|
1691
|
+
|
1692
|
+
scale_offset = ggml_nelements(src0) * sizeof(int8_t);
|
1693
|
+
ggml_cann_pool_alloc dequant_buffer_allocator(
|
1694
|
+
ctx.pool(), ggml_nelements(src0) * sizeof(float_t));
|
1695
|
+
|
1696
|
+
aclTensor* acl_weight_tensor = ggml_cann_create_tensor(
|
1697
|
+
src0->data, ACL_INT8, sizeof(int8_t), weight_ne, weight_nb,
|
1698
|
+
GGML_MAX_DIMS + 1);
|
1699
|
+
aclTensor* acl_scale_tensor = ggml_cann_create_tensor(
|
1700
|
+
src0->data, ACL_FLOAT16, sizeof(uint16_t), scale_ne, scale_nb,
|
1701
|
+
GGML_MAX_DIMS + 1, ACL_FORMAT_ND, scale_offset);
|
1702
|
+
aclTensor* dequant_tensor = ggml_cann_create_tensor(
|
1703
|
+
dequant_buffer_allocator.get(), ACL_FLOAT, sizeof(float_t),
|
1704
|
+
dequant_ne, dequant_nb, GGML_MAX_DIMS + 1);
|
1705
|
+
|
1706
|
+
aclnn_mul(ctx, acl_weight_tensor, acl_scale_tensor, dequant_tensor);
|
1707
|
+
dequant_nb[0] = sizeof(float_t);
|
1708
|
+
dequant_ne = src0->ne;
|
1709
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
1710
|
+
dequant_nb[i] = dequant_nb[i - 1] * src0->ne[i - 1];
|
1711
|
+
}
|
1712
|
+
|
1713
|
+
aclnn_embedding_4d(ctx, dequant_buffer_allocator.get(),
|
1714
|
+
dequant_ne, dequant_nb, src1, dst);
|
1715
|
+
|
1716
|
+
ggml_cann_release_resources(ctx, dequant_tensor);
|
1717
|
+
break;
|
1718
|
+
}
|
1719
|
+
default:
|
1720
|
+
GGML_ABORT("Unsupported tensor type for GGML_OP_GET_ROWS");
|
1721
|
+
break;
|
1722
|
+
}
|
1723
|
+
}
|
1724
|
+
|
1725
|
+
/**
|
1726
|
+
* @brief Repeats elements of a tensor along a specified dimension.
|
1727
|
+
*
|
1728
|
+
* This function repeats each element of the source tensor `acl_src` a specified
|
1729
|
+
* number of times (`repeats`) along the specified dimension `dim` and stores
|
1730
|
+
* the result in the destination tensor `acl_dst`.
|
1731
|
+
*
|
1732
|
+
* @param ctx The context for the CANN backend operations.
|
1733
|
+
* @param acl_src The source tensor whose elements will be repeated.
|
1734
|
+
* @param acl_dst The destination tensor where the repeated elements will be
|
1735
|
+
* stored.
|
1736
|
+
* @param dim The dimension along which the elements will be repeated.
|
1737
|
+
* @param repeats The number of times each element will be repeated.
|
1738
|
+
* @param output_size The size of the output tensor.
|
1739
|
+
*/
|
1740
|
+
static void aclnn_repeat_interleave(ggml_backend_cann_context& ctx,
|
1741
|
+
aclTensor* acl_src, aclTensor* acl_dst,
|
1742
|
+
int64_t dim, int64_t repeats,
|
1743
|
+
int64_t output_size) {
|
1744
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, RepeatInterleaveIntWithDim, acl_src, repeats, dim,
|
1745
|
+
output_size, acl_dst);
|
1746
|
+
}
|
1747
|
+
|
1748
|
+
/**
|
1749
|
+
* @brief Performs matrix multiplication with floating-point precision on
|
1750
|
+
* tensors using the CANN backend.
|
1751
|
+
*
|
1752
|
+
* This function performs matrix multiplication of the input tensor and the
|
1753
|
+
* weight tensor, handling broadcasting and transposing as needed, and stores
|
1754
|
+
* the result in the destination tensor `dst`.
|
1755
|
+
*
|
1756
|
+
* @param ctx The context for the CANN backend operations.
|
1757
|
+
* @param dst The destination tensor where the result of the matrix
|
1758
|
+
* multiplication will be stored.
|
1759
|
+
*/
|
1760
|
+
static void ggml_cann_mat_mul_fp(ggml_backend_cann_context& ctx,
|
1761
|
+
ggml_tensor* dst) {
|
1762
|
+
ggml_tensor* weight = dst->src[0]; // weight
|
1763
|
+
ggml_tensor* input = dst->src[1]; // input
|
1764
|
+
|
1765
|
+
// when weight ne2 or ne3 is 1, aclnnMatmulGetWorkspaceSize will auto
|
1766
|
+
// broadcast, when weight ne2 or ne3 is not 1, weight need repeat.
|
1767
|
+
BCAST_MUL_MAT_SHAPE(input, weight, dst);
|
1768
|
+
|
1769
|
+
int64_t n_dims = bcast_dims;
|
1770
|
+
if (bcast_input_ne[3] == bcast_weight_ne[3] && bcast_input_ne[3] == 1) {
|
1771
|
+
if (bcast_input_ne[2] == 1 && bcast_weight_ne[2] == 1) {
|
1772
|
+
n_dims = 2;
|
1773
|
+
} else if (bcast_input_ne[2] == 1) {
|
1774
|
+
n_dims = 3;
|
1775
|
+
}
|
1776
|
+
}
|
1777
|
+
|
1778
|
+
aclTensor* acl_input_tensor =
|
1779
|
+
ggml_cann_create_tensor(input, bcast_input_ne, bcast_input_nb, n_dims);
|
1780
|
+
int64_t transpose_ne[] = {bcast_weight_ne[1], bcast_weight_ne[0],
|
1781
|
+
bcast_weight_ne[2], bcast_weight_ne[3],
|
1782
|
+
bcast_weight_ne[4], bcast_weight_ne[5]};
|
1783
|
+
size_t transpose_nb[] = {bcast_weight_nb[1], bcast_weight_nb[0],
|
1784
|
+
bcast_weight_nb[2], bcast_weight_nb[3],
|
1785
|
+
bcast_weight_nb[4], bcast_weight_nb[5]};
|
1786
|
+
aclTensor* acl_weight_tensor =
|
1787
|
+
ggml_cann_create_tensor(weight, transpose_ne, transpose_nb, n_dims);
|
1788
|
+
aclTensor* acl_dst =
|
1789
|
+
ggml_cann_create_tensor(dst, bcast_dst_ne, bcast_dst_nb, n_dims);
|
1790
|
+
|
1791
|
+
switch (n_dims) {
|
1792
|
+
case 2:
|
1793
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Mm, acl_input_tensor, acl_weight_tensor, acl_dst, 2);
|
1794
|
+
break;
|
1795
|
+
case 3:
|
1796
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, BatchMatMul, acl_input_tensor, acl_weight_tensor, acl_dst, 2);
|
1797
|
+
break;
|
1798
|
+
default:
|
1799
|
+
// ALLOW_FP32_DOWN_PRECISION, when input is
|
1800
|
+
// fp32, atlas a2 will transpose it to HFLOAT32.
|
1801
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Matmul, acl_input_tensor, acl_weight_tensor, acl_dst, 1);
|
1802
|
+
break;
|
1803
|
+
}
|
1804
|
+
|
1805
|
+
ggml_cann_release_resources(ctx, acl_weight_tensor, acl_input_tensor, acl_dst);
|
1806
|
+
}
|
1807
|
+
|
1808
|
+
/**
|
1809
|
+
* @brief Performs matrix multiplication with quantized weights and
|
1810
|
+
* floating-point inputs using the CANN backend.
|
1811
|
+
*
|
1812
|
+
* This function performs matrix multiplication of the input tensor `src1` and
|
1813
|
+
* the weight tensor `src0`, handling broadcasting, transposing, and
|
1814
|
+
* quantization as needed, and stores the result in the destination tensor
|
1815
|
+
* `dst`.
|
1816
|
+
*
|
1817
|
+
* @param ctx The context for the CANN backend operations.
|
1818
|
+
* @param dst The destination tensor where the result of the matrix
|
1819
|
+
* multiplication will be stored.
|
1820
|
+
*/
|
1821
|
+
static void ggml_cann_mul_mat_quant(ggml_backend_cann_context& ctx,
|
1822
|
+
ggml_tensor* dst,
|
1823
|
+
const enum ggml_type type) {
|
1824
|
+
ggml_tensor* src0 = dst->src[0]; // weight
|
1825
|
+
ggml_tensor* src1 = dst->src[1]; // input
|
1826
|
+
|
1827
|
+
// The shape of the weight is NCHW.
|
1828
|
+
// Matrix multiplication uses HW dims.
|
1829
|
+
// HC is regarded as batch.
|
1830
|
+
// weight need transpose.
|
1831
|
+
float weight_elem_size;
|
1832
|
+
if (type == GGML_TYPE_Q4_0) {
|
1833
|
+
weight_elem_size = float(sizeof(uint8_t)) / 2;
|
1834
|
+
} else if (type == GGML_TYPE_Q8_0) {
|
1835
|
+
weight_elem_size = float(sizeof(uint8_t));
|
1836
|
+
} else {
|
1837
|
+
GGML_ABORT("Only support Q4_0 and Q8_0 MUL_MAT");
|
1838
|
+
}
|
1839
|
+
float weight_nb[] = {src0->ne[0] * weight_elem_size, weight_elem_size};
|
1840
|
+
size_t weight_stride = src0->ne[1] * src0->ne[0] * weight_elem_size;
|
1841
|
+
size_t weight_size = weight_stride * src0->ne[2] * src0->ne[3];
|
1842
|
+
|
1843
|
+
// scale stored at the end of weight. Also need transpose.
|
1844
|
+
size_t scale_elem_size = sizeof(uint16_t);
|
1845
|
+
size_t scale_nb[] = {src0->ne[0] / QK8_0 * scale_elem_size,
|
1846
|
+
scale_elem_size};
|
1847
|
+
size_t scale_stride = src0->ne[1] * src0->ne[0] / QK8_0 * scale_elem_size;
|
1848
|
+
char* scale_offset = (char*)src0->data + weight_size;
|
1849
|
+
|
1850
|
+
// input
|
1851
|
+
size_t input_elem_size = sizeof(uint16_t);
|
1852
|
+
int64_t input_ne[] = {src1->ne[0], src1->ne[1]};
|
1853
|
+
size_t input_nb[] = {input_elem_size, input_ne[0] * input_elem_size};
|
1854
|
+
size_t input_stride = input_ne[0] * input_ne[1] * input_elem_size;
|
1855
|
+
ggml_cann_pool_alloc input_alloctor(ctx.pool());
|
1856
|
+
void* input_buffer = src1->data;
|
1857
|
+
|
1858
|
+
// case in
|
1859
|
+
if (src1->type != GGML_TYPE_F16) {
|
1860
|
+
aclTensor* acl_src1_tensor = ggml_cann_create_tensor(src1);
|
1861
|
+
input_buffer =
|
1862
|
+
input_alloctor.alloc(ggml_nelements(src1) * input_elem_size);
|
1863
|
+
|
1864
|
+
int64_t* input_cast_ne = src1->ne;
|
1865
|
+
size_t input_cast_nb[GGML_MAX_DIMS];
|
1866
|
+
input_cast_nb[0] = sizeof(uint16_t);
|
1867
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
1868
|
+
input_cast_nb[i] = input_cast_nb[i - 1] * input_cast_ne[i - 1];
|
1869
|
+
}
|
1870
|
+
|
1871
|
+
aclTensor* acl_input_tensor = ggml_cann_create_tensor(
|
1872
|
+
input_buffer, ACL_FLOAT16, input_elem_size, input_cast_ne,
|
1873
|
+
input_cast_nb, GGML_MAX_DIMS);
|
1874
|
+
aclnn_cast(ctx, acl_src1_tensor, acl_input_tensor, ACL_FLOAT16);
|
1875
|
+
ggml_cann_release_resources(ctx, acl_input_tensor, acl_src1_tensor);
|
1876
|
+
}
|
1877
|
+
|
1878
|
+
// output
|
1879
|
+
size_t output_elem_size = sizeof(uint16_t);
|
1880
|
+
size_t output_nb[] = {output_elem_size, dst->ne[0] * output_elem_size};
|
1881
|
+
ggml_cann_pool_alloc output_allocator(ctx.pool());
|
1882
|
+
void* output_buffer =
|
1883
|
+
output_allocator.alloc(ggml_nelements(dst) * output_elem_size);
|
1884
|
+
size_t output_stride = dst->ne[0] * dst->ne[1] * output_elem_size;
|
1885
|
+
|
1886
|
+
// aclnn
|
1887
|
+
int64_t max_elem_size = 65535;
|
1888
|
+
int64_t split_size = (src0->ne[1] / max_elem_size) + 1;
|
1889
|
+
ggml_cann_pool_alloc workspace_allocator(ctx.pool());
|
1890
|
+
for (int64_t n1 = 0; n1 < src1->ne[3]; n1++) {
|
1891
|
+
for (int64_t c1 = 0; c1 < src1->ne[2]; c1++) {
|
1892
|
+
int64_t n0 = n1 / (src1->ne[3] / src0->ne[3]);
|
1893
|
+
int64_t c0 = c1 / (src1->ne[2] / src0->ne[2]);
|
1894
|
+
|
1895
|
+
int64_t batch1 = (n1 * src1->ne[2]) + c1;
|
1896
|
+
int64_t batch0 = (n0 * src0->ne[2]) + c0;
|
1897
|
+
|
1898
|
+
aclTensor* acl_input_tensor = ggml_cann_create_tensor(
|
1899
|
+
(char*)input_buffer + batch1 * input_stride, ACL_FLOAT16,
|
1900
|
+
input_elem_size, input_ne, input_nb, 2);
|
1901
|
+
|
1902
|
+
// first split
|
1903
|
+
int64_t weight_ne_offset = 0;
|
1904
|
+
int64_t weight_ne[2] = {
|
1905
|
+
max_elem_size > src0->ne[1] ? src0->ne[1] : max_elem_size,
|
1906
|
+
src0->ne[0]};
|
1907
|
+
int64_t scale_ne_offset = 0;
|
1908
|
+
int64_t scale_ne[2] = {weight_ne[0], weight_ne[1] / QK8_0};
|
1909
|
+
int64_t output_ne_offset = 0;
|
1910
|
+
int64_t output_ne[2] = {weight_ne[0], dst->ne[1]};
|
1911
|
+
|
1912
|
+
aclTensor* acl_weight_tensor = ggml_cann_create_tensor(
|
1913
|
+
(char*)src0->data + batch0 * weight_stride,
|
1914
|
+
ggml_cann_type_mapping(type), weight_elem_size, weight_ne,
|
1915
|
+
weight_nb, 2, ACL_FORMAT_ND, weight_ne_offset);
|
1916
|
+
aclTensor* acl_scale_tensor = ggml_cann_create_tensor(
|
1917
|
+
scale_offset + batch0 * scale_stride, ACL_FLOAT16,
|
1918
|
+
scale_elem_size, scale_ne, scale_nb, 2, ACL_FORMAT_ND,
|
1919
|
+
scale_ne_offset);
|
1920
|
+
aclTensor* acl_output_tensor = ggml_cann_create_tensor(
|
1921
|
+
(char*)output_buffer + batch1 * output_stride, ACL_FLOAT16,
|
1922
|
+
output_elem_size, output_ne, output_nb, 2, ACL_FORMAT_ND,
|
1923
|
+
output_ne_offset);
|
1924
|
+
int64_t antiquantGroupSize = 0;
|
1925
|
+
if (src0->ne[0] > QK8_0) {
|
1926
|
+
antiquantGroupSize = QK8_0;
|
1927
|
+
}
|
1928
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, WeightQuantBatchMatmulV2, acl_input_tensor,
|
1929
|
+
acl_weight_tensor, acl_scale_tensor, nullptr,
|
1930
|
+
nullptr, nullptr, nullptr, antiquantGroupSize,
|
1931
|
+
acl_output_tensor);
|
1932
|
+
ggml_cann_release_resources(ctx, acl_weight_tensor, acl_scale_tensor, acl_output_tensor);
|
1933
|
+
|
1934
|
+
// other splits
|
1935
|
+
for (int64_t split = 1; split < split_size; split++) {
|
1936
|
+
weight_ne_offset +=
|
1937
|
+
weight_elem_size * weight_ne[0] * weight_ne[1];
|
1938
|
+
weight_ne[0] = max_elem_size * (split + 1) > src0->ne[1]
|
1939
|
+
? src0->ne[1] - (max_elem_size * split)
|
1940
|
+
: max_elem_size;
|
1941
|
+
scale_ne_offset += scale_elem_size * scale_ne[0] * scale_ne[1];
|
1942
|
+
scale_ne[0] = weight_ne[0];
|
1943
|
+
output_ne_offset +=
|
1944
|
+
output_elem_size * output_ne[0] * output_ne[1];
|
1945
|
+
output_ne[0] = weight_ne[0];
|
1946
|
+
|
1947
|
+
acl_weight_tensor = ggml_cann_create_tensor(
|
1948
|
+
(char*)src0->data + batch0 * weight_stride,
|
1949
|
+
ggml_cann_type_mapping(type), weight_elem_size, weight_ne,
|
1950
|
+
weight_nb, 2, ACL_FORMAT_ND, weight_ne_offset);
|
1951
|
+
acl_scale_tensor = ggml_cann_create_tensor(
|
1952
|
+
scale_offset + batch0 * scale_stride, ACL_FLOAT16,
|
1953
|
+
scale_elem_size, scale_ne, scale_nb, 2, ACL_FORMAT_ND,
|
1954
|
+
scale_ne_offset);
|
1955
|
+
acl_output_tensor = ggml_cann_create_tensor(
|
1956
|
+
(char*)output_buffer + batch1 * output_stride, ACL_FLOAT16,
|
1957
|
+
output_elem_size, output_ne, output_nb, 2, ACL_FORMAT_ND,
|
1958
|
+
output_ne_offset);
|
1959
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, WeightQuantBatchMatmulV2, acl_input_tensor,
|
1960
|
+
acl_weight_tensor, acl_scale_tensor, nullptr,
|
1961
|
+
nullptr, nullptr, nullptr, antiquantGroupSize,
|
1962
|
+
acl_output_tensor);
|
1963
|
+
ggml_cann_release_resources(ctx, acl_weight_tensor, acl_scale_tensor, acl_output_tensor);
|
1964
|
+
}
|
1965
|
+
|
1966
|
+
ggml_cann_release_resources(ctx, acl_input_tensor);
|
1967
|
+
}
|
1968
|
+
}
|
1969
|
+
|
1970
|
+
// cast out
|
1971
|
+
if (dst->type != GGML_TYPE_F16) {
|
1972
|
+
int64_t* output_cast_ne = dst->ne;
|
1973
|
+
size_t output_cast_nb[GGML_MAX_DIMS];
|
1974
|
+
output_cast_nb[0] = sizeof(uint16_t);
|
1975
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
1976
|
+
output_cast_nb[i] = output_cast_nb[i - 1] * output_cast_ne[i - 1];
|
1977
|
+
}
|
1978
|
+
|
1979
|
+
aclTensor* acl_output_tensor = ggml_cann_create_tensor(
|
1980
|
+
output_buffer, ACL_FLOAT16, output_elem_size, output_cast_ne,
|
1981
|
+
output_cast_nb, GGML_MAX_DIMS);
|
1982
|
+
aclTensor* acl_dst_tensor = ggml_cann_create_tensor(dst);
|
1983
|
+
aclnn_cast(ctx, acl_output_tensor, acl_dst_tensor, ggml_cann_type_mapping(dst->type));
|
1984
|
+
|
1985
|
+
ggml_cann_release_resources(ctx, acl_output_tensor, acl_dst_tensor);
|
1986
|
+
}
|
1987
|
+
}
|
1988
|
+
|
1989
|
+
void ggml_cann_mul_mat(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
1990
|
+
const enum ggml_type type = dst->src[0]->type;
|
1991
|
+
switch (type) {
|
1992
|
+
case GGML_TYPE_F32:
|
1993
|
+
case GGML_TYPE_F16:
|
1994
|
+
ggml_cann_mat_mul_fp(ctx, dst);
|
1995
|
+
break;
|
1996
|
+
case GGML_TYPE_Q4_0:
|
1997
|
+
case GGML_TYPE_Q8_0:
|
1998
|
+
ggml_cann_mul_mat_quant(ctx, dst, type);
|
1999
|
+
break;
|
2000
|
+
default:
|
2001
|
+
GGML_ABORT("Unsupported type for mul_mat");
|
2002
|
+
break;
|
2003
|
+
}
|
2004
|
+
}
|
2005
|
+
|
2006
|
+
/**
|
2007
|
+
* @brief Rolls the elements of a tensor along a specified dimension.
|
2008
|
+
*
|
2009
|
+
* This function rolls the elements of the source tensor `acl_src` by the
|
2010
|
+
* specified shifts `shifts` along the specified dimensions `dims`, and stores
|
2011
|
+
* the result in the destination tensor `acl_dst`.
|
2012
|
+
*
|
2013
|
+
* @param ctx The context for the CANN backend operations.
|
2014
|
+
* @param acl_src The source tensor whose elements will be rolled.
|
2015
|
+
* @param acl_dst The destination tensor where the rolled elements will be
|
2016
|
+
* stored.
|
2017
|
+
* @param shifts An array specifying the number of positions by which elements
|
2018
|
+
* are shifted.
|
2019
|
+
* @param dims An array specifying the dimensions along which elements are
|
2020
|
+
* shifted.
|
2021
|
+
*/
|
2022
|
+
static void aclnn_roll(ggml_backend_cann_context& ctx, aclTensor* acl_src,
|
2023
|
+
aclTensor* acl_dst, int64_t* shifts, int64_t* dims) {
|
2024
|
+
aclIntArray* acl_shifts = aclCreateIntArray(shifts, 1);
|
2025
|
+
aclIntArray* acl_dims = aclCreateIntArray(dims, 1);
|
2026
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Roll, acl_src, acl_shifts, acl_dims, acl_dst);
|
2027
|
+
ggml_cann_release_resources(ctx, acl_shifts, acl_dims);
|
2028
|
+
}
|
2029
|
+
|
2030
|
+
/**
|
2031
|
+
* @brief Fills specified positions of a tensor with a scalar value.
|
2032
|
+
*
|
2033
|
+
* This function fills the positions in the source tensor `acl_src` specified by
|
2034
|
+
* `index` along the dimension `dim` with the scalar value `value`.
|
2035
|
+
*
|
2036
|
+
* @param ctx The context for the CANN backend operations.
|
2037
|
+
* @param acl_src The source tensor where the positions will be filled.
|
2038
|
+
* @param dim The dimension along which the positions are specified.
|
2039
|
+
* @param index An array specifying the positions to be filled.
|
2040
|
+
* @param index_num The number of positions specified in the index array.
|
2041
|
+
* @param value The scalar value used to fill the specified positions.
|
2042
|
+
*/
|
2043
|
+
static void aclnn_index_fill_tensor(ggml_backend_cann_context& ctx,
|
2044
|
+
aclTensor* acl_src, int64_t dim,
|
2045
|
+
int64_t* index, int64_t index_num,
|
2046
|
+
float value) {
|
2047
|
+
aclIntArray* acl_index = aclCreateIntArray(index, index_num);
|
2048
|
+
aclScalar* acl_value = aclCreateScalar(&value, aclDataType::ACL_FLOAT);
|
2049
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceIndexFillTensor, acl_src, dim, acl_index, acl_value);
|
2050
|
+
ggml_cann_release_resources(ctx, acl_index, acl_value);
|
2051
|
+
}
|
2052
|
+
|
2053
|
+
static void aclnn_cache_init(ggml_backend_cann_context& ctx, ggml_tensor* dst,
|
2054
|
+
aclTensor* acl_cos_repeat_tensor,
|
2055
|
+
aclTensor* acl_sin_repeat_tensor,
|
2056
|
+
float theta_scale, float freq_scale,
|
2057
|
+
float attn_factor, bool is_neox) {
|
2058
|
+
// int sin/cos cache, cache has different repeat method depond on
|
2059
|
+
// @param.is_neox
|
2060
|
+
|
2061
|
+
ggml_tensor* src0 = dst->src[0]; // input
|
2062
|
+
ggml_tensor* src1 = dst->src[1]; // position
|
2063
|
+
ggml_tensor* src2 = dst->src[2]; // freq_factors
|
2064
|
+
|
2065
|
+
GGML_TENSOR_BINARY_OP_LOCALS
|
2066
|
+
|
2067
|
+
// theta_scale arange, [0,1,...,ne00/2 - 1]
|
2068
|
+
int64_t theta_scale_length = ne00 / 2;
|
2069
|
+
ggml_cann_pool_alloc theta_scale_allocator(ctx.pool(),
|
2070
|
+
theta_scale_length * sizeof(float_t));
|
2071
|
+
void* theta_scale_buffer = theta_scale_allocator.get();
|
2072
|
+
int64_t theta_scale_ne[] = {theta_scale_length, 1, 1, 1};
|
2073
|
+
size_t theta_scale_nb[] = {sizeof(float_t), sizeof(float_t), sizeof(float_t),
|
2074
|
+
theta_scale_length * sizeof(float_t)};
|
2075
|
+
|
2076
|
+
aclTensor* acl_theta_scale_tensor =
|
2077
|
+
ggml_cann_create_tensor(theta_scale_buffer, ACL_FLOAT, sizeof(float_t),
|
2078
|
+
theta_scale_ne, theta_scale_nb, GGML_MAX_DIMS);
|
2079
|
+
float start = 0;
|
2080
|
+
float step = 1;
|
2081
|
+
float stop = ne00 / 2;
|
2082
|
+
float n_elements = ne00 / 2;
|
2083
|
+
aclnn_arange(ctx, acl_theta_scale_tensor, start, stop, step, n_elements);
|
2084
|
+
|
2085
|
+
// power
|
2086
|
+
aclScalar* acl_theta_scale = aclCreateScalar(&theta_scale, aclDataType::ACL_FLOAT);
|
2087
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, PowScalarTensor, acl_theta_scale, acl_theta_scale_tensor,
|
2088
|
+
acl_theta_scale_tensor);
|
2089
|
+
|
2090
|
+
// freq_scale
|
2091
|
+
if (freq_scale != 1) {
|
2092
|
+
aclnn_muls(ctx, acl_theta_scale_tensor, freq_scale, nullptr, true);
|
2093
|
+
}
|
2094
|
+
|
2095
|
+
// freq_factors
|
2096
|
+
if (src2) {
|
2097
|
+
aclTensor* acl_freq_factors_tensor = ggml_cann_create_tensor(
|
2098
|
+
src2->data, ggml_cann_type_mapping(src2->type),
|
2099
|
+
ggml_type_size(src2->type), theta_scale_ne, theta_scale_nb, GGML_MAX_DIMS);
|
2100
|
+
aclnn_div(ctx, acl_theta_scale_tensor, acl_freq_factors_tensor);
|
2101
|
+
ggml_cann_release_resources(ctx, acl_freq_factors_tensor);
|
2102
|
+
}
|
2103
|
+
|
2104
|
+
// position
|
2105
|
+
GGML_ASSERT(src1->type == GGML_TYPE_I32);
|
2106
|
+
int64_t position_length = src1->ne[0];
|
2107
|
+
int64_t position_ne[] = {1, 1, position_length, 1};
|
2108
|
+
size_t position_nb[] = {sizeof(int32_t), sizeof(int32_t), sizeof(int32_t),
|
2109
|
+
sizeof(int32_t) * position_length};
|
2110
|
+
aclTensor* acl_position_tensor = ggml_cann_create_tensor(
|
2111
|
+
src1->data, ggml_cann_type_mapping(src1->type),
|
2112
|
+
ggml_type_size(src1->type), position_ne, position_nb, GGML_MAX_DIMS);
|
2113
|
+
|
2114
|
+
// power * position
|
2115
|
+
int64_t theta_length = theta_scale_length * position_length;
|
2116
|
+
ggml_cann_pool_alloc theta_allocator(ctx.pool(),
|
2117
|
+
theta_length * sizeof(float_t));
|
2118
|
+
void* theta_buffer = theta_allocator.get();
|
2119
|
+
int64_t theta_ne[] = {theta_scale_length, 1, position_length, 1};
|
2120
|
+
size_t theta_nb[GGML_MAX_DIMS];
|
2121
|
+
theta_nb[0] = sizeof(float_t);
|
2122
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
2123
|
+
theta_nb[i] = theta_nb[i - 1] * theta_ne[i - 1];
|
2124
|
+
}
|
2125
|
+
aclTensor* acl_theta_tensor =
|
2126
|
+
ggml_cann_create_tensor(theta_buffer, ACL_FLOAT, sizeof(float_t),
|
2127
|
+
theta_ne, theta_nb, GGML_MAX_DIMS);
|
2128
|
+
aclnn_mul(ctx, acl_position_tensor, acl_theta_scale_tensor,
|
2129
|
+
acl_theta_tensor);
|
2130
|
+
|
2131
|
+
// sin/cos
|
2132
|
+
ggml_cann_pool_alloc sin_allocator(ctx.pool(),
|
2133
|
+
theta_length * sizeof(float_t));
|
2134
|
+
void* sin_buffer = sin_allocator.get();
|
2135
|
+
aclTensor* acl_sin_tensor = ggml_cann_create_tensor(
|
2136
|
+
sin_buffer, ACL_FLOAT, sizeof(float_t), theta_ne, theta_nb,
|
2137
|
+
GGML_MAX_DIMS, ACL_FORMAT_ND);
|
2138
|
+
aclnn_sin(ctx, acl_theta_tensor, acl_sin_tensor);
|
2139
|
+
|
2140
|
+
ggml_cann_pool_alloc cos_allocator(ctx.pool(),
|
2141
|
+
theta_length * sizeof(float_t));
|
2142
|
+
void* cos_buffer = cos_allocator.get();
|
2143
|
+
aclTensor* acl_cos_tensor = ggml_cann_create_tensor(
|
2144
|
+
cos_buffer, ACL_FLOAT, sizeof(float_t), theta_ne, theta_nb,
|
2145
|
+
GGML_MAX_DIMS, ACL_FORMAT_ND);
|
2146
|
+
aclnn_cos(ctx, acl_theta_tensor, acl_cos_tensor);
|
2147
|
+
|
2148
|
+
// attn_factor
|
2149
|
+
if (attn_factor != 1) {
|
2150
|
+
aclnn_muls(ctx, acl_sin_tensor, attn_factor, nullptr, true);
|
2151
|
+
aclnn_muls(ctx, acl_cos_tensor, attn_factor, nullptr, true);
|
2152
|
+
}
|
2153
|
+
|
2154
|
+
// repeat
|
2155
|
+
if (is_neox) {
|
2156
|
+
int64_t repeatsArray[] = {1, 1, 1, 2};
|
2157
|
+
aclnn_repeat(ctx, acl_sin_tensor, acl_sin_repeat_tensor, repeatsArray);
|
2158
|
+
aclnn_repeat(ctx, acl_cos_tensor, acl_cos_repeat_tensor, repeatsArray);
|
2159
|
+
} else {
|
2160
|
+
int64_t num_repeats = 2;
|
2161
|
+
int64_t dim = 3;
|
2162
|
+
int64_t output_size = theta_scale_length * num_repeats;
|
2163
|
+
aclnn_repeat_interleave(ctx, acl_sin_tensor, acl_sin_repeat_tensor, dim,
|
2164
|
+
num_repeats, output_size);
|
2165
|
+
aclnn_repeat_interleave(ctx, acl_cos_tensor, acl_cos_repeat_tensor, dim,
|
2166
|
+
num_repeats, output_size);
|
2167
|
+
}
|
2168
|
+
|
2169
|
+
// release
|
2170
|
+
ggml_cann_release_resources(ctx, acl_theta_scale_tensor, acl_position_tensor,
|
2171
|
+
acl_theta_tensor, acl_sin_tensor, acl_cos_tensor, acl_theta_scale);
|
2172
|
+
}
|
2173
|
+
|
2174
|
+
#ifdef __cplusplus
|
2175
|
+
extern "C" {
|
2176
|
+
#endif
|
2177
|
+
aclnnStatus aclnnRotaryPositionEmbeddingGetWorkspaceSize(
|
2178
|
+
const aclTensor* x, const aclTensor* cos, const aclTensor* sin,
|
2179
|
+
int64_t mode, const aclTensor* yOut, uint64_t* workspaceSize,
|
2180
|
+
aclOpExecutor** executor);
|
2181
|
+
aclnnStatus aclnnRotaryPositionEmbedding(void* workspace,
|
2182
|
+
uint64_t workspaceSize,
|
2183
|
+
aclOpExecutor* executor,
|
2184
|
+
aclrtStream stream);
|
2185
|
+
#ifdef __cplusplus
|
2186
|
+
}
|
2187
|
+
#endif
|
2188
|
+
|
2189
|
+
void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
2190
|
+
// TODO: use ascendc
|
2191
|
+
// Only test with LLAMA model.
|
2192
|
+
ggml_tensor* src0 = dst->src[0]; // input
|
2193
|
+
|
2194
|
+
// param
|
2195
|
+
float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
|
2196
|
+
// const int n_past = ((int32_t *) dst->op_params)[0];
|
2197
|
+
const int n_dims = ((int32_t*)dst->op_params)[1];
|
2198
|
+
const int mode = ((int32_t*)dst->op_params)[2];
|
2199
|
+
// const int n_ctx = ((int32_t *) dst->op_params)[3];
|
2200
|
+
const int n_ctx_orig = ((int32_t*)dst->op_params)[4];
|
2201
|
+
|
2202
|
+
GGML_TENSOR_UNARY_OP_LOCALS
|
2203
|
+
|
2204
|
+
memcpy(&freq_base, (int32_t*)dst->op_params + 5, sizeof(float));
|
2205
|
+
memcpy(&freq_scale, (int32_t*)dst->op_params + 6, sizeof(float));
|
2206
|
+
memcpy(&ext_factor, (int32_t*)dst->op_params + 7, sizeof(float));
|
2207
|
+
memcpy(&attn_factor, (int32_t*)dst->op_params + 8, sizeof(float));
|
2208
|
+
memcpy(&beta_fast, (int32_t*)dst->op_params + 9, sizeof(float));
|
2209
|
+
memcpy(&beta_slow, (int32_t*)dst->op_params + 10, sizeof(float));
|
2210
|
+
|
2211
|
+
// TODO: n_dims <= ne0
|
2212
|
+
GGML_ASSERT(n_dims == ne0);
|
2213
|
+
GGML_ASSERT(n_dims % 2 == 0);
|
2214
|
+
// TODO: ext_factor != 0
|
2215
|
+
GGML_ASSERT(ext_factor == 0);
|
2216
|
+
|
2217
|
+
const float theta_scale = powf(freq_base, -2.0f / n_dims);
|
2218
|
+
|
2219
|
+
float corr_dims[2];
|
2220
|
+
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast,
|
2221
|
+
beta_slow, corr_dims);
|
2222
|
+
|
2223
|
+
const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
|
2224
|
+
|
2225
|
+
// init cos/sin cache
|
2226
|
+
ggml_cann_pool_alloc sin_allocator(
|
2227
|
+
ctx.pool(), ne00 * ne02 * sizeof(float_t));
|
2228
|
+
ggml_cann_pool_alloc cos_allocator(
|
2229
|
+
ctx.pool(), ne00 * ne02 * sizeof(float_t));
|
2230
|
+
void* sin_buffer = sin_allocator.get();
|
2231
|
+
void* cos_buffer = cos_allocator.get();
|
2232
|
+
|
2233
|
+
int64_t sin_reshape_ne[4] = {ne00, 1, ne02, 1};
|
2234
|
+
size_t sin_reshape_nb[GGML_MAX_DIMS];
|
2235
|
+
sin_reshape_nb[0] = sizeof(float_t);
|
2236
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
2237
|
+
sin_reshape_nb[i] = sin_reshape_nb[i - 1] * sin_reshape_ne[i - 1];
|
2238
|
+
}
|
2239
|
+
aclTensor* acl_sin_reshape_tensor =
|
2240
|
+
ggml_cann_create_tensor(sin_buffer, ACL_FLOAT, sizeof(float_t),
|
2241
|
+
sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);
|
2242
|
+
aclTensor* acl_cos_reshape_tensor =
|
2243
|
+
ggml_cann_create_tensor(cos_buffer, ACL_FLOAT, sizeof(float_t),
|
2244
|
+
sin_reshape_ne, sin_reshape_nb, GGML_MAX_DIMS);
|
2245
|
+
aclnn_cache_init(ctx, dst, acl_cos_reshape_tensor, acl_sin_reshape_tensor,
|
2246
|
+
theta_scale, freq_scale, attn_factor, is_neox);
|
2247
|
+
|
2248
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src0);
|
2249
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
2250
|
+
|
2251
|
+
#ifdef ASCEND_310P
|
2252
|
+
// Special ROPE operation for 310P
|
2253
|
+
|
2254
|
+
// roll input
|
2255
|
+
void* input_roll_buffer;
|
2256
|
+
aclTensor* acl_minus_one_tensor;
|
2257
|
+
void* minus_one_scale_buffer = nullptr;
|
2258
|
+
ggml_cann_pool_alloc roll_allocator(ctx.pool(), ggml_nbytes(src0));
|
2259
|
+
ggml_cann_pool_alloc minus_one_scale_allocator(
|
2260
|
+
ctx.pool(), sizeof(float_t) * src0->ne[0]);
|
2261
|
+
if (!is_neox) {
|
2262
|
+
// roll input: [q0,q1,q2,q3,...] -> [q1,q0,q3,q2,...]
|
2263
|
+
input_roll_buffer = roll_allocator.get();
|
2264
|
+
int64_t input_roll_ne[4] = {2, src0->ne[1] * (src0->ne[0] / 2),
|
2265
|
+
src0->ne[2], src0->ne[3]};
|
2266
|
+
size_t input_roll_nb[GGML_MAX_DIMS];
|
2267
|
+
input_roll_nb[0] = ggml_type_size(src0->type);
|
2268
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
2269
|
+
input_roll_nb[i] = input_roll_nb[i - 1] * input_roll_ne[i - 1];
|
2270
|
+
}
|
2271
|
+
aclTensor* acl_input_roll_tensor = ggml_cann_create_tensor(
|
2272
|
+
input_roll_buffer, ggml_cann_type_mapping(src0->type),
|
2273
|
+
ggml_type_size(src0->type), input_roll_ne, input_roll_nb,
|
2274
|
+
GGML_MAX_DIMS);
|
2275
|
+
aclTensor* acl_input_tensor = ggml_cann_create_tensor(
|
2276
|
+
src0->data, ggml_cann_type_mapping(src0->type),
|
2277
|
+
ggml_type_size(src0->type), input_roll_ne, input_roll_nb,
|
2278
|
+
GGML_MAX_DIMS);
|
2279
|
+
|
2280
|
+
int64_t shifts[] = {1};
|
2281
|
+
int64_t dims[] = {3};
|
2282
|
+
aclnn_roll(ctx, acl_input_tensor, acl_input_roll_tensor, shifts, dims);
|
2283
|
+
ggml_cann_release_resources(ctx, acl_input_roll_tensor, acl_input_tensor);
|
2284
|
+
|
2285
|
+
// init [-1, 1, -1, 1, ...]
|
2286
|
+
minus_one_scale_buffer = minus_one_scale_allocator.get();
|
2287
|
+
|
2288
|
+
int64_t minus_one_ne[4] = {src0->ne[0], 1, 1, 1};
|
2289
|
+
size_t minus_one_nb[GGML_MAX_DIMS];
|
2290
|
+
minus_one_nb[0] = sizeof(float_t);
|
2291
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
2292
|
+
minus_one_nb[i] = minus_one_nb[i - 1] * minus_one_ne[i - 1];
|
2293
|
+
}
|
2294
|
+
acl_minus_one_tensor = aclnn_values(
|
2295
|
+
ctx, minus_one_scale_buffer, sizeof(float_t) * src0->ne[0],
|
2296
|
+
minus_one_ne, GGML_MAX_DIMS, ACL_FLOAT, sizeof(float_t), 1);
|
2297
|
+
int64_t dim = 3;
|
2298
|
+
int64_t* index = new int64_t[src0->ne[0]];
|
2299
|
+
for (int i = 0; i < src0->ne[0]; i++) {
|
2300
|
+
index[i] = i / 2 * 2;
|
2301
|
+
}
|
2302
|
+
int64_t index_num = src0->ne[0];
|
2303
|
+
float value = -1;
|
2304
|
+
aclnn_index_fill_tensor(ctx, acl_minus_one_tensor, dim, index,
|
2305
|
+
index_num, value);
|
2306
|
+
} else {
|
2307
|
+
// roll input: [q0,q1,q2,...] ->
|
2308
|
+
// [q_half,q_half+1,...,q_end,q0,q1,...q_half-1]
|
2309
|
+
input_roll_buffer = roll_allocator.get();
|
2310
|
+
aclTensor* acl_input_roll_tensor = ggml_cann_create_tensor(
|
2311
|
+
input_roll_buffer, ggml_cann_type_mapping(src0->type),
|
2312
|
+
ggml_type_size(src0->type), src0->ne, src0->nb, GGML_MAX_DIMS);
|
2313
|
+
aclTensor* acl_input_tensor = ggml_cann_create_tensor(src0);
|
2314
|
+
|
2315
|
+
int64_t shifts[] = {src0->ne[0] / 2};
|
2316
|
+
int64_t dims[] = {3};
|
2317
|
+
aclnn_roll(ctx, acl_input_tensor, acl_input_roll_tensor, shifts, dims);
|
2318
|
+
|
2319
|
+
ggml_cann_release_resources(ctx, acl_input_roll_tensor, acl_input_tensor);
|
2320
|
+
// init [-1, -1, -1, 1, 1,1,...]
|
2321
|
+
minus_one_scale_buffer = minus_one_scale_allocator.get();
|
2322
|
+
int64_t minus_one_ne[4] = {src0->ne[0], 1, 1, 1};
|
2323
|
+
size_t minus_one_nb[GGML_MAX_DIMS];
|
2324
|
+
minus_one_nb[0] = sizeof(float_t);
|
2325
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
2326
|
+
minus_one_nb[i] = minus_one_nb[i - 1] * minus_one_ne[i - 1];
|
2327
|
+
}
|
2328
|
+
acl_minus_one_tensor = aclnn_values(
|
2329
|
+
ctx, minus_one_scale_buffer, sizeof(float_t) * src0->ne[0],
|
2330
|
+
minus_one_ne, GGML_MAX_DIMS, ACL_FLOAT, sizeof(float_t), 1);
|
2331
|
+
// -1 * first half
|
2332
|
+
int64_t first_half_ne[4] = {src0->ne[0] / 2, 1, 1, 1};
|
2333
|
+
size_t first_half_nb[GGML_MAX_DIMS];
|
2334
|
+
first_half_nb[0] = sizeof(float_t);
|
2335
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
2336
|
+
first_half_nb[i] = first_half_nb[i - 1] * first_half_ne[i - 1];
|
2337
|
+
}
|
2338
|
+
aclTensor* acl_first_half_tensor = ggml_cann_create_tensor(
|
2339
|
+
minus_one_scale_buffer, ACL_FLOAT, sizeof(float_t), first_half_ne,
|
2340
|
+
first_half_nb, GGML_MAX_DIMS);
|
2341
|
+
bool inplace = true;
|
2342
|
+
float scale = -1;
|
2343
|
+
aclnn_muls(ctx, acl_first_half_tensor, scale, nullptr, inplace);
|
2344
|
+
ggml_cann_release_resources(ctx, acl_first_half_tensor);
|
2345
|
+
}
|
2346
|
+
|
2347
|
+
// TODO: n_dims < ne0
|
2348
|
+
GGML_ASSERT(n_dims == src0->ne[0]);
|
2349
|
+
|
2350
|
+
// input * scale
|
2351
|
+
ggml_cann_pool_alloc roll_mul_scale_allocator(ctx.pool(),
|
2352
|
+
ggml_nbytes(src0));
|
2353
|
+
void* input_roll_mul_scale_buffer = roll_mul_scale_allocator.get();
|
2354
|
+
size_t input_nb[GGML_MAX_DIMS];
|
2355
|
+
input_nb[0] = ggml_type_size(src0->type);
|
2356
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
2357
|
+
input_nb[i] = input_nb[i - 1] * src0->ne[i - 1];
|
2358
|
+
}
|
2359
|
+
aclTensor* acl_input_roll_mul_scale_tensor = ggml_cann_create_tensor(
|
2360
|
+
input_roll_mul_scale_buffer, ggml_cann_type_mapping(src0->type),
|
2361
|
+
ggml_type_size(src0->type), src0->ne, input_nb, GGML_MAX_DIMS);
|
2362
|
+
aclTensor* acl_input_roll_reshape_tensor = ggml_cann_create_tensor(
|
2363
|
+
input_roll_buffer, ggml_cann_type_mapping(src0->type),
|
2364
|
+
ggml_type_size(src0->type), src0->ne, input_nb, GGML_MAX_DIMS);
|
2365
|
+
|
2366
|
+
aclnn_mul(ctx, acl_input_roll_reshape_tensor, acl_minus_one_tensor,
|
2367
|
+
acl_input_roll_mul_scale_tensor);
|
2368
|
+
|
2369
|
+
// output
|
2370
|
+
void* output_fp32_buffer;
|
2371
|
+
if (src0->type == GGML_TYPE_F32) {
|
2372
|
+
aclnn_mul(ctx, acl_src, acl_cos_reshape_tensor);
|
2373
|
+
aclnn_mul(ctx, acl_input_roll_mul_scale_tensor,
|
2374
|
+
acl_sin_reshape_tensor);
|
2375
|
+
aclnn_add(ctx, acl_src, acl_input_roll_mul_scale_tensor, acl_dst);
|
2376
|
+
// TODO: ne0 != n_dims in mode2
|
2377
|
+
} else if (src0->type == GGML_TYPE_F16) {
|
2378
|
+
size_t input_fp32_nb[GGML_MAX_DIMS];
|
2379
|
+
input_fp32_nb[0] = sizeof(float_t);
|
2380
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
2381
|
+
input_fp32_nb[i] = input_fp32_nb[i - 1] * dst->ne[i - 1];
|
2382
|
+
}
|
2383
|
+
ggml_cann_pool_alloc fp32_allocator1(
|
2384
|
+
ctx.pool(), ggml_nelements(dst) * sizeof(float_t));
|
2385
|
+
void* input_fp32_buffer1 = fp32_allocator1.get();
|
2386
|
+
aclTensor* input_fp32_tensor1 = ggml_cann_create_tensor(
|
2387
|
+
input_fp32_buffer1, ACL_FLOAT, sizeof(float_t), dst->ne,
|
2388
|
+
input_fp32_nb, GGML_MAX_DIMS);
|
2389
|
+
ggml_cann_pool_alloc fp32_allocator2(
|
2390
|
+
ctx.pool(), ggml_nelements(dst) * sizeof(float_t));
|
2391
|
+
void* input_fp32_buffer2 = fp32_allocator2.get();
|
2392
|
+
aclTensor* input_fp32_tensor2 = ggml_cann_create_tensor(
|
2393
|
+
input_fp32_buffer2, ACL_FLOAT, sizeof(float_t), dst->ne,
|
2394
|
+
input_fp32_nb, GGML_MAX_DIMS);
|
2395
|
+
|
2396
|
+
ggml_cann_pool_alloc fp32_allocator(
|
2397
|
+
ctx.pool(), ggml_nelements(dst) * sizeof(float_t));
|
2398
|
+
output_fp32_buffer = fp32_allocator.get();
|
2399
|
+
aclTensor* output_fp32_tensor = ggml_cann_create_tensor(
|
2400
|
+
output_fp32_buffer, ACL_FLOAT, sizeof(float_t), dst->ne,
|
2401
|
+
input_fp32_nb, GGML_MAX_DIMS);
|
2402
|
+
aclnn_mul(ctx, acl_src, acl_cos_reshape_tensor, input_fp32_tensor1);
|
2403
|
+
aclnn_mul(ctx, acl_input_roll_mul_scale_tensor, acl_sin_reshape_tensor,
|
2404
|
+
input_fp32_tensor2);
|
2405
|
+
aclnn_add(ctx, input_fp32_tensor1, input_fp32_tensor2,
|
2406
|
+
output_fp32_tensor);
|
2407
|
+
aclnn_cast(ctx, output_fp32_tensor, acl_dst, ACL_FLOAT16);
|
2408
|
+
|
2409
|
+
ggml_cann_release_resources(ctx, input_fp32_tensor1, input_fp32_tensor2,
|
2410
|
+
output_fp32_tensor, acl_sin_reshape_tensor,
|
2411
|
+
acl_minus_one_tensor, acl_input_roll_mul_scale_tensor,
|
2412
|
+
acl_input_roll_reshape_tensor, acl_src);
|
2413
|
+
}
|
2414
|
+
return;
|
2415
|
+
#endif
|
2416
|
+
|
2417
|
+
// ggml_mode = 0 --> aclnn_model = 1
|
2418
|
+
int64_t acl_mode = mode == 0 ? 1 : mode;
|
2419
|
+
|
2420
|
+
switch (src0->type) {
|
2421
|
+
case GGML_TYPE_F32: {
|
2422
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, RotaryPositionEmbedding, acl_src,
|
2423
|
+
acl_cos_reshape_tensor, acl_sin_reshape_tensor, acl_mode, acl_dst);
|
2424
|
+
break;
|
2425
|
+
}
|
2426
|
+
case GGML_TYPE_F16: {
|
2427
|
+
ggml_cann_pool_alloc src_trans_allocator(
|
2428
|
+
ctx.pool(), ggml_nelements(src0) * sizeof(float));
|
2429
|
+
void* src_trans_buffer = src_trans_allocator.get();
|
2430
|
+
ggml_cann_pool_alloc dst_trans_allocator(
|
2431
|
+
ctx.pool(), ggml_nelements(dst) * sizeof(float));
|
2432
|
+
void* dst_trans_buffer = dst_trans_allocator.get();
|
2433
|
+
|
2434
|
+
size_t src_trans_nb[GGML_MAX_DIMS];
|
2435
|
+
src_trans_nb[0] = sizeof(float);
|
2436
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
2437
|
+
src_trans_nb[i] = src_trans_nb[i - 1] * src0->ne[i - 1];
|
2438
|
+
}
|
2439
|
+
|
2440
|
+
aclTensor* acl_src_trans_tensor = ggml_cann_create_tensor(
|
2441
|
+
src_trans_buffer, ACL_FLOAT, sizeof(float), src0->ne, src_trans_nb,
|
2442
|
+
GGML_MAX_DIMS);
|
2443
|
+
aclTensor* acl_dst_trans_tensor = ggml_cann_create_tensor(
|
2444
|
+
dst_trans_buffer, ACL_FLOAT, sizeof(float), dst->ne, src_trans_nb,
|
2445
|
+
GGML_MAX_DIMS);
|
2446
|
+
|
2447
|
+
aclnn_cast(ctx, acl_src, acl_src_trans_tensor, ACL_FLOAT);
|
2448
|
+
|
2449
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, RotaryPositionEmbedding, acl_src_trans_tensor,
|
2450
|
+
acl_cos_reshape_tensor, acl_sin_reshape_tensor, acl_mode,
|
2451
|
+
acl_dst_trans_tensor);
|
2452
|
+
|
2453
|
+
aclnn_cast(ctx, acl_dst_trans_tensor, acl_dst, ACL_FLOAT16);
|
2454
|
+
|
2455
|
+
ggml_cann_release_resources(ctx, acl_src_trans_tensor,
|
2456
|
+
acl_dst_trans_tensor);
|
2457
|
+
break;
|
2458
|
+
}
|
2459
|
+
default:
|
2460
|
+
GGML_ABORT("Unsupported tensor type for GGML_OP_ROPE");
|
2461
|
+
break;
|
2462
|
+
}
|
2463
|
+
ggml_cann_release_resources(ctx, acl_cos_reshape_tensor,
|
2464
|
+
acl_sin_reshape_tensor, acl_src, acl_dst);
|
2465
|
+
}
|
2466
|
+
|
2467
|
+
|
2468
|
+
void ggml_cann_argmax(ggml_backend_cann_context& ctx, ggml_tensor* dst){
|
2469
|
+
ggml_tensor * src0 = dst->src[0];
|
2470
|
+
|
2471
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src0);
|
2472
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst, dst->ne, dst->nb, 3);
|
2473
|
+
|
2474
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, ArgMax, acl_src, 3, false, acl_dst);
|
2475
|
+
|
2476
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst);
|
2477
|
+
}
|
2478
|
+
|
2479
|
+
void ggml_cann_conv_transpose_1d(ggml_backend_cann_context& ctx, ggml_tensor* dst){
|
2480
|
+
ggml_tensor * src0 = dst->src[0];
|
2481
|
+
ggml_tensor * src1 = dst->src[1];
|
2482
|
+
|
2483
|
+
// stride
|
2484
|
+
int64_t s0 = ((const int32_t*)(dst->op_params))[0];
|
2485
|
+
|
2486
|
+
aclTensor* acl_input = ggml_cann_create_tensor(src1, src1->ne, src1->nb, 3, ACL_FORMAT_NCL);
|
2487
|
+
aclTensor* acl_weight = ggml_cann_create_tensor(src0, src0->ne, src0->nb, 3, ACL_FORMAT_NCL);
|
2488
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst, dst->ne, dst->nb, 3, ACL_FORMAT_NCL);
|
2489
|
+
|
2490
|
+
int64_t strideVal[1];
|
2491
|
+
strideVal[0] = s0;
|
2492
|
+
aclIntArray *stride = aclCreateIntArray(strideVal, 1);
|
2493
|
+
int64_t paddingVal[] = {0};
|
2494
|
+
aclIntArray *padding = aclCreateIntArray(paddingVal, 1);
|
2495
|
+
int64_t dilationVal[] = {1};
|
2496
|
+
aclIntArray *dilation = aclCreateIntArray(dilationVal, 1);
|
2497
|
+
bool transposed = true;
|
2498
|
+
int64_t groups = 1;
|
2499
|
+
int8_t cubeMathType = 0;
|
2500
|
+
|
2501
|
+
#ifdef ASCEND_310P
|
2502
|
+
cubeMathType = 1;
|
2503
|
+
#endif
|
2504
|
+
|
2505
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Convolution, acl_input, acl_weight, nullptr, stride,
|
2506
|
+
padding, dilation, transposed, padding, groups, acl_dst, cubeMathType);
|
2507
|
+
|
2508
|
+
ggml_cann_release_resources(ctx, acl_weight, acl_dst, stride, padding, dilation);
|
2509
|
+
}
|
2510
|
+
|
2511
|
+
void ggml_cann_elu(ggml_backend_cann_context& ctx, ggml_tensor* dst){
|
2512
|
+
ggml_tensor * src0 = dst->src[0];
|
2513
|
+
|
2514
|
+
aclTensor* acl_input = ggml_cann_create_tensor(src0);
|
2515
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
2516
|
+
|
2517
|
+
float alphaValue = 1.0f;
|
2518
|
+
aclScalar* alpha = nullptr;
|
2519
|
+
alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
|
2520
|
+
|
2521
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Elu, acl_input, alpha, alpha, alpha,
|
2522
|
+
acl_dst);
|
2523
|
+
|
2524
|
+
ggml_cann_release_resources(ctx, acl_input, acl_dst, alpha);
|
2525
|
+
}
|
2526
|
+
|
2527
|
+
void ggml_cann_mean(ggml_backend_cann_context& ctx, ggml_tensor* dst){
|
2528
|
+
ggml_tensor * src0 = dst->src[0];
|
2529
|
+
|
2530
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src0);
|
2531
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
2532
|
+
|
2533
|
+
int64_t reduceDimValue[] = {3};
|
2534
|
+
aclIntArray* reduceDim = aclCreateIntArray(reduceDimValue, 1);
|
2535
|
+
bool keepDim = true;
|
2536
|
+
|
2537
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Mean, acl_src, reduceDim, keepDim, ACL_FLOAT, acl_dst);
|
2538
|
+
|
2539
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst, reduceDim);
|
2540
|
+
}
|
2541
|
+
|
2542
|
+
void ggml_cann_pad_reflect_1d(ggml_backend_cann_context& ctx, ggml_tensor* dst){
|
2543
|
+
ggml_tensor * src0 = dst->src[0];
|
2544
|
+
int32_t *opts = (int32_t *) dst->op_params;
|
2545
|
+
int64_t paddingsArray[2] = {opts[0], opts[1]};
|
2546
|
+
aclIntArray* paddings = aclCreateIntArray(paddingsArray, 2);
|
2547
|
+
|
2548
|
+
for (int64_t i = 0; i < src0->ne[3]; i++) {
|
2549
|
+
aclTensor* acl_src = ggml_cann_create_tensor(
|
2550
|
+
(char*)src0->data + i * src0->ne[3],
|
2551
|
+
ggml_cann_type_mapping(src0->type), ggml_element_size(src0),
|
2552
|
+
src0->ne, src0->nb, 3);
|
2553
|
+
|
2554
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(
|
2555
|
+
(char*)dst->data + i * src0->ne[3],
|
2556
|
+
ggml_cann_type_mapping(dst->type), ggml_element_size(dst),
|
2557
|
+
dst->ne, dst->nb, 3);
|
2558
|
+
|
2559
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, ReflectionPad1d, acl_src, paddings, acl_dst);
|
2560
|
+
|
2561
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst);
|
2562
|
+
}
|
2563
|
+
ggml_cann_release_resources(ctx, paddings);
|
2564
|
+
}
|
2565
|
+
|
2566
|
+
void ggml_cann_count_equal(ggml_backend_cann_context& ctx, ggml_tensor* dst){
|
2567
|
+
ggml_tensor * src0 = dst->src[0];
|
2568
|
+
ggml_tensor * src1 = dst->src[1];
|
2569
|
+
|
2570
|
+
aclTensor* acl_self = ggml_cann_create_tensor(src0);
|
2571
|
+
aclTensor* acl_other = ggml_cann_create_tensor(src1);
|
2572
|
+
|
2573
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceEqTensor, acl_self, acl_other);
|
2574
|
+
|
2575
|
+
ggml_cann_sum(ctx, dst);
|
2576
|
+
|
2577
|
+
ggml_cann_release_resources(ctx, acl_self, acl_other);
|
2578
|
+
}
|
2579
|
+
|
2580
|
+
void ggml_cann_step(ggml_backend_cann_context& ctx, ggml_tensor* dst){
|
2581
|
+
ggml_tensor * src0 = dst->src[0];
|
2582
|
+
|
2583
|
+
aclTensor* acl_src = ggml_cann_create_tensor(src0);
|
2584
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst);
|
2585
|
+
|
2586
|
+
float alphaValue = 0.0f;
|
2587
|
+
aclScalar* alpha = nullptr;
|
2588
|
+
alpha = aclCreateScalar(&alphaValue, aclDataType::ACL_FLOAT);
|
2589
|
+
|
2590
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, GtScalar, acl_src, alpha, acl_dst);
|
2591
|
+
|
2592
|
+
ggml_cann_release_resources(ctx, acl_src, acl_dst, alpha);
|
2593
|
+
}
|
2594
|
+
|
2595
|
+
/**
|
2596
|
+
* @brief Performs expert-specific matrix multiplication (MoE) with
|
2597
|
+
* floating-point precision using the CANN backend.
|
2598
|
+
*
|
2599
|
+
* This function executes a matrix multiplication operation tailored for
|
2600
|
+
* Mixture of Experts (MoE) models, where the input tensor is multiplied
|
2601
|
+
* with expert-specific weight matrices. It uses the CANN backend for
|
2602
|
+
* efficient computation and stores the result in the destination tensor `dst`.
|
2603
|
+
* The operation may leverage identity-based optimizations or routing masks
|
2604
|
+
* as part of sparse expert selection.
|
2605
|
+
*
|
2606
|
+
* @param ctx The context for executing CANN backend operations.
|
2607
|
+
* @param dst The destination tensor where the MoE multiplication result
|
2608
|
+
* will be stored.
|
2609
|
+
*
|
2610
|
+
* @note This function assumes floating-point data types and is designed for
|
2611
|
+
* MoE architectures, possibly involving sparse expert routing.
|
2612
|
+
*/
|
2613
|
+
static void ggml_cann_mul_mat_id_fp(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
2614
|
+
//dst [M, K, N, 1]
|
2615
|
+
ggml_tensor * src0 = dst->src[0]; //src0 [D, M, A, 1]
|
2616
|
+
ggml_tensor * src1 = dst->src[1]; //src1 [D, B, N, 1], B = K or B = 1
|
2617
|
+
ggml_tensor * ids = dst->src[2]; //ids [K, N]
|
2618
|
+
|
2619
|
+
GGML_TENSOR_BINARY_OP_LOCALS
|
2620
|
+
|
2621
|
+
// copy index from npu to cpu
|
2622
|
+
int64_t n_as = ne02; // A
|
2623
|
+
int64_t n_ids = ids->ne[0]; // K
|
2624
|
+
|
2625
|
+
std::vector<char> ids_host(ggml_nbytes(ids));
|
2626
|
+
ggml_cann_async_memcpy(ctx, ids_host.data(), ids->data, ggml_nbytes(ids),
|
2627
|
+
ACL_MEMCPY_DEVICE_TO_HOST);
|
2628
|
+
ACL_CHECK(aclrtSynchronizeStream(ctx.stream()));
|
2629
|
+
|
2630
|
+
char * src0_original = (char *) src0->data;
|
2631
|
+
char * src1_original = (char *) src1->data;
|
2632
|
+
char * dst_original = (char *) dst->data;
|
2633
|
+
size_t ori_src0_nb[4] = {nb00, nb01, nb02, nb03};
|
2634
|
+
|
2635
|
+
// src0 is F16, src1 is F32, dst is F32
|
2636
|
+
ggml_cann_pool_alloc src0_cast_allocator;
|
2637
|
+
if (src0->type == GGML_TYPE_F16) {
|
2638
|
+
src0_cast_allocator.alloc(ctx.pool(), sizeof(float) * ggml_nelements(src0));
|
2639
|
+
void* src0_cast_buf = src0_cast_allocator.get();
|
2640
|
+
|
2641
|
+
size_t cast_nb[GGML_MAX_DIMS];
|
2642
|
+
cast_nb[0] = sizeof(float_t);
|
2643
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
2644
|
+
cast_nb[i] = cast_nb[i - 1] * src0->ne[i - 1];
|
2645
|
+
}
|
2646
|
+
|
2647
|
+
aclTensor* acl_src0_f16 = ggml_cann_create_tensor(src0);
|
2648
|
+
aclTensor* acl_cast = ggml_cann_create_tensor(src0_cast_buf,
|
2649
|
+
ACL_FLOAT, sizeof(float), src0->ne, cast_nb, 4);
|
2650
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, Cast, acl_src0_f16, ACL_FLOAT, acl_cast);
|
2651
|
+
ggml_cann_release_resources(ctx, acl_cast, acl_src0_f16);
|
2652
|
+
|
2653
|
+
src0_original = (char *) src0_cast_buf;
|
2654
|
+
memcpy(ori_src0_nb, cast_nb, sizeof(ori_src0_nb));
|
2655
|
+
}
|
2656
|
+
|
2657
|
+
std::vector<aclTensor*> src0_tensor_vec;
|
2658
|
+
std::vector<aclTensor*> src1_tensor_vec;
|
2659
|
+
std::vector<aclTensor*> dst_tensor_vec;
|
2660
|
+
for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
|
2661
|
+
for (int64_t id = 0; id < n_ids; id++) {
|
2662
|
+
// src0_row [M, D] -> weight && permute
|
2663
|
+
int64_t src0_ne[2] = {ne01, ne00};
|
2664
|
+
size_t src0_nb[2] = {ori_src0_nb[1], ori_src0_nb[0]};
|
2665
|
+
// src1_row [D, 1] -> input
|
2666
|
+
int64_t src1_ne[2] = {ne10, 1};
|
2667
|
+
size_t src1_nb[2] = {nb10, nb11};
|
2668
|
+
// dst_row [M, 1] -> out
|
2669
|
+
int64_t dst_ne[2] = {ne0, 1};
|
2670
|
+
size_t dst_nb[2] = {nb0, nb1};
|
2671
|
+
|
2672
|
+
// expert index
|
2673
|
+
int32_t i02 = *(int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]);
|
2674
|
+
GGML_ASSERT(i02 >= 0 && i02 < n_as);
|
2675
|
+
|
2676
|
+
// If B = 1 (broadcast), always use 0; otherwise, use id.
|
2677
|
+
int64_t i11 = (ne11 == 1 ? 0 : id);
|
2678
|
+
int64_t i12 = iid1;
|
2679
|
+
|
2680
|
+
int64_t i1 = id;
|
2681
|
+
int64_t i2 = i12;
|
2682
|
+
|
2683
|
+
void* src0_tmp_ptr = src0_original + i02*ori_src0_nb[2];
|
2684
|
+
void* src1_tmp_ptr = src1_original + i11*nb11 + i12*nb12;
|
2685
|
+
void* dst_tmp_ptr = dst_original + i1*nb1 + i2*nb2;
|
2686
|
+
|
2687
|
+
aclTensor* acl_src0 = ggml_cann_create_tensor(src0_tmp_ptr,
|
2688
|
+
ACL_FLOAT, sizeof(float),
|
2689
|
+
src0_ne, src0_nb, 2);
|
2690
|
+
aclTensor* acl_src1 = ggml_cann_create_tensor(src1_tmp_ptr,
|
2691
|
+
ACL_FLOAT, sizeof(float),
|
2692
|
+
src1_ne, src1_nb, 2);
|
2693
|
+
aclTensor* acl_dst = ggml_cann_create_tensor(dst_tmp_ptr,
|
2694
|
+
ACL_FLOAT, sizeof(float),
|
2695
|
+
dst_ne, dst_nb, 2);
|
2696
|
+
|
2697
|
+
src0_tensor_vec.push_back(acl_src0);
|
2698
|
+
src1_tensor_vec.push_back(acl_src1);
|
2699
|
+
dst_tensor_vec.push_back(acl_dst);
|
2700
|
+
}
|
2701
|
+
}
|
2702
|
+
|
2703
|
+
size_t GROUP_SIZE = 128;
|
2704
|
+
// GroupedMatmulV2 required tensor_list.size < 128
|
2705
|
+
for (size_t i = 0; i < src0_tensor_vec.size(); i += GROUP_SIZE) {
|
2706
|
+
// split and call GroupedMatmulV2
|
2707
|
+
size_t end = std::min(i + GROUP_SIZE, src0_tensor_vec.size());
|
2708
|
+
std::vector<aclTensor*> src0_tensor_vec_split(src0_tensor_vec.begin() + i, src0_tensor_vec.begin() + end);
|
2709
|
+
std::vector<aclTensor*> src1_tensor_vec_split(src1_tensor_vec.begin() + i, src1_tensor_vec.begin() + end);
|
2710
|
+
std::vector<aclTensor*> dst_tensor_vec_split(dst_tensor_vec.begin() + i, dst_tensor_vec.begin() + end);
|
2711
|
+
|
2712
|
+
aclTensorList* src0_tensor_list = aclCreateTensorList(src0_tensor_vec_split.data(), src0_tensor_vec_split.size());
|
2713
|
+
aclTensorList* src1_tensor_list = aclCreateTensorList(src1_tensor_vec_split.data(), src1_tensor_vec_split.size());
|
2714
|
+
aclTensorList* dst_tensor_list = aclCreateTensorList(dst_tensor_vec_split.data(), dst_tensor_vec_split.size());
|
2715
|
+
|
2716
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, GroupedMatmulV2, src1_tensor_list, src0_tensor_list,
|
2717
|
+
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, 0, -1, dst_tensor_list);
|
2718
|
+
|
2719
|
+
ggml_cann_release_resources(ctx, src0_tensor_list, src1_tensor_list, dst_tensor_list);
|
2720
|
+
}
|
2721
|
+
return;
|
2722
|
+
}
|
2723
|
+
|
2724
|
+
/**
|
2725
|
+
* @brief Performs expert-specific matrix multiplication (MoE) with
|
2726
|
+
* quantized precision using the CANN backend.
|
2727
|
+
*
|
2728
|
+
* This function executes a matrix multiplication operation tailored for
|
2729
|
+
* Mixture of Experts (MoE) models, where the input tensor is multiplied
|
2730
|
+
* with expert-specific quantized weight matrices. It leverages the CANN
|
2731
|
+
* backend to perform efficient low-precision computations and stores the
|
2732
|
+
* quantized result in the destination tensor `dst`.
|
2733
|
+
*
|
2734
|
+
* Quantization techniques reduce memory footprint and improve performance
|
2735
|
+
* by using lower-bit representations (e.g., int8) instead of floating-point.
|
2736
|
+
* This function is designed to work with such formats and may incorporate
|
2737
|
+
* optimizations like identity-based fast paths or routing masks for sparse
|
2738
|
+
* expert selection.
|
2739
|
+
*
|
2740
|
+
* @param ctx The context for executing CANN backend operations.
|
2741
|
+
* @param dst The destination tensor where the quantized MoE multiplication result
|
2742
|
+
* will be stored.
|
2743
|
+
*
|
2744
|
+
* @note This function assumes quantized data types and is designed for
|
2745
|
+
* MoE architectures with potential sparse expert routing.
|
2746
|
+
*/
|
2747
|
+
static void ggml_cann_mul_mat_id_quant(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
2748
|
+
// TODO: Use aclnnGroupedMatMul
|
2749
|
+
//dst [M, K, N, 1]
|
2750
|
+
ggml_tensor * src0 = dst->src[0]; //src0 [D, M, A, 1]
|
2751
|
+
ggml_tensor * src1 = dst->src[1]; //src1 [D, B, N, 1], B = K or B = 1
|
2752
|
+
ggml_tensor * ids = dst->src[2]; //ids [K, N]
|
2753
|
+
|
2754
|
+
GGML_TENSOR_BINARY_OP_LOCALS
|
2755
|
+
|
2756
|
+
// copy index from npu to cpu
|
2757
|
+
int64_t n_as = ne02; // A
|
2758
|
+
int64_t n_ids = ids->ne[0]; // K
|
2759
|
+
|
2760
|
+
std::vector<char> ids_host(ggml_nbytes(ids));
|
2761
|
+
ggml_cann_async_memcpy(ctx, ids_host.data(), ids->data, ggml_nbytes(ids),
|
2762
|
+
ACL_MEMCPY_DEVICE_TO_HOST);
|
2763
|
+
ACL_CHECK(aclrtSynchronizeStream(ctx.stream()));
|
2764
|
+
|
2765
|
+
char * src0_original = (char *) src0->data;
|
2766
|
+
char * src1_original = (char *) src1->data;
|
2767
|
+
char * dst_original = (char *) dst->data;
|
2768
|
+
|
2769
|
+
ggml_tensor src0_row = *src0;
|
2770
|
+
ggml_tensor src1_row = *src1;
|
2771
|
+
ggml_tensor dst_row = *dst;
|
2772
|
+
|
2773
|
+
const enum ggml_type type = dst->src[0]->type;
|
2774
|
+
float weight_elem_size;
|
2775
|
+
if (type == GGML_TYPE_Q4_0) {
|
2776
|
+
weight_elem_size = float(sizeof(uint8_t)) / 2;
|
2777
|
+
} else if (type == GGML_TYPE_Q8_0) {
|
2778
|
+
weight_elem_size = float(sizeof(uint8_t));
|
2779
|
+
} else {
|
2780
|
+
GGML_ABORT("MUL_MAT_ID only support quant type Q4_0 and Q8_0 ");
|
2781
|
+
}
|
2782
|
+
|
2783
|
+
// src0_row [D, M, 1, 1] weight without permute
|
2784
|
+
src0_row.ne[2] = 1;
|
2785
|
+
src0_row.ne[3] = 1;
|
2786
|
+
src0_row.nb[0] = weight_elem_size;
|
2787
|
+
src0_row.nb[1] = weight_elem_size * ne00;
|
2788
|
+
src0_row.nb[2] = weight_elem_size * ne00;
|
2789
|
+
src0_row.nb[3] = weight_elem_size * ne00;
|
2790
|
+
size_t weight_stride = ne00 * ne01 * weight_elem_size;
|
2791
|
+
size_t weight_size = weight_stride * ne02 * ne03;
|
2792
|
+
|
2793
|
+
// scale [D, M, 1, 1] -> scale && permute
|
2794
|
+
size_t scale_elem_size = sizeof(uint16_t);
|
2795
|
+
size_t scale_stride = src0->ne[1] * src0->ne[0] / QK8_0 * scale_elem_size;
|
2796
|
+
|
2797
|
+
// src1_row [D, 1, 1, 1] -> input
|
2798
|
+
src1_row.ne[1] = 1;
|
2799
|
+
src1_row.ne[2] = 1;
|
2800
|
+
src1_row.ne[3] = 1;
|
2801
|
+
src1_row.nb[2] = nb11;
|
2802
|
+
src1_row.nb[3] = nb11;
|
2803
|
+
|
2804
|
+
// dst_row [M, 1, 1, 1] -> out
|
2805
|
+
dst_row.ne[1] = 1;
|
2806
|
+
dst_row.ne[2] = 1;
|
2807
|
+
dst_row.ne[3] = 1;
|
2808
|
+
dst_row.nb[2] = nb1;
|
2809
|
+
dst_row.nb[3] = nb1;
|
2810
|
+
|
2811
|
+
//create weight for one row
|
2812
|
+
ggml_cann_pool_alloc weight_allocator(ctx.pool());
|
2813
|
+
void* weight_buffer = weight_allocator.alloc(nb02);
|
2814
|
+
for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
|
2815
|
+
for (int64_t id = 0; id < n_ids; id++) {
|
2816
|
+
// expert index
|
2817
|
+
int32_t i02 = *(int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]);
|
2818
|
+
GGML_ASSERT(i02 >= 0 && i02 < n_as);
|
2819
|
+
|
2820
|
+
// If B = 1 (broadcast), always use 0; otherwise, use id.
|
2821
|
+
int64_t i11 = (ne11 == 1 ? 0 : id);
|
2822
|
+
int64_t i12 = iid1;
|
2823
|
+
|
2824
|
+
int64_t i1 = id;
|
2825
|
+
int64_t i2 = i12;
|
2826
|
+
|
2827
|
+
void* src0_tmp_ptr = src0_original + i02*weight_stride;
|
2828
|
+
void* scale_tmp_ptr = src0_original + weight_size + i02*scale_stride;
|
2829
|
+
void* src1_tmp_ptr = src1_original + i11*nb11 + i12*nb12;
|
2830
|
+
void* dst_tmp_ptr = dst_original + i1*nb1 + i2*nb2;
|
2831
|
+
|
2832
|
+
// mem cpy
|
2833
|
+
ggml_cann_async_memcpy(ctx, weight_buffer, src0_tmp_ptr, weight_stride,
|
2834
|
+
ACL_MEMCPY_DEVICE_TO_DEVICE);
|
2835
|
+
void* scale_buffer = (char*)weight_buffer + weight_stride;
|
2836
|
+
ggml_cann_async_memcpy(ctx, scale_buffer, scale_tmp_ptr, scale_stride,
|
2837
|
+
ACL_MEMCPY_DEVICE_TO_DEVICE);
|
2838
|
+
|
2839
|
+
src0_row.data = weight_buffer;
|
2840
|
+
src1_row.data = src1_tmp_ptr;
|
2841
|
+
dst_row.data = dst_tmp_ptr;
|
2842
|
+
dst_row.src[0] = &src0_row;
|
2843
|
+
dst_row.src[1] = &src1_row;
|
2844
|
+
|
2845
|
+
ggml_cann_mul_mat(ctx, &dst_row);
|
2846
|
+
}
|
2847
|
+
}
|
2848
|
+
return;
|
2849
|
+
}
|
2850
|
+
|
2851
|
+
void ggml_cann_mul_mat_id(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
|
2852
|
+
const enum ggml_type type = dst->src[0]->type;
|
2853
|
+
switch (type) {
|
2854
|
+
case GGML_TYPE_F32:
|
2855
|
+
case GGML_TYPE_F16:
|
2856
|
+
ggml_cann_mul_mat_id_fp(ctx, dst);
|
2857
|
+
break;
|
2858
|
+
case GGML_TYPE_Q4_0:
|
2859
|
+
case GGML_TYPE_Q8_0:
|
2860
|
+
ggml_cann_mul_mat_id_quant(ctx, dst);
|
2861
|
+
break;
|
2862
|
+
default:
|
2863
|
+
GGML_ABORT("Unsupported type for mul_mat_id");
|
2864
|
+
break;
|
2865
|
+
}
|
2866
|
+
}
|
2867
|
+
|
2868
|
+
void ggml_cann_flash_attn_ext(ggml_backend_cann_context& ctx, ggml_tensor* dst){
|
2869
|
+
|
2870
|
+
ggml_tensor* src0 = dst->src[0]; // q, fp32
|
2871
|
+
ggml_tensor* src1 = dst->src[1]; // k, fp16
|
2872
|
+
ggml_tensor* src2 = dst->src[2]; // v, fp16
|
2873
|
+
ggml_tensor* src3 = dst->src[3]; // mask, fp16
|
2874
|
+
|
2875
|
+
float maxBias = 0.0f;
|
2876
|
+
float scaleValue = 1.0f;
|
2877
|
+
float logitSoftcap = 0.0f;
|
2878
|
+
memcpy(&scaleValue, (float*)dst->op_params + 0, sizeof(float));
|
2879
|
+
memcpy(&maxBias, (float*)dst->op_params + 1, sizeof(float));
|
2880
|
+
memcpy(&logitSoftcap, (float*)dst->op_params + 2, sizeof(float));
|
2881
|
+
|
2882
|
+
if(logitSoftcap == 0.0f){
|
2883
|
+
size_t faElemSize = sizeof(uint16_t);
|
2884
|
+
auto faDataType = ACL_FLOAT16; //ACL_BF16;
|
2885
|
+
|
2886
|
+
aclTensor* acl_src0_f16_tensor = nullptr;
|
2887
|
+
aclTensor* acl_src1_f16_tensor = nullptr;
|
2888
|
+
aclTensor* acl_src2_f16_tensor = nullptr;
|
2889
|
+
aclTensor* acl_dst_f16_tensor = nullptr;
|
2890
|
+
|
2891
|
+
// Step 1: cast the src0 (Query) to fp16 if needed
|
2892
|
+
ggml_cann_pool_alloc src0_f16_allocator(ctx.pool());
|
2893
|
+
void* src0_f16_buffer = nullptr;
|
2894
|
+
|
2895
|
+
if(ggml_cann_type_mapping(src0->type) != faDataType){
|
2896
|
+
aclTensor* acl_src0_f32_tensor = ggml_cann_create_tensor(src0);
|
2897
|
+
src0_f16_buffer = src0_f16_allocator.alloc(
|
2898
|
+
ggml_nelements(src0) * faElemSize);
|
2899
|
+
|
2900
|
+
int64_t* src0_f16_ne = src0->ne;
|
2901
|
+
size_t src0_f16_nb[GGML_MAX_DIMS];
|
2902
|
+
src0_f16_nb[0] = sizeof(uint16_t);
|
2903
|
+
for(int i = 1; i < GGML_MAX_DIMS; ++i){
|
2904
|
+
src0_f16_nb[i] = src0_f16_nb[i - 1] * src0_f16_ne[i - 1];
|
2905
|
+
}
|
2906
|
+
|
2907
|
+
acl_src0_f16_tensor = ggml_cann_create_tensor(
|
2908
|
+
src0_f16_buffer, faDataType, faElemSize,
|
2909
|
+
src0_f16_ne, src0_f16_nb, GGML_MAX_DIMS
|
2910
|
+
);
|
2911
|
+
aclnn_cast(ctx, acl_src0_f32_tensor, acl_src0_f16_tensor, faDataType);
|
2912
|
+
ggml_cann_release_resources(ctx, acl_src0_f32_tensor);
|
2913
|
+
}else{
|
2914
|
+
acl_src0_f16_tensor = ggml_cann_create_tensor(src0);
|
2915
|
+
}
|
2916
|
+
|
2917
|
+
// Step 2: create the acl tensors for src1 (Key), src2 (Value),
|
2918
|
+
// and the direct output from FusedInferAttention
|
2919
|
+
|
2920
|
+
acl_src1_f16_tensor = ggml_cann_create_tensor(src1);
|
2921
|
+
acl_src2_f16_tensor = ggml_cann_create_tensor(src2);
|
2922
|
+
|
2923
|
+
ggml_cann_pool_alloc out_f16_allocator(ctx.pool());
|
2924
|
+
void* out_f16_buffer = out_f16_allocator.alloc(
|
2925
|
+
ggml_nelements(dst) * faElemSize);
|
2926
|
+
|
2927
|
+
int64_t* out_f16_ne = src0->ne;
|
2928
|
+
size_t out_f16_nb[GGML_MAX_DIMS];
|
2929
|
+
out_f16_nb[0] = faElemSize;
|
2930
|
+
for(int i = 1; i < GGML_MAX_DIMS; ++i){
|
2931
|
+
out_f16_nb[i] = out_f16_nb[i - 1] * out_f16_ne[i - 1];
|
2932
|
+
}
|
2933
|
+
|
2934
|
+
acl_dst_f16_tensor = ggml_cann_create_tensor(
|
2935
|
+
out_f16_buffer, faDataType, faElemSize,
|
2936
|
+
out_f16_ne, out_f16_nb, GGML_MAX_DIMS
|
2937
|
+
);
|
2938
|
+
|
2939
|
+
// Step 3: create the PSEShift tensor if needed
|
2940
|
+
// this tensor is considered as mask (f16) in the llama.cpp
|
2941
|
+
|
2942
|
+
aclTensor* bcast_pse_tensor = nullptr;
|
2943
|
+
int64_t bcast_pse_ne[GGML_MAX_DIMS];
|
2944
|
+
size_t bcast_pse_nb[GGML_MAX_DIMS];
|
2945
|
+
ggml_cann_pool_alloc bcast_pse_allocator(ctx.pool());
|
2946
|
+
void* bcast_pse_buffer = nullptr;
|
2947
|
+
|
2948
|
+
if(src3 != nullptr){
|
2949
|
+
bcast_pse_buffer = bcast_pse_allocator.alloc(
|
2950
|
+
ggml_nelements(src3) * src0->ne[2] * sizeof(uint16_t));
|
2951
|
+
|
2952
|
+
if(src0->ne[1] > 1){
|
2953
|
+
// Case 1: broadcast pse for prefill stage with multiple head
|
2954
|
+
aclTensor* acl_mask_f16_tensor = ggml_cann_create_tensor(src3);
|
2955
|
+
bcast_pse_ne[0] = src3->ne[0];
|
2956
|
+
bcast_pse_ne[1] = src3->ne[1];
|
2957
|
+
bcast_pse_ne[2] = src0->ne[2];
|
2958
|
+
bcast_pse_ne[3] = src3->ne[3];
|
2959
|
+
|
2960
|
+
bcast_pse_nb[0] = sizeof(uint16_t);
|
2961
|
+
for(int i = 1; i < GGML_MAX_DIMS; ++i){
|
2962
|
+
bcast_pse_nb[i] = bcast_pse_nb[i - 1] * bcast_pse_ne[i - 1];
|
2963
|
+
}
|
2964
|
+
|
2965
|
+
bcast_pse_tensor = ggml_cann_create_tensor(
|
2966
|
+
bcast_pse_buffer, ACL_FLOAT16, sizeof(uint16_t),
|
2967
|
+
bcast_pse_ne, bcast_pse_nb, GGML_MAX_DIMS);
|
2968
|
+
|
2969
|
+
int64_t repeats[] = {1, src0->ne[2], 1, 1};
|
2970
|
+
aclnn_repeat(ctx, acl_mask_f16_tensor, bcast_pse_tensor, repeats);
|
2971
|
+
|
2972
|
+
ggml_cann_release_resources(ctx, acl_mask_f16_tensor);
|
2973
|
+
}else{
|
2974
|
+
// Case 2: trunc the first row and broadcast pse for decode stage with multiple head
|
2975
|
+
int64_t trunc_pse_ne[GGML_MAX_DIMS] = {src3->ne[0], src0->ne[1], src3->ne[2], src3->ne[3]};
|
2976
|
+
size_t* trunc_pse_nb = src3->nb;
|
2977
|
+
|
2978
|
+
aclTensor* acl_mask_f16_trunc_tensor = ggml_cann_create_tensor(
|
2979
|
+
src3->data, ACL_FLOAT16, sizeof(uint16_t),
|
2980
|
+
trunc_pse_ne, trunc_pse_nb, GGML_MAX_DIMS);
|
2981
|
+
|
2982
|
+
bcast_pse_ne[0] = src3->ne[0];
|
2983
|
+
bcast_pse_ne[1] = src0->ne[1];
|
2984
|
+
bcast_pse_ne[2] = src0->ne[2];
|
2985
|
+
bcast_pse_ne[3] = src3->ne[3];
|
2986
|
+
|
2987
|
+
bcast_pse_nb[0] = sizeof(uint16_t);
|
2988
|
+
for(int i = 1; i < GGML_MAX_DIMS; ++i){
|
2989
|
+
bcast_pse_nb[i] = bcast_pse_nb[i - 1] * bcast_pse_ne[i - 1];
|
2990
|
+
}
|
2991
|
+
|
2992
|
+
bcast_pse_tensor = ggml_cann_create_tensor(
|
2993
|
+
bcast_pse_buffer, ACL_FLOAT16, sizeof(uint16_t),
|
2994
|
+
bcast_pse_ne, bcast_pse_nb, GGML_MAX_DIMS);
|
2995
|
+
|
2996
|
+
int64_t repeats[] = {1, src0->ne[2], 1, 1};
|
2997
|
+
aclnn_repeat(ctx, acl_mask_f16_trunc_tensor, bcast_pse_tensor, repeats);
|
2998
|
+
|
2999
|
+
ggml_cann_release_resources(ctx, acl_mask_f16_trunc_tensor);
|
3000
|
+
}
|
3001
|
+
|
3002
|
+
// Compute the slope if needed. Derived from ggml_cann_softmax().
|
3003
|
+
if(maxBias != 0.0f){
|
3004
|
+
// alibi
|
3005
|
+
const int64_t ne2_ne3 = src0->ne[2] * src0->ne[3];
|
3006
|
+
const int64_t n_head = src0->ne[2];
|
3007
|
+
const int n_heads_log2_floor = 1u << (uint32_t)floor(log2(n_head));
|
3008
|
+
float m0 = powf(2.0f, -(maxBias) / n_heads_log2_floor);
|
3009
|
+
float m1 = powf(2.0f, -(maxBias / 2.0f) / n_heads_log2_floor);
|
3010
|
+
// init arange
|
3011
|
+
ggml_cann_pool_alloc arange_allocator(ctx.pool(),
|
3012
|
+
ne2_ne3 * faElemSize);
|
3013
|
+
void* tmp_arange_buffer = arange_allocator.get();
|
3014
|
+
|
3015
|
+
// arange1: [1, ..., n_heads_log2_floor+1)
|
3016
|
+
float start = 1;
|
3017
|
+
float stop = n_heads_log2_floor + 1;
|
3018
|
+
float step = 1;
|
3019
|
+
int64_t n_elements_arange = n_heads_log2_floor;
|
3020
|
+
|
3021
|
+
int64_t tmp_arange1_ne[] = {n_heads_log2_floor};
|
3022
|
+
size_t tmp_arange1_nb[] = {faElemSize};
|
3023
|
+
aclTensor* tmp_arange1_tensor = ggml_cann_create_tensor(
|
3024
|
+
tmp_arange_buffer, faDataType, faElemSize,
|
3025
|
+
tmp_arange1_ne, tmp_arange1_nb,
|
3026
|
+
GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
3027
|
+
|
3028
|
+
aclnn_arange(ctx, tmp_arange1_tensor, start, stop, step, n_elements_arange);
|
3029
|
+
|
3030
|
+
aclTensor* tmp_arange2_tensor = nullptr;
|
3031
|
+
if (n_heads_log2_floor < ne2_ne3) {
|
3032
|
+
// arange2: [1, ..., 2 * (k - n_heads_log2_floor) + 1)
|
3033
|
+
start = 1;
|
3034
|
+
stop = 2 * (ne2_ne3 - n_heads_log2_floor) + 1;
|
3035
|
+
step = 2;
|
3036
|
+
n_elements_arange = ne2_ne3 - n_heads_log2_floor;
|
3037
|
+
int64_t tmp_arange2_ne[] = {ne2_ne3 - n_heads_log2_floor};
|
3038
|
+
size_t tmp_arange2_nb[] = {faElemSize};
|
3039
|
+
|
3040
|
+
aclTensor* tmp_arange2_tensor = ggml_cann_create_tensor(
|
3041
|
+
(char*)tmp_arange_buffer +
|
3042
|
+
n_heads_log2_floor * faElemSize,
|
3043
|
+
faDataType, faElemSize,
|
3044
|
+
tmp_arange2_ne, tmp_arange2_nb, GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
3045
|
+
aclnn_arange(ctx, tmp_arange2_tensor, start, stop, step,
|
3046
|
+
n_elements_arange);
|
3047
|
+
}
|
3048
|
+
|
3049
|
+
// init mk_base
|
3050
|
+
ggml_cann_pool_alloc mk_base_allocator(ctx.pool(),
|
3051
|
+
ne2_ne3 * faElemSize);
|
3052
|
+
void* tmp_mk_base_buffer = mk_base_allocator.get();
|
3053
|
+
int64_t tmp_mk_base1_ne[] = {n_heads_log2_floor};
|
3054
|
+
size_t tmp_mk_base1_nb[] = {faElemSize};
|
3055
|
+
aclTensor* tmp_mk_base1_tensor = ggml_cann_create_tensor(
|
3056
|
+
tmp_mk_base_buffer, faDataType, faElemSize,
|
3057
|
+
tmp_mk_base1_ne, tmp_mk_base1_nb,
|
3058
|
+
GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
3059
|
+
|
3060
|
+
aclnn_fill_scalar(ctx, m0, tmp_mk_base1_tensor);
|
3061
|
+
|
3062
|
+
aclTensor* tmp_mk_base2_tensor = nullptr;
|
3063
|
+
if (n_heads_log2_floor < ne2_ne3) {
|
3064
|
+
int64_t tmp_mk_base2_ne[] = {ne2_ne3 - n_heads_log2_floor};
|
3065
|
+
size_t tmp_mk_base2_nb[] = {faElemSize};
|
3066
|
+
aclTensor* tmp_mk_base2_tensor = ggml_cann_create_tensor(
|
3067
|
+
(char*)tmp_mk_base_buffer +
|
3068
|
+
n_heads_log2_floor * faElemSize,
|
3069
|
+
faDataType, faElemSize,
|
3070
|
+
tmp_mk_base2_ne, tmp_mk_base2_nb, GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
3071
|
+
aclnn_fill_scalar(ctx, m1, tmp_mk_base2_tensor);
|
3072
|
+
}
|
3073
|
+
|
3074
|
+
// init mk
|
3075
|
+
int64_t tmp_mk_base_ne[] = {ne2_ne3};
|
3076
|
+
size_t tmp_mk_base_nb[] = {faElemSize};
|
3077
|
+
aclTensor* tmp_mk_base_tensor = ggml_cann_create_tensor(
|
3078
|
+
tmp_mk_base_buffer, faDataType, faElemSize,
|
3079
|
+
tmp_mk_base_ne, tmp_mk_base_nb,
|
3080
|
+
GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
3081
|
+
aclTensor* tmp_arange_tensor = ggml_cann_create_tensor(
|
3082
|
+
tmp_arange_buffer, faDataType, faElemSize,
|
3083
|
+
tmp_mk_base_ne, tmp_mk_base_nb,
|
3084
|
+
GGML_MAX_DIMS - 3, ACL_FORMAT_ND);
|
3085
|
+
aclnn_pow_tensor_tensor(ctx, tmp_mk_base_tensor, tmp_arange_tensor);
|
3086
|
+
|
3087
|
+
// reshape mk
|
3088
|
+
int64_t tmp_mk_ne[] = {1, 1, src0->ne[2], src0->ne[3]};
|
3089
|
+
size_t tmp_mk_nb[GGML_MAX_DIMS];
|
3090
|
+
tmp_mk_nb[0] = faElemSize;
|
3091
|
+
for (int i = 1; i < GGML_MAX_DIMS; i++) {
|
3092
|
+
tmp_mk_nb[i] = tmp_mk_nb[i - 1] * tmp_mk_ne[i - 1];
|
3093
|
+
}
|
3094
|
+
aclTensor* tmp_mk_tensor = ggml_cann_create_tensor(
|
3095
|
+
tmp_mk_base_buffer, faDataType, faElemSize,
|
3096
|
+
tmp_mk_ne, tmp_mk_nb, GGML_MAX_DIMS,
|
3097
|
+
ACL_FORMAT_ND);
|
3098
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, InplaceMul, bcast_pse_tensor, tmp_mk_tensor);
|
3099
|
+
|
3100
|
+
ggml_cann_release_resources(ctx, tmp_arange1_tensor, tmp_arange2_tensor,
|
3101
|
+
tmp_mk_base1_tensor, tmp_mk_base2_tensor, tmp_mk_base_tensor,
|
3102
|
+
tmp_arange_tensor, tmp_mk_tensor);
|
3103
|
+
}
|
3104
|
+
}
|
3105
|
+
|
3106
|
+
// Step 4: set the inputs for FusedInferAttention.
|
3107
|
+
int kvTensorNum = 1;
|
3108
|
+
aclTensor* acl_q_tensor = acl_src0_f16_tensor;
|
3109
|
+
aclTensor* acl_k_tensors[] = {acl_src1_f16_tensor};
|
3110
|
+
aclTensor* acl_v_tensors[] = {acl_src2_f16_tensor};
|
3111
|
+
auto acl_k_tensor_list = aclCreateTensorList(acl_k_tensors, kvTensorNum);
|
3112
|
+
auto acl_v_tensor_list = aclCreateTensorList(acl_v_tensors, kvTensorNum);
|
3113
|
+
|
3114
|
+
int64_t numHeads = src0->ne[2]; // N
|
3115
|
+
int64_t numKeyValueHeads = src1->ne[2];
|
3116
|
+
// double scaleValue = 1 / sqrt(src0->ne[0]); // 1/sqrt(d)
|
3117
|
+
int64_t preTokens = 65535;
|
3118
|
+
int64_t nextTokens = 65535;
|
3119
|
+
char layout[5] = {'B', 'N', 'S', 'D', 0};
|
3120
|
+
int64_t sparseMode = 0;
|
3121
|
+
int64_t innerPrecise = (src0->ne[1] == 1) ? 0 : 2;
|
3122
|
+
int64_t blockSize = 0;
|
3123
|
+
int64_t antiquantMode = 0;
|
3124
|
+
bool softmaxLseFlag = false;
|
3125
|
+
int64_t keyAntiquantMode = 0;
|
3126
|
+
int64_t valueAntiquantMode = 0;
|
3127
|
+
|
3128
|
+
// Step 5: launch the FusedInferAttentionScoreV2 kernel.
|
3129
|
+
// Refer to https://gitee.com/ascend/cann-ops-adv/blob/master/docs/FusedInferAttentionScoreV2.md
|
3130
|
+
|
3131
|
+
GGML_CANN_CALL_ACLNN_OP(ctx, FusedInferAttentionScoreV2,
|
3132
|
+
acl_q_tensor, acl_k_tensor_list, acl_v_tensor_list, // q, k, v
|
3133
|
+
bcast_pse_tensor, nullptr, // pse, mask
|
3134
|
+
nullptr, nullptr, // actSeqLen, actSeqLenkv
|
3135
|
+
nullptr, nullptr, // deqScale1, quantScale1
|
3136
|
+
nullptr, nullptr, nullptr, // deqScale2, quantScale2, quantOffset2
|
3137
|
+
nullptr, nullptr, // antiquantScale, antiquantOffset
|
3138
|
+
nullptr, // blockTable
|
3139
|
+
nullptr, nullptr, // qPadSize, kvPadSize
|
3140
|
+
nullptr, nullptr, // kAntiquantScale, kAntiQuantOffset
|
3141
|
+
nullptr, nullptr, // vAntiquantScale, vAntiQuantOffset
|
3142
|
+
nullptr, nullptr, nullptr, // kSharedPrefix, vSharedPrefix, actSharedLen
|
3143
|
+
numHeads, scaleValue, // heads, scaleValue
|
3144
|
+
preTokens, nextTokens, // preTokens, nextTokens
|
3145
|
+
layout, // inputLayout
|
3146
|
+
numKeyValueHeads, // numKVHeads
|
3147
|
+
sparseMode, innerPrecise, // sparseMode, innerPrecise
|
3148
|
+
blockSize, antiquantMode, // blockSize, antiquantMode
|
3149
|
+
softmaxLseFlag, // softmaxLseFlag
|
3150
|
+
keyAntiquantMode, valueAntiquantMode, // keyAntiqMode, valueAntiqMode
|
3151
|
+
acl_dst_f16_tensor, // attentionOut
|
3152
|
+
nullptr // softmaxLse
|
3153
|
+
);
|
3154
|
+
|
3155
|
+
// Step 6: post-processing, permute and cast to f32
|
3156
|
+
|
3157
|
+
int64_t new_dim[] = {0, 2, 1, 3};
|
3158
|
+
aclTensor* acl_dst_tensor = ggml_cann_create_tensor(dst);
|
3159
|
+
|
3160
|
+
if(ggml_cann_type_mapping(dst->type) != faDataType){
|
3161
|
+
ggml_cann_pool_alloc perm_out_f16_allocator(ctx.pool());
|
3162
|
+
perm_out_f16_allocator.alloc(ggml_nelements(dst) * faElemSize);
|
3163
|
+
void* perm_out_f16_buffer = perm_out_f16_allocator.get();
|
3164
|
+
|
3165
|
+
int64_t* perm_out_f16_ne = dst->ne;
|
3166
|
+
size_t perm_out_f16_nb[GGML_MAX_DIMS];
|
3167
|
+
perm_out_f16_nb[0] = faElemSize;
|
3168
|
+
for(int i = 1; i < GGML_MAX_DIMS; ++i){
|
3169
|
+
perm_out_f16_nb[i] = perm_out_f16_nb[i - 1] * perm_out_f16_ne[i - 1];
|
3170
|
+
}
|
3171
|
+
aclTensor* acl_perm_out_f16_tensor = ggml_cann_create_tensor(
|
3172
|
+
perm_out_f16_buffer, faDataType, faElemSize,
|
3173
|
+
perm_out_f16_ne, perm_out_f16_nb, GGML_MAX_DIMS);
|
3174
|
+
aclnn_permute(ctx, acl_dst_f16_tensor, acl_perm_out_f16_tensor, new_dim, GGML_MAX_DIMS);
|
3175
|
+
aclnn_cast(ctx,
|
3176
|
+
acl_perm_out_f16_tensor, acl_dst_tensor, ggml_cann_type_mapping(dst->type));
|
3177
|
+
ggml_cann_release_resources(ctx, acl_perm_out_f16_tensor);
|
3178
|
+
}else{
|
3179
|
+
// only need to permute
|
3180
|
+
aclnn_permute(ctx, acl_dst_f16_tensor, acl_dst_tensor, new_dim, GGML_MAX_DIMS);
|
3181
|
+
}
|
3182
|
+
ggml_cann_release_resources(ctx, acl_src0_f16_tensor,
|
3183
|
+
acl_src1_f16_tensor,
|
3184
|
+
acl_src2_f16_tensor,
|
3185
|
+
acl_dst_f16_tensor,
|
3186
|
+
acl_dst_tensor);
|
3187
|
+
if(src3 != nullptr){
|
3188
|
+
ggml_cann_release_resources(ctx, bcast_pse_tensor);
|
3189
|
+
}
|
3190
|
+
}else{
|
3191
|
+
GGML_ABORT("Function is not implemented.");
|
3192
|
+
}
|
3193
|
+
}
|