whispercpp 1.3.1 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +4 -3
- data/README.md +92 -31
- data/Rakefile +26 -7
- data/ext/.gitignore +5 -7
- data/ext/dependencies.rb +61 -0
- data/ext/extconf.rb +21 -198
- data/ext/options.rb +221 -0
- data/ext/ruby_whisper.c +159 -0
- data/ext/ruby_whisper.h +17 -2
- data/ext/ruby_whisper_context.c +641 -0
- data/ext/ruby_whisper_error.c +52 -0
- data/ext/ruby_whisper_model.c +232 -0
- data/ext/ruby_whisper_params.c +1301 -0
- data/ext/ruby_whisper_segment.c +143 -0
- data/ext/ruby_whisper_transcribe.cpp +87 -0
- data/ext/ruby_whisper_vad_params.c +288 -0
- data/ext/sources/.dockerignore +3 -0
- data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
- data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
- data/ext/sources/CMakeLists.txt +251 -0
- data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
- data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
- data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
- data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
- data/ext/sources/bindings/javascript/package.json +26 -0
- data/ext/sources/bindings/javascript/whisper.js +19 -0
- data/ext/sources/build-xcframework.sh +547 -0
- data/ext/sources/ci/run.sh +336 -0
- data/ext/sources/close-issue.yml +28 -0
- data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
- data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
- data/ext/sources/cmake/build-info.cmake +60 -0
- data/ext/sources/cmake/git-vars.cmake +22 -0
- data/ext/sources/cmake/whisper-config.cmake.in +65 -0
- data/ext/sources/cmake/whisper.pc.in +10 -0
- data/ext/sources/examples/CMakeLists.txt +124 -0
- data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
- data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
- data/ext/sources/examples/addon.node/addon.cpp +438 -0
- data/ext/sources/examples/addon.node/index.js +54 -0
- data/ext/sources/examples/addon.node/package.json +16 -0
- data/ext/sources/examples/bench/CMakeLists.txt +8 -0
- data/ext/sources/examples/bench/bench.cpp +175 -0
- data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
- data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
- data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
- data/ext/sources/examples/cli/CMakeLists.txt +8 -0
- data/ext/sources/examples/cli/cli.cpp +1294 -0
- data/ext/sources/examples/coi-serviceworker.js +146 -0
- data/ext/sources/examples/command/CMakeLists.txt +10 -0
- data/ext/sources/examples/command/command.cpp +776 -0
- data/ext/sources/examples/command/commands.txt +9 -0
- data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
- data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
- data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
- data/ext/sources/examples/common-ggml.cpp +238 -0
- data/ext/sources/examples/common-ggml.h +18 -0
- data/ext/sources/examples/common-sdl.cpp +227 -0
- data/ext/sources/examples/common-sdl.h +49 -0
- data/ext/sources/examples/common-whisper.cpp +168 -0
- data/ext/sources/examples/common-whisper.h +24 -0
- data/ext/sources/examples/common.cpp +675 -0
- data/ext/sources/examples/common.h +322 -0
- data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
- data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
- data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
- data/ext/sources/examples/generate-karaoke.sh +57 -0
- data/ext/sources/examples/grammar-parser.cpp +423 -0
- data/ext/sources/examples/grammar-parser.h +29 -0
- data/ext/sources/examples/helpers.js +191 -0
- data/ext/sources/examples/json.hpp +24596 -0
- data/ext/sources/examples/livestream.sh +112 -0
- data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
- data/ext/sources/examples/lsp/lsp.cpp +467 -0
- data/ext/sources/examples/lsp/whisper.vim +362 -0
- data/ext/sources/examples/miniaudio.h +93468 -0
- data/ext/sources/examples/python/test_whisper_processor.py +7 -0
- data/ext/sources/examples/python/whisper_processor.py +54 -0
- data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
- data/ext/sources/examples/quantize/quantize.cpp +223 -0
- data/ext/sources/examples/server/CMakeLists.txt +12 -0
- data/ext/sources/examples/server/bench.js +29 -0
- data/ext/sources/examples/server/httplib.h +10497 -0
- data/ext/sources/examples/server/server.cpp +1091 -0
- data/ext/sources/examples/server.py +115 -0
- data/ext/sources/examples/stb_vorbis.c +5584 -0
- data/ext/sources/examples/stream/CMakeLists.txt +10 -0
- data/ext/sources/examples/stream/stream.cpp +429 -0
- data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
- data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
- data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
- data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
- data/ext/sources/examples/sycl/build.sh +22 -0
- data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
- data/ext/sources/examples/sycl/run-whisper.sh +17 -0
- data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
- data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
- data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
- data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
- data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
- data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
- data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
- data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
- data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
- data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
- data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
- data/ext/sources/examples/talk-llama/llama-context.h +276 -0
- data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
- data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
- data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
- data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
- data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
- data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
- data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
- data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
- data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
- data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
- data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
- data/ext/sources/examples/talk-llama/llama-io.h +35 -0
- data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
- data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
- data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
- data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
- data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
- data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
- data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
- data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
- data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
- data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
- data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
- data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
- data/ext/sources/examples/talk-llama/llama-model.h +425 -0
- data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
- data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
- data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
- data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
- data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
- data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
- data/ext/sources/examples/talk-llama/llama.cpp +354 -0
- data/ext/sources/examples/talk-llama/llama.h +1377 -0
- data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
- data/ext/sources/examples/talk-llama/speak +40 -0
- data/ext/sources/examples/talk-llama/speak.bat +1 -0
- data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
- data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
- data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
- data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
- data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
- data/ext/sources/examples/talk-llama/unicode.h +66 -0
- data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
- data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
- data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
- data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
- data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
- data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
- data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
- data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
- data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
- data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
- data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
- data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
- data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
- data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
- data/ext/sources/ggml/CMakeLists.txt +390 -0
- data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
- data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
- data/ext/sources/ggml/cmake/common.cmake +26 -0
- data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
- data/ext/{ggml → sources/ggml}/include/ggml-alloc.h +1 -1
- data/ext/{ggml → sources/ggml}/include/ggml-backend.h +9 -7
- data/ext/{ggml → sources/ggml}/include/ggml-cpp.h +2 -1
- data/ext/{ggml → sources/ggml}/include/ggml-cpu.h +9 -1
- data/ext/{ggml → sources/ggml}/include/ggml-metal.h +1 -1
- data/ext/{ggml → sources/ggml}/include/ggml-opt.h +49 -28
- data/ext/{ggml → sources/ggml}/include/ggml-rpc.h +6 -1
- data/ext/{ggml → sources/ggml}/include/ggml-vulkan.h +0 -2
- data/ext/{ggml → sources/ggml}/include/ggml.h +182 -265
- data/ext/sources/ggml/include/gguf.h +202 -0
- data/ext/sources/ggml/src/CMakeLists.txt +346 -0
- data/ext/{ggml → sources/ggml}/src/ggml-alloc.c +34 -29
- data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
- data/ext/{ggml → sources/ggml}/src/ggml-backend-impl.h +1 -2
- data/ext/{ggml → sources/ggml}/src/ggml-backend-reg.cpp +87 -53
- data/ext/{ggml → sources/ggml}/src/ggml-backend.cpp +26 -14
- data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
- data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
- data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.cpp +10 -4
- data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.h +5 -5
- data/ext/{ggml → sources/ggml}/src/ggml-cann/aclnn_ops.cpp +1272 -1506
- data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/common.h +135 -1
- data/ext/{ggml → sources/ggml}/src/ggml-cann/ggml-cann.cpp +564 -146
- data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/dup.cpp +3 -5
- data/ext/{ggml → sources/ggml}/src/ggml-common.h +12 -8
- data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.cpp +2 -1
- data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
- data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/cpu-feats-x86.cpp +5 -1
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-impl.h +163 -41
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.c +4029 -1117
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu.cpp +67 -18
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
- data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
- data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
- data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
- data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
- data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
- data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
- data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
- data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
- data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
- data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
- data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
- data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
- data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
- data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
- data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
- data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
- data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
- data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
- data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
- data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
- data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
- data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
- data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
- data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
- data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
- data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
- data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
- data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
- data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
- data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
- data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
- data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
- data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
- data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
- data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
- data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
- data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
- data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
- data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
- data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
- data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
- data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
- data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
- data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
- data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
- data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
- data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
- data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
- data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/cuda.h +1 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/hip.h +57 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/musa.h +7 -1
- data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
- data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
- data/ext/{ggml → sources/ggml}/src/ggml-impl.h +64 -19
- data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
- data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
- data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
- data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.m +2178 -1064
- data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.metal +1575 -1218
- data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
- data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
- data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
- data/ext/{ggml → sources/ggml}/src/ggml-opt.cpp +373 -190
- data/ext/{ggml → sources/ggml}/src/ggml-quants.c +114 -120
- data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
- data/ext/{ggml → sources/ggml}/src/ggml-rpc/ggml-rpc.cpp +480 -73
- data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
- data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
- data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
- data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/common.cpp +20 -32
- data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/concat.cpp +32 -33
- data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/conv.cpp +4 -2
- data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/convert.cpp +104 -28
- data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
- data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
- data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
- data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/dmmv.cpp +156 -17
- data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
- data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
- data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
- data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
- data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
- data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/ggml-sycl.cpp +1004 -1240
- data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
- data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmq.cpp +0 -1
- data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmvq.cpp +261 -166
- data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/norm.cpp +204 -81
- data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/outprod.cpp +8 -17
- data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
- data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
- data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
- data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/softmax.cpp +35 -25
- data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
- data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/tsembd.cpp +3 -3
- data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
- data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
- data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/ext/{ggml → sources/ggml}/src/ggml-vulkan/ggml-vulkan.cpp +3130 -1087
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
- data/ext/{ggml → sources/ggml}/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +193 -35
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/ext/{ggml → sources/ggml}/src/ggml.c +676 -1820
- data/ext/sources/ggml/src/gguf.cpp +1330 -0
- data/ext/{include → sources/include}/whisper.h +68 -2
- data/ext/sources/src/CMakeLists.txt +143 -0
- data/ext/{src → sources/src}/coreml/whisper-decoder-impl.h +27 -15
- data/ext/{src → sources/src}/coreml/whisper-decoder-impl.m +35 -10
- data/ext/{src → sources/src}/coreml/whisper-encoder-impl.h +21 -9
- data/ext/{src → sources/src}/coreml/whisper-encoder-impl.m +28 -3
- data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
- data/ext/sources/src/whisper-arch.h +197 -0
- data/ext/{src → sources/src}/whisper.cpp +1905 -374
- data/ext/sources/tests/CMakeLists.txt +105 -0
- data/ext/sources/tests/earnings21/eval.mk +58 -0
- data/ext/sources/tests/earnings21/eval.py +68 -0
- data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
- data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
- data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
- data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
- data/ext/sources/tests/earnings21/requirements.txt +6 -0
- data/ext/sources/tests/en-0-ref.txt +1 -0
- data/ext/sources/tests/en-1-ref.txt +1 -0
- data/ext/sources/tests/en-2-ref.txt +1 -0
- data/ext/sources/tests/es-0-ref.txt +1 -0
- data/ext/sources/tests/librispeech/eval.mk +39 -0
- data/ext/sources/tests/librispeech/eval.py +47 -0
- data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
- data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
- data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
- data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
- data/ext/sources/tests/librispeech/requirements.txt +6 -0
- data/ext/sources/tests/run-tests.sh +130 -0
- data/ext/sources/tests/test-c.c +3 -0
- data/ext/sources/tests/test-vad-full.cpp +54 -0
- data/ext/sources/tests/test-vad.cpp +83 -0
- data/ext/sources/tests/test-whisper.js +58 -0
- data/extsources.rb +33 -5
- data/lib/whisper/model/uri.rb +149 -128
- data/sig/whisper.rbs +480 -0
- data/tests/helper.rb +28 -0
- data/tests/test_callback.rb +45 -3
- data/tests/test_error.rb +2 -2
- data/tests/test_model.rb +38 -0
- data/tests/test_package.rb +18 -3
- data/tests/test_params.rb +145 -8
- data/tests/test_segment.rb +10 -19
- data/tests/test_vad.rb +19 -0
- data/tests/test_vad_params.rb +103 -0
- data/tests/test_whisper.rb +37 -37
- data/whispercpp.gemspec +5 -4
- metadata +766 -111
- data/ext/cpu.mk +0 -9
- data/ext/examples/dr_wav.h +0 -8815
- data/ext/ggml/src/ggml-cann/aclnn_ops.h +0 -592
- data/ext/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +0 -4262
- data/ext/ggml/src/ggml-cpu/ggml-cpu.c +0 -14123
- data/ext/ggml/src/ggml-cpu/llamafile/sgemm.cpp +0 -1884
- data/ext/ggml/src/ggml-cpu/llamafile/sgemm.h +0 -14
- data/ext/ggml/src/ggml-metal/ggml-metal-impl.h +0 -288
- data/ext/ggml/src/ggml-sycl/element_wise.cpp +0 -1030
- data/ext/ggml/src/ggml-sycl/im2col.cpp +0 -126
- data/ext/ggml/src/ggml-sycl/rope.cpp +0 -276
- data/ext/ggml/src/ggml-sycl/wkv6.cpp +0 -141
- data/ext/metal-embed.mk +0 -17
- data/ext/metal.mk +0 -6
- data/ext/ruby_whisper.cpp +0 -1909
- data/ext/scripts/get-flags.mk +0 -38
- data/lib/whisper.rb +0 -2
- /data/ext/{ggml → sources/ggml}/include/ggml-blas.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-cann.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-cuda.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-kompute.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-opencl.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-sycl.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/common.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/ggml-amx.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-blas/ggml-blas.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/ascendc_kernels.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f16.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f32.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q4_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/common.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-aarch64.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-kompute/ggml-kompute.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-quants.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-threading.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-threading.h +0 -0
- /data/ext/{src → sources/src}/coreml/whisper-encoder.h +0 -0
- /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.cpp +0 -0
- /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.h +0 -0
@@ -0,0 +1,2676 @@
|
|
1
|
+
#include "llama-context.h"
|
2
|
+
|
3
|
+
#include "llama-impl.h"
|
4
|
+
#include "llama-io.h"
|
5
|
+
#include "llama-mmap.h"
|
6
|
+
#include "llama-model.h"
|
7
|
+
#include "llama-kv-cache.h"
|
8
|
+
|
9
|
+
#include <cstring>
|
10
|
+
#include <stdexcept>
|
11
|
+
#include <cinttypes>
|
12
|
+
|
13
|
+
//
|
14
|
+
// llama_context
|
15
|
+
//
|
16
|
+
|
17
|
+
llama_context::llama_context(
|
18
|
+
const llama_model & model,
|
19
|
+
llama_context_params params) :
|
20
|
+
model(model) {
|
21
|
+
LLAMA_LOG_INFO("%s: constructing llama_context\n", __func__);
|
22
|
+
|
23
|
+
t_start_us = model.t_start_us;
|
24
|
+
t_load_us = model.t_load_us;
|
25
|
+
|
26
|
+
const auto & hparams = model.hparams;
|
27
|
+
|
28
|
+
cparams.n_seq_max = std::max(1u, params.n_seq_max);
|
29
|
+
if (cparams.n_seq_max > LLAMA_MAX_PARALLEL_SEQUENCES) {
|
30
|
+
throw std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_PARALLEL_SEQUENCES));
|
31
|
+
}
|
32
|
+
|
33
|
+
cparams.n_threads = params.n_threads;
|
34
|
+
cparams.n_threads_batch = params.n_threads_batch;
|
35
|
+
cparams.yarn_ext_factor = params.yarn_ext_factor;
|
36
|
+
cparams.yarn_attn_factor = params.yarn_attn_factor;
|
37
|
+
cparams.yarn_beta_fast = params.yarn_beta_fast;
|
38
|
+
cparams.yarn_beta_slow = params.yarn_beta_slow;
|
39
|
+
cparams.defrag_thold = params.defrag_thold;
|
40
|
+
cparams.embeddings = params.embeddings;
|
41
|
+
cparams.offload_kqv = params.offload_kqv;
|
42
|
+
cparams.flash_attn = params.flash_attn;
|
43
|
+
cparams.no_perf = params.no_perf;
|
44
|
+
cparams.pooling_type = params.pooling_type;
|
45
|
+
cparams.warmup = false;
|
46
|
+
|
47
|
+
cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;
|
48
|
+
cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;
|
49
|
+
cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;
|
50
|
+
|
51
|
+
cparams.n_ctx_orig_yarn = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx :
|
52
|
+
hparams.n_ctx_orig_yarn != 0 ? hparams.n_ctx_orig_yarn :
|
53
|
+
hparams.n_ctx_train;
|
54
|
+
|
55
|
+
cparams.cb_eval = params.cb_eval;
|
56
|
+
cparams.cb_eval_user_data = params.cb_eval_user_data;
|
57
|
+
|
58
|
+
auto rope_scaling_type = params.rope_scaling_type;
|
59
|
+
if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {
|
60
|
+
rope_scaling_type = hparams.rope_scaling_type_train;
|
61
|
+
}
|
62
|
+
|
63
|
+
if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {
|
64
|
+
cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none
|
65
|
+
}
|
66
|
+
|
67
|
+
if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'
|
68
|
+
cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;
|
69
|
+
}
|
70
|
+
|
71
|
+
cparams.yarn_attn_factor *= hparams.rope_attn_factor;
|
72
|
+
|
73
|
+
if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {
|
74
|
+
if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {
|
75
|
+
cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;
|
76
|
+
} else {
|
77
|
+
cparams.pooling_type = hparams.pooling_type;
|
78
|
+
}
|
79
|
+
}
|
80
|
+
|
81
|
+
if (params.attention_type == LLAMA_ATTENTION_TYPE_UNSPECIFIED) {
|
82
|
+
cparams.causal_attn = hparams.causal_attn;
|
83
|
+
} else {
|
84
|
+
cparams.causal_attn = params.attention_type == LLAMA_ATTENTION_TYPE_CAUSAL;
|
85
|
+
}
|
86
|
+
|
87
|
+
// with causal attention, the batch size is limited by the context size
|
88
|
+
cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;
|
89
|
+
|
90
|
+
// the batch has to be at least GGML_KQ_MASK_PAD because we will be padding the KQ_mask
|
91
|
+
// this is required by GPU kernels in order to avoid out-of-bounds accesses (e.g. ggml_flash_attn_ext)
|
92
|
+
// ref: https://github.com/ggerganov/llama.cpp/pull/5021
|
93
|
+
// TODO: this padding is not needed for the cache-less context so we should probably move it to llama_context_kv_self
|
94
|
+
if (cparams.n_batch < GGML_KQ_MASK_PAD) {
|
95
|
+
LLAMA_LOG_WARN("%s: n_batch is less than GGML_KQ_MASK_PAD - increasing to %d\n", __func__, GGML_KQ_MASK_PAD);
|
96
|
+
cparams.n_batch = GGML_KQ_MASK_PAD;
|
97
|
+
}
|
98
|
+
|
99
|
+
cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);
|
100
|
+
|
101
|
+
cparams.op_offload = params.op_offload;
|
102
|
+
|
103
|
+
const uint32_t n_ctx_per_seq = cparams.n_ctx / cparams.n_seq_max;
|
104
|
+
|
105
|
+
LLAMA_LOG_INFO("%s: n_seq_max = %u\n", __func__, cparams.n_seq_max);
|
106
|
+
LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx);
|
107
|
+
LLAMA_LOG_INFO("%s: n_ctx_per_seq = %u\n", __func__, n_ctx_per_seq);
|
108
|
+
LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch);
|
109
|
+
LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch);
|
110
|
+
LLAMA_LOG_INFO("%s: causal_attn = %d\n", __func__, cparams.causal_attn);
|
111
|
+
LLAMA_LOG_INFO("%s: flash_attn = %d\n", __func__, cparams.flash_attn);
|
112
|
+
LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);
|
113
|
+
LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);
|
114
|
+
|
115
|
+
if (n_ctx_per_seq < hparams.n_ctx_train) {
|
116
|
+
LLAMA_LOG_WARN("%s: n_ctx_per_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n",
|
117
|
+
__func__, n_ctx_per_seq, hparams.n_ctx_train);
|
118
|
+
}
|
119
|
+
|
120
|
+
if (n_ctx_per_seq > hparams.n_ctx_train) {
|
121
|
+
LLAMA_LOG_WARN("%s: n_ctx_per_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n",
|
122
|
+
__func__, n_ctx_per_seq, hparams.n_ctx_train);
|
123
|
+
}
|
124
|
+
|
125
|
+
if (!hparams.vocab_only) {
|
126
|
+
// GPU backends
|
127
|
+
for (auto * dev : model.devices) {
|
128
|
+
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
|
129
|
+
if (backend == nullptr) {
|
130
|
+
throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev)));
|
131
|
+
}
|
132
|
+
backends.emplace_back(backend);
|
133
|
+
}
|
134
|
+
|
135
|
+
// add ACCEL backends (such as BLAS)
|
136
|
+
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
|
137
|
+
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
|
138
|
+
if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {
|
139
|
+
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
|
140
|
+
if (backend == nullptr) {
|
141
|
+
throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev)));
|
142
|
+
}
|
143
|
+
backends.emplace_back(backend);
|
144
|
+
}
|
145
|
+
}
|
146
|
+
|
147
|
+
// add CPU backend
|
148
|
+
backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
|
149
|
+
if (backend_cpu == nullptr) {
|
150
|
+
throw std::runtime_error("failed to initialize CPU backend");
|
151
|
+
}
|
152
|
+
backends.emplace_back(backend_cpu);
|
153
|
+
|
154
|
+
// create a list of the set_n_threads functions in the backends
|
155
|
+
for (auto & backend : backends) {
|
156
|
+
ggml_backend_dev_t dev = ggml_backend_get_device(backend.get());
|
157
|
+
ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
|
158
|
+
if (reg) {
|
159
|
+
auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
|
160
|
+
if (ggml_backend_set_n_threads_fn) {
|
161
|
+
set_n_threads_fns.emplace_back(backend.get(), ggml_backend_set_n_threads_fn);
|
162
|
+
}
|
163
|
+
}
|
164
|
+
}
|
165
|
+
|
166
|
+
llama_set_abort_callback(this, params.abort_callback, params.abort_callback_data);
|
167
|
+
|
168
|
+
// graph outputs buffer
|
169
|
+
{
|
170
|
+
// resized during inference when a batch uses more outputs
|
171
|
+
if ((uint32_t) output_reserve(params.n_seq_max) < params.n_seq_max) {
|
172
|
+
throw std::runtime_error("failed to reserve initial output buffer");
|
173
|
+
}
|
174
|
+
|
175
|
+
LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__,
|
176
|
+
ggml_backend_buffer_name (buf_output.get()),
|
177
|
+
ggml_backend_buffer_get_size(buf_output.get()) / 1024.0 / 1024.0);
|
178
|
+
}
|
179
|
+
}
|
180
|
+
|
181
|
+
// init the memory module
|
182
|
+
if (!hparams.vocab_only) {
|
183
|
+
llama_memory_params params_mem = {
|
184
|
+
/*.type_k =*/ params.type_k,
|
185
|
+
/*.type_v =*/ params.type_v,
|
186
|
+
/*.swa_full =*/ params.swa_full,
|
187
|
+
};
|
188
|
+
|
189
|
+
memory.reset(model.create_memory(params_mem, cparams));
|
190
|
+
}
|
191
|
+
|
192
|
+
// init backends
|
193
|
+
if (!hparams.vocab_only) {
|
194
|
+
LLAMA_LOG_DEBUG("%s: enumerating backends\n", __func__);
|
195
|
+
|
196
|
+
backend_buft.clear();
|
197
|
+
backend_ptrs.clear();
|
198
|
+
|
199
|
+
for (auto & backend : backends) {
|
200
|
+
auto * buft = ggml_backend_get_default_buffer_type(backend.get());
|
201
|
+
auto backend_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));
|
202
|
+
|
203
|
+
if (backend_type == GGML_BACKEND_DEVICE_TYPE_CPU && !model.devices.empty()) {
|
204
|
+
// use the host buffer of the first device CPU for faster transfer of the intermediate state
|
205
|
+
auto * dev = model.devices[0];
|
206
|
+
auto * host_buft = ggml_backend_dev_host_buffer_type(dev);
|
207
|
+
if (host_buft) {
|
208
|
+
buft = host_buft;
|
209
|
+
}
|
210
|
+
}
|
211
|
+
|
212
|
+
backend_buft.push_back(buft);
|
213
|
+
backend_ptrs.push_back(backend.get());
|
214
|
+
}
|
215
|
+
|
216
|
+
LLAMA_LOG_DEBUG("%s: backend_ptrs.size() = %zu\n", __func__, backend_ptrs.size());
|
217
|
+
|
218
|
+
const size_t max_nodes = this->graph_max_nodes();
|
219
|
+
|
220
|
+
LLAMA_LOG_DEBUG("%s: max_nodes = %zu\n", __func__, max_nodes);
|
221
|
+
|
222
|
+
// buffer used to store the computation graph and the tensor meta data
|
223
|
+
buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false));
|
224
|
+
|
225
|
+
// TODO: move these checks to ggml_backend_sched
|
226
|
+
// enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary
|
227
|
+
bool pipeline_parallel =
|
228
|
+
model.n_devices() > 1 &&
|
229
|
+
model.params.n_gpu_layers > (int) model.hparams.n_layer &&
|
230
|
+
model.params.split_mode == LLAMA_SPLIT_MODE_LAYER &&
|
231
|
+
cparams.offload_kqv &&
|
232
|
+
!model.has_tensor_overrides();
|
233
|
+
|
234
|
+
// pipeline parallelism requires support for async compute and events in all devices
|
235
|
+
if (pipeline_parallel) {
|
236
|
+
for (auto & backend : backends) {
|
237
|
+
auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));
|
238
|
+
if (dev_type == GGML_BACKEND_DEVICE_TYPE_CPU) {
|
239
|
+
// ignore CPU backend
|
240
|
+
continue;
|
241
|
+
}
|
242
|
+
auto * dev = ggml_backend_get_device(backend.get());
|
243
|
+
ggml_backend_dev_props props;
|
244
|
+
ggml_backend_dev_get_props(dev, &props);
|
245
|
+
if (!props.caps.async || !props.caps.events) {
|
246
|
+
// device does not support async compute or events
|
247
|
+
pipeline_parallel = false;
|
248
|
+
break;
|
249
|
+
}
|
250
|
+
}
|
251
|
+
}
|
252
|
+
|
253
|
+
sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, pipeline_parallel, cparams.op_offload));
|
254
|
+
|
255
|
+
if (pipeline_parallel) {
|
256
|
+
LLAMA_LOG_INFO("%s: pipeline parallelism enabled (n_copies=%d)\n", __func__, ggml_backend_sched_get_n_copies(sched.get()));
|
257
|
+
}
|
258
|
+
}
|
259
|
+
|
260
|
+
// reserve worst-case graph
|
261
|
+
if (!hparams.vocab_only && memory) {
|
262
|
+
const uint32_t n_seqs = 1; // TODO: worst-case number of sequences
|
263
|
+
const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);
|
264
|
+
|
265
|
+
llama_token token = model.vocab.token_bos(); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph
|
266
|
+
|
267
|
+
// restore later
|
268
|
+
// TODO: something cleaner
|
269
|
+
const auto n_outputs_save = n_outputs;
|
270
|
+
|
271
|
+
LLAMA_LOG_DEBUG("%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n", __func__, n_tokens, n_seqs, n_outputs);
|
272
|
+
|
273
|
+
int n_splits_pp = -1;
|
274
|
+
int n_nodes_pp = -1;
|
275
|
+
|
276
|
+
int n_splits_tg = -1;
|
277
|
+
int n_nodes_tg = -1;
|
278
|
+
|
279
|
+
// simulate full KV cache
|
280
|
+
llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
|
281
|
+
|
282
|
+
kv_self->set_full();
|
283
|
+
|
284
|
+
cross.v_embd.clear();
|
285
|
+
|
286
|
+
// reserve pp graph first so that buffers are only allocated once
|
287
|
+
{
|
288
|
+
llama_ubatch ubatch_pp = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr};
|
289
|
+
|
290
|
+
// max number of outputs
|
291
|
+
n_outputs = ubatch_pp.n_tokens;
|
292
|
+
|
293
|
+
LLAMA_LOG_DEBUG("%s: reserving graph for n_tokens = %d, n_seqs = %d\n", __func__, ubatch_pp.n_tokens, ubatch_pp.n_seqs);
|
294
|
+
|
295
|
+
auto * gf = graph_init();
|
296
|
+
graph_build(ctx_compute.get(), gf, ubatch_pp, LLM_GRAPH_TYPE_DEFAULT);
|
297
|
+
|
298
|
+
if (!ggml_backend_sched_reserve(sched.get(), gf)) {
|
299
|
+
throw std::runtime_error("failed to allocate compute pp buffers");
|
300
|
+
}
|
301
|
+
|
302
|
+
n_splits_pp = ggml_backend_sched_get_n_splits(sched.get());
|
303
|
+
n_nodes_pp = ggml_graph_n_nodes(gf);
|
304
|
+
}
|
305
|
+
|
306
|
+
// reserve with tg graph to get the number of splits and nodes
|
307
|
+
{
|
308
|
+
llama_ubatch ubatch_tg = { true, 1, 1, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr};
|
309
|
+
|
310
|
+
n_outputs = ubatch_tg.n_tokens;
|
311
|
+
|
312
|
+
LLAMA_LOG_DEBUG("%s: reserving graph for n_tokens = %d, n_seqs = %d\n", __func__, ubatch_tg.n_tokens, ubatch_tg.n_seqs);
|
313
|
+
|
314
|
+
auto * gf = graph_init();
|
315
|
+
graph_build(ctx_compute.get(), gf, ubatch_tg, LLM_GRAPH_TYPE_DEFAULT);
|
316
|
+
|
317
|
+
if (!ggml_backend_sched_reserve(sched.get(), gf)) {
|
318
|
+
throw std::runtime_error("failed to allocate compute tg buffers");
|
319
|
+
}
|
320
|
+
|
321
|
+
n_splits_tg = ggml_backend_sched_get_n_splits(sched.get());
|
322
|
+
n_nodes_tg = ggml_graph_n_nodes(gf);
|
323
|
+
}
|
324
|
+
|
325
|
+
// reserve again with pp graph to avoid ggml-alloc reallocations during inference
|
326
|
+
{
|
327
|
+
llama_ubatch ubatch_pp = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr};
|
328
|
+
|
329
|
+
n_outputs = ubatch_pp.n_tokens;
|
330
|
+
|
331
|
+
LLAMA_LOG_DEBUG("%s: reserving graph for n_tokens = %d, n_seqs = %d\n", __func__, ubatch_pp.n_tokens, ubatch_pp.n_seqs);
|
332
|
+
|
333
|
+
auto * gf = graph_init();
|
334
|
+
graph_build(ctx_compute.get(), gf, ubatch_pp, LLM_GRAPH_TYPE_DEFAULT);
|
335
|
+
|
336
|
+
if (!ggml_backend_sched_reserve(sched.get(), gf)) {
|
337
|
+
throw std::runtime_error("failed to allocate compute pp buffers");
|
338
|
+
}
|
339
|
+
}
|
340
|
+
|
341
|
+
n_outputs = n_outputs_save;
|
342
|
+
|
343
|
+
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
|
344
|
+
ggml_backend_t backend = backend_ptrs[i];
|
345
|
+
ggml_backend_buffer_type_t buft = backend_buft[i];
|
346
|
+
size_t size = ggml_backend_sched_get_buffer_size(sched.get(), backend);
|
347
|
+
if (size > 1) {
|
348
|
+
LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__,
|
349
|
+
ggml_backend_buft_name(buft),
|
350
|
+
size / 1024.0 / 1024.0);
|
351
|
+
}
|
352
|
+
}
|
353
|
+
|
354
|
+
if (n_nodes_pp == n_nodes_tg) {
|
355
|
+
LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, n_nodes_pp);
|
356
|
+
} else {
|
357
|
+
LLAMA_LOG_INFO("%s: graph nodes = %d (with bs=%d), %d (with bs=1)\n", __func__, n_nodes_pp, n_tokens, n_nodes_tg);
|
358
|
+
}
|
359
|
+
|
360
|
+
if (n_splits_pp == n_splits_tg) {
|
361
|
+
LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits_pp);
|
362
|
+
} else {
|
363
|
+
LLAMA_LOG_INFO("%s: graph splits = %d (with bs=%d), %d (with bs=1)\n", __func__, n_splits_pp, n_tokens, n_splits_tg);
|
364
|
+
}
|
365
|
+
}
|
366
|
+
}
|
367
|
+
|
368
|
+
llama_context::~llama_context() {
|
369
|
+
ggml_opt_free(opt_ctx);
|
370
|
+
}
|
371
|
+
|
372
|
+
void llama_context::synchronize() {
|
373
|
+
ggml_backend_sched_synchronize(sched.get());
|
374
|
+
|
375
|
+
// FIXME: if multiple single tokens are evaluated without a synchronization,
|
376
|
+
// the stats will be added to the prompt evaluation stats
|
377
|
+
// this should only happen when using batch size 1 to evaluate a batch
|
378
|
+
|
379
|
+
// add the evaluation to the stats
|
380
|
+
if (n_queued_tokens == 1) {
|
381
|
+
if (!cparams.no_perf) {
|
382
|
+
t_eval_us += ggml_time_us() - t_compute_start_us;
|
383
|
+
}
|
384
|
+
n_eval++;
|
385
|
+
} else if (n_queued_tokens > 1) {
|
386
|
+
if (!cparams.no_perf) {
|
387
|
+
t_p_eval_us += ggml_time_us() - t_compute_start_us;
|
388
|
+
}
|
389
|
+
n_p_eval += n_queued_tokens;
|
390
|
+
}
|
391
|
+
|
392
|
+
// get a more accurate load time, upon first eval
|
393
|
+
if (n_queued_tokens > 0 && !has_evaluated_once) {
|
394
|
+
t_load_us = ggml_time_us() - t_start_us;
|
395
|
+
has_evaluated_once = true;
|
396
|
+
}
|
397
|
+
|
398
|
+
n_queued_tokens = 0;
|
399
|
+
t_compute_start_us = 0;
|
400
|
+
}
|
401
|
+
|
402
|
+
const llama_model & llama_context::get_model() const {
|
403
|
+
return model;
|
404
|
+
}
|
405
|
+
|
406
|
+
const llama_cparams & llama_context::get_cparams() const {
|
407
|
+
return cparams;
|
408
|
+
}
|
409
|
+
|
410
|
+
ggml_backend_sched_t llama_context::get_sched() const {
|
411
|
+
return sched.get();
|
412
|
+
}
|
413
|
+
|
414
|
+
ggml_context * llama_context::get_ctx_compute() const {
|
415
|
+
return ctx_compute.get();
|
416
|
+
}
|
417
|
+
|
418
|
+
uint32_t llama_context::n_ctx() const {
|
419
|
+
return cparams.n_ctx;
|
420
|
+
}
|
421
|
+
|
422
|
+
uint32_t llama_context::n_ctx_per_seq() const {
|
423
|
+
return cparams.n_ctx / cparams.n_seq_max;
|
424
|
+
}
|
425
|
+
|
426
|
+
uint32_t llama_context::n_batch() const {
|
427
|
+
return cparams.n_batch;
|
428
|
+
}
|
429
|
+
|
430
|
+
uint32_t llama_context::n_ubatch() const {
|
431
|
+
return cparams.n_ubatch;
|
432
|
+
}
|
433
|
+
|
434
|
+
uint32_t llama_context::n_seq_max() const {
|
435
|
+
return cparams.n_seq_max;
|
436
|
+
}
|
437
|
+
|
438
|
+
uint32_t llama_context::n_threads() const {
|
439
|
+
return cparams.n_threads;
|
440
|
+
}
|
441
|
+
|
442
|
+
uint32_t llama_context::n_threads_batch() const {
|
443
|
+
return cparams.n_threads_batch;
|
444
|
+
}
|
445
|
+
|
446
|
+
llama_kv_cache * llama_context::get_kv_self() {
|
447
|
+
llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
|
448
|
+
return kv_self;
|
449
|
+
}
|
450
|
+
|
451
|
+
const llama_kv_cache * llama_context::get_kv_self() const {
|
452
|
+
llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
|
453
|
+
return kv_self;
|
454
|
+
}
|
455
|
+
|
456
|
+
void llama_context::kv_self_update() {
|
457
|
+
bool need_reserve = false;
|
458
|
+
|
459
|
+
llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
|
460
|
+
|
461
|
+
need_reserve = kv_self->update(*this);
|
462
|
+
|
463
|
+
// reserve a worst case graph if needed
|
464
|
+
if (need_reserve) {
|
465
|
+
LLAMA_LOG_DEBUG("%s: reserving a worst case graph\n", __func__);
|
466
|
+
|
467
|
+
// build worst-case graph
|
468
|
+
uint32_t n_seqs = 1; // TODO: worst-case number of sequences
|
469
|
+
uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);
|
470
|
+
|
471
|
+
// simulate full KV cache
|
472
|
+
kv_self->set_full();
|
473
|
+
|
474
|
+
llama_token token = model.vocab.token_bos(); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graph
|
475
|
+
llama_ubatch ubatch = { true, n_tokens, n_tokens / n_seqs, n_seqs, &token, nullptr, nullptr, nullptr, nullptr, nullptr};
|
476
|
+
|
477
|
+
auto * gf = graph_init();
|
478
|
+
graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_DEFAULT);
|
479
|
+
|
480
|
+
// initialize scheduler with the worst-case graph
|
481
|
+
ggml_backend_sched_reset(sched.get());
|
482
|
+
if (!ggml_backend_sched_reserve(sched.get(), gf)) {
|
483
|
+
LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__);
|
484
|
+
}
|
485
|
+
}
|
486
|
+
}
|
487
|
+
|
488
|
+
enum llama_pooling_type llama_context::pooling_type() const {
|
489
|
+
return cparams.pooling_type;
|
490
|
+
}
|
491
|
+
|
492
|
+
float * llama_context::get_logits() {
|
493
|
+
return logits;
|
494
|
+
}
|
495
|
+
|
496
|
+
float * llama_context::get_logits_ith(int32_t i) {
|
497
|
+
int32_t j = -1;
|
498
|
+
|
499
|
+
try {
|
500
|
+
if (logits == nullptr) {
|
501
|
+
throw std::runtime_error("no logits");
|
502
|
+
}
|
503
|
+
|
504
|
+
if (i < 0) {
|
505
|
+
j = n_outputs + i;
|
506
|
+
if (j < 0) {
|
507
|
+
throw std::runtime_error(format("negative index out of range [0, %d)", n_outputs));
|
508
|
+
}
|
509
|
+
} else if ((size_t) i >= output_ids.size()) {
|
510
|
+
throw std::runtime_error(format("out of range [0, %zu)", output_ids.size()));
|
511
|
+
} else {
|
512
|
+
j = output_ids[i];
|
513
|
+
}
|
514
|
+
|
515
|
+
if (j < 0) {
|
516
|
+
throw std::runtime_error(format("batch.logits[%d] != true", i));
|
517
|
+
}
|
518
|
+
if (j >= n_outputs) {
|
519
|
+
// This should not happen
|
520
|
+
throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, n_outputs));
|
521
|
+
}
|
522
|
+
|
523
|
+
return logits + j*model.vocab.n_tokens();
|
524
|
+
} catch (const std::exception & err) {
|
525
|
+
LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, err.what());
|
526
|
+
#ifndef NDEBUG
|
527
|
+
GGML_ABORT("fatal error");
|
528
|
+
#else
|
529
|
+
return nullptr;
|
530
|
+
#endif
|
531
|
+
}
|
532
|
+
}
|
533
|
+
|
534
|
+
float * llama_context::get_embeddings() {
|
535
|
+
return embd;
|
536
|
+
}
|
537
|
+
|
538
|
+
float * llama_context::get_embeddings_ith(int32_t i) {
|
539
|
+
int32_t j = -1;
|
540
|
+
|
541
|
+
try {
|
542
|
+
if (embd == nullptr) {
|
543
|
+
throw std::runtime_error("no embeddings");
|
544
|
+
}
|
545
|
+
|
546
|
+
if (i < 0) {
|
547
|
+
j = n_outputs + i;
|
548
|
+
if (j < 0) {
|
549
|
+
throw std::runtime_error(format("negative index out of range [0, %d)", n_outputs));
|
550
|
+
}
|
551
|
+
} else if ((size_t) i >= output_ids.size()) {
|
552
|
+
throw std::runtime_error(format("out of range [0, %zu)", output_ids.size()));
|
553
|
+
} else {
|
554
|
+
j = output_ids[i];
|
555
|
+
}
|
556
|
+
|
557
|
+
if (j < 0) {
|
558
|
+
throw std::runtime_error(format("batch.logits[%d] != true", i));
|
559
|
+
}
|
560
|
+
if (j >= n_outputs) {
|
561
|
+
// This should not happen
|
562
|
+
throw std::runtime_error(format("corrupt output buffer (j=%d, n_outputs=%d)", j, n_outputs));
|
563
|
+
}
|
564
|
+
|
565
|
+
return embd + j*model.hparams.n_embd;
|
566
|
+
} catch (const std::exception & err) {
|
567
|
+
LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, err.what());
|
568
|
+
#ifndef NDEBUG
|
569
|
+
GGML_ABORT("fatal error");
|
570
|
+
#else
|
571
|
+
return nullptr;
|
572
|
+
#endif
|
573
|
+
}
|
574
|
+
}
|
575
|
+
|
576
|
+
float * llama_context::get_embeddings_seq(llama_seq_id seq_id) {
|
577
|
+
auto it = embd_seq.find(seq_id);
|
578
|
+
if (it == embd_seq.end()) {
|
579
|
+
return nullptr;
|
580
|
+
}
|
581
|
+
|
582
|
+
return it->second.data();
|
583
|
+
}
|
584
|
+
|
585
|
+
void llama_context::attach_threadpool(
|
586
|
+
ggml_threadpool_t threadpool,
|
587
|
+
ggml_threadpool_t threadpool_batch) {
|
588
|
+
LLAMA_LOG_DEBUG("%s: call\n", __func__);
|
589
|
+
|
590
|
+
this->threadpool = threadpool;
|
591
|
+
this->threadpool_batch = threadpool_batch ? threadpool_batch : threadpool;
|
592
|
+
}
|
593
|
+
|
594
|
+
void llama_context::detach_threadpool() {
|
595
|
+
LLAMA_LOG_DEBUG("%s: call\n", __func__);
|
596
|
+
|
597
|
+
this->threadpool = nullptr;
|
598
|
+
this->threadpool_batch = nullptr;
|
599
|
+
}
|
600
|
+
|
601
|
+
void llama_context::set_n_threads(int32_t n_threads, int32_t n_threads_batch) {
|
602
|
+
LLAMA_LOG_DEBUG("%s: n_threads = %d, n_threads_batch = %d\n", __func__, n_threads, n_threads_batch);
|
603
|
+
|
604
|
+
cparams.n_threads = n_threads;
|
605
|
+
cparams.n_threads_batch = n_threads_batch;
|
606
|
+
}
|
607
|
+
|
608
|
+
void llama_context::set_abort_callback(bool (*abort_callback)(void * data), void * abort_callback_data) {
|
609
|
+
LLAMA_LOG_DEBUG("%s: call\n", __func__);
|
610
|
+
|
611
|
+
this->abort_callback = abort_callback;
|
612
|
+
this->abort_callback_data = abort_callback_data;
|
613
|
+
|
614
|
+
for (auto & backend : backends) {
|
615
|
+
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get()));
|
616
|
+
auto * set_abort_callback_fn = (ggml_backend_set_abort_callback_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_abort_callback");
|
617
|
+
if (set_abort_callback_fn) {
|
618
|
+
set_abort_callback_fn(backend.get(), this->abort_callback, this->abort_callback_data);
|
619
|
+
}
|
620
|
+
}
|
621
|
+
}
|
622
|
+
|
623
|
+
void llama_context::set_embeddings(bool value) {
|
624
|
+
LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);
|
625
|
+
|
626
|
+
cparams.embeddings = value;
|
627
|
+
}
|
628
|
+
|
629
|
+
void llama_context::set_causal_attn(bool value) {
|
630
|
+
LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);
|
631
|
+
|
632
|
+
cparams.causal_attn = value;
|
633
|
+
}
|
634
|
+
|
635
|
+
void llama_context::set_warmup(bool value) {
|
636
|
+
LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);
|
637
|
+
|
638
|
+
cparams.warmup = value;
|
639
|
+
}
|
640
|
+
|
641
|
+
void llama_context::set_adapter_lora(
|
642
|
+
llama_adapter_lora * adapter,
|
643
|
+
float scale) {
|
644
|
+
LLAMA_LOG_DEBUG("%s: adapter = %p, scale = %f\n", __func__, (void *) adapter, scale);
|
645
|
+
|
646
|
+
loras[adapter] = scale;
|
647
|
+
}
|
648
|
+
|
649
|
+
bool llama_context::rm_adapter_lora(
|
650
|
+
llama_adapter_lora * adapter) {
|
651
|
+
LLAMA_LOG_DEBUG("%s: adapter = %p\n", __func__, (void *) adapter);
|
652
|
+
|
653
|
+
auto pos = loras.find(adapter);
|
654
|
+
if (pos != loras.end()) {
|
655
|
+
loras.erase(pos);
|
656
|
+
return true;
|
657
|
+
}
|
658
|
+
|
659
|
+
return false;
|
660
|
+
}
|
661
|
+
|
662
|
+
void llama_context::clear_adapter_lora() {
|
663
|
+
LLAMA_LOG_DEBUG("%s: call\n", __func__);
|
664
|
+
|
665
|
+
loras.clear();
|
666
|
+
}
|
667
|
+
|
668
|
+
bool llama_context::apply_adapter_cvec(
|
669
|
+
const float * data,
|
670
|
+
size_t len,
|
671
|
+
int32_t n_embd,
|
672
|
+
int32_t il_start,
|
673
|
+
int32_t il_end) {
|
674
|
+
LLAMA_LOG_DEBUG("%s: il_start = %d, il_end = %d\n", __func__, il_start, il_end);
|
675
|
+
|
676
|
+
return cvec.apply(model, data, len, n_embd, il_start, il_end);
|
677
|
+
}
|
678
|
+
|
679
|
+
int llama_context::encode(llama_batch & inp_batch) {
|
680
|
+
if (inp_batch.n_tokens == 0) {
|
681
|
+
LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__);
|
682
|
+
return -1;
|
683
|
+
}
|
684
|
+
|
685
|
+
// temporary allocate memory for the input batch if needed
|
686
|
+
// note: during encode, we always pass the full sequence starting from pos = 0
|
687
|
+
llama_batch_allocr batch_allocr(inp_batch, inp_batch.pos ? -1 : 0);
|
688
|
+
|
689
|
+
const llama_batch & batch = batch_allocr.batch;
|
690
|
+
const int32_t n_tokens = batch.n_tokens;
|
691
|
+
|
692
|
+
const auto & hparams = model.hparams;
|
693
|
+
|
694
|
+
GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT
|
695
|
+
|
696
|
+
// TODO: move the validation to the llama_batch_allocr
|
697
|
+
if (batch.token) {
|
698
|
+
for (int32_t i = 0; i < n_tokens; ++i) {
|
699
|
+
if (batch.token[i] < 0 || (uint32_t) batch.token[i] >= model.vocab.n_tokens()) {
|
700
|
+
LLAMA_LOG_ERROR("%s: invalid token[%d] = %d\n", __func__, i, batch.token[i]);
|
701
|
+
return -1;
|
702
|
+
}
|
703
|
+
|
704
|
+
if (batch.seq_id && (batch.seq_id[i][0] < 0 || batch.seq_id[i][0] >= LLAMA_MAX_PARALLEL_SEQUENCES)) {
|
705
|
+
LLAMA_LOG_ERROR("%s: invalid seq_id[%d] = %d > %d\n", __func__, i, batch.seq_id[i][0], LLAMA_MAX_PARALLEL_SEQUENCES);
|
706
|
+
throw -1;
|
707
|
+
}
|
708
|
+
}
|
709
|
+
}
|
710
|
+
|
711
|
+
// micro-batching is not possible for non-causal encoding, so we process the batch in a single shot
|
712
|
+
GGML_ASSERT(cparams.n_ubatch >= (uint32_t) n_tokens && "encoder requires n_ubatch >= n_tokens");
|
713
|
+
|
714
|
+
if (t_compute_start_us == 0) {
|
715
|
+
t_compute_start_us = ggml_time_us();
|
716
|
+
}
|
717
|
+
|
718
|
+
embd_seq.clear();
|
719
|
+
|
720
|
+
n_queued_tokens += n_tokens;
|
721
|
+
|
722
|
+
const int64_t n_embd = hparams.n_embd;
|
723
|
+
|
724
|
+
llama_sbatch sbatch = llama_sbatch(batch, n_embd, /* simple_split */ true, /* logits_all */ true);
|
725
|
+
|
726
|
+
const llama_ubatch ubatch = sbatch.split_simple(n_tokens);
|
727
|
+
|
728
|
+
// reserve output buffer
|
729
|
+
if (output_reserve(n_tokens) < n_tokens) {
|
730
|
+
LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens);
|
731
|
+
return -2;
|
732
|
+
};
|
733
|
+
|
734
|
+
for (int32_t i = 0; i < n_tokens; ++i) {
|
735
|
+
output_ids[i] = i;
|
736
|
+
}
|
737
|
+
|
738
|
+
n_outputs = n_tokens;
|
739
|
+
|
740
|
+
//batch_manager->prepare(ubatch);
|
741
|
+
|
742
|
+
ggml_backend_sched_reset(sched.get());
|
743
|
+
ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data);
|
744
|
+
|
745
|
+
const auto causal_attn_org = cparams.causal_attn;
|
746
|
+
|
747
|
+
// always use non-causal attention for encoder graphs
|
748
|
+
// TODO: this is a tmp solution until we have a proper way to support enc-dec models
|
749
|
+
// ref: https://github.com/ggml-org/llama.cpp/pull/12181#issuecomment-2730451223
|
750
|
+
cparams.causal_attn = false;
|
751
|
+
|
752
|
+
auto * gf = graph_init();
|
753
|
+
auto res = graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_ENCODER);
|
754
|
+
|
755
|
+
ggml_backend_sched_alloc_graph(sched.get(), gf);
|
756
|
+
|
757
|
+
res->set_inputs(&ubatch);
|
758
|
+
|
759
|
+
cparams.causal_attn = causal_attn_org;
|
760
|
+
|
761
|
+
const auto compute_status = graph_compute(gf, n_tokens > 1);
|
762
|
+
switch (compute_status) {
|
763
|
+
case GGML_STATUS_SUCCESS:
|
764
|
+
break;
|
765
|
+
case GGML_STATUS_ABORTED:
|
766
|
+
return 2;
|
767
|
+
case GGML_STATUS_ALLOC_FAILED:
|
768
|
+
return -2;
|
769
|
+
case GGML_STATUS_FAILED:
|
770
|
+
default:
|
771
|
+
return -3;
|
772
|
+
}
|
773
|
+
|
774
|
+
auto * t_embd = res->get_embd_pooled() ? res->get_embd_pooled() : res->get_embd();
|
775
|
+
|
776
|
+
// extract embeddings
|
777
|
+
if (t_embd) {
|
778
|
+
ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd);
|
779
|
+
GGML_ASSERT(backend_embd != nullptr);
|
780
|
+
|
781
|
+
switch (cparams.pooling_type) {
|
782
|
+
case LLAMA_POOLING_TYPE_NONE:
|
783
|
+
{
|
784
|
+
// extract token embeddings
|
785
|
+
GGML_ASSERT(embd != nullptr);
|
786
|
+
|
787
|
+
GGML_ASSERT(n_tokens*n_embd <= (int64_t) embd_size);
|
788
|
+
ggml_backend_tensor_get_async(backend_embd, t_embd, embd, 0, n_tokens*n_embd*sizeof(float));
|
789
|
+
} break;
|
790
|
+
case LLAMA_POOLING_TYPE_MEAN:
|
791
|
+
case LLAMA_POOLING_TYPE_CLS:
|
792
|
+
case LLAMA_POOLING_TYPE_LAST:
|
793
|
+
{
|
794
|
+
// extract sequence embeddings
|
795
|
+
auto & embd_seq_out = embd_seq;
|
796
|
+
embd_seq_out.clear();
|
797
|
+
|
798
|
+
GGML_ASSERT(!ubatch.equal_seqs); // TODO: handle equal splits
|
799
|
+
|
800
|
+
for (int32_t i = 0; i < n_tokens; i++) {
|
801
|
+
const llama_seq_id seq_id = ubatch.seq_id[i][0];
|
802
|
+
if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {
|
803
|
+
continue;
|
804
|
+
}
|
805
|
+
embd_seq_out[seq_id].resize(n_embd);
|
806
|
+
ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float));
|
807
|
+
}
|
808
|
+
} break;
|
809
|
+
case LLAMA_POOLING_TYPE_RANK:
|
810
|
+
{
|
811
|
+
// extract the rerank score - a single float per sequence
|
812
|
+
auto & embd_seq_out = embd_seq;
|
813
|
+
|
814
|
+
for (uint32_t s = 0; s < ubatch.n_seqs; ++s) {
|
815
|
+
const llama_seq_id seq_id = ubatch.seq_id[s][0];
|
816
|
+
if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {
|
817
|
+
continue;
|
818
|
+
}
|
819
|
+
embd_seq_out[seq_id].resize(1);
|
820
|
+
ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (seq_id)*sizeof(float), sizeof(float));
|
821
|
+
}
|
822
|
+
} break;
|
823
|
+
case LLAMA_POOLING_TYPE_UNSPECIFIED:
|
824
|
+
{
|
825
|
+
GGML_ABORT("unknown pooling type");
|
826
|
+
}
|
827
|
+
}
|
828
|
+
}
|
829
|
+
|
830
|
+
// Reset state for the next token before backend sync, to allow the CPU activities in the reset to
|
831
|
+
// overlap with device computation.
|
832
|
+
ggml_backend_sched_reset(sched.get());
|
833
|
+
|
834
|
+
// TODO: hacky solution
|
835
|
+
if (model.arch == LLM_ARCH_T5 && t_embd) {
|
836
|
+
//cross.t_embd = t_embd;
|
837
|
+
|
838
|
+
synchronize();
|
839
|
+
|
840
|
+
cross.n_embd = t_embd->ne[0];
|
841
|
+
cross.n_enc = t_embd->ne[1];
|
842
|
+
cross.v_embd.resize(cross.n_embd*cross.n_enc);
|
843
|
+
memcpy(cross.v_embd.data(), embd, ggml_nbytes(t_embd));
|
844
|
+
|
845
|
+
// remember the sequence ids used during the encoding - needed for cross attention later
|
846
|
+
cross.seq_ids_enc.resize(n_tokens);
|
847
|
+
for (int32_t i = 0; i < n_tokens; i++) {
|
848
|
+
cross.seq_ids_enc[i].clear();
|
849
|
+
for (int s = 0; s < ubatch.n_seq_id[i]; s++) {
|
850
|
+
llama_seq_id seq_id = ubatch.seq_id[i][s];
|
851
|
+
cross.seq_ids_enc[i].insert(seq_id);
|
852
|
+
}
|
853
|
+
}
|
854
|
+
}
|
855
|
+
|
856
|
+
return 0;
|
857
|
+
}
|
858
|
+
|
859
|
+
int llama_context::decode(llama_batch & inp_batch) {
|
860
|
+
if (!memory) {
|
861
|
+
LLAMA_LOG_DEBUG("%s: cannot decode batches with this context (calling encode() instead)\n", __func__);
|
862
|
+
return encode(inp_batch);
|
863
|
+
}
|
864
|
+
|
865
|
+
if (inp_batch.n_tokens == 0) {
|
866
|
+
LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__);
|
867
|
+
return -1;
|
868
|
+
}
|
869
|
+
|
870
|
+
if (!inp_batch.pos) {
|
871
|
+
if (inp_batch.seq_id) {
|
872
|
+
LLAMA_LOG_ERROR("%s: pos == NULL, but seq_id != NULL\n", __func__);
|
873
|
+
return -1;
|
874
|
+
}
|
875
|
+
}
|
876
|
+
|
877
|
+
llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
|
878
|
+
|
879
|
+
// temporary allocate memory for the input batch if needed
|
880
|
+
llama_batch_allocr batch_allocr(inp_batch, inp_batch.pos ? -1 : kv_self->seq_pos_max(0) + 1);
|
881
|
+
|
882
|
+
const llama_batch & batch = batch_allocr.batch;
|
883
|
+
|
884
|
+
const auto & vocab = model.vocab;
|
885
|
+
const auto & hparams = model.hparams;
|
886
|
+
|
887
|
+
const int32_t n_vocab = vocab.n_tokens();
|
888
|
+
|
889
|
+
const int64_t n_tokens_all = batch.n_tokens;
|
890
|
+
const int64_t n_embd = hparams.n_embd;
|
891
|
+
|
892
|
+
llama_kv_cache_guard kv_guard(kv_self);
|
893
|
+
|
894
|
+
GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); // NOLINT
|
895
|
+
|
896
|
+
// TODO: move the validation to the llama_batch_allocr
|
897
|
+
if (batch.token) {
|
898
|
+
for (int64_t i = 0; i < n_tokens_all; ++i) {
|
899
|
+
if (batch.token[i] < 0 || (uint32_t) batch.token[i] >= model.vocab.n_tokens()) {
|
900
|
+
LLAMA_LOG_ERROR("%s: invalid token[%" PRId64 "] = %d\n", __func__, i, batch.token[i]);
|
901
|
+
return -1;
|
902
|
+
}
|
903
|
+
|
904
|
+
if (batch.seq_id && (batch.seq_id[i][0] < 0 || batch.seq_id[i][0] >= LLAMA_MAX_PARALLEL_SEQUENCES)) {
|
905
|
+
LLAMA_LOG_ERROR("%s: invalid seq_id[%" PRId64 "] = %d >= %d\n", __func__, i, batch.seq_id[i][0], LLAMA_MAX_PARALLEL_SEQUENCES);
|
906
|
+
return -1;
|
907
|
+
}
|
908
|
+
}
|
909
|
+
}
|
910
|
+
|
911
|
+
GGML_ASSERT(n_tokens_all <= cparams.n_batch);
|
912
|
+
|
913
|
+
GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens");
|
914
|
+
|
915
|
+
if (t_compute_start_us == 0) {
|
916
|
+
t_compute_start_us = ggml_time_us();
|
917
|
+
}
|
918
|
+
n_queued_tokens += n_tokens_all;
|
919
|
+
|
920
|
+
// this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens
|
921
|
+
const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE;
|
922
|
+
|
923
|
+
embd_seq.clear();
|
924
|
+
|
925
|
+
int64_t n_outputs_all = 0;
|
926
|
+
|
927
|
+
// count outputs
|
928
|
+
if (batch.logits && !embd_pooled) {
|
929
|
+
for (uint32_t i = 0; i < n_tokens_all; ++i) {
|
930
|
+
n_outputs_all += batch.logits[i] != 0;
|
931
|
+
}
|
932
|
+
} else if (embd_pooled) {
|
933
|
+
n_outputs_all = n_tokens_all;
|
934
|
+
} else {
|
935
|
+
// keep last output only
|
936
|
+
n_outputs_all = 1;
|
937
|
+
}
|
938
|
+
|
939
|
+
llama_sbatch sbatch = kv_self->sbatch_init(batch, /* logits_all */ n_outputs_all == n_tokens_all);
|
940
|
+
|
941
|
+
// reserve output buffer
|
942
|
+
if (output_reserve(n_outputs_all) < n_outputs_all) {
|
943
|
+
LLAMA_LOG_ERROR("%s: could not reserve space for batch with %" PRId64 " outputs\n", __func__, n_outputs_all);
|
944
|
+
return -2;
|
945
|
+
};
|
946
|
+
|
947
|
+
// handle any pending defrags/shifts
|
948
|
+
kv_self_update();
|
949
|
+
|
950
|
+
int64_t n_outputs_prev = 0;
|
951
|
+
|
952
|
+
while (sbatch.n_tokens > 0) {
|
953
|
+
llama_ubatch ubatch = kv_self->ubatch_next(sbatch, cparams.n_ubatch, embd_pooled);
|
954
|
+
|
955
|
+
// count the outputs in this u_batch
|
956
|
+
{
|
957
|
+
int32_t n_outputs_new = 0;
|
958
|
+
|
959
|
+
if (n_outputs_all == n_tokens_all) {
|
960
|
+
n_outputs_new = ubatch.n_tokens;
|
961
|
+
} else {
|
962
|
+
GGML_ASSERT(ubatch.output);
|
963
|
+
for (uint32_t i = 0; i < ubatch.n_tokens; i++) {
|
964
|
+
n_outputs_new += (int32_t) (ubatch.output[i] != 0);
|
965
|
+
}
|
966
|
+
}
|
967
|
+
|
968
|
+
// needs to happen before the graph is built
|
969
|
+
n_outputs = n_outputs_new;
|
970
|
+
}
|
971
|
+
|
972
|
+
// find KV slot
|
973
|
+
if (!kv_self->find_slot(ubatch)) {
|
974
|
+
return 1;
|
975
|
+
}
|
976
|
+
|
977
|
+
ggml_backend_sched_reset(sched.get());
|
978
|
+
ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data);
|
979
|
+
|
980
|
+
auto * gf = graph_init();
|
981
|
+
auto res = graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_DECODER);
|
982
|
+
|
983
|
+
// LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);
|
984
|
+
|
985
|
+
ggml_backend_sched_alloc_graph(sched.get(), gf);
|
986
|
+
|
987
|
+
res->set_inputs(&ubatch);
|
988
|
+
|
989
|
+
const auto compute_status = graph_compute(gf, ubatch.n_tokens > 1);
|
990
|
+
if (compute_status != GGML_STATUS_SUCCESS) {
|
991
|
+
switch (compute_status) {
|
992
|
+
case GGML_STATUS_ABORTED:
|
993
|
+
return 2;
|
994
|
+
case GGML_STATUS_ALLOC_FAILED:
|
995
|
+
return -2;
|
996
|
+
case GGML_STATUS_FAILED:
|
997
|
+
default:
|
998
|
+
return -3;
|
999
|
+
}
|
1000
|
+
}
|
1001
|
+
|
1002
|
+
// plot the computation graph in dot format (for debugging purposes)
|
1003
|
+
//if (n_past%100 == 0) {
|
1004
|
+
// ggml_graph_dump_dot(gf, NULL, "llama.dot");
|
1005
|
+
//}
|
1006
|
+
|
1007
|
+
auto * t_logits = cparams.embeddings ? nullptr : res->get_logits();
|
1008
|
+
auto * t_embd = cparams.embeddings ? res->get_embd() : nullptr;
|
1009
|
+
|
1010
|
+
if (t_embd && res->get_embd_pooled()) {
|
1011
|
+
t_embd = res->get_embd_pooled();
|
1012
|
+
}
|
1013
|
+
|
1014
|
+
// extract logits
|
1015
|
+
if (t_logits && n_outputs > 0) {
|
1016
|
+
ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(sched.get(), t_logits);
|
1017
|
+
GGML_ASSERT(backend_res != nullptr);
|
1018
|
+
GGML_ASSERT(logits != nullptr);
|
1019
|
+
|
1020
|
+
float * logits_out = logits + n_outputs_prev*n_vocab;
|
1021
|
+
|
1022
|
+
if (n_outputs) {
|
1023
|
+
GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all);
|
1024
|
+
GGML_ASSERT((n_outputs_prev + n_outputs)*n_vocab <= (int64_t) logits_size);
|
1025
|
+
ggml_backend_tensor_get_async(backend_res, t_logits, logits_out, 0, n_outputs*n_vocab*sizeof(float));
|
1026
|
+
}
|
1027
|
+
}
|
1028
|
+
|
1029
|
+
// extract embeddings
|
1030
|
+
if (t_embd && n_outputs > 0) {
|
1031
|
+
ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd);
|
1032
|
+
GGML_ASSERT(backend_embd != nullptr);
|
1033
|
+
|
1034
|
+
switch (cparams.pooling_type) {
|
1035
|
+
case LLAMA_POOLING_TYPE_NONE:
|
1036
|
+
{
|
1037
|
+
// extract token embeddings
|
1038
|
+
GGML_ASSERT(embd != nullptr);
|
1039
|
+
float * embd_out = embd + n_outputs_prev*n_embd;
|
1040
|
+
|
1041
|
+
if (n_outputs) {
|
1042
|
+
GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all);
|
1043
|
+
GGML_ASSERT((n_outputs_prev + n_outputs)*n_embd <= (int64_t) embd_size);
|
1044
|
+
ggml_backend_tensor_get_async(backend_embd, t_embd, embd_out, 0, n_outputs*n_embd*sizeof(float));
|
1045
|
+
}
|
1046
|
+
} break;
|
1047
|
+
case LLAMA_POOLING_TYPE_MEAN:
|
1048
|
+
case LLAMA_POOLING_TYPE_CLS:
|
1049
|
+
case LLAMA_POOLING_TYPE_LAST:
|
1050
|
+
{
|
1051
|
+
// extract sequence embeddings (cleared before processing each batch)
|
1052
|
+
auto & embd_seq_out = embd_seq;
|
1053
|
+
|
1054
|
+
for (uint32_t s = 0; s < ubatch.n_seqs; ++s) {
|
1055
|
+
const llama_seq_id seq_id = ubatch.seq_id[s][0];
|
1056
|
+
if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {
|
1057
|
+
continue;
|
1058
|
+
}
|
1059
|
+
embd_seq_out[seq_id].resize(n_embd);
|
1060
|
+
ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float));
|
1061
|
+
}
|
1062
|
+
} break;
|
1063
|
+
case LLAMA_POOLING_TYPE_RANK:
|
1064
|
+
{
|
1065
|
+
// extract the rerank score - a single float per sequence
|
1066
|
+
auto & embd_seq_out = embd_seq;
|
1067
|
+
|
1068
|
+
for (uint32_t s = 0; s < ubatch.n_seqs; ++s) {
|
1069
|
+
const llama_seq_id seq_id = ubatch.seq_id[s][0];
|
1070
|
+
if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {
|
1071
|
+
continue;
|
1072
|
+
}
|
1073
|
+
embd_seq_out[seq_id].resize(1);
|
1074
|
+
ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (seq_id)*sizeof(float), sizeof(float));
|
1075
|
+
}
|
1076
|
+
} break;
|
1077
|
+
case LLAMA_POOLING_TYPE_UNSPECIFIED:
|
1078
|
+
{
|
1079
|
+
GGML_ABORT("unknown pooling type");
|
1080
|
+
}
|
1081
|
+
}
|
1082
|
+
}
|
1083
|
+
|
1084
|
+
n_outputs_prev += n_outputs;
|
1085
|
+
}
|
1086
|
+
|
1087
|
+
// finalize the batch processing
|
1088
|
+
kv_guard.commit();
|
1089
|
+
|
1090
|
+
// set to total number of outputs in the batch, for use in llama_get_logits_ith
|
1091
|
+
n_outputs = n_outputs_all;
|
1092
|
+
|
1093
|
+
// set output mappings
|
1094
|
+
{
|
1095
|
+
bool sorted_output = true;
|
1096
|
+
|
1097
|
+
auto & out_ids = sbatch.out_ids;
|
1098
|
+
|
1099
|
+
GGML_ASSERT(out_ids.size() == (size_t) n_outputs_all);
|
1100
|
+
|
1101
|
+
for (int64_t i = 0; i < n_outputs_all; ++i) {
|
1102
|
+
int64_t out_id = out_ids[i];
|
1103
|
+
output_ids[out_id] = i;
|
1104
|
+
if (out_id != i) {
|
1105
|
+
sorted_output = false;
|
1106
|
+
}
|
1107
|
+
}
|
1108
|
+
|
1109
|
+
// make the outputs have the same order they had in the user-provided batch
|
1110
|
+
// note: this is mostly relevant for recurrent models atm
|
1111
|
+
if (!sorted_output) {
|
1112
|
+
const uint32_t n_vocab = model.vocab.n_tokens();
|
1113
|
+
const uint32_t n_embd = model.hparams.n_embd;
|
1114
|
+
|
1115
|
+
GGML_ASSERT((size_t) n_outputs == out_ids.size());
|
1116
|
+
|
1117
|
+
// TODO: is there something more efficient which also minimizes swaps?
|
1118
|
+
// selection sort, to minimize swaps (from https://en.wikipedia.org/wiki/Selection_sort)
|
1119
|
+
for (int32_t i = 0; i < n_outputs - 1; ++i) {
|
1120
|
+
int32_t j_min = i;
|
1121
|
+
for (int32_t j = i + 1; j < n_outputs; ++j) {
|
1122
|
+
if (out_ids[j] < out_ids[j_min]) {
|
1123
|
+
j_min = j;
|
1124
|
+
}
|
1125
|
+
}
|
1126
|
+
if (j_min == i) { continue; }
|
1127
|
+
std::swap(out_ids[i], out_ids[j_min]);
|
1128
|
+
if (logits_size > 0) {
|
1129
|
+
for (uint32_t k = 0; k < n_vocab; k++) {
|
1130
|
+
std::swap(logits[i*n_vocab + k], logits[j_min*n_vocab + k]);
|
1131
|
+
}
|
1132
|
+
}
|
1133
|
+
if (embd_size > 0) {
|
1134
|
+
for (uint32_t k = 0; k < n_embd; k++) {
|
1135
|
+
std::swap(embd[i*n_embd + k], embd[j_min*n_embd + k]);
|
1136
|
+
}
|
1137
|
+
}
|
1138
|
+
}
|
1139
|
+
std::fill(output_ids.begin(), output_ids.end(), -1);
|
1140
|
+
for (int32_t i = 0; i < n_outputs; ++i) {
|
1141
|
+
output_ids[out_ids[i]] = i;
|
1142
|
+
}
|
1143
|
+
}
|
1144
|
+
}
|
1145
|
+
|
1146
|
+
// wait for the computation to finish (automatically done when obtaining the model output)
|
1147
|
+
//synchronize();
|
1148
|
+
|
1149
|
+
// decide if we need to defrag the kv cache
|
1150
|
+
if (cparams.defrag_thold > 0.0f) {
|
1151
|
+
kv_self->defrag_sched(cparams.defrag_thold);
|
1152
|
+
}
|
1153
|
+
|
1154
|
+
// Reset state for the next token before backend sync, to allow the CPU activities in the reset to
|
1155
|
+
// overlap with device computation.
|
1156
|
+
ggml_backend_sched_reset(sched.get());
|
1157
|
+
|
1158
|
+
return 0;
|
1159
|
+
}
|
1160
|
+
|
1161
|
+
//
|
1162
|
+
// output
|
1163
|
+
//
|
1164
|
+
|
1165
|
+
int32_t llama_context::output_reserve(int32_t n_outputs) {
|
1166
|
+
const auto & hparams = model.hparams;
|
1167
|
+
const auto & vocab = model.vocab;
|
1168
|
+
|
1169
|
+
const int64_t n_outputs_max = std::max<int64_t>(n_outputs, n_seq_max());
|
1170
|
+
|
1171
|
+
const auto n_batch = cparams.n_batch;
|
1172
|
+
const auto n_vocab = vocab.n_tokens();
|
1173
|
+
const auto n_embd = hparams.n_embd;
|
1174
|
+
|
1175
|
+
// TODO: use a per-batch flag for logits presence instead
|
1176
|
+
bool has_logits = !cparams.embeddings;
|
1177
|
+
bool has_embd = cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE);
|
1178
|
+
|
1179
|
+
// TODO: hacky enc-dec support
|
1180
|
+
if (model.arch == LLM_ARCH_T5) {
|
1181
|
+
has_logits = true;
|
1182
|
+
has_embd = true;
|
1183
|
+
}
|
1184
|
+
|
1185
|
+
logits_size = has_logits ? n_vocab*n_outputs_max : 0;
|
1186
|
+
embd_size = has_embd ? n_embd*n_outputs_max : 0;
|
1187
|
+
|
1188
|
+
if (output_ids.empty()) {
|
1189
|
+
// init, never resized afterwards
|
1190
|
+
output_ids.resize(n_batch);
|
1191
|
+
}
|
1192
|
+
|
1193
|
+
const size_t prev_size = buf_output ? ggml_backend_buffer_get_size(buf_output.get()) : 0;
|
1194
|
+
const size_t new_size = (logits_size + embd_size) * sizeof(float);
|
1195
|
+
|
1196
|
+
// alloc only when more than the current capacity is required
|
1197
|
+
// TODO: also consider shrinking the buffer
|
1198
|
+
if (!buf_output || prev_size < new_size) {
|
1199
|
+
if (buf_output) {
|
1200
|
+
#ifndef NDEBUG
|
1201
|
+
// This doesn't happen often, but may be annoying in some cases (like the HellaSwag benchmark)
|
1202
|
+
LLAMA_LOG_INFO("%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);
|
1203
|
+
#endif
|
1204
|
+
buf_output = nullptr;
|
1205
|
+
logits = nullptr;
|
1206
|
+
embd = nullptr;
|
1207
|
+
}
|
1208
|
+
|
1209
|
+
auto * buft = ggml_backend_cpu_buffer_type();
|
1210
|
+
// try to use the host buffer of the device where the output tensor is allocated for faster transfer to system memory
|
1211
|
+
auto * output_dev = model.dev_output();
|
1212
|
+
auto * output_dev_host_buft = output_dev ? ggml_backend_dev_host_buffer_type(output_dev) : nullptr;
|
1213
|
+
if (output_dev_host_buft) {
|
1214
|
+
buft = output_dev_host_buft;
|
1215
|
+
}
|
1216
|
+
buf_output.reset(ggml_backend_buft_alloc_buffer(buft, new_size));
|
1217
|
+
if (buf_output == nullptr) {
|
1218
|
+
LLAMA_LOG_ERROR("%s: failed to allocate output buffer of size %.2f MiB\n", __func__, new_size / (1024.0 * 1024.0));
|
1219
|
+
return 0;
|
1220
|
+
}
|
1221
|
+
}
|
1222
|
+
|
1223
|
+
float * output_base = (float *) ggml_backend_buffer_get_base(buf_output.get());
|
1224
|
+
|
1225
|
+
logits = has_logits ? output_base : nullptr;
|
1226
|
+
embd = has_embd ? output_base + logits_size : nullptr;
|
1227
|
+
|
1228
|
+
// set all ids as invalid (negative)
|
1229
|
+
std::fill(output_ids.begin(), output_ids.end(), -1);
|
1230
|
+
|
1231
|
+
this->n_outputs = 0;
|
1232
|
+
this->n_outputs_max = n_outputs_max;
|
1233
|
+
|
1234
|
+
return n_outputs_max;
|
1235
|
+
}
|
1236
|
+
|
1237
|
+
//
|
1238
|
+
// graph
|
1239
|
+
//
|
1240
|
+
|
1241
|
+
int32_t llama_context::graph_max_nodes() const {
|
1242
|
+
return std::max<int32_t>(65536, 5*model.n_tensors());
|
1243
|
+
}
|
1244
|
+
|
1245
|
+
ggml_cgraph * llama_context::graph_init() {
|
1246
|
+
ggml_init_params params = {
|
1247
|
+
/*.mem_size =*/ buf_compute_meta.size(),
|
1248
|
+
/*.mem_buffer =*/ buf_compute_meta.data(),
|
1249
|
+
/*.no_alloc =*/ true,
|
1250
|
+
};
|
1251
|
+
|
1252
|
+
ctx_compute.reset(ggml_init(params));
|
1253
|
+
|
1254
|
+
return ggml_new_graph_custom(ctx_compute.get(), graph_max_nodes(), false);
|
1255
|
+
}
|
1256
|
+
|
1257
|
+
llm_graph_result_ptr llama_context::graph_build(
|
1258
|
+
ggml_context * ctx,
|
1259
|
+
ggml_cgraph * gf,
|
1260
|
+
const llama_ubatch & ubatch,
|
1261
|
+
llm_graph_type gtype) {
|
1262
|
+
return model.build_graph(
|
1263
|
+
{
|
1264
|
+
/*.ctx =*/ ctx,
|
1265
|
+
/*.arch =*/ model.arch,
|
1266
|
+
/*.hparams =*/ model.hparams,
|
1267
|
+
/*.cparams =*/ cparams,
|
1268
|
+
/*.ubatch =*/ ubatch,
|
1269
|
+
/*.sched =*/ sched.get(),
|
1270
|
+
/*.backend_cpu =*/ backend_cpu,
|
1271
|
+
/*.cvec =*/ &cvec,
|
1272
|
+
/*.loras =*/ &loras,
|
1273
|
+
/*.memory =*/ memory.get(),
|
1274
|
+
/*.cross =*/ &cross,
|
1275
|
+
/*.n_outputs =*/ n_outputs,
|
1276
|
+
/*.cb =*/ graph_get_cb(),
|
1277
|
+
}, gf, gtype);
|
1278
|
+
}
|
1279
|
+
|
1280
|
+
ggml_status llama_context::graph_compute(
|
1281
|
+
ggml_cgraph * gf,
|
1282
|
+
bool batched) {
|
1283
|
+
int n_threads = batched ? cparams.n_threads_batch : cparams.n_threads;
|
1284
|
+
ggml_threadpool_t tp = batched ? threadpool_batch : threadpool;
|
1285
|
+
|
1286
|
+
if (backend_cpu != nullptr) {
|
1287
|
+
auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu));
|
1288
|
+
auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool");
|
1289
|
+
set_threadpool_fn(backend_cpu, tp);
|
1290
|
+
}
|
1291
|
+
|
1292
|
+
// set the number of threads for all the backends
|
1293
|
+
for (const auto & set_n_threads_fn : set_n_threads_fns) {
|
1294
|
+
set_n_threads_fn.second(set_n_threads_fn.first, n_threads);
|
1295
|
+
}
|
1296
|
+
|
1297
|
+
auto status = ggml_backend_sched_graph_compute_async(sched.get(), gf);
|
1298
|
+
if (status != GGML_STATUS_SUCCESS) {
|
1299
|
+
LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async failed with error %d\n", __func__, status);
|
1300
|
+
}
|
1301
|
+
|
1302
|
+
// fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(sched));
|
1303
|
+
|
1304
|
+
return status;
|
1305
|
+
}
|
1306
|
+
|
1307
|
+
llm_graph_cb llama_context::graph_get_cb() const {
|
1308
|
+
return [&](const llama_ubatch & ubatch, ggml_tensor * cur, const char * name, int il) {
|
1309
|
+
if (il >= 0) {
|
1310
|
+
ggml_format_name(cur, "%s-%d", name, il);
|
1311
|
+
} else {
|
1312
|
+
ggml_set_name(cur, name);
|
1313
|
+
}
|
1314
|
+
|
1315
|
+
if (!cparams.offload_kqv) {
|
1316
|
+
if (strcmp(name, "kqv_merged_cont") == 0) {
|
1317
|
+
// all nodes between the KV store and the attention output are run on the CPU
|
1318
|
+
ggml_backend_sched_set_tensor_backend(sched.get(), cur, backend_cpu);
|
1319
|
+
}
|
1320
|
+
}
|
1321
|
+
|
1322
|
+
// norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends
|
1323
|
+
// FIXME: fix in ggml_backend_sched
|
1324
|
+
const bool full_offload = model.params.n_gpu_layers > (int) model.hparams.n_layer;
|
1325
|
+
if (ubatch.n_tokens < 32 || full_offload) {
|
1326
|
+
if (il != -1 && strcmp(name, "norm") == 0) {
|
1327
|
+
const auto & dev_layer = model.dev_layer(il);
|
1328
|
+
for (const auto & backend : backends) {
|
1329
|
+
if (ggml_backend_get_device(backend.get()) == dev_layer) {
|
1330
|
+
if (ggml_backend_supports_op(backend.get(), cur)) {
|
1331
|
+
ggml_backend_sched_set_tensor_backend(sched.get(), cur, backend.get());
|
1332
|
+
}
|
1333
|
+
}
|
1334
|
+
}
|
1335
|
+
}
|
1336
|
+
}
|
1337
|
+
};
|
1338
|
+
}
|
1339
|
+
|
1340
|
+
//
|
1341
|
+
// state save/load
|
1342
|
+
//
|
1343
|
+
|
1344
|
+
class llama_io_write_dummy : public llama_io_write_i {
|
1345
|
+
public:
|
1346
|
+
llama_io_write_dummy() = default;
|
1347
|
+
|
1348
|
+
void write(const void * /* src */, size_t size) override {
|
1349
|
+
size_written += size;
|
1350
|
+
}
|
1351
|
+
|
1352
|
+
void write_tensor(const ggml_tensor * /* tensor */, size_t /* offset */, size_t size) override {
|
1353
|
+
size_written += size;
|
1354
|
+
}
|
1355
|
+
|
1356
|
+
size_t n_bytes() override {
|
1357
|
+
return size_written;
|
1358
|
+
}
|
1359
|
+
|
1360
|
+
private:
|
1361
|
+
size_t size_written = 0;
|
1362
|
+
};
|
1363
|
+
|
1364
|
+
class llama_io_write_buffer : public llama_io_write_i {
|
1365
|
+
public:
|
1366
|
+
llama_io_write_buffer(
|
1367
|
+
uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
|
1368
|
+
|
1369
|
+
void write(const void * src, size_t size) override {
|
1370
|
+
if (size > buf_size) {
|
1371
|
+
throw std::runtime_error("unexpectedly reached end of buffer");
|
1372
|
+
}
|
1373
|
+
memcpy(ptr, src, size);
|
1374
|
+
ptr += size;
|
1375
|
+
size_written += size;
|
1376
|
+
buf_size -= size;
|
1377
|
+
}
|
1378
|
+
|
1379
|
+
void write_tensor(const ggml_tensor * tensor, size_t offset, size_t size) override {
|
1380
|
+
if (size > buf_size) {
|
1381
|
+
throw std::runtime_error("unexpectedly reached end of buffer");
|
1382
|
+
}
|
1383
|
+
ggml_backend_tensor_get(tensor, ptr, offset, size);
|
1384
|
+
ptr += size;
|
1385
|
+
size_written += size;
|
1386
|
+
buf_size -= size;
|
1387
|
+
}
|
1388
|
+
|
1389
|
+
size_t n_bytes() override {
|
1390
|
+
return size_written;
|
1391
|
+
}
|
1392
|
+
|
1393
|
+
private:
|
1394
|
+
uint8_t * ptr;
|
1395
|
+
size_t buf_size = 0;
|
1396
|
+
size_t size_written = 0;
|
1397
|
+
};
|
1398
|
+
|
1399
|
+
class llama_io_read_buffer : public llama_io_read_i {
|
1400
|
+
public:
|
1401
|
+
llama_io_read_buffer(const uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
|
1402
|
+
|
1403
|
+
const uint8_t * read(size_t size) override {
|
1404
|
+
const uint8_t * base_ptr = ptr;
|
1405
|
+
if (size > buf_size) {
|
1406
|
+
throw std::runtime_error("unexpectedly reached end of buffer");
|
1407
|
+
}
|
1408
|
+
ptr += size;
|
1409
|
+
size_read += size;
|
1410
|
+
buf_size -= size;
|
1411
|
+
return base_ptr;
|
1412
|
+
}
|
1413
|
+
|
1414
|
+
void read_to(void * dst, size_t size) override {
|
1415
|
+
memcpy(dst, read(size), size);
|
1416
|
+
}
|
1417
|
+
|
1418
|
+
size_t n_bytes() override {
|
1419
|
+
return size_read;
|
1420
|
+
}
|
1421
|
+
|
1422
|
+
private:
|
1423
|
+
const uint8_t * ptr;
|
1424
|
+
size_t buf_size = 0;
|
1425
|
+
size_t size_read = 0;
|
1426
|
+
};
|
1427
|
+
|
1428
|
+
class llama_io_write_file : public llama_io_write_i {
|
1429
|
+
public:
|
1430
|
+
llama_io_write_file(llama_file * f) : file(f) {}
|
1431
|
+
|
1432
|
+
void write(const void * src, size_t size) override {
|
1433
|
+
file->write_raw(src, size);
|
1434
|
+
size_written += size;
|
1435
|
+
}
|
1436
|
+
|
1437
|
+
void write_tensor(const ggml_tensor * tensor, size_t offset, size_t size) override {
|
1438
|
+
temp_buffer.resize(size);
|
1439
|
+
ggml_backend_tensor_get(tensor, temp_buffer.data(), offset, size);
|
1440
|
+
write(temp_buffer.data(), temp_buffer.size());
|
1441
|
+
}
|
1442
|
+
|
1443
|
+
size_t n_bytes() override {
|
1444
|
+
return size_written;
|
1445
|
+
}
|
1446
|
+
|
1447
|
+
private:
|
1448
|
+
llama_file * file;
|
1449
|
+
size_t size_written = 0;
|
1450
|
+
std::vector<uint8_t> temp_buffer;
|
1451
|
+
};
|
1452
|
+
|
1453
|
+
class llama_io_read_file : public llama_io_read_i {
|
1454
|
+
public:
|
1455
|
+
llama_io_read_file(llama_file * f) : file(f) {}
|
1456
|
+
|
1457
|
+
void read_to(void * dst, size_t size) override {
|
1458
|
+
file->read_raw(dst, size);
|
1459
|
+
size_read += size;
|
1460
|
+
}
|
1461
|
+
|
1462
|
+
const uint8_t * read(size_t size) override {
|
1463
|
+
temp_buffer.resize(size);
|
1464
|
+
read_to(temp_buffer.data(), size);
|
1465
|
+
return temp_buffer.data();
|
1466
|
+
}
|
1467
|
+
|
1468
|
+
size_t n_bytes() override {
|
1469
|
+
return size_read;
|
1470
|
+
}
|
1471
|
+
|
1472
|
+
private:
|
1473
|
+
llama_file * file;
|
1474
|
+
size_t size_read = 0;
|
1475
|
+
std::vector<uint8_t> temp_buffer;
|
1476
|
+
};
|
1477
|
+
|
1478
|
+
size_t llama_context::state_get_size() {
|
1479
|
+
llama_io_write_dummy io;
|
1480
|
+
try {
|
1481
|
+
return state_write_data(io);
|
1482
|
+
} catch (const std::exception & err) {
|
1483
|
+
LLAMA_LOG_ERROR("%s: error getting state size: %s\n", __func__, err.what());
|
1484
|
+
return 0;
|
1485
|
+
}
|
1486
|
+
}
|
1487
|
+
|
1488
|
+
size_t llama_context::state_get_data(uint8_t * dst, size_t size) {
|
1489
|
+
llama_io_write_buffer io(dst, size);
|
1490
|
+
try {
|
1491
|
+
return state_write_data(io);
|
1492
|
+
} catch (const std::exception & err) {
|
1493
|
+
LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what());
|
1494
|
+
return 0;
|
1495
|
+
}
|
1496
|
+
}
|
1497
|
+
|
1498
|
+
size_t llama_context::state_set_data(const uint8_t * src, size_t size) {
|
1499
|
+
llama_io_read_buffer io(src, size);
|
1500
|
+
try {
|
1501
|
+
return state_read_data(io);
|
1502
|
+
} catch (const std::exception & err) {
|
1503
|
+
LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what());
|
1504
|
+
return 0;
|
1505
|
+
}
|
1506
|
+
}
|
1507
|
+
|
1508
|
+
size_t llama_context::state_seq_get_size(llama_seq_id seq_id) {
|
1509
|
+
llama_io_write_dummy io;
|
1510
|
+
try {
|
1511
|
+
return state_seq_write_data(io, seq_id);
|
1512
|
+
} catch (const std::exception & err) {
|
1513
|
+
LLAMA_LOG_ERROR("%s: error getting state size: %s\n", __func__, err.what());
|
1514
|
+
return 0;
|
1515
|
+
}
|
1516
|
+
}
|
1517
|
+
|
1518
|
+
size_t llama_context::state_seq_get_data(llama_seq_id seq_id, uint8_t * dst, size_t size) {
|
1519
|
+
llama_io_write_buffer io(dst, size);
|
1520
|
+
try {
|
1521
|
+
return state_seq_write_data(io, seq_id);
|
1522
|
+
} catch (const std::exception & err) {
|
1523
|
+
LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what());
|
1524
|
+
return 0;
|
1525
|
+
}
|
1526
|
+
}
|
1527
|
+
|
1528
|
+
size_t llama_context::state_seq_set_data(llama_seq_id seq_id, const uint8_t * src, size_t size) {
|
1529
|
+
llama_io_read_buffer io(src, size);
|
1530
|
+
try {
|
1531
|
+
return state_seq_read_data(io, seq_id);
|
1532
|
+
} catch (const std::exception & err) {
|
1533
|
+
LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what());
|
1534
|
+
return 0;
|
1535
|
+
}
|
1536
|
+
}
|
1537
|
+
|
1538
|
+
bool llama_context::state_load_file(const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
|
1539
|
+
llama_file file(filepath, "rb");
|
1540
|
+
|
1541
|
+
// sanity checks
|
1542
|
+
{
|
1543
|
+
const uint32_t magic = file.read_u32();
|
1544
|
+
const uint32_t version = file.read_u32();
|
1545
|
+
|
1546
|
+
if (magic != LLAMA_SESSION_MAGIC || version != LLAMA_SESSION_VERSION) {
|
1547
|
+
LLAMA_LOG_ERROR("%s: unknown (magic, version) for session file: %08x, %08x\n", __func__, magic, version);
|
1548
|
+
return false;
|
1549
|
+
}
|
1550
|
+
}
|
1551
|
+
|
1552
|
+
// load the prompt
|
1553
|
+
{
|
1554
|
+
const uint32_t n_token_count = file.read_u32();
|
1555
|
+
|
1556
|
+
if (n_token_count > n_token_capacity) {
|
1557
|
+
LLAMA_LOG_ERROR("%s: token count in session file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity);
|
1558
|
+
return false;
|
1559
|
+
}
|
1560
|
+
|
1561
|
+
file.read_raw(tokens_out, sizeof(llama_token) * n_token_count);
|
1562
|
+
*n_token_count_out = n_token_count;
|
1563
|
+
}
|
1564
|
+
|
1565
|
+
// restore the context state
|
1566
|
+
{
|
1567
|
+
const size_t n_state_size_cur = file.size() - file.tell();
|
1568
|
+
|
1569
|
+
llama_io_read_file io( &file);
|
1570
|
+
const size_t n_read = state_read_data(io);
|
1571
|
+
|
1572
|
+
if (n_read != n_state_size_cur) {
|
1573
|
+
LLAMA_LOG_ERROR("%s: did not read all of the session file data! size %zu, got %zu\n", __func__, n_state_size_cur, n_read);
|
1574
|
+
return false;
|
1575
|
+
}
|
1576
|
+
}
|
1577
|
+
|
1578
|
+
return true;
|
1579
|
+
}
|
1580
|
+
|
1581
|
+
bool llama_context::state_save_file(const char * filepath, const llama_token * tokens, size_t n_token_count) {
|
1582
|
+
llama_file file(filepath, "wb");
|
1583
|
+
|
1584
|
+
file.write_u32(LLAMA_SESSION_MAGIC);
|
1585
|
+
file.write_u32(LLAMA_SESSION_VERSION);
|
1586
|
+
|
1587
|
+
// save the prompt
|
1588
|
+
file.write_u32((uint32_t) n_token_count);
|
1589
|
+
file.write_raw(tokens, sizeof(llama_token) * n_token_count);
|
1590
|
+
|
1591
|
+
// save the context state using stream saving
|
1592
|
+
llama_io_write_file io(&file);
|
1593
|
+
state_write_data(io);
|
1594
|
+
|
1595
|
+
return true;
|
1596
|
+
}
|
1597
|
+
|
1598
|
+
size_t llama_context::state_seq_load_file(llama_seq_id seq_id, const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
|
1599
|
+
llama_file file(filepath, "rb");
|
1600
|
+
|
1601
|
+
// version checks
|
1602
|
+
{
|
1603
|
+
const uint32_t magic = file.read_u32();
|
1604
|
+
const uint32_t version = file.read_u32();
|
1605
|
+
|
1606
|
+
if (magic != LLAMA_STATE_SEQ_MAGIC || version != LLAMA_STATE_SEQ_VERSION) {
|
1607
|
+
LLAMA_LOG_ERROR("%s: unknown (magic, version) for sequence state file: %08x, %08x\n", __func__, magic, version);
|
1608
|
+
return 0;
|
1609
|
+
}
|
1610
|
+
}
|
1611
|
+
|
1612
|
+
// load the prompt
|
1613
|
+
{
|
1614
|
+
const uint32_t n_token_count = file.read_u32();
|
1615
|
+
|
1616
|
+
if (n_token_count > n_token_capacity) {
|
1617
|
+
LLAMA_LOG_ERROR("%s: token count in sequence state file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity);
|
1618
|
+
return 0;
|
1619
|
+
}
|
1620
|
+
|
1621
|
+
file.read_raw(tokens_out, sizeof(llama_token) * n_token_count);
|
1622
|
+
*n_token_count_out = n_token_count;
|
1623
|
+
}
|
1624
|
+
|
1625
|
+
// restore the context state
|
1626
|
+
{
|
1627
|
+
const size_t state_size = file.size() - file.tell();
|
1628
|
+
llama_io_read_file io(&file);
|
1629
|
+
const size_t nread = state_seq_read_data(io, seq_id);
|
1630
|
+
if (!nread) {
|
1631
|
+
LLAMA_LOG_ERROR("%s: failed to restore sequence state\n", __func__);
|
1632
|
+
return 0;
|
1633
|
+
}
|
1634
|
+
GGML_ASSERT(nread <= state_size);
|
1635
|
+
GGML_ASSERT(nread + sizeof(uint32_t) * 3 + sizeof(llama_token) * *n_token_count_out == file.tell());
|
1636
|
+
}
|
1637
|
+
|
1638
|
+
return file.tell();
|
1639
|
+
}
|
1640
|
+
|
1641
|
+
size_t llama_context::state_seq_save_file(llama_seq_id seq_id, const char * filepath, const llama_token * tokens, size_t n_token_count) {
|
1642
|
+
llama_file file(filepath, "wb");
|
1643
|
+
|
1644
|
+
file.write_u32(LLAMA_STATE_SEQ_MAGIC);
|
1645
|
+
file.write_u32(LLAMA_STATE_SEQ_VERSION);
|
1646
|
+
|
1647
|
+
// save the prompt
|
1648
|
+
file.write_u32((uint32_t) n_token_count);
|
1649
|
+
file.write_raw(tokens, sizeof(llama_token) * n_token_count);
|
1650
|
+
|
1651
|
+
// save the context state using stream saving
|
1652
|
+
llama_io_write_file io(&file);
|
1653
|
+
state_seq_write_data(io, seq_id);
|
1654
|
+
|
1655
|
+
const size_t res = file.tell();
|
1656
|
+
GGML_ASSERT(res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + io.n_bytes());
|
1657
|
+
|
1658
|
+
return res;
|
1659
|
+
}
|
1660
|
+
|
1661
|
+
size_t llama_context::state_write_data(llama_io_write_i & io) {
|
1662
|
+
LLAMA_LOG_DEBUG("%s: writing state\n", __func__);
|
1663
|
+
|
1664
|
+
// write model info
|
1665
|
+
{
|
1666
|
+
LLAMA_LOG_DEBUG("%s: - writing model info\n", __func__);
|
1667
|
+
|
1668
|
+
const std::string arch_str = llm_arch_name(model.arch);
|
1669
|
+
io.write_string(arch_str);
|
1670
|
+
// TODO: add more model-specific info which should prevent loading the session file if not identical
|
1671
|
+
}
|
1672
|
+
|
1673
|
+
// write output ids
|
1674
|
+
{
|
1675
|
+
LLAMA_LOG_DEBUG("%s: - writing output ids\n", __func__);
|
1676
|
+
|
1677
|
+
const auto n_outputs = this->n_outputs;
|
1678
|
+
const auto & output_ids = this->output_ids;
|
1679
|
+
|
1680
|
+
std::vector<int32_t> w_output_pos;
|
1681
|
+
|
1682
|
+
GGML_ASSERT(n_outputs <= n_outputs_max);
|
1683
|
+
|
1684
|
+
w_output_pos.resize(n_outputs);
|
1685
|
+
|
1686
|
+
// build a more compact representation of the output ids
|
1687
|
+
for (size_t i = 0; i < n_batch(); ++i) {
|
1688
|
+
// map an output id to a position in the batch
|
1689
|
+
int32_t pos = output_ids[i];
|
1690
|
+
if (pos >= 0) {
|
1691
|
+
GGML_ASSERT(pos < n_outputs);
|
1692
|
+
w_output_pos[pos] = i;
|
1693
|
+
}
|
1694
|
+
}
|
1695
|
+
|
1696
|
+
io.write(&n_outputs, sizeof(n_outputs));
|
1697
|
+
|
1698
|
+
if (n_outputs) {
|
1699
|
+
io.write(w_output_pos.data(), n_outputs * sizeof(int32_t));
|
1700
|
+
}
|
1701
|
+
}
|
1702
|
+
|
1703
|
+
// write logits
|
1704
|
+
{
|
1705
|
+
LLAMA_LOG_DEBUG("%s: - writing logits\n", __func__);
|
1706
|
+
|
1707
|
+
const uint64_t logits_size = std::min((uint64_t) this->logits_size, (uint64_t) n_outputs * model.vocab.n_tokens());
|
1708
|
+
|
1709
|
+
io.write(&logits_size, sizeof(logits_size));
|
1710
|
+
|
1711
|
+
if (logits_size) {
|
1712
|
+
io.write(logits, logits_size * sizeof(float));
|
1713
|
+
}
|
1714
|
+
}
|
1715
|
+
|
1716
|
+
// write embeddings
|
1717
|
+
{
|
1718
|
+
LLAMA_LOG_DEBUG("%s: - writing embeddings\n", __func__);
|
1719
|
+
|
1720
|
+
const uint64_t embd_size = std::min((uint64_t) this->embd_size, (uint64_t) n_outputs * model.hparams.n_embd);
|
1721
|
+
|
1722
|
+
io.write(&embd_size, sizeof(embd_size));
|
1723
|
+
|
1724
|
+
if (embd_size) {
|
1725
|
+
io.write(embd, embd_size * sizeof(float));
|
1726
|
+
}
|
1727
|
+
}
|
1728
|
+
|
1729
|
+
llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
|
1730
|
+
|
1731
|
+
if (kv_self != nullptr) {
|
1732
|
+
LLAMA_LOG_DEBUG("%s: - writing KV self\n", __func__);
|
1733
|
+
kv_self->state_write(io);
|
1734
|
+
}
|
1735
|
+
|
1736
|
+
return io.n_bytes();
|
1737
|
+
}
|
1738
|
+
|
1739
|
+
size_t llama_context::state_read_data(llama_io_read_i & io) {
|
1740
|
+
LLAMA_LOG_DEBUG("%s: reading state\n", __func__);
|
1741
|
+
|
1742
|
+
// read model info
|
1743
|
+
{
|
1744
|
+
LLAMA_LOG_DEBUG("%s: - reading model info\n", __func__);
|
1745
|
+
|
1746
|
+
const std::string cur_arch_str = llm_arch_name(model.arch);
|
1747
|
+
|
1748
|
+
std::string arch_str;
|
1749
|
+
io.read_string(arch_str);
|
1750
|
+
if (cur_arch_str != arch_str) {
|
1751
|
+
throw std::runtime_error(format("wrong model arch: '%s' instead of '%s'", arch_str.c_str(), cur_arch_str.c_str()));
|
1752
|
+
}
|
1753
|
+
// TODO: add more info which needs to be identical but which is not verified otherwise
|
1754
|
+
}
|
1755
|
+
|
1756
|
+
// read output ids
|
1757
|
+
{
|
1758
|
+
LLAMA_LOG_DEBUG("%s: - reading output ids\n", __func__);
|
1759
|
+
|
1760
|
+
auto n_outputs = this->n_outputs;
|
1761
|
+
io.read_to(&n_outputs, sizeof(n_outputs));
|
1762
|
+
|
1763
|
+
if (n_outputs > output_reserve(n_outputs)) {
|
1764
|
+
throw std::runtime_error("could not reserve outputs");
|
1765
|
+
}
|
1766
|
+
|
1767
|
+
std::vector<int32_t> output_pos;
|
1768
|
+
|
1769
|
+
if (n_outputs) {
|
1770
|
+
output_pos.resize(n_outputs);
|
1771
|
+
io.read_to(output_pos.data(), n_outputs * sizeof(int32_t));
|
1772
|
+
|
1773
|
+
for (int32_t i = 0; i < (int32_t) output_pos.size(); ++i) {
|
1774
|
+
int32_t id = output_pos[i];
|
1775
|
+
if ((uint32_t) id >= n_batch()) {
|
1776
|
+
throw std::runtime_error(format("invalid output id, %d does not fit in batch size of %u", id, n_batch()));
|
1777
|
+
}
|
1778
|
+
this->output_ids[id] = i;
|
1779
|
+
}
|
1780
|
+
|
1781
|
+
this->n_outputs = n_outputs;
|
1782
|
+
}
|
1783
|
+
}
|
1784
|
+
|
1785
|
+
// read logits
|
1786
|
+
{
|
1787
|
+
LLAMA_LOG_DEBUG("%s: - reading logits\n", __func__);
|
1788
|
+
|
1789
|
+
uint64_t logits_size;
|
1790
|
+
io.read_to(&logits_size, sizeof(logits_size));
|
1791
|
+
|
1792
|
+
if (this->logits_size < logits_size) {
|
1793
|
+
throw std::runtime_error("logits buffer too small");
|
1794
|
+
}
|
1795
|
+
|
1796
|
+
if (logits_size) {
|
1797
|
+
io.read_to(this->logits, logits_size * sizeof(float));
|
1798
|
+
}
|
1799
|
+
}
|
1800
|
+
|
1801
|
+
// read embeddings
|
1802
|
+
{
|
1803
|
+
LLAMA_LOG_DEBUG("%s: - reading embeddings\n", __func__);
|
1804
|
+
|
1805
|
+
uint64_t embd_size;
|
1806
|
+
io.read_to(&embd_size, sizeof(embd_size));
|
1807
|
+
|
1808
|
+
if (this->embd_size < embd_size) {
|
1809
|
+
throw std::runtime_error("embeddings buffer too small");
|
1810
|
+
}
|
1811
|
+
|
1812
|
+
if (embd_size) {
|
1813
|
+
io.read_to(this->embd, embd_size * sizeof(float));
|
1814
|
+
}
|
1815
|
+
}
|
1816
|
+
|
1817
|
+
if (memory) {
|
1818
|
+
LLAMA_LOG_DEBUG("%s: - reading KV self\n", __func__);
|
1819
|
+
|
1820
|
+
llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
|
1821
|
+
|
1822
|
+
kv_self->state_read(io);
|
1823
|
+
}
|
1824
|
+
|
1825
|
+
return io.n_bytes();
|
1826
|
+
}
|
1827
|
+
|
1828
|
+
size_t llama_context::state_seq_write_data(llama_io_write_i & io, llama_seq_id seq_id) {
|
1829
|
+
GGML_UNUSED(seq_id);
|
1830
|
+
|
1831
|
+
if (memory) {
|
1832
|
+
llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
|
1833
|
+
|
1834
|
+
kv_self->state_write(io, seq_id);
|
1835
|
+
}
|
1836
|
+
|
1837
|
+
return io.n_bytes();
|
1838
|
+
}
|
1839
|
+
|
1840
|
+
size_t llama_context::state_seq_read_data(llama_io_read_i & io, llama_seq_id seq_id) {
|
1841
|
+
GGML_UNUSED(seq_id);
|
1842
|
+
|
1843
|
+
if (memory) {
|
1844
|
+
llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
|
1845
|
+
|
1846
|
+
kv_self->state_read(io, seq_id);
|
1847
|
+
}
|
1848
|
+
|
1849
|
+
return io.n_bytes();
|
1850
|
+
}
|
1851
|
+
|
1852
|
+
//
|
1853
|
+
// perf
|
1854
|
+
//
|
1855
|
+
|
1856
|
+
llama_perf_context_data llama_context::perf_get_data() const {
|
1857
|
+
llama_perf_context_data data = {};
|
1858
|
+
|
1859
|
+
data.t_start_ms = 1e-3 * t_start_us;
|
1860
|
+
data.t_load_ms = 1e-3 * t_load_us;
|
1861
|
+
data.t_p_eval_ms = 1e-3 * t_p_eval_us;
|
1862
|
+
data.t_eval_ms = 1e-3 * t_eval_us;
|
1863
|
+
data.n_p_eval = std::max(1, n_p_eval);
|
1864
|
+
data.n_eval = std::max(1, n_eval);
|
1865
|
+
|
1866
|
+
return data;
|
1867
|
+
}
|
1868
|
+
|
1869
|
+
void llama_context::perf_reset() {
|
1870
|
+
t_start_us = ggml_time_us();
|
1871
|
+
t_eval_us = n_eval = 0;
|
1872
|
+
t_p_eval_us = n_p_eval = 0;
|
1873
|
+
}
|
1874
|
+
|
1875
|
+
//
|
1876
|
+
// training
|
1877
|
+
//
|
1878
|
+
|
1879
|
+
static void llama_set_param(struct ggml_tensor * tensor, llama_opt_param_filter param_filter, void * userdata) {
|
1880
|
+
if (!tensor || tensor->type != GGML_TYPE_F32) {
|
1881
|
+
return;
|
1882
|
+
}
|
1883
|
+
if (!param_filter(tensor, userdata)) {
|
1884
|
+
return;
|
1885
|
+
}
|
1886
|
+
if (strcmp(tensor->name, "token_embd.weight") == 0) {
|
1887
|
+
return; // FIXME
|
1888
|
+
}
|
1889
|
+
if (strcmp(tensor->name, "rope_freqs.weight") == 0) {
|
1890
|
+
return; // FIXME
|
1891
|
+
}
|
1892
|
+
ggml_set_param(tensor);
|
1893
|
+
}
|
1894
|
+
|
1895
|
+
void llama_context::opt_init(struct llama_model * model, struct llama_opt_params lopt_params) {
|
1896
|
+
GGML_ASSERT(!opt_ctx);
|
1897
|
+
model->hparams.n_ctx_train = lopt_params.n_ctx_train > 0 ? lopt_params.n_ctx_train : n_ctx();
|
1898
|
+
const uint32_t n_batch = std::min(this->n_batch(), model->hparams.n_ctx_train);
|
1899
|
+
const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch);
|
1900
|
+
GGML_ASSERT(model->hparams.n_ctx_train % n_batch == 0);
|
1901
|
+
GGML_ASSERT(n_batch % n_ubatch == 0);
|
1902
|
+
|
1903
|
+
ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);
|
1904
|
+
opt_params.opt_period = n_batch / n_ubatch;
|
1905
|
+
opt_params.get_opt_pars = lopt_params.get_opt_pars;
|
1906
|
+
opt_params.get_opt_pars_ud = lopt_params.get_opt_pars_ud;
|
1907
|
+
|
1908
|
+
opt_ctx = ggml_opt_init(opt_params);
|
1909
|
+
|
1910
|
+
llama_opt_param_filter param_filter = lopt_params.param_filter;
|
1911
|
+
void * param_filter_ud = lopt_params.param_filter_ud;
|
1912
|
+
|
1913
|
+
//llama_set_param(model->tok_embd, param_filter, param_filter_ud); // FIXME
|
1914
|
+
llama_set_param(model->type_embd, param_filter, param_filter_ud);
|
1915
|
+
llama_set_param(model->pos_embd, param_filter, param_filter_ud);
|
1916
|
+
llama_set_param(model->tok_norm, param_filter, param_filter_ud);
|
1917
|
+
llama_set_param(model->tok_norm_b, param_filter, param_filter_ud);
|
1918
|
+
llama_set_param(model->output_norm, param_filter, param_filter_ud);
|
1919
|
+
llama_set_param(model->output_norm_b, param_filter, param_filter_ud);
|
1920
|
+
llama_set_param(model->output, param_filter, param_filter_ud);
|
1921
|
+
llama_set_param(model->output_b, param_filter, param_filter_ud);
|
1922
|
+
llama_set_param(model->output_norm_enc, param_filter, param_filter_ud);
|
1923
|
+
llama_set_param(model->cls, param_filter, param_filter_ud);
|
1924
|
+
llama_set_param(model->cls_b, param_filter, param_filter_ud);
|
1925
|
+
llama_set_param(model->cls_out, param_filter, param_filter_ud);
|
1926
|
+
llama_set_param(model->cls_out_b, param_filter, param_filter_ud);
|
1927
|
+
|
1928
|
+
for (struct llama_layer & layer : model->layers) {
|
1929
|
+
for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) {
|
1930
|
+
llama_set_param(reinterpret_cast<struct ggml_tensor **>(&layer)[i], param_filter, param_filter_ud);
|
1931
|
+
}
|
1932
|
+
}
|
1933
|
+
}
|
1934
|
+
|
1935
|
+
void llama_context::opt_epoch_iter(
|
1936
|
+
ggml_opt_dataset_t dataset,
|
1937
|
+
ggml_opt_result_t result,
|
1938
|
+
const std::vector<llama_token> & tokens,
|
1939
|
+
const std::vector<llama_token> & labels_sparse,
|
1940
|
+
llama_batch & batch,
|
1941
|
+
ggml_opt_epoch_callback callback,
|
1942
|
+
bool train,
|
1943
|
+
int64_t idata_in_loop,
|
1944
|
+
int64_t ndata_in_loop,
|
1945
|
+
int64_t t_loop_start) {
|
1946
|
+
GGML_ASSERT(opt_ctx);
|
1947
|
+
const uint32_t n_ctx = llama_model_n_ctx_train(&model);
|
1948
|
+
const uint32_t n_batch = std::min(this->n_batch(), n_ctx);
|
1949
|
+
const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch);
|
1950
|
+
|
1951
|
+
llama_kv_cache * kv_self = static_cast<llama_kv_cache *>(memory.get());
|
1952
|
+
|
1953
|
+
kv_self->clear();
|
1954
|
+
llama_kv_cache_guard kv_guard(kv_self);
|
1955
|
+
|
1956
|
+
for (uint32_t pos_ctx = 0; pos_ctx < n_ctx; pos_ctx += n_batch) {
|
1957
|
+
batch.n_tokens = n_batch;
|
1958
|
+
for (uint32_t pos_batch = 0; pos_batch < n_batch; ++pos_batch) {
|
1959
|
+
batch.token [pos_batch] = tokens[pos_ctx + pos_batch];
|
1960
|
+
batch.pos [pos_batch] = pos_ctx + pos_batch;
|
1961
|
+
batch.n_seq_id[pos_batch] = 1;
|
1962
|
+
batch.seq_id [pos_batch][0] = 0;
|
1963
|
+
batch.logits [pos_batch] = true;
|
1964
|
+
}
|
1965
|
+
|
1966
|
+
const auto n_tokens_all = batch.n_tokens;
|
1967
|
+
|
1968
|
+
n_queued_tokens += n_tokens_all;
|
1969
|
+
|
1970
|
+
// this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens
|
1971
|
+
const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE;
|
1972
|
+
|
1973
|
+
embd_seq.clear();
|
1974
|
+
|
1975
|
+
int64_t n_outputs_all = n_tokens_all;
|
1976
|
+
|
1977
|
+
llama_sbatch sbatch = kv_self->sbatch_init(batch, /*logits_all =*/ true);
|
1978
|
+
|
1979
|
+
// reserve output buffer
|
1980
|
+
if (output_reserve(n_outputs_all) < n_outputs_all) {
|
1981
|
+
LLAMA_LOG_ERROR("%s: could not reserve space for batch with %" PRId64 " outputs\n", __func__, n_outputs_all);
|
1982
|
+
GGML_ABORT("TODO: handle this error");
|
1983
|
+
};
|
1984
|
+
|
1985
|
+
for (uint32_t pos_batch = 0; pos_batch < n_batch; pos_batch += n_ubatch) {
|
1986
|
+
llama_ubatch ubatch = kv_self->ubatch_next(sbatch, cparams.n_ubatch, embd_pooled);
|
1987
|
+
|
1988
|
+
n_outputs = ubatch.n_tokens;
|
1989
|
+
|
1990
|
+
// TODO: not sure if this is needed
|
1991
|
+
if (!kv_self->find_slot(ubatch)) {
|
1992
|
+
LLAMA_LOG_WARN("%s: failed to find KV cache slot for ubatch of size %d\n", __func__, ubatch.n_tokens);
|
1993
|
+
|
1994
|
+
GGML_ABORT("TODO: handle this error");
|
1995
|
+
}
|
1996
|
+
|
1997
|
+
auto * gf = graph_init();
|
1998
|
+
auto res = graph_build(ctx_compute.get(), gf, ubatch, LLM_GRAPH_TYPE_DEFAULT);
|
1999
|
+
|
2000
|
+
struct ggml_context * ctx_compute_opt;
|
2001
|
+
{
|
2002
|
+
const size_t size_gf = ggml_graph_size(gf);
|
2003
|
+
const size_t size_meta = 4*size_gf*ggml_tensor_overhead() + 2*ggml_graph_overhead_custom(size_gf, /*grads = */ true);
|
2004
|
+
struct ggml_init_params params = {
|
2005
|
+
/*.mem_size =*/ size_meta,
|
2006
|
+
/*.mem_buffer =*/ nullptr,
|
2007
|
+
/*.no_alloc =*/ true,
|
2008
|
+
};
|
2009
|
+
ctx_compute_opt = ggml_init(params);
|
2010
|
+
}
|
2011
|
+
ggml_opt_prepare_alloc(opt_ctx, ctx_compute_opt, gf, res->get_tokens(), res->get_logits());
|
2012
|
+
ggml_opt_alloc(opt_ctx, train);
|
2013
|
+
res->set_inputs(&ubatch);
|
2014
|
+
{
|
2015
|
+
struct ggml_tensor * labels = ggml_opt_labels(opt_ctx);
|
2016
|
+
GGML_ASSERT(labels->ne[1] == n_ubatch);
|
2017
|
+
ggml_set_zero(labels);
|
2018
|
+
const float onef = 1.0f;
|
2019
|
+
for (uint32_t pos_ubatch = 0; pos_ubatch < n_ubatch; ++pos_ubatch) {
|
2020
|
+
const uint32_t ilabel = pos_ctx + pos_batch + pos_ubatch;
|
2021
|
+
GGML_ASSERT(labels_sparse[ilabel] < labels->ne[0]);
|
2022
|
+
ggml_backend_tensor_set(labels, &onef, (pos_ubatch*labels->ne[0] + labels_sparse[ilabel])*sizeof(float), sizeof(float));
|
2023
|
+
}
|
2024
|
+
}
|
2025
|
+
ggml_opt_eval(opt_ctx, result);
|
2026
|
+
if (callback) {
|
2027
|
+
callback(train, opt_ctx, dataset, result, idata_in_loop + (pos_ctx + pos_batch)/n_ubatch + 1, ndata_in_loop, t_loop_start);
|
2028
|
+
}
|
2029
|
+
ggml_free(ctx_compute_opt);
|
2030
|
+
}
|
2031
|
+
}
|
2032
|
+
|
2033
|
+
kv_guard.commit();
|
2034
|
+
}
|
2035
|
+
|
2036
|
+
void llama_context::opt_epoch(
|
2037
|
+
ggml_opt_dataset_t dataset,
|
2038
|
+
ggml_opt_result_t result_train,
|
2039
|
+
ggml_opt_result_t result_eval,
|
2040
|
+
int64_t idata_split,
|
2041
|
+
ggml_opt_epoch_callback callback_train,
|
2042
|
+
ggml_opt_epoch_callback callback_eval) {
|
2043
|
+
const uint32_t n_ctx = this->n_ctx();
|
2044
|
+
const uint32_t n_batch = std::min(cparams.n_batch, n_ctx);
|
2045
|
+
const uint32_t n_ubatch = std::min(cparams.n_ubatch, n_batch);
|
2046
|
+
const int64_t ndata = ggml_opt_dataset_ndata(dataset);
|
2047
|
+
|
2048
|
+
GGML_ASSERT(idata_split >= 0);
|
2049
|
+
GGML_ASSERT(idata_split <= ndata);
|
2050
|
+
|
2051
|
+
const uint32_t ubatch_per_ctx = n_ctx / n_ubatch;
|
2052
|
+
|
2053
|
+
struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
|
2054
|
+
std::vector<llama_token> tokens(n_ctx);
|
2055
|
+
std::vector<llama_token> labels_sparse(n_ctx);
|
2056
|
+
|
2057
|
+
int64_t idata = 0;
|
2058
|
+
|
2059
|
+
int64_t t_loop_start = ggml_time_us();
|
2060
|
+
int64_t ndata_in_loop = idata_split*ubatch_per_ctx;
|
2061
|
+
for (; idata < idata_split; ++idata) {
|
2062
|
+
constexpr bool train = true;
|
2063
|
+
const int64_t idata_in_loop = idata*ubatch_per_ctx;
|
2064
|
+
|
2065
|
+
ggml_opt_dataset_get_batch_host(dataset, tokens.data(), n_ctx*sizeof(llama_token), labels_sparse.data(), idata);
|
2066
|
+
opt_epoch_iter(dataset, result_train, tokens, labels_sparse, batch,
|
2067
|
+
callback_train, train, idata_in_loop, ndata_in_loop, t_loop_start);
|
2068
|
+
}
|
2069
|
+
|
2070
|
+
t_loop_start = ggml_time_us();
|
2071
|
+
ndata_in_loop = (ndata - idata_split)*ubatch_per_ctx;
|
2072
|
+
for (; idata < ndata; ++idata) {
|
2073
|
+
constexpr bool train = false;
|
2074
|
+
const int64_t idata_in_loop = (idata - idata_split)*ubatch_per_ctx;
|
2075
|
+
|
2076
|
+
ggml_opt_dataset_get_batch_host(dataset, tokens.data(), n_ctx*sizeof(llama_token), labels_sparse.data(), idata);
|
2077
|
+
opt_epoch_iter(dataset, result_eval, tokens, labels_sparse, batch,
|
2078
|
+
callback_eval, train, idata_in_loop, ndata_in_loop, t_loop_start);
|
2079
|
+
}
|
2080
|
+
|
2081
|
+
llama_batch_free(batch);
|
2082
|
+
}
|
2083
|
+
|
2084
|
+
//
|
2085
|
+
// interface implementation
|
2086
|
+
//
|
2087
|
+
|
2088
|
+
llama_context_params llama_context_default_params() {
|
2089
|
+
llama_context_params result = {
|
2090
|
+
/*.n_ctx =*/ 512,
|
2091
|
+
/*.n_batch =*/ 2048,
|
2092
|
+
/*.n_ubatch =*/ 512,
|
2093
|
+
/*.n_seq_max =*/ 1,
|
2094
|
+
/*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default
|
2095
|
+
/*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS,
|
2096
|
+
/*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED,
|
2097
|
+
/*.pooling_type =*/ LLAMA_POOLING_TYPE_UNSPECIFIED,
|
2098
|
+
/*.attention_type =*/ LLAMA_ATTENTION_TYPE_UNSPECIFIED,
|
2099
|
+
/*.rope_freq_base =*/ 0.0f,
|
2100
|
+
/*.rope_freq_scale =*/ 0.0f,
|
2101
|
+
/*.yarn_ext_factor =*/ -1.0f,
|
2102
|
+
/*.yarn_attn_factor =*/ 1.0f,
|
2103
|
+
/*.yarn_beta_fast =*/ 32.0f,
|
2104
|
+
/*.yarn_beta_slow =*/ 1.0f,
|
2105
|
+
/*.yarn_orig_ctx =*/ 0,
|
2106
|
+
/*.defrag_thold =*/ -1.0f,
|
2107
|
+
/*.cb_eval =*/ nullptr,
|
2108
|
+
/*.cb_eval_user_data =*/ nullptr,
|
2109
|
+
/*.type_k =*/ GGML_TYPE_F16,
|
2110
|
+
/*.type_v =*/ GGML_TYPE_F16,
|
2111
|
+
/*.abort_callback =*/ nullptr,
|
2112
|
+
/*.abort_callback_data =*/ nullptr,
|
2113
|
+
/*.embeddings =*/ false,
|
2114
|
+
/*.offload_kqv =*/ true,
|
2115
|
+
/*.flash_attn =*/ false,
|
2116
|
+
/*.no_perf =*/ true,
|
2117
|
+
/*.op_offload =*/ true,
|
2118
|
+
/*.swa_full =*/ true,
|
2119
|
+
};
|
2120
|
+
|
2121
|
+
return result;
|
2122
|
+
}
|
2123
|
+
|
2124
|
+
llama_context * llama_init_from_model(
|
2125
|
+
llama_model * model,
|
2126
|
+
llama_context_params params) {
|
2127
|
+
if (!model) {
|
2128
|
+
LLAMA_LOG_ERROR("%s: model cannot be NULL\n", __func__);
|
2129
|
+
return nullptr;
|
2130
|
+
}
|
2131
|
+
|
2132
|
+
if (params.n_batch == 0 && params.n_ubatch == 0) {
|
2133
|
+
LLAMA_LOG_ERROR("%s: n_batch and n_ubatch cannot both be zero\n", __func__);
|
2134
|
+
return nullptr;
|
2135
|
+
}
|
2136
|
+
|
2137
|
+
if (params.n_ctx == 0 && model->hparams.n_ctx_train == 0) {
|
2138
|
+
LLAMA_LOG_ERROR("%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\n", __func__);
|
2139
|
+
return nullptr;
|
2140
|
+
}
|
2141
|
+
|
2142
|
+
if (params.flash_attn && model->arch == LLM_ARCH_GROK) {
|
2143
|
+
LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__);
|
2144
|
+
params.flash_attn = false;
|
2145
|
+
}
|
2146
|
+
|
2147
|
+
if (ggml_is_quantized(params.type_v) && !params.flash_attn) {
|
2148
|
+
LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__);
|
2149
|
+
return nullptr;
|
2150
|
+
}
|
2151
|
+
|
2152
|
+
try {
|
2153
|
+
auto * ctx = new llama_context(*model, params);
|
2154
|
+
return ctx;
|
2155
|
+
} catch (const std::exception & err) {
|
2156
|
+
LLAMA_LOG_ERROR("%s: failed to initialize the context: %s\n", __func__, err.what());
|
2157
|
+
}
|
2158
|
+
|
2159
|
+
return nullptr;
|
2160
|
+
}
|
2161
|
+
|
2162
|
+
// deprecated
|
2163
|
+
llama_context * llama_new_context_with_model(
|
2164
|
+
llama_model * model,
|
2165
|
+
llama_context_params params) {
|
2166
|
+
return llama_init_from_model(model, params);
|
2167
|
+
}
|
2168
|
+
|
2169
|
+
void llama_free(llama_context * ctx) {
|
2170
|
+
delete ctx;
|
2171
|
+
}
|
2172
|
+
|
2173
|
+
uint32_t llama_n_ctx(const llama_context * ctx) {
|
2174
|
+
return ctx->n_ctx();
|
2175
|
+
}
|
2176
|
+
|
2177
|
+
uint32_t llama_n_batch(const llama_context * ctx) {
|
2178
|
+
return ctx->n_batch();
|
2179
|
+
}
|
2180
|
+
|
2181
|
+
uint32_t llama_n_ubatch(const llama_context * ctx) {
|
2182
|
+
return ctx->n_ubatch();
|
2183
|
+
}
|
2184
|
+
|
2185
|
+
uint32_t llama_n_seq_max(const llama_context * ctx) {
|
2186
|
+
return ctx->n_seq_max();
|
2187
|
+
}
|
2188
|
+
|
2189
|
+
const llama_model * llama_get_model(const llama_context * ctx) {
|
2190
|
+
return &ctx->get_model();
|
2191
|
+
}
|
2192
|
+
|
2193
|
+
llama_kv_cache * llama_get_kv_self(llama_context * ctx) {
|
2194
|
+
return ctx->get_kv_self();
|
2195
|
+
}
|
2196
|
+
|
2197
|
+
void llama_kv_self_update(llama_context * ctx) {
|
2198
|
+
ctx->kv_self_update();
|
2199
|
+
}
|
2200
|
+
|
2201
|
+
enum llama_pooling_type llama_pooling_type(const llama_context * ctx) {
|
2202
|
+
return ctx->pooling_type();
|
2203
|
+
}
|
2204
|
+
|
2205
|
+
void llama_attach_threadpool(
|
2206
|
+
llama_context * ctx,
|
2207
|
+
ggml_threadpool_t threadpool,
|
2208
|
+
ggml_threadpool_t threadpool_batch) {
|
2209
|
+
ctx->attach_threadpool(threadpool, threadpool_batch);
|
2210
|
+
}
|
2211
|
+
|
2212
|
+
void llama_detach_threadpool(llama_context * ctx) {
|
2213
|
+
ctx->detach_threadpool();
|
2214
|
+
}
|
2215
|
+
|
2216
|
+
void llama_set_n_threads(llama_context * ctx, int32_t n_threads, int32_t n_threads_batch) {
|
2217
|
+
ctx->set_n_threads(n_threads, n_threads_batch);
|
2218
|
+
}
|
2219
|
+
|
2220
|
+
int32_t llama_n_threads(llama_context * ctx) {
|
2221
|
+
return ctx->n_threads();
|
2222
|
+
}
|
2223
|
+
|
2224
|
+
int32_t llama_n_threads_batch(llama_context * ctx) {
|
2225
|
+
return ctx->n_threads_batch();
|
2226
|
+
}
|
2227
|
+
|
2228
|
+
void llama_set_abort_callback(llama_context * ctx, bool (*abort_callback)(void * data), void * abort_callback_data) {
|
2229
|
+
ctx->set_abort_callback(abort_callback, abort_callback_data);
|
2230
|
+
}
|
2231
|
+
|
2232
|
+
void llama_set_embeddings(llama_context * ctx, bool embeddings) {
|
2233
|
+
ctx->set_embeddings(embeddings);
|
2234
|
+
}
|
2235
|
+
|
2236
|
+
void llama_set_causal_attn(llama_context * ctx, bool causal_attn) {
|
2237
|
+
ctx->set_causal_attn(causal_attn);
|
2238
|
+
}
|
2239
|
+
|
2240
|
+
void llama_set_warmup(llama_context * ctx, bool warmup) {
|
2241
|
+
ctx->set_warmup(warmup);
|
2242
|
+
}
|
2243
|
+
|
2244
|
+
void llama_synchronize(llama_context * ctx) {
|
2245
|
+
ctx->synchronize();
|
2246
|
+
}
|
2247
|
+
|
2248
|
+
float * llama_get_logits(llama_context * ctx) {
|
2249
|
+
ctx->synchronize();
|
2250
|
+
|
2251
|
+
return ctx->get_logits();
|
2252
|
+
}
|
2253
|
+
|
2254
|
+
float * llama_get_logits_ith(llama_context * ctx, int32_t i) {
|
2255
|
+
ctx->synchronize();
|
2256
|
+
|
2257
|
+
return ctx->get_logits_ith(i);
|
2258
|
+
}
|
2259
|
+
|
2260
|
+
float * llama_get_embeddings(llama_context * ctx) {
|
2261
|
+
ctx->synchronize();
|
2262
|
+
|
2263
|
+
return ctx->get_embeddings();
|
2264
|
+
}
|
2265
|
+
|
2266
|
+
float * llama_get_embeddings_ith(llama_context * ctx, int32_t i) {
|
2267
|
+
ctx->synchronize();
|
2268
|
+
|
2269
|
+
return ctx->get_embeddings_ith(i);
|
2270
|
+
}
|
2271
|
+
|
2272
|
+
float * llama_get_embeddings_seq(llama_context * ctx, llama_seq_id seq_id) {
|
2273
|
+
ctx->synchronize();
|
2274
|
+
|
2275
|
+
return ctx->get_embeddings_seq(seq_id);
|
2276
|
+
}
|
2277
|
+
|
2278
|
+
// llama adapter API
|
2279
|
+
|
2280
|
+
int32_t llama_set_adapter_lora(
|
2281
|
+
llama_context * ctx,
|
2282
|
+
llama_adapter_lora * adapter,
|
2283
|
+
float scale) {
|
2284
|
+
ctx->set_adapter_lora(adapter, scale);
|
2285
|
+
|
2286
|
+
return 0;
|
2287
|
+
}
|
2288
|
+
|
2289
|
+
int32_t llama_rm_adapter_lora(
|
2290
|
+
llama_context * ctx,
|
2291
|
+
llama_adapter_lora * adapter) {
|
2292
|
+
bool res = ctx->rm_adapter_lora(adapter);
|
2293
|
+
|
2294
|
+
return res ? 0 : -1;
|
2295
|
+
}
|
2296
|
+
|
2297
|
+
void llama_clear_adapter_lora(llama_context * ctx) {
|
2298
|
+
ctx->clear_adapter_lora();
|
2299
|
+
}
|
2300
|
+
|
2301
|
+
int32_t llama_apply_adapter_cvec(
|
2302
|
+
llama_context * ctx,
|
2303
|
+
const float * data,
|
2304
|
+
size_t len,
|
2305
|
+
int32_t n_embd,
|
2306
|
+
int32_t il_start,
|
2307
|
+
int32_t il_end) {
|
2308
|
+
bool res = ctx->apply_adapter_cvec(data, len, n_embd, il_start, il_end);
|
2309
|
+
|
2310
|
+
return res ? 0 : -1;
|
2311
|
+
}
|
2312
|
+
|
2313
|
+
//
|
2314
|
+
// kv cache
|
2315
|
+
//
|
2316
|
+
|
2317
|
+
// deprecated
|
2318
|
+
int32_t llama_kv_self_n_tokens(const llama_context * ctx) {
|
2319
|
+
const auto * kv = ctx->get_kv_self();
|
2320
|
+
if (!kv) {
|
2321
|
+
return 0;
|
2322
|
+
}
|
2323
|
+
|
2324
|
+
int32_t res = 0;
|
2325
|
+
|
2326
|
+
for (uint32_t s = 0; s < ctx->get_cparams().n_seq_max; s++) {
|
2327
|
+
const llama_pos p0 = kv->seq_pos_min(s);
|
2328
|
+
const llama_pos p1 = kv->seq_pos_max(s);
|
2329
|
+
|
2330
|
+
if (p0 >= 0) {
|
2331
|
+
res += (p1 - p0) + 1;
|
2332
|
+
}
|
2333
|
+
}
|
2334
|
+
|
2335
|
+
return res;
|
2336
|
+
}
|
2337
|
+
|
2338
|
+
// deprecated
|
2339
|
+
// note: this is the same as above - will be removed anyway, so it's ok
|
2340
|
+
int32_t llama_kv_self_used_cells(const llama_context * ctx) {
|
2341
|
+
const auto * kv = ctx->get_kv_self();
|
2342
|
+
if (!kv) {
|
2343
|
+
return 0;
|
2344
|
+
}
|
2345
|
+
|
2346
|
+
int32_t res = 0;
|
2347
|
+
|
2348
|
+
for (uint32_t s = 0; s < ctx->get_cparams().n_seq_max; s++) {
|
2349
|
+
const llama_pos p0 = kv->seq_pos_min(s);
|
2350
|
+
const llama_pos p1 = kv->seq_pos_max(s);
|
2351
|
+
|
2352
|
+
if (p0 >= 0) {
|
2353
|
+
res += (p1 - p0) + 1;
|
2354
|
+
}
|
2355
|
+
}
|
2356
|
+
|
2357
|
+
return res;
|
2358
|
+
}
|
2359
|
+
|
2360
|
+
void llama_kv_self_clear(llama_context * ctx) {
|
2361
|
+
auto * kv = ctx->get_kv_self();
|
2362
|
+
if (!kv) {
|
2363
|
+
return;
|
2364
|
+
}
|
2365
|
+
|
2366
|
+
kv->clear();
|
2367
|
+
}
|
2368
|
+
|
2369
|
+
bool llama_kv_self_seq_rm(
|
2370
|
+
llama_context * ctx,
|
2371
|
+
llama_seq_id seq_id,
|
2372
|
+
llama_pos p0,
|
2373
|
+
llama_pos p1) {
|
2374
|
+
auto * kv = ctx->get_kv_self();
|
2375
|
+
if (!kv) {
|
2376
|
+
return true;
|
2377
|
+
}
|
2378
|
+
|
2379
|
+
return kv->seq_rm(seq_id, p0, p1);
|
2380
|
+
}
|
2381
|
+
|
2382
|
+
void llama_kv_self_seq_cp(
|
2383
|
+
llama_context * ctx,
|
2384
|
+
llama_seq_id seq_id_src,
|
2385
|
+
llama_seq_id seq_id_dst,
|
2386
|
+
llama_pos p0,
|
2387
|
+
llama_pos p1) {
|
2388
|
+
auto * kv = ctx->get_kv_self();
|
2389
|
+
if (!kv) {
|
2390
|
+
return;
|
2391
|
+
}
|
2392
|
+
|
2393
|
+
kv->seq_cp(seq_id_src, seq_id_dst, p0, p1);
|
2394
|
+
}
|
2395
|
+
|
2396
|
+
void llama_kv_self_seq_keep(llama_context * ctx, llama_seq_id seq_id) {
|
2397
|
+
auto * kv = ctx->get_kv_self();
|
2398
|
+
if (!kv) {
|
2399
|
+
return;
|
2400
|
+
}
|
2401
|
+
|
2402
|
+
kv->seq_keep(seq_id);
|
2403
|
+
}
|
2404
|
+
|
2405
|
+
void llama_kv_self_seq_add(
|
2406
|
+
llama_context * ctx,
|
2407
|
+
llama_seq_id seq_id,
|
2408
|
+
llama_pos p0,
|
2409
|
+
llama_pos p1,
|
2410
|
+
llama_pos delta) {
|
2411
|
+
auto * kv = ctx->get_kv_self();
|
2412
|
+
if (!kv) {
|
2413
|
+
return;
|
2414
|
+
}
|
2415
|
+
|
2416
|
+
kv->seq_add(seq_id, p0, p1, delta);
|
2417
|
+
}
|
2418
|
+
|
2419
|
+
void llama_kv_self_seq_div(
|
2420
|
+
llama_context * ctx,
|
2421
|
+
llama_seq_id seq_id,
|
2422
|
+
llama_pos p0,
|
2423
|
+
llama_pos p1,
|
2424
|
+
int d) {
|
2425
|
+
auto * kv = ctx->get_kv_self();
|
2426
|
+
if (!kv) {
|
2427
|
+
return;
|
2428
|
+
}
|
2429
|
+
|
2430
|
+
kv->seq_div(seq_id, p0, p1, d);
|
2431
|
+
}
|
2432
|
+
|
2433
|
+
llama_pos llama_kv_self_seq_pos_min(llama_context * ctx, llama_seq_id seq_id) {
|
2434
|
+
const auto * kv = ctx->get_kv_self();
|
2435
|
+
if (!kv) {
|
2436
|
+
return -1;
|
2437
|
+
}
|
2438
|
+
|
2439
|
+
return kv->seq_pos_min(seq_id);
|
2440
|
+
}
|
2441
|
+
|
2442
|
+
llama_pos llama_kv_self_seq_pos_max(llama_context * ctx, llama_seq_id seq_id) {
|
2443
|
+
const auto * kv = ctx->get_kv_self();
|
2444
|
+
if (!kv) {
|
2445
|
+
return -1;
|
2446
|
+
}
|
2447
|
+
|
2448
|
+
return kv->seq_pos_max(seq_id);
|
2449
|
+
}
|
2450
|
+
|
2451
|
+
void llama_kv_self_defrag(llama_context * ctx) {
|
2452
|
+
auto * kv = ctx->get_kv_self();
|
2453
|
+
if (!kv) {
|
2454
|
+
return;
|
2455
|
+
}
|
2456
|
+
|
2457
|
+
// force defrag
|
2458
|
+
kv->defrag_sched(-1.0f);
|
2459
|
+
}
|
2460
|
+
|
2461
|
+
bool llama_kv_self_can_shift(const llama_context * ctx) {
|
2462
|
+
const auto * kv = ctx->get_kv_self();
|
2463
|
+
if (!kv) {
|
2464
|
+
return false;
|
2465
|
+
}
|
2466
|
+
|
2467
|
+
return kv->get_can_shift();
|
2468
|
+
}
|
2469
|
+
|
2470
|
+
// llama state API
|
2471
|
+
|
2472
|
+
// deprecated
|
2473
|
+
size_t llama_get_state_size(llama_context * ctx) {
|
2474
|
+
return llama_state_get_size(ctx);
|
2475
|
+
}
|
2476
|
+
|
2477
|
+
// deprecated
|
2478
|
+
size_t llama_copy_state_data(llama_context * ctx, uint8_t * dst) {
|
2479
|
+
return llama_state_get_data(ctx, dst, -1);
|
2480
|
+
}
|
2481
|
+
|
2482
|
+
// deprecated
|
2483
|
+
size_t llama_set_state_data(llama_context * ctx, const uint8_t * src) {
|
2484
|
+
return llama_state_set_data(ctx, src, -1);
|
2485
|
+
}
|
2486
|
+
|
2487
|
+
// deprecated
|
2488
|
+
bool llama_load_session_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
|
2489
|
+
return llama_state_load_file(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out);
|
2490
|
+
}
|
2491
|
+
|
2492
|
+
// deprecated
|
2493
|
+
bool llama_save_session_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {
|
2494
|
+
return llama_state_save_file(ctx, path_session, tokens, n_token_count);
|
2495
|
+
}
|
2496
|
+
|
2497
|
+
// Returns the *actual* size of the state.
|
2498
|
+
// Intended to be used when saving to state to a buffer.
|
2499
|
+
size_t llama_state_get_size(llama_context * ctx) {
|
2500
|
+
return ctx->state_get_size();
|
2501
|
+
}
|
2502
|
+
|
2503
|
+
size_t llama_state_get_data(llama_context * ctx, uint8_t * dst, size_t size) {
|
2504
|
+
ctx->synchronize();
|
2505
|
+
|
2506
|
+
return ctx->state_get_data(dst, size);
|
2507
|
+
}
|
2508
|
+
|
2509
|
+
// Sets the state reading from the specified source address
|
2510
|
+
size_t llama_state_set_data(llama_context * ctx, const uint8_t * src, size_t size) {
|
2511
|
+
ctx->synchronize();
|
2512
|
+
|
2513
|
+
return ctx->state_set_data(src, size);
|
2514
|
+
}
|
2515
|
+
|
2516
|
+
bool llama_state_load_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
|
2517
|
+
ctx->synchronize();
|
2518
|
+
|
2519
|
+
try {
|
2520
|
+
return ctx->state_load_file(path_session, tokens_out, n_token_capacity, n_token_count_out);
|
2521
|
+
} catch (const std::exception & err) {
|
2522
|
+
LLAMA_LOG_ERROR("%s: error loading session file: %s\n", __func__, err.what());
|
2523
|
+
return false;
|
2524
|
+
}
|
2525
|
+
}
|
2526
|
+
|
2527
|
+
bool llama_state_save_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {
|
2528
|
+
ctx->synchronize();
|
2529
|
+
|
2530
|
+
try {
|
2531
|
+
return ctx->state_save_file(path_session, tokens, n_token_count);
|
2532
|
+
} catch (const std::exception & err) {
|
2533
|
+
LLAMA_LOG_ERROR("%s: error saving session file: %s\n", __func__, err.what());
|
2534
|
+
return false;
|
2535
|
+
}
|
2536
|
+
}
|
2537
|
+
|
2538
|
+
size_t llama_state_seq_get_size(llama_context * ctx, llama_seq_id seq_id) {
|
2539
|
+
return ctx->state_seq_get_size(seq_id);
|
2540
|
+
}
|
2541
|
+
|
2542
|
+
size_t llama_state_seq_get_data(llama_context * ctx, uint8_t * dst, size_t size, llama_seq_id seq_id) {
|
2543
|
+
ctx->synchronize();
|
2544
|
+
|
2545
|
+
return ctx->state_seq_get_data(seq_id, dst, size);
|
2546
|
+
}
|
2547
|
+
|
2548
|
+
size_t llama_state_seq_set_data(llama_context * ctx, const uint8_t * src, size_t size, llama_seq_id seq_id) {
|
2549
|
+
ctx->synchronize();
|
2550
|
+
|
2551
|
+
return ctx->state_seq_set_data(seq_id, src, size);
|
2552
|
+
}
|
2553
|
+
|
2554
|
+
size_t llama_state_seq_save_file(llama_context * ctx, const char * filepath, llama_seq_id seq_id, const llama_token * tokens, size_t n_token_count) {
|
2555
|
+
ctx->synchronize();
|
2556
|
+
|
2557
|
+
try {
|
2558
|
+
return ctx->state_seq_save_file(seq_id, filepath, tokens, n_token_count);
|
2559
|
+
} catch (const std::exception & err) {
|
2560
|
+
LLAMA_LOG_ERROR("%s: error saving sequence state file: %s\n", __func__, err.what());
|
2561
|
+
return 0;
|
2562
|
+
}
|
2563
|
+
}
|
2564
|
+
|
2565
|
+
size_t llama_state_seq_load_file(llama_context * ctx, const char * filepath, llama_seq_id dest_seq_id, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
|
2566
|
+
ctx->synchronize();
|
2567
|
+
|
2568
|
+
try {
|
2569
|
+
return ctx->state_seq_load_file(dest_seq_id, filepath, tokens_out, n_token_capacity, n_token_count_out);
|
2570
|
+
} catch (const std::exception & err) {
|
2571
|
+
LLAMA_LOG_ERROR("%s: error loading sequence state file: %s\n", __func__, err.what());
|
2572
|
+
return 0;
|
2573
|
+
}
|
2574
|
+
}
|
2575
|
+
|
2576
|
+
///
|
2577
|
+
|
2578
|
+
int32_t llama_encode(
|
2579
|
+
llama_context * ctx,
|
2580
|
+
llama_batch batch) {
|
2581
|
+
const int ret = ctx->encode(batch);
|
2582
|
+
if (ret != 0) {
|
2583
|
+
LLAMA_LOG_ERROR("%s: failed to encode, ret = %d\n", __func__, ret);
|
2584
|
+
}
|
2585
|
+
|
2586
|
+
return ret;
|
2587
|
+
}
|
2588
|
+
|
2589
|
+
int32_t llama_decode(
|
2590
|
+
llama_context * ctx,
|
2591
|
+
llama_batch batch) {
|
2592
|
+
int ret = ctx->decode(batch);
|
2593
|
+
|
2594
|
+
// defrag and try again
|
2595
|
+
// TODO: distinguish return code when we are sure that even after defrag there is no space available
|
2596
|
+
if (ret == 1) {
|
2597
|
+
llama_kv_self_defrag(ctx);
|
2598
|
+
ret = ctx->decode(batch);
|
2599
|
+
|
2600
|
+
if (ret == 1) {
|
2601
|
+
LLAMA_LOG_WARN("%s: failed to find KV cache slot for batch of size %d\n", __func__, batch.n_tokens);
|
2602
|
+
|
2603
|
+
return ret;
|
2604
|
+
}
|
2605
|
+
}
|
2606
|
+
|
2607
|
+
if (ret != 0) {
|
2608
|
+
LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret);
|
2609
|
+
}
|
2610
|
+
|
2611
|
+
return ret;
|
2612
|
+
}
|
2613
|
+
|
2614
|
+
//
|
2615
|
+
// perf
|
2616
|
+
//
|
2617
|
+
|
2618
|
+
llama_perf_context_data llama_perf_context(const llama_context * ctx) {
|
2619
|
+
llama_perf_context_data data = {};
|
2620
|
+
|
2621
|
+
if (ctx == nullptr) {
|
2622
|
+
return data;
|
2623
|
+
}
|
2624
|
+
|
2625
|
+
data = ctx->perf_get_data();
|
2626
|
+
|
2627
|
+
return data;
|
2628
|
+
}
|
2629
|
+
|
2630
|
+
void llama_perf_context_print(const llama_context * ctx) {
|
2631
|
+
const auto data = llama_perf_context(ctx);
|
2632
|
+
|
2633
|
+
const double t_end_ms = 1e-3 * ggml_time_us();
|
2634
|
+
|
2635
|
+
LLAMA_LOG_INFO("%s: load time = %10.2f ms\n", __func__, data.t_load_ms);
|
2636
|
+
LLAMA_LOG_INFO("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n",
|
2637
|
+
__func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval);
|
2638
|
+
LLAMA_LOG_INFO("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",
|
2639
|
+
__func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval);
|
2640
|
+
LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval));
|
2641
|
+
}
|
2642
|
+
|
2643
|
+
void llama_perf_context_reset(llama_context * ctx) {
|
2644
|
+
ctx->perf_reset();
|
2645
|
+
}
|
2646
|
+
|
2647
|
+
//
|
2648
|
+
// training
|
2649
|
+
//
|
2650
|
+
|
2651
|
+
bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata) {
|
2652
|
+
GGML_UNUSED(tensor);
|
2653
|
+
GGML_UNUSED(userdata);
|
2654
|
+
return true;
|
2655
|
+
}
|
2656
|
+
|
2657
|
+
void llama_opt_init(struct llama_context * ctx, struct llama_model * model, struct llama_opt_params lopt_params) {
|
2658
|
+
ctx->opt_init(model, lopt_params);
|
2659
|
+
}
|
2660
|
+
|
2661
|
+
void llama_opt_epoch(
|
2662
|
+
struct llama_context * ctx,
|
2663
|
+
ggml_opt_dataset_t dataset,
|
2664
|
+
ggml_opt_result_t result_train,
|
2665
|
+
ggml_opt_result_t result_eval,
|
2666
|
+
int64_t idata_split,
|
2667
|
+
ggml_opt_epoch_callback callback_train,
|
2668
|
+
ggml_opt_epoch_callback callback_eval) {
|
2669
|
+
ctx->opt_epoch(
|
2670
|
+
dataset,
|
2671
|
+
result_train,
|
2672
|
+
result_eval,
|
2673
|
+
idata_split,
|
2674
|
+
callback_train,
|
2675
|
+
callback_eval);
|
2676
|
+
}
|