whispercpp 1.3.1 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +4 -3
- data/README.md +92 -31
- data/Rakefile +26 -7
- data/ext/.gitignore +5 -7
- data/ext/dependencies.rb +61 -0
- data/ext/extconf.rb +21 -198
- data/ext/options.rb +221 -0
- data/ext/ruby_whisper.c +159 -0
- data/ext/ruby_whisper.h +17 -2
- data/ext/ruby_whisper_context.c +641 -0
- data/ext/ruby_whisper_error.c +52 -0
- data/ext/ruby_whisper_model.c +232 -0
- data/ext/ruby_whisper_params.c +1301 -0
- data/ext/ruby_whisper_segment.c +143 -0
- data/ext/ruby_whisper_transcribe.cpp +87 -0
- data/ext/ruby_whisper_vad_params.c +288 -0
- data/ext/sources/.dockerignore +3 -0
- data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
- data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
- data/ext/sources/CMakeLists.txt +251 -0
- data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
- data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
- data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
- data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
- data/ext/sources/bindings/javascript/package.json +26 -0
- data/ext/sources/bindings/javascript/whisper.js +19 -0
- data/ext/sources/build-xcframework.sh +547 -0
- data/ext/sources/ci/run.sh +336 -0
- data/ext/sources/close-issue.yml +28 -0
- data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
- data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
- data/ext/sources/cmake/build-info.cmake +60 -0
- data/ext/sources/cmake/git-vars.cmake +22 -0
- data/ext/sources/cmake/whisper-config.cmake.in +65 -0
- data/ext/sources/cmake/whisper.pc.in +10 -0
- data/ext/sources/examples/CMakeLists.txt +124 -0
- data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
- data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
- data/ext/sources/examples/addon.node/addon.cpp +438 -0
- data/ext/sources/examples/addon.node/index.js +54 -0
- data/ext/sources/examples/addon.node/package.json +16 -0
- data/ext/sources/examples/bench/CMakeLists.txt +8 -0
- data/ext/sources/examples/bench/bench.cpp +175 -0
- data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
- data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
- data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
- data/ext/sources/examples/cli/CMakeLists.txt +8 -0
- data/ext/sources/examples/cli/cli.cpp +1294 -0
- data/ext/sources/examples/coi-serviceworker.js +146 -0
- data/ext/sources/examples/command/CMakeLists.txt +10 -0
- data/ext/sources/examples/command/command.cpp +776 -0
- data/ext/sources/examples/command/commands.txt +9 -0
- data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
- data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
- data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
- data/ext/sources/examples/common-ggml.cpp +238 -0
- data/ext/sources/examples/common-ggml.h +18 -0
- data/ext/sources/examples/common-sdl.cpp +227 -0
- data/ext/sources/examples/common-sdl.h +49 -0
- data/ext/sources/examples/common-whisper.cpp +168 -0
- data/ext/sources/examples/common-whisper.h +24 -0
- data/ext/sources/examples/common.cpp +675 -0
- data/ext/sources/examples/common.h +322 -0
- data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
- data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
- data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
- data/ext/sources/examples/generate-karaoke.sh +57 -0
- data/ext/sources/examples/grammar-parser.cpp +423 -0
- data/ext/sources/examples/grammar-parser.h +29 -0
- data/ext/sources/examples/helpers.js +191 -0
- data/ext/sources/examples/json.hpp +24596 -0
- data/ext/sources/examples/livestream.sh +112 -0
- data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
- data/ext/sources/examples/lsp/lsp.cpp +467 -0
- data/ext/sources/examples/lsp/whisper.vim +362 -0
- data/ext/sources/examples/miniaudio.h +93468 -0
- data/ext/sources/examples/python/test_whisper_processor.py +7 -0
- data/ext/sources/examples/python/whisper_processor.py +54 -0
- data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
- data/ext/sources/examples/quantize/quantize.cpp +223 -0
- data/ext/sources/examples/server/CMakeLists.txt +12 -0
- data/ext/sources/examples/server/bench.js +29 -0
- data/ext/sources/examples/server/httplib.h +10497 -0
- data/ext/sources/examples/server/server.cpp +1091 -0
- data/ext/sources/examples/server.py +115 -0
- data/ext/sources/examples/stb_vorbis.c +5584 -0
- data/ext/sources/examples/stream/CMakeLists.txt +10 -0
- data/ext/sources/examples/stream/stream.cpp +429 -0
- data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
- data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
- data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
- data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
- data/ext/sources/examples/sycl/build.sh +22 -0
- data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
- data/ext/sources/examples/sycl/run-whisper.sh +17 -0
- data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
- data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
- data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
- data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
- data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
- data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
- data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
- data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
- data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
- data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
- data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
- data/ext/sources/examples/talk-llama/llama-context.h +276 -0
- data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
- data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
- data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
- data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
- data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
- data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
- data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
- data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
- data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
- data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
- data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
- data/ext/sources/examples/talk-llama/llama-io.h +35 -0
- data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
- data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
- data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
- data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
- data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
- data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
- data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
- data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
- data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
- data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
- data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
- data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
- data/ext/sources/examples/talk-llama/llama-model.h +425 -0
- data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
- data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
- data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
- data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
- data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
- data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
- data/ext/sources/examples/talk-llama/llama.cpp +354 -0
- data/ext/sources/examples/talk-llama/llama.h +1377 -0
- data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
- data/ext/sources/examples/talk-llama/speak +40 -0
- data/ext/sources/examples/talk-llama/speak.bat +1 -0
- data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
- data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
- data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
- data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
- data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
- data/ext/sources/examples/talk-llama/unicode.h +66 -0
- data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
- data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
- data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
- data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
- data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
- data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
- data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
- data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
- data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
- data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
- data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
- data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
- data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
- data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
- data/ext/sources/ggml/CMakeLists.txt +390 -0
- data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
- data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
- data/ext/sources/ggml/cmake/common.cmake +26 -0
- data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
- data/ext/{ggml → sources/ggml}/include/ggml-alloc.h +1 -1
- data/ext/{ggml → sources/ggml}/include/ggml-backend.h +9 -7
- data/ext/{ggml → sources/ggml}/include/ggml-cpp.h +2 -1
- data/ext/{ggml → sources/ggml}/include/ggml-cpu.h +9 -1
- data/ext/{ggml → sources/ggml}/include/ggml-metal.h +1 -1
- data/ext/{ggml → sources/ggml}/include/ggml-opt.h +49 -28
- data/ext/{ggml → sources/ggml}/include/ggml-rpc.h +6 -1
- data/ext/{ggml → sources/ggml}/include/ggml-vulkan.h +0 -2
- data/ext/{ggml → sources/ggml}/include/ggml.h +182 -265
- data/ext/sources/ggml/include/gguf.h +202 -0
- data/ext/sources/ggml/src/CMakeLists.txt +346 -0
- data/ext/{ggml → sources/ggml}/src/ggml-alloc.c +34 -29
- data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
- data/ext/{ggml → sources/ggml}/src/ggml-backend-impl.h +1 -2
- data/ext/{ggml → sources/ggml}/src/ggml-backend-reg.cpp +87 -53
- data/ext/{ggml → sources/ggml}/src/ggml-backend.cpp +26 -14
- data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
- data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
- data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.cpp +10 -4
- data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.h +5 -5
- data/ext/{ggml → sources/ggml}/src/ggml-cann/aclnn_ops.cpp +1272 -1506
- data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/common.h +135 -1
- data/ext/{ggml → sources/ggml}/src/ggml-cann/ggml-cann.cpp +564 -146
- data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/dup.cpp +3 -5
- data/ext/{ggml → sources/ggml}/src/ggml-common.h +12 -8
- data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.cpp +2 -1
- data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
- data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/cpu-feats-x86.cpp +5 -1
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-impl.h +163 -41
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.c +4029 -1117
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu.cpp +67 -18
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
- data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
- data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
- data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
- data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
- data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
- data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
- data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
- data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
- data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
- data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
- data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
- data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
- data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
- data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
- data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
- data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
- data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
- data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
- data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
- data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
- data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
- data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
- data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
- data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
- data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
- data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
- data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
- data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
- data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
- data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
- data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
- data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
- data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
- data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
- data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
- data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
- data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
- data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
- data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
- data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
- data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
- data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
- data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
- data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
- data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
- data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
- data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
- data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
- data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/cuda.h +1 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/hip.h +57 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/musa.h +7 -1
- data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
- data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
- data/ext/{ggml → sources/ggml}/src/ggml-impl.h +64 -19
- data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
- data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
- data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
- data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.m +2178 -1064
- data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.metal +1575 -1218
- data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
- data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
- data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
- data/ext/{ggml → sources/ggml}/src/ggml-opt.cpp +373 -190
- data/ext/{ggml → sources/ggml}/src/ggml-quants.c +114 -120
- data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
- data/ext/{ggml → sources/ggml}/src/ggml-rpc/ggml-rpc.cpp +480 -73
- data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
- data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
- data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
- data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/common.cpp +20 -32
- data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/concat.cpp +32 -33
- data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/conv.cpp +4 -2
- data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/convert.cpp +104 -28
- data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
- data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
- data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
- data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/dmmv.cpp +156 -17
- data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
- data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
- data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
- data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
- data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
- data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/ggml-sycl.cpp +1004 -1240
- data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
- data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmq.cpp +0 -1
- data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmvq.cpp +261 -166
- data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/norm.cpp +204 -81
- data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/outprod.cpp +8 -17
- data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
- data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
- data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
- data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/softmax.cpp +35 -25
- data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
- data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/tsembd.cpp +3 -3
- data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
- data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
- data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/ext/{ggml → sources/ggml}/src/ggml-vulkan/ggml-vulkan.cpp +3130 -1087
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
- data/ext/{ggml → sources/ggml}/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +193 -35
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/ext/{ggml → sources/ggml}/src/ggml.c +676 -1820
- data/ext/sources/ggml/src/gguf.cpp +1330 -0
- data/ext/{include → sources/include}/whisper.h +68 -2
- data/ext/sources/src/CMakeLists.txt +143 -0
- data/ext/{src → sources/src}/coreml/whisper-decoder-impl.h +27 -15
- data/ext/{src → sources/src}/coreml/whisper-decoder-impl.m +35 -10
- data/ext/{src → sources/src}/coreml/whisper-encoder-impl.h +21 -9
- data/ext/{src → sources/src}/coreml/whisper-encoder-impl.m +28 -3
- data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
- data/ext/sources/src/whisper-arch.h +197 -0
- data/ext/{src → sources/src}/whisper.cpp +1905 -374
- data/ext/sources/tests/CMakeLists.txt +105 -0
- data/ext/sources/tests/earnings21/eval.mk +58 -0
- data/ext/sources/tests/earnings21/eval.py +68 -0
- data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
- data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
- data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
- data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
- data/ext/sources/tests/earnings21/requirements.txt +6 -0
- data/ext/sources/tests/en-0-ref.txt +1 -0
- data/ext/sources/tests/en-1-ref.txt +1 -0
- data/ext/sources/tests/en-2-ref.txt +1 -0
- data/ext/sources/tests/es-0-ref.txt +1 -0
- data/ext/sources/tests/librispeech/eval.mk +39 -0
- data/ext/sources/tests/librispeech/eval.py +47 -0
- data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
- data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
- data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
- data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
- data/ext/sources/tests/librispeech/requirements.txt +6 -0
- data/ext/sources/tests/run-tests.sh +130 -0
- data/ext/sources/tests/test-c.c +3 -0
- data/ext/sources/tests/test-vad-full.cpp +54 -0
- data/ext/sources/tests/test-vad.cpp +83 -0
- data/ext/sources/tests/test-whisper.js +58 -0
- data/extsources.rb +33 -5
- data/lib/whisper/model/uri.rb +149 -128
- data/sig/whisper.rbs +480 -0
- data/tests/helper.rb +28 -0
- data/tests/test_callback.rb +45 -3
- data/tests/test_error.rb +2 -2
- data/tests/test_model.rb +38 -0
- data/tests/test_package.rb +18 -3
- data/tests/test_params.rb +145 -8
- data/tests/test_segment.rb +10 -19
- data/tests/test_vad.rb +19 -0
- data/tests/test_vad_params.rb +103 -0
- data/tests/test_whisper.rb +37 -37
- data/whispercpp.gemspec +5 -4
- metadata +766 -111
- data/ext/cpu.mk +0 -9
- data/ext/examples/dr_wav.h +0 -8815
- data/ext/ggml/src/ggml-cann/aclnn_ops.h +0 -592
- data/ext/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +0 -4262
- data/ext/ggml/src/ggml-cpu/ggml-cpu.c +0 -14123
- data/ext/ggml/src/ggml-cpu/llamafile/sgemm.cpp +0 -1884
- data/ext/ggml/src/ggml-cpu/llamafile/sgemm.h +0 -14
- data/ext/ggml/src/ggml-metal/ggml-metal-impl.h +0 -288
- data/ext/ggml/src/ggml-sycl/element_wise.cpp +0 -1030
- data/ext/ggml/src/ggml-sycl/im2col.cpp +0 -126
- data/ext/ggml/src/ggml-sycl/rope.cpp +0 -276
- data/ext/ggml/src/ggml-sycl/wkv6.cpp +0 -141
- data/ext/metal-embed.mk +0 -17
- data/ext/metal.mk +0 -6
- data/ext/ruby_whisper.cpp +0 -1909
- data/ext/scripts/get-flags.mk +0 -38
- data/lib/whisper.rb +0 -2
- /data/ext/{ggml → sources/ggml}/include/ggml-blas.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-cann.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-cuda.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-kompute.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-opencl.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-sycl.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/common.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/ggml-amx.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-blas/ggml-blas.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/ascendc_kernels.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f16.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f32.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q4_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/common.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-aarch64.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-kompute/ggml-kompute.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-quants.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-threading.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-threading.h +0 -0
- /data/ext/{src → sources/src}/coreml/whisper-encoder.h +0 -0
- /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.cpp +0 -0
- /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.h +0 -0
@@ -0,0 +1,1138 @@
|
|
1
|
+
#include "llama-model-loader.h"
|
2
|
+
|
3
|
+
#include "ggml.h"
|
4
|
+
|
5
|
+
#include <array>
|
6
|
+
#include <cinttypes>
|
7
|
+
#include <cstring>
|
8
|
+
#include <future>
|
9
|
+
|
10
|
+
static const size_t kiB = 1024;
|
11
|
+
static const size_t MiB = 1024*kiB;
|
12
|
+
static const size_t GiB = 1024*MiB;
|
13
|
+
|
14
|
+
const char * llama_file_version_name(llama_fver version) {
|
15
|
+
switch (version) {
|
16
|
+
case GGUF_FILE_VERSION_V1: return "GGUF V1 (support until nov 2023)";
|
17
|
+
case GGUF_FILE_VERSION_V2: return "GGUF V2";
|
18
|
+
case GGUF_FILE_VERSION_V3: return "GGUF V3 (latest)";
|
19
|
+
}
|
20
|
+
|
21
|
+
return "unknown";
|
22
|
+
}
|
23
|
+
|
24
|
+
static std::string llama_model_ftype_name(llama_ftype ftype) {
|
25
|
+
if (ftype & LLAMA_FTYPE_GUESSED) {
|
26
|
+
return llama_model_ftype_name((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) + " (guessed)";
|
27
|
+
}
|
28
|
+
|
29
|
+
switch (ftype) {
|
30
|
+
case LLAMA_FTYPE_ALL_F32: return "all F32";
|
31
|
+
case LLAMA_FTYPE_MOSTLY_F16: return "F16";
|
32
|
+
case LLAMA_FTYPE_MOSTLY_BF16: return "BF16";
|
33
|
+
case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";
|
34
|
+
case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";
|
35
|
+
case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0";
|
36
|
+
case LLAMA_FTYPE_MOSTLY_Q5_1: return "Q5_1";
|
37
|
+
case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0";
|
38
|
+
case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium";
|
39
|
+
case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small";
|
40
|
+
case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small";
|
41
|
+
case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium";
|
42
|
+
case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large";
|
43
|
+
case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "Q4_K - Small";
|
44
|
+
case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "Q4_K - Medium";
|
45
|
+
case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small";
|
46
|
+
case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium";
|
47
|
+
case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K";
|
48
|
+
case LLAMA_FTYPE_MOSTLY_TQ1_0: return "TQ1_0 - 1.69 bpw ternary";
|
49
|
+
case LLAMA_FTYPE_MOSTLY_TQ2_0: return "TQ2_0 - 2.06 bpw ternary";
|
50
|
+
case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return "IQ2_XXS - 2.0625 bpw";
|
51
|
+
case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw";
|
52
|
+
case LLAMA_FTYPE_MOSTLY_IQ2_S: return "IQ2_S - 2.5 bpw";
|
53
|
+
case LLAMA_FTYPE_MOSTLY_IQ2_M: return "IQ2_M - 2.7 bpw";
|
54
|
+
case LLAMA_FTYPE_MOSTLY_IQ3_XS: return "IQ3_XS - 3.3 bpw";
|
55
|
+
case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return "IQ3_XXS - 3.0625 bpw";
|
56
|
+
case LLAMA_FTYPE_MOSTLY_IQ1_S: return "IQ1_S - 1.5625 bpw";
|
57
|
+
case LLAMA_FTYPE_MOSTLY_IQ1_M: return "IQ1_M - 1.75 bpw";
|
58
|
+
case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw";
|
59
|
+
case LLAMA_FTYPE_MOSTLY_IQ4_XS: return "IQ4_XS - 4.25 bpw";
|
60
|
+
case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw";
|
61
|
+
case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw";
|
62
|
+
|
63
|
+
default: return "unknown, may not work";
|
64
|
+
}
|
65
|
+
}
|
66
|
+
|
67
|
+
// return a list of splits for a given path
|
68
|
+
// for example, given "<name>-00002-of-00004.gguf", returns list of all 4 splits
|
69
|
+
static std::vector<std::string> llama_get_list_splits(const std::string & path, const int idx, const int n_split) {
|
70
|
+
std::vector<std::string> paths;
|
71
|
+
std::string split_prefix;
|
72
|
+
std::vector<char> buf(llama_path_max(), 0);
|
73
|
+
|
74
|
+
{
|
75
|
+
int ret = llama_split_prefix(buf.data(), buf.size(), path.c_str(), idx, n_split);
|
76
|
+
if (!ret) {
|
77
|
+
throw std::runtime_error(format("invalid split file name: %s", path.c_str()));
|
78
|
+
}
|
79
|
+
split_prefix = std::string(buf.data(), ret);
|
80
|
+
}
|
81
|
+
|
82
|
+
if (split_prefix.empty()) {
|
83
|
+
throw std::runtime_error(format("invalid split file: %s", path.c_str()));
|
84
|
+
}
|
85
|
+
|
86
|
+
for (int idx = 0; idx < n_split; ++idx) {
|
87
|
+
int ret = llama_split_path(buf.data(), buf.size(), split_prefix.c_str(), idx, n_split);
|
88
|
+
paths.push_back(std::string(buf.data(), ret));
|
89
|
+
}
|
90
|
+
|
91
|
+
return paths;
|
92
|
+
}
|
93
|
+
|
94
|
+
namespace GGUFMeta {
|
95
|
+
template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int64_t)>
|
96
|
+
struct GKV_Base_Type {
|
97
|
+
static constexpr gguf_type gt = gt_;
|
98
|
+
|
99
|
+
static T getter(const gguf_context * ctx, const int kid) {
|
100
|
+
return gfun(ctx, kid);
|
101
|
+
}
|
102
|
+
};
|
103
|
+
|
104
|
+
template<typename T> struct GKV_Base;
|
105
|
+
|
106
|
+
template<> struct GKV_Base<bool >: GKV_Base_Type<bool, GGUF_TYPE_BOOL, gguf_get_val_bool> {};
|
107
|
+
template<> struct GKV_Base<uint8_t >: GKV_Base_Type<uint8_t, GGUF_TYPE_UINT8, gguf_get_val_u8 > {};
|
108
|
+
template<> struct GKV_Base<uint16_t >: GKV_Base_Type<uint16_t, GGUF_TYPE_UINT16, gguf_get_val_u16 > {};
|
109
|
+
template<> struct GKV_Base<uint32_t >: GKV_Base_Type<uint32_t, GGUF_TYPE_UINT32, gguf_get_val_u32 > {};
|
110
|
+
template<> struct GKV_Base<uint64_t >: GKV_Base_Type<uint64_t, GGUF_TYPE_UINT64, gguf_get_val_u64 > {};
|
111
|
+
template<> struct GKV_Base<int8_t >: GKV_Base_Type<int8_t, GGUF_TYPE_INT8, gguf_get_val_i8 > {};
|
112
|
+
template<> struct GKV_Base<int16_t >: GKV_Base_Type<int16_t, GGUF_TYPE_INT16, gguf_get_val_i16 > {};
|
113
|
+
template<> struct GKV_Base<int32_t >: GKV_Base_Type<int32_t, GGUF_TYPE_INT32, gguf_get_val_i32 > {};
|
114
|
+
template<> struct GKV_Base<int64_t >: GKV_Base_Type<int64_t, GGUF_TYPE_INT64, gguf_get_val_i64 > {};
|
115
|
+
template<> struct GKV_Base<float >: GKV_Base_Type<float, GGUF_TYPE_FLOAT32, gguf_get_val_f32 > {};
|
116
|
+
template<> struct GKV_Base<double >: GKV_Base_Type<double, GGUF_TYPE_FLOAT64, gguf_get_val_f64 > {};
|
117
|
+
template<> struct GKV_Base<const char *>: GKV_Base_Type<const char *, GGUF_TYPE_STRING, gguf_get_val_str > {};
|
118
|
+
|
119
|
+
template<> struct GKV_Base<std::string> {
|
120
|
+
static constexpr gguf_type gt = GGUF_TYPE_STRING;
|
121
|
+
|
122
|
+
static std::string getter(const gguf_context * ctx, const int kid) {
|
123
|
+
return gguf_get_val_str(ctx, kid);
|
124
|
+
}
|
125
|
+
};
|
126
|
+
|
127
|
+
struct ArrayInfo {
|
128
|
+
const gguf_type gt;
|
129
|
+
const size_t length;
|
130
|
+
const void * data;
|
131
|
+
};
|
132
|
+
|
133
|
+
template<> struct GKV_Base<ArrayInfo> {
|
134
|
+
public:
|
135
|
+
static constexpr gguf_type gt = GGUF_TYPE_ARRAY;
|
136
|
+
static ArrayInfo getter(const gguf_context *ctx, const int k) {
|
137
|
+
const enum gguf_type arr_type = gguf_get_arr_type(ctx, k);
|
138
|
+
return ArrayInfo {
|
139
|
+
arr_type,
|
140
|
+
size_t(gguf_get_arr_n(ctx, k)),
|
141
|
+
arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx, k),
|
142
|
+
};
|
143
|
+
}
|
144
|
+
};
|
145
|
+
|
146
|
+
template<typename T>
|
147
|
+
class GKV : public GKV_Base<T> {
|
148
|
+
GKV() = delete;
|
149
|
+
|
150
|
+
public:
|
151
|
+
static T get_kv(const gguf_context * ctx, const int k) {
|
152
|
+
const enum gguf_type kt = gguf_get_kv_type(ctx, k);
|
153
|
+
|
154
|
+
if (kt != GKV::gt) {
|
155
|
+
throw std::runtime_error(format("key %s has wrong type %s but expected type %s",
|
156
|
+
gguf_get_key(ctx, k), gguf_type_name(kt), gguf_type_name(GKV::gt)));
|
157
|
+
}
|
158
|
+
return GKV::getter(ctx, k);
|
159
|
+
}
|
160
|
+
|
161
|
+
static const char * override_type_to_str(const llama_model_kv_override_type ty) {
|
162
|
+
switch (ty) {
|
163
|
+
case LLAMA_KV_OVERRIDE_TYPE_BOOL: return "bool";
|
164
|
+
case LLAMA_KV_OVERRIDE_TYPE_INT: return "int";
|
165
|
+
case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return "float";
|
166
|
+
case LLAMA_KV_OVERRIDE_TYPE_STR: return "str";
|
167
|
+
}
|
168
|
+
return "unknown";
|
169
|
+
}
|
170
|
+
|
171
|
+
static bool validate_override(const llama_model_kv_override_type expected_type, const struct llama_model_kv_override * ovrd) {
|
172
|
+
if (!ovrd) { return false; }
|
173
|
+
if (ovrd->tag == expected_type) {
|
174
|
+
LLAMA_LOG_INFO("%s: Using metadata override (%5s) '%s' = ",
|
175
|
+
__func__, override_type_to_str(ovrd->tag), ovrd->key);
|
176
|
+
switch (ovrd->tag) {
|
177
|
+
case LLAMA_KV_OVERRIDE_TYPE_BOOL: {
|
178
|
+
LLAMA_LOG_INFO("%s\n", ovrd->val_bool ? "true" : "false");
|
179
|
+
} break;
|
180
|
+
case LLAMA_KV_OVERRIDE_TYPE_INT: {
|
181
|
+
LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->val_i64);
|
182
|
+
} break;
|
183
|
+
case LLAMA_KV_OVERRIDE_TYPE_FLOAT: {
|
184
|
+
LLAMA_LOG_INFO("%.6f\n", ovrd->val_f64);
|
185
|
+
} break;
|
186
|
+
case LLAMA_KV_OVERRIDE_TYPE_STR: {
|
187
|
+
LLAMA_LOG_INFO("%s\n", ovrd->val_str);
|
188
|
+
} break;
|
189
|
+
default:
|
190
|
+
// Shouldn't be possible to end up here, but just in case...
|
191
|
+
throw std::runtime_error(
|
192
|
+
format("Unsupported attempt to override %s type for metadata key %s\n",
|
193
|
+
override_type_to_str(ovrd->tag), ovrd->key));
|
194
|
+
}
|
195
|
+
return true;
|
196
|
+
}
|
197
|
+
LLAMA_LOG_WARN("%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\n",
|
198
|
+
__func__, ovrd->key, override_type_to_str(expected_type), override_type_to_str(ovrd->tag));
|
199
|
+
return false;
|
200
|
+
}
|
201
|
+
|
202
|
+
template<typename OT>
|
203
|
+
static typename std::enable_if<std::is_same<OT, bool>::value, bool>::type
|
204
|
+
try_override(OT & target, const struct llama_model_kv_override * ovrd) {
|
205
|
+
if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, ovrd)) {
|
206
|
+
target = ovrd->val_bool;
|
207
|
+
return true;
|
208
|
+
}
|
209
|
+
return false;
|
210
|
+
}
|
211
|
+
|
212
|
+
template<typename OT>
|
213
|
+
static typename std::enable_if<!std::is_same<OT, bool>::value && std::is_integral<OT>::value, bool>::type
|
214
|
+
try_override(OT & target, const struct llama_model_kv_override * ovrd) {
|
215
|
+
if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, ovrd)) {
|
216
|
+
target = ovrd->val_i64;
|
217
|
+
return true;
|
218
|
+
}
|
219
|
+
return false;
|
220
|
+
}
|
221
|
+
|
222
|
+
template<typename OT>
|
223
|
+
static typename std::enable_if<std::is_floating_point<OT>::value, bool>::type
|
224
|
+
try_override(T & target, const struct llama_model_kv_override * ovrd) {
|
225
|
+
if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, ovrd)) {
|
226
|
+
target = ovrd->val_f64;
|
227
|
+
return true;
|
228
|
+
}
|
229
|
+
return false;
|
230
|
+
}
|
231
|
+
|
232
|
+
template<typename OT>
|
233
|
+
static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type
|
234
|
+
try_override(T & target, const struct llama_model_kv_override * ovrd) {
|
235
|
+
if (validate_override(LLAMA_KV_OVERRIDE_TYPE_STR, ovrd)) {
|
236
|
+
target = ovrd->val_str;
|
237
|
+
return true;
|
238
|
+
}
|
239
|
+
return false;
|
240
|
+
}
|
241
|
+
|
242
|
+
static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override * ovrd = nullptr) {
|
243
|
+
if (try_override<T>(target, ovrd)) {
|
244
|
+
return true;
|
245
|
+
}
|
246
|
+
if (k < 0) { return false; }
|
247
|
+
target = get_kv(ctx, k);
|
248
|
+
return true;
|
249
|
+
}
|
250
|
+
|
251
|
+
static bool set(const gguf_context * ctx, const char * key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {
|
252
|
+
return set(ctx, gguf_find_key(ctx, key), target, ovrd);
|
253
|
+
}
|
254
|
+
|
255
|
+
static bool set(const gguf_context * ctx, const std::string & key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {
|
256
|
+
return set(ctx, key.c_str(), target, ovrd);
|
257
|
+
}
|
258
|
+
};
|
259
|
+
}
|
260
|
+
|
261
|
+
template<typename T>
|
262
|
+
typename std::enable_if<std::is_integral<T>::value, bool>::type
|
263
|
+
llama_model_loader::get_arr_n(const std::string & key, T & result, bool required) {
|
264
|
+
const int kid = gguf_find_key(meta.get(), key.c_str());
|
265
|
+
|
266
|
+
if (kid < 0) {
|
267
|
+
if (required) {
|
268
|
+
throw std::runtime_error(format("key not found in model: %s", key.c_str()));
|
269
|
+
}
|
270
|
+
return false;
|
271
|
+
}
|
272
|
+
|
273
|
+
struct GGUFMeta::ArrayInfo arr_info =
|
274
|
+
GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta.get(), kid);
|
275
|
+
|
276
|
+
|
277
|
+
result = arr_info.length;
|
278
|
+
return true;
|
279
|
+
}
|
280
|
+
|
281
|
+
template<typename T>
|
282
|
+
typename std::enable_if<std::is_integral<T>::value, bool>::type
|
283
|
+
llama_model_loader::get_arr_n(enum llm_kv kid, T & result, bool required) {
|
284
|
+
return get_arr_n(llm_kv(kid), result, required);
|
285
|
+
}
|
286
|
+
|
287
|
+
template bool llama_model_loader::get_arr_n(enum llm_kv kid, uint32_t & result, bool required);
|
288
|
+
|
289
|
+
template<typename T>
|
290
|
+
bool llama_model_loader::get_arr(const std::string & key, std::vector<T> & result, bool required) {
|
291
|
+
const int kid = gguf_find_key(meta.get(), key.c_str());
|
292
|
+
|
293
|
+
if (kid < 0 || gguf_get_kv_type(meta.get(), kid) != GGUF_TYPE_ARRAY) {
|
294
|
+
if (required) {
|
295
|
+
throw std::runtime_error(format("array key not found in model: %s", key.c_str()));
|
296
|
+
}
|
297
|
+
return false;
|
298
|
+
}
|
299
|
+
|
300
|
+
struct GGUFMeta::ArrayInfo arr_info =
|
301
|
+
GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta.get(), kid);
|
302
|
+
|
303
|
+
switch (arr_info.gt) {
|
304
|
+
case GGUF_TYPE_UINT32:
|
305
|
+
case GGUF_TYPE_INT32: GGML_ASSERT((std::is_same<T, int32_t>::value) ||
|
306
|
+
(std::is_same<T, uint32_t>::value)); break;
|
307
|
+
case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;
|
308
|
+
default:
|
309
|
+
throw std::runtime_error(format("%s is not a float32/uint32/int32 array", key.c_str()));
|
310
|
+
}
|
311
|
+
|
312
|
+
result.resize(arr_info.length);
|
313
|
+
result.assign((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length);
|
314
|
+
|
315
|
+
return true;
|
316
|
+
}
|
317
|
+
|
318
|
+
template<typename T, size_t N_MAX>
|
319
|
+
bool llama_model_loader::get_arr(const std::string & key, std::array<T, N_MAX> & result, bool required) {
|
320
|
+
const int kid = gguf_find_key(meta.get(), key.c_str());
|
321
|
+
|
322
|
+
if (kid < 0 || gguf_get_kv_type(meta.get(), kid) != GGUF_TYPE_ARRAY) {
|
323
|
+
if (required) {
|
324
|
+
throw std::runtime_error(format("array key not found in model: %s", key.c_str()));
|
325
|
+
}
|
326
|
+
return false;
|
327
|
+
}
|
328
|
+
|
329
|
+
struct GGUFMeta::ArrayInfo arr_info =
|
330
|
+
GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta.get(), kid);
|
331
|
+
|
332
|
+
switch (arr_info.gt) {
|
333
|
+
case GGUF_TYPE_UINT32:
|
334
|
+
case GGUF_TYPE_INT32: GGML_ASSERT((std::is_same<T, int32_t>::value) ||
|
335
|
+
(std::is_same<T, uint32_t>::value)); break;
|
336
|
+
case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;
|
337
|
+
default:
|
338
|
+
throw std::runtime_error(format("%s is not a float32/uint32/int32 array", key.c_str()));
|
339
|
+
}
|
340
|
+
|
341
|
+
if (arr_info.length > N_MAX) {
|
342
|
+
throw std::runtime_error(format("array length %u for key %s exceeds max %u", (uint32_t) arr_info.length, key.c_str(), (uint32_t) N_MAX));
|
343
|
+
}
|
344
|
+
|
345
|
+
std::copy((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length, result.begin());
|
346
|
+
|
347
|
+
return true;
|
348
|
+
}
|
349
|
+
|
350
|
+
template<typename T>
|
351
|
+
bool llama_model_loader::get_arr(enum llm_kv kid, T & result, bool required) {
|
352
|
+
return get_arr(llm_kv(kid), result, required);
|
353
|
+
}
|
354
|
+
|
355
|
+
template<typename T>
|
356
|
+
bool llama_model_loader::get_key(const std::string & key, T & result, bool required) {
|
357
|
+
auto it = kv_overrides.find(key);
|
358
|
+
|
359
|
+
const struct llama_model_kv_override * override =
|
360
|
+
it != kv_overrides.end() ? &it->second : nullptr;
|
361
|
+
|
362
|
+
const bool found = GGUFMeta::GKV<T>::set(meta.get(), key, result, override);
|
363
|
+
|
364
|
+
if (required && !found) {
|
365
|
+
throw std::runtime_error(format("key not found in model: %s", key.c_str()));
|
366
|
+
}
|
367
|
+
|
368
|
+
return found;
|
369
|
+
}
|
370
|
+
|
371
|
+
template<typename T>
|
372
|
+
bool llama_model_loader::get_key(enum llm_kv kid, T & result, bool required) {
|
373
|
+
return get_key(llm_kv(kid), result, required);
|
374
|
+
}
|
375
|
+
|
376
|
+
template bool llama_model_loader::get_key<bool> (enum llm_kv kid, bool & result, bool required);
|
377
|
+
template bool llama_model_loader::get_key<float> (enum llm_kv kid, float & result, bool required);
|
378
|
+
template bool llama_model_loader::get_key<uint32_t> (enum llm_kv kid, uint32_t & result, bool required);
|
379
|
+
template bool llama_model_loader::get_key<std::string>(enum llm_kv kid, std::string & result, bool required);
|
380
|
+
|
381
|
+
template<>
|
382
|
+
bool llama_model_loader::get_key(enum llm_kv kid, enum llama_pooling_type & result, bool required) {
|
383
|
+
uint32_t tmp;
|
384
|
+
const bool found = get_key(kid, tmp, required);
|
385
|
+
if (found) {
|
386
|
+
result = (enum llama_pooling_type) tmp;
|
387
|
+
} else {
|
388
|
+
result = LLAMA_POOLING_TYPE_UNSPECIFIED;
|
389
|
+
}
|
390
|
+
return found;
|
391
|
+
}
|
392
|
+
|
393
|
+
// get array of n <= N_MAX elements, or a single element repeated n times
|
394
|
+
template<typename T, size_t N_MAX>
|
395
|
+
bool llama_model_loader::get_key_or_arr(const std::string & key, std::array<T, N_MAX> & result, uint32_t n, bool required) {
|
396
|
+
const int kid = gguf_find_key(meta.get(), key.c_str());
|
397
|
+
|
398
|
+
if (kid < 0) {
|
399
|
+
if (required) {
|
400
|
+
throw std::runtime_error(format("key not found in model: %s", key.c_str()));
|
401
|
+
}
|
402
|
+
return false;
|
403
|
+
}
|
404
|
+
|
405
|
+
if (n > N_MAX) {
|
406
|
+
throw std::runtime_error(format("n > N_MAX: %u > %u for key %s", (uint32_t) n, (uint32_t) N_MAX, key.c_str()));
|
407
|
+
}
|
408
|
+
|
409
|
+
if (gguf_get_kv_type(meta.get(), kid) == GGUF_TYPE_ARRAY) {
|
410
|
+
struct GGUFMeta::ArrayInfo arr_info =
|
411
|
+
GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta.get(), kid);
|
412
|
+
|
413
|
+
if (n != arr_info.length) {
|
414
|
+
throw std::runtime_error(format("key %s has wrong array length; expected %u, got %u", key.c_str(), n, (uint32_t) arr_info.length));
|
415
|
+
}
|
416
|
+
|
417
|
+
return get_arr(key, result, required);
|
418
|
+
}
|
419
|
+
|
420
|
+
T value;
|
421
|
+
|
422
|
+
bool ok = get_key(key, value, required);
|
423
|
+
if (!ok) {
|
424
|
+
return false;
|
425
|
+
}
|
426
|
+
|
427
|
+
for (uint32_t i = 0; i < n; i++) {
|
428
|
+
result[i] = value;
|
429
|
+
}
|
430
|
+
|
431
|
+
return true;
|
432
|
+
}
|
433
|
+
|
434
|
+
template<typename T>
|
435
|
+
bool llama_model_loader::get_key_or_arr(enum llm_kv kid, T & result, uint32_t n, bool required) {
|
436
|
+
return get_key_or_arr(llm_kv(kid), result, n, required);
|
437
|
+
}
|
438
|
+
|
439
|
+
// TODO: this is not very clever - figure out something better
|
440
|
+
template bool llama_model_loader::get_key_or_arr<std::array<int, 4>>(enum llm_kv kid, std::array<int, 4> & result, uint32_t n, bool required);
|
441
|
+
template bool llama_model_loader::get_key_or_arr<std::array<uint32_t, 512>>(enum llm_kv kid, std::array<uint32_t, 512> & result, uint32_t n, bool required);
|
442
|
+
|
443
|
+
llama_model_loader::llama_model_loader(
|
444
|
+
const std::string & fname,
|
445
|
+
std::vector<std::string> & splits,
|
446
|
+
bool use_mmap,
|
447
|
+
bool check_tensors,
|
448
|
+
const llama_model_kv_override * param_overrides_p,
|
449
|
+
const llama_model_tensor_buft_override * param_tensor_buft_overrides_p) {
|
450
|
+
int trace = 0;
|
451
|
+
if (getenv("LLAMA_TRACE")) {
|
452
|
+
trace = atoi(getenv("LLAMA_TRACE"));
|
453
|
+
}
|
454
|
+
|
455
|
+
if (param_overrides_p != nullptr) {
|
456
|
+
for (const struct llama_model_kv_override * p = param_overrides_p; p->key[0] != 0; p++) {
|
457
|
+
kv_overrides.insert({std::string(p->key), *p});
|
458
|
+
}
|
459
|
+
}
|
460
|
+
|
461
|
+
tensor_buft_overrides = param_tensor_buft_overrides_p;
|
462
|
+
|
463
|
+
// Load the main GGUF
|
464
|
+
struct ggml_context * ctx = NULL;
|
465
|
+
struct gguf_init_params params = {
|
466
|
+
/*.no_alloc = */ true,
|
467
|
+
/*.ctx = */ &ctx,
|
468
|
+
};
|
469
|
+
|
470
|
+
meta.reset(gguf_init_from_file(fname.c_str(), params));
|
471
|
+
if (!meta) {
|
472
|
+
throw std::runtime_error(format("%s: failed to load model from %s", __func__, fname.c_str()));
|
473
|
+
}
|
474
|
+
|
475
|
+
get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);
|
476
|
+
llm_kv = LLM_KV(llm_arch_from_string(arch_name));
|
477
|
+
|
478
|
+
files.emplace_back(new llama_file(fname.c_str(), "rb"));
|
479
|
+
contexts.emplace_back(ctx);
|
480
|
+
|
481
|
+
// Save tensors data offset of the main file.
|
482
|
+
// For subsidiary files, `meta` tensor data offset must not be used,
|
483
|
+
// so we build a unified tensors index for weights.
|
484
|
+
for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
|
485
|
+
std::string tensor_name = std::string(cur->name);
|
486
|
+
// make sure there is no duplicated tensor names
|
487
|
+
if (weights_map.find(tensor_name) != weights_map.end()) {
|
488
|
+
throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
|
489
|
+
}
|
490
|
+
n_elements += ggml_nelements(cur);
|
491
|
+
n_bytes += ggml_nbytes(cur);
|
492
|
+
weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, meta.get(), cur));
|
493
|
+
}
|
494
|
+
uint16_t n_split = 0;
|
495
|
+
get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false);
|
496
|
+
|
497
|
+
// Load additional GGML contexts
|
498
|
+
if (n_split > 1) {
|
499
|
+
// make sure the main file is loaded first
|
500
|
+
uint16_t idx = 0;
|
501
|
+
const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO);
|
502
|
+
get_key(kv_split_no, idx);
|
503
|
+
if (idx != 0) {
|
504
|
+
throw std::runtime_error(format("illegal split file idx: %d (file: %s), model must be loaded with the first split", idx, fname.c_str()));
|
505
|
+
}
|
506
|
+
|
507
|
+
// generate list of splits if needed
|
508
|
+
if (splits.empty()) {
|
509
|
+
splits = llama_get_list_splits(fname, idx, n_split);
|
510
|
+
}
|
511
|
+
|
512
|
+
// in case user give a custom list of splits, check if it matches the expected number
|
513
|
+
if (n_split != (uint16_t)splits.size()) {
|
514
|
+
throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", splits.size(), n_split));
|
515
|
+
}
|
516
|
+
|
517
|
+
if (trace > 0) {
|
518
|
+
LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split);
|
519
|
+
}
|
520
|
+
|
521
|
+
// load other splits
|
522
|
+
for (idx = 1; idx < n_split; idx++) {
|
523
|
+
const char * fname_split = splits[idx].c_str();
|
524
|
+
|
525
|
+
struct gguf_init_params split_params = {
|
526
|
+
/*.no_alloc = */ true,
|
527
|
+
/*.ctx = */ &ctx,
|
528
|
+
};
|
529
|
+
gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) };
|
530
|
+
if (!ctx_gguf) {
|
531
|
+
throw std::runtime_error(format("%s: failed to load GGUF split from %s", __func__, fname_split));
|
532
|
+
}
|
533
|
+
|
534
|
+
// check idx
|
535
|
+
{
|
536
|
+
const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str());
|
537
|
+
if (kid < 0) {
|
538
|
+
throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split));
|
539
|
+
}
|
540
|
+
int idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid);
|
541
|
+
if (idx_gguf != idx) {
|
542
|
+
throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %d", idx_gguf, fname_split, idx));
|
543
|
+
}
|
544
|
+
}
|
545
|
+
|
546
|
+
files.emplace_back(new llama_file(fname_split, "rb"));
|
547
|
+
contexts.emplace_back(ctx);
|
548
|
+
|
549
|
+
// Save tensors data offset info of the shard.
|
550
|
+
for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
|
551
|
+
std::string tensor_name = std::string(cur->name);
|
552
|
+
// make sure there is no duplicated tensor names
|
553
|
+
if (weights_map.find(tensor_name) != weights_map.end()) {
|
554
|
+
throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));
|
555
|
+
}
|
556
|
+
n_elements += ggml_nelements(cur);
|
557
|
+
n_bytes += ggml_nbytes(cur);
|
558
|
+
weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), idx, ctx_gguf.get(), cur));
|
559
|
+
}
|
560
|
+
}
|
561
|
+
|
562
|
+
get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors);
|
563
|
+
|
564
|
+
// sanity check
|
565
|
+
{
|
566
|
+
const int n_tensors_loaded = (int) weights_map.size();
|
567
|
+
if (n_tensors != n_tensors_loaded) {
|
568
|
+
throw std::runtime_error(format("corrupted model: %d tensors expected but %d found", n_tensors, n_tensors_loaded));
|
569
|
+
}
|
570
|
+
}
|
571
|
+
|
572
|
+
LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1);
|
573
|
+
}
|
574
|
+
|
575
|
+
n_kv = gguf_get_n_kv(meta.get());
|
576
|
+
n_tensors = weights_map.size();
|
577
|
+
|
578
|
+
fver = (enum llama_fver) gguf_get_version(meta.get());
|
579
|
+
|
580
|
+
LLAMA_LOG_INFO("%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n",
|
581
|
+
__func__, n_kv, n_tensors, fname.c_str(), llama_file_version_name(fver));
|
582
|
+
|
583
|
+
// determine file type based on the number of tensors for each quantization and print meta data
|
584
|
+
// TODO: make optional
|
585
|
+
{
|
586
|
+
std::map<enum ggml_type, uint32_t> n_type;
|
587
|
+
|
588
|
+
uint32_t n_type_max = 0;
|
589
|
+
enum ggml_type type_max = GGML_TYPE_F32;
|
590
|
+
|
591
|
+
for (const auto & it : weights_map) {
|
592
|
+
const llama_tensor_weight & w = it.second;
|
593
|
+
const ggml_tensor * tensor = w.tensor;
|
594
|
+
|
595
|
+
enum ggml_type type = tensor->type;
|
596
|
+
|
597
|
+
n_type[type]++;
|
598
|
+
|
599
|
+
if (n_type_max < n_type[type]) {
|
600
|
+
n_type_max = n_type[type];
|
601
|
+
type_max = type;
|
602
|
+
}
|
603
|
+
|
604
|
+
if (trace > 0) {
|
605
|
+
const uint16_t sid = w.idx;
|
606
|
+
LLAMA_LOG_INFO("%s: - tensor split %2d: %32s %-8s [ %s ] %8.2f MiB\n", __func__,
|
607
|
+
sid, ggml_get_name(tensor), ggml_type_name(type), llama_format_tensor_shape(tensor).c_str(),
|
608
|
+
ggml_nbytes(tensor)/1024.0f/1024.0f);
|
609
|
+
}
|
610
|
+
}
|
611
|
+
|
612
|
+
switch (type_max) {
|
613
|
+
case GGML_TYPE_F32: ftype = LLAMA_FTYPE_ALL_F32; break;
|
614
|
+
case GGML_TYPE_F16: ftype = LLAMA_FTYPE_MOSTLY_F16; break;
|
615
|
+
case GGML_TYPE_BF16: ftype = LLAMA_FTYPE_MOSTLY_BF16; break;
|
616
|
+
case GGML_TYPE_Q4_0: ftype = LLAMA_FTYPE_MOSTLY_Q4_0; break;
|
617
|
+
case GGML_TYPE_Q4_1: ftype = LLAMA_FTYPE_MOSTLY_Q4_1; break;
|
618
|
+
case GGML_TYPE_Q5_0: ftype = LLAMA_FTYPE_MOSTLY_Q5_0; break;
|
619
|
+
case GGML_TYPE_Q5_1: ftype = LLAMA_FTYPE_MOSTLY_Q5_1; break;
|
620
|
+
case GGML_TYPE_Q8_0: ftype = LLAMA_FTYPE_MOSTLY_Q8_0; break;
|
621
|
+
case GGML_TYPE_Q2_K: ftype = LLAMA_FTYPE_MOSTLY_Q2_K; break;
|
622
|
+
case GGML_TYPE_Q3_K: ftype = LLAMA_FTYPE_MOSTLY_Q3_K_M; break;
|
623
|
+
case GGML_TYPE_Q4_K: ftype = LLAMA_FTYPE_MOSTLY_Q4_K_M; break;
|
624
|
+
case GGML_TYPE_Q5_K: ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M; break;
|
625
|
+
case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break;
|
626
|
+
case GGML_TYPE_TQ1_0: ftype = LLAMA_FTYPE_MOSTLY_TQ1_0; break;
|
627
|
+
case GGML_TYPE_TQ2_0: ftype = LLAMA_FTYPE_MOSTLY_TQ2_0; break;
|
628
|
+
case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break;
|
629
|
+
case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break;
|
630
|
+
case GGML_TYPE_IQ2_S: ftype = LLAMA_FTYPE_MOSTLY_IQ2_S; break;
|
631
|
+
case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break;
|
632
|
+
case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break;
|
633
|
+
case GGML_TYPE_IQ1_M: ftype = LLAMA_FTYPE_MOSTLY_IQ1_M; break;
|
634
|
+
case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break;
|
635
|
+
case GGML_TYPE_IQ4_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS; break;
|
636
|
+
case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break;
|
637
|
+
default:
|
638
|
+
{
|
639
|
+
LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));
|
640
|
+
ftype = LLAMA_FTYPE_ALL_F32;
|
641
|
+
} break;
|
642
|
+
}
|
643
|
+
|
644
|
+
// this is a way to mark that we have "guessed" the file type
|
645
|
+
ftype = (llama_ftype) (ftype | LLAMA_FTYPE_GUESSED);
|
646
|
+
|
647
|
+
{
|
648
|
+
uint32_t ftype_val = 0;
|
649
|
+
if (get_key(LLM_KV_GENERAL_FILE_TYPE, ftype_val, false)) {
|
650
|
+
ftype = (llama_ftype) ftype_val;
|
651
|
+
}
|
652
|
+
}
|
653
|
+
|
654
|
+
LLAMA_LOG_INFO("%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", __func__);
|
655
|
+
|
656
|
+
for (int i = 0; i < n_kv; i++) {
|
657
|
+
const char * name = gguf_get_key(meta.get(), i);
|
658
|
+
const enum gguf_type type = gguf_get_kv_type(meta.get(), i);
|
659
|
+
const std::string type_name =
|
660
|
+
type == GGUF_TYPE_ARRAY
|
661
|
+
? format("%s[%s,%zu]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(meta.get(), i)), gguf_get_arr_n(meta.get(), i))
|
662
|
+
: gguf_type_name(type);
|
663
|
+
|
664
|
+
std::string value = gguf_kv_to_str(meta.get(), i);
|
665
|
+
const size_t MAX_VALUE_LEN = 40;
|
666
|
+
if (value.size() > MAX_VALUE_LEN) {
|
667
|
+
value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str());
|
668
|
+
}
|
669
|
+
replace_all(value, "\n", "\\n");
|
670
|
+
|
671
|
+
LLAMA_LOG_INFO("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), value.c_str());
|
672
|
+
}
|
673
|
+
|
674
|
+
// print type counts
|
675
|
+
for (auto & kv : n_type) {
|
676
|
+
if (kv.second == 0) {
|
677
|
+
continue;
|
678
|
+
}
|
679
|
+
|
680
|
+
LLAMA_LOG_INFO("%s: - type %4s: %4d tensors\n", __func__, ggml_type_name(kv.first), kv.second);
|
681
|
+
}
|
682
|
+
}
|
683
|
+
|
684
|
+
if (!llama_mmap::SUPPORTED) {
|
685
|
+
LLAMA_LOG_WARN("%s: mmap is not supported on this platform\n", __func__);
|
686
|
+
use_mmap = false;
|
687
|
+
}
|
688
|
+
|
689
|
+
this->use_mmap = use_mmap;
|
690
|
+
this->check_tensors = check_tensors;
|
691
|
+
}
|
692
|
+
|
693
|
+
std::string llama_model_loader::get_arch_name() const {
|
694
|
+
return arch_name;
|
695
|
+
}
|
696
|
+
|
697
|
+
enum llm_arch llama_model_loader::get_arch() const {
|
698
|
+
return llm_kv.arch;
|
699
|
+
}
|
700
|
+
|
701
|
+
const llama_model_loader::llama_tensor_weight * llama_model_loader::get_weight(const char * name) const {
|
702
|
+
auto pos = weights_map.find(name);
|
703
|
+
if (pos != weights_map.end()) {
|
704
|
+
return &pos->second;
|
705
|
+
}
|
706
|
+
|
707
|
+
return nullptr;
|
708
|
+
}
|
709
|
+
|
710
|
+
const llama_model_loader::llama_tensor_weight & llama_model_loader::require_weight(const char * name) const {
|
711
|
+
const llama_tensor_weight * weight = get_weight(name);
|
712
|
+
if (!weight) {
|
713
|
+
throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name));
|
714
|
+
}
|
715
|
+
return *weight;
|
716
|
+
}
|
717
|
+
|
718
|
+
struct ggml_tensor * llama_model_loader::get_tensor_meta(const char * name) const {
|
719
|
+
const auto * weight = get_weight(name);
|
720
|
+
if (!weight) {
|
721
|
+
return nullptr;
|
722
|
+
}
|
723
|
+
return weight->tensor;
|
724
|
+
}
|
725
|
+
|
726
|
+
struct ggml_tensor * llama_model_loader::require_tensor_meta(const std::string & name) const {
|
727
|
+
struct ggml_tensor * tensor = get_tensor_meta(name.c_str());
|
728
|
+
if (!tensor) {
|
729
|
+
throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));
|
730
|
+
}
|
731
|
+
return tensor;
|
732
|
+
}
|
733
|
+
|
734
|
+
const struct ggml_tensor * llama_model_loader::check_tensor_dims(const std::string & name, const std::vector<int64_t> & ne, bool required) const {
|
735
|
+
const struct ggml_tensor * cur = get_tensor_meta(name.c_str());
|
736
|
+
|
737
|
+
if (cur == NULL) {
|
738
|
+
if (!required) {
|
739
|
+
return NULL;
|
740
|
+
}
|
741
|
+
throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));
|
742
|
+
}
|
743
|
+
|
744
|
+
{
|
745
|
+
bool is_ok = true;
|
746
|
+
for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {
|
747
|
+
if ((i < ne.size() && ne[i] != cur->ne[i]) || (i >= ne.size() && cur->ne[i] != 1)) {
|
748
|
+
is_ok = false;
|
749
|
+
break;
|
750
|
+
}
|
751
|
+
}
|
752
|
+
if (!is_ok) {
|
753
|
+
throw std::runtime_error(
|
754
|
+
format("%s: tensor '%s' has wrong shape; expected %s, got %s",
|
755
|
+
__func__, name.c_str(),
|
756
|
+
llama_format_tensor_shape(ne).c_str(),
|
757
|
+
llama_format_tensor_shape(cur).c_str()));
|
758
|
+
}
|
759
|
+
}
|
760
|
+
|
761
|
+
return cur;
|
762
|
+
}
|
763
|
+
|
764
|
+
struct ggml_tensor * llama_model_loader::create_tensor(struct ggml_context * ctx, const std::string & name, const std::initializer_list<int64_t> & ne, int flags) {
|
765
|
+
const struct ggml_tensor * cur = check_tensor_dims(name, ne, !(flags & TENSOR_NOT_REQUIRED));
|
766
|
+
|
767
|
+
if (cur == NULL) {
|
768
|
+
return NULL;
|
769
|
+
}
|
770
|
+
|
771
|
+
bool duplicated = flags & TENSOR_DUPLICATED;
|
772
|
+
|
773
|
+
struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur);
|
774
|
+
ggml_set_name(tensor, ggml_get_name(cur));
|
775
|
+
|
776
|
+
if (duplicated) {
|
777
|
+
size_data += ggml_nbytes(cur);
|
778
|
+
} else {
|
779
|
+
n_created++;
|
780
|
+
}
|
781
|
+
|
782
|
+
return tensor;
|
783
|
+
|
784
|
+
}
|
785
|
+
|
786
|
+
struct ggml_tensor * llama_model_loader::create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::initializer_list<int64_t> & ne, size_t offset, bool required) {
|
787
|
+
const struct ggml_tensor * cur = check_tensor_dims(name, ne, required);
|
788
|
+
|
789
|
+
if (cur == NULL) {
|
790
|
+
return NULL;
|
791
|
+
}
|
792
|
+
|
793
|
+
if (cur->type != base->type) {
|
794
|
+
throw std::runtime_error(format("%s: tensor '%s' has wrong type; expected %s, got %s", __func__, name.c_str(), ggml_type_name(base->type), ggml_type_name(cur->type)));
|
795
|
+
}
|
796
|
+
|
797
|
+
std::array<int64_t, GGML_MAX_DIMS> dims;
|
798
|
+
for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {
|
799
|
+
dims[i] = i < ne.size() ? ne.begin()[i] : 1;
|
800
|
+
}
|
801
|
+
|
802
|
+
struct ggml_tensor * tensor = ggml_view_4d(ctx, base,
|
803
|
+
dims[0], dims[1], dims[2], dims[3],
|
804
|
+
cur->nb[1], cur->nb[2], cur->nb[3],
|
805
|
+
offset);
|
806
|
+
|
807
|
+
ggml_set_name(tensor, name.c_str());
|
808
|
+
|
809
|
+
n_created++;
|
810
|
+
|
811
|
+
return tensor;
|
812
|
+
}
|
813
|
+
|
814
|
+
void llama_model_loader::done_getting_tensors() const {
|
815
|
+
if (n_created != n_tensors) {
|
816
|
+
throw std::runtime_error(format("%s: wrong number of tensors; expected %d, got %d", __func__, n_tensors, n_created));
|
817
|
+
}
|
818
|
+
}
|
819
|
+
|
820
|
+
void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps) {
|
821
|
+
if (use_mmap) {
|
822
|
+
mappings.reserve(files.size());
|
823
|
+
mmaps_used.reserve(files.size());
|
824
|
+
for (const auto & file : files) {
|
825
|
+
bool is_numa = false;
|
826
|
+
|
827
|
+
auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
|
828
|
+
if (dev) {
|
829
|
+
auto * reg = ggml_backend_dev_backend_reg(dev);
|
830
|
+
auto * is_numa_fn = (decltype(ggml_is_numa) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_is_numa");
|
831
|
+
if (is_numa_fn) {
|
832
|
+
is_numa = is_numa_fn();
|
833
|
+
}
|
834
|
+
}
|
835
|
+
|
836
|
+
std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa);
|
837
|
+
mmaps_used.emplace_back(mapping->size(), 0);
|
838
|
+
if (mlock_mmaps) {
|
839
|
+
std::unique_ptr<llama_mlock> mlock_mmap(new llama_mlock());
|
840
|
+
mlock_mmap->init(mapping->addr());
|
841
|
+
mlock_mmaps->emplace_back(std::move(mlock_mmap));
|
842
|
+
}
|
843
|
+
mappings.emplace_back(std::move(mapping));
|
844
|
+
}
|
845
|
+
}
|
846
|
+
|
847
|
+
// compute the total size of all tensors for progress reporting
|
848
|
+
for (const auto & it : weights_map) {
|
849
|
+
size_data += ggml_nbytes(it.second.tensor);
|
850
|
+
}
|
851
|
+
}
|
852
|
+
|
853
|
+
void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const {
|
854
|
+
GGML_ASSERT(!mappings.empty());
|
855
|
+
const auto & mapping = mappings.at(idx);
|
856
|
+
|
857
|
+
*first = mapping->size();
|
858
|
+
*last = 0;
|
859
|
+
*addr = mapping->addr();
|
860
|
+
for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) {
|
861
|
+
const auto * weight = get_weight(ggml_get_name(tensor));
|
862
|
+
if (!weight || weight->idx != idx) {
|
863
|
+
continue;
|
864
|
+
}
|
865
|
+
*first = std::min(*first, weight->offs);
|
866
|
+
*last = std::max(*last, weight->offs + ggml_nbytes(tensor));
|
867
|
+
}
|
868
|
+
}
|
869
|
+
|
870
|
+
void llama_model_loader::load_data_for(struct ggml_tensor * cur) const {
|
871
|
+
const auto & w = require_weight(ggml_get_name(cur));
|
872
|
+
|
873
|
+
if (use_mmap) {
|
874
|
+
const auto & mapping = mappings.at(w.idx);
|
875
|
+
if (cur->data == nullptr) {
|
876
|
+
cur->data = (uint8_t *)mapping->addr() + w.offs;
|
877
|
+
} else {
|
878
|
+
memcpy(cur->data, (uint8_t *)mapping->addr() + w.offs, ggml_nbytes(cur));
|
879
|
+
}
|
880
|
+
} else {
|
881
|
+
GGML_ASSERT(cur->data != nullptr);
|
882
|
+
GGML_ASSERT(w.idx < files.size());
|
883
|
+
const auto & file = files.at(w.idx);
|
884
|
+
file->seek(w.offs, SEEK_SET);
|
885
|
+
file->read_raw(cur->data, ggml_nbytes(cur));
|
886
|
+
}
|
887
|
+
|
888
|
+
if (check_tensors && !ggml_validate_row_data(cur->type, cur->data, ggml_nbytes(cur))) {
|
889
|
+
throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
|
890
|
+
}
|
891
|
+
}
|
892
|
+
|
893
|
+
bool llama_model_loader::load_all_data(
|
894
|
+
struct ggml_context * ctx,
|
895
|
+
llama_buf_map & bufs,
|
896
|
+
llama_mlocks * lmlocks,
|
897
|
+
llama_progress_callback progress_callback,
|
898
|
+
void * progress_callback_user_data) {
|
899
|
+
GGML_ASSERT(size_data != 0 && "call init_mappings() first");
|
900
|
+
|
901
|
+
std::vector<no_init<uint8_t>> read_buf;
|
902
|
+
std::vector<std::future<std::pair<ggml_tensor *, bool>>> validation_result;
|
903
|
+
|
904
|
+
// 4 staging buffers for async uploads, each sized 1MB seems to be a good default for single NVMe drives.
|
905
|
+
// NVMe raid configurations might require more / larger buffers.
|
906
|
+
constexpr size_t n_buffers = 4;
|
907
|
+
constexpr size_t buffer_size = 1 * 1024 * 1024; // 1MB
|
908
|
+
|
909
|
+
std::vector<ggml_backend_buffer_t> host_buffers;
|
910
|
+
std::vector<ggml_backend_event_t> events;
|
911
|
+
std::vector<void *> host_ptrs;
|
912
|
+
size_t buffer_idx = 0; // buffer to use for async loads
|
913
|
+
ggml_backend_t upload_backend = [&](const char * func) -> ggml_backend_t {
|
914
|
+
if (use_mmap || check_tensors) {
|
915
|
+
return nullptr;
|
916
|
+
}
|
917
|
+
// When not using mmaped io use async uploads from pinned memory to GPU memory.
|
918
|
+
// First determine if the backend supports the necessary features for async uploads.
|
919
|
+
auto * buf = bufs.count(0) ? bufs.at(0) : nullptr;
|
920
|
+
if (!buf) {
|
921
|
+
LLAMA_LOG_DEBUG("%s: no buffer found for async uploads\n", func);
|
922
|
+
return nullptr;
|
923
|
+
}
|
924
|
+
|
925
|
+
auto * buft = ggml_backend_buffer_get_type(buf);
|
926
|
+
auto * dev = ggml_backend_buft_get_device(buft);
|
927
|
+
if (!dev) {
|
928
|
+
LLAMA_LOG_DEBUG("%s: no device found for buffer type %s for async uploads\n", func,
|
929
|
+
ggml_backend_buft_name(buft));
|
930
|
+
return nullptr;
|
931
|
+
}
|
932
|
+
|
933
|
+
if (buft != ggml_backend_dev_buffer_type(dev)) {
|
934
|
+
LLAMA_LOG_DEBUG("%s: buffer type %s is not the default buffer type for device %s for async uploads\n", func,
|
935
|
+
ggml_backend_buft_name(buft), ggml_backend_dev_name(dev));
|
936
|
+
return nullptr;
|
937
|
+
}
|
938
|
+
|
939
|
+
ggml_backend_dev_props props;
|
940
|
+
ggml_backend_dev_get_props(dev, &props);
|
941
|
+
if (!props.caps.async || !props.caps.host_buffer || !props.caps.events) {
|
942
|
+
LLAMA_LOG_DEBUG("%s: device %s does not support async, host buffers or events\n", func,
|
943
|
+
ggml_backend_dev_name(dev));
|
944
|
+
return nullptr;
|
945
|
+
}
|
946
|
+
|
947
|
+
auto * host_buft = ggml_backend_dev_host_buffer_type(dev);
|
948
|
+
if (!host_buft) {
|
949
|
+
LLAMA_LOG_DEBUG("%s: no host buffer type found for device %s\n", func,
|
950
|
+
ggml_backend_dev_name(dev));
|
951
|
+
return nullptr;
|
952
|
+
}
|
953
|
+
|
954
|
+
// If the backend is supported, create pinned memory buffers and events for synchronisation.
|
955
|
+
for (size_t idx = 0; idx < n_buffers; ++idx) {
|
956
|
+
auto * buf = ggml_backend_buft_alloc_buffer(host_buft, buffer_size);
|
957
|
+
if (!buf) {
|
958
|
+
LLAMA_LOG_DEBUG("%s: failed to allocate host buffer for async uploads for device %s\n", func,
|
959
|
+
ggml_backend_dev_name(dev));
|
960
|
+
return nullptr;
|
961
|
+
}
|
962
|
+
|
963
|
+
host_buffers.emplace_back(buf);
|
964
|
+
host_ptrs.emplace_back(ggml_backend_buffer_get_base(buf));
|
965
|
+
|
966
|
+
auto * event = ggml_backend_event_new(dev);
|
967
|
+
if (!event) {
|
968
|
+
LLAMA_LOG_DEBUG("%s: failed to create event for async uploads for device %s\n", func,
|
969
|
+
ggml_backend_dev_name(dev));
|
970
|
+
return nullptr;
|
971
|
+
}
|
972
|
+
|
973
|
+
events.emplace_back(event);
|
974
|
+
}
|
975
|
+
|
976
|
+
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
|
977
|
+
if (!backend) {
|
978
|
+
LLAMA_LOG_DEBUG("%s: failed to initialize backend for device %s for async uploads\n", func,
|
979
|
+
ggml_backend_dev_name(dev));
|
980
|
+
return nullptr;
|
981
|
+
}
|
982
|
+
|
983
|
+
return backend;
|
984
|
+
}(__func__);
|
985
|
+
|
986
|
+
if (upload_backend) {
|
987
|
+
LLAMA_LOG_DEBUG("%s: using async uploads for device %s, buffer type %s, backend %s\n", __func__,
|
988
|
+
ggml_backend_dev_name(ggml_backend_get_device(upload_backend)),
|
989
|
+
ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.at(0))),
|
990
|
+
ggml_backend_name(upload_backend));
|
991
|
+
}
|
992
|
+
|
993
|
+
for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) {
|
994
|
+
const auto * weight = get_weight(ggml_get_name(cur));
|
995
|
+
if (weight == nullptr) {
|
996
|
+
// this can happen with split experts models
|
997
|
+
continue;
|
998
|
+
}
|
999
|
+
|
1000
|
+
if (progress_callback) {
|
1001
|
+
if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) {
|
1002
|
+
return false;
|
1003
|
+
}
|
1004
|
+
}
|
1005
|
+
|
1006
|
+
size_t n_size = ggml_nbytes(cur);
|
1007
|
+
|
1008
|
+
if (use_mmap) {
|
1009
|
+
const auto & mapping = mappings.at(weight->idx);
|
1010
|
+
ggml_backend_buffer_t buf_mmap = nullptr;
|
1011
|
+
if (bufs.count(weight->idx)) {
|
1012
|
+
buf_mmap = bufs.at(weight->idx);
|
1013
|
+
}
|
1014
|
+
uint8_t * data = (uint8_t *) mapping->addr() + weight->offs;
|
1015
|
+
|
1016
|
+
if (check_tensors) {
|
1017
|
+
validation_result.emplace_back(std::async(std::launch::async, [cur, data, n_size] {
|
1018
|
+
return std::make_pair(cur, ggml_validate_row_data(cur->type, data, n_size));
|
1019
|
+
}));
|
1020
|
+
}
|
1021
|
+
|
1022
|
+
GGML_ASSERT(buf_mmap || cur->data); // either we have a buffer to allocate the tensor in, or it is already allocated
|
1023
|
+
if (buf_mmap && cur->data == nullptr) {
|
1024
|
+
ggml_backend_tensor_alloc(buf_mmap, cur, data);
|
1025
|
+
if (lmlocks) {
|
1026
|
+
const auto & lmlock = lmlocks->at(weight->idx);
|
1027
|
+
lmlock->grow_to(weight->offs + n_size);
|
1028
|
+
}
|
1029
|
+
|
1030
|
+
auto & mmap_used = mmaps_used[weight->idx];
|
1031
|
+
mmap_used.first = std::min(mmap_used.first, weight->offs);
|
1032
|
+
mmap_used.second = std::max(mmap_used.second, weight->offs + n_size);
|
1033
|
+
} else {
|
1034
|
+
ggml_backend_tensor_set(cur, data, 0, n_size);
|
1035
|
+
}
|
1036
|
+
} else {
|
1037
|
+
const auto & file = files.at(weight->idx);
|
1038
|
+
if (ggml_backend_buffer_is_host(cur->buffer)) {
|
1039
|
+
file->seek(weight->offs, SEEK_SET);
|
1040
|
+
file->read_raw(cur->data, n_size);
|
1041
|
+
if (check_tensors) {
|
1042
|
+
validation_result.emplace_back(std::async(std::launch::async, [cur, n_size] {
|
1043
|
+
return std::make_pair(cur, ggml_validate_row_data(cur->type, cur->data, n_size));
|
1044
|
+
}));
|
1045
|
+
}
|
1046
|
+
} else {
|
1047
|
+
// If upload_backend is valid load the tensor in chunks to pinned memory and upload the buffers asynchronously to the GPU.
|
1048
|
+
if (upload_backend) {
|
1049
|
+
file->seek(weight->offs, SEEK_SET);
|
1050
|
+
|
1051
|
+
size_t bytes_read = 0;
|
1052
|
+
|
1053
|
+
while (bytes_read < n_size) {
|
1054
|
+
size_t read_iteration = std::min<size_t>(buffer_size, n_size - bytes_read);
|
1055
|
+
|
1056
|
+
ggml_backend_event_synchronize(events[buffer_idx]);
|
1057
|
+
file->read_raw(host_ptrs[buffer_idx], read_iteration);
|
1058
|
+
ggml_backend_tensor_set_async(upload_backend, cur, host_ptrs[buffer_idx], bytes_read, read_iteration);
|
1059
|
+
ggml_backend_event_record(events[buffer_idx], upload_backend);
|
1060
|
+
|
1061
|
+
bytes_read += read_iteration;
|
1062
|
+
++buffer_idx;
|
1063
|
+
buffer_idx %= n_buffers;
|
1064
|
+
}
|
1065
|
+
} else {
|
1066
|
+
read_buf.resize(n_size);
|
1067
|
+
file->seek(weight->offs, SEEK_SET);
|
1068
|
+
file->read_raw(read_buf.data(), n_size);
|
1069
|
+
ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size);
|
1070
|
+
if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) {
|
1071
|
+
throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
|
1072
|
+
}
|
1073
|
+
}
|
1074
|
+
}
|
1075
|
+
}
|
1076
|
+
|
1077
|
+
size_done += n_size;
|
1078
|
+
}
|
1079
|
+
|
1080
|
+
// free temporary resources used for async uploads
|
1081
|
+
for (auto * event : events) {
|
1082
|
+
ggml_backend_event_synchronize(event);
|
1083
|
+
ggml_backend_event_free(event);
|
1084
|
+
}
|
1085
|
+
for (auto * buf : host_buffers) {
|
1086
|
+
ggml_backend_buffer_free(buf);
|
1087
|
+
}
|
1088
|
+
ggml_backend_free(upload_backend);
|
1089
|
+
|
1090
|
+
// check validation results
|
1091
|
+
bool validation_failed = false;
|
1092
|
+
for (auto & future : validation_result) {
|
1093
|
+
auto result = future.get();
|
1094
|
+
if (!result.second) {
|
1095
|
+
LLAMA_LOG_ERROR("%s: tensor '%s' has invalid data\n", __func__, ggml_get_name(result.first));
|
1096
|
+
validation_failed = true;
|
1097
|
+
}
|
1098
|
+
}
|
1099
|
+
if (validation_failed) {
|
1100
|
+
throw std::runtime_error("found tensors with invalid data");
|
1101
|
+
}
|
1102
|
+
|
1103
|
+
// check if this is the last call and do final cleanup
|
1104
|
+
if (size_done >= size_data) {
|
1105
|
+
// unmap offloaded tensors and metadata
|
1106
|
+
if (use_mmap) {
|
1107
|
+
for (uint32_t idx = 0; idx < mappings.size(); idx++) {
|
1108
|
+
const auto & mmap_used = mmaps_used.at(idx);
|
1109
|
+
auto & mapping = mappings.at(idx);
|
1110
|
+
mapping->unmap_fragment(0, mmap_used.first);
|
1111
|
+
if (mmap_used.second != 0) {
|
1112
|
+
mapping->unmap_fragment(mmap_used.second, mapping->size());
|
1113
|
+
}
|
1114
|
+
}
|
1115
|
+
}
|
1116
|
+
if (progress_callback) {
|
1117
|
+
// Even though the model is done loading, we still honor
|
1118
|
+
// cancellation since we need to free allocations.
|
1119
|
+
return progress_callback(1.0f, progress_callback_user_data);
|
1120
|
+
}
|
1121
|
+
}
|
1122
|
+
|
1123
|
+
return true;
|
1124
|
+
}
|
1125
|
+
|
1126
|
+
std::string llama_model_loader::ftype_name() const {
|
1127
|
+
return llama_model_ftype_name(ftype);
|
1128
|
+
}
|
1129
|
+
|
1130
|
+
void llama_model_loader::print_info() const {
|
1131
|
+
LLAMA_LOG_INFO("%s: file format = %s\n", __func__, llama_file_version_name(fver));
|
1132
|
+
LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_model_ftype_name(ftype).c_str());
|
1133
|
+
if (n_bytes < GiB) {
|
1134
|
+
LLAMA_LOG_INFO("%s: file size = %.2f MiB (%.2f BPW) \n", __func__, n_bytes/1024.0/1024.0, n_bytes*8.0/n_elements);
|
1135
|
+
} else {
|
1136
|
+
LLAMA_LOG_INFO("%s: file size = %.2f GiB (%.2f BPW) \n", __func__, n_bytes/1024.0/1024.0/1024.0, n_bytes*8.0/n_elements);
|
1137
|
+
}
|
1138
|
+
}
|