whispercpp 1.3.1 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +4 -3
- data/README.md +92 -31
- data/Rakefile +26 -7
- data/ext/.gitignore +5 -7
- data/ext/dependencies.rb +61 -0
- data/ext/extconf.rb +21 -198
- data/ext/options.rb +221 -0
- data/ext/ruby_whisper.c +159 -0
- data/ext/ruby_whisper.h +17 -2
- data/ext/ruby_whisper_context.c +641 -0
- data/ext/ruby_whisper_error.c +52 -0
- data/ext/ruby_whisper_model.c +232 -0
- data/ext/ruby_whisper_params.c +1301 -0
- data/ext/ruby_whisper_segment.c +143 -0
- data/ext/ruby_whisper_transcribe.cpp +87 -0
- data/ext/ruby_whisper_vad_params.c +288 -0
- data/ext/sources/.dockerignore +3 -0
- data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
- data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
- data/ext/sources/CMakeLists.txt +251 -0
- data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
- data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
- data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
- data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
- data/ext/sources/bindings/javascript/package.json +26 -0
- data/ext/sources/bindings/javascript/whisper.js +19 -0
- data/ext/sources/build-xcframework.sh +547 -0
- data/ext/sources/ci/run.sh +336 -0
- data/ext/sources/close-issue.yml +28 -0
- data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
- data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
- data/ext/sources/cmake/build-info.cmake +60 -0
- data/ext/sources/cmake/git-vars.cmake +22 -0
- data/ext/sources/cmake/whisper-config.cmake.in +65 -0
- data/ext/sources/cmake/whisper.pc.in +10 -0
- data/ext/sources/examples/CMakeLists.txt +124 -0
- data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
- data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
- data/ext/sources/examples/addon.node/addon.cpp +438 -0
- data/ext/sources/examples/addon.node/index.js +54 -0
- data/ext/sources/examples/addon.node/package.json +16 -0
- data/ext/sources/examples/bench/CMakeLists.txt +8 -0
- data/ext/sources/examples/bench/bench.cpp +175 -0
- data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
- data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
- data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
- data/ext/sources/examples/cli/CMakeLists.txt +8 -0
- data/ext/sources/examples/cli/cli.cpp +1294 -0
- data/ext/sources/examples/coi-serviceworker.js +146 -0
- data/ext/sources/examples/command/CMakeLists.txt +10 -0
- data/ext/sources/examples/command/command.cpp +776 -0
- data/ext/sources/examples/command/commands.txt +9 -0
- data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
- data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
- data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
- data/ext/sources/examples/common-ggml.cpp +238 -0
- data/ext/sources/examples/common-ggml.h +18 -0
- data/ext/sources/examples/common-sdl.cpp +227 -0
- data/ext/sources/examples/common-sdl.h +49 -0
- data/ext/sources/examples/common-whisper.cpp +168 -0
- data/ext/sources/examples/common-whisper.h +24 -0
- data/ext/sources/examples/common.cpp +675 -0
- data/ext/sources/examples/common.h +322 -0
- data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
- data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
- data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
- data/ext/sources/examples/generate-karaoke.sh +57 -0
- data/ext/sources/examples/grammar-parser.cpp +423 -0
- data/ext/sources/examples/grammar-parser.h +29 -0
- data/ext/sources/examples/helpers.js +191 -0
- data/ext/sources/examples/json.hpp +24596 -0
- data/ext/sources/examples/livestream.sh +112 -0
- data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
- data/ext/sources/examples/lsp/lsp.cpp +467 -0
- data/ext/sources/examples/lsp/whisper.vim +362 -0
- data/ext/sources/examples/miniaudio.h +93468 -0
- data/ext/sources/examples/python/test_whisper_processor.py +7 -0
- data/ext/sources/examples/python/whisper_processor.py +54 -0
- data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
- data/ext/sources/examples/quantize/quantize.cpp +223 -0
- data/ext/sources/examples/server/CMakeLists.txt +12 -0
- data/ext/sources/examples/server/bench.js +29 -0
- data/ext/sources/examples/server/httplib.h +10497 -0
- data/ext/sources/examples/server/server.cpp +1091 -0
- data/ext/sources/examples/server.py +115 -0
- data/ext/sources/examples/stb_vorbis.c +5584 -0
- data/ext/sources/examples/stream/CMakeLists.txt +10 -0
- data/ext/sources/examples/stream/stream.cpp +429 -0
- data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
- data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
- data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
- data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
- data/ext/sources/examples/sycl/build.sh +22 -0
- data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
- data/ext/sources/examples/sycl/run-whisper.sh +17 -0
- data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
- data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
- data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
- data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
- data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
- data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
- data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
- data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
- data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
- data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
- data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
- data/ext/sources/examples/talk-llama/llama-context.h +276 -0
- data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
- data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
- data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
- data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
- data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
- data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
- data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
- data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
- data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
- data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
- data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
- data/ext/sources/examples/talk-llama/llama-io.h +35 -0
- data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
- data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
- data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
- data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
- data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
- data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
- data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
- data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
- data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
- data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
- data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
- data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
- data/ext/sources/examples/talk-llama/llama-model.h +425 -0
- data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
- data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
- data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
- data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
- data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
- data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
- data/ext/sources/examples/talk-llama/llama.cpp +354 -0
- data/ext/sources/examples/talk-llama/llama.h +1377 -0
- data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
- data/ext/sources/examples/talk-llama/speak +40 -0
- data/ext/sources/examples/talk-llama/speak.bat +1 -0
- data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
- data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
- data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
- data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
- data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
- data/ext/sources/examples/talk-llama/unicode.h +66 -0
- data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
- data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
- data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
- data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
- data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
- data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
- data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
- data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
- data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
- data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
- data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
- data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
- data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
- data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
- data/ext/sources/ggml/CMakeLists.txt +390 -0
- data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
- data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
- data/ext/sources/ggml/cmake/common.cmake +26 -0
- data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
- data/ext/{ggml → sources/ggml}/include/ggml-alloc.h +1 -1
- data/ext/{ggml → sources/ggml}/include/ggml-backend.h +9 -7
- data/ext/{ggml → sources/ggml}/include/ggml-cpp.h +2 -1
- data/ext/{ggml → sources/ggml}/include/ggml-cpu.h +9 -1
- data/ext/{ggml → sources/ggml}/include/ggml-metal.h +1 -1
- data/ext/{ggml → sources/ggml}/include/ggml-opt.h +49 -28
- data/ext/{ggml → sources/ggml}/include/ggml-rpc.h +6 -1
- data/ext/{ggml → sources/ggml}/include/ggml-vulkan.h +0 -2
- data/ext/{ggml → sources/ggml}/include/ggml.h +182 -265
- data/ext/sources/ggml/include/gguf.h +202 -0
- data/ext/sources/ggml/src/CMakeLists.txt +346 -0
- data/ext/{ggml → sources/ggml}/src/ggml-alloc.c +34 -29
- data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
- data/ext/{ggml → sources/ggml}/src/ggml-backend-impl.h +1 -2
- data/ext/{ggml → sources/ggml}/src/ggml-backend-reg.cpp +87 -53
- data/ext/{ggml → sources/ggml}/src/ggml-backend.cpp +26 -14
- data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
- data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
- data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.cpp +10 -4
- data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.h +5 -5
- data/ext/{ggml → sources/ggml}/src/ggml-cann/aclnn_ops.cpp +1272 -1506
- data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/common.h +135 -1
- data/ext/{ggml → sources/ggml}/src/ggml-cann/ggml-cann.cpp +564 -146
- data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/dup.cpp +3 -5
- data/ext/{ggml → sources/ggml}/src/ggml-common.h +12 -8
- data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.cpp +2 -1
- data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
- data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/cpu-feats-x86.cpp +5 -1
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-impl.h +163 -41
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.c +4029 -1117
- data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu.cpp +67 -18
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
- data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
- data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
- data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
- data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
- data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
- data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
- data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
- data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
- data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
- data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
- data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
- data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
- data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
- data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
- data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
- data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
- data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
- data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
- data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
- data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
- data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
- data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
- data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
- data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
- data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
- data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
- data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
- data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
- data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
- data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
- data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
- data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
- data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
- data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
- data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
- data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
- data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
- data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
- data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
- data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
- data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
- data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
- data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
- data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
- data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
- data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
- data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
- data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
- data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
- data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
- data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
- data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/cuda.h +1 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/hip.h +57 -0
- data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/musa.h +7 -1
- data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
- data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
- data/ext/{ggml → sources/ggml}/src/ggml-impl.h +64 -19
- data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
- data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
- data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
- data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
- data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.m +2178 -1064
- data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.metal +1575 -1218
- data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
- data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
- data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
- data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
- data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
- data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
- data/ext/{ggml → sources/ggml}/src/ggml-opt.cpp +373 -190
- data/ext/{ggml → sources/ggml}/src/ggml-quants.c +114 -120
- data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
- data/ext/{ggml → sources/ggml}/src/ggml-rpc/ggml-rpc.cpp +480 -73
- data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
- data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
- data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
- data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/common.cpp +20 -32
- data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/concat.cpp +32 -33
- data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/conv.cpp +4 -2
- data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/convert.cpp +104 -28
- data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
- data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
- data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
- data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/dmmv.cpp +156 -17
- data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
- data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
- data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
- data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
- data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
- data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/ggml-sycl.cpp +1004 -1240
- data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
- data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
- data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
- data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmq.cpp +0 -1
- data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmvq.cpp +261 -166
- data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/norm.cpp +204 -81
- data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/outprod.cpp +8 -17
- data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
- data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
- data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
- data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
- data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/softmax.cpp +35 -25
- data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
- data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
- data/ext/{ggml → sources/ggml}/src/ggml-sycl/tsembd.cpp +3 -3
- data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
- data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
- data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
- data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
- data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- data/ext/{ggml → sources/ggml}/src/ggml-vulkan/ggml-vulkan.cpp +3130 -1087
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
- data/ext/{ggml → sources/ggml}/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +193 -35
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- data/ext/{ggml → sources/ggml}/src/ggml.c +676 -1820
- data/ext/sources/ggml/src/gguf.cpp +1330 -0
- data/ext/{include → sources/include}/whisper.h +68 -2
- data/ext/sources/src/CMakeLists.txt +143 -0
- data/ext/{src → sources/src}/coreml/whisper-decoder-impl.h +27 -15
- data/ext/{src → sources/src}/coreml/whisper-decoder-impl.m +35 -10
- data/ext/{src → sources/src}/coreml/whisper-encoder-impl.h +21 -9
- data/ext/{src → sources/src}/coreml/whisper-encoder-impl.m +28 -3
- data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
- data/ext/sources/src/whisper-arch.h +197 -0
- data/ext/{src → sources/src}/whisper.cpp +1905 -374
- data/ext/sources/tests/CMakeLists.txt +105 -0
- data/ext/sources/tests/earnings21/eval.mk +58 -0
- data/ext/sources/tests/earnings21/eval.py +68 -0
- data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
- data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
- data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
- data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
- data/ext/sources/tests/earnings21/requirements.txt +6 -0
- data/ext/sources/tests/en-0-ref.txt +1 -0
- data/ext/sources/tests/en-1-ref.txt +1 -0
- data/ext/sources/tests/en-2-ref.txt +1 -0
- data/ext/sources/tests/es-0-ref.txt +1 -0
- data/ext/sources/tests/librispeech/eval.mk +39 -0
- data/ext/sources/tests/librispeech/eval.py +47 -0
- data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
- data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
- data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
- data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
- data/ext/sources/tests/librispeech/requirements.txt +6 -0
- data/ext/sources/tests/run-tests.sh +130 -0
- data/ext/sources/tests/test-c.c +3 -0
- data/ext/sources/tests/test-vad-full.cpp +54 -0
- data/ext/sources/tests/test-vad.cpp +83 -0
- data/ext/sources/tests/test-whisper.js +58 -0
- data/extsources.rb +33 -5
- data/lib/whisper/model/uri.rb +149 -128
- data/sig/whisper.rbs +480 -0
- data/tests/helper.rb +28 -0
- data/tests/test_callback.rb +45 -3
- data/tests/test_error.rb +2 -2
- data/tests/test_model.rb +38 -0
- data/tests/test_package.rb +18 -3
- data/tests/test_params.rb +145 -8
- data/tests/test_segment.rb +10 -19
- data/tests/test_vad.rb +19 -0
- data/tests/test_vad_params.rb +103 -0
- data/tests/test_whisper.rb +37 -37
- data/whispercpp.gemspec +5 -4
- metadata +766 -111
- data/ext/cpu.mk +0 -9
- data/ext/examples/dr_wav.h +0 -8815
- data/ext/ggml/src/ggml-cann/aclnn_ops.h +0 -592
- data/ext/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +0 -4262
- data/ext/ggml/src/ggml-cpu/ggml-cpu.c +0 -14123
- data/ext/ggml/src/ggml-cpu/llamafile/sgemm.cpp +0 -1884
- data/ext/ggml/src/ggml-cpu/llamafile/sgemm.h +0 -14
- data/ext/ggml/src/ggml-metal/ggml-metal-impl.h +0 -288
- data/ext/ggml/src/ggml-sycl/element_wise.cpp +0 -1030
- data/ext/ggml/src/ggml-sycl/im2col.cpp +0 -126
- data/ext/ggml/src/ggml-sycl/rope.cpp +0 -276
- data/ext/ggml/src/ggml-sycl/wkv6.cpp +0 -141
- data/ext/metal-embed.mk +0 -17
- data/ext/metal.mk +0 -6
- data/ext/ruby_whisper.cpp +0 -1909
- data/ext/scripts/get-flags.mk +0 -38
- data/lib/whisper.rb +0 -2
- /data/ext/{ggml → sources/ggml}/include/ggml-blas.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-cann.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-cuda.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-kompute.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-opencl.h +0 -0
- /data/ext/{ggml → sources/ggml}/include/ggml-sycl.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/common.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/ggml-amx.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-blas/ggml-blas.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/ascendc_kernels.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f16.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f32.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q4_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/common.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-aarch64.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-kompute/ggml-kompute.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-quants.h +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-threading.cpp +0 -0
- /data/ext/{ggml → sources/ggml}/src/ggml-threading.h +0 -0
- /data/ext/{src → sources/src}/coreml/whisper-encoder.h +0 -0
- /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.cpp +0 -0
- /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.h +0 -0
@@ -0,0 +1,1135 @@
|
|
1
|
+
#pragma once
|
2
|
+
|
3
|
+
#include "common.cuh"
|
4
|
+
#include <cstdint>
|
5
|
+
|
6
|
+
static __device__ __forceinline__ int get_int_b2(const void * x, const int & i32) {
|
7
|
+
const uint16_t * x16 = (const uint16_t *) x; // assume at least 2 byte alignment
|
8
|
+
|
9
|
+
int x32 = x16[2*i32 + 0] << 0;
|
10
|
+
x32 |= x16[2*i32 + 1] << 16;
|
11
|
+
|
12
|
+
return x32;
|
13
|
+
}
|
14
|
+
|
15
|
+
static __device__ __forceinline__ int get_int_b4(const void * x, const int & i32) {
|
16
|
+
return ((const int *) x)[i32]; // assume at least 4 byte alignment
|
17
|
+
}
|
18
|
+
|
19
|
+
// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called
|
20
|
+
// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q
|
21
|
+
|
22
|
+
#define VDR_Q4_0_Q8_1_MMVQ 2
|
23
|
+
#define VDR_Q4_0_Q8_1_MMQ 4
|
24
|
+
|
25
|
+
template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(
|
26
|
+
const int * v, const int * u, const float & d4, const half2 & ds8) {
|
27
|
+
|
28
|
+
int sumi = 0;
|
29
|
+
|
30
|
+
#pragma unroll
|
31
|
+
for (int i = 0; i < vdr; ++i) {
|
32
|
+
const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
|
33
|
+
const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
|
34
|
+
|
35
|
+
// SIMD dot product of quantized values
|
36
|
+
sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);
|
37
|
+
sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);
|
38
|
+
}
|
39
|
+
|
40
|
+
const float2 ds8f = __half22float2(ds8);
|
41
|
+
|
42
|
+
// second part effectively subtracts 8 from each quant value
|
43
|
+
return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);
|
44
|
+
}
|
45
|
+
|
46
|
+
#define VDR_Q4_1_Q8_1_MMVQ 2
|
47
|
+
#define VDR_Q4_1_Q8_1_MMQ 4
|
48
|
+
|
49
|
+
template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(
|
50
|
+
const int * v, const int * u, const half2 & dm4, const half2 & ds8) {
|
51
|
+
|
52
|
+
int sumi = 0;
|
53
|
+
|
54
|
+
#pragma unroll
|
55
|
+
for (int i = 0; i < vdr; ++i) {
|
56
|
+
const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
|
57
|
+
const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
|
58
|
+
|
59
|
+
// SIMD dot product of quantized values
|
60
|
+
sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);
|
61
|
+
sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);
|
62
|
+
}
|
63
|
+
|
64
|
+
#ifdef GGML_CUDA_F16
|
65
|
+
const float2 tmp = __half22float2(__hmul2(dm4, ds8));
|
66
|
+
const float d4d8 = tmp.x;
|
67
|
+
const float m4s8 = tmp.y;
|
68
|
+
#else
|
69
|
+
const float2 dm4f = __half22float2(dm4);
|
70
|
+
const float2 ds8f = __half22float2(ds8);
|
71
|
+
const float d4d8 = dm4f.x * ds8f.x;
|
72
|
+
const float m4s8 = dm4f.y * ds8f.y;
|
73
|
+
#endif // GGML_CUDA_F16
|
74
|
+
|
75
|
+
// scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it
|
76
|
+
return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));
|
77
|
+
}
|
78
|
+
|
79
|
+
#define VDR_Q5_0_Q8_1_MMVQ 2
|
80
|
+
#define VDR_Q5_0_Q8_1_MMQ 4
|
81
|
+
|
82
|
+
template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(
|
83
|
+
const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {
|
84
|
+
|
85
|
+
int sumi = 0;
|
86
|
+
|
87
|
+
#pragma unroll
|
88
|
+
for (int i = 0; i < vdr; ++i) {
|
89
|
+
int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
|
90
|
+
vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
|
91
|
+
vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
|
92
|
+
vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
|
93
|
+
vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
|
94
|
+
sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
|
95
|
+
|
96
|
+
int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
|
97
|
+
vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
|
98
|
+
vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
|
99
|
+
vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
|
100
|
+
vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
|
101
|
+
sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
|
102
|
+
}
|
103
|
+
|
104
|
+
const float2 ds8f = __half22float2(ds8);
|
105
|
+
|
106
|
+
// second part effectively subtracts 16 from each quant value
|
107
|
+
return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);
|
108
|
+
}
|
109
|
+
|
110
|
+
#define VDR_Q5_1_Q8_1_MMVQ 2
|
111
|
+
#define VDR_Q5_1_Q8_1_MMQ 4
|
112
|
+
|
113
|
+
template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(
|
114
|
+
const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {
|
115
|
+
|
116
|
+
int sumi = 0;
|
117
|
+
|
118
|
+
#pragma unroll
|
119
|
+
for (int i = 0; i < vdr; ++i) {
|
120
|
+
int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
|
121
|
+
vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
|
122
|
+
vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
|
123
|
+
vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
|
124
|
+
vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
|
125
|
+
sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
|
126
|
+
|
127
|
+
int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
|
128
|
+
vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
|
129
|
+
vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
|
130
|
+
vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
|
131
|
+
vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
|
132
|
+
sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
|
133
|
+
}
|
134
|
+
|
135
|
+
#ifdef GGML_CUDA_F16
|
136
|
+
const float2 tmp = __half22float2(__hmul2(dm5, ds8));
|
137
|
+
const float d5d8 = tmp.x;
|
138
|
+
const float m5s8 = tmp.y;
|
139
|
+
#else
|
140
|
+
const float2 dm5f = __half22float2(dm5);
|
141
|
+
const float2 ds8f = __half22float2(ds8);
|
142
|
+
const float d5d8 = dm5f.x * ds8f.x;
|
143
|
+
const float m5s8 = dm5f.y * ds8f.y;
|
144
|
+
#endif // GGML_CUDA_F16
|
145
|
+
|
146
|
+
// scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it
|
147
|
+
return sumi*d5d8 + m5s8 / (QI5_1 / vdr);
|
148
|
+
}
|
149
|
+
|
150
|
+
#define VDR_Q8_0_Q8_1_MMVQ 2
|
151
|
+
#define VDR_Q8_0_Q8_1_MMQ 8
|
152
|
+
|
153
|
+
template <typename T, int vdr> static __device__ __forceinline__ T vec_dot_q8_0_q8_1_impl(
|
154
|
+
const int * v, const int * u, const T & d8_0, const T & d8_1) {
|
155
|
+
|
156
|
+
int sumi = 0;
|
157
|
+
|
158
|
+
#pragma unroll
|
159
|
+
for (int i = 0; i < vdr; ++i) {
|
160
|
+
// SIMD dot product of quantized values
|
161
|
+
sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
|
162
|
+
}
|
163
|
+
|
164
|
+
return d8_0*d8_1 * ((T) sumi);
|
165
|
+
}
|
166
|
+
|
167
|
+
template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(
|
168
|
+
const int * v, const int * u, const half2 & dm8, const half2 & ds8) {
|
169
|
+
|
170
|
+
int sumi = 0;
|
171
|
+
|
172
|
+
#pragma unroll
|
173
|
+
for (int i = 0; i < vdr; ++i) {
|
174
|
+
// SIMD dot product of quantized values
|
175
|
+
sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
|
176
|
+
}
|
177
|
+
|
178
|
+
#ifdef GGML_CUDA_F16
|
179
|
+
const float2 tmp = __half22float2(__hmul2(dm8, ds8));
|
180
|
+
const float d8d8 = tmp.x;
|
181
|
+
const float m8s8 = tmp.y;
|
182
|
+
#else
|
183
|
+
const float2 dm8f = __half22float2(dm8);
|
184
|
+
const float2 ds8f = __half22float2(ds8);
|
185
|
+
const float d8d8 = dm8f.x * ds8f.x;
|
186
|
+
const float m8s8 = dm8f.y * ds8f.y;
|
187
|
+
#endif // GGML_CUDA_F16
|
188
|
+
|
189
|
+
// scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it
|
190
|
+
return sumi*d8d8 + m8s8 / (QI8_1 / vdr);
|
191
|
+
}
|
192
|
+
|
193
|
+
template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_16_q8_1_impl(
|
194
|
+
const int * v, const int * u, const float * d8_0, const float & d8_1) {
|
195
|
+
|
196
|
+
float sumf = 0.0f;
|
197
|
+
|
198
|
+
#pragma unroll
|
199
|
+
for (int i0 = 0; i0 < vdr; i0 += QI8_0/2) {
|
200
|
+
int sumi = 0;
|
201
|
+
|
202
|
+
#pragma unroll
|
203
|
+
for (int i = i0; i < i0 + QI8_0/2; ++i) {
|
204
|
+
// SIMD dot product of quantized values
|
205
|
+
sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
|
206
|
+
}
|
207
|
+
|
208
|
+
sumf += d8_0[i0/(QI8_0/2)]*sumi;
|
209
|
+
}
|
210
|
+
|
211
|
+
return d8_1*sumf;
|
212
|
+
}
|
213
|
+
|
214
|
+
#define VDR_Q2_K_Q8_1_MMVQ 1
|
215
|
+
#define VDR_Q2_K_Q8_1_MMQ 4
|
216
|
+
|
217
|
+
// contiguous v/x values
|
218
|
+
static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
|
219
|
+
const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
|
220
|
+
const half2 & dm2, const float * __restrict__ d8) {
|
221
|
+
|
222
|
+
float sumf_d = 0.0f;
|
223
|
+
float sumf_m = 0.0f;
|
224
|
+
|
225
|
+
#pragma unroll
|
226
|
+
for (int i = 0; i < QR2_K; ++i) {
|
227
|
+
const int sc = scales[2*i];
|
228
|
+
|
229
|
+
const int vi = (v >> (2*i)) & 0x03030303;
|
230
|
+
|
231
|
+
sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product
|
232
|
+
|
233
|
+
// fill int with 4x m
|
234
|
+
int m = sc >> 4;
|
235
|
+
m |= m << 8;
|
236
|
+
m |= m << 16;
|
237
|
+
sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values
|
238
|
+
}
|
239
|
+
|
240
|
+
const float2 dm2f = __half22float2(dm2);
|
241
|
+
|
242
|
+
return dm2f.x*sumf_d - dm2f.y*sumf_m;
|
243
|
+
}
|
244
|
+
|
245
|
+
// contiguous v/x + u/y values
|
246
|
+
template <int ns8>
|
247
|
+
static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
|
248
|
+
const int * __restrict__ v, const int * __restrict__ u, const half2 * dm2, const float & d8, const half2 * s8) {
|
249
|
+
|
250
|
+
float sumf = 0.0f;
|
251
|
+
float sumf_d8 = 0.0f;
|
252
|
+
|
253
|
+
#pragma unroll
|
254
|
+
for (int i0 = 0; i0 < QR2_K*VDR_Q2_K_Q8_1_MMQ; i0 += QI8_1) {
|
255
|
+
const float2 dm2f0 = __half22float2(dm2[i0/(QI8_1/2) + 0]);
|
256
|
+
int sumi_d0 = 0;
|
257
|
+
|
258
|
+
const float2 dm2f1 = __half22float2(dm2[i0/(QI8_1/2) + 1]);
|
259
|
+
int sumi_d1 = 0;
|
260
|
+
|
261
|
+
#pragma unroll
|
262
|
+
for (int i = i0; i < i0 + QI8_1/2; ++i) {
|
263
|
+
sumi_d0 = ggml_cuda_dp4a(v[i], u[i], sumi_d0);
|
264
|
+
}
|
265
|
+
sumf_d8 += dm2f0.x * sumi_d0;
|
266
|
+
|
267
|
+
#pragma unroll
|
268
|
+
for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {
|
269
|
+
sumi_d1 = ggml_cuda_dp4a(v[i], u[i], sumi_d1);
|
270
|
+
}
|
271
|
+
sumf_d8 += dm2f1.x * sumi_d1;
|
272
|
+
|
273
|
+
if (i0/QI8_1 < ns8) {
|
274
|
+
const float2 s8f = __half22float2(s8[i0/QI8_1]);
|
275
|
+
sumf -= dm2f0.y*s8f.x;
|
276
|
+
sumf -= dm2f1.y*s8f.y;
|
277
|
+
} else {
|
278
|
+
int sumi_m0 = 0;
|
279
|
+
#pragma unroll
|
280
|
+
for (int i = i0; i < i0 + QI8_1/2; ++i) {
|
281
|
+
sumi_m0 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m0);
|
282
|
+
}
|
283
|
+
sumf_d8 -= dm2f0.y * sumi_m0;
|
284
|
+
|
285
|
+
int sumi_m1 = 0;
|
286
|
+
#pragma unroll
|
287
|
+
for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {
|
288
|
+
sumi_m1 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m1);
|
289
|
+
}
|
290
|
+
sumf_d8 -= dm2f1.y * sumi_m1;
|
291
|
+
}
|
292
|
+
}
|
293
|
+
|
294
|
+
return sumf + d8*sumf_d8;
|
295
|
+
}
|
296
|
+
|
297
|
+
#define VDR_Q3_K_Q8_1_MMVQ 1
|
298
|
+
#define VDR_Q3_K_Q8_1_MMQ 2
|
299
|
+
|
300
|
+
// contiguous v/x values
|
301
|
+
static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
|
302
|
+
const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,
|
303
|
+
const int & scale_offset, const float & d3, const float * __restrict__ d8) {
|
304
|
+
|
305
|
+
float sumf = 0.0f;
|
306
|
+
|
307
|
+
#pragma unroll
|
308
|
+
for (int i = 0; i < QR3_K; ++i) {
|
309
|
+
const int isc = scale_offset + 2*i;
|
310
|
+
|
311
|
+
const int isc_low = isc % (QK_K/32);
|
312
|
+
const int sc_shift_low = 4 * (isc / (QK_K/32));
|
313
|
+
const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;
|
314
|
+
|
315
|
+
const int isc_high = isc % (QK_K/64);
|
316
|
+
const int sc_shift_high = 2 * (isc / (QK_K/64));
|
317
|
+
const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;
|
318
|
+
|
319
|
+
const int sc = (sc_low | sc_high) - 32;
|
320
|
+
|
321
|
+
const int vil = (vl >> (2*i)) & 0x03030303;
|
322
|
+
|
323
|
+
const int vih = ((vh >> i) << 2) & 0x04040404;
|
324
|
+
|
325
|
+
const int vi = __vsubss4(vil, vih);
|
326
|
+
|
327
|
+
sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
|
328
|
+
}
|
329
|
+
|
330
|
+
return d3 * sumf;
|
331
|
+
}
|
332
|
+
|
333
|
+
// contiguous v/x + u/y values
|
334
|
+
static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
|
335
|
+
const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,
|
336
|
+
const float & d3, const float & d8) {
|
337
|
+
|
338
|
+
int sumi = 0;
|
339
|
+
|
340
|
+
#pragma unroll
|
341
|
+
for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {
|
342
|
+
int sumi_sc = 0;
|
343
|
+
|
344
|
+
#pragma unroll
|
345
|
+
for (int i = i0; i < i0 + QI8_1/2; ++i) {
|
346
|
+
sumi_sc = ggml_cuda_dp4a(v[i], u[i], sumi_sc); // SIMD dot product
|
347
|
+
}
|
348
|
+
|
349
|
+
sumi += sumi_sc * scales[i0 / (QI8_1/2)];
|
350
|
+
}
|
351
|
+
|
352
|
+
return d3*d8 * sumi;
|
353
|
+
}
|
354
|
+
|
355
|
+
#define VDR_Q4_K_Q8_1_MMVQ 2
|
356
|
+
#define VDR_Q4_K_Q8_1_MMQ 8
|
357
|
+
|
358
|
+
// contiguous v/x values
|
359
|
+
static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
|
360
|
+
const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
|
361
|
+
const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {
|
362
|
+
|
363
|
+
float sumf_d = 0.0f;
|
364
|
+
float sumf_m = 0.0f;
|
365
|
+
|
366
|
+
#pragma unroll
|
367
|
+
for (int i = 0; i < QR4_K; ++i) {
|
368
|
+
const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;
|
369
|
+
const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;
|
370
|
+
|
371
|
+
const int dot1 = ggml_cuda_dp4a(v1i, u[2*i+1], ggml_cuda_dp4a(v0i, u[2*i+0], 0)); // SIMD dot product
|
372
|
+
const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+1], ggml_cuda_dp4a(0x01010101, u[2*i+0], 0)); // sum of u
|
373
|
+
|
374
|
+
sumf_d += d8[i] * (dot1 * sc[i]);
|
375
|
+
sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values
|
376
|
+
}
|
377
|
+
|
378
|
+
const float2 dm4f = __half22float2(dm4);
|
379
|
+
|
380
|
+
return dm4f.x*sumf_d - dm4f.y*sumf_m;
|
381
|
+
}
|
382
|
+
|
383
|
+
// contiguous v/x + u/y values
|
384
|
+
static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
|
385
|
+
const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
|
386
|
+
const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
|
387
|
+
|
388
|
+
float sumf_d = 0.0f;
|
389
|
+
float sumf_m = 0.0f;
|
390
|
+
|
391
|
+
#pragma unroll
|
392
|
+
for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {
|
393
|
+
int sumi_d = 0;
|
394
|
+
|
395
|
+
#pragma unroll
|
396
|
+
for (int j = 0; j < QI8_1; ++j) {
|
397
|
+
sumi_d = ggml_cuda_dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product
|
398
|
+
}
|
399
|
+
|
400
|
+
const float2 ds8f = __half22float2(ds8[i]);
|
401
|
+
|
402
|
+
sumf_d += ds8f.x * (sc[i] * sumi_d);
|
403
|
+
sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
|
404
|
+
}
|
405
|
+
|
406
|
+
const float2 dm4f = __half22float2(dm4);
|
407
|
+
|
408
|
+
return dm4f.x*sumf_d - dm4f.y*sumf_m;
|
409
|
+
}
|
410
|
+
|
411
|
+
#define VDR_Q5_K_Q8_1_MMVQ 2
|
412
|
+
#define VDR_Q5_K_Q8_1_MMQ 8
|
413
|
+
|
414
|
+
// contiguous v/x values
|
415
|
+
static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
|
416
|
+
const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,
|
417
|
+
const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {
|
418
|
+
|
419
|
+
float sumf_d = 0.0f;
|
420
|
+
float sumf_m = 0.0f;
|
421
|
+
|
422
|
+
#pragma unroll
|
423
|
+
for (int i = 0; i < QR5_K; ++i) {
|
424
|
+
const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;
|
425
|
+
const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;
|
426
|
+
|
427
|
+
const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;
|
428
|
+
const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;
|
429
|
+
|
430
|
+
const int v0i = vl0i | vh0i;
|
431
|
+
const int v1i = vl1i | vh1i;
|
432
|
+
|
433
|
+
const int dot1 = ggml_cuda_dp4a(v0i, u[2*i+0], ggml_cuda_dp4a(v1i, u[2*i+1], 0)); // SIMD dot product
|
434
|
+
const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+0], ggml_cuda_dp4a(0x01010101, u[2*i+1], 0)); // sum of u
|
435
|
+
|
436
|
+
sumf_d += d8[i] * (dot1 * sc[i]);
|
437
|
+
sumf_m += d8[i] * (dot2 * m[i]);
|
438
|
+
|
439
|
+
}
|
440
|
+
|
441
|
+
const float2 dm5f = __half22float2(dm5);
|
442
|
+
|
443
|
+
return dm5f.x*sumf_d - dm5f.y*sumf_m;
|
444
|
+
}
|
445
|
+
|
446
|
+
// contiguous v/x + u/y values
|
447
|
+
static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
|
448
|
+
const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
|
449
|
+
const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
|
450
|
+
|
451
|
+
float sumf_d = 0.0f;
|
452
|
+
float sumf_m = 0.0f;
|
453
|
+
|
454
|
+
#pragma unroll
|
455
|
+
for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {
|
456
|
+
int sumi_d = 0;
|
457
|
+
|
458
|
+
#pragma unroll
|
459
|
+
for (int j = 0; j < QI8_1; ++j) {
|
460
|
+
sumi_d = ggml_cuda_dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product
|
461
|
+
}
|
462
|
+
|
463
|
+
const float2 ds8f = __half22float2(ds8[i]);
|
464
|
+
|
465
|
+
sumf_d += ds8f.x * (sc[i] * sumi_d);
|
466
|
+
sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
|
467
|
+
}
|
468
|
+
|
469
|
+
const float2 dm4f = __half22float2(dm4);
|
470
|
+
|
471
|
+
return dm4f.x*sumf_d - dm4f.y*sumf_m;
|
472
|
+
}
|
473
|
+
|
474
|
+
#define VDR_Q6_K_Q8_1_MMVQ 1
|
475
|
+
#define VDR_Q6_K_Q8_1_MMQ 8
|
476
|
+
|
477
|
+
// contiguous v/x values
|
478
|
+
static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
|
479
|
+
const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
|
480
|
+
const float & d, const float * __restrict__ d8) {
|
481
|
+
|
482
|
+
float sumf = 0.0f;
|
483
|
+
|
484
|
+
#pragma unroll
|
485
|
+
for (int i = 0; i < QR6_K; ++i) {
|
486
|
+
const int sc = scales[4*i];
|
487
|
+
|
488
|
+
const int vil = (vl >> (4*i)) & 0x0F0F0F0F;
|
489
|
+
|
490
|
+
const int vih = ((vh >> (4*i)) << 4) & 0x30303030;
|
491
|
+
|
492
|
+
const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32
|
493
|
+
|
494
|
+
sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
|
495
|
+
}
|
496
|
+
|
497
|
+
return d*sumf;
|
498
|
+
}
|
499
|
+
|
500
|
+
// contiguous v/x + u/y values
|
501
|
+
static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(
|
502
|
+
const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,
|
503
|
+
const float & d6, const float * __restrict__ d8) {
|
504
|
+
|
505
|
+
float sumf_d = 0.0f;
|
506
|
+
|
507
|
+
const int sc_packed = get_int_b4(sc, 0);
|
508
|
+
const int8_t * sc_reg = (const int8_t *) &sc_packed;
|
509
|
+
|
510
|
+
#pragma unroll
|
511
|
+
for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {
|
512
|
+
int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale
|
513
|
+
|
514
|
+
#pragma unroll
|
515
|
+
for (int i = i0; i < i0 + 2; ++i) {
|
516
|
+
sumi_d.x = ggml_cuda_dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product
|
517
|
+
sumi_d.x = ggml_cuda_dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product
|
518
|
+
|
519
|
+
sumi_d.y = ggml_cuda_dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product
|
520
|
+
sumi_d.y = ggml_cuda_dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product
|
521
|
+
}
|
522
|
+
|
523
|
+
sumf_d += d8[i0/4] * (sc_reg[i0/2+0]*sumi_d.x + sc_reg[i0/2+1]*sumi_d.y);
|
524
|
+
}
|
525
|
+
|
526
|
+
return d6 * sumf_d;
|
527
|
+
}
|
528
|
+
|
529
|
+
static __device__ __forceinline__ float vec_dot_q4_0_q8_1(
|
530
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
531
|
+
|
532
|
+
const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq + kbx;
|
533
|
+
|
534
|
+
int v[VDR_Q4_0_Q8_1_MMVQ];
|
535
|
+
int u[2*VDR_Q4_0_Q8_1_MMVQ];
|
536
|
+
|
537
|
+
#pragma unroll
|
538
|
+
for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {
|
539
|
+
v[i] = get_int_b2(bq4_0->qs, iqs + i);
|
540
|
+
u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
|
541
|
+
u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_0);
|
542
|
+
}
|
543
|
+
|
544
|
+
return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, bq4_0->d, bq8_1->ds);
|
545
|
+
}
|
546
|
+
|
547
|
+
|
548
|
+
static __device__ __forceinline__ float vec_dot_q4_1_q8_1(
|
549
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
550
|
+
|
551
|
+
const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq + kbx;
|
552
|
+
|
553
|
+
int v[VDR_Q4_1_Q8_1_MMVQ];
|
554
|
+
int u[2*VDR_Q4_1_Q8_1_MMVQ];
|
555
|
+
|
556
|
+
#pragma unroll
|
557
|
+
for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {
|
558
|
+
v[i] = get_int_b4(bq4_1->qs, iqs + i);
|
559
|
+
u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
|
560
|
+
u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_1);
|
561
|
+
}
|
562
|
+
|
563
|
+
return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);
|
564
|
+
}
|
565
|
+
|
566
|
+
static __device__ __forceinline__ float vec_dot_q5_0_q8_1(
|
567
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
568
|
+
|
569
|
+
const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq + kbx;
|
570
|
+
|
571
|
+
int vl[VDR_Q5_0_Q8_1_MMVQ];
|
572
|
+
int vh[VDR_Q5_0_Q8_1_MMVQ];
|
573
|
+
int u[2*VDR_Q5_0_Q8_1_MMVQ];
|
574
|
+
|
575
|
+
#pragma unroll
|
576
|
+
for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {
|
577
|
+
vl[i] = get_int_b2(bq5_0->qs, iqs + i);
|
578
|
+
vh[i] = get_int_b2(bq5_0->qh, 0) >> (4 * (iqs + i));
|
579
|
+
u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
|
580
|
+
u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_0);
|
581
|
+
}
|
582
|
+
|
583
|
+
return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, bq5_0->d, bq8_1->ds);
|
584
|
+
}
|
585
|
+
|
586
|
+
static __device__ __forceinline__ float vec_dot_q5_1_q8_1(
|
587
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
588
|
+
|
589
|
+
const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq + kbx;
|
590
|
+
|
591
|
+
int vl[VDR_Q5_1_Q8_1_MMVQ];
|
592
|
+
int vh[VDR_Q5_1_Q8_1_MMVQ];
|
593
|
+
int u[2*VDR_Q5_1_Q8_1_MMVQ];
|
594
|
+
|
595
|
+
#pragma unroll
|
596
|
+
for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {
|
597
|
+
vl[i] = get_int_b4(bq5_1->qs, iqs + i);
|
598
|
+
vh[i] = get_int_b4(bq5_1->qh, 0) >> (4 * (iqs + i));
|
599
|
+
u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
|
600
|
+
u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_1);
|
601
|
+
}
|
602
|
+
|
603
|
+
return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);
|
604
|
+
}
|
605
|
+
|
606
|
+
static __device__ __forceinline__ float vec_dot_q8_0_q8_1(
|
607
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
608
|
+
|
609
|
+
const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq + kbx;
|
610
|
+
|
611
|
+
int v[VDR_Q8_0_Q8_1_MMVQ];
|
612
|
+
int u[VDR_Q8_0_Q8_1_MMVQ];
|
613
|
+
|
614
|
+
#pragma unroll
|
615
|
+
for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {
|
616
|
+
v[i] = get_int_b2(bq8_0->qs, iqs + i);
|
617
|
+
u[i] = get_int_b4(bq8_1->qs, iqs + i);
|
618
|
+
}
|
619
|
+
|
620
|
+
return vec_dot_q8_0_q8_1_impl<float, VDR_Q8_0_Q8_1_MMVQ>(v, u, bq8_0->d, __low2half(bq8_1->ds));
|
621
|
+
}
|
622
|
+
|
623
|
+
static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
|
624
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
625
|
+
|
626
|
+
const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx;
|
627
|
+
|
628
|
+
const int bq8_offset = QR2_K * (iqs / QI8_1);
|
629
|
+
const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
|
630
|
+
|
631
|
+
const uint8_t * scales = bq2_K->scales + scale_offset;
|
632
|
+
|
633
|
+
const int v = get_int_b4(bq2_K->qs, iqs);
|
634
|
+
int u[QR2_K];
|
635
|
+
float d8[QR2_K];
|
636
|
+
|
637
|
+
#pragma unroll
|
638
|
+
for (int i = 0; i < QR2_K; ++ i) {
|
639
|
+
u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
|
640
|
+
d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
|
641
|
+
}
|
642
|
+
|
643
|
+
return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);
|
644
|
+
}
|
645
|
+
|
646
|
+
static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
|
647
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
648
|
+
|
649
|
+
const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx;
|
650
|
+
|
651
|
+
const int bq8_offset = QR3_K * (iqs / (QI3_K/2));
|
652
|
+
const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
|
653
|
+
|
654
|
+
const float d = bq3_K->d;
|
655
|
+
|
656
|
+
const int vl = get_int_b2(bq3_K->qs, iqs);
|
657
|
+
|
658
|
+
// invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
|
659
|
+
const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;
|
660
|
+
|
661
|
+
int u[QR3_K];
|
662
|
+
float d8[QR3_K];
|
663
|
+
|
664
|
+
#pragma unroll
|
665
|
+
for (int i = 0; i < QR3_K; ++i) {
|
666
|
+
u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
|
667
|
+
d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
|
668
|
+
}
|
669
|
+
|
670
|
+
return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);
|
671
|
+
}
|
672
|
+
|
673
|
+
static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
|
674
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
675
|
+
|
676
|
+
const block_q4_K * bq4_K = (const block_q4_K *) vbq + kbx;
|
677
|
+
|
678
|
+
int v[2];
|
679
|
+
int u[2*QR4_K];
|
680
|
+
float d8[QR4_K];
|
681
|
+
|
682
|
+
// iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6
|
683
|
+
const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));
|
684
|
+
|
685
|
+
// iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12
|
686
|
+
// iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44
|
687
|
+
// iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76
|
688
|
+
// iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108
|
689
|
+
|
690
|
+
const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
|
691
|
+
v[0] = q4[0];
|
692
|
+
v[1] = q4[4];
|
693
|
+
|
694
|
+
const uint16_t * scales = (const uint16_t *)bq4_K->scales;
|
695
|
+
uint16_t aux[2];
|
696
|
+
const int j = bq8_offset/2;
|
697
|
+
if (j < 2) {
|
698
|
+
aux[0] = scales[j+0] & 0x3f3f;
|
699
|
+
aux[1] = scales[j+2] & 0x3f3f;
|
700
|
+
} else {
|
701
|
+
aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
|
702
|
+
aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
|
703
|
+
}
|
704
|
+
const uint8_t * sc = (const uint8_t *)aux;
|
705
|
+
const uint8_t * m = sc + 2;
|
706
|
+
|
707
|
+
for (int i = 0; i < QR4_K; ++i) {
|
708
|
+
const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
|
709
|
+
d8[i] = __low2float(bq8i->ds);
|
710
|
+
|
711
|
+
const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
|
712
|
+
u[2*i+0] = q8[0];
|
713
|
+
u[2*i+1] = q8[4];
|
714
|
+
}
|
715
|
+
|
716
|
+
return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);
|
717
|
+
}
|
718
|
+
|
719
|
+
static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
|
720
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
721
|
+
|
722
|
+
const block_q5_K * bq5_K = (const block_q5_K *) vbq + kbx;
|
723
|
+
|
724
|
+
int vl[2];
|
725
|
+
int vh[2];
|
726
|
+
int u[2*QR5_K];
|
727
|
+
float d8[QR5_K];
|
728
|
+
|
729
|
+
const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));
|
730
|
+
const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
|
731
|
+
const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));
|
732
|
+
|
733
|
+
vl[0] = ql[0];
|
734
|
+
vl[1] = ql[4];
|
735
|
+
|
736
|
+
vh[0] = qh[0] >> bq8_offset;
|
737
|
+
vh[1] = qh[4] >> bq8_offset;
|
738
|
+
|
739
|
+
const uint16_t * scales = (const uint16_t *)bq5_K->scales;
|
740
|
+
uint16_t aux[2];
|
741
|
+
const int j = bq8_offset/2;
|
742
|
+
if (j < 2) {
|
743
|
+
aux[0] = scales[j+0] & 0x3f3f;
|
744
|
+
aux[1] = scales[j+2] & 0x3f3f;
|
745
|
+
} else {
|
746
|
+
aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
|
747
|
+
aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
|
748
|
+
}
|
749
|
+
const uint8_t * sc = (const uint8_t *)aux;
|
750
|
+
const uint8_t * m = sc + 2;
|
751
|
+
|
752
|
+
#pragma unroll
|
753
|
+
for (int i = 0; i < QR5_K; ++i) {
|
754
|
+
const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
|
755
|
+
d8[i] = __low2float(bq8i->ds);
|
756
|
+
|
757
|
+
const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
|
758
|
+
u[2*i+0] = q8[0];
|
759
|
+
u[2*i+1] = q8[4];
|
760
|
+
}
|
761
|
+
|
762
|
+
return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);
|
763
|
+
}
|
764
|
+
|
765
|
+
static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
|
766
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
767
|
+
|
768
|
+
const block_q6_K * bq6_K = (const block_q6_K *) vbq + kbx;
|
769
|
+
|
770
|
+
const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);
|
771
|
+
const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);
|
772
|
+
const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));
|
773
|
+
|
774
|
+
const int vl = get_int_b2(bq6_K->ql, iqs);
|
775
|
+
const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;
|
776
|
+
|
777
|
+
const int8_t * scales = bq6_K->scales + scale_offset;
|
778
|
+
|
779
|
+
int u[QR6_K];
|
780
|
+
float d8[QR6_K];
|
781
|
+
|
782
|
+
#pragma unroll
|
783
|
+
for (int i = 0; i < QR6_K; ++i) {
|
784
|
+
u[i] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);
|
785
|
+
d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);
|
786
|
+
}
|
787
|
+
|
788
|
+
return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8);
|
789
|
+
}
|
790
|
+
|
791
|
+
#define VDR_IQ2_XXS_Q8_1_MMVQ 2
|
792
|
+
#define VDR_IQ2_XXS_Q8_1_MMQ 2
|
793
|
+
|
794
|
+
static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(
|
795
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
796
|
+
|
797
|
+
const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq + kbx;
|
798
|
+
|
799
|
+
const int q2 = get_int_b2(bq2->qs, iqs);
|
800
|
+
const uint8_t * aux8 = (const uint8_t *) &q2;
|
801
|
+
const uint32_t aux32 = get_int_b2(bq2->qs, iqs + 1);
|
802
|
+
|
803
|
+
int sumi = 0;
|
804
|
+
#pragma unroll
|
805
|
+
for (int k0 = 0; k0 < 8; k0 += 2) {
|
806
|
+
const int * grid_pos = (const int *) (iq2xxs_grid + aux8[k0/2]);
|
807
|
+
const int signs_packed = ksigns_iq2xs[(aux32 >> (7*k0/2)) & 0x7F];
|
808
|
+
|
809
|
+
const int signs0 = __vcmpne4(((signs_packed & 0x03) << 7) | ((signs_packed & 0x0C) << 21), 0x00000000);
|
810
|
+
const int grid0 = __vsub4(grid_pos[0] ^ signs0, signs0);
|
811
|
+
const int u0 = get_int_b4(bq8_1[iqs/2].qs, k0 + 0);
|
812
|
+
sumi = ggml_cuda_dp4a(grid0, u0, sumi);
|
813
|
+
|
814
|
+
const int signs1 = __vcmpne4(((signs_packed & 0x30) << 3) | ((signs_packed & 0xC0) << 17), 0x00000000);
|
815
|
+
const int grid1 = __vsub4(grid_pos[1] ^ signs1, signs1);
|
816
|
+
const int u1 = get_int_b4(bq8_1[iqs/2].qs, k0 + 1);
|
817
|
+
sumi = ggml_cuda_dp4a(grid1, u1, sumi);
|
818
|
+
}
|
819
|
+
|
820
|
+
const int ls = aux32 >> 28;
|
821
|
+
sumi = (ls*sumi + sumi/2)/4;
|
822
|
+
const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
|
823
|
+
return d * sumi;
|
824
|
+
}
|
825
|
+
|
826
|
+
#define VDR_IQ2_XS_Q8_1_MMVQ 2
|
827
|
+
#define VDR_IQ2_XS_Q8_1_MMQ 2
|
828
|
+
|
829
|
+
static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(
|
830
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
831
|
+
|
832
|
+
const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq + kbx;
|
833
|
+
|
834
|
+
const int2 q2_packed = make_int2(get_int_b2(bq2->qs, iqs + 0), get_int_b2(bq2->qs, iqs + 1));
|
835
|
+
const uint16_t * q2 = (const uint16_t *) &q2_packed;
|
836
|
+
const int ls0 = bq2->scales[iqs/2] & 0x0F;
|
837
|
+
const int ls1 = bq2->scales[iqs/2] >> 4;
|
838
|
+
|
839
|
+
int sumi0 = 0;
|
840
|
+
int sumi1 = 0;
|
841
|
+
#pragma unroll
|
842
|
+
for (int l0 = 0; l0 < 8; l0 += 2) {
|
843
|
+
const uint32_t * grid_pos = (const uint32_t *)(iq2xs_grid + (q2[l0/2] & 0x000001FF));
|
844
|
+
const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l0/2] >> 9));
|
845
|
+
|
846
|
+
const int grid_l = __vsub4(grid_pos[0] ^ signs[0], signs[0]);
|
847
|
+
const int grid_h = __vsub4(grid_pos[1] ^ signs[1], signs[1]);
|
848
|
+
|
849
|
+
const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
|
850
|
+
const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
|
851
|
+
|
852
|
+
if (l0 < 4) {
|
853
|
+
sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);
|
854
|
+
sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);
|
855
|
+
} else {
|
856
|
+
sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);
|
857
|
+
sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);
|
858
|
+
}
|
859
|
+
}
|
860
|
+
const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;
|
861
|
+
const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
|
862
|
+
return d * sumi;
|
863
|
+
}
|
864
|
+
|
865
|
+
#define VDR_IQ2_S_Q8_1_MMVQ 2
|
866
|
+
#define VDR_IQ2_S_Q8_1_MMQ 2
|
867
|
+
|
868
|
+
static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(
|
869
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
870
|
+
|
871
|
+
const block_iq2_s * bq2 = (const block_iq2_s *) vbq + kbx;
|
872
|
+
|
873
|
+
const int qs_packed = get_int_b2(bq2->qs, iqs/2);
|
874
|
+
const uint8_t * qs = (const uint8_t *) &qs_packed;
|
875
|
+
|
876
|
+
const int qh = bq2->qh[iqs/2];
|
877
|
+
|
878
|
+
const int signs_packed_32 = get_int_b2(bq2->qs, QK_K/32 + iqs/2);
|
879
|
+
const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
|
880
|
+
|
881
|
+
const int ls0 = bq2->scales[iqs/2] & 0x0F;
|
882
|
+
const int ls1 = bq2->scales[iqs/2] >> 4;
|
883
|
+
|
884
|
+
int sumi0 = 0;
|
885
|
+
int sumi1 = 0;
|
886
|
+
#pragma unroll
|
887
|
+
for (int l0 = 0; l0 < 8; l0 += 2) {
|
888
|
+
const int * grid_pos = (const int *)(iq2s_grid + (qs[l0/2] | ((qh << (8-l0)) & 0x300)));
|
889
|
+
|
890
|
+
const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);
|
891
|
+
const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);
|
892
|
+
|
893
|
+
const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);
|
894
|
+
const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);
|
895
|
+
|
896
|
+
const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
|
897
|
+
const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
|
898
|
+
|
899
|
+
if (l0 < 4) {
|
900
|
+
sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);
|
901
|
+
sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);
|
902
|
+
} else {
|
903
|
+
sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);
|
904
|
+
sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);
|
905
|
+
}
|
906
|
+
}
|
907
|
+
const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;
|
908
|
+
|
909
|
+
const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
|
910
|
+
return d * sumi;
|
911
|
+
}
|
912
|
+
|
913
|
+
#define VDR_IQ3_XXS_Q8_1_MMVQ 2
|
914
|
+
#define VDR_IQ3_XXS_Q8_1_MMQ 2
|
915
|
+
|
916
|
+
static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(
|
917
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
918
|
+
|
919
|
+
const block_iq3_xxs * bq3 = (const block_iq3_xxs *) vbq + kbx;
|
920
|
+
|
921
|
+
const int2 q3_packed = make_int2(get_int_b2(bq3->qs, iqs), get_int_b2(bq3->qs, iqs+1));
|
922
|
+
const uint8_t * q3 = (const uint8_t *) &q3_packed;
|
923
|
+
const uint32_t aux32 = get_int_b2(bq3->qs, QK_K/16 + iqs/2);
|
924
|
+
|
925
|
+
int sumi = 0;
|
926
|
+
#pragma unroll
|
927
|
+
for (int l0 = 0; l0 < 8; l0 += 2) {
|
928
|
+
const int2 grid_pos = make_int2(iq3xxs_grid[q3[l0 + 0]], iq3xxs_grid[q3[l0 + 1]]);
|
929
|
+
|
930
|
+
const int * signs = (const int *)(ksigns64 + ((aux32 >> (7*l0/2)) & 0x7F));
|
931
|
+
|
932
|
+
const int grid_l = __vsub4(grid_pos.x ^ signs[0], signs[0]);
|
933
|
+
const int grid_h = __vsub4(grid_pos.y ^ signs[1], signs[1]);
|
934
|
+
|
935
|
+
const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
|
936
|
+
const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
|
937
|
+
|
938
|
+
sumi = ggml_cuda_dp4a(grid_l, u0, sumi);
|
939
|
+
sumi = ggml_cuda_dp4a(grid_h, u1, sumi);
|
940
|
+
}
|
941
|
+
|
942
|
+
const int ls = aux32 >> 28;
|
943
|
+
sumi = (ls*sumi + sumi/2)/2;
|
944
|
+
const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);
|
945
|
+
return d * sumi;
|
946
|
+
}
|
947
|
+
|
948
|
+
#define VDR_IQ3_S_Q8_1_MMVQ 2
|
949
|
+
#define VDR_IQ3_S_Q8_1_MMQ 2
|
950
|
+
|
951
|
+
// TODO: don't use lookup table for signs
|
952
|
+
static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
|
953
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
954
|
+
|
955
|
+
const block_iq3_s * bq3 = (const block_iq3_s *) vbq + kbx;
|
956
|
+
|
957
|
+
const int2 qs_packed = make_int2(get_int_b2(bq3->qs, iqs + 0), get_int_b2(bq3->qs, iqs + 1));
|
958
|
+
const uint8_t * qs = (const uint8_t *) &qs_packed;
|
959
|
+
|
960
|
+
const int qh = bq3->qh[iqs/2];
|
961
|
+
|
962
|
+
const int signs_packed_32 = get_int_b2(bq3->signs, iqs/2);
|
963
|
+
const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
|
964
|
+
|
965
|
+
int sumi = 0;
|
966
|
+
#pragma unroll
|
967
|
+
for (int l0 = 0; l0 < 8; l0 += 2) {
|
968
|
+
const int2 grid_pos = make_int2(
|
969
|
+
iq3s_grid[qs[l0 + 0] | ((qh << (8 - l0)) & 0x100)],
|
970
|
+
iq3s_grid[qs[l0 + 1] | ((qh << (7 - l0)) & 0x100)]);
|
971
|
+
|
972
|
+
const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);
|
973
|
+
const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);
|
974
|
+
|
975
|
+
const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);
|
976
|
+
const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);
|
977
|
+
|
978
|
+
const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
|
979
|
+
const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
|
980
|
+
|
981
|
+
sumi = ggml_cuda_dp4a(grid_l, u0, sumi);
|
982
|
+
sumi = ggml_cuda_dp4a(grid_h, u1, sumi);
|
983
|
+
}
|
984
|
+
|
985
|
+
sumi *= 1 + 2*((bq3->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F);
|
986
|
+
|
987
|
+
const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);
|
988
|
+
return d * sumi;
|
989
|
+
}
|
990
|
+
|
991
|
+
#define VDR_IQ1_S_Q8_1_MMVQ 1
|
992
|
+
#define VDR_IQ1_S_Q8_1_MMQ 1
|
993
|
+
|
994
|
+
static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
|
995
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
996
|
+
const block_iq1_s * bq1 = (const block_iq1_s *) vbq + kbx;
|
997
|
+
|
998
|
+
const int qs_packed = get_int_b2(bq1->qs, iqs);
|
999
|
+
const uint8_t * qs = (const uint8_t *) &qs_packed;
|
1000
|
+
|
1001
|
+
const int qh = bq1->qh[iqs];
|
1002
|
+
|
1003
|
+
int sumi = 0;
|
1004
|
+
#pragma unroll
|
1005
|
+
for (int l0 = 0; l0 < 8; l0 += 2) {
|
1006
|
+
const int grid = iq1s_grid_gpu[qs[l0/2] | (((qh >> 3*(l0/2)) & 0x07) << 8)];
|
1007
|
+
|
1008
|
+
const int grid0 = (grid >> 0) & 0x0F0F0F0F;
|
1009
|
+
const int grid1 = (grid >> 4) & 0x0F0F0F0F;
|
1010
|
+
|
1011
|
+
const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
|
1012
|
+
const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
|
1013
|
+
|
1014
|
+
sumi = ggml_cuda_dp4a(grid0, u0, sumi);
|
1015
|
+
sumi = ggml_cuda_dp4a(grid1, u1, sumi);
|
1016
|
+
}
|
1017
|
+
|
1018
|
+
const float d1q = __half2float(bq1->d) * (((qh >> 11) & 0x0E) + 1);
|
1019
|
+
const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);
|
1020
|
+
const float2 ds = __half22float2(bq8_1[iqs].ds);
|
1021
|
+
return d1q * (ds.x*sumi + ds.y*delta);
|
1022
|
+
}
|
1023
|
+
|
1024
|
+
#define VDR_IQ1_M_Q8_1_MMVQ 1
|
1025
|
+
#define VDR_IQ1_M_Q8_1_MMQ 1
|
1026
|
+
|
1027
|
+
static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(
|
1028
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
1029
|
+
|
1030
|
+
const block_iq1_m * bq1 = (const block_iq1_m *) vbq + kbx;
|
1031
|
+
|
1032
|
+
const int qs_packed = get_int_b4(bq1->qs, iqs);
|
1033
|
+
const uint8_t * qs = (const uint8_t *) &qs_packed;
|
1034
|
+
|
1035
|
+
int sumi[2] = {0};
|
1036
|
+
float sumf[2] = {0.0f};
|
1037
|
+
#pragma unroll
|
1038
|
+
for (int l0 = 0; l0 < 8; l0 += 2) {
|
1039
|
+
const int qhl = bq1->qh[2*iqs + l0/4] >> (4 * ((l0/2) % 2));
|
1040
|
+
|
1041
|
+
const int grid = iq1s_grid_gpu[qs[l0/2] | ((qhl & 0x07) << 8)];
|
1042
|
+
|
1043
|
+
const int grid0 = (grid >> 0) & 0x0F0F0F0F;
|
1044
|
+
const int grid1 = (grid >> 4) & 0x0F0F0F0F;
|
1045
|
+
|
1046
|
+
const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
|
1047
|
+
const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
|
1048
|
+
|
1049
|
+
sumi[l0/4] = ggml_cuda_dp4a(grid0, u0, sumi[l0/4]);
|
1050
|
+
sumi[l0/4] = ggml_cuda_dp4a(grid1, u1, sumi[l0/4]);
|
1051
|
+
|
1052
|
+
const float delta = -1.0f + IQ1M_DELTA - (qhl & 0x08) * (2.0f*IQ1M_DELTA/0x08);
|
1053
|
+
int sumy = 0;
|
1054
|
+
sumy = ggml_cuda_dp4a(u0, 0x01010101, sumy);
|
1055
|
+
sumy = ggml_cuda_dp4a(u1, 0x01010101, sumy);
|
1056
|
+
sumf[l0/4] += delta*sumy;
|
1057
|
+
}
|
1058
|
+
|
1059
|
+
const uint16_t * sc = (const uint16_t *) bq1->scales;
|
1060
|
+
|
1061
|
+
iq1m_scale_t scale;
|
1062
|
+
scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);
|
1063
|
+
const float d = __half2float(scale.f16) * __low2float(bq8_1[iqs].ds);
|
1064
|
+
|
1065
|
+
const int tmp = sc[iqs/2] >> (6*(iqs%2));
|
1066
|
+
const int sc0 = 2*((tmp >> 0) & 0x07) + 1;
|
1067
|
+
const int sc1 = 2*((tmp >> 3) & 0x07) + 1;
|
1068
|
+
return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1);
|
1069
|
+
}
|
1070
|
+
|
1071
|
+
static __device__ __forceinline__ int2 get_int_from_table_16(const int & q4) {
|
1072
|
+
const int q0_32 = (q4 >> 0) & 0x0F0F0F0F;
|
1073
|
+
const int8_t * q0_8 = (const int8_t *) &q0_32;
|
1074
|
+
const char4 val0_8 = make_char4(
|
1075
|
+
kvalues_iq4nl[q0_8[0]], kvalues_iq4nl[q0_8[1]], kvalues_iq4nl[q0_8[2]], kvalues_iq4nl[q0_8[3]]);
|
1076
|
+
|
1077
|
+
const int q1_32 = (q4 >> 4) & 0x0F0F0F0F;
|
1078
|
+
const int8_t * q1_8 = (const int8_t *) &q1_32;
|
1079
|
+
const char4 val1_8 = make_char4(
|
1080
|
+
kvalues_iq4nl[q1_8[0]], kvalues_iq4nl[q1_8[1]], kvalues_iq4nl[q1_8[2]], kvalues_iq4nl[q1_8[3]]);
|
1081
|
+
|
1082
|
+
return make_int2(*((const int *) &val0_8), *((const int *) &val1_8));
|
1083
|
+
}
|
1084
|
+
|
1085
|
+
#define VDR_IQ4_NL_Q8_1_MMVQ 2
|
1086
|
+
#define VDR_IQ4_NL_Q8_1_MMQ 4
|
1087
|
+
|
1088
|
+
static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(
|
1089
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
1090
|
+
|
1091
|
+
const block_iq4_nl * bq4 = (const block_iq4_nl *) vbq + kbx;
|
1092
|
+
|
1093
|
+
const int * q8 = (const int *) bq8_1->qs + iqs;
|
1094
|
+
|
1095
|
+
int sumi = 0;
|
1096
|
+
#pragma unroll
|
1097
|
+
for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {
|
1098
|
+
const int aux_q4 = get_int_b2(bq4->qs, iqs + l);
|
1099
|
+
const int2 v = get_int_from_table_16(aux_q4);
|
1100
|
+
|
1101
|
+
sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi);
|
1102
|
+
sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi);
|
1103
|
+
}
|
1104
|
+
|
1105
|
+
const float d = __half2float(bq4->d) * __low2float(bq8_1->ds);
|
1106
|
+
return d * sumi;
|
1107
|
+
}
|
1108
|
+
|
1109
|
+
#define VDR_IQ4_XS_Q8_1_MMVQ 4
|
1110
|
+
#define VDR_IQ4_XS_Q8_1_MMQ 4
|
1111
|
+
|
1112
|
+
static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(
|
1113
|
+
const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
|
1114
|
+
|
1115
|
+
const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq + kbx;
|
1116
|
+
|
1117
|
+
int sumi = 0;
|
1118
|
+
#pragma unroll
|
1119
|
+
for (int j = 0; j < 4; ++j) {
|
1120
|
+
const int aux_q4 = get_int_b4(bq4->qs, iqs + j);
|
1121
|
+
const int2 v = get_int_from_table_16(aux_q4);
|
1122
|
+
|
1123
|
+
const int u0 = get_int_b4(bq8_1[iqs/4].qs, j + 0);
|
1124
|
+
const int u1 = get_int_b4(bq8_1[iqs/4].qs, j + 4);
|
1125
|
+
|
1126
|
+
sumi = ggml_cuda_dp4a(v.x, u0, sumi);
|
1127
|
+
sumi = ggml_cuda_dp4a(v.y, u1, sumi);
|
1128
|
+
}
|
1129
|
+
|
1130
|
+
const int ls = ((bq4->scales_l[iqs/8] >> (iqs & 0x04)) & 0x0F) | (((bq4->scales_h >> (iqs/2)) & 0x03) << 4);
|
1131
|
+
sumi *= ls - 32;
|
1132
|
+
|
1133
|
+
const float d = __half2float(bq4->d) * __low2float(bq8_1[iqs/4].ds);
|
1134
|
+
return d * sumi;
|
1135
|
+
}
|