whispercpp 1.3.1 → 1.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (797) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -3
  3. data/README.md +92 -31
  4. data/Rakefile +26 -7
  5. data/ext/.gitignore +5 -7
  6. data/ext/dependencies.rb +61 -0
  7. data/ext/extconf.rb +21 -198
  8. data/ext/options.rb +221 -0
  9. data/ext/ruby_whisper.c +159 -0
  10. data/ext/ruby_whisper.h +17 -2
  11. data/ext/ruby_whisper_context.c +641 -0
  12. data/ext/ruby_whisper_error.c +52 -0
  13. data/ext/ruby_whisper_model.c +232 -0
  14. data/ext/ruby_whisper_params.c +1301 -0
  15. data/ext/ruby_whisper_segment.c +143 -0
  16. data/ext/ruby_whisper_transcribe.cpp +87 -0
  17. data/ext/ruby_whisper_vad_params.c +288 -0
  18. data/ext/sources/.dockerignore +3 -0
  19. data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
  20. data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
  21. data/ext/sources/CMakeLists.txt +251 -0
  22. data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
  23. data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
  24. data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
  25. data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
  26. data/ext/sources/bindings/javascript/package.json +26 -0
  27. data/ext/sources/bindings/javascript/whisper.js +19 -0
  28. data/ext/sources/build-xcframework.sh +547 -0
  29. data/ext/sources/ci/run.sh +336 -0
  30. data/ext/sources/close-issue.yml +28 -0
  31. data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
  32. data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
  33. data/ext/sources/cmake/build-info.cmake +60 -0
  34. data/ext/sources/cmake/git-vars.cmake +22 -0
  35. data/ext/sources/cmake/whisper-config.cmake.in +65 -0
  36. data/ext/sources/cmake/whisper.pc.in +10 -0
  37. data/ext/sources/examples/CMakeLists.txt +124 -0
  38. data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
  39. data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
  40. data/ext/sources/examples/addon.node/addon.cpp +438 -0
  41. data/ext/sources/examples/addon.node/index.js +54 -0
  42. data/ext/sources/examples/addon.node/package.json +16 -0
  43. data/ext/sources/examples/bench/CMakeLists.txt +8 -0
  44. data/ext/sources/examples/bench/bench.cpp +175 -0
  45. data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
  46. data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
  47. data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
  48. data/ext/sources/examples/cli/CMakeLists.txt +8 -0
  49. data/ext/sources/examples/cli/cli.cpp +1294 -0
  50. data/ext/sources/examples/coi-serviceworker.js +146 -0
  51. data/ext/sources/examples/command/CMakeLists.txt +10 -0
  52. data/ext/sources/examples/command/command.cpp +776 -0
  53. data/ext/sources/examples/command/commands.txt +9 -0
  54. data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
  55. data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
  56. data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
  57. data/ext/sources/examples/common-ggml.cpp +238 -0
  58. data/ext/sources/examples/common-ggml.h +18 -0
  59. data/ext/sources/examples/common-sdl.cpp +227 -0
  60. data/ext/sources/examples/common-sdl.h +49 -0
  61. data/ext/sources/examples/common-whisper.cpp +168 -0
  62. data/ext/sources/examples/common-whisper.h +24 -0
  63. data/ext/sources/examples/common.cpp +675 -0
  64. data/ext/sources/examples/common.h +322 -0
  65. data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
  66. data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
  67. data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
  68. data/ext/sources/examples/generate-karaoke.sh +57 -0
  69. data/ext/sources/examples/grammar-parser.cpp +423 -0
  70. data/ext/sources/examples/grammar-parser.h +29 -0
  71. data/ext/sources/examples/helpers.js +191 -0
  72. data/ext/sources/examples/json.hpp +24596 -0
  73. data/ext/sources/examples/livestream.sh +112 -0
  74. data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
  75. data/ext/sources/examples/lsp/lsp.cpp +467 -0
  76. data/ext/sources/examples/lsp/whisper.vim +362 -0
  77. data/ext/sources/examples/miniaudio.h +93468 -0
  78. data/ext/sources/examples/python/test_whisper_processor.py +7 -0
  79. data/ext/sources/examples/python/whisper_processor.py +54 -0
  80. data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
  81. data/ext/sources/examples/quantize/quantize.cpp +223 -0
  82. data/ext/sources/examples/server/CMakeLists.txt +12 -0
  83. data/ext/sources/examples/server/bench.js +29 -0
  84. data/ext/sources/examples/server/httplib.h +10497 -0
  85. data/ext/sources/examples/server/server.cpp +1091 -0
  86. data/ext/sources/examples/server.py +115 -0
  87. data/ext/sources/examples/stb_vorbis.c +5584 -0
  88. data/ext/sources/examples/stream/CMakeLists.txt +10 -0
  89. data/ext/sources/examples/stream/stream.cpp +429 -0
  90. data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
  91. data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
  92. data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
  93. data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
  94. data/ext/sources/examples/sycl/build.sh +22 -0
  95. data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
  96. data/ext/sources/examples/sycl/run-whisper.sh +17 -0
  97. data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
  98. data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
  99. data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
  100. data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
  101. data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
  102. data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
  103. data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
  104. data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
  105. data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
  106. data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
  107. data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
  108. data/ext/sources/examples/talk-llama/llama-context.h +276 -0
  109. data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
  110. data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
  111. data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
  112. data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
  113. data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
  114. data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
  115. data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
  116. data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
  117. data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
  118. data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
  119. data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
  120. data/ext/sources/examples/talk-llama/llama-io.h +35 -0
  121. data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
  122. data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
  123. data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
  124. data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
  125. data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
  126. data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
  127. data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
  128. data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
  129. data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
  130. data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
  131. data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
  132. data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
  133. data/ext/sources/examples/talk-llama/llama-model.h +425 -0
  134. data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
  135. data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
  136. data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
  137. data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
  138. data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
  139. data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
  140. data/ext/sources/examples/talk-llama/llama.cpp +354 -0
  141. data/ext/sources/examples/talk-llama/llama.h +1377 -0
  142. data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
  143. data/ext/sources/examples/talk-llama/speak +40 -0
  144. data/ext/sources/examples/talk-llama/speak.bat +1 -0
  145. data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
  146. data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
  147. data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
  148. data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
  149. data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
  150. data/ext/sources/examples/talk-llama/unicode.h +66 -0
  151. data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
  152. data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
  153. data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
  154. data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
  155. data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
  156. data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
  157. data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
  158. data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
  159. data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
  160. data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
  161. data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
  162. data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
  163. data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
  164. data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
  165. data/ext/sources/ggml/CMakeLists.txt +390 -0
  166. data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
  167. data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
  168. data/ext/sources/ggml/cmake/common.cmake +26 -0
  169. data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
  170. data/ext/{ggml → sources/ggml}/include/ggml-alloc.h +1 -1
  171. data/ext/{ggml → sources/ggml}/include/ggml-backend.h +9 -7
  172. data/ext/{ggml → sources/ggml}/include/ggml-cpp.h +2 -1
  173. data/ext/{ggml → sources/ggml}/include/ggml-cpu.h +9 -1
  174. data/ext/{ggml → sources/ggml}/include/ggml-metal.h +1 -1
  175. data/ext/{ggml → sources/ggml}/include/ggml-opt.h +49 -28
  176. data/ext/{ggml → sources/ggml}/include/ggml-rpc.h +6 -1
  177. data/ext/{ggml → sources/ggml}/include/ggml-vulkan.h +0 -2
  178. data/ext/{ggml → sources/ggml}/include/ggml.h +182 -265
  179. data/ext/sources/ggml/include/gguf.h +202 -0
  180. data/ext/sources/ggml/src/CMakeLists.txt +346 -0
  181. data/ext/{ggml → sources/ggml}/src/ggml-alloc.c +34 -29
  182. data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
  183. data/ext/{ggml → sources/ggml}/src/ggml-backend-impl.h +1 -2
  184. data/ext/{ggml → sources/ggml}/src/ggml-backend-reg.cpp +87 -53
  185. data/ext/{ggml → sources/ggml}/src/ggml-backend.cpp +26 -14
  186. data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
  187. data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
  188. data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
  189. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.cpp +10 -4
  190. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.h +5 -5
  191. data/ext/{ggml → sources/ggml}/src/ggml-cann/aclnn_ops.cpp +1272 -1506
  192. data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
  193. data/ext/{ggml → sources/ggml}/src/ggml-cann/common.h +135 -1
  194. data/ext/{ggml → sources/ggml}/src/ggml-cann/ggml-cann.cpp +564 -146
  195. data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
  196. data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/dup.cpp +3 -5
  197. data/ext/{ggml → sources/ggml}/src/ggml-common.h +12 -8
  198. data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
  199. data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.cpp +2 -1
  200. data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
  201. data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
  202. data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  203. data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
  204. data/ext/{ggml → sources/ggml}/src/ggml-cpu/cpu-feats-x86.cpp +5 -1
  205. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
  206. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-impl.h +163 -41
  207. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.c +4029 -1117
  208. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
  209. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu.cpp +67 -18
  210. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
  211. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
  212. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
  213. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  214. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
  215. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
  216. data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
  217. data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
  218. data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
  219. data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
  220. data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
  221. data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
  222. data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
  223. data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
  224. data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
  225. data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
  226. data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
  227. data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
  228. data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
  229. data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
  230. data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
  231. data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
  232. data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
  233. data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
  234. data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
  235. data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
  236. data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
  237. data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
  238. data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
  239. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
  240. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  241. data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
  242. data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
  243. data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
  244. data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  245. data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  246. data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
  247. data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
  248. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
  249. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  250. data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
  251. data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
  252. data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  253. data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
  254. data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
  255. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
  256. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
  257. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
  258. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
  259. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
  260. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
  261. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
  262. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
  263. data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
  264. data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
  265. data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
  266. data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
  267. data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
  268. data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
  269. data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
  270. data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
  271. data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
  272. data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
  273. data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
  274. data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
  275. data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
  276. data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
  277. data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
  278. data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
  279. data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
  280. data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
  281. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  282. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  283. data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
  284. data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  285. data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
  286. data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
  287. data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
  288. data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  289. data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
  290. data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
  291. data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
  292. data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
  293. data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
  294. data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
  295. data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
  296. data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
  297. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
  298. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  299. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
  300. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  301. data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
  302. data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
  303. data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
  304. data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
  305. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
  306. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
  307. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  308. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
  309. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
  310. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
  311. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
  312. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
  313. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  314. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
  315. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
  316. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
  317. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
  318. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
  319. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  320. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  321. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
  322. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
  323. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
  324. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
  325. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
  326. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
  327. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
  328. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
  329. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
  330. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
  331. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
  332. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
  333. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
  334. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
  335. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
  336. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
  337. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
  338. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
  339. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
  340. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
  341. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
  342. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
  343. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
  344. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
  345. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
  346. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
  347. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
  348. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
  349. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
  350. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
  351. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
  352. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
  353. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
  354. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
  355. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
  356. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
  357. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
  358. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
  359. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
  360. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
  361. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
  362. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
  363. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
  364. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
  365. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
  366. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
  367. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
  368. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
  369. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
  370. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
  371. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
  372. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
  373. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
  374. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
  375. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
  376. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
  377. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
  378. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
  379. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
  380. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
  381. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
  382. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
  383. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
  384. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
  385. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
  386. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
  387. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
  388. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
  389. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
  390. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
  391. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
  392. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
  393. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
  394. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
  395. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
  396. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
  397. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
  398. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
  399. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
  400. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
  401. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
  402. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
  403. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
  404. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
  405. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
  406. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
  407. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
  408. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
  409. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
  410. data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
  411. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  412. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  413. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  414. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  415. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  416. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  417. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  418. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  419. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  420. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  421. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  422. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  423. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  424. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  425. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  426. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  427. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  428. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  429. data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
  430. data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  431. data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
  432. data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
  433. data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
  434. data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
  435. data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
  436. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/cuda.h +1 -0
  437. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/hip.h +57 -0
  438. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/musa.h +7 -1
  439. data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
  440. data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
  441. data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
  442. data/ext/{ggml → sources/ggml}/src/ggml-impl.h +64 -19
  443. data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
  444. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
  445. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
  446. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
  447. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
  448. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
  449. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
  450. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
  451. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
  452. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
  453. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
  454. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
  455. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
  456. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
  457. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
  458. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
  459. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
  460. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
  461. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
  462. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
  463. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
  464. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
  465. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
  466. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
  467. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
  468. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
  469. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
  470. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
  471. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
  472. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
  473. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
  474. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
  475. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
  476. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
  477. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
  478. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
  479. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
  480. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
  481. data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
  482. data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
  483. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.m +2178 -1064
  484. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.metal +1575 -1218
  485. data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
  486. data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
  487. data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
  488. data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
  489. data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
  490. data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
  491. data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  492. data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
  493. data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
  494. data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  495. data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  496. data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
  497. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
  498. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
  499. data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
  500. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  501. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  502. data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
  503. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
  504. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
  505. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
  506. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
  507. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
  508. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  509. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  510. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  511. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  512. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  513. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  514. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
  515. data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
  516. data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  517. data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
  518. data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
  519. data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
  520. data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  521. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
  522. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
  523. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
  524. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
  525. data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
  526. data/ext/{ggml → sources/ggml}/src/ggml-opt.cpp +373 -190
  527. data/ext/{ggml → sources/ggml}/src/ggml-quants.c +114 -120
  528. data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
  529. data/ext/{ggml → sources/ggml}/src/ggml-rpc/ggml-rpc.cpp +480 -73
  530. data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
  531. data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
  532. data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
  533. data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
  534. data/ext/{ggml → sources/ggml}/src/ggml-sycl/common.cpp +20 -32
  535. data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
  536. data/ext/{ggml → sources/ggml}/src/ggml-sycl/concat.cpp +32 -33
  537. data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
  538. data/ext/{ggml → sources/ggml}/src/ggml-sycl/conv.cpp +4 -2
  539. data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
  540. data/ext/{ggml → sources/ggml}/src/ggml-sycl/convert.cpp +104 -28
  541. data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
  542. data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
  543. data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
  544. data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
  545. data/ext/{ggml → sources/ggml}/src/ggml-sycl/dmmv.cpp +156 -17
  546. data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  547. data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
  548. data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
  549. data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
  550. data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
  551. data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
  552. data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
  553. data/ext/{ggml → sources/ggml}/src/ggml-sycl/ggml-sycl.cpp +1004 -1240
  554. data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
  555. data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
  556. data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
  557. data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
  558. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmq.cpp +0 -1
  559. data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
  560. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmvq.cpp +261 -166
  561. data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
  562. data/ext/{ggml → sources/ggml}/src/ggml-sycl/norm.cpp +204 -81
  563. data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
  564. data/ext/{ggml → sources/ggml}/src/ggml-sycl/outprod.cpp +8 -17
  565. data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
  566. data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
  567. data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
  568. data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
  569. data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
  570. data/ext/{ggml → sources/ggml}/src/ggml-sycl/softmax.cpp +35 -25
  571. data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
  572. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
  573. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
  574. data/ext/{ggml → sources/ggml}/src/ggml-sycl/tsembd.cpp +3 -3
  575. data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  576. data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
  577. data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
  578. data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
  579. data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
  580. data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  581. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/ggml-vulkan.cpp +3130 -1087
  582. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
  583. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
  584. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
  585. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
  586. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
  587. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
  588. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  589. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
  590. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  591. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
  592. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  593. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
  594. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
  595. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  596. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  597. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
  598. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
  599. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
  600. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  601. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  602. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  603. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  604. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
  605. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
  606. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
  607. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  608. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  609. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  610. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  611. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  612. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  613. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  614. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  615. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  616. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  617. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  618. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
  619. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  620. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  621. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
  622. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
  623. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
  624. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
  625. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
  626. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
  627. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
  628. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
  629. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
  630. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
  631. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
  632. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
  633. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  634. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
  635. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  636. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
  637. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  638. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  639. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
  640. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
  641. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
  642. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
  643. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  644. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
  645. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  646. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
  647. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  648. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
  649. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
  650. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
  651. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  652. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
  653. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
  654. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  655. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
  656. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
  657. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
  658. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
  659. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  660. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  661. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
  662. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  663. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
  664. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
  665. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  666. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  667. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
  668. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  669. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
  670. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
  671. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
  672. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
  673. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
  674. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  675. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
  676. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
  677. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  678. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
  679. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
  680. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
  681. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
  682. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  683. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
  684. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
  685. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
  686. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
  687. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
  688. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
  689. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
  690. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
  691. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
  692. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +193 -35
  693. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  694. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  695. data/ext/{ggml → sources/ggml}/src/ggml.c +676 -1820
  696. data/ext/sources/ggml/src/gguf.cpp +1330 -0
  697. data/ext/{include → sources/include}/whisper.h +68 -2
  698. data/ext/sources/src/CMakeLists.txt +143 -0
  699. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.h +27 -15
  700. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.m +35 -10
  701. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.h +21 -9
  702. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.m +28 -3
  703. data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
  704. data/ext/sources/src/whisper-arch.h +197 -0
  705. data/ext/{src → sources/src}/whisper.cpp +1905 -374
  706. data/ext/sources/tests/CMakeLists.txt +105 -0
  707. data/ext/sources/tests/earnings21/eval.mk +58 -0
  708. data/ext/sources/tests/earnings21/eval.py +68 -0
  709. data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
  710. data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
  711. data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
  712. data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
  713. data/ext/sources/tests/earnings21/requirements.txt +6 -0
  714. data/ext/sources/tests/en-0-ref.txt +1 -0
  715. data/ext/sources/tests/en-1-ref.txt +1 -0
  716. data/ext/sources/tests/en-2-ref.txt +1 -0
  717. data/ext/sources/tests/es-0-ref.txt +1 -0
  718. data/ext/sources/tests/librispeech/eval.mk +39 -0
  719. data/ext/sources/tests/librispeech/eval.py +47 -0
  720. data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
  721. data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
  722. data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
  723. data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
  724. data/ext/sources/tests/librispeech/requirements.txt +6 -0
  725. data/ext/sources/tests/run-tests.sh +130 -0
  726. data/ext/sources/tests/test-c.c +3 -0
  727. data/ext/sources/tests/test-vad-full.cpp +54 -0
  728. data/ext/sources/tests/test-vad.cpp +83 -0
  729. data/ext/sources/tests/test-whisper.js +58 -0
  730. data/extsources.rb +33 -5
  731. data/lib/whisper/model/uri.rb +149 -128
  732. data/sig/whisper.rbs +480 -0
  733. data/tests/helper.rb +28 -0
  734. data/tests/test_callback.rb +45 -3
  735. data/tests/test_error.rb +2 -2
  736. data/tests/test_model.rb +38 -0
  737. data/tests/test_package.rb +18 -3
  738. data/tests/test_params.rb +145 -8
  739. data/tests/test_segment.rb +10 -19
  740. data/tests/test_vad.rb +19 -0
  741. data/tests/test_vad_params.rb +103 -0
  742. data/tests/test_whisper.rb +37 -37
  743. data/whispercpp.gemspec +5 -4
  744. metadata +766 -111
  745. data/ext/cpu.mk +0 -9
  746. data/ext/examples/dr_wav.h +0 -8815
  747. data/ext/ggml/src/ggml-cann/aclnn_ops.h +0 -592
  748. data/ext/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +0 -4262
  749. data/ext/ggml/src/ggml-cpu/ggml-cpu.c +0 -14123
  750. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.cpp +0 -1884
  751. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.h +0 -14
  752. data/ext/ggml/src/ggml-metal/ggml-metal-impl.h +0 -288
  753. data/ext/ggml/src/ggml-sycl/element_wise.cpp +0 -1030
  754. data/ext/ggml/src/ggml-sycl/im2col.cpp +0 -126
  755. data/ext/ggml/src/ggml-sycl/rope.cpp +0 -276
  756. data/ext/ggml/src/ggml-sycl/wkv6.cpp +0 -141
  757. data/ext/metal-embed.mk +0 -17
  758. data/ext/metal.mk +0 -6
  759. data/ext/ruby_whisper.cpp +0 -1909
  760. data/ext/scripts/get-flags.mk +0 -38
  761. data/lib/whisper.rb +0 -2
  762. /data/ext/{ggml → sources/ggml}/include/ggml-blas.h +0 -0
  763. /data/ext/{ggml → sources/ggml}/include/ggml-cann.h +0 -0
  764. /data/ext/{ggml → sources/ggml}/include/ggml-cuda.h +0 -0
  765. /data/ext/{ggml → sources/ggml}/include/ggml-kompute.h +0 -0
  766. /data/ext/{ggml → sources/ggml}/include/ggml-opencl.h +0 -0
  767. /data/ext/{ggml → sources/ggml}/include/ggml-sycl.h +0 -0
  768. /data/ext/{ggml → sources/ggml}/src/ggml-amx/common.h +0 -0
  769. /data/ext/{ggml → sources/ggml}/src/ggml-amx/ggml-amx.cpp +0 -0
  770. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.cpp +0 -0
  771. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.h +0 -0
  772. /data/ext/{ggml → sources/ggml}/src/ggml-blas/ggml-blas.cpp +0 -0
  773. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/ascendc_kernels.h +0 -0
  774. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f16.cpp +0 -0
  775. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f32.cpp +0 -0
  776. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q4_0.cpp +0 -0
  777. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q8_0.cpp +0 -0
  778. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +0 -0
  779. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +0 -0
  780. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +0 -0
  781. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.h +0 -0
  782. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/common.h +0 -0
  783. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.cpp +0 -0
  784. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.h +0 -0
  785. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-aarch64.h +0 -0
  786. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.cpp +0 -0
  787. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.h +0 -0
  788. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.h +0 -0
  789. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.cpp +0 -0
  790. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.h +0 -0
  791. /data/ext/{ggml → sources/ggml}/src/ggml-kompute/ggml-kompute.cpp +0 -0
  792. /data/ext/{ggml → sources/ggml}/src/ggml-quants.h +0 -0
  793. /data/ext/{ggml → sources/ggml}/src/ggml-threading.cpp +0 -0
  794. /data/ext/{ggml → sources/ggml}/src/ggml-threading.h +0 -0
  795. /data/ext/{src → sources/src}/coreml/whisper-encoder.h +0 -0
  796. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.cpp +0 -0
  797. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.h +0 -0
@@ -0,0 +1,1135 @@
1
+ #pragma once
2
+
3
+ #include "common.cuh"
4
+ #include <cstdint>
5
+
6
+ static __device__ __forceinline__ int get_int_b2(const void * x, const int & i32) {
7
+ const uint16_t * x16 = (const uint16_t *) x; // assume at least 2 byte alignment
8
+
9
+ int x32 = x16[2*i32 + 0] << 0;
10
+ x32 |= x16[2*i32 + 1] << 16;
11
+
12
+ return x32;
13
+ }
14
+
15
+ static __device__ __forceinline__ int get_int_b4(const void * x, const int & i32) {
16
+ return ((const int *) x)[i32]; // assume at least 4 byte alignment
17
+ }
18
+
19
+ // VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called
20
+ // MMVQ = mul_mat_vec_q, MMQ = mul_mat_q
21
+
22
+ #define VDR_Q4_0_Q8_1_MMVQ 2
23
+ #define VDR_Q4_0_Q8_1_MMQ 4
24
+
25
+ template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(
26
+ const int * v, const int * u, const float & d4, const half2 & ds8) {
27
+
28
+ int sumi = 0;
29
+
30
+ #pragma unroll
31
+ for (int i = 0; i < vdr; ++i) {
32
+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
33
+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
34
+
35
+ // SIMD dot product of quantized values
36
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);
37
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);
38
+ }
39
+
40
+ const float2 ds8f = __half22float2(ds8);
41
+
42
+ // second part effectively subtracts 8 from each quant value
43
+ return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);
44
+ }
45
+
46
+ #define VDR_Q4_1_Q8_1_MMVQ 2
47
+ #define VDR_Q4_1_Q8_1_MMQ 4
48
+
49
+ template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(
50
+ const int * v, const int * u, const half2 & dm4, const half2 & ds8) {
51
+
52
+ int sumi = 0;
53
+
54
+ #pragma unroll
55
+ for (int i = 0; i < vdr; ++i) {
56
+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;
57
+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;
58
+
59
+ // SIMD dot product of quantized values
60
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);
61
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);
62
+ }
63
+
64
+ #ifdef GGML_CUDA_F16
65
+ const float2 tmp = __half22float2(__hmul2(dm4, ds8));
66
+ const float d4d8 = tmp.x;
67
+ const float m4s8 = tmp.y;
68
+ #else
69
+ const float2 dm4f = __half22float2(dm4);
70
+ const float2 ds8f = __half22float2(ds8);
71
+ const float d4d8 = dm4f.x * ds8f.x;
72
+ const float m4s8 = dm4f.y * ds8f.y;
73
+ #endif // GGML_CUDA_F16
74
+
75
+ // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it
76
+ return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));
77
+ }
78
+
79
+ #define VDR_Q5_0_Q8_1_MMVQ 2
80
+ #define VDR_Q5_0_Q8_1_MMQ 4
81
+
82
+ template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(
83
+ const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {
84
+
85
+ int sumi = 0;
86
+
87
+ #pragma unroll
88
+ for (int i = 0; i < vdr; ++i) {
89
+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
90
+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
91
+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
92
+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
93
+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
94
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
95
+
96
+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
97
+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
98
+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
99
+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
100
+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
101
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
102
+ }
103
+
104
+ const float2 ds8f = __half22float2(ds8);
105
+
106
+ // second part effectively subtracts 16 from each quant value
107
+ return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);
108
+ }
109
+
110
+ #define VDR_Q5_1_Q8_1_MMVQ 2
111
+ #define VDR_Q5_1_Q8_1_MMQ 4
112
+
113
+ template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(
114
+ const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {
115
+
116
+ int sumi = 0;
117
+
118
+ #pragma unroll
119
+ for (int i = 0; i < vdr; ++i) {
120
+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits
121
+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4
122
+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12
123
+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20
124
+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28
125
+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values
126
+
127
+ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits
128
+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4
129
+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12
130
+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20
131
+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28
132
+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values
133
+ }
134
+
135
+ #ifdef GGML_CUDA_F16
136
+ const float2 tmp = __half22float2(__hmul2(dm5, ds8));
137
+ const float d5d8 = tmp.x;
138
+ const float m5s8 = tmp.y;
139
+ #else
140
+ const float2 dm5f = __half22float2(dm5);
141
+ const float2 ds8f = __half22float2(ds8);
142
+ const float d5d8 = dm5f.x * ds8f.x;
143
+ const float m5s8 = dm5f.y * ds8f.y;
144
+ #endif // GGML_CUDA_F16
145
+
146
+ // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it
147
+ return sumi*d5d8 + m5s8 / (QI5_1 / vdr);
148
+ }
149
+
150
+ #define VDR_Q8_0_Q8_1_MMVQ 2
151
+ #define VDR_Q8_0_Q8_1_MMQ 8
152
+
153
+ template <typename T, int vdr> static __device__ __forceinline__ T vec_dot_q8_0_q8_1_impl(
154
+ const int * v, const int * u, const T & d8_0, const T & d8_1) {
155
+
156
+ int sumi = 0;
157
+
158
+ #pragma unroll
159
+ for (int i = 0; i < vdr; ++i) {
160
+ // SIMD dot product of quantized values
161
+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
162
+ }
163
+
164
+ return d8_0*d8_1 * ((T) sumi);
165
+ }
166
+
167
+ template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(
168
+ const int * v, const int * u, const half2 & dm8, const half2 & ds8) {
169
+
170
+ int sumi = 0;
171
+
172
+ #pragma unroll
173
+ for (int i = 0; i < vdr; ++i) {
174
+ // SIMD dot product of quantized values
175
+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
176
+ }
177
+
178
+ #ifdef GGML_CUDA_F16
179
+ const float2 tmp = __half22float2(__hmul2(dm8, ds8));
180
+ const float d8d8 = tmp.x;
181
+ const float m8s8 = tmp.y;
182
+ #else
183
+ const float2 dm8f = __half22float2(dm8);
184
+ const float2 ds8f = __half22float2(ds8);
185
+ const float d8d8 = dm8f.x * ds8f.x;
186
+ const float m8s8 = dm8f.y * ds8f.y;
187
+ #endif // GGML_CUDA_F16
188
+
189
+ // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it
190
+ return sumi*d8d8 + m8s8 / (QI8_1 / vdr);
191
+ }
192
+
193
+ template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_16_q8_1_impl(
194
+ const int * v, const int * u, const float * d8_0, const float & d8_1) {
195
+
196
+ float sumf = 0.0f;
197
+
198
+ #pragma unroll
199
+ for (int i0 = 0; i0 < vdr; i0 += QI8_0/2) {
200
+ int sumi = 0;
201
+
202
+ #pragma unroll
203
+ for (int i = i0; i < i0 + QI8_0/2; ++i) {
204
+ // SIMD dot product of quantized values
205
+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);
206
+ }
207
+
208
+ sumf += d8_0[i0/(QI8_0/2)]*sumi;
209
+ }
210
+
211
+ return d8_1*sumf;
212
+ }
213
+
214
+ #define VDR_Q2_K_Q8_1_MMVQ 1
215
+ #define VDR_Q2_K_Q8_1_MMQ 4
216
+
217
+ // contiguous v/x values
218
+ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(
219
+ const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,
220
+ const half2 & dm2, const float * __restrict__ d8) {
221
+
222
+ float sumf_d = 0.0f;
223
+ float sumf_m = 0.0f;
224
+
225
+ #pragma unroll
226
+ for (int i = 0; i < QR2_K; ++i) {
227
+ const int sc = scales[2*i];
228
+
229
+ const int vi = (v >> (2*i)) & 0x03030303;
230
+
231
+ sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product
232
+
233
+ // fill int with 4x m
234
+ int m = sc >> 4;
235
+ m |= m << 8;
236
+ m |= m << 16;
237
+ sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values
238
+ }
239
+
240
+ const float2 dm2f = __half22float2(dm2);
241
+
242
+ return dm2f.x*sumf_d - dm2f.y*sumf_m;
243
+ }
244
+
245
+ // contiguous v/x + u/y values
246
+ template <int ns8>
247
+ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(
248
+ const int * __restrict__ v, const int * __restrict__ u, const half2 * dm2, const float & d8, const half2 * s8) {
249
+
250
+ float sumf = 0.0f;
251
+ float sumf_d8 = 0.0f;
252
+
253
+ #pragma unroll
254
+ for (int i0 = 0; i0 < QR2_K*VDR_Q2_K_Q8_1_MMQ; i0 += QI8_1) {
255
+ const float2 dm2f0 = __half22float2(dm2[i0/(QI8_1/2) + 0]);
256
+ int sumi_d0 = 0;
257
+
258
+ const float2 dm2f1 = __half22float2(dm2[i0/(QI8_1/2) + 1]);
259
+ int sumi_d1 = 0;
260
+
261
+ #pragma unroll
262
+ for (int i = i0; i < i0 + QI8_1/2; ++i) {
263
+ sumi_d0 = ggml_cuda_dp4a(v[i], u[i], sumi_d0);
264
+ }
265
+ sumf_d8 += dm2f0.x * sumi_d0;
266
+
267
+ #pragma unroll
268
+ for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {
269
+ sumi_d1 = ggml_cuda_dp4a(v[i], u[i], sumi_d1);
270
+ }
271
+ sumf_d8 += dm2f1.x * sumi_d1;
272
+
273
+ if (i0/QI8_1 < ns8) {
274
+ const float2 s8f = __half22float2(s8[i0/QI8_1]);
275
+ sumf -= dm2f0.y*s8f.x;
276
+ sumf -= dm2f1.y*s8f.y;
277
+ } else {
278
+ int sumi_m0 = 0;
279
+ #pragma unroll
280
+ for (int i = i0; i < i0 + QI8_1/2; ++i) {
281
+ sumi_m0 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m0);
282
+ }
283
+ sumf_d8 -= dm2f0.y * sumi_m0;
284
+
285
+ int sumi_m1 = 0;
286
+ #pragma unroll
287
+ for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {
288
+ sumi_m1 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m1);
289
+ }
290
+ sumf_d8 -= dm2f1.y * sumi_m1;
291
+ }
292
+ }
293
+
294
+ return sumf + d8*sumf_d8;
295
+ }
296
+
297
+ #define VDR_Q3_K_Q8_1_MMVQ 1
298
+ #define VDR_Q3_K_Q8_1_MMQ 2
299
+
300
+ // contiguous v/x values
301
+ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(
302
+ const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,
303
+ const int & scale_offset, const float & d3, const float * __restrict__ d8) {
304
+
305
+ float sumf = 0.0f;
306
+
307
+ #pragma unroll
308
+ for (int i = 0; i < QR3_K; ++i) {
309
+ const int isc = scale_offset + 2*i;
310
+
311
+ const int isc_low = isc % (QK_K/32);
312
+ const int sc_shift_low = 4 * (isc / (QK_K/32));
313
+ const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;
314
+
315
+ const int isc_high = isc % (QK_K/64);
316
+ const int sc_shift_high = 2 * (isc / (QK_K/64));
317
+ const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;
318
+
319
+ const int sc = (sc_low | sc_high) - 32;
320
+
321
+ const int vil = (vl >> (2*i)) & 0x03030303;
322
+
323
+ const int vih = ((vh >> i) << 2) & 0x04040404;
324
+
325
+ const int vi = __vsubss4(vil, vih);
326
+
327
+ sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
328
+ }
329
+
330
+ return d3 * sumf;
331
+ }
332
+
333
+ // contiguous v/x + u/y values
334
+ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(
335
+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,
336
+ const float & d3, const float & d8) {
337
+
338
+ int sumi = 0;
339
+
340
+ #pragma unroll
341
+ for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {
342
+ int sumi_sc = 0;
343
+
344
+ #pragma unroll
345
+ for (int i = i0; i < i0 + QI8_1/2; ++i) {
346
+ sumi_sc = ggml_cuda_dp4a(v[i], u[i], sumi_sc); // SIMD dot product
347
+ }
348
+
349
+ sumi += sumi_sc * scales[i0 / (QI8_1/2)];
350
+ }
351
+
352
+ return d3*d8 * sumi;
353
+ }
354
+
355
+ #define VDR_Q4_K_Q8_1_MMVQ 2
356
+ #define VDR_Q4_K_Q8_1_MMQ 8
357
+
358
+ // contiguous v/x values
359
+ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(
360
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
361
+ const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {
362
+
363
+ float sumf_d = 0.0f;
364
+ float sumf_m = 0.0f;
365
+
366
+ #pragma unroll
367
+ for (int i = 0; i < QR4_K; ++i) {
368
+ const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;
369
+ const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;
370
+
371
+ const int dot1 = ggml_cuda_dp4a(v1i, u[2*i+1], ggml_cuda_dp4a(v0i, u[2*i+0], 0)); // SIMD dot product
372
+ const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+1], ggml_cuda_dp4a(0x01010101, u[2*i+0], 0)); // sum of u
373
+
374
+ sumf_d += d8[i] * (dot1 * sc[i]);
375
+ sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values
376
+ }
377
+
378
+ const float2 dm4f = __half22float2(dm4);
379
+
380
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
381
+ }
382
+
383
+ // contiguous v/x + u/y values
384
+ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(
385
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
386
+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
387
+
388
+ float sumf_d = 0.0f;
389
+ float sumf_m = 0.0f;
390
+
391
+ #pragma unroll
392
+ for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {
393
+ int sumi_d = 0;
394
+
395
+ #pragma unroll
396
+ for (int j = 0; j < QI8_1; ++j) {
397
+ sumi_d = ggml_cuda_dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product
398
+ }
399
+
400
+ const float2 ds8f = __half22float2(ds8[i]);
401
+
402
+ sumf_d += ds8f.x * (sc[i] * sumi_d);
403
+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
404
+ }
405
+
406
+ const float2 dm4f = __half22float2(dm4);
407
+
408
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
409
+ }
410
+
411
+ #define VDR_Q5_K_Q8_1_MMVQ 2
412
+ #define VDR_Q5_K_Q8_1_MMQ 8
413
+
414
+ // contiguous v/x values
415
+ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(
416
+ const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,
417
+ const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {
418
+
419
+ float sumf_d = 0.0f;
420
+ float sumf_m = 0.0f;
421
+
422
+ #pragma unroll
423
+ for (int i = 0; i < QR5_K; ++i) {
424
+ const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;
425
+ const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;
426
+
427
+ const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;
428
+ const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;
429
+
430
+ const int v0i = vl0i | vh0i;
431
+ const int v1i = vl1i | vh1i;
432
+
433
+ const int dot1 = ggml_cuda_dp4a(v0i, u[2*i+0], ggml_cuda_dp4a(v1i, u[2*i+1], 0)); // SIMD dot product
434
+ const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+0], ggml_cuda_dp4a(0x01010101, u[2*i+1], 0)); // sum of u
435
+
436
+ sumf_d += d8[i] * (dot1 * sc[i]);
437
+ sumf_m += d8[i] * (dot2 * m[i]);
438
+
439
+ }
440
+
441
+ const float2 dm5f = __half22float2(dm5);
442
+
443
+ return dm5f.x*sumf_d - dm5f.y*sumf_m;
444
+ }
445
+
446
+ // contiguous v/x + u/y values
447
+ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(
448
+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,
449
+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {
450
+
451
+ float sumf_d = 0.0f;
452
+ float sumf_m = 0.0f;
453
+
454
+ #pragma unroll
455
+ for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {
456
+ int sumi_d = 0;
457
+
458
+ #pragma unroll
459
+ for (int j = 0; j < QI8_1; ++j) {
460
+ sumi_d = ggml_cuda_dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product
461
+ }
462
+
463
+ const float2 ds8f = __half22float2(ds8[i]);
464
+
465
+ sumf_d += ds8f.x * (sc[i] * sumi_d);
466
+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val
467
+ }
468
+
469
+ const float2 dm4f = __half22float2(dm4);
470
+
471
+ return dm4f.x*sumf_d - dm4f.y*sumf_m;
472
+ }
473
+
474
+ #define VDR_Q6_K_Q8_1_MMVQ 1
475
+ #define VDR_Q6_K_Q8_1_MMQ 8
476
+
477
+ // contiguous v/x values
478
+ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(
479
+ const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,
480
+ const float & d, const float * __restrict__ d8) {
481
+
482
+ float sumf = 0.0f;
483
+
484
+ #pragma unroll
485
+ for (int i = 0; i < QR6_K; ++i) {
486
+ const int sc = scales[4*i];
487
+
488
+ const int vil = (vl >> (4*i)) & 0x0F0F0F0F;
489
+
490
+ const int vih = ((vh >> (4*i)) << 4) & 0x30303030;
491
+
492
+ const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32
493
+
494
+ sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product
495
+ }
496
+
497
+ return d*sumf;
498
+ }
499
+
500
+ // contiguous v/x + u/y values
501
+ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(
502
+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,
503
+ const float & d6, const float * __restrict__ d8) {
504
+
505
+ float sumf_d = 0.0f;
506
+
507
+ const int sc_packed = get_int_b4(sc, 0);
508
+ const int8_t * sc_reg = (const int8_t *) &sc_packed;
509
+
510
+ #pragma unroll
511
+ for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {
512
+ int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale
513
+
514
+ #pragma unroll
515
+ for (int i = i0; i < i0 + 2; ++i) {
516
+ sumi_d.x = ggml_cuda_dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product
517
+ sumi_d.x = ggml_cuda_dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product
518
+
519
+ sumi_d.y = ggml_cuda_dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product
520
+ sumi_d.y = ggml_cuda_dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product
521
+ }
522
+
523
+ sumf_d += d8[i0/4] * (sc_reg[i0/2+0]*sumi_d.x + sc_reg[i0/2+1]*sumi_d.y);
524
+ }
525
+
526
+ return d6 * sumf_d;
527
+ }
528
+
529
+ static __device__ __forceinline__ float vec_dot_q4_0_q8_1(
530
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
531
+
532
+ const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq + kbx;
533
+
534
+ int v[VDR_Q4_0_Q8_1_MMVQ];
535
+ int u[2*VDR_Q4_0_Q8_1_MMVQ];
536
+
537
+ #pragma unroll
538
+ for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {
539
+ v[i] = get_int_b2(bq4_0->qs, iqs + i);
540
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
541
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_0);
542
+ }
543
+
544
+ return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, bq4_0->d, bq8_1->ds);
545
+ }
546
+
547
+
548
+ static __device__ __forceinline__ float vec_dot_q4_1_q8_1(
549
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
550
+
551
+ const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq + kbx;
552
+
553
+ int v[VDR_Q4_1_Q8_1_MMVQ];
554
+ int u[2*VDR_Q4_1_Q8_1_MMVQ];
555
+
556
+ #pragma unroll
557
+ for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {
558
+ v[i] = get_int_b4(bq4_1->qs, iqs + i);
559
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
560
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_1);
561
+ }
562
+
563
+ return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);
564
+ }
565
+
566
+ static __device__ __forceinline__ float vec_dot_q5_0_q8_1(
567
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
568
+
569
+ const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq + kbx;
570
+
571
+ int vl[VDR_Q5_0_Q8_1_MMVQ];
572
+ int vh[VDR_Q5_0_Q8_1_MMVQ];
573
+ int u[2*VDR_Q5_0_Q8_1_MMVQ];
574
+
575
+ #pragma unroll
576
+ for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {
577
+ vl[i] = get_int_b2(bq5_0->qs, iqs + i);
578
+ vh[i] = get_int_b2(bq5_0->qh, 0) >> (4 * (iqs + i));
579
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
580
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_0);
581
+ }
582
+
583
+ return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, bq5_0->d, bq8_1->ds);
584
+ }
585
+
586
+ static __device__ __forceinline__ float vec_dot_q5_1_q8_1(
587
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
588
+
589
+ const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq + kbx;
590
+
591
+ int vl[VDR_Q5_1_Q8_1_MMVQ];
592
+ int vh[VDR_Q5_1_Q8_1_MMVQ];
593
+ int u[2*VDR_Q5_1_Q8_1_MMVQ];
594
+
595
+ #pragma unroll
596
+ for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {
597
+ vl[i] = get_int_b4(bq5_1->qs, iqs + i);
598
+ vh[i] = get_int_b4(bq5_1->qh, 0) >> (4 * (iqs + i));
599
+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);
600
+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_1);
601
+ }
602
+
603
+ return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);
604
+ }
605
+
606
+ static __device__ __forceinline__ float vec_dot_q8_0_q8_1(
607
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
608
+
609
+ const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq + kbx;
610
+
611
+ int v[VDR_Q8_0_Q8_1_MMVQ];
612
+ int u[VDR_Q8_0_Q8_1_MMVQ];
613
+
614
+ #pragma unroll
615
+ for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {
616
+ v[i] = get_int_b2(bq8_0->qs, iqs + i);
617
+ u[i] = get_int_b4(bq8_1->qs, iqs + i);
618
+ }
619
+
620
+ return vec_dot_q8_0_q8_1_impl<float, VDR_Q8_0_Q8_1_MMVQ>(v, u, bq8_0->d, __low2half(bq8_1->ds));
621
+ }
622
+
623
+ static __device__ __forceinline__ float vec_dot_q2_K_q8_1(
624
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
625
+
626
+ const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx;
627
+
628
+ const int bq8_offset = QR2_K * (iqs / QI8_1);
629
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
630
+
631
+ const uint8_t * scales = bq2_K->scales + scale_offset;
632
+
633
+ const int v = get_int_b4(bq2_K->qs, iqs);
634
+ int u[QR2_K];
635
+ float d8[QR2_K];
636
+
637
+ #pragma unroll
638
+ for (int i = 0; i < QR2_K; ++ i) {
639
+ u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
640
+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
641
+ }
642
+
643
+ return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);
644
+ }
645
+
646
+ static __device__ __forceinline__ float vec_dot_q3_K_q8_1(
647
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
648
+
649
+ const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx;
650
+
651
+ const int bq8_offset = QR3_K * (iqs / (QI3_K/2));
652
+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);
653
+
654
+ const float d = bq3_K->d;
655
+
656
+ const int vl = get_int_b2(bq3_K->qs, iqs);
657
+
658
+ // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted
659
+ const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;
660
+
661
+ int u[QR3_K];
662
+ float d8[QR3_K];
663
+
664
+ #pragma unroll
665
+ for (int i = 0; i < QR3_K; ++i) {
666
+ u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);
667
+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);
668
+ }
669
+
670
+ return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);
671
+ }
672
+
673
+ static __device__ __forceinline__ float vec_dot_q4_K_q8_1(
674
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
675
+
676
+ const block_q4_K * bq4_K = (const block_q4_K *) vbq + kbx;
677
+
678
+ int v[2];
679
+ int u[2*QR4_K];
680
+ float d8[QR4_K];
681
+
682
+ // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6
683
+ const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));
684
+
685
+ // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12
686
+ // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44
687
+ // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76
688
+ // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108
689
+
690
+ const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
691
+ v[0] = q4[0];
692
+ v[1] = q4[4];
693
+
694
+ const uint16_t * scales = (const uint16_t *)bq4_K->scales;
695
+ uint16_t aux[2];
696
+ const int j = bq8_offset/2;
697
+ if (j < 2) {
698
+ aux[0] = scales[j+0] & 0x3f3f;
699
+ aux[1] = scales[j+2] & 0x3f3f;
700
+ } else {
701
+ aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
702
+ aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
703
+ }
704
+ const uint8_t * sc = (const uint8_t *)aux;
705
+ const uint8_t * m = sc + 2;
706
+
707
+ for (int i = 0; i < QR4_K; ++i) {
708
+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
709
+ d8[i] = __low2float(bq8i->ds);
710
+
711
+ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
712
+ u[2*i+0] = q8[0];
713
+ u[2*i+1] = q8[4];
714
+ }
715
+
716
+ return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);
717
+ }
718
+
719
+ static __device__ __forceinline__ float vec_dot_q5_K_q8_1(
720
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
721
+
722
+ const block_q5_K * bq5_K = (const block_q5_K *) vbq + kbx;
723
+
724
+ int vl[2];
725
+ int vh[2];
726
+ int u[2*QR5_K];
727
+ float d8[QR5_K];
728
+
729
+ const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));
730
+ const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));
731
+ const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));
732
+
733
+ vl[0] = ql[0];
734
+ vl[1] = ql[4];
735
+
736
+ vh[0] = qh[0] >> bq8_offset;
737
+ vh[1] = qh[4] >> bq8_offset;
738
+
739
+ const uint16_t * scales = (const uint16_t *)bq5_K->scales;
740
+ uint16_t aux[2];
741
+ const int j = bq8_offset/2;
742
+ if (j < 2) {
743
+ aux[0] = scales[j+0] & 0x3f3f;
744
+ aux[1] = scales[j+2] & 0x3f3f;
745
+ } else {
746
+ aux[0] = ((scales[j+2] >> 0) & 0x0f0f) | ((scales[j-2] & 0xc0c0) >> 2);
747
+ aux[1] = ((scales[j+2] >> 4) & 0x0f0f) | ((scales[j-0] & 0xc0c0) >> 2);
748
+ }
749
+ const uint8_t * sc = (const uint8_t *)aux;
750
+ const uint8_t * m = sc + 2;
751
+
752
+ #pragma unroll
753
+ for (int i = 0; i < QR5_K; ++i) {
754
+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;
755
+ d8[i] = __low2float(bq8i->ds);
756
+
757
+ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);
758
+ u[2*i+0] = q8[0];
759
+ u[2*i+1] = q8[4];
760
+ }
761
+
762
+ return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);
763
+ }
764
+
765
+ static __device__ __forceinline__ float vec_dot_q6_K_q8_1(
766
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
767
+
768
+ const block_q6_K * bq6_K = (const block_q6_K *) vbq + kbx;
769
+
770
+ const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);
771
+ const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);
772
+ const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));
773
+
774
+ const int vl = get_int_b2(bq6_K->ql, iqs);
775
+ const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;
776
+
777
+ const int8_t * scales = bq6_K->scales + scale_offset;
778
+
779
+ int u[QR6_K];
780
+ float d8[QR6_K];
781
+
782
+ #pragma unroll
783
+ for (int i = 0; i < QR6_K; ++i) {
784
+ u[i] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);
785
+ d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);
786
+ }
787
+
788
+ return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8);
789
+ }
790
+
791
+ #define VDR_IQ2_XXS_Q8_1_MMVQ 2
792
+ #define VDR_IQ2_XXS_Q8_1_MMQ 2
793
+
794
+ static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(
795
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
796
+
797
+ const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq + kbx;
798
+
799
+ const int q2 = get_int_b2(bq2->qs, iqs);
800
+ const uint8_t * aux8 = (const uint8_t *) &q2;
801
+ const uint32_t aux32 = get_int_b2(bq2->qs, iqs + 1);
802
+
803
+ int sumi = 0;
804
+ #pragma unroll
805
+ for (int k0 = 0; k0 < 8; k0 += 2) {
806
+ const int * grid_pos = (const int *) (iq2xxs_grid + aux8[k0/2]);
807
+ const int signs_packed = ksigns_iq2xs[(aux32 >> (7*k0/2)) & 0x7F];
808
+
809
+ const int signs0 = __vcmpne4(((signs_packed & 0x03) << 7) | ((signs_packed & 0x0C) << 21), 0x00000000);
810
+ const int grid0 = __vsub4(grid_pos[0] ^ signs0, signs0);
811
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, k0 + 0);
812
+ sumi = ggml_cuda_dp4a(grid0, u0, sumi);
813
+
814
+ const int signs1 = __vcmpne4(((signs_packed & 0x30) << 3) | ((signs_packed & 0xC0) << 17), 0x00000000);
815
+ const int grid1 = __vsub4(grid_pos[1] ^ signs1, signs1);
816
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, k0 + 1);
817
+ sumi = ggml_cuda_dp4a(grid1, u1, sumi);
818
+ }
819
+
820
+ const int ls = aux32 >> 28;
821
+ sumi = (ls*sumi + sumi/2)/4;
822
+ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
823
+ return d * sumi;
824
+ }
825
+
826
+ #define VDR_IQ2_XS_Q8_1_MMVQ 2
827
+ #define VDR_IQ2_XS_Q8_1_MMQ 2
828
+
829
+ static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(
830
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
831
+
832
+ const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq + kbx;
833
+
834
+ const int2 q2_packed = make_int2(get_int_b2(bq2->qs, iqs + 0), get_int_b2(bq2->qs, iqs + 1));
835
+ const uint16_t * q2 = (const uint16_t *) &q2_packed;
836
+ const int ls0 = bq2->scales[iqs/2] & 0x0F;
837
+ const int ls1 = bq2->scales[iqs/2] >> 4;
838
+
839
+ int sumi0 = 0;
840
+ int sumi1 = 0;
841
+ #pragma unroll
842
+ for (int l0 = 0; l0 < 8; l0 += 2) {
843
+ const uint32_t * grid_pos = (const uint32_t *)(iq2xs_grid + (q2[l0/2] & 0x000001FF));
844
+ const uint32_t * signs = (const uint32_t *)(ksigns64 + (q2[l0/2] >> 9));
845
+
846
+ const int grid_l = __vsub4(grid_pos[0] ^ signs[0], signs[0]);
847
+ const int grid_h = __vsub4(grid_pos[1] ^ signs[1], signs[1]);
848
+
849
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
850
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
851
+
852
+ if (l0 < 4) {
853
+ sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);
854
+ sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);
855
+ } else {
856
+ sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);
857
+ sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);
858
+ }
859
+ }
860
+ const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;
861
+ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
862
+ return d * sumi;
863
+ }
864
+
865
+ #define VDR_IQ2_S_Q8_1_MMVQ 2
866
+ #define VDR_IQ2_S_Q8_1_MMQ 2
867
+
868
+ static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(
869
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
870
+
871
+ const block_iq2_s * bq2 = (const block_iq2_s *) vbq + kbx;
872
+
873
+ const int qs_packed = get_int_b2(bq2->qs, iqs/2);
874
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
875
+
876
+ const int qh = bq2->qh[iqs/2];
877
+
878
+ const int signs_packed_32 = get_int_b2(bq2->qs, QK_K/32 + iqs/2);
879
+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
880
+
881
+ const int ls0 = bq2->scales[iqs/2] & 0x0F;
882
+ const int ls1 = bq2->scales[iqs/2] >> 4;
883
+
884
+ int sumi0 = 0;
885
+ int sumi1 = 0;
886
+ #pragma unroll
887
+ for (int l0 = 0; l0 < 8; l0 += 2) {
888
+ const int * grid_pos = (const int *)(iq2s_grid + (qs[l0/2] | ((qh << (8-l0)) & 0x300)));
889
+
890
+ const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);
891
+ const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);
892
+
893
+ const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);
894
+ const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);
895
+
896
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
897
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
898
+
899
+ if (l0 < 4) {
900
+ sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);
901
+ sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);
902
+ } else {
903
+ sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);
904
+ sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);
905
+ }
906
+ }
907
+ const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;
908
+
909
+ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);
910
+ return d * sumi;
911
+ }
912
+
913
+ #define VDR_IQ3_XXS_Q8_1_MMVQ 2
914
+ #define VDR_IQ3_XXS_Q8_1_MMQ 2
915
+
916
+ static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(
917
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
918
+
919
+ const block_iq3_xxs * bq3 = (const block_iq3_xxs *) vbq + kbx;
920
+
921
+ const int2 q3_packed = make_int2(get_int_b2(bq3->qs, iqs), get_int_b2(bq3->qs, iqs+1));
922
+ const uint8_t * q3 = (const uint8_t *) &q3_packed;
923
+ const uint32_t aux32 = get_int_b2(bq3->qs, QK_K/16 + iqs/2);
924
+
925
+ int sumi = 0;
926
+ #pragma unroll
927
+ for (int l0 = 0; l0 < 8; l0 += 2) {
928
+ const int2 grid_pos = make_int2(iq3xxs_grid[q3[l0 + 0]], iq3xxs_grid[q3[l0 + 1]]);
929
+
930
+ const int * signs = (const int *)(ksigns64 + ((aux32 >> (7*l0/2)) & 0x7F));
931
+
932
+ const int grid_l = __vsub4(grid_pos.x ^ signs[0], signs[0]);
933
+ const int grid_h = __vsub4(grid_pos.y ^ signs[1], signs[1]);
934
+
935
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
936
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
937
+
938
+ sumi = ggml_cuda_dp4a(grid_l, u0, sumi);
939
+ sumi = ggml_cuda_dp4a(grid_h, u1, sumi);
940
+ }
941
+
942
+ const int ls = aux32 >> 28;
943
+ sumi = (ls*sumi + sumi/2)/2;
944
+ const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);
945
+ return d * sumi;
946
+ }
947
+
948
+ #define VDR_IQ3_S_Q8_1_MMVQ 2
949
+ #define VDR_IQ3_S_Q8_1_MMQ 2
950
+
951
+ // TODO: don't use lookup table for signs
952
+ static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
953
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
954
+
955
+ const block_iq3_s * bq3 = (const block_iq3_s *) vbq + kbx;
956
+
957
+ const int2 qs_packed = make_int2(get_int_b2(bq3->qs, iqs + 0), get_int_b2(bq3->qs, iqs + 1));
958
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
959
+
960
+ const int qh = bq3->qh[iqs/2];
961
+
962
+ const int signs_packed_32 = get_int_b2(bq3->signs, iqs/2);
963
+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
964
+
965
+ int sumi = 0;
966
+ #pragma unroll
967
+ for (int l0 = 0; l0 < 8; l0 += 2) {
968
+ const int2 grid_pos = make_int2(
969
+ iq3s_grid[qs[l0 + 0] | ((qh << (8 - l0)) & 0x100)],
970
+ iq3s_grid[qs[l0 + 1] | ((qh << (7 - l0)) & 0x100)]);
971
+
972
+ const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);
973
+ const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);
974
+
975
+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);
976
+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);
977
+
978
+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
979
+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
980
+
981
+ sumi = ggml_cuda_dp4a(grid_l, u0, sumi);
982
+ sumi = ggml_cuda_dp4a(grid_h, u1, sumi);
983
+ }
984
+
985
+ sumi *= 1 + 2*((bq3->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F);
986
+
987
+ const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);
988
+ return d * sumi;
989
+ }
990
+
991
+ #define VDR_IQ1_S_Q8_1_MMVQ 1
992
+ #define VDR_IQ1_S_Q8_1_MMQ 1
993
+
994
+ static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(
995
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
996
+ const block_iq1_s * bq1 = (const block_iq1_s *) vbq + kbx;
997
+
998
+ const int qs_packed = get_int_b2(bq1->qs, iqs);
999
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
1000
+
1001
+ const int qh = bq1->qh[iqs];
1002
+
1003
+ int sumi = 0;
1004
+ #pragma unroll
1005
+ for (int l0 = 0; l0 < 8; l0 += 2) {
1006
+ const int grid = iq1s_grid_gpu[qs[l0/2] | (((qh >> 3*(l0/2)) & 0x07) << 8)];
1007
+
1008
+ const int grid0 = (grid >> 0) & 0x0F0F0F0F;
1009
+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;
1010
+
1011
+ const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
1012
+ const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
1013
+
1014
+ sumi = ggml_cuda_dp4a(grid0, u0, sumi);
1015
+ sumi = ggml_cuda_dp4a(grid1, u1, sumi);
1016
+ }
1017
+
1018
+ const float d1q = __half2float(bq1->d) * (((qh >> 11) & 0x0E) + 1);
1019
+ const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);
1020
+ const float2 ds = __half22float2(bq8_1[iqs].ds);
1021
+ return d1q * (ds.x*sumi + ds.y*delta);
1022
+ }
1023
+
1024
+ #define VDR_IQ1_M_Q8_1_MMVQ 1
1025
+ #define VDR_IQ1_M_Q8_1_MMQ 1
1026
+
1027
+ static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(
1028
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
1029
+
1030
+ const block_iq1_m * bq1 = (const block_iq1_m *) vbq + kbx;
1031
+
1032
+ const int qs_packed = get_int_b4(bq1->qs, iqs);
1033
+ const uint8_t * qs = (const uint8_t *) &qs_packed;
1034
+
1035
+ int sumi[2] = {0};
1036
+ float sumf[2] = {0.0f};
1037
+ #pragma unroll
1038
+ for (int l0 = 0; l0 < 8; l0 += 2) {
1039
+ const int qhl = bq1->qh[2*iqs + l0/4] >> (4 * ((l0/2) % 2));
1040
+
1041
+ const int grid = iq1s_grid_gpu[qs[l0/2] | ((qhl & 0x07) << 8)];
1042
+
1043
+ const int grid0 = (grid >> 0) & 0x0F0F0F0F;
1044
+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;
1045
+
1046
+ const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);
1047
+ const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);
1048
+
1049
+ sumi[l0/4] = ggml_cuda_dp4a(grid0, u0, sumi[l0/4]);
1050
+ sumi[l0/4] = ggml_cuda_dp4a(grid1, u1, sumi[l0/4]);
1051
+
1052
+ const float delta = -1.0f + IQ1M_DELTA - (qhl & 0x08) * (2.0f*IQ1M_DELTA/0x08);
1053
+ int sumy = 0;
1054
+ sumy = ggml_cuda_dp4a(u0, 0x01010101, sumy);
1055
+ sumy = ggml_cuda_dp4a(u1, 0x01010101, sumy);
1056
+ sumf[l0/4] += delta*sumy;
1057
+ }
1058
+
1059
+ const uint16_t * sc = (const uint16_t *) bq1->scales;
1060
+
1061
+ iq1m_scale_t scale;
1062
+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);
1063
+ const float d = __half2float(scale.f16) * __low2float(bq8_1[iqs].ds);
1064
+
1065
+ const int tmp = sc[iqs/2] >> (6*(iqs%2));
1066
+ const int sc0 = 2*((tmp >> 0) & 0x07) + 1;
1067
+ const int sc1 = 2*((tmp >> 3) & 0x07) + 1;
1068
+ return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1);
1069
+ }
1070
+
1071
+ static __device__ __forceinline__ int2 get_int_from_table_16(const int & q4) {
1072
+ const int q0_32 = (q4 >> 0) & 0x0F0F0F0F;
1073
+ const int8_t * q0_8 = (const int8_t *) &q0_32;
1074
+ const char4 val0_8 = make_char4(
1075
+ kvalues_iq4nl[q0_8[0]], kvalues_iq4nl[q0_8[1]], kvalues_iq4nl[q0_8[2]], kvalues_iq4nl[q0_8[3]]);
1076
+
1077
+ const int q1_32 = (q4 >> 4) & 0x0F0F0F0F;
1078
+ const int8_t * q1_8 = (const int8_t *) &q1_32;
1079
+ const char4 val1_8 = make_char4(
1080
+ kvalues_iq4nl[q1_8[0]], kvalues_iq4nl[q1_8[1]], kvalues_iq4nl[q1_8[2]], kvalues_iq4nl[q1_8[3]]);
1081
+
1082
+ return make_int2(*((const int *) &val0_8), *((const int *) &val1_8));
1083
+ }
1084
+
1085
+ #define VDR_IQ4_NL_Q8_1_MMVQ 2
1086
+ #define VDR_IQ4_NL_Q8_1_MMQ 4
1087
+
1088
+ static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(
1089
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
1090
+
1091
+ const block_iq4_nl * bq4 = (const block_iq4_nl *) vbq + kbx;
1092
+
1093
+ const int * q8 = (const int *) bq8_1->qs + iqs;
1094
+
1095
+ int sumi = 0;
1096
+ #pragma unroll
1097
+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {
1098
+ const int aux_q4 = get_int_b2(bq4->qs, iqs + l);
1099
+ const int2 v = get_int_from_table_16(aux_q4);
1100
+
1101
+ sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi);
1102
+ sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi);
1103
+ }
1104
+
1105
+ const float d = __half2float(bq4->d) * __low2float(bq8_1->ds);
1106
+ return d * sumi;
1107
+ }
1108
+
1109
+ #define VDR_IQ4_XS_Q8_1_MMVQ 4
1110
+ #define VDR_IQ4_XS_Q8_1_MMQ 4
1111
+
1112
+ static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(
1113
+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
1114
+
1115
+ const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq + kbx;
1116
+
1117
+ int sumi = 0;
1118
+ #pragma unroll
1119
+ for (int j = 0; j < 4; ++j) {
1120
+ const int aux_q4 = get_int_b4(bq4->qs, iqs + j);
1121
+ const int2 v = get_int_from_table_16(aux_q4);
1122
+
1123
+ const int u0 = get_int_b4(bq8_1[iqs/4].qs, j + 0);
1124
+ const int u1 = get_int_b4(bq8_1[iqs/4].qs, j + 4);
1125
+
1126
+ sumi = ggml_cuda_dp4a(v.x, u0, sumi);
1127
+ sumi = ggml_cuda_dp4a(v.y, u1, sumi);
1128
+ }
1129
+
1130
+ const int ls = ((bq4->scales_l[iqs/8] >> (iqs & 0x04)) & 0x0F) | (((bq4->scales_h >> (iqs/2)) & 0x03) << 4);
1131
+ sumi *= ls - 32;
1132
+
1133
+ const float d = __half2float(bq4->d) * __low2float(bq8_1[iqs/4].ds);
1134
+ return d * sumi;
1135
+ }