whispercpp 1.3.1 → 1.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (797) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -3
  3. data/README.md +92 -31
  4. data/Rakefile +26 -7
  5. data/ext/.gitignore +5 -7
  6. data/ext/dependencies.rb +61 -0
  7. data/ext/extconf.rb +21 -198
  8. data/ext/options.rb +221 -0
  9. data/ext/ruby_whisper.c +159 -0
  10. data/ext/ruby_whisper.h +17 -2
  11. data/ext/ruby_whisper_context.c +641 -0
  12. data/ext/ruby_whisper_error.c +52 -0
  13. data/ext/ruby_whisper_model.c +232 -0
  14. data/ext/ruby_whisper_params.c +1301 -0
  15. data/ext/ruby_whisper_segment.c +143 -0
  16. data/ext/ruby_whisper_transcribe.cpp +87 -0
  17. data/ext/ruby_whisper_vad_params.c +288 -0
  18. data/ext/sources/.dockerignore +3 -0
  19. data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
  20. data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
  21. data/ext/sources/CMakeLists.txt +251 -0
  22. data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
  23. data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
  24. data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
  25. data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
  26. data/ext/sources/bindings/javascript/package.json +26 -0
  27. data/ext/sources/bindings/javascript/whisper.js +19 -0
  28. data/ext/sources/build-xcframework.sh +547 -0
  29. data/ext/sources/ci/run.sh +336 -0
  30. data/ext/sources/close-issue.yml +28 -0
  31. data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
  32. data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
  33. data/ext/sources/cmake/build-info.cmake +60 -0
  34. data/ext/sources/cmake/git-vars.cmake +22 -0
  35. data/ext/sources/cmake/whisper-config.cmake.in +65 -0
  36. data/ext/sources/cmake/whisper.pc.in +10 -0
  37. data/ext/sources/examples/CMakeLists.txt +124 -0
  38. data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
  39. data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
  40. data/ext/sources/examples/addon.node/addon.cpp +438 -0
  41. data/ext/sources/examples/addon.node/index.js +54 -0
  42. data/ext/sources/examples/addon.node/package.json +16 -0
  43. data/ext/sources/examples/bench/CMakeLists.txt +8 -0
  44. data/ext/sources/examples/bench/bench.cpp +175 -0
  45. data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
  46. data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
  47. data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
  48. data/ext/sources/examples/cli/CMakeLists.txt +8 -0
  49. data/ext/sources/examples/cli/cli.cpp +1294 -0
  50. data/ext/sources/examples/coi-serviceworker.js +146 -0
  51. data/ext/sources/examples/command/CMakeLists.txt +10 -0
  52. data/ext/sources/examples/command/command.cpp +776 -0
  53. data/ext/sources/examples/command/commands.txt +9 -0
  54. data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
  55. data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
  56. data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
  57. data/ext/sources/examples/common-ggml.cpp +238 -0
  58. data/ext/sources/examples/common-ggml.h +18 -0
  59. data/ext/sources/examples/common-sdl.cpp +227 -0
  60. data/ext/sources/examples/common-sdl.h +49 -0
  61. data/ext/sources/examples/common-whisper.cpp +168 -0
  62. data/ext/sources/examples/common-whisper.h +24 -0
  63. data/ext/sources/examples/common.cpp +675 -0
  64. data/ext/sources/examples/common.h +322 -0
  65. data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
  66. data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
  67. data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
  68. data/ext/sources/examples/generate-karaoke.sh +57 -0
  69. data/ext/sources/examples/grammar-parser.cpp +423 -0
  70. data/ext/sources/examples/grammar-parser.h +29 -0
  71. data/ext/sources/examples/helpers.js +191 -0
  72. data/ext/sources/examples/json.hpp +24596 -0
  73. data/ext/sources/examples/livestream.sh +112 -0
  74. data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
  75. data/ext/sources/examples/lsp/lsp.cpp +467 -0
  76. data/ext/sources/examples/lsp/whisper.vim +362 -0
  77. data/ext/sources/examples/miniaudio.h +93468 -0
  78. data/ext/sources/examples/python/test_whisper_processor.py +7 -0
  79. data/ext/sources/examples/python/whisper_processor.py +54 -0
  80. data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
  81. data/ext/sources/examples/quantize/quantize.cpp +223 -0
  82. data/ext/sources/examples/server/CMakeLists.txt +12 -0
  83. data/ext/sources/examples/server/bench.js +29 -0
  84. data/ext/sources/examples/server/httplib.h +10497 -0
  85. data/ext/sources/examples/server/server.cpp +1091 -0
  86. data/ext/sources/examples/server.py +115 -0
  87. data/ext/sources/examples/stb_vorbis.c +5584 -0
  88. data/ext/sources/examples/stream/CMakeLists.txt +10 -0
  89. data/ext/sources/examples/stream/stream.cpp +429 -0
  90. data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
  91. data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
  92. data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
  93. data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
  94. data/ext/sources/examples/sycl/build.sh +22 -0
  95. data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
  96. data/ext/sources/examples/sycl/run-whisper.sh +17 -0
  97. data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
  98. data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
  99. data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
  100. data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
  101. data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
  102. data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
  103. data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
  104. data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
  105. data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
  106. data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
  107. data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
  108. data/ext/sources/examples/talk-llama/llama-context.h +276 -0
  109. data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
  110. data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
  111. data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
  112. data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
  113. data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
  114. data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
  115. data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
  116. data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
  117. data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
  118. data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
  119. data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
  120. data/ext/sources/examples/talk-llama/llama-io.h +35 -0
  121. data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
  122. data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
  123. data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
  124. data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
  125. data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
  126. data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
  127. data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
  128. data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
  129. data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
  130. data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
  131. data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
  132. data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
  133. data/ext/sources/examples/talk-llama/llama-model.h +425 -0
  134. data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
  135. data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
  136. data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
  137. data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
  138. data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
  139. data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
  140. data/ext/sources/examples/talk-llama/llama.cpp +354 -0
  141. data/ext/sources/examples/talk-llama/llama.h +1377 -0
  142. data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
  143. data/ext/sources/examples/talk-llama/speak +40 -0
  144. data/ext/sources/examples/talk-llama/speak.bat +1 -0
  145. data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
  146. data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
  147. data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
  148. data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
  149. data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
  150. data/ext/sources/examples/talk-llama/unicode.h +66 -0
  151. data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
  152. data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
  153. data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
  154. data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
  155. data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
  156. data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
  157. data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
  158. data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
  159. data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
  160. data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
  161. data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
  162. data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
  163. data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
  164. data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
  165. data/ext/sources/ggml/CMakeLists.txt +390 -0
  166. data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
  167. data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
  168. data/ext/sources/ggml/cmake/common.cmake +26 -0
  169. data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
  170. data/ext/{ggml → sources/ggml}/include/ggml-alloc.h +1 -1
  171. data/ext/{ggml → sources/ggml}/include/ggml-backend.h +9 -7
  172. data/ext/{ggml → sources/ggml}/include/ggml-cpp.h +2 -1
  173. data/ext/{ggml → sources/ggml}/include/ggml-cpu.h +9 -1
  174. data/ext/{ggml → sources/ggml}/include/ggml-metal.h +1 -1
  175. data/ext/{ggml → sources/ggml}/include/ggml-opt.h +49 -28
  176. data/ext/{ggml → sources/ggml}/include/ggml-rpc.h +6 -1
  177. data/ext/{ggml → sources/ggml}/include/ggml-vulkan.h +0 -2
  178. data/ext/{ggml → sources/ggml}/include/ggml.h +182 -265
  179. data/ext/sources/ggml/include/gguf.h +202 -0
  180. data/ext/sources/ggml/src/CMakeLists.txt +346 -0
  181. data/ext/{ggml → sources/ggml}/src/ggml-alloc.c +34 -29
  182. data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
  183. data/ext/{ggml → sources/ggml}/src/ggml-backend-impl.h +1 -2
  184. data/ext/{ggml → sources/ggml}/src/ggml-backend-reg.cpp +87 -53
  185. data/ext/{ggml → sources/ggml}/src/ggml-backend.cpp +26 -14
  186. data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
  187. data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
  188. data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
  189. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.cpp +10 -4
  190. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.h +5 -5
  191. data/ext/{ggml → sources/ggml}/src/ggml-cann/aclnn_ops.cpp +1272 -1506
  192. data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
  193. data/ext/{ggml → sources/ggml}/src/ggml-cann/common.h +135 -1
  194. data/ext/{ggml → sources/ggml}/src/ggml-cann/ggml-cann.cpp +564 -146
  195. data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
  196. data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/dup.cpp +3 -5
  197. data/ext/{ggml → sources/ggml}/src/ggml-common.h +12 -8
  198. data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
  199. data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.cpp +2 -1
  200. data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
  201. data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
  202. data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  203. data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
  204. data/ext/{ggml → sources/ggml}/src/ggml-cpu/cpu-feats-x86.cpp +5 -1
  205. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
  206. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-impl.h +163 -41
  207. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.c +4029 -1117
  208. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
  209. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu.cpp +67 -18
  210. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
  211. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
  212. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
  213. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  214. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
  215. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
  216. data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
  217. data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
  218. data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
  219. data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
  220. data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
  221. data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
  222. data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
  223. data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
  224. data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
  225. data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
  226. data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
  227. data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
  228. data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
  229. data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
  230. data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
  231. data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
  232. data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
  233. data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
  234. data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
  235. data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
  236. data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
  237. data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
  238. data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
  239. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
  240. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  241. data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
  242. data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
  243. data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
  244. data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  245. data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  246. data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
  247. data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
  248. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
  249. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  250. data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
  251. data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
  252. data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  253. data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
  254. data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
  255. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
  256. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
  257. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
  258. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
  259. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
  260. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
  261. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
  262. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
  263. data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
  264. data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
  265. data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
  266. data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
  267. data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
  268. data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
  269. data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
  270. data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
  271. data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
  272. data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
  273. data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
  274. data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
  275. data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
  276. data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
  277. data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
  278. data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
  279. data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
  280. data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
  281. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  282. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  283. data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
  284. data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  285. data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
  286. data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
  287. data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
  288. data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  289. data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
  290. data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
  291. data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
  292. data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
  293. data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
  294. data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
  295. data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
  296. data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
  297. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
  298. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  299. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
  300. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  301. data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
  302. data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
  303. data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
  304. data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
  305. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
  306. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
  307. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  308. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
  309. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
  310. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
  311. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
  312. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
  313. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  314. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
  315. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
  316. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
  317. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
  318. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
  319. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  320. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  321. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
  322. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
  323. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
  324. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
  325. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
  326. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
  327. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
  328. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
  329. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
  330. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
  331. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
  332. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
  333. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
  334. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
  335. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
  336. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
  337. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
  338. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
  339. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
  340. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
  341. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
  342. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
  343. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
  344. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
  345. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
  346. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
  347. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
  348. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
  349. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
  350. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
  351. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
  352. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
  353. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
  354. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
  355. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
  356. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
  357. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
  358. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
  359. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
  360. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
  361. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
  362. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
  363. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
  364. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
  365. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
  366. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
  367. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
  368. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
  369. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
  370. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
  371. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
  372. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
  373. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
  374. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
  375. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
  376. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
  377. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
  378. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
  379. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
  380. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
  381. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
  382. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
  383. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
  384. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
  385. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
  386. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
  387. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
  388. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
  389. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
  390. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
  391. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
  392. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
  393. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
  394. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
  395. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
  396. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
  397. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
  398. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
  399. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
  400. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
  401. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
  402. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
  403. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
  404. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
  405. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
  406. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
  407. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
  408. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
  409. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
  410. data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
  411. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  412. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  413. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  414. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  415. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  416. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  417. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  418. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  419. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  420. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  421. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  422. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  423. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  424. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  425. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  426. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  427. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  428. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  429. data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
  430. data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  431. data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
  432. data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
  433. data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
  434. data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
  435. data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
  436. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/cuda.h +1 -0
  437. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/hip.h +57 -0
  438. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/musa.h +7 -1
  439. data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
  440. data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
  441. data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
  442. data/ext/{ggml → sources/ggml}/src/ggml-impl.h +64 -19
  443. data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
  444. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
  445. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
  446. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
  447. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
  448. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
  449. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
  450. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
  451. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
  452. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
  453. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
  454. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
  455. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
  456. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
  457. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
  458. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
  459. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
  460. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
  461. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
  462. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
  463. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
  464. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
  465. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
  466. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
  467. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
  468. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
  469. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
  470. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
  471. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
  472. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
  473. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
  474. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
  475. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
  476. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
  477. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
  478. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
  479. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
  480. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
  481. data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
  482. data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
  483. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.m +2178 -1064
  484. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.metal +1575 -1218
  485. data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
  486. data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
  487. data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
  488. data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
  489. data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
  490. data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
  491. data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  492. data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
  493. data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
  494. data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  495. data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  496. data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
  497. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
  498. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
  499. data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
  500. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  501. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  502. data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
  503. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
  504. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
  505. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
  506. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
  507. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
  508. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  509. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  510. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  511. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  512. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  513. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  514. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
  515. data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
  516. data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  517. data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
  518. data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
  519. data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
  520. data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  521. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
  522. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
  523. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
  524. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
  525. data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
  526. data/ext/{ggml → sources/ggml}/src/ggml-opt.cpp +373 -190
  527. data/ext/{ggml → sources/ggml}/src/ggml-quants.c +114 -120
  528. data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
  529. data/ext/{ggml → sources/ggml}/src/ggml-rpc/ggml-rpc.cpp +480 -73
  530. data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
  531. data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
  532. data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
  533. data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
  534. data/ext/{ggml → sources/ggml}/src/ggml-sycl/common.cpp +20 -32
  535. data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
  536. data/ext/{ggml → sources/ggml}/src/ggml-sycl/concat.cpp +32 -33
  537. data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
  538. data/ext/{ggml → sources/ggml}/src/ggml-sycl/conv.cpp +4 -2
  539. data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
  540. data/ext/{ggml → sources/ggml}/src/ggml-sycl/convert.cpp +104 -28
  541. data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
  542. data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
  543. data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
  544. data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
  545. data/ext/{ggml → sources/ggml}/src/ggml-sycl/dmmv.cpp +156 -17
  546. data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  547. data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
  548. data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
  549. data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
  550. data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
  551. data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
  552. data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
  553. data/ext/{ggml → sources/ggml}/src/ggml-sycl/ggml-sycl.cpp +1004 -1240
  554. data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
  555. data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
  556. data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
  557. data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
  558. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmq.cpp +0 -1
  559. data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
  560. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmvq.cpp +261 -166
  561. data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
  562. data/ext/{ggml → sources/ggml}/src/ggml-sycl/norm.cpp +204 -81
  563. data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
  564. data/ext/{ggml → sources/ggml}/src/ggml-sycl/outprod.cpp +8 -17
  565. data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
  566. data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
  567. data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
  568. data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
  569. data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
  570. data/ext/{ggml → sources/ggml}/src/ggml-sycl/softmax.cpp +35 -25
  571. data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
  572. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
  573. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
  574. data/ext/{ggml → sources/ggml}/src/ggml-sycl/tsembd.cpp +3 -3
  575. data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  576. data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
  577. data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
  578. data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
  579. data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
  580. data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  581. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/ggml-vulkan.cpp +3130 -1087
  582. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
  583. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
  584. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
  585. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
  586. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
  587. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
  588. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  589. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
  590. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  591. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
  592. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  593. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
  594. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
  595. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  596. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  597. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
  598. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
  599. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
  600. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  601. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  602. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  603. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  604. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
  605. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
  606. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
  607. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  608. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  609. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  610. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  611. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  612. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  613. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  614. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  615. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  616. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  617. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  618. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
  619. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  620. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  621. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
  622. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
  623. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
  624. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
  625. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
  626. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
  627. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
  628. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
  629. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
  630. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
  631. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
  632. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
  633. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  634. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
  635. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  636. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
  637. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  638. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  639. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
  640. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
  641. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
  642. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
  643. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  644. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
  645. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  646. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
  647. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  648. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
  649. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
  650. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
  651. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  652. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
  653. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
  654. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  655. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
  656. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
  657. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
  658. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
  659. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  660. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  661. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
  662. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  663. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
  664. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
  665. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  666. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  667. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
  668. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  669. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
  670. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
  671. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
  672. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
  673. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
  674. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  675. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
  676. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
  677. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  678. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
  679. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
  680. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
  681. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
  682. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  683. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
  684. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
  685. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
  686. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
  687. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
  688. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
  689. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
  690. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
  691. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
  692. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +193 -35
  693. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  694. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  695. data/ext/{ggml → sources/ggml}/src/ggml.c +676 -1820
  696. data/ext/sources/ggml/src/gguf.cpp +1330 -0
  697. data/ext/{include → sources/include}/whisper.h +68 -2
  698. data/ext/sources/src/CMakeLists.txt +143 -0
  699. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.h +27 -15
  700. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.m +35 -10
  701. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.h +21 -9
  702. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.m +28 -3
  703. data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
  704. data/ext/sources/src/whisper-arch.h +197 -0
  705. data/ext/{src → sources/src}/whisper.cpp +1905 -374
  706. data/ext/sources/tests/CMakeLists.txt +105 -0
  707. data/ext/sources/tests/earnings21/eval.mk +58 -0
  708. data/ext/sources/tests/earnings21/eval.py +68 -0
  709. data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
  710. data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
  711. data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
  712. data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
  713. data/ext/sources/tests/earnings21/requirements.txt +6 -0
  714. data/ext/sources/tests/en-0-ref.txt +1 -0
  715. data/ext/sources/tests/en-1-ref.txt +1 -0
  716. data/ext/sources/tests/en-2-ref.txt +1 -0
  717. data/ext/sources/tests/es-0-ref.txt +1 -0
  718. data/ext/sources/tests/librispeech/eval.mk +39 -0
  719. data/ext/sources/tests/librispeech/eval.py +47 -0
  720. data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
  721. data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
  722. data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
  723. data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
  724. data/ext/sources/tests/librispeech/requirements.txt +6 -0
  725. data/ext/sources/tests/run-tests.sh +130 -0
  726. data/ext/sources/tests/test-c.c +3 -0
  727. data/ext/sources/tests/test-vad-full.cpp +54 -0
  728. data/ext/sources/tests/test-vad.cpp +83 -0
  729. data/ext/sources/tests/test-whisper.js +58 -0
  730. data/extsources.rb +33 -5
  731. data/lib/whisper/model/uri.rb +149 -128
  732. data/sig/whisper.rbs +480 -0
  733. data/tests/helper.rb +28 -0
  734. data/tests/test_callback.rb +45 -3
  735. data/tests/test_error.rb +2 -2
  736. data/tests/test_model.rb +38 -0
  737. data/tests/test_package.rb +18 -3
  738. data/tests/test_params.rb +145 -8
  739. data/tests/test_segment.rb +10 -19
  740. data/tests/test_vad.rb +19 -0
  741. data/tests/test_vad_params.rb +103 -0
  742. data/tests/test_whisper.rb +37 -37
  743. data/whispercpp.gemspec +5 -4
  744. metadata +766 -111
  745. data/ext/cpu.mk +0 -9
  746. data/ext/examples/dr_wav.h +0 -8815
  747. data/ext/ggml/src/ggml-cann/aclnn_ops.h +0 -592
  748. data/ext/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +0 -4262
  749. data/ext/ggml/src/ggml-cpu/ggml-cpu.c +0 -14123
  750. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.cpp +0 -1884
  751. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.h +0 -14
  752. data/ext/ggml/src/ggml-metal/ggml-metal-impl.h +0 -288
  753. data/ext/ggml/src/ggml-sycl/element_wise.cpp +0 -1030
  754. data/ext/ggml/src/ggml-sycl/im2col.cpp +0 -126
  755. data/ext/ggml/src/ggml-sycl/rope.cpp +0 -276
  756. data/ext/ggml/src/ggml-sycl/wkv6.cpp +0 -141
  757. data/ext/metal-embed.mk +0 -17
  758. data/ext/metal.mk +0 -6
  759. data/ext/ruby_whisper.cpp +0 -1909
  760. data/ext/scripts/get-flags.mk +0 -38
  761. data/lib/whisper.rb +0 -2
  762. /data/ext/{ggml → sources/ggml}/include/ggml-blas.h +0 -0
  763. /data/ext/{ggml → sources/ggml}/include/ggml-cann.h +0 -0
  764. /data/ext/{ggml → sources/ggml}/include/ggml-cuda.h +0 -0
  765. /data/ext/{ggml → sources/ggml}/include/ggml-kompute.h +0 -0
  766. /data/ext/{ggml → sources/ggml}/include/ggml-opencl.h +0 -0
  767. /data/ext/{ggml → sources/ggml}/include/ggml-sycl.h +0 -0
  768. /data/ext/{ggml → sources/ggml}/src/ggml-amx/common.h +0 -0
  769. /data/ext/{ggml → sources/ggml}/src/ggml-amx/ggml-amx.cpp +0 -0
  770. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.cpp +0 -0
  771. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.h +0 -0
  772. /data/ext/{ggml → sources/ggml}/src/ggml-blas/ggml-blas.cpp +0 -0
  773. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/ascendc_kernels.h +0 -0
  774. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f16.cpp +0 -0
  775. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f32.cpp +0 -0
  776. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q4_0.cpp +0 -0
  777. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q8_0.cpp +0 -0
  778. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +0 -0
  779. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +0 -0
  780. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +0 -0
  781. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.h +0 -0
  782. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/common.h +0 -0
  783. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.cpp +0 -0
  784. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.h +0 -0
  785. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-aarch64.h +0 -0
  786. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.cpp +0 -0
  787. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.h +0 -0
  788. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.h +0 -0
  789. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.cpp +0 -0
  790. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.h +0 -0
  791. /data/ext/{ggml → sources/ggml}/src/ggml-kompute/ggml-kompute.cpp +0 -0
  792. /data/ext/{ggml → sources/ggml}/src/ggml-quants.h +0 -0
  793. /data/ext/{ggml → sources/ggml}/src/ggml-threading.cpp +0 -0
  794. /data/ext/{ggml → sources/ggml}/src/ggml-threading.h +0 -0
  795. /data/ext/{src → sources/src}/coreml/whisper-encoder.h +0 -0
  796. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.cpp +0 -0
  797. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.h +0 -0
@@ -0,0 +1,1125 @@
1
+ /**
2
+ * Copyright (c) 2023-2024 The ggml authors
3
+ *
4
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
5
+ * of this software and associated documentation files (the "Software"), to
6
+ * deal in the Software without restriction, including without limitation the
7
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8
+ * sell copies of the Software, and to permit persons to whom the Software is
9
+ * furnished to do so, subject to the following conditions:
10
+ *
11
+ * The above copyright notice and this permission notice shall be included in
12
+ * all copies or substantial portions of the Software.
13
+ *
14
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
20
+ * IN THE SOFTWARE.
21
+ */
22
+
23
+ #ifndef CANN_ACLNN_OPS
24
+ #define CANN_ACLNN_OPS
25
+
26
+ #include <functional>
27
+ #include <aclnnop/aclnn_abs.h>
28
+ #include <aclnnop/aclnn_neg.h>
29
+ #include <aclnnop/aclnn_exp.h>
30
+ #include <aclnnop/aclnn_arange.h>
31
+ #include <aclnnop/aclnn_argsort.h>
32
+ #include <aclnnop/aclnn_cat.h>
33
+ #include <aclnnop/aclnn_clamp.h>
34
+ #include <aclnnop/aclnn_gelu.h>
35
+ #include <aclnnop/aclnn_gelu_v2.h>
36
+ #include <aclnnop/aclnn_sigmoid.h>
37
+ #include <aclnnop/aclnn_hardsigmoid.h>
38
+ #include <aclnnop/aclnn_hardswish.h>
39
+ #include <aclnnop/aclnn_leaky_relu.h>
40
+ #include <aclnnop/aclnn_relu.h>
41
+ #include <aclnnop/aclnn_silu.h>
42
+ #include <aclnnop/aclnn_tanh.h>
43
+ #include <aclnnop/aclnn_sqrt.h>
44
+ #include <aclnnop/aclnn_sin.h>
45
+ #include <aclnnop/aclnn_cos.h>
46
+ #include <aclnnop/aclnn_log.h>
47
+ #include <aclnnop/aclnn_sign.h>
48
+ #include "acl_tensor.h"
49
+ #include "common.h"
50
+
51
+ /**
52
+ * @brief Repeats a ggml tensor along each dimension to match the dimensions
53
+ * of another tensor.
54
+ *
55
+ * @details This function repeats the elements of a source ggml tensor along
56
+ * each dimension to create a destination tensor with the specified
57
+ * dimensions. The operation is performed using the ACL backend and
58
+ * executed asynchronously on the device.
59
+ *
60
+ * @param ctx The CANN context used for operations.
61
+ * @param dst The ggml tensor representing the destination, which op is
62
+ * GGML_OP_REPEAT and specifies the desired dimensions.
63
+ */
64
+ void ggml_cann_repeat(ggml_backend_cann_context& ctx, ggml_tensor* dst);
65
+
66
+ /**
67
+ * @brief Applies the Leaky ReLU activation function to a tensor using the CANN
68
+ * backend.
69
+ *
70
+ * @details This function computes the Leaky ReLU activation for each element of
71
+ * the input tensor. The Leaky ReLU function allows a small gradient
72
+ * when the unit is not active (i.e., when the input is negative). The
73
+ * Leaky ReLU function is defined as:
74
+ * \f[
75
+ * \text{dst} = \max(0, src) + \text{negativeSlope} \cdot \min(0,
76
+ * src)
77
+ * \f]
78
+ * `negativeSlope` is in dst->params.
79
+ *
80
+ * @param ctx The CANN context used for operations.
81
+ * @param dst The destination tensor where the result of the Leaky ReLU
82
+ * activation is stored, which op is `GGML_OP_LEAKY_RELU`
83
+ */
84
+ void ggml_cann_leaky_relu(ggml_backend_cann_context& ctx, ggml_tensor* dst);
85
+
86
+ /**
87
+ * @brief Concatenates multiple tensors along a specified dimension using the
88
+ * CANN backend.
89
+ *
90
+ * @param ctx The CANN context used for operations.
91
+ * @param tensorList A pointer to the list of tensors to be concatenated.
92
+ * @param dst The destination tensor where the result of the
93
+ * concatenation is stored. dst->op is `GGML_OP_CONCAT`.
94
+ * @param concat_dim The dimension along which the tensors are concatenated.
95
+ *
96
+ * @attention tensorList length should be 2 and the dimension using for concat
97
+ * default to 1.
98
+ */
99
+ void ggml_cann_concat(ggml_backend_cann_context& ctx, ggml_tensor* dst);
100
+
101
+ /**
102
+ * @brief Generates a sequence of evenly spaced values within a specified
103
+ * interval for a ggml tensor using the CANN backend.
104
+ *
105
+ * @details This function creates a sequence of numbers over a specified i
106
+ * nterval, starting from `start`, ending before `stop`, and
107
+ * incrementing by `step`. The sequence is stored in the destination
108
+ * tensor `dst`.
109
+ *
110
+ * @param ctx The CANN context used for operations.
111
+ * @param dst The destination tensor where the generated sequence will be stored.
112
+ * `start`, 'stop' and 'step' are in dst->op_params and dst->op is
113
+ * `GGML_OP_ARANGE`.
114
+ */
115
+ void ggml_cann_arange(ggml_backend_cann_context& ctx, ggml_tensor* dst);
116
+
117
+ /**
118
+ * @brief Applies a clamp operation to the elements of a ggml tensor using the
119
+ * CANN backend.
120
+ *
121
+ * @details This function clamps the elements of the input tensor `src` to a
122
+ * specified range defined by `min` and `max` values. The result is
123
+ * stored in the destination tensor `dst`. The operation is defined as:
124
+ * \f[
125
+ * y = \max(\min(x, max\_value), min\_value)
126
+ * \f]
127
+ * where `x` is an element of the input tensor, and `y` is the
128
+ * corresponding element in the output tensor.
129
+ * @param ctx The CANN context used for operations.
130
+ * @param dst The destination tensor where the clamped values will be stored.
131
+ * dst->op is `GGML_OP_CLAMP`, `min` and `max` value is in dst->params.
132
+ */
133
+ void ggml_cann_clamp(ggml_backend_cann_context& ctx, ggml_tensor* dst);
134
+
135
+ /**
136
+ * @brief Scales the elements of a ggml tensor by a constant factor using the
137
+ * CANN backend.
138
+ *
139
+ * @details This function multiplies each element of the input tensor `src` by
140
+ * a scaling factor `scale`, storing the result in the destination
141
+ * tensor `dst`. The operation is defined as:
142
+ * \f[
143
+ * dst = src \times scale
144
+ * \f]
145
+ *
146
+ * @param ctx The CANN context used for operations.
147
+ * @param dst The destination tensor where the scaled values will be stored.
148
+ * dst->op is `GGML_OP_SCALE` and `scale` value is in dst->params.
149
+ */
150
+ void ggml_cann_scale(ggml_backend_cann_context& ctx, ggml_tensor* dst);
151
+
152
+ /**
153
+ * @brief Sorts the elements of a ggml tensor and returns the indices that
154
+ * would sort the tensor using the CANN backend.
155
+ *
156
+ * @details This function performs an argsort operation on the input tensor
157
+ * `src`. It sorts the elements of `src` in either ascending or
158
+ * descending order, depending on the `GGML_SORT_ORDER_DESC`,
159
+ * and returns the indices that would sort the original tensor.
160
+ *
161
+ * @param ctx The CANN context used for operations.
162
+ * @param dst The destination tensor where the sorted indices will be stored.
163
+ * dst->op is `GGML_OP_ARGSORT`.
164
+ */
165
+ void ggml_cann_argsort(ggml_backend_cann_context& ctx, ggml_tensor* dst);
166
+
167
+ /**
168
+ * @brief Computes the Layer Normalization for a ggml tensor using the CANN
169
+ * backend.
170
+ *
171
+ * @details This function applies the Layer Normalization operation on the
172
+ * input tensor `src` and stores the result in the destination tensor
173
+ * `dst`. Layer Normalization normalizes the features at each sample in
174
+ * a mini-batch independently. It is commonly used in neural networks
175
+ * to normalize the activations of a layer by adjusting and scaling
176
+ * the outputs.
177
+ * The operation is defined as:
178
+ * \f[
179
+ * \text { out }=\frac{x-\mathrm{E}[x]}{\sqrt{\text{Var}[x]+eps}}
180
+ * \f]
181
+ * `Var` defaults dst->ne[0]. `eps` is in dst->params.
182
+ *
183
+ * @param ctx The CANN context used for operations.
184
+ * @param dst The destination tensor where the normalized values will be stored.
185
+ * @attention `Var` defaults to dst->ne[0].
186
+ */
187
+ void ggml_cann_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst);
188
+
189
+ /**
190
+ * @brief Computes the Group Normalization for a ggml tensor using the CANN
191
+ * backend.
192
+ *
193
+ * @brief This function applies the Group Normalization operation on the input
194
+ * tensor `src` and stores the result in the destination tensor `dst`.
195
+ * Group Normalization divides the channels into groups and normalizes
196
+ * the features within each group across spatial locations.
197
+ * It is commonly used in convolutional neural networks to improve
198
+ * training stability and performance.
199
+ * The operation is defined as:
200
+ * \f[
201
+ * \text { out }=\frac{x-\mathrm{E}[x]}{\sqrt{\text{Var}[x]+eps}}
202
+ * \f]
203
+ *
204
+ * @param ctx The CANN context used for operations.
205
+ * @param dst The destination tensor where the normalized values will be stored.
206
+ * `n_groups` is in dst->params, which split C channel to `n_groups`.
207
+ * dst->op is `GGML_OP_GROUP_NORM`.
208
+ *
209
+ * @attention eps defaults to 1e-6f.
210
+ */
211
+ void ggml_cann_group_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst);
212
+
213
+ /**
214
+ * @brief Computes the accumulation of tensors using the CANN backend.
215
+ *
216
+ * @details This function performs an accumulation operation on two tensors.
217
+ * Depending on the `inplace` flag, it either updates the destination
218
+ * tensor `dst` in place by adding `alpha * src1` to it, or it creates
219
+ * a new tensor as the result of `src0 + alpha * src1` and stores it in
220
+ * `dst`.
221
+ * The operation is defined as:
222
+ * \f[
223
+ * dst = src0 + alpha \times src1
224
+ * \f]
225
+ * if `inplace` is `true`, `src0` is equal to 'dst'.
226
+ * @param ctx The CANN context used for operations.
227
+ * @param dst The destination tensor where the accumulated values will be stored.
228
+ * `inplace` is in dst->params, and dst->op is `GGML_OP_ACC`.
229
+ */
230
+ void ggml_cann_acc(ggml_backend_cann_context& ctx, ggml_tensor* dst);
231
+
232
+ /**
233
+ * @brief Computes the sum of elements along the last dimension of a ggml tensor
234
+ * using the CANN backend.
235
+ *
236
+ * @details This function performs a reduction sum operation along the last
237
+ * dimension of the input tensor `src`. The result of the sum is stored
238
+ * in the destination tensor `dst`.
239
+ *
240
+ * @param ctx The CANN context used for operations.
241
+ * @param dst The destination tensor where the reduced values will be stored。
242
+ * dst->op is `GGML_OP_SUM_ROWS`.
243
+ *
244
+ * @attention `reduce_dims` defaults to 3, which means the last dimension.
245
+ */
246
+ void ggml_cann_sum_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst);
247
+
248
+ /**
249
+ * @brief Computes the sum of elements in a ggml tensor.
250
+ *
251
+ * @details This function performs a reduction sum operation along the last
252
+ * dimension of the input tensor `src`. The result of the sum is stored
253
+ * in the destination tensor `dst`.
254
+ *
255
+ * @param ctx The CANN context used for operations.
256
+ * @param dst The destination tensor where the reduced values will be stored。
257
+ *
258
+ */
259
+
260
+ void ggml_cann_sum(ggml_backend_cann_context& ctx, ggml_tensor* dst);
261
+
262
+ /**
263
+ * @brief Upsamples a ggml tensor using nearest neighbor interpolation using
264
+ * the CANN backend.
265
+ *
266
+ * @details This function performs upsampling of the input tensor `src` using
267
+ * nearest neighbor interpolation. The upsampling is applied to the
268
+ * height and width dimensions (last two dimensions) of the tensor. The
269
+ * result is stored in the destination tensor `dst`, which must have
270
+ * the appropriate dimensions for the upsampled output.
271
+ *
272
+ * @param ctx The CANN context used for operations.
273
+ * @param dst The destination tensor where the upsampled values will be stored.
274
+ * dst->op is `GGML_OP_UPSCALE`.
275
+ */
276
+ void ggml_cann_upsample_nearest2d(ggml_backend_cann_context& ctx,
277
+ ggml_tensor* dst);
278
+
279
+ /**
280
+ * @brief Pads a ggml tensor to match the dimensions of the destination tensor
281
+ * using the CANN backend.
282
+ *
283
+ * @details This function pads the input tensor `src` so that it matches the
284
+ * dimensions of the destination tensor `dst`. The amount of padding
285
+ * is calculated based on the difference in sizes between `src` and
286
+ * `dst` along each dimension. The padded tensor is stored in `dst`.
287
+ *
288
+ * @param ctx The CANN context used for operations.
289
+ * @param dst The destination tensor, which specifies the target dimensions for
290
+ * padding. dst->op is `GGML_OP_PAD`.
291
+ */
292
+ void ggml_cann_pad(ggml_backend_cann_context& ctx, ggml_tensor* dst);
293
+
294
+ /**
295
+ * @brief Executes a 2D pooling operation on a ggml tensor using the CANN
296
+ * backend.
297
+ *
298
+ * @details This function dispatches the execution of a 2D pooling operation on
299
+ * the input tensor `dst`. The type of pooling (average or max) is
300
+ * determined by the `op` parameter, which is read from the operation
301
+ * parameters of `dst`. The function supports average pooling
302
+ * (`GGML_OP_POOL_AVG`) and max pooling (`GGML_OP_POOL_MAX`). If an
303
+ * invalid operation is encountered, the function asserts a failure.
304
+ *
305
+ * @param ctx The CANN context used for operations.
306
+ * @param dst The destination tensor on which the pooling operation is to be
307
+ * performed. dst->op is `GGML_OP_POOL_2D`.
308
+ */
309
+ void ggml_cann_pool2d(ggml_backend_cann_context& ctx, ggml_tensor* dst);
310
+
311
+ /**
312
+ * @brief Duplicates a ggml tensor using the CANN backend.
313
+ *
314
+ * @details This function duplicates the contents of the source tensor `src` to
315
+ * the destination tensor `dst`. The function supports various tensor
316
+ * types and configurations, including handling of extra data, type
317
+ * conversions, and special cases for contiguous and non-contiguous
318
+ * tensors.
319
+ *
320
+ * @param ctx The CANN context used for operations.
321
+ * @param dst The destination tensor where the duplicated data will be stored.
322
+ * dst->op is `GGML_OP_DUP`
323
+ *
324
+ * @attention Only support Fp16/FP32. Not support when src and dst have
325
+ * different shape and dst is no-contiguous.
326
+ * @note: This func need to simplify.
327
+ */
328
+ void ggml_cann_dup(ggml_backend_cann_context& ctx, ggml_tensor* dst);
329
+
330
+ /**
331
+ * @brief Computes the Root Mean Square (RMS) normalization of a ggml tensor
332
+ * using the CANN backend.
333
+ *
334
+ * @details This function applies RMS normalization to the input tensor `src`
335
+ * and stores the result in the destination tensor `dst`. RMS
336
+ * normalization involves computing the root mean square of the input
337
+ * tensor along a specified dimension and then dividing each element of
338
+ * the tensor by this value, adjusted by a small epsilon value to
339
+ * prevent division by zero.
340
+ * The operation is defined as:
341
+ * \f[
342
+ * \text{RmsNorm}\left(x_i\right)=\frac{x_i}{\text{Rms}(\mathbf{x})} g_i,
343
+ * \quad \text { where } \text{Rms}(\mathbf{x})=\sqrt{\frac{1}{n} \sum_{i=1}^n x_i^2+e p s}
344
+ * \f]
345
+ * `eps` is in dst->op_params.
346
+ * @param ctx The CANN context used for operations.
347
+ * @param dst The destination tensor where the normalized values will be stored.
348
+ * dst->op is `GGML_OP_RMS_NORM`.
349
+ */
350
+ void ggml_cann_rms_norm(ggml_backend_cann_context& ctx, ggml_tensor* dst);
351
+
352
+ /**
353
+ * @brief Applies a diagonal mask to the tensor with a specified value.
354
+ *
355
+ * @details This function creates a mask tensor filled with ones, then applies
356
+ * an upper triangular and lower triangular operation to it based on
357
+ * the number of past elements specified. Afterward, it adds the masked
358
+ * tensor to the destination tensor in-place.
359
+ *
360
+ * @param ctx The backend CANN context used for operations.
361
+ * @param dst The destination tensor where the result will be stored. dst->op is
362
+ * `GGML_OP_DIAG_MASK`
363
+ * @param value The value to use for masking.
364
+ */
365
+ void ggml_cann_diag_mask(ggml_backend_cann_context& ctx, ggml_tensor* dst, float value);
366
+
367
+ /**
368
+ * @brief Performs an image-to-column transformation on the input tensor.
369
+ *
370
+ * @details This function takes an input tensor and applies an image-to-column
371
+ * operation, converting spatial dimensions into column-like
372
+ * structures suitable for convolutional operations. It supports both
373
+ * half-precision (F16) and single-precision (F32) floating-point data
374
+ * types.
375
+ *
376
+ * @param ctx The backend CANN context for executing operations.
377
+ * @param dst The destination tensor that stores the result of the operation.
378
+ * dst->op is `GGML_OP_IM2COL`.
379
+ */
380
+ void ggml_cann_im2col(ggml_backend_cann_context& ctx, ggml_tensor* dst);
381
+
382
+ /**
383
+ * @brief Computes time step embeddings using sine and cosine functions.
384
+ *
385
+ * @details This function calculates time step embeddings by applying sine and
386
+ * cosine transformations to a given input tensor, which is typically
387
+ * used in temporal models like diffusion models or transformers to
388
+ * encode time information effectively.
389
+ *
390
+ * @param ctx The backend CANN context for executing operations.
391
+ * @param dst The destination tensor where the result of the embedding operation
392
+ * will be stored. dst->op is `GGML_OP_TIMESTEP_EMBEDDING`.
393
+ */
394
+ void ggml_cann_timestep_embedding(ggml_backend_cann_context& ctx, ggml_tensor* dst);
395
+
396
+ // @see ggml_cann_dup.
397
+ void ggml_cann_cpy(ggml_backend_cann_context& ctx, ggml_tensor* dst);
398
+
399
+ /**
400
+ * @brief Computes the softmax activation with optional masking.
401
+ *
402
+ * @details This function computes the softmax activation over the input tensor,
403
+ * optionally applying a mask and scaling factor. It supports both FP16
404
+ * and FP32 data types and can handle masking by broadcasting the mask
405
+ * across rows if necessary.
406
+ * The function performs the following steps:
407
+ * 1. Multiplies the input tensor by a scale factor.
408
+ * 2. Optionally casts the mask tensor to FP32 if it is in FP16 format.
409
+ * 3. Broadcasts the mask tensor if its dimensions do not match the
410
+ * input tensor's dimensions.
411
+ * 4. Adds the mask to the scaled input tensor.
412
+ * 5. Applies the softmax activation function along the specified
413
+ * dimension.
414
+ *
415
+ * @param ctx The backend CANN context for executing operations.
416
+ * @param dst The destination tensor where the result will be stored. dst->op is
417
+ * `GGML_OP_SOFTMAX`.
418
+ */
419
+ void ggml_cann_softmax(ggml_backend_cann_context& ctx, ggml_tensor* dst);
420
+
421
+ /**
422
+ * @brief Extracts specific rows from a tensor based on indices.
423
+ *
424
+ * @details This function retrieves rows from a source tensor src0 according to
425
+ * the indices provided in another tensor src1 and stores the result in
426
+ * a destination tensor (\p dst). It supports different data types
427
+ * including F32, F16, Q4_0, and Q8_0.
428
+ *
429
+ * @param ctx The backend CANN context for executing operations.
430
+ * @param dst The destination tensor where the extracted rows will be stored.
431
+ * dst->op is `GGML_OP_GET_ROWS`.
432
+ */
433
+ void ggml_cann_get_rows(ggml_backend_cann_context& ctx, ggml_tensor* dst);
434
+
435
+ /**
436
+ * @brief Executes matrix multiplication for the given tensor.
437
+ *
438
+ * @details This function performs matrix multiplication on the source tensors
439
+ * associated with the destination tensor. It supports matrix
440
+ * multiplication F32, F16, and Q8_0.
441
+ *
442
+ * @param ctx The backend CANN context for executing operations.
443
+ * @param dst The destination tensor for storing the result of the matrix
444
+ * multiplication. dst->op is `GGML_OP_MUL_MAT`.
445
+ */
446
+ void ggml_cann_mul_mat(ggml_backend_cann_context& ctx, ggml_tensor* dst);
447
+
448
+ /**
449
+ * @brief Applies Rotary Positional Embedding (RoPE) to the input tensor.
450
+ *
451
+ * @details This function implements the RoPE mechanism, which is a method to
452
+ * encode positional information into sequence data, particularly
453
+ * useful in transformer models. It supports both F32 and F16 data
454
+ * types.
455
+ *
456
+ * @param ctx The backend CANN context for executing operations.
457
+ * @param dst The destination tensor where the RoPE-transformed data will be
458
+ * stored. dst->op is `GGML_OP_ROPE`.
459
+ *
460
+ * @note The function currently does not support cases where the n_dims is less
461
+ * than the input tensor's first dimension.
462
+ * @note The function currently does not support cases where the freq_factors is
463
+ * not NULL.
464
+ * @note The function currently does not support cases where the ext_factor is
465
+ * not equal 0.
466
+ * @note The function currently does not support cases where the freq_scale is
467
+ * not equal 1.
468
+ */
469
+ void ggml_cann_rope(ggml_backend_cann_context& ctx, ggml_tensor* dst);
470
+
471
+ /**
472
+ * @brief Computes the index of the maximum value along the specified dimension
473
+ * of a ggml tensor using the CANN backend.
474
+ *
475
+ * @details This function performs an argmax operation on the input tensor.
476
+ * It finds the index of the maximum value along the specified axis
477
+ * and stores these indices in the destination tensor `dst`. The
478
+ * operation is executed using the CANN backend for optimized performance.
479
+ *
480
+ * @param ctx The CANN context used for operations.
481
+ * @param dst The destination tensor where the indices of the maximum values will
482
+ * be stored. dst->op is `GGML_OP_ARGMAX`.
483
+ */
484
+ void ggml_cann_argmax(ggml_backend_cann_context& ctx, ggml_tensor* dst);
485
+
486
+ /**
487
+ * @brief Adds two tensors element-wise and stores the result in a destination
488
+ * tensor.
489
+ *
490
+ * This function performs the operation:
491
+ * \f[
492
+ * dst = acl\_src0 + alpha \times acl\_src1
493
+ * \f]
494
+ * where alpha is a scalar value and defaults to 1.0f.
495
+ *
496
+ * @param ctx The context for the CANN backend operations.
497
+ * @param acl_src0 The first source tensor.
498
+ * @param acl_src1 The second source tensor.
499
+ * @param acl_dst The destination tensor where the result will be stored.
500
+ */
501
+ void aclnn_add(ggml_backend_cann_context& ctx, aclTensor* acl_src0,
502
+ aclTensor* acl_src1, aclTensor* acl_dst = nullptr);
503
+
504
+ /**
505
+ * @brief Sub two tensors element-wise and stores the result in a destination
506
+ * tensor.
507
+ *
508
+ * This function performs the operation:
509
+ * \f[
510
+ * dst = acl\_src0 - alpha \times acl\_src1
511
+ * \f]
512
+ * where alpha is a scalar value and defaults to 1.0f.
513
+ *
514
+ * @param ctx The context for the CANN backend operations.
515
+ * @param acl_src0 The first source tensor.
516
+ * @param acl_src1 The second source tensor.
517
+ * @param acl_dst The destination tensor where the result will be stored.
518
+ */
519
+ void aclnn_sub(ggml_backend_cann_context& ctx, aclTensor* acl_src0,
520
+ aclTensor* acl_src1, aclTensor* acl_dst = nullptr);
521
+
522
+ /**
523
+ * @brief Performs element-wise multiplication of two tensors and stores the
524
+ * result in a destination tensor.
525
+ *
526
+ * This function performs element-wise multiplication of the tensors `acl_src`
527
+ * and `acl_other` and stores the result in the destination tensor `acl_dst`.
528
+ * The operation is defined as:
529
+ * \f[
530
+ * \text {acl_dst }_i=\text {acl_src }_i \times \text {acl_other }_i
531
+ * \f]
532
+ *
533
+ * @param ctx The context for the CANN backend operations.
534
+ * @param acl_src The first tensor for element-wise multiplication.
535
+ * @param acl_other The second tensor for element-wise multiplication.
536
+ * @param acl_dst The destination tensor where the result will be stored.
537
+ */
538
+ void aclnn_mul(ggml_backend_cann_context& ctx, aclTensor* acl_src,
539
+ aclTensor* acl_other, aclTensor* acl_dst = nullptr);
540
+
541
+ /**
542
+ * @brief Matrix division, optionally in-place.
543
+ *
544
+ * This function division each element of the source tensor `acl_src` by the
545
+ * tensor `acl_other` and stores the result in the destination tensor `acl_dst`.
546
+ * If `inplace` is true, `acl_dst` will not be used and the operation is
547
+ * performed in-place on `acl_src`. The operation is defined as: \f[
548
+ * \text{dst}_i = \frac{\text{acl_src}_i}{\text{acl_other}_i}
549
+ * \f]
550
+ *
551
+ * @param ctx The context for the CANN backend operations.
552
+ * @param acl_src Numerator tensor..
553
+ * @param acl_other Denominator tensor.
554
+ * @param acl_dst The destination tensor where the result will be stored if
555
+ * `inplace` is false.
556
+ * @param inplace Flag indicating whether to perform the operation in-place on
557
+ * `acl_src`.
558
+ */
559
+ void aclnn_div(ggml_backend_cann_context& ctx, aclTensor* acl_src,
560
+ aclTensor* acl_other, aclTensor* acl_dst = nullptr);
561
+
562
+ /**
563
+ * @brief Applies element-wise cosine function to the elements of a tensor.
564
+ *
565
+ * This function computes the cosine of each element in the source tensor
566
+ * `acl_src` and stores the result in the destination tensor `acl_dst`. The
567
+ * operation is defined as: \f[ \text {acl_dst }_i=\cos \left(\text {acl_src
568
+ * }_i\right) \f]
569
+ *
570
+ * @param ctx The context for the CANN backend operations.
571
+ * @param acl_src The source tensor on which the cosine function will be
572
+ * applied.
573
+ * @param acl_dst The destination tensor where the cosine results will be
574
+ * stored.
575
+ */
576
+ void aclnn_cos(ggml_backend_cann_context& ctx, aclTensor* acl_src,
577
+ aclTensor* acl_dst);
578
+
579
+ /**
580
+ * @brief Applies element-wise sine function to the elements of a tensor.
581
+ *
582
+ * This function computes the sine of each element in the source tensor
583
+ `acl_src`
584
+ * and stores the result in the destination tensor `acl_dst`.
585
+ * The operation is defined as:
586
+ * \f[
587
+ * \text {acl_dst }_i=\sin \left(\text {acl_src }_i\right)
588
+ * \f]
589
+
590
+ * @param ctx The context for the CANN backend operations.
591
+ * @param acl_src The source tensor on which the sine function will be applied.
592
+ * @param acl_dst The destination tensor where the sine results will be stored.
593
+ */
594
+ void aclnn_sin(ggml_backend_cann_context& ctx, aclTensor* acl_src,
595
+ aclTensor* acl_dst);
596
+
597
+ /**
598
+ * @brief Prepares broadcast-compatible ACL tensors for two input tensors and one
599
+ * output tensor.
600
+ *
601
+ * This function checks whether broadcasting is needed between `src0` and `src1`.
602
+ * If broadcasting is required, it calculates the proper shapes and creates
603
+ * ACL tensors with broadcast parameters. Otherwise, it directly creates ACL tensors
604
+ * based on the original tensor shapes.
605
+ *
606
+ * @param src0 The first input tensor (reference shape).
607
+ * @param src1 The second input tensor (possibly broadcasted).
608
+ * @param dst The destination/output tensor.
609
+ * @param acl_src0 Output pointer to the created ACL tensor corresponding to src0.
610
+ * @param acl_src1 Output pointer to the created ACL tensor corresponding to src1.
611
+ * @param acl_dst Output pointer to the created ACL tensor corresponding to dst.
612
+ */
613
+ void bcast_shape(ggml_tensor * src0, ggml_tensor * src1, ggml_tensor * dst,
614
+ aclTensor ** acl_src0, aclTensor ** acl_src1, aclTensor ** acl_dst);
615
+
616
+ /**
617
+ * @brief Computes the 1D transposed convolution (deconvolution) of a ggml
618
+ * tensor using the CANN backend.
619
+ *
620
+ * @details This function performs a 1D transposed convolution (also known as
621
+ * deconvolution) operation on the input tensor. The computed result is stored
622
+ * in the destination tensor `dst`. The operation is optimized using the CANN
623
+ * backend for improved performance.
624
+ *
625
+ * @param ctx The CANN context used for operations.
626
+ * @param dst The destination tensor where the transposed convolution result
627
+ * will be stored. dst->op is `GGML_OP_CONV_TRANSPOSE_1D`.
628
+ */
629
+ void ggml_cann_conv_transpose_1d(ggml_backend_cann_context& ctx, ggml_tensor* dst);
630
+
631
+ /**
632
+ * @brief Applies the ELU (Exponential Linear Unit) activation to a ggml tensor
633
+ * using the CANN backend.
634
+ *
635
+ * @details This function performs an element-wise ELU activation on the input
636
+ * tensor.
637
+ * The result is written to the destination tensor `dst` in-place.
638
+ * The ELU function is defined as:
639
+ *
640
+ * \text{ELU}(x) =
641
+ * \begin{cases}
642
+ * x, & \text{if } x > 0 \\
643
+ * \alpha \left( \exp(x) - 1 \right), & \text{if } x \leq 0
644
+ * \end{cases}
645
+ *
646
+ * where α (alpha) is a hyperparameter, typically set to 1.0.
647
+ * This operation is optimized using the CANN backend for high-performance
648
+ * inference or training.
649
+ *
650
+ * @param ctx The CANN context used for operations.
651
+ * @param dst The destination tensor where the ELU-activated result will be stored.
652
+ * dst->op is expected to be `GGML_OP_ELU`.
653
+ */
654
+ void ggml_cann_elu(ggml_backend_cann_context& ctx, ggml_tensor* dst);
655
+
656
+ /**
657
+ * @brief Computes the mean of a ggml tensor element-wise using the CANN backend.
658
+ *
659
+ * @details This function calculates the element-wise mean of the input tensor.
660
+ * The result is written to the destination tensor `dst`.
661
+ * The mean is computed by averaging the values across the entire tensor.
662
+ *
663
+ * This operation is optimized using the CANN backend for high-performance inference or training.
664
+ *
665
+ * @param ctx The CANN context used for operations.
666
+ * @param dst The destination tensor where the mean result will be stored.
667
+ * dst->op is expected to be `GGML_OP_MEAN`.
668
+ */
669
+ void ggml_cann_mean(ggml_backend_cann_context& ctx, ggml_tensor* dst);
670
+
671
+ /**
672
+ * @brief Applies 1D reflect padding to a ggml tensor using the CANN backend.
673
+ *
674
+ * @details This function performs 1D reflect padding on the input tensor.
675
+ * The amount of padding on each side is specified by parameters stored in `dst->op_params`.
676
+ * The operation reflects the values at the borders of the tensor to generate the padded output.
677
+ *
678
+ * This operation is optimized using the CANN backend for high-performance inference or training.
679
+ *
680
+ * @param ctx The CANN context used for operations.
681
+ * @param dst The destination tensor where the padded result will be stored.
682
+ * dst->op is expected to be `GGML_OP_PAD_REFLECT_1D`.
683
+ */
684
+ void ggml_cann_pad_reflect_1d(ggml_backend_cann_context& ctx, ggml_tensor* dst);
685
+
686
+ /**
687
+ * @brief Counts the number of equal elements in two ggml tensors using the CANN backend.
688
+ *
689
+ * @details This function performs an element-wise comparison between two input tensors,
690
+ * and counts the number of positions where the elements are equal. The result is
691
+ * stored in the destination tensor `dst` as a scalar.
692
+ *
693
+ * The operation is optimized using the CANN backend, making it suitable for
694
+ * high-performance inference or training scenarios.
695
+ *
696
+ * @param ctx The CANN context used for operations.
697
+ * @param dst The destination tensor where the result will be stored.
698
+ * dst->op is expected to be `GGML_OP_COUNT_EQUAL`.
699
+ */
700
+ void ggml_cann_count_equal(ggml_backend_cann_context& ctx, ggml_tensor* dst);
701
+
702
+ /**
703
+ * @brief Applies the Step activation function to a ggml tensor using the CANN backend.
704
+ *
705
+ * @details This function applies a step function element-wise to the input tensor, where
706
+ * each element is transformed to 1.0 if it is greater than 0, and 0.0 otherwise.
707
+ * The result is stored in the destination tensor `dst`.
708
+ *
709
+ * This operation is accelerated using the CANN backend to improve runtime performance.
710
+ *
711
+ * @param ctx The CANN context used for operations.
712
+ * @param dst The destination tensor where the result will be stored.
713
+ * dst->op is expected to be `GGML_OP_STEP`.
714
+ */
715
+ void ggml_cann_step(ggml_backend_cann_context& ctx, ggml_tensor* dst);
716
+
717
+ /**
718
+ * @brief Performs the Flash Attention extended operator using the CANN backend.
719
+ *
720
+ * @details This function implements the memory-efficient Flash Attention algorithm
721
+ * for computing scaled dot-product attention with hardware acceleration.
722
+ * The result is stored in the destination tensor `dst`.
723
+ *
724
+ * This operation is accelerated using the CANN backend to improve runtime performance.
725
+ *
726
+ * @param ctx The CANN context used for operations.
727
+ * @param dst The destination tensor where the result will be stored.
728
+ * dst->op is expected to be `GGML_OP_FLASH_ATTN_EXT`.
729
+ */
730
+ void ggml_cann_flash_attn_ext(ggml_backend_cann_context& ctx, ggml_tensor* dst);
731
+
732
+ /*
733
+ * @brief A generic wrapper for ACL resources with custom deleter support.
734
+ */
735
+ using any_acl_resource = std::unique_ptr<void, std::function<void(void*)>>;
736
+
737
+ /**
738
+ * @brief Trait structure used to define how to destroy a given ACL resource type.
739
+ *
740
+ * @tparam T ACL resource type.
741
+ */
742
+ template<typename T>
743
+ struct acl_resource_traits;
744
+
745
+ /**
746
+ * @brief Specialization for aclTensor, defines how to destroy an aclTensor resource.
747
+ */
748
+ template<>
749
+ struct acl_resource_traits<aclTensor> {
750
+ static void destroy(void* p) {
751
+ ACL_CHECK(aclDestroyTensor(static_cast<aclTensor*>(p)));
752
+ }
753
+ };
754
+
755
+ /**
756
+ * @brief Specialization for aclIntArray, defines how to destroy an aclIntArray resource.
757
+ */
758
+ template<>
759
+ struct acl_resource_traits<aclIntArray> {
760
+ static void destroy(void* p) {
761
+ ACL_CHECK(aclDestroyIntArray(static_cast<aclIntArray*>(p)));
762
+ }
763
+ };
764
+
765
+ /**
766
+ * @brief Specialization for aclScalar, defines how to destroy an aclScalar resource.
767
+ */
768
+ template<>
769
+ struct acl_resource_traits<aclScalar> {
770
+ static void destroy(void* p) {
771
+ ACL_CHECK(aclDestroyScalar(static_cast<aclScalar*>(p)));
772
+ }
773
+ };
774
+
775
+ /**
776
+ * @brief Specialization for aclTensorList, defines how to destroy an aclTensorList resource.
777
+ */
778
+ template<>
779
+ struct acl_resource_traits<aclTensorList> {
780
+ static void destroy(void* p) {
781
+ ACL_CHECK(aclDestroyTensorList(static_cast<aclTensorList*>(p)));
782
+ }
783
+ };
784
+
785
+ /**
786
+ * @brief Creates a generic ACL resource wrapper with proper destruction logic.
787
+ *
788
+ * @tparam T ACL resource type.
789
+ * @param ptr Raw pointer to ACL resource.
790
+ * @return any_acl_resource Smart pointer that handles destruction.
791
+ */
792
+ template<typename T>
793
+ any_acl_resource make_acl_resource(T* ptr) {
794
+ return any_acl_resource(
795
+ static_cast<void*>(ptr),
796
+ [](void* p) {
797
+ acl_resource_traits<T>::destroy(p);
798
+ }
799
+ );
800
+ }
801
+
802
+ /**
803
+ * @brief Registers multiple ACL resources into a vector for lifetime management.
804
+ *
805
+ * @tparam Args Variadic list of ACL resource types.
806
+ * @param vec Target vector to hold ACL resources.
807
+ * @param args Raw pointers to ACL resources.
808
+ */
809
+ template<typename... Args>
810
+ void register_acl_resources(std::vector<any_acl_resource>& vec, Args*... args) {
811
+ (vec.emplace_back(make_acl_resource(args)), ...);
812
+ }
813
+
814
+ /**
815
+ * @brief Task class that wraps the execution of an aclnn function call.
816
+ */
817
+ class aclnn_task : public cann_task {
818
+ public:
819
+ aclnn_task(aclnn_func_t aclnn_func, void * workspace_addr,
820
+ uint64_t workspace_size, aclOpExecutor * executor,
821
+ aclrtStream stream) :
822
+ aclnn_func_(aclnn_func),
823
+ workspace_addr_(workspace_addr),
824
+ workspace_size_(workspace_size),
825
+ executor_(executor),
826
+ stream_(stream) {}
827
+ virtual void run_task() override {
828
+ ACL_CHECK(aclnn_func_(workspace_addr_, workspace_size_, executor_, stream_));
829
+ }
830
+ private:
831
+ aclnn_func_t aclnn_func_;
832
+ void * workspace_addr_;
833
+ uint64_t workspace_size_;
834
+ aclOpExecutor * executor_;
835
+ aclrtStream stream_;
836
+ };
837
+
838
+ /**
839
+ * @brief Task class that releases ACL resources after usage.
840
+ */
841
+ class release_resource_task : public cann_task {
842
+ public:
843
+ release_resource_task(std::vector<any_acl_resource>&& resources){
844
+ resource_ = std::move(resources);
845
+ }
846
+
847
+ virtual void run_task() override {
848
+ resource_.clear();
849
+ }
850
+ private:
851
+ std::vector<any_acl_resource> resource_;
852
+ };
853
+
854
+ /**
855
+ * @brief Task class for performing asynchronous memory copy operations.
856
+ */
857
+ class async_memcpy_task : public cann_task {
858
+ public:
859
+ async_memcpy_task(void* dst, const void* src, size_t size,
860
+ aclrtMemcpyKind kind, aclrtStream stream)
861
+ : dst_(dst), src_(src), size_(size), kind_(kind), stream_(stream) {}
862
+
863
+ virtual void run_task() override {
864
+ ACL_CHECK(aclrtMemcpyAsync(dst_, size_, src_, size_, kind_, stream_));
865
+ }
866
+ private:
867
+ void* dst_;
868
+ const void* src_;
869
+ size_t size_;
870
+ aclrtMemcpyKind kind_;
871
+ aclrtStream stream_;
872
+ };
873
+
874
+ /**
875
+ * @brief Task class for performing asynchronous memory set operations.
876
+ */
877
+ class async_memset_task : public cann_task {
878
+ public:
879
+ async_memset_task(void* buffer, size_t size, int32_t value, aclrtStream stream)
880
+ : buffer_(buffer), size_(size), value_(value), stream_(stream) {}
881
+
882
+ virtual void run_task() override {
883
+ ACL_CHECK(aclrtMemsetAsync(buffer_, size_, value_, size_, stream_));
884
+ }
885
+ private:
886
+ void* buffer_;
887
+ size_t size_;
888
+ int32_t value_;
889
+ aclrtStream stream_;
890
+ };
891
+
892
+ /**
893
+ * @brief Launches an asynchronous task using the memory allocator.
894
+ *
895
+ * This macro submit an asynchronous task on the specified stream.
896
+ * The task uses memory allocated by the allocator. It is guaranteed
897
+ * that the memory will not be accessed by other tasks until this task
898
+ * completes, due to the sequential execution order within the same stream.
899
+ *
900
+ * @param OP_NAME aclnn operator name.
901
+ * @param args Additional arguments required by the task.
902
+ *
903
+ * @note
904
+ * Memory from the allocator will be "freed" immediately and can be
905
+ * reallocated to other pointers. However, it won't be accessed by any
906
+ * other task before this asynchronous task ends, because all tasks in the
907
+ * same stream are executed in queue order.
908
+ */
909
+
910
+ #define GGML_CANN_CALL_ACLNN_OP(CTX, OP_NAME, ...) \
911
+ do { \
912
+ uint64_t workspaceSize = 0; \
913
+ aclOpExecutor * executor; \
914
+ void * workspaceAddr = nullptr; \
915
+ ACL_CHECK(aclnn##OP_NAME##GetWorkspaceSize(__VA_ARGS__, &workspaceSize, &executor));\
916
+ /* workspace should alloced in main thread to keep malloc order when using vmm. */ \
917
+ if (workspaceSize > 0) { \
918
+ ggml_cann_pool_alloc workspace_allocator(CTX.pool(), workspaceSize); \
919
+ workspaceAddr = workspace_allocator.get(); \
920
+ } \
921
+ if (CTX.async_mode) { \
922
+ auto task = \
923
+ std::make_unique<aclnn_task>(aclnn##OP_NAME, workspaceAddr, workspaceSize, \
924
+ executor, CTX.stream()); \
925
+ CTX.task_queue.submit_task(std::move(task)); \
926
+ } else { \
927
+ ACL_CHECK(aclnn##OP_NAME(workspaceAddr, workspaceSize, executor, CTX.stream()));\
928
+ } \
929
+ } while (0)
930
+
931
+ /**
932
+ * @brief Registers and releases multiple ACL resources, optionally deferring the release
933
+ * using a task.
934
+ *
935
+ * @tparam Args Types of the ACL resources.
936
+ * @param ctx Backend context which manages task submission and async mode.
937
+ * @param args Pointers to ACL resources to be released.
938
+ */
939
+ template <typename... Args>
940
+ void ggml_cann_release_resources(ggml_backend_cann_context & ctx, Args &&... args) {
941
+ std::vector<any_acl_resource> resources;
942
+ register_acl_resources(resources, std::forward<Args>(args)...);
943
+ if(ctx.async_mode) {
944
+ auto task = std::make_unique<release_resource_task>(std::move(resources));
945
+ ctx.task_queue.submit_task(std::move(task));
946
+ }
947
+ }
948
+
949
+ /**
950
+ * @brief Performs an asynchronous memory copy operation, optionally deferred via task submission.
951
+ *
952
+ * @param ctx Backend context containing stream and async configuration.
953
+ * @param dst Destination memory address.
954
+ * @param src Source memory address.
955
+ * @param len Size of memory to copy (in bytes).
956
+ * @param kind Type of memory copy (host-to-device, device-to-host, etc).
957
+ */
958
+ inline void ggml_cann_async_memcpy(ggml_backend_cann_context & ctx, void * dst,
959
+ const void * src, size_t len, aclrtMemcpyKind kind) {
960
+ if (ctx.async_mode) {
961
+ auto task = std::make_unique<async_memcpy_task>(dst, const_cast<void *>(src), len, kind, ctx.stream());
962
+ ctx.task_queue.submit_task(std::move(task));
963
+ } else {
964
+ ACL_CHECK(aclrtMemcpyAsync(dst, len, src, len, kind, ctx.stream()));
965
+ }
966
+ }
967
+
968
+ inline void ggml_cann_async_memcpy(ggml_backend_cann_context * ctx, void * dst,
969
+ const void * src, size_t len, aclrtMemcpyKind kind) {
970
+ if (ctx->async_mode) {
971
+ auto task = std::make_unique<async_memcpy_task>(dst, const_cast<void *>(src), len, kind, ctx->stream());
972
+ ctx->task_queue.submit_task(std::move(task));
973
+ } else {
974
+ ACL_CHECK(aclrtMemcpyAsync(dst, len, src, len, kind, ctx->stream()));
975
+ }
976
+ }
977
+
978
+ /**
979
+ * @brief Performs an asynchronous memory set operation, optionally deferred via task submission.
980
+ *
981
+ * @param ctx Backend context containing stream and async configuration.
982
+ * @param buffer Memory buffer to be set.
983
+ * @param size Size of the memory buffer (in bytes).
984
+ * @param value Value to set in the buffer.
985
+ */
986
+ inline void ggml_cann_async_memset(ggml_backend_cann_context & ctx, void * buffer,
987
+ size_t size, int value) {
988
+ if (ctx.async_mode) {
989
+ auto task = std::make_unique<async_memset_task>(buffer, size, value, ctx.stream());
990
+ ctx.task_queue.submit_task(std::move(task));
991
+ } else {
992
+ ACL_CHECK(aclrtMemsetAsync(buffer, size, value, size, ctx.stream()));
993
+ }
994
+ }
995
+
996
+ /**
997
+ * @brief Performs sparse expert-based matrix multiplication using the CANN backend.
998
+ *
999
+ * @details This function implements a MoE-style batched matrix multiplication, where each input token
1000
+ * is routed to one or more experts, and each expert corresponds to a specific [D, M] weight matrix
1001
+ * in the source tensor `src0`. The routing indices are provided via the `ids` tensor.
1002
+ *
1003
+ * For each token (from `src1`), the function selects the corresponding expert(s) as specified by `ids`,
1004
+ * performs the matrix multiplication with the selected expert's weight submatrix (from `src0`),
1005
+ * and stores the results in `dst`. This operation is optimized and executed on the CANN backend.
1006
+ *
1007
+ * Dimensions:
1008
+ * - src0: [D, M, A, 1], where A is the number of experts
1009
+ * - src1: [D, B, N, 1], where N is batch size and B is the slot count per sample
1010
+ * - ids : [K, N], where K is the number of experts each token is routed to
1011
+ * - dst : [M, K, N, 1], output tensor storing the result of expert × token multiplication
1012
+ *
1013
+ * The function handles two main modes:
1014
+ * - If `ne12 == 1`, a simpler per-token loop is used.
1015
+ * - TODO: If `ne12 > 1`, grouped multiplication and memory copying is used for efficiency.
1016
+ *
1017
+ * @param ctx The CANN context used for operations.
1018
+ * @param dst The destination tensor where the expert-weighted token outputs are stored.
1019
+ * Expected to be of shape [M, K, N, 1].
1020
+ */
1021
+ void ggml_cann_mul_mat_id(ggml_backend_cann_context& ctx, ggml_tensor* dst);
1022
+
1023
+ /**
1024
+ * @brief Applies a element-wise operation to two input tensors using the CANN
1025
+ * backend.
1026
+ *
1027
+ * This templated function takes a binary operator and applies it to two source
1028
+ * tensors
1029
+ * associated with the destination tensor. The function handles broadcasting as
1030
+ * needed.
1031
+ *
1032
+ * @tparam binary_op A callable object (e.g., lambda or function pointer) representing
1033
+ * the binary operation to be performed. It must take three arguments:
1034
+ * (ggml_backend_cann_context&, aclTensor*, aclTensor*, aclTensor*).
1035
+ *
1036
+ * @param ctx The CANN backend context used to manage execution and resources.
1037
+ * @param dst The destination tensor.
1038
+ */
1039
+ template <auto binary_op>
1040
+ void ggml_cann_binary_op(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
1041
+ ggml_tensor* src0 = dst->src[0];
1042
+ ggml_tensor* src1 = dst->src[1];
1043
+
1044
+ aclTensor* acl_src0;
1045
+ aclTensor* acl_src1;
1046
+ aclTensor* acl_dst;
1047
+
1048
+ // Need bcast
1049
+ bcast_shape(src0, src1, dst, &acl_src0, &acl_src1, &acl_dst);
1050
+ binary_op(ctx, acl_src0, acl_src1, acl_dst);
1051
+
1052
+ ggml_cann_release_resources(ctx, acl_src0, acl_src1, acl_dst);
1053
+ }
1054
+
1055
+
1056
+ /**
1057
+ * @brief Applies a unary operation to an input tensor using the CANN backend.
1058
+ *
1059
+ * This templated function applies a unary operator to the source tensor of `dst`
1060
+ * and stores the result in the destination tensor.
1061
+ *
1062
+ * @tparam unary_op A callable with the signature:
1063
+ * void(ggml_backend_cann_context&, aclTensor*, aclTensor*)
1064
+ * where the first aclTensor is the source and the second is the destination.
1065
+ * @param ctx The CANN backend context for managing resources and execution.
1066
+ * @param dst The destination tensor. Its src[0] is treated as the input tensor.
1067
+ */
1068
+ template <void unary_op(ggml_backend_cann_context&, aclTensor*, aclTensor*)>
1069
+ void ggml_cann_unary_op(ggml_backend_cann_context& ctx, ggml_tensor* dst) {
1070
+ ggml_tensor* src = dst->src[0];
1071
+
1072
+ aclTensor* acl_src = ggml_cann_create_tensor(src);
1073
+ aclTensor* acl_dst = ggml_cann_create_tensor(dst);
1074
+
1075
+ unary_op(ctx, acl_src, acl_dst);
1076
+ ggml_cann_release_resources(ctx, acl_src, acl_dst);
1077
+ }
1078
+
1079
+ /**
1080
+ * @brief Applies a unary operation to a ggml tensor using the CANN backend.
1081
+ *
1082
+ * @details This function performs a unary operation on the input tensor using
1083
+ * a user-provided lambda or callable object `unary_op`, which accepts the CANN
1084
+ * context and two ACL tensors (source and destination). Internally, this function
1085
+ * creates ACL representations of the ggml tensors and invokes the unary operation.
1086
+ * The result is stored in the destination tensor `dst`. This utility abstracts the
1087
+ * common boilerplate of tensor conversion and cleanup when implementing unary ops.
1088
+ *
1089
+ * @param unary_op A callable that performs the unary operation using CANN APIs.
1090
+ * @param ctx The CANN context used for operations.
1091
+ * @param dst The destination tensor where the result will be stored.
1092
+ * The source tensor is retrieved from `dst->src[0]`.
1093
+ */
1094
+ void ggml_cann_unary_op(
1095
+ std::function<void(ggml_backend_cann_context&, aclTensor*, aclTensor*)> unary_op,
1096
+ ggml_backend_cann_context& ctx, ggml_tensor* dst);
1097
+
1098
+ /**
1099
+ * @brief Helper macro to invoke a unary ACL operation using ggml_cann_unary_op.
1100
+ *
1101
+ * This macro defines an inline lambda wrapping a specific ACL operation name,
1102
+ * and passes it to the templated ggml_cann_unary_op function. It simplifies
1103
+ * calling unary ops by hiding the lambda boilerplate.
1104
+ *
1105
+ * Internally, the lambda will call:
1106
+ * @code
1107
+ * GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst);
1108
+ * @endcode
1109
+ *
1110
+ * @param OP_NAME The name of the ACL unary operator to invoke via GGML_CANN_CALL_ACLNN_OP.
1111
+ *
1112
+ * @see ggml_cann_unary_op
1113
+ * @see GGML_CANN_CALL_ACLNN_OP
1114
+ */
1115
+ #define GGML_CANN_CALL_UNARY_OP(OP_NAME) \
1116
+ do { \
1117
+ auto lambda = [](ggml_backend_cann_context& ctx, \
1118
+ aclTensor* acl_src, \
1119
+ aclTensor* acl_dst) { \
1120
+ GGML_CANN_CALL_ACLNN_OP(ctx, OP_NAME, acl_src, acl_dst); \
1121
+ }; \
1122
+ ggml_cann_unary_op(lambda, ctx, dst); \
1123
+ } \
1124
+ while (0)
1125
+ #endif // CANN_ACLNN_OPS